405 lines
No EOL
15 KiB
TypeScript
405 lines
No EOL
15 KiB
TypeScript
import { useState, useEffect } from 'react';
|
|
import {
|
|
ArrowPathIcon,
|
|
CircleStackIcon,
|
|
CloudArrowDownIcon,
|
|
ExclamationTriangleIcon,
|
|
CheckCircleIcon,
|
|
} from '@heroicons/react/24/outline';
|
|
import { usePipeline } from './hooks/usePipeline';
|
|
import type { PipelineOperation } from './types';
|
|
|
|
const operations: PipelineOperation[] = [
|
|
// Symbol operations
|
|
{
|
|
id: 'sync-qm-symbols',
|
|
name: 'Sync QM Symbols',
|
|
description: 'Sync symbols from QuestionsAndMethods API',
|
|
endpoint: '/symbols',
|
|
method: 'POST',
|
|
category: 'sync',
|
|
},
|
|
{
|
|
id: 'sync-provider-symbols',
|
|
name: 'Sync Provider Symbols',
|
|
description: 'Sync symbols from a specific provider',
|
|
endpoint: '/symbols/:provider',
|
|
method: 'POST',
|
|
category: 'sync',
|
|
params: { provider: 'yahoo' }, // Default provider
|
|
},
|
|
// Exchange operations
|
|
{
|
|
id: 'sync-qm-exchanges',
|
|
name: 'Sync QM Exchanges',
|
|
description: 'Sync exchanges from QuestionsAndMethods API',
|
|
endpoint: '/exchanges',
|
|
method: 'POST',
|
|
category: 'sync',
|
|
},
|
|
{
|
|
id: 'sync-all-exchanges',
|
|
name: 'Sync All Exchanges',
|
|
description: 'Sync all exchanges with optional clear',
|
|
endpoint: '/exchanges/all',
|
|
method: 'POST',
|
|
category: 'sync',
|
|
},
|
|
// Provider mapping operations
|
|
{
|
|
id: 'sync-qm-provider-mappings',
|
|
name: 'Sync QM Provider Mappings',
|
|
description: 'Sync provider mappings from QuestionsAndMethods',
|
|
endpoint: '/provider-mappings/qm',
|
|
method: 'POST',
|
|
category: 'sync',
|
|
},
|
|
{
|
|
id: 'sync-ib-exchanges',
|
|
name: 'Sync IB Exchanges',
|
|
description: 'Sync exchanges from Interactive Brokers',
|
|
endpoint: '/provider-mappings/ib',
|
|
method: 'POST',
|
|
category: 'sync',
|
|
},
|
|
// Maintenance operations
|
|
{
|
|
id: 'clear-postgresql',
|
|
name: 'Clear PostgreSQL Data',
|
|
description: 'Clear exchange and provider mapping data',
|
|
endpoint: '/clear/postgresql',
|
|
method: 'POST',
|
|
category: 'maintenance',
|
|
dangerous: true,
|
|
},
|
|
];
|
|
|
|
export function PipelinePage() {
|
|
const {
|
|
loading,
|
|
error,
|
|
lastJobResult,
|
|
syncQMSymbols,
|
|
syncProviderSymbols,
|
|
syncQMExchanges,
|
|
syncAllExchanges,
|
|
syncQMProviderMappings,
|
|
syncIBExchanges,
|
|
clearPostgreSQLData,
|
|
getExchangeStats,
|
|
getProviderMappingStats,
|
|
} = usePipeline();
|
|
|
|
const [selectedProvider, setSelectedProvider] = useState('yahoo');
|
|
const [clearFirst, setClearFirst] = useState(false);
|
|
const [clearDataType, setClearDataType] = useState<'all' | 'exchanges' | 'provider_mappings'>('all');
|
|
const [stats, setStats] = useState<{ exchanges?: any; providerMappings?: any }>({});
|
|
|
|
// Load stats on mount
|
|
useEffect(() => {
|
|
loadStats();
|
|
}, []);
|
|
|
|
const loadStats = async () => {
|
|
const [exchangeStats, mappingStats] = await Promise.all([
|
|
getExchangeStats(),
|
|
getProviderMappingStats(),
|
|
]);
|
|
setStats({
|
|
exchanges: exchangeStats,
|
|
providerMappings: mappingStats,
|
|
});
|
|
};
|
|
|
|
const handleOperation = async (op: PipelineOperation) => {
|
|
switch (op.id) {
|
|
case 'sync-qm-symbols':
|
|
await syncQMSymbols();
|
|
break;
|
|
case 'sync-provider-symbols':
|
|
await syncProviderSymbols(selectedProvider);
|
|
break;
|
|
case 'sync-qm-exchanges':
|
|
await syncQMExchanges();
|
|
break;
|
|
case 'sync-all-exchanges':
|
|
await syncAllExchanges(clearFirst);
|
|
break;
|
|
case 'sync-qm-provider-mappings':
|
|
await syncQMProviderMappings();
|
|
break;
|
|
case 'sync-ib-exchanges':
|
|
await syncIBExchanges();
|
|
break;
|
|
case 'clear-postgresql':
|
|
if (confirm(`Are you sure you want to clear ${clearDataType} data? This action cannot be undone.`)) {
|
|
await clearPostgreSQLData(clearDataType);
|
|
}
|
|
break;
|
|
}
|
|
// Reload stats after operation
|
|
await loadStats();
|
|
};
|
|
|
|
const getCategoryIcon = (category: string) => {
|
|
switch (category) {
|
|
case 'sync':
|
|
return <CloudArrowDownIcon className="h-4 w-4" />;
|
|
case 'maintenance':
|
|
return <ExclamationTriangleIcon className="h-4 w-4" />;
|
|
default:
|
|
return <CircleStackIcon className="h-4 w-4" />;
|
|
}
|
|
};
|
|
|
|
const getCategoryColor = (category: string) => {
|
|
switch (category) {
|
|
case 'sync':
|
|
return 'text-primary-400';
|
|
case 'maintenance':
|
|
return 'text-warning';
|
|
default:
|
|
return 'text-text-secondary';
|
|
}
|
|
};
|
|
|
|
return (
|
|
<div className="flex flex-col h-full space-y-4">
|
|
<div className="flex-shrink-0">
|
|
<h1 className="text-lg font-bold text-text-primary mb-2">Data Pipeline Management</h1>
|
|
<p className="text-text-secondary text-sm">
|
|
Manage data synchronization and maintenance operations
|
|
</p>
|
|
</div>
|
|
|
|
{/* Stats Overview */}
|
|
{(stats.exchanges || stats.providerMappings) && (
|
|
<div className="grid grid-cols-1 md:grid-cols-2 gap-3 flex-shrink-0">
|
|
{stats.exchanges && (
|
|
<div className="bg-surface-secondary p-3 rounded-lg border border-border">
|
|
<h3 className="text-base font-semibold text-text-primary mb-2">Exchange Statistics</h3>
|
|
<div className="space-y-1.5 text-xs">
|
|
<div className="flex justify-between">
|
|
<span className="text-text-secondary">Total Exchanges:</span>
|
|
<span className="text-text-primary font-medium">{stats.exchanges.totalExchanges}</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span className="text-text-secondary">Active Exchanges:</span>
|
|
<span className="text-success font-medium">{stats.exchanges.activeExchanges}</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span className="text-text-secondary">Total Provider Mappings:</span>
|
|
<span className="text-text-primary font-medium">{stats.exchanges.totalProviderMappings}</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span className="text-text-secondary">Active Mappings:</span>
|
|
<span className="text-success font-medium">{stats.exchanges.activeProviderMappings}</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{stats.providerMappings && (
|
|
<div className="bg-surface-secondary p-3 rounded-lg border border-border">
|
|
<h3 className="text-base font-semibold text-text-primary mb-2">Provider Mapping Statistics</h3>
|
|
<div className="space-y-1.5 text-xs">
|
|
<div className="flex justify-between">
|
|
<span className="text-text-secondary">Coverage:</span>
|
|
<span className="text-primary-400 font-medium">
|
|
{stats.providerMappings.coveragePercentage?.toFixed(1)}%
|
|
</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span className="text-text-secondary">Verified Mappings:</span>
|
|
<span className="text-success font-medium">{stats.providerMappings.verifiedMappings}</span>
|
|
</div>
|
|
<div className="flex justify-between">
|
|
<span className="text-text-secondary">Auto-mapped:</span>
|
|
<span className="text-text-primary font-medium">{stats.providerMappings.autoMappedCount}</span>
|
|
</div>
|
|
{stats.providerMappings.mappingsByProvider && (
|
|
<div className="mt-2 pt-2 border-t border-border">
|
|
<span className="text-text-secondary text-xs">By Provider:</span>
|
|
<div className="mt-1 flex flex-wrap gap-2">
|
|
{Object.entries(stats.providerMappings.mappingsByProvider).map(([provider, count]) => (
|
|
<span key={provider} className="text-xs bg-surface px-2 py-1 rounded">
|
|
{provider}: {String(count)}
|
|
</span>
|
|
))}
|
|
</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
)}
|
|
|
|
{/* Status Messages */}
|
|
{error && (
|
|
<div className="p-3 bg-danger/10 border border-danger/20 rounded-lg flex-shrink-0">
|
|
<div className="flex items-center gap-2">
|
|
<ExclamationTriangleIcon className="h-4 w-4 text-danger" />
|
|
<span className="text-danger text-sm">{error}</span>
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{lastJobResult && (
|
|
<div className={`p-3 rounded-lg border flex-shrink-0 ${
|
|
lastJobResult.success
|
|
? 'bg-success/10 border-success/20'
|
|
: 'bg-danger/10 border-danger/20'
|
|
}`}>
|
|
<div className="flex items-center gap-2">
|
|
{lastJobResult.success ? (
|
|
<CheckCircleIcon className="h-4 w-4 text-success" />
|
|
) : (
|
|
<ExclamationTriangleIcon className="h-4 w-4 text-danger" />
|
|
)}
|
|
<span className={`text-sm ${lastJobResult.success ? 'text-success' : 'text-danger'}`}>
|
|
{lastJobResult.message || lastJobResult.error}
|
|
</span>
|
|
{lastJobResult.jobId && (
|
|
<span className="text-text-muted text-xs ml-auto">
|
|
Job ID: {lastJobResult.jobId}
|
|
</span>
|
|
)}
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{/* Operations Grid */}
|
|
<div className="flex-1 min-h-0 space-y-4 overflow-y-auto">
|
|
{/* Sync Operations */}
|
|
<div>
|
|
<h2 className="text-base font-semibold text-text-primary mb-3 flex items-center gap-2">
|
|
<CloudArrowDownIcon className="h-4 w-4 text-primary-400" />
|
|
Sync Operations
|
|
</h2>
|
|
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-3">
|
|
{operations.filter(op => op.category === 'sync').map(op => (
|
|
<div
|
|
key={op.id}
|
|
className="bg-surface-secondary p-3 rounded-lg border border-border hover:border-primary-500/50 transition-colors"
|
|
>
|
|
<div className="flex items-start justify-between mb-2">
|
|
<h3 className="font-medium text-sm text-text-primary">{op.name}</h3>
|
|
<div className={getCategoryColor(op.category)}>
|
|
{getCategoryIcon(op.category)}
|
|
</div>
|
|
</div>
|
|
<p className="text-xs text-text-secondary mb-2">{op.description}</p>
|
|
|
|
{/* Special inputs for specific operations */}
|
|
{op.id === 'sync-provider-symbols' && (
|
|
<div className="mb-2">
|
|
<label className="block text-xs text-text-muted mb-1">Provider</label>
|
|
<select
|
|
value={selectedProvider}
|
|
onChange={e => setSelectedProvider(e.target.value)}
|
|
className="w-full px-2 py-1 text-xs bg-surface border border-border rounded focus:ring-1 focus:ring-primary-500 focus:border-primary-500"
|
|
>
|
|
<option value="yahoo">Yahoo</option>
|
|
<option value="qm">QuestionsAndMethods</option>
|
|
<option value="ib">Interactive Brokers</option>
|
|
</select>
|
|
</div>
|
|
)}
|
|
|
|
{op.id === 'sync-all-exchanges' && (
|
|
<div className="mb-2">
|
|
<label className="flex items-center gap-2 text-xs">
|
|
<input
|
|
type="checkbox"
|
|
checked={clearFirst}
|
|
onChange={e => setClearFirst(e.target.checked)}
|
|
className="rounded border-border text-primary-500 focus:ring-primary-500"
|
|
/>
|
|
<span className="text-text-secondary">Clear existing data first</span>
|
|
</label>
|
|
</div>
|
|
)}
|
|
|
|
<button
|
|
onClick={() => handleOperation(op)}
|
|
disabled={loading}
|
|
className="w-full px-2.5 py-1.5 bg-primary-500/20 text-primary-400 rounded text-xs font-medium hover:bg-primary-500/30 disabled:opacity-50 disabled:cursor-not-allowed transition-colors flex items-center justify-center gap-2"
|
|
>
|
|
{loading ? (
|
|
<>
|
|
<ArrowPathIcon className="h-3 w-3 animate-spin" />
|
|
Processing...
|
|
</>
|
|
) : (
|
|
<>
|
|
<CloudArrowDownIcon className="h-3 w-3" />
|
|
Execute
|
|
</>
|
|
)}
|
|
</button>
|
|
</div>
|
|
))}
|
|
</div>
|
|
</div>
|
|
|
|
{/* Maintenance Operations */}
|
|
<div>
|
|
<h2 className="text-base font-semibold text-text-primary mb-3 flex items-center gap-2">
|
|
<ExclamationTriangleIcon className="h-4 w-4 text-warning" />
|
|
Maintenance Operations
|
|
</h2>
|
|
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-3">
|
|
{operations.filter(op => op.category === 'maintenance').map(op => (
|
|
<div
|
|
key={op.id}
|
|
className="bg-surface-secondary p-4 rounded-lg border border-warning/20 hover:border-warning/50 transition-colors"
|
|
>
|
|
<div className="flex items-start justify-between mb-2">
|
|
<h3 className="font-medium text-sm text-text-primary">{op.name}</h3>
|
|
<div className={getCategoryColor(op.category)}>
|
|
{getCategoryIcon(op.category)}
|
|
</div>
|
|
</div>
|
|
<p className="text-xs text-text-secondary mb-2">{op.description}</p>
|
|
|
|
{op.id === 'clear-postgresql' && (
|
|
<div className="mb-2">
|
|
<label className="block text-xs text-text-muted mb-1">Data Type</label>
|
|
<select
|
|
value={clearDataType}
|
|
onChange={e => setClearDataType(e.target.value as any)}
|
|
className="w-full px-2 py-1 text-xs bg-surface border border-border rounded focus:ring-1 focus:ring-warning focus:border-warning"
|
|
>
|
|
<option value="all">All Data</option>
|
|
<option value="exchanges">Exchanges Only</option>
|
|
<option value="provider_mappings">Provider Mappings Only</option>
|
|
</select>
|
|
</div>
|
|
)}
|
|
|
|
<button
|
|
onClick={() => handleOperation(op)}
|
|
disabled={loading}
|
|
className="w-full px-2.5 py-1.5 bg-warning/20 text-warning rounded text-xs font-medium hover:bg-warning/30 disabled:opacity-50 disabled:cursor-not-allowed transition-colors flex items-center justify-center gap-2"
|
|
>
|
|
{loading ? (
|
|
<>
|
|
<ArrowPathIcon className="h-3 w-3 animate-spin" />
|
|
Processing...
|
|
</>
|
|
) : (
|
|
<>
|
|
<ExclamationTriangleIcon className="h-3 w-3" />
|
|
Execute
|
|
</>
|
|
)}
|
|
</button>
|
|
</div>
|
|
))}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
);
|
|
} |