moved most api stuff to web-api and built out a better monitoring solution for web-app
This commit is contained in:
parent
fbff428e90
commit
da1c52a841
45 changed files with 2986 additions and 312 deletions
406
apps/stock/web-app/src/features/pipeline/PipelinePage.tsx
Normal file
406
apps/stock/web-app/src/features/pipeline/PipelinePage.tsx
Normal file
|
|
@ -0,0 +1,406 @@
|
|||
import { useState, useEffect } from 'react';
|
||||
import {
|
||||
ArrowPathIcon,
|
||||
CircleStackIcon,
|
||||
CloudArrowDownIcon,
|
||||
ExclamationTriangleIcon,
|
||||
CheckCircleIcon,
|
||||
ClockIcon,
|
||||
} from '@heroicons/react/24/outline';
|
||||
import { usePipeline } from './hooks/usePipeline';
|
||||
import type { PipelineOperation } from './types';
|
||||
|
||||
const operations: PipelineOperation[] = [
|
||||
// Symbol operations
|
||||
{
|
||||
id: 'sync-qm-symbols',
|
||||
name: 'Sync QM Symbols',
|
||||
description: 'Sync symbols from QuestionsAndMethods API',
|
||||
endpoint: '/symbols',
|
||||
method: 'POST',
|
||||
category: 'sync',
|
||||
},
|
||||
{
|
||||
id: 'sync-provider-symbols',
|
||||
name: 'Sync Provider Symbols',
|
||||
description: 'Sync symbols from a specific provider',
|
||||
endpoint: '/symbols/:provider',
|
||||
method: 'POST',
|
||||
category: 'sync',
|
||||
params: { provider: 'yahoo' }, // Default provider
|
||||
},
|
||||
// Exchange operations
|
||||
{
|
||||
id: 'sync-qm-exchanges',
|
||||
name: 'Sync QM Exchanges',
|
||||
description: 'Sync exchanges from QuestionsAndMethods API',
|
||||
endpoint: '/exchanges',
|
||||
method: 'POST',
|
||||
category: 'sync',
|
||||
},
|
||||
{
|
||||
id: 'sync-all-exchanges',
|
||||
name: 'Sync All Exchanges',
|
||||
description: 'Sync all exchanges with optional clear',
|
||||
endpoint: '/exchanges/all',
|
||||
method: 'POST',
|
||||
category: 'sync',
|
||||
},
|
||||
// Provider mapping operations
|
||||
{
|
||||
id: 'sync-qm-provider-mappings',
|
||||
name: 'Sync QM Provider Mappings',
|
||||
description: 'Sync provider mappings from QuestionsAndMethods',
|
||||
endpoint: '/provider-mappings/qm',
|
||||
method: 'POST',
|
||||
category: 'sync',
|
||||
},
|
||||
{
|
||||
id: 'sync-ib-exchanges',
|
||||
name: 'Sync IB Exchanges',
|
||||
description: 'Sync exchanges from Interactive Brokers',
|
||||
endpoint: '/provider-mappings/ib',
|
||||
method: 'POST',
|
||||
category: 'sync',
|
||||
},
|
||||
// Maintenance operations
|
||||
{
|
||||
id: 'clear-postgresql',
|
||||
name: 'Clear PostgreSQL Data',
|
||||
description: 'Clear exchange and provider mapping data',
|
||||
endpoint: '/clear/postgresql',
|
||||
method: 'POST',
|
||||
category: 'maintenance',
|
||||
dangerous: true,
|
||||
},
|
||||
];
|
||||
|
||||
export function PipelinePage() {
|
||||
const {
|
||||
loading,
|
||||
error,
|
||||
lastJobResult,
|
||||
syncQMSymbols,
|
||||
syncProviderSymbols,
|
||||
syncQMExchanges,
|
||||
syncAllExchanges,
|
||||
syncQMProviderMappings,
|
||||
syncIBExchanges,
|
||||
clearPostgreSQLData,
|
||||
getExchangeStats,
|
||||
getProviderMappingStats,
|
||||
} = usePipeline();
|
||||
|
||||
const [selectedProvider, setSelectedProvider] = useState('yahoo');
|
||||
const [clearFirst, setClearFirst] = useState(false);
|
||||
const [clearDataType, setClearDataType] = useState<'all' | 'exchanges' | 'provider_mappings'>('all');
|
||||
const [stats, setStats] = useState<{ exchanges?: any; providerMappings?: any }>({});
|
||||
|
||||
// Load stats on mount
|
||||
useEffect(() => {
|
||||
loadStats();
|
||||
}, []);
|
||||
|
||||
const loadStats = async () => {
|
||||
const [exchangeStats, mappingStats] = await Promise.all([
|
||||
getExchangeStats(),
|
||||
getProviderMappingStats(),
|
||||
]);
|
||||
setStats({
|
||||
exchanges: exchangeStats,
|
||||
providerMappings: mappingStats,
|
||||
});
|
||||
};
|
||||
|
||||
const handleOperation = async (op: PipelineOperation) => {
|
||||
switch (op.id) {
|
||||
case 'sync-qm-symbols':
|
||||
await syncQMSymbols();
|
||||
break;
|
||||
case 'sync-provider-symbols':
|
||||
await syncProviderSymbols(selectedProvider);
|
||||
break;
|
||||
case 'sync-qm-exchanges':
|
||||
await syncQMExchanges();
|
||||
break;
|
||||
case 'sync-all-exchanges':
|
||||
await syncAllExchanges(clearFirst);
|
||||
break;
|
||||
case 'sync-qm-provider-mappings':
|
||||
await syncQMProviderMappings();
|
||||
break;
|
||||
case 'sync-ib-exchanges':
|
||||
await syncIBExchanges();
|
||||
break;
|
||||
case 'clear-postgresql':
|
||||
if (confirm(`Are you sure you want to clear ${clearDataType} data? This action cannot be undone.`)) {
|
||||
await clearPostgreSQLData(clearDataType);
|
||||
}
|
||||
break;
|
||||
}
|
||||
// Reload stats after operation
|
||||
await loadStats();
|
||||
};
|
||||
|
||||
const getCategoryIcon = (category: string) => {
|
||||
switch (category) {
|
||||
case 'sync':
|
||||
return <CloudArrowDownIcon className="h-5 w-5" />;
|
||||
case 'maintenance':
|
||||
return <ExclamationTriangleIcon className="h-5 w-5" />;
|
||||
default:
|
||||
return <CircleStackIcon className="h-5 w-5" />;
|
||||
}
|
||||
};
|
||||
|
||||
const getCategoryColor = (category: string) => {
|
||||
switch (category) {
|
||||
case 'sync':
|
||||
return 'text-primary-400';
|
||||
case 'maintenance':
|
||||
return 'text-warning';
|
||||
default:
|
||||
return 'text-text-secondary';
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="p-6 max-w-7xl mx-auto">
|
||||
<div className="mb-8">
|
||||
<h1 className="text-2xl font-bold text-text-primary mb-2">Data Pipeline Management</h1>
|
||||
<p className="text-text-secondary">
|
||||
Manage data synchronization and maintenance operations
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Stats Overview */}
|
||||
{(stats.exchanges || stats.providerMappings) && (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4 mb-8">
|
||||
{stats.exchanges && (
|
||||
<div className="bg-surface-secondary p-4 rounded-lg border border-border">
|
||||
<h3 className="text-lg font-semibold text-text-primary mb-3">Exchange Statistics</h3>
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-text-secondary">Total Exchanges:</span>
|
||||
<span className="text-text-primary font-medium">{stats.exchanges.totalExchanges}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-text-secondary">Active Exchanges:</span>
|
||||
<span className="text-success font-medium">{stats.exchanges.activeExchanges}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-text-secondary">Total Provider Mappings:</span>
|
||||
<span className="text-text-primary font-medium">{stats.exchanges.totalProviderMappings}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-text-secondary">Active Mappings:</span>
|
||||
<span className="text-success font-medium">{stats.exchanges.activeProviderMappings}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{stats.providerMappings && (
|
||||
<div className="bg-surface-secondary p-4 rounded-lg border border-border">
|
||||
<h3 className="text-lg font-semibold text-text-primary mb-3">Provider Mapping Statistics</h3>
|
||||
<div className="space-y-2 text-sm">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-text-secondary">Coverage:</span>
|
||||
<span className="text-primary-400 font-medium">
|
||||
{stats.providerMappings.coveragePercentage?.toFixed(1)}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-text-secondary">Verified Mappings:</span>
|
||||
<span className="text-success font-medium">{stats.providerMappings.verifiedMappings}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-text-secondary">Auto-mapped:</span>
|
||||
<span className="text-text-primary font-medium">{stats.providerMappings.autoMappedCount}</span>
|
||||
</div>
|
||||
{stats.providerMappings.mappingsByProvider && (
|
||||
<div className="mt-2 pt-2 border-t border-border">
|
||||
<span className="text-text-secondary text-xs">By Provider:</span>
|
||||
<div className="mt-1 flex flex-wrap gap-2">
|
||||
{Object.entries(stats.providerMappings.mappingsByProvider).map(([provider, count]) => (
|
||||
<span key={provider} className="text-xs bg-surface px-2 py-1 rounded">
|
||||
{provider}: {count}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Status Messages */}
|
||||
{error && (
|
||||
<div className="mb-6 p-4 bg-danger/10 border border-danger/20 rounded-lg">
|
||||
<div className="flex items-center gap-2">
|
||||
<ExclamationTriangleIcon className="h-5 w-5 text-danger" />
|
||||
<span className="text-danger">{error}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{lastJobResult && (
|
||||
<div className={`mb-6 p-4 rounded-lg border ${
|
||||
lastJobResult.success
|
||||
? 'bg-success/10 border-success/20'
|
||||
: 'bg-danger/10 border-danger/20'
|
||||
}`}>
|
||||
<div className="flex items-center gap-2">
|
||||
{lastJobResult.success ? (
|
||||
<CheckCircleIcon className="h-5 w-5 text-success" />
|
||||
) : (
|
||||
<ExclamationTriangleIcon className="h-5 w-5 text-danger" />
|
||||
)}
|
||||
<span className={lastJobResult.success ? 'text-success' : 'text-danger'}>
|
||||
{lastJobResult.message || lastJobResult.error}
|
||||
</span>
|
||||
{lastJobResult.jobId && (
|
||||
<span className="text-text-muted text-xs ml-auto">
|
||||
Job ID: {lastJobResult.jobId}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Operations Grid */}
|
||||
<div className="space-y-6">
|
||||
{/* Sync Operations */}
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-text-primary mb-4 flex items-center gap-2">
|
||||
<CloudArrowDownIcon className="h-5 w-5 text-primary-400" />
|
||||
Sync Operations
|
||||
</h2>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||
{operations.filter(op => op.category === 'sync').map(op => (
|
||||
<div
|
||||
key={op.id}
|
||||
className="bg-surface-secondary p-4 rounded-lg border border-border hover:border-primary-500/50 transition-colors"
|
||||
>
|
||||
<div className="flex items-start justify-between mb-2">
|
||||
<h3 className="font-medium text-text-primary">{op.name}</h3>
|
||||
<div className={getCategoryColor(op.category)}>
|
||||
{getCategoryIcon(op.category)}
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-sm text-text-secondary mb-3">{op.description}</p>
|
||||
|
||||
{/* Special inputs for specific operations */}
|
||||
{op.id === 'sync-provider-symbols' && (
|
||||
<div className="mb-3">
|
||||
<label className="block text-xs text-text-muted mb-1">Provider</label>
|
||||
<select
|
||||
value={selectedProvider}
|
||||
onChange={e => setSelectedProvider(e.target.value)}
|
||||
className="w-full px-2 py-1 text-sm bg-surface border border-border rounded focus:ring-1 focus:ring-primary-500 focus:border-primary-500"
|
||||
>
|
||||
<option value="yahoo">Yahoo</option>
|
||||
<option value="qm">QuestionsAndMethods</option>
|
||||
<option value="ib">Interactive Brokers</option>
|
||||
</select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{op.id === 'sync-all-exchanges' && (
|
||||
<div className="mb-3">
|
||||
<label className="flex items-center gap-2 text-sm">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={clearFirst}
|
||||
onChange={e => setClearFirst(e.target.checked)}
|
||||
className="rounded border-border text-primary-500 focus:ring-primary-500"
|
||||
/>
|
||||
<span className="text-text-secondary">Clear existing data first</span>
|
||||
</label>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<button
|
||||
onClick={() => handleOperation(op)}
|
||||
disabled={loading}
|
||||
className="w-full px-3 py-2 bg-primary-500/20 text-primary-400 rounded text-sm font-medium hover:bg-primary-500/30 disabled:opacity-50 disabled:cursor-not-allowed transition-colors flex items-center justify-center gap-2"
|
||||
>
|
||||
{loading ? (
|
||||
<>
|
||||
<ArrowPathIcon className="h-4 w-4 animate-spin" />
|
||||
Processing...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<CloudArrowDownIcon className="h-4 w-4" />
|
||||
Execute
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Maintenance Operations */}
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-text-primary mb-4 flex items-center gap-2">
|
||||
<ExclamationTriangleIcon className="h-5 w-5 text-warning" />
|
||||
Maintenance Operations
|
||||
</h2>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||
{operations.filter(op => op.category === 'maintenance').map(op => (
|
||||
<div
|
||||
key={op.id}
|
||||
className="bg-surface-secondary p-4 rounded-lg border border-warning/20 hover:border-warning/50 transition-colors"
|
||||
>
|
||||
<div className="flex items-start justify-between mb-2">
|
||||
<h3 className="font-medium text-text-primary">{op.name}</h3>
|
||||
<div className={getCategoryColor(op.category)}>
|
||||
{getCategoryIcon(op.category)}
|
||||
</div>
|
||||
</div>
|
||||
<p className="text-sm text-text-secondary mb-3">{op.description}</p>
|
||||
|
||||
{op.id === 'clear-postgresql' && (
|
||||
<div className="mb-3">
|
||||
<label className="block text-xs text-text-muted mb-1">Data Type</label>
|
||||
<select
|
||||
value={clearDataType}
|
||||
onChange={e => setClearDataType(e.target.value as any)}
|
||||
className="w-full px-2 py-1 text-sm bg-surface border border-border rounded focus:ring-1 focus:ring-warning focus:border-warning"
|
||||
>
|
||||
<option value="all">All Data</option>
|
||||
<option value="exchanges">Exchanges Only</option>
|
||||
<option value="provider_mappings">Provider Mappings Only</option>
|
||||
</select>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<button
|
||||
onClick={() => handleOperation(op)}
|
||||
disabled={loading}
|
||||
className="w-full px-3 py-2 bg-warning/20 text-warning rounded text-sm font-medium hover:bg-warning/30 disabled:opacity-50 disabled:cursor-not-allowed transition-colors flex items-center justify-center gap-2"
|
||||
>
|
||||
{loading ? (
|
||||
<>
|
||||
<ArrowPathIcon className="h-4 w-4 animate-spin" />
|
||||
Processing...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<ExclamationTriangleIcon className="h-4 w-4" />
|
||||
Execute
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue