diff --git a/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl b/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl index bd62e5b..0f0affb 100644 Binary files a/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl and b/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl differ diff --git a/apps/stock/web-api/src/index.ts b/apps/stock/web-api/src/index.ts index ffb7d2b..24b07ff 100644 --- a/apps/stock/web-api/src/index.ts +++ b/apps/stock/web-api/src/index.ts @@ -4,19 +4,23 @@ */ import { initializeStockConfig } from '@stock-bot/stock-config'; -import { - ServiceApplication, - createServiceContainerFromConfig, - initializeServices as initializeAwilixServices, -} from '@stock-bot/di'; +import { ServiceApplication } from '@stock-bot/di'; import { getLogger } from '@stock-bot/logger'; // Local imports import { createRoutes } from './routes/create-routes'; -import { setupServiceContainer } from './container-setup'; // Initialize configuration with service-specific overrides const config = initializeStockConfig('webApi'); + +// Override queue settings for web-api (no workers needed) +if (config.queue) { + config.queue.workers = 0; + config.queue.concurrency = 0; + config.queue.enableScheduledJobs = false; + config.queue.delayWorkerStart = true; +} + console.log('Web API Service Configuration:', JSON.stringify(config, null, 2)); // Create service application @@ -44,9 +48,15 @@ const app = new ServiceApplication( { // Custom lifecycle hooks onContainerReady: (container) => { - // Setup service-specific configuration - const enhancedContainer = setupServiceContainer(config, container); - return enhancedContainer; + // Override queue configuration to disable workers + const config = container.cradle.config; + if (config.queue) { + config.queue.workers = 0; + config.queue.concurrency = 0; + config.queue.enableScheduledJobs = false; + config.queue.delayWorkerStart = true; + } + return container; }, onStarted: (port) => { const logger = getLogger('web-api'); @@ -57,16 +67,21 @@ const app = new ServiceApplication( // Container factory function async function createContainer(config: any) { - const container = createServiceContainerFromConfig(config, { - enableQuestDB: false, // Web API doesn't need QuestDB - enableMongoDB: true, - enablePostgres: true, - enableCache: true, - enableQueue: false, // Web API doesn't need queue processing - enableBrowser: false, // Web API doesn't need browser - enableProxy: false, // Web API doesn't need proxy - }); - await initializeAwilixServices(container); + const { ServiceContainerBuilder } = await import('@stock-bot/di'); + + const container = await new ServiceContainerBuilder() + .withConfig(config) + .withOptions({ + enableQuestDB: false, // Disable QuestDB for now + enableMongoDB: true, + enablePostgres: true, + enableCache: true, + enableQueue: true, // Enable for pipeline operations + enableBrowser: false, // Web API doesn't need browser + enableProxy: false, // Web API doesn't need proxy + }) + .build(); // This automatically initializes services + return container; } diff --git a/apps/stock/web-api/src/routes/create-routes.ts b/apps/stock/web-api/src/routes/create-routes.ts index 5549ea4..6f6eee3 100644 --- a/apps/stock/web-api/src/routes/create-routes.ts +++ b/apps/stock/web-api/src/routes/create-routes.ts @@ -1,23 +1,29 @@ -/** - * Route factory for web API service - * Creates routes with access to the service container - */ - -import { Hono } from 'hono'; -import type { IServiceContainer } from '@stock-bot/handlers'; -import { createHealthRoutes } from './health.routes'; -import { createExchangeRoutes } from './exchange.routes'; - -export function createRoutes(container: IServiceContainer): Hono { - const app = new Hono(); - - // Create routes with container - const healthRoutes = createHealthRoutes(container); - const exchangeRoutes = createExchangeRoutes(container); - - // Mount routes - app.route('/health', healthRoutes); - app.route('/api/exchanges', exchangeRoutes); - - return app; +/** + * Route factory for web API service + * Creates routes with access to the service container + */ + +import { Hono } from 'hono'; +import type { IServiceContainer } from '@stock-bot/handlers'; +import { createHealthRoutes } from './health.routes'; +import { createExchangeRoutes } from './exchange.routes'; +import { createMonitoringRoutes } from './monitoring.routes'; +import { createPipelineRoutes } from './pipeline.routes'; + +export function createRoutes(container: IServiceContainer): Hono { + const app = new Hono(); + + // Create routes with container + const healthRoutes = createHealthRoutes(container); + const exchangeRoutes = createExchangeRoutes(container); + const monitoringRoutes = createMonitoringRoutes(container); + const pipelineRoutes = createPipelineRoutes(container); + + // Mount routes + app.route('/health', healthRoutes); + app.route('/api/exchanges', exchangeRoutes); + app.route('/api/system/monitoring', monitoringRoutes); + app.route('/api/pipeline', pipelineRoutes); + + return app; } \ No newline at end of file diff --git a/apps/stock/web-api/src/routes/monitoring.routes.ts b/apps/stock/web-api/src/routes/monitoring.routes.ts new file mode 100644 index 0000000..8efd6e7 --- /dev/null +++ b/apps/stock/web-api/src/routes/monitoring.routes.ts @@ -0,0 +1,183 @@ +/** + * Monitoring routes for system health and metrics + */ + +import { Hono } from 'hono'; +import type { IServiceContainer } from '@stock-bot/handlers'; +import { MonitoringService } from '../services/monitoring.service'; + +export function createMonitoringRoutes(container: IServiceContainer) { + const monitoring = new Hono(); + const monitoringService = new MonitoringService(container); + + /** + * Get overall system health + */ + monitoring.get('/', async (c) => { + try { + const health = await monitoringService.getSystemHealth(); + + // Set appropriate status code based on health + const statusCode = health.status === 'healthy' ? 200 : + health.status === 'degraded' ? 503 : 500; + + return c.json(health, statusCode); + } catch (error) { + return c.json({ + status: 'error', + message: 'Failed to retrieve system health', + error: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Get cache/Dragonfly statistics + */ + monitoring.get('/cache', async (c) => { + try { + const stats = await monitoringService.getCacheStats(); + return c.json(stats); + } catch (error) { + return c.json({ + error: 'Failed to retrieve cache statistics', + message: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Get queue statistics + */ + monitoring.get('/queues', async (c) => { + try { + const stats = await monitoringService.getQueueStats(); + return c.json({ queues: stats }); + } catch (error) { + return c.json({ + error: 'Failed to retrieve queue statistics', + message: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Get specific queue statistics + */ + monitoring.get('/queues/:name', async (c) => { + try { + const queueName = c.req.param('name'); + const stats = await monitoringService.getQueueStats(); + const queueStats = stats.find(q => q.name === queueName); + + if (!queueStats) { + return c.json({ + error: 'Queue not found', + message: `Queue '${queueName}' does not exist`, + }, 404); + } + + return c.json(queueStats); + } catch (error) { + return c.json({ + error: 'Failed to retrieve queue statistics', + message: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Get database statistics + */ + monitoring.get('/databases', async (c) => { + try { + const stats = await monitoringService.getDatabaseStats(); + return c.json({ databases: stats }); + } catch (error) { + return c.json({ + error: 'Failed to retrieve database statistics', + message: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Get specific database statistics + */ + monitoring.get('/databases/:type', async (c) => { + try { + const dbType = c.req.param('type') as 'postgres' | 'mongodb' | 'questdb'; + const stats = await monitoringService.getDatabaseStats(); + const dbStats = stats.find(db => db.type === dbType); + + if (!dbStats) { + return c.json({ + error: 'Database not found', + message: `Database type '${dbType}' not found or not enabled`, + }, 404); + } + + return c.json(dbStats); + } catch (error) { + return c.json({ + error: 'Failed to retrieve database statistics', + message: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Get service metrics + */ + monitoring.get('/metrics', async (c) => { + try { + const metrics = await monitoringService.getServiceMetrics(); + return c.json(metrics); + } catch (error) { + return c.json({ + error: 'Failed to retrieve service metrics', + message: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Get detailed cache info (Redis INFO command output) + */ + monitoring.get('/cache/info', async (c) => { + try { + if (!container.cache) { + return c.json({ + error: 'Cache not available', + message: 'Cache service is not enabled', + }, 503); + } + + const info = await container.cache.info(); + const stats = await monitoringService.getCacheStats(); + + return c.json({ + parsed: stats, + raw: info, + }); + } catch (error) { + return c.json({ + error: 'Failed to retrieve cache info', + message: error instanceof Error ? error.message : 'Unknown error', + }, 500); + } + }); + + /** + * Health check endpoint for monitoring + */ + monitoring.get('/ping', (c) => { + return c.json({ + status: 'ok', + timestamp: new Date().toISOString(), + service: 'monitoring', + }); + }); + + return monitoring; +} \ No newline at end of file diff --git a/apps/stock/web-api/src/routes/pipeline.routes.ts b/apps/stock/web-api/src/routes/pipeline.routes.ts new file mode 100644 index 0000000..1e19fc2 --- /dev/null +++ b/apps/stock/web-api/src/routes/pipeline.routes.ts @@ -0,0 +1,135 @@ +/** + * Pipeline Routes + * API endpoints for data pipeline operations + */ + +import { Hono } from 'hono'; +import type { IServiceContainer } from '@stock-bot/handlers'; +import { getLogger } from '@stock-bot/logger'; +import { PipelineService } from '../services/pipeline.service'; + +const logger = getLogger('pipeline-routes'); + +export function createPipelineRoutes(container: IServiceContainer) { + const pipeline = new Hono(); + const pipelineService = new PipelineService(container); + + // Symbol sync endpoints + pipeline.post('/symbols', async c => { + try { + const result = await pipelineService.syncQMSymbols(); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in POST /symbols', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + pipeline.post('/symbols/:provider', async c => { + try { + const provider = c.req.param('provider'); + const result = await pipelineService.syncProviderSymbols(provider); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in POST /symbols/:provider', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + // Exchange sync endpoints + pipeline.post('/exchanges', async c => { + try { + const result = await pipelineService.syncQMExchanges(); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in POST /exchanges', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + pipeline.post('/exchanges/all', async c => { + try { + const clearFirst = c.req.query('clear') === 'true'; + const result = await pipelineService.syncAllExchanges(clearFirst); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in POST /exchanges/all', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + // Provider mapping sync endpoints + pipeline.post('/provider-mappings/qm', async c => { + try { + const result = await pipelineService.syncQMProviderMappings(); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in POST /provider-mappings/qm', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + pipeline.post('/provider-mappings/ib', async c => { + try { + const result = await pipelineService.syncIBExchanges(); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in POST /provider-mappings/ib', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + // Status endpoint + pipeline.get('/status', async c => { + try { + const result = await pipelineService.getSyncStatus(); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in GET /status', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + // Clear data endpoint + pipeline.post('/clear/postgresql', async c => { + try { + const dataType = c.req.query('type') as 'exchanges' | 'provider_mappings' | 'all'; + const result = await pipelineService.clearPostgreSQLData(dataType || 'all'); + return c.json(result, result.success ? 200 : 503); + } catch (error) { + logger.error('Error in POST /clear/postgresql', { error }); + return c.json({ success: false, error: 'Internal server error' }, 500); + } + }); + + // Statistics endpoints + pipeline.get('/stats/exchanges', async c => { + try { + const result = await pipelineService.getExchangeStats(); + if (result.success) { + return c.json(result.data); + } else { + return c.json({ error: result.error }, 503); + } + } catch (error) { + logger.error('Error in GET /stats/exchanges', { error }); + return c.json({ error: 'Internal server error' }, 500); + } + }); + + pipeline.get('/stats/provider-mappings', async c => { + try { + const result = await pipelineService.getProviderMappingStats(); + if (result.success) { + return c.json(result.data); + } else { + return c.json({ error: result.error }, 503); + } + } catch (error) { + logger.error('Error in GET /stats/provider-mappings', { error }); + return c.json({ error: 'Internal server error' }, 500); + } + }); + + return pipeline; +} \ No newline at end of file diff --git a/apps/stock/web-api/src/services/monitoring.service.ts b/apps/stock/web-api/src/services/monitoring.service.ts new file mode 100644 index 0000000..40a2268 --- /dev/null +++ b/apps/stock/web-api/src/services/monitoring.service.ts @@ -0,0 +1,356 @@ +/** + * Monitoring Service + * Collects health and performance metrics from all system components + */ + +import type { IServiceContainer } from '@stock-bot/handlers'; +import { getLogger } from '@stock-bot/logger'; +import type { + CacheStats, + QueueStats, + DatabaseStats, + SystemHealth, + ServiceMetrics, + MetricSnapshot +} from '../types/monitoring.types'; + +export class MonitoringService { + private readonly logger = getLogger('monitoring-service'); + private startTime = Date.now(); + + constructor(private readonly container: IServiceContainer) {} + + /** + * Get cache/Dragonfly statistics + */ + async getCacheStats(): Promise { + try { + if (!this.container.cache) { + return { + provider: 'dragonfly', + connected: false, + }; + } + + // Get Redis/Dragonfly info + const info = await this.container.cache.info(); + const dbSize = await this.container.cache.dbsize(); + + // Parse memory stats from info + const memoryUsed = this.parseInfoValue(info, 'used_memory'); + const memoryPeak = this.parseInfoValue(info, 'used_memory_peak'); + + // Parse stats + const hits = this.parseInfoValue(info, 'keyspace_hits'); + const misses = this.parseInfoValue(info, 'keyspace_misses'); + const evictedKeys = this.parseInfoValue(info, 'evicted_keys'); + const expiredKeys = this.parseInfoValue(info, 'expired_keys'); + + return { + provider: 'dragonfly', + connected: true, + uptime: this.parseInfoValue(info, 'uptime_in_seconds'), + memoryUsage: { + used: memoryUsed, + peak: memoryPeak, + }, + stats: { + hits, + misses, + keys: dbSize, + evictedKeys, + expiredKeys, + }, + info: this.parseRedisInfo(info), + }; + } catch (error) { + this.logger.error('Failed to get cache stats', { error }); + return { + provider: 'dragonfly', + connected: false, + }; + } + } + + /** + * Get queue statistics + */ + async getQueueStats(): Promise { + const stats: QueueStats[] = []; + + try { + if (!this.container.queue) { + return stats; + } + + // Get all queue names from the queue manager + const queueManager = this.container.queue; + const queueNames = ['default', 'proxy', 'qm', 'ib', 'ceo', 'webshare']; // Add your queue names + + for (const queueName of queueNames) { + try { + const queue = queueManager.getQueue(queueName); + if (!queue) continue; + + const [waiting, active, completed, failed, delayed, paused] = await Promise.all([ + queue.getWaitingCount(), + queue.getActiveCount(), + queue.getCompletedCount(), + queue.getFailedCount(), + queue.getDelayedCount(), + queue.getPausedCount(), + ]); + + // Get worker info if available + const workers = queueManager.getWorker(queueName); + const workerInfo = workers ? { + count: 1, // Assuming single worker per queue + concurrency: workers.concurrency || 1, + } : undefined; + + stats.push({ + name: queueName, + connected: true, + jobs: { + waiting, + active, + completed, + failed, + delayed, + paused, + }, + workers: workerInfo, + }); + } catch (error) { + this.logger.warn(`Failed to get stats for queue ${queueName}`, { error }); + stats.push({ + name: queueName, + connected: false, + jobs: { + waiting: 0, + active: 0, + completed: 0, + failed: 0, + delayed: 0, + paused: 0, + }, + }); + } + } + } catch (error) { + this.logger.error('Failed to get queue stats', { error }); + } + + return stats; + } + + /** + * Get database statistics + */ + async getDatabaseStats(): Promise { + const stats: DatabaseStats[] = []; + + // PostgreSQL stats + if (this.container.postgres) { + try { + const startTime = Date.now(); + const result = await this.container.postgres.query('SELECT 1'); + const latency = Date.now() - startTime; + + // Get pool stats + const pool = (this.container.postgres as any).pool; + const poolStats = pool ? { + size: pool.totalCount || 0, + active: pool.idleCount || 0, + idle: pool.waitingCount || 0, + max: pool.options?.max || 0, + } : undefined; + + stats.push({ + type: 'postgres', + name: 'PostgreSQL', + connected: true, + latency, + pool: poolStats, + }); + } catch (error) { + this.logger.error('Failed to get PostgreSQL stats', { error }); + stats.push({ + type: 'postgres', + name: 'PostgreSQL', + connected: false, + }); + } + } + + // MongoDB stats + if (this.container.mongodb) { + try { + const startTime = Date.now(); + const db = this.container.mongodb.db(); + await db.admin().ping(); + const latency = Date.now() - startTime; + + const serverStatus = await db.admin().serverStatus(); + + stats.push({ + type: 'mongodb', + name: 'MongoDB', + connected: true, + latency, + stats: { + version: serverStatus.version, + uptime: serverStatus.uptime, + connections: serverStatus.connections, + opcounters: serverStatus.opcounters, + }, + }); + } catch (error) { + this.logger.error('Failed to get MongoDB stats', { error }); + stats.push({ + type: 'mongodb', + name: 'MongoDB', + connected: false, + }); + } + } + + // QuestDB stats + if (this.container.questdb) { + try { + const startTime = Date.now(); + // QuestDB health check + const response = await fetch(`http://${process.env.QUESTDB_HOST || 'localhost'}:9000/exec?query=SELECT%201`); + const latency = Date.now() - startTime; + + stats.push({ + type: 'questdb', + name: 'QuestDB', + connected: response.ok, + latency, + }); + } catch (error) { + this.logger.error('Failed to get QuestDB stats', { error }); + stats.push({ + type: 'questdb', + name: 'QuestDB', + connected: false, + }); + } + } + + return stats; + } + + /** + * Get system health summary + */ + async getSystemHealth(): Promise { + const [cacheStats, queueStats, databaseStats] = await Promise.all([ + this.getCacheStats(), + this.getQueueStats(), + this.getDatabaseStats(), + ]); + + const memory = process.memoryUsage(); + const uptime = Date.now() - this.startTime; + + // Determine overall health status + const errors: string[] = []; + + if (!cacheStats.connected) { + errors.push('Cache service is disconnected'); + } + + const disconnectedQueues = queueStats.filter(q => !q.connected); + if (disconnectedQueues.length > 0) { + errors.push(`${disconnectedQueues.length} queue(s) are disconnected`); + } + + const disconnectedDbs = databaseStats.filter(db => !db.connected); + if (disconnectedDbs.length > 0) { + errors.push(`${disconnectedDbs.length} database(s) are disconnected`); + } + + const status = errors.length === 0 ? 'healthy' : + errors.length < 3 ? 'degraded' : 'unhealthy'; + + return { + status, + timestamp: new Date().toISOString(), + uptime, + memory: { + used: memory.heapUsed, + total: memory.heapTotal, + percentage: (memory.heapUsed / memory.heapTotal) * 100, + }, + services: { + cache: cacheStats, + queues: queueStats, + databases: databaseStats, + }, + errors: errors.length > 0 ? errors : undefined, + }; + } + + /** + * Get service metrics (placeholder for future implementation) + */ + async getServiceMetrics(): Promise { + const now = new Date().toISOString(); + + return { + requestsPerSecond: { + timestamp: now, + value: 0, + unit: 'req/s', + }, + averageResponseTime: { + timestamp: now, + value: 0, + unit: 'ms', + }, + errorRate: { + timestamp: now, + value: 0, + unit: '%', + }, + activeConnections: { + timestamp: now, + value: 0, + unit: 'connections', + }, + }; + } + + /** + * Parse value from Redis INFO output + */ + private parseInfoValue(info: string, key: string): number { + const match = info.match(new RegExp(`${key}:(\\d+)`)); + return match ? parseInt(match[1], 10) : 0; + } + + /** + * Parse Redis INFO into structured object + */ + private parseRedisInfo(info: string): Record { + const result: Record = {}; + const sections = info.split('\r\n\r\n'); + + for (const section of sections) { + const lines = section.split('\r\n'); + const sectionName = lines[0]?.replace('# ', '') || 'general'; + result[sectionName] = {}; + + for (let i = 1; i < lines.length; i++) { + const [key, value] = lines[i].split(':'); + if (key && value) { + result[sectionName][key] = value; + } + } + } + + return result; + } +} \ No newline at end of file diff --git a/apps/stock/web-api/src/services/pipeline.service.ts b/apps/stock/web-api/src/services/pipeline.service.ts new file mode 100644 index 0000000..f95906f --- /dev/null +++ b/apps/stock/web-api/src/services/pipeline.service.ts @@ -0,0 +1,335 @@ +/** + * Pipeline Service + * Manages data pipeline operations by queuing jobs for the data-pipeline service + */ + +import type { IServiceContainer } from '@stock-bot/handlers'; +import { getLogger } from '@stock-bot/logger'; + +const logger = getLogger('pipeline-service'); + +export interface PipelineJobResult { + success: boolean; + jobId?: string; + message?: string; + error?: string; + data?: any; +} + +export interface PipelineStatsResult { + success: boolean; + data?: any; + error?: string; +} + +export class PipelineService { + constructor(private container: IServiceContainer) {} + + /** + * Queue a job to sync symbols from QuestionsAndMethods + */ + async syncQMSymbols(): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const symbolsQueue = queueManager.getQueue('symbols'); + const job = await symbolsQueue.addJob('sync-qm-symbols', { + handler: 'symbols', + operation: 'sync-qm-symbols', + payload: {}, + }); + + logger.info('QM symbols sync job queued', { jobId: job.id }); + return { success: true, jobId: job.id, message: 'QM symbols sync job queued' }; + } catch (error) { + logger.error('Failed to queue QM symbols sync job', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue sync job', + }; + } + } + + /** + * Queue a job to sync exchanges from QuestionsAndMethods + */ + async syncQMExchanges(): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const exchangesQueue = queueManager.getQueue('exchanges'); + const job = await exchangesQueue.addJob('sync-qm-exchanges', { + handler: 'exchanges', + operation: 'sync-qm-exchanges', + payload: {}, + }); + + logger.info('QM exchanges sync job queued', { jobId: job.id }); + return { success: true, jobId: job.id, message: 'QM exchanges sync job queued' }; + } catch (error) { + logger.error('Failed to queue QM exchanges sync job', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue sync job', + }; + } + } + + /** + * Queue a job to sync symbols from a specific provider + */ + async syncProviderSymbols(provider: string): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const symbolsQueue = queueManager.getQueue('symbols'); + const job = await symbolsQueue.addJob('sync-symbols-from-provider', { + handler: 'symbols', + operation: 'sync-symbols-from-provider', + payload: { provider }, + }); + + logger.info(`${provider} symbols sync job queued`, { jobId: job.id, provider }); + return { + success: true, + jobId: job.id, + message: `${provider} symbols sync job queued`, + }; + } catch (error) { + logger.error('Failed to queue provider symbols sync job', { error, provider }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue sync job', + }; + } + } + + /** + * Queue a job to sync all exchanges + */ + async syncAllExchanges(clearFirst: boolean = false): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const exchangesQueue = queueManager.getQueue('exchanges'); + const job = await exchangesQueue.addJob('sync-all-exchanges', { + handler: 'exchanges', + operation: 'sync-all-exchanges', + payload: { clearFirst }, + }); + + logger.info('Enhanced exchanges sync job queued', { jobId: job.id, clearFirst }); + return { + success: true, + jobId: job.id, + message: 'Enhanced exchanges sync job queued', + }; + } catch (error) { + logger.error('Failed to queue enhanced exchanges sync job', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue sync job', + }; + } + } + + /** + * Queue a job to sync QM provider mappings + */ + async syncQMProviderMappings(): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const exchangesQueue = queueManager.getQueue('exchanges'); + const job = await exchangesQueue.addJob('sync-qm-provider-mappings', { + handler: 'exchanges', + operation: 'sync-qm-provider-mappings', + payload: {}, + }); + + logger.info('QM provider mappings sync job queued', { jobId: job.id }); + return { + success: true, + jobId: job.id, + message: 'QM provider mappings sync job queued', + }; + } catch (error) { + logger.error('Failed to queue QM provider mappings sync job', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue sync job', + }; + } + } + + /** + * Queue a job to sync IB exchanges + */ + async syncIBExchanges(): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const exchangesQueue = queueManager.getQueue('exchanges'); + const job = await exchangesQueue.addJob('sync-ib-exchanges', { + handler: 'exchanges', + operation: 'sync-ib-exchanges', + payload: {}, + }); + + logger.info('IB exchanges sync job queued', { jobId: job.id }); + return { + success: true, + jobId: job.id, + message: 'IB exchanges sync job queued', + }; + } catch (error) { + logger.error('Failed to queue IB exchanges sync job', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue sync job', + }; + } + } + + /** + * Get sync status + */ + async getSyncStatus(): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const symbolsQueue = queueManager.getQueue('symbols'); + const job = await symbolsQueue.addJob('sync-status', { + handler: 'symbols', + operation: 'sync-status', + payload: {}, + }); + + logger.info('Sync status job queued', { jobId: job.id }); + return { + success: true, + jobId: job.id, + message: 'Sync status job queued', + }; + } catch (error) { + logger.error('Failed to queue sync status job', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue status job', + }; + } + } + + /** + * Clear PostgreSQL data + */ + async clearPostgreSQLData( + dataType: 'exchanges' | 'provider_mappings' | 'all' = 'all' + ): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const exchangesQueue = queueManager.getQueue('exchanges'); + const job = await exchangesQueue.addJob('clear-postgresql-data', { + handler: 'exchanges', + operation: 'clear-postgresql-data', + payload: { dataType }, + }); + + logger.info('PostgreSQL data clear job queued', { jobId: job.id, dataType }); + return { + success: true, + jobId: job.id, + message: 'PostgreSQL data clear job queued', + }; + } catch (error) { + logger.error('Failed to queue PostgreSQL clear job', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to queue clear job', + }; + } + } + + /** + * Get exchange statistics (waits for result) + */ + async getExchangeStats(): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const exchangesQueue = queueManager.getQueue('exchanges'); + const job = await exchangesQueue.addJob('get-exchange-stats', { + handler: 'exchanges', + operation: 'get-exchange-stats', + payload: {}, + }); + + // Wait for job to complete and return result + const result = await job.waitUntilFinished(); + return { success: true, data: result }; + } catch (error) { + logger.error('Failed to get exchange stats', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to get stats', + }; + } + } + + /** + * Get provider mapping statistics (waits for result) + */ + async getProviderMappingStats(): Promise { + try { + const queueManager = this.container.queue; + if (!queueManager) { + return { success: false, error: 'Queue manager not available' }; + } + + const exchangesQueue = queueManager.getQueue('exchanges'); + const job = await exchangesQueue.addJob('get-provider-mapping-stats', { + handler: 'exchanges', + operation: 'get-provider-mapping-stats', + payload: {}, + }); + + // Wait for job to complete and return result + const result = await job.waitUntilFinished(); + return { success: true, data: result }; + } catch (error) { + logger.error('Failed to get provider mapping stats', { error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Failed to get stats', + }; + } + } +} \ No newline at end of file diff --git a/apps/stock/web-api/src/types/monitoring.types.ts b/apps/stock/web-api/src/types/monitoring.types.ts new file mode 100644 index 0000000..f9c0992 --- /dev/null +++ b/apps/stock/web-api/src/types/monitoring.types.ts @@ -0,0 +1,93 @@ +/** + * Monitoring types for system health and metrics + */ + +export interface CacheStats { + provider: string; + connected: boolean; + uptime?: number; + memoryUsage?: { + used: number; + peak: number; + total?: number; + }; + stats?: { + hits: number; + misses: number; + keys: number; + evictedKeys?: number; + expiredKeys?: number; + }; + info?: Record; +} + +export interface QueueStats { + name: string; + connected: boolean; + jobs: { + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused: number; + }; + workers?: { + count: number; + concurrency: number; + }; + throughput?: { + processed: number; + failed: number; + avgProcessingTime?: number; + }; +} + +export interface DatabaseStats { + type: 'postgres' | 'mongodb' | 'questdb'; + name: string; + connected: boolean; + latency?: number; + pool?: { + size: number; + active: number; + idle: number; + waiting?: number; + max: number; + }; + stats?: Record; +} + +export interface SystemHealth { + status: 'healthy' | 'degraded' | 'unhealthy'; + timestamp: string; + uptime: number; + memory: { + used: number; + total: number; + percentage: number; + }; + cpu?: { + usage: number; + loadAverage?: number[]; + }; + services: { + cache: CacheStats; + queues: QueueStats[]; + databases: DatabaseStats[]; + }; + errors?: string[]; +} + +export interface MetricSnapshot { + timestamp: string; + value: number; + unit?: string; +} + +export interface ServiceMetrics { + requestsPerSecond: MetricSnapshot; + averageResponseTime: MetricSnapshot; + errorRate: MetricSnapshot; + activeConnections: MetricSnapshot; +} \ No newline at end of file diff --git a/apps/stock/web-app/.env b/apps/stock/web-app/.env index 1988dde..7b014ba 100644 --- a/apps/stock/web-app/.env +++ b/apps/stock/web-app/.env @@ -1,5 +1,5 @@ # API Configuration -VITE_API_BASE_URL=http://localhost:4000/api +VITE_API_BASE_URL=http://localhost:2003 VITE_DATA_SERVICE_URL=http://localhost:3001 VITE_PORTFOLIO_SERVICE_URL=http://localhost:3002 VITE_STRATEGY_SERVICE_URL=http://localhost:3003 diff --git a/apps/stock/web-app/.env.example b/apps/stock/web-app/.env.example index feaddbf..79c7087 100644 --- a/apps/stock/web-app/.env.example +++ b/apps/stock/web-app/.env.example @@ -1,9 +1,12 @@ # API Configuration -VITE_API_BASE_URL=http://localhost:8080 +# Web API service URL (default port: 2003) +VITE_API_BASE_URL=http://localhost:2003 + +# Other services (if needed in the future) VITE_DATA_SERVICE_URL=http://localhost:3001 VITE_PORTFOLIO_SERVICE_URL=http://localhost:3002 VITE_STRATEGY_SERVICE_URL=http://localhost:3003 VITE_EXECUTION_SERVICE_URL=http://localhost:3004 # Environment -VITE_NODE_ENV=development +VITE_NODE_ENV=development \ No newline at end of file diff --git a/apps/stock/web-app/src/app/App.tsx b/apps/stock/web-app/src/app/App.tsx index e73c2b0..aed1130 100644 --- a/apps/stock/web-app/src/app/App.tsx +++ b/apps/stock/web-app/src/app/App.tsx @@ -1,6 +1,8 @@ import { Layout } from '@/components/layout'; import { DashboardPage } from '@/features/dashboard'; import { ExchangesPage } from '@/features/exchanges'; +import { MonitoringPage } from '@/features/monitoring'; +import { PipelinePage } from '@/features/pipeline'; import { BrowserRouter, Navigate, Route, Routes } from 'react-router-dom'; export function App() { @@ -24,6 +26,8 @@ export function App() { element={
Analytics Page - Coming Soon
} /> Settings Page - Coming Soon} /> + } /> + } /> diff --git a/apps/stock/web-app/src/components/layout/Layout.tsx b/apps/stock/web-app/src/components/layout/Layout.tsx index 6f50405..dc777ca 100644 --- a/apps/stock/web-app/src/components/layout/Layout.tsx +++ b/apps/stock/web-app/src/components/layout/Layout.tsx @@ -10,7 +10,17 @@ export function Layout() { // Determine title from current route const getTitle = () => { const path = location.pathname.replace('/', ''); - if (!path || path === 'dashboard') {return 'Dashboard';} + if (!path || path === 'dashboard') return 'Dashboard'; + + // Handle nested routes + if (path.includes('/')) { + const parts = path.split('/'); + // For system routes, show the sub-page name + if (parts[0] === 'system' && parts[1]) { + return parts[1].charAt(0).toUpperCase() + parts[1].slice(1); + } + } + return path.charAt(0).toUpperCase() + path.slice(1); }; diff --git a/apps/stock/web-app/src/components/layout/Sidebar.tsx b/apps/stock/web-app/src/components/layout/Sidebar.tsx index f6822b0..8289993 100644 --- a/apps/stock/web-app/src/components/layout/Sidebar.tsx +++ b/apps/stock/web-app/src/components/layout/Sidebar.tsx @@ -1,9 +1,9 @@ import { navigation } from '@/lib/constants'; import { cn } from '@/lib/utils'; import { Dialog, Transition } from '@headlessui/react'; -import { XMarkIcon } from '@heroicons/react/24/outline'; -import { Fragment } from 'react'; -import { NavLink } from 'react-router-dom'; +import { XMarkIcon, ChevronDownIcon, ChevronRightIcon } from '@heroicons/react/24/outline'; +import { Fragment, useState } from 'react'; +import { NavLink, useLocation } from 'react-router-dom'; interface SidebarProps { sidebarOpen: boolean; @@ -76,6 +76,35 @@ export function Sidebar({ sidebarOpen, setSidebarOpen }: SidebarProps) { } function SidebarContent() { + const location = useLocation(); + + // Auto-expand items that have active children + const getInitialExpanded = () => { + const expanded = new Set(); + navigation.forEach(item => { + if (item.children && item.children.some(child => location.pathname === child.href)) { + expanded.add(item.name); + } + }); + return expanded; + }; + + const [expandedItems, setExpandedItems] = useState>(getInitialExpanded()); + + const toggleExpanded = (name: string) => { + const newExpanded = new Set(expandedItems); + if (newExpanded.has(name)) { + newExpanded.delete(name); + } else { + newExpanded.add(name); + } + setExpandedItems(newExpanded); + }; + + const isChildActive = (children: any[]) => { + return children.some(child => location.pathname === child.href); + }; + return (
@@ -87,32 +116,96 @@ function SidebarContent() {
    {navigation.map(item => (
  • - - cn( - isActive - ? 'bg-surface-secondary text-primary-400 border-l-2 border-primary-500' - : 'text-text-secondary hover:text-primary-400 hover:bg-surface-secondary', - 'group flex gap-x-2 rounded-r-md px-2 py-1.5 text-sm leading-tight font-medium transition-colors' - ) - } - > - {({ isActive }) => ( - <> + {item.children ? ( + <> + + {expandedItems.has(item.name) && ( +
      + {item.children.map(child => ( +
    • + + cn( + isActive + ? 'bg-surface-secondary text-primary-400 border-l-2 border-primary-500' + : 'text-text-secondary hover:text-primary-400 hover:bg-surface-secondary', + 'group flex gap-x-2 rounded-r-md px-2 py-1 text-sm leading-tight font-medium transition-colors' + ) + } + > + {({ isActive }) => ( + <> + +
      • + ))} +
    + )} + + ) : ( + + cn( + isActive + ? 'bg-surface-secondary text-primary-400 border-l-2 border-primary-500' + : 'text-text-secondary hover:text-primary-400 hover:bg-surface-secondary', + 'group flex gap-x-2 rounded-r-md px-2 py-1.5 text-sm leading-tight font-medium transition-colors' + ) + } + > + {({ isActive }) => ( + <> + + )}
    • ))}
diff --git a/apps/stock/web-app/src/components/ui/DataTable/DataTable.tsx b/apps/stock/web-app/src/components/ui/DataTable/DataTable.tsx index 49f110d..e9a3eb2 100644 --- a/apps/stock/web-app/src/components/ui/DataTable/DataTable.tsx +++ b/apps/stock/web-app/src/components/ui/DataTable/DataTable.tsx @@ -1,15 +1,13 @@ import { cn } from '@/lib/utils'; import { ChevronDownIcon, ChevronUpIcon } from '@heroicons/react/24/outline'; import { - ColumnDef, flexRender, getCoreRowModel, getExpandedRowModel, getSortedRowModel, - Row, - SortingState, useReactTable, } from '@tanstack/react-table'; +import type { ColumnDef, Row, SortingState } from '@tanstack/react-table'; import { useState, useRef } from 'react'; import { TableVirtuoso } from 'react-virtuoso'; diff --git a/apps/stock/web-app/src/features/dashboard/components/PortfolioTable.tsx b/apps/stock/web-app/src/features/dashboard/components/PortfolioTable.tsx index 0132578..b31f574 100644 --- a/apps/stock/web-app/src/features/dashboard/components/PortfolioTable.tsx +++ b/apps/stock/web-app/src/features/dashboard/components/PortfolioTable.tsx @@ -1,5 +1,5 @@ import { DataTable } from '@/components/ui'; -import { ColumnDef } from '@tanstack/react-table'; +import type { ColumnDef } from '@tanstack/react-table'; import React from 'react'; interface PortfolioItem { diff --git a/apps/stock/web-app/src/features/exchanges/components/AddExchangeDialog.tsx b/apps/stock/web-app/src/features/exchanges/components/AddExchangeDialog.tsx index 3c7ca75..0e7a0ff 100644 --- a/apps/stock/web-app/src/features/exchanges/components/AddExchangeDialog.tsx +++ b/apps/stock/web-app/src/features/exchanges/components/AddExchangeDialog.tsx @@ -1,8 +1,8 @@ -import { Dialog, DialogContent, DialogHeader, DialogTitle, Button } from '@/components/ui'; +import { Button, Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui'; import { useCallback } from 'react'; -import { CreateExchangeRequest, AddExchangeDialogProps } from '../types'; -import { validateExchangeForm } from '../utils/validation'; import { useFormValidation } from '../hooks/useFormValidation'; +import type { AddExchangeDialogProps, CreateExchangeRequest } from '../types'; +import { validateExchangeForm } from '../utils/validation'; const initialFormData: CreateExchangeRequest = { code: '', diff --git a/apps/stock/web-app/src/features/exchanges/components/AddProviderMappingDialog.tsx b/apps/stock/web-app/src/features/exchanges/components/AddProviderMappingDialog.tsx index f571c15..8052875 100644 --- a/apps/stock/web-app/src/features/exchanges/components/AddProviderMappingDialog.tsx +++ b/apps/stock/web-app/src/features/exchanges/components/AddProviderMappingDialog.tsx @@ -1,7 +1,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle, Button } from '@/components/ui'; import { useCallback, useEffect, useState } from 'react'; import { useExchanges } from '../hooks/useExchanges'; -import { CreateProviderMappingRequest } from '../types'; +import type { CreateProviderMappingRequest } from '../types/index'; interface AddProviderMappingDialogProps { isOpen: boolean; diff --git a/apps/stock/web-app/src/features/exchanges/components/AddSourceDialog.tsx b/apps/stock/web-app/src/features/exchanges/components/AddSourceDialog.tsx deleted file mode 100644 index bc44c1b..0000000 --- a/apps/stock/web-app/src/features/exchanges/components/AddSourceDialog.tsx +++ /dev/null @@ -1,216 +0,0 @@ -import { Dialog, Transition } from '@headlessui/react'; -import { XMarkIcon } from '@heroicons/react/24/outline'; -import React, { useState } from 'react'; -import { AddSourceRequest } from '../types'; - -interface AddSourceDialogProps { - isOpen: boolean; - onClose: () => void; - onAddSource: (request: AddSourceRequest) => Promise; - exchangeId: string; - exchangeName: string; -} - -export function AddSourceDialog({ - isOpen, - onClose, - onAddSource, - exchangeName, -}: AddSourceDialogProps) { - const [source, setSource] = useState(''); - const [sourceCode, setSourceCode] = useState(''); - const [id, setId] = useState(''); - const [name, setName] = useState(''); - const [code, setCode] = useState(''); - const [aliases, setAliases] = useState(''); - const [loading, setLoading] = useState(false); - - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - if (!source || !sourceCode || !id || !name || !code) {return;} - - setLoading(true); - try { - await onAddSource({ - source, - source_code: sourceCode, - mapping: { - id, - name, - code, - aliases: aliases - .split(',') - .map(a => a.trim()) - .filter(Boolean), - }, - }); - - // Reset form - setSource(''); - setSourceCode(''); - setId(''); - setName(''); - setCode(''); - setAliases(''); - } catch (_error) { - // TODO: Implement proper error handling/toast notification - // eslint-disable-next-line no-console - console.error('Error adding source:', _error); - } finally { - setLoading(false); - } - }; - - return ( - - - -
- - -
-
- - -
- - Add Source to {exchangeName} - - -
- -
-
- - -
- -
- - setSourceCode(e.target.value)} - className="w-full bg-surface border border-border rounded px-3 py-2 text-text-primary focus:ring-1 focus:ring-primary-500 focus:border-primary-500" - placeholder="e.g., IB, ALP, POLY" - required - /> -
- -
- - setId(e.target.value)} - className="w-full bg-surface border border-border rounded px-3 py-2 text-text-primary focus:ring-1 focus:ring-primary-500 focus:border-primary-500" - placeholder="e.g., NYSE, NASDAQ" - required - /> -
- -
- - setName(e.target.value)} - className="w-full bg-surface border border-border rounded px-3 py-2 text-text-primary focus:ring-1 focus:ring-primary-500 focus:border-primary-500" - placeholder="e.g., New York Stock Exchange" - required - /> -
- -
- - setCode(e.target.value)} - className="w-full bg-surface border border-border rounded px-3 py-2 text-text-primary focus:ring-1 focus:ring-primary-500 focus:border-primary-500" - placeholder="e.g., NYSE" - required - /> -
- -
- - setAliases(e.target.value)} - className="w-full bg-surface border border-border rounded px-3 py-2 text-text-primary focus:ring-1 focus:ring-primary-500 focus:border-primary-500" - placeholder="e.g., NYSE, New York, Big Board" - /> -
- -
- - -
-
- - -
-
-
- - ); -} diff --git a/apps/stock/web-app/src/features/exchanges/components/ExchangesTable.tsx b/apps/stock/web-app/src/features/exchanges/components/ExchangesTable.tsx index 52133f0..96018b5 100644 --- a/apps/stock/web-app/src/features/exchanges/components/ExchangesTable.tsx +++ b/apps/stock/web-app/src/features/exchanges/components/ExchangesTable.tsx @@ -1,12 +1,12 @@ import { DataTable } from '@/components/ui'; import { PlusIcon, TrashIcon } from '@heroicons/react/24/outline'; -import { ColumnDef } from '@tanstack/react-table'; +import type { ColumnDef } from '@tanstack/react-table'; import { useCallback, useMemo, useState } from 'react'; import { useExchanges } from '../hooks/useExchanges'; -import { Exchange, EditingCell, AddProviderMappingDialogState, DeleteDialogState } from '../types'; +import type { AddProviderMappingDialogState, DeleteDialogState, EditingCell, Exchange } from '../types'; +import { formatDate, formatProviderMapping, getProviderMappingColor, sortProviderMappings } from '../utils/formatters'; import { AddProviderMappingDialog } from './AddProviderMappingDialog'; import { DeleteExchangeDialog } from './DeleteExchangeDialog'; -import { sortProviderMappings, getProviderMappingColor, formatProviderMapping, formatDate } from '../utils/formatters'; export function ExchangesTable() { const { @@ -235,7 +235,7 @@ export function ExchangesTable() { cell: ({ getValue, row }) => { const totalMappings = parseInt(getValue() as string) || 0; const activeMappings = parseInt(row.original.active_mapping_count) || 0; - const _verifiedMappings = parseInt(row.original.verified_mapping_count) || 0; + // const _verifiedMappings = parseInt(row.original.verified_mapping_count) || 0; // Get provider mappings directly from the exchange data const mappings = row.original.provider_mappings || []; @@ -329,7 +329,7 @@ export function ExchangesTable() {

Error Loading Exchanges

{error}

- Make sure the web-api service is running on localhost:4000 + Make sure the web-api service is running on localhost:2003

); diff --git a/apps/stock/web-app/src/features/exchanges/components/index.ts b/apps/stock/web-app/src/features/exchanges/components/index.ts index fd15d0d..2f6b5a7 100644 --- a/apps/stock/web-app/src/features/exchanges/components/index.ts +++ b/apps/stock/web-app/src/features/exchanges/components/index.ts @@ -1,4 +1,4 @@ -export { AddSourceDialog } from './AddSourceDialog'; + export { AddProviderMappingDialog } from './AddProviderMappingDialog'; export { AddExchangeDialog } from './AddExchangeDialog'; export { DeleteExchangeDialog } from './DeleteExchangeDialog'; diff --git a/apps/stock/web-app/src/features/exchanges/hooks/useExchanges.ts b/apps/stock/web-app/src/features/exchanges/hooks/useExchanges.ts index 89b9688..2333984 100644 --- a/apps/stock/web-app/src/features/exchanges/hooks/useExchanges.ts +++ b/apps/stock/web-app/src/features/exchanges/hooks/useExchanges.ts @@ -1,6 +1,6 @@ import { useCallback, useEffect, useState } from 'react'; import { exchangeApi } from '../services/exchangeApi'; -import { +import type { CreateExchangeRequest, CreateProviderMappingRequest, Exchange, @@ -10,7 +10,7 @@ import { ProviderMapping, UpdateExchangeRequest, UpdateProviderMappingRequest, -} from '../types'; +} from '../types/index'; export function useExchanges() { const [exchanges, setExchanges] = useState([]); diff --git a/apps/stock/web-app/src/features/exchanges/hooks/useFormValidation.ts b/apps/stock/web-app/src/features/exchanges/hooks/useFormValidation.ts index 5b2d2f3..8c1a44b 100644 --- a/apps/stock/web-app/src/features/exchanges/hooks/useFormValidation.ts +++ b/apps/stock/web-app/src/features/exchanges/hooks/useFormValidation.ts @@ -1,5 +1,5 @@ import { useCallback, useState } from 'react'; -import { FormErrors } from '../types'; +import type { FormErrors } from '../types'; export function useFormValidation(initialData: T, validateFn: (data: T) => FormErrors) { const [formData, setFormData] = useState(initialData); diff --git a/apps/stock/web-app/src/features/exchanges/services/exchangeApi.ts b/apps/stock/web-app/src/features/exchanges/services/exchangeApi.ts index fb3d7c9..3f1f448 100644 --- a/apps/stock/web-app/src/features/exchanges/services/exchangeApi.ts +++ b/apps/stock/web-app/src/features/exchanges/services/exchangeApi.ts @@ -1,4 +1,4 @@ -import { +import type { ApiResponse, CreateExchangeRequest, CreateProviderMappingRequest, @@ -9,13 +9,13 @@ import { ProviderMapping, UpdateExchangeRequest, UpdateProviderMappingRequest, -} from '../types'; +} from '../types/index'; -const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || 'http://localhost:4000/api'; +const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || 'http://localhost:2003'; class ExchangeApiService { private async request(endpoint: string, options?: RequestInit): Promise> { - const url = `${API_BASE_URL}${endpoint}`; + const url = `${API_BASE_URL}/api${endpoint}`; const response = await fetch(url, { headers: { diff --git a/apps/stock/web-app/src/features/exchanges/types/index.ts b/apps/stock/web-app/src/features/exchanges/types/index.ts index f5d7cbc..ff687b3 100644 --- a/apps/stock/web-app/src/features/exchanges/types/index.ts +++ b/apps/stock/web-app/src/features/exchanges/types/index.ts @@ -1,7 +1,154 @@ -// Re-export all types from organized files -export * from './api.types'; -export * from './request.types'; -export * from './component.types'; +// API Response types +export interface ApiResponse { + success: boolean; + data?: T; + error?: string; + message?: string; + total?: number; +} + +// Base entity types +export interface BaseEntity { + id: string; + created_at: string; + updated_at: string; +} + +export interface ProviderMapping extends BaseEntity { + provider: string; + provider_exchange_code: string; + provider_exchange_name: string; + master_exchange_id: string; + country_code: string | null; + currency: string | null; + confidence: number; + active: boolean; + verified: boolean; + auto_mapped: boolean; + master_exchange_code?: string; + master_exchange_name?: string; + master_exchange_active?: boolean; +} + +export interface Exchange extends BaseEntity { + code: string; + name: string; + country: string; + currency: string; + active: boolean; + visible: boolean; + provider_mapping_count: string; + active_mapping_count: string; + verified_mapping_count: string; + providers: string | null; + provider_mappings: ProviderMapping[]; +} + +export interface ExchangeDetails { + exchange: Exchange; + provider_mappings: ProviderMapping[]; +} + +export interface ProviderExchange { + provider_exchange_code: string; + provider_exchange_name: string; + country_code: string | null; + currency: string | null; + symbol_count: number | null; +} + +export interface ExchangeStats { + total_exchanges: string; + active_exchanges: string; + countries: string; + currencies: string; + total_provider_mappings: string; + active_provider_mappings: string; + verified_provider_mappings: string; + providers: string; +} + +// Request types for API calls +export interface CreateExchangeRequest { + code: string; + name: string; + country: string; + currency: string; + active?: boolean; +} + +export interface UpdateExchangeRequest { + name?: string; + active?: boolean; + visible?: boolean; + country?: string; + currency?: string; +} + +export interface CreateProviderMappingRequest { + provider: string; + provider_exchange_code: string; + provider_exchange_name?: string; + master_exchange_id: string; + country_code?: string; + currency?: string; + confidence?: number; + active?: boolean; + verified?: boolean; +} + +export interface UpdateProviderMappingRequest { + active?: boolean; + verified?: boolean; + confidence?: number; + master_exchange_id?: string; +} + +// Component-specific types +export interface EditingCell { + id: string; + field: string; +} + +export interface AddProviderMappingDialogState { + exchangeId: string; + exchangeName: string; +} + +export interface DeleteDialogState { + exchangeId: string; + exchangeName: string; + providerMappingCount: number; +} + +export interface FormErrors { + [key: string]: string; +} + +// Dialog props interfaces +export interface BaseDialogProps { + isOpen: boolean; + onClose: () => void; +} + +export interface AddExchangeDialogProps extends BaseDialogProps { + onCreateExchange: (request: CreateExchangeRequest) => Promise; +} + +export interface AddProviderMappingDialogProps extends BaseDialogProps { + exchangeId: string; + exchangeName: string; + onCreateMapping: ( + request: CreateProviderMappingRequest + ) => Promise; +} + +export interface DeleteExchangeDialogProps extends BaseDialogProps { + exchangeId: string; + exchangeName: string; + providerMappingCount: number; + onConfirmDelete: (exchangeId: string) => Promise; +} // Legacy compatibility - can be removed later -export type ExchangesApiResponse = import('./api.types').ApiResponse; +export type ExchangesApiResponse = ApiResponse; \ No newline at end of file diff --git a/apps/stock/web-app/src/features/exchanges/utils/formatters.ts b/apps/stock/web-app/src/features/exchanges/utils/formatters.ts index 2572c33..cf583f3 100644 --- a/apps/stock/web-app/src/features/exchanges/utils/formatters.ts +++ b/apps/stock/web-app/src/features/exchanges/utils/formatters.ts @@ -1,4 +1,4 @@ -import { ProviderMapping } from '../types'; +import type { ProviderMapping } from '../types'; export function formatDate(dateString: string): string { return new Date(dateString).toLocaleDateString(); diff --git a/apps/stock/web-app/src/features/exchanges/utils/validation.ts b/apps/stock/web-app/src/features/exchanges/utils/validation.ts index 61e03ee..007ac48 100644 --- a/apps/stock/web-app/src/features/exchanges/utils/validation.ts +++ b/apps/stock/web-app/src/features/exchanges/utils/validation.ts @@ -1,4 +1,4 @@ -import { FormErrors } from '../types'; +import type { FormErrors } from '../types'; export function validateExchangeForm(data: { code: string; diff --git a/apps/stock/web-app/src/features/monitoring/MonitoringPage.tsx b/apps/stock/web-app/src/features/monitoring/MonitoringPage.tsx new file mode 100644 index 0000000..35e4a2c --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/MonitoringPage.tsx @@ -0,0 +1,104 @@ +/** + * System Monitoring Page + */ + +import React, { useState } from 'react'; +import { useSystemHealth, useCacheStats, useQueueStats, useDatabaseStats } from './hooks'; +import { SystemHealthCard, CacheStatsCard, QueueStatsTable, DatabaseStatsGrid } from './components'; + +export function MonitoringPage() { + const [refreshInterval, setRefreshInterval] = useState(5000); // 5 seconds default + + const { data: health, loading: healthLoading, error: healthError } = useSystemHealth(refreshInterval); + const { data: cache, loading: cacheLoading, error: cacheError } = useCacheStats(refreshInterval); + const { data: queues, loading: queuesLoading, error: queuesError } = useQueueStats(refreshInterval); + const { data: databases, loading: dbLoading, error: dbError } = useDatabaseStats(refreshInterval); + + const handleRefreshIntervalChange = (e: React.ChangeEvent) => { + setRefreshInterval(Number(e.target.value)); + }; + + if (healthLoading || cacheLoading || queuesLoading || dbLoading) { + return ( +
+
+
+

Loading monitoring data...

+
+
+ ); + } + + const hasErrors = healthError || cacheError || queuesError || dbError; + + return ( +
+
+
+

System Monitoring

+
+
+ + +
+
+
+
+ + {hasErrors && ( +
+

Errors occurred while fetching data:

+
    + {healthError &&
  • System Health: {healthError}
    • } + {cacheError &&
  • Cache Stats: {cacheError}
    • } + {queuesError &&
  • Queue Stats: {queuesError}
    • } + {dbError &&
  • Database Stats: {dbError}
    • } +
+
+ )} + +
+ {/* System Health */} + {health && ( +
+
+ +
+
+ {cache && } +
+
+ )} + + {/* Database Stats */} + {databases && databases.length > 0 && ( +
+

Database Connections

+ +
+ )} + + {/* Queue Stats */} + {queues && queues.length > 0 && ( +
+

Queue Status

+ +
+ )} +
+
+ ); +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/components/CacheStatsCard.tsx b/apps/stock/web-app/src/features/monitoring/components/CacheStatsCard.tsx new file mode 100644 index 0000000..39bfe06 --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/components/CacheStatsCard.tsx @@ -0,0 +1,96 @@ +/** + * Cache Statistics Card Component + */ + +import React from 'react'; +import { Card } from '../../../components/ui/Card'; +import type { CacheStats } from '../types'; + +interface CacheStatsCardProps { + stats: CacheStats; +} + +export function CacheStatsCard({ stats }: CacheStatsCardProps) { + const formatBytes = (bytes: number) => { + const mb = bytes / 1024 / 1024; + return mb.toFixed(2) + ' MB'; + }; + + const hitRate = stats.stats && (stats.stats.hits + stats.stats.misses) > 0 + ? (stats.stats.hits / (stats.stats.hits + stats.stats.misses) * 100).toFixed(1) + : '0'; + + return ( + +
+

Cache (Dragonfly)

+ + {stats.connected ? 'Connected' : 'Disconnected'} + +
+ + {stats.connected ? ( +
+ {stats.memoryUsage && ( +
+
+
Memory Used
+
{formatBytes(stats.memoryUsage.used)}
+
+
+
Peak Memory
+
{formatBytes(stats.memoryUsage.peak)}
+
+
+ )} + + {stats.stats && ( + <> +
+
+
Hit Rate
+
{hitRate}%
+
+
+
Total Keys
+
{stats.stats.keys.toLocaleString()}
+
+
+ +
+
+ Hits: {stats.stats.hits.toLocaleString()} +
+
+ Misses: {stats.stats.misses.toLocaleString()} +
+ {stats.stats.evictedKeys !== undefined && ( +
+ Evicted: {stats.stats.evictedKeys.toLocaleString()} +
+ )} + {stats.stats.expiredKeys !== undefined && ( +
+ Expired: {stats.stats.expiredKeys.toLocaleString()} +
+ )} +
+ + )} + + {stats.uptime && ( +
+ Uptime: {Math.floor(stats.uptime / 3600)}h {Math.floor((stats.uptime % 3600) / 60)}m +
+ )} +
+ ) : ( +
+ Cache service is not available +
+ )} + + ); +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/components/DatabaseStatsGrid.tsx b/apps/stock/web-app/src/features/monitoring/components/DatabaseStatsGrid.tsx new file mode 100644 index 0000000..5850057 --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/components/DatabaseStatsGrid.tsx @@ -0,0 +1,104 @@ +/** + * Database Statistics Grid Component + */ + +import React from 'react'; +import { Card } from '../../../components/ui/Card'; +import type { DatabaseStats } from '../types'; + +interface DatabaseStatsGridProps { + databases: DatabaseStats[]; +} + +export function DatabaseStatsGrid({ databases }: DatabaseStatsGridProps) { + const getDbIcon = (type: string) => { + switch (type) { + case 'postgres': + return '🐘'; + case 'mongodb': + return '🍃'; + case 'questdb': + return '⚡'; + default: + return '💾'; + } + }; + + return ( +
+ {databases.map((db) => ( + +
+
+ {getDbIcon(db.type)} +

{db.name}

+
+ + {db.connected ? 'Connected' : 'Disconnected'} + +
+ + {db.connected ? ( +
+ {db.latency !== undefined && ( +
+
Latency
+
{db.latency}ms
+
+ )} + + {db.pool && ( +
+
Connection Pool
+
+
+ Active: {db.pool.active} +
+
+ Idle: {db.pool.idle} +
+
+ Size: {db.pool.size} +
+
+ Max: {db.pool.max} +
+
+ + {db.pool.max > 0 && ( +
+
+
+
+
+ {((db.pool.size / db.pool.max) * 100).toFixed(0)}% utilized +
+
+ )} +
+ )} + + {db.type === 'mongodb' && db.stats && ( +
+
Version: {db.stats.version}
+ {db.stats.connections && ( +
Connections: {db.stats.connections.current}/{db.stats.connections.available}
+ )} +
+ )} +
+ ) : ( +
+ Database is not available +
+ )} + + ))} +
+ ); +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/components/QueueStatsTable.tsx b/apps/stock/web-app/src/features/monitoring/components/QueueStatsTable.tsx new file mode 100644 index 0000000..96d51ad --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/components/QueueStatsTable.tsx @@ -0,0 +1,77 @@ +/** + * Queue Statistics Table Component + */ + +import React from 'react'; +import { Card } from '../../../components/ui/Card'; +import type { QueueStats } from '../types'; + +interface QueueStatsTableProps { + queues: QueueStats[]; +} + +export function QueueStatsTable({ queues }: QueueStatsTableProps) { + const totalJobs = (queue: QueueStats) => { + const { jobs } = queue; + return jobs.waiting + jobs.active + jobs.completed + jobs.failed + jobs.delayed + jobs.paused; + }; + + return ( + +

Queue Statistics

+ + {queues.length > 0 ? ( +
+ + + + + + + + + + + + + + + {queues.map((queue) => ( + + + + + + + + + + + ))} + +
QueueStatusWaitingActiveCompletedFailedDelayedTotal
{queue.name} + + {queue.jobs.waiting.toLocaleString()} + {queue.jobs.active > 0 ? ( + {queue.jobs.active} + ) : ( + queue.jobs.active + )} + {queue.jobs.completed.toLocaleString()} + {queue.jobs.failed > 0 ? ( + {queue.jobs.failed} + ) : ( + queue.jobs.failed + )} + {queue.jobs.delayed.toLocaleString()}{totalJobs(queue).toLocaleString()}
+
+ ) : ( +
+ No queue data available +
+ )} + + ); +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/components/SystemHealthCard.tsx b/apps/stock/web-app/src/features/monitoring/components/SystemHealthCard.tsx new file mode 100644 index 0000000..3d2901a --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/components/SystemHealthCard.tsx @@ -0,0 +1,87 @@ +/** + * System Health Card Component + */ + +import React from 'react'; +import { Card } from '../../../components/ui/Card'; +import type { SystemHealth } from '../types'; + +interface SystemHealthCardProps { + health: SystemHealth; +} + +export function SystemHealthCard({ health }: SystemHealthCardProps) { + const statusColor = { + healthy: 'text-green-600 bg-green-100', + degraded: 'text-yellow-600 bg-yellow-100', + unhealthy: 'text-red-600 bg-red-100', + }[health.status]; + + const formatUptime = (ms: number) => { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + const hours = Math.floor(minutes / 60); + const days = Math.floor(hours / 24); + + if (days > 0) return `${days}d ${hours % 24}h`; + if (hours > 0) return `${hours}h ${minutes % 60}m`; + if (minutes > 0) return `${minutes}m ${seconds % 60}s`; + return `${seconds}s`; + }; + + const formatBytes = (bytes: number) => { + const gb = bytes / 1024 / 1024 / 1024; + return gb.toFixed(2) + ' GB'; + }; + + return ( + +
+

System Health

+ + {health.status.toUpperCase()} + +
+ +
+
+
Uptime
+
{formatUptime(health.uptime)}
+
+ +
+
Memory Usage
+
+
+ {formatBytes(health.memory.used)} / {formatBytes(health.memory.total)} + {health.memory.percentage.toFixed(1)}% +
+
+
+
+
+
+ + {health.errors && health.errors.length > 0 && ( +
+
Issues
+
    + {health.errors.map((error, index) => ( +
  • + • {error} +
    • + ))} +
+
+ )} + +
+ Last updated: {new Date(health.timestamp).toLocaleTimeString()} +
+
+ + ); +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/components/index.ts b/apps/stock/web-app/src/features/monitoring/components/index.ts new file mode 100644 index 0000000..d6c4b60 --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/components/index.ts @@ -0,0 +1,8 @@ +/** + * Monitoring components exports + */ + +export { SystemHealthCard } from './SystemHealthCard'; +export { CacheStatsCard } from './CacheStatsCard'; +export { QueueStatsTable } from './QueueStatsTable'; +export { DatabaseStatsGrid } from './DatabaseStatsGrid'; \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/hooks/index.ts b/apps/stock/web-app/src/features/monitoring/hooks/index.ts new file mode 100644 index 0000000..22327de --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/hooks/index.ts @@ -0,0 +1,5 @@ +/** + * Monitoring hooks exports + */ + +export * from './useMonitoring'; \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/hooks/useMonitoring.ts b/apps/stock/web-app/src/features/monitoring/hooks/useMonitoring.ts new file mode 100644 index 0000000..d22cd6d --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/hooks/useMonitoring.ts @@ -0,0 +1,123 @@ +/** + * Custom hook for monitoring data + */ + +import { useState, useEffect, useCallback } from 'react'; +import { monitoringApi } from '../services/monitoringApi'; +import type { SystemHealth, CacheStats, QueueStats, DatabaseStats } from '../types'; + +export function useSystemHealth(refreshInterval: number = 5000) { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + try { + const health = await monitoringApi.getSystemHealth(); + setData(health); + setError(null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to fetch system health'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + + if (refreshInterval > 0) { + const interval = setInterval(fetchData, refreshInterval); + return () => clearInterval(interval); + } + }, [fetchData, refreshInterval]); + + return { data, loading, error, refetch: fetchData }; +} + +export function useCacheStats(refreshInterval: number = 5000) { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + try { + const stats = await monitoringApi.getCacheStats(); + setData(stats); + setError(null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to fetch cache stats'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + + if (refreshInterval > 0) { + const interval = setInterval(fetchData, refreshInterval); + return () => clearInterval(interval); + } + }, [fetchData, refreshInterval]); + + return { data, loading, error, refetch: fetchData }; +} + +export function useQueueStats(refreshInterval: number = 5000) { + const [data, setData] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + try { + const result = await monitoringApi.getQueueStats(); + setData(result.queues); + setError(null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to fetch queue stats'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + + if (refreshInterval > 0) { + const interval = setInterval(fetchData, refreshInterval); + return () => clearInterval(interval); + } + }, [fetchData, refreshInterval]); + + return { data, loading, error, refetch: fetchData }; +} + +export function useDatabaseStats(refreshInterval: number = 5000) { + const [data, setData] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + try { + const result = await monitoringApi.getDatabaseStats(); + setData(result.databases); + setError(null); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to fetch database stats'); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + + if (refreshInterval > 0) { + const interval = setInterval(fetchData, refreshInterval); + return () => clearInterval(interval); + } + }, [fetchData, refreshInterval]); + + return { data, loading, error, refetch: fetchData }; +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/index.ts b/apps/stock/web-app/src/features/monitoring/index.ts new file mode 100644 index 0000000..f56a62a --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/index.ts @@ -0,0 +1,8 @@ +/** + * Monitoring feature exports + */ + +export { MonitoringPage } from './MonitoringPage'; +export * from './types'; +export * from './hooks/useMonitoring'; +export * from './services/monitoringApi'; \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/services/monitoringApi.ts b/apps/stock/web-app/src/features/monitoring/services/monitoringApi.ts new file mode 100644 index 0000000..52b2581 --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/services/monitoringApi.ts @@ -0,0 +1,87 @@ +/** + * Monitoring API Service + */ + +import type { SystemHealth, CacheStats, QueueStats, DatabaseStats } from '../types'; + +const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || 'http://localhost:2003'; +const MONITORING_BASE = `${API_BASE_URL}/api/system/monitoring`; + +export const monitoringApi = { + /** + * Get overall system health + */ + async getSystemHealth(): Promise { + const response = await fetch(MONITORING_BASE); + if (!response.ok) { + throw new Error(`Failed to fetch system health: ${response.statusText}`); + } + return response.json(); + }, + + /** + * Get cache statistics + */ + async getCacheStats(): Promise { + const response = await fetch(`${MONITORING_BASE}/cache`); + if (!response.ok) { + throw new Error(`Failed to fetch cache stats: ${response.statusText}`); + } + return response.json(); + }, + + /** + * Get queue statistics + */ + async getQueueStats(): Promise<{ queues: QueueStats[] }> { + const response = await fetch(`${MONITORING_BASE}/queues`); + if (!response.ok) { + throw new Error(`Failed to fetch queue stats: ${response.statusText}`); + } + return response.json(); + }, + + /** + * Get specific queue statistics + */ + async getQueueStatsByName(name: string): Promise { + const response = await fetch(`${MONITORING_BASE}/queues/${name}`); + if (!response.ok) { + throw new Error(`Failed to fetch queue ${name} stats: ${response.statusText}`); + } + return response.json(); + }, + + /** + * Get database statistics + */ + async getDatabaseStats(): Promise<{ databases: DatabaseStats[] }> { + const response = await fetch(`${MONITORING_BASE}/databases`); + if (!response.ok) { + throw new Error(`Failed to fetch database stats: ${response.statusText}`); + } + return response.json(); + }, + + /** + * Get specific database statistics + */ + async getDatabaseStatsByType(type: 'postgres' | 'mongodb' | 'questdb'): Promise { + const response = await fetch(`${MONITORING_BASE}/databases/${type}`); + if (!response.ok) { + throw new Error(`Failed to fetch ${type} stats: ${response.statusText}`); + } + return response.json(); + }, + + /** + * Get detailed cache info + */ + async getCacheInfo(): Promise<{ parsed: CacheStats; raw: string }> { + const response = await fetch(`${MONITORING_BASE}/cache/info`); + if (!response.ok) { + throw new Error(`Failed to fetch cache info: ${response.statusText}`); + } + return response.json(); + }, +}; \ No newline at end of file diff --git a/apps/stock/web-app/src/features/monitoring/types/index.ts b/apps/stock/web-app/src/features/monitoring/types/index.ts new file mode 100644 index 0000000..9365e44 --- /dev/null +++ b/apps/stock/web-app/src/features/monitoring/types/index.ts @@ -0,0 +1,80 @@ +/** + * Monitoring types for system health and metrics + */ + +export interface CacheStats { + provider: string; + connected: boolean; + uptime?: number; + memoryUsage?: { + used: number; + peak: number; + total?: number; + }; + stats?: { + hits: number; + misses: number; + keys: number; + evictedKeys?: number; + expiredKeys?: number; + }; + info?: Record; +} + +export interface QueueStats { + name: string; + connected: boolean; + jobs: { + waiting: number; + active: number; + completed: number; + failed: number; + delayed: number; + paused: number; + }; + workers?: { + count: number; + concurrency: number; + }; + throughput?: { + processed: number; + failed: number; + avgProcessingTime?: number; + }; +} + +export interface DatabaseStats { + type: 'postgres' | 'mongodb' | 'questdb'; + name: string; + connected: boolean; + latency?: number; + pool?: { + size: number; + active: number; + idle: number; + waiting?: number; + max: number; + }; + stats?: Record; +} + +export interface SystemHealth { + status: 'healthy' | 'degraded' | 'unhealthy'; + timestamp: string; + uptime: number; + memory: { + used: number; + total: number; + percentage: number; + }; + cpu?: { + usage: number; + loadAverage?: number[]; + }; + services: { + cache: CacheStats; + queues: QueueStats[]; + databases: DatabaseStats[]; + }; + errors?: string[]; +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/pipeline/PipelinePage.tsx b/apps/stock/web-app/src/features/pipeline/PipelinePage.tsx new file mode 100644 index 0000000..2e69dfa --- /dev/null +++ b/apps/stock/web-app/src/features/pipeline/PipelinePage.tsx @@ -0,0 +1,406 @@ +import { useState, useEffect } from 'react'; +import { + ArrowPathIcon, + CircleStackIcon, + CloudArrowDownIcon, + ExclamationTriangleIcon, + CheckCircleIcon, + ClockIcon, +} from '@heroicons/react/24/outline'; +import { usePipeline } from './hooks/usePipeline'; +import type { PipelineOperation } from './types'; + +const operations: PipelineOperation[] = [ + // Symbol operations + { + id: 'sync-qm-symbols', + name: 'Sync QM Symbols', + description: 'Sync symbols from QuestionsAndMethods API', + endpoint: '/symbols', + method: 'POST', + category: 'sync', + }, + { + id: 'sync-provider-symbols', + name: 'Sync Provider Symbols', + description: 'Sync symbols from a specific provider', + endpoint: '/symbols/:provider', + method: 'POST', + category: 'sync', + params: { provider: 'yahoo' }, // Default provider + }, + // Exchange operations + { + id: 'sync-qm-exchanges', + name: 'Sync QM Exchanges', + description: 'Sync exchanges from QuestionsAndMethods API', + endpoint: '/exchanges', + method: 'POST', + category: 'sync', + }, + { + id: 'sync-all-exchanges', + name: 'Sync All Exchanges', + description: 'Sync all exchanges with optional clear', + endpoint: '/exchanges/all', + method: 'POST', + category: 'sync', + }, + // Provider mapping operations + { + id: 'sync-qm-provider-mappings', + name: 'Sync QM Provider Mappings', + description: 'Sync provider mappings from QuestionsAndMethods', + endpoint: '/provider-mappings/qm', + method: 'POST', + category: 'sync', + }, + { + id: 'sync-ib-exchanges', + name: 'Sync IB Exchanges', + description: 'Sync exchanges from Interactive Brokers', + endpoint: '/provider-mappings/ib', + method: 'POST', + category: 'sync', + }, + // Maintenance operations + { + id: 'clear-postgresql', + name: 'Clear PostgreSQL Data', + description: 'Clear exchange and provider mapping data', + endpoint: '/clear/postgresql', + method: 'POST', + category: 'maintenance', + dangerous: true, + }, +]; + +export function PipelinePage() { + const { + loading, + error, + lastJobResult, + syncQMSymbols, + syncProviderSymbols, + syncQMExchanges, + syncAllExchanges, + syncQMProviderMappings, + syncIBExchanges, + clearPostgreSQLData, + getExchangeStats, + getProviderMappingStats, + } = usePipeline(); + + const [selectedProvider, setSelectedProvider] = useState('yahoo'); + const [clearFirst, setClearFirst] = useState(false); + const [clearDataType, setClearDataType] = useState<'all' | 'exchanges' | 'provider_mappings'>('all'); + const [stats, setStats] = useState<{ exchanges?: any; providerMappings?: any }>({}); + + // Load stats on mount + useEffect(() => { + loadStats(); + }, []); + + const loadStats = async () => { + const [exchangeStats, mappingStats] = await Promise.all([ + getExchangeStats(), + getProviderMappingStats(), + ]); + setStats({ + exchanges: exchangeStats, + providerMappings: mappingStats, + }); + }; + + const handleOperation = async (op: PipelineOperation) => { + switch (op.id) { + case 'sync-qm-symbols': + await syncQMSymbols(); + break; + case 'sync-provider-symbols': + await syncProviderSymbols(selectedProvider); + break; + case 'sync-qm-exchanges': + await syncQMExchanges(); + break; + case 'sync-all-exchanges': + await syncAllExchanges(clearFirst); + break; + case 'sync-qm-provider-mappings': + await syncQMProviderMappings(); + break; + case 'sync-ib-exchanges': + await syncIBExchanges(); + break; + case 'clear-postgresql': + if (confirm(`Are you sure you want to clear ${clearDataType} data? This action cannot be undone.`)) { + await clearPostgreSQLData(clearDataType); + } + break; + } + // Reload stats after operation + await loadStats(); + }; + + const getCategoryIcon = (category: string) => { + switch (category) { + case 'sync': + return ; + case 'maintenance': + return ; + default: + return ; + } + }; + + const getCategoryColor = (category: string) => { + switch (category) { + case 'sync': + return 'text-primary-400'; + case 'maintenance': + return 'text-warning'; + default: + return 'text-text-secondary'; + } + }; + + return ( +
+
+

Data Pipeline Management

+

+ Manage data synchronization and maintenance operations +

+
+ + {/* Stats Overview */} + {(stats.exchanges || stats.providerMappings) && ( +
+ {stats.exchanges && ( +
+

Exchange Statistics

+
+
+ Total Exchanges: + {stats.exchanges.totalExchanges} +
+
+ Active Exchanges: + {stats.exchanges.activeExchanges} +
+
+ Total Provider Mappings: + {stats.exchanges.totalProviderMappings} +
+
+ Active Mappings: + {stats.exchanges.activeProviderMappings} +
+
+
+ )} + + {stats.providerMappings && ( +
+

Provider Mapping Statistics

+
+
+ Coverage: + + {stats.providerMappings.coveragePercentage?.toFixed(1)}% + +
+
+ Verified Mappings: + {stats.providerMappings.verifiedMappings} +
+
+ Auto-mapped: + {stats.providerMappings.autoMappedCount} +
+ {stats.providerMappings.mappingsByProvider && ( +
+ By Provider: +
+ {Object.entries(stats.providerMappings.mappingsByProvider).map(([provider, count]) => ( + + {provider}: {count} + + ))} +
+
+ )} +
+
+ )} +
+ )} + + {/* Status Messages */} + {error && ( +
+
+ + {error} +
+
+ )} + + {lastJobResult && ( +
+
+ {lastJobResult.success ? ( + + ) : ( + + )} + + {lastJobResult.message || lastJobResult.error} + + {lastJobResult.jobId && ( + + Job ID: {lastJobResult.jobId} + + )} +
+
+ )} + + {/* Operations Grid */} +
+ {/* Sync Operations */} +
+

+ + Sync Operations +

+
+ {operations.filter(op => op.category === 'sync').map(op => ( +
+
+

{op.name}

+
+ {getCategoryIcon(op.category)} +
+
+

{op.description}

+ + {/* Special inputs for specific operations */} + {op.id === 'sync-provider-symbols' && ( +
+ + +
+ )} + + {op.id === 'sync-all-exchanges' && ( +
+ +
+ )} + + +
+ ))} +
+
+ + {/* Maintenance Operations */} +
+

+ + Maintenance Operations +

+
+ {operations.filter(op => op.category === 'maintenance').map(op => ( +
+
+

{op.name}

+
+ {getCategoryIcon(op.category)} +
+
+

{op.description}

+ + {op.id === 'clear-postgresql' && ( +
+ + +
+ )} + + +
+ ))} +
+
+
+
+ ); +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/pipeline/hooks/usePipeline.ts b/apps/stock/web-app/src/features/pipeline/hooks/usePipeline.ts new file mode 100644 index 0000000..9a14810 --- /dev/null +++ b/apps/stock/web-app/src/features/pipeline/hooks/usePipeline.ts @@ -0,0 +1,159 @@ +import { useCallback, useState } from 'react'; +import { pipelineApi } from '../services/pipelineApi'; +import type { + DataClearType, + ExchangeStats, + PipelineJobResult, + ProviderMappingStats, +} from '../types'; + +export function usePipeline() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [lastJobResult, setLastJobResult] = useState(null); + + const executeOperation = useCallback(async ( + operation: () => Promise + ): Promise => { + try { + setLoading(true); + setError(null); + const result = await operation(); + setLastJobResult(result); + if (!result.success) { + setError(result.error || 'Operation failed'); + return false; + } + return true; + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Unknown error occurred'; + setError(errorMessage); + setLastJobResult({ success: false, error: errorMessage }); + return false; + } finally { + setLoading(false); + } + }, []); + + // Symbol sync operations + const syncQMSymbols = useCallback( + () => executeOperation(() => pipelineApi.syncQMSymbols()), + [executeOperation] + ); + + const syncProviderSymbols = useCallback( + (provider: string) => executeOperation(() => pipelineApi.syncProviderSymbols(provider)), + [executeOperation] + ); + + // Exchange sync operations + const syncQMExchanges = useCallback( + () => executeOperation(() => pipelineApi.syncQMExchanges()), + [executeOperation] + ); + + const syncAllExchanges = useCallback( + (clearFirst: boolean = false) => + executeOperation(() => pipelineApi.syncAllExchanges(clearFirst)), + [executeOperation] + ); + + // Provider mapping sync operations + const syncQMProviderMappings = useCallback( + () => executeOperation(() => pipelineApi.syncQMProviderMappings()), + [executeOperation] + ); + + const syncIBExchanges = useCallback( + () => executeOperation(() => pipelineApi.syncIBExchanges()), + [executeOperation] + ); + + // Maintenance operations + const clearPostgreSQLData = useCallback( + (dataType: DataClearType = 'all') => + executeOperation(() => pipelineApi.clearPostgreSQLData(dataType)), + [executeOperation] + ); + + // Status and stats operations + const getSyncStatus = useCallback(async () => { + try { + setLoading(true); + setError(null); + const result = await pipelineApi.getSyncStatus(); + return result; + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to get sync status'; + setError(errorMessage); + return null; + } finally { + setLoading(false); + } + }, []); + + const getExchangeStats = useCallback(async (): Promise => { + try { + setLoading(true); + setError(null); + const result = await pipelineApi.getExchangeStats(); + if (result.success && result.data) { + return result.data as ExchangeStats; + } + setError(result.error || 'Failed to get exchange stats'); + return null; + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to get exchange stats'; + setError(errorMessage); + return null; + } finally { + setLoading(false); + } + }, []); + + const getProviderMappingStats = useCallback(async (): Promise => { + try { + setLoading(true); + setError(null); + const result = await pipelineApi.getProviderMappingStats(); + if (result.success && result.data) { + return result.data as ProviderMappingStats; + } + setError(result.error || 'Failed to get provider mapping stats'); + return null; + } catch (err) { + const errorMessage = err instanceof Error ? err.message : 'Failed to get provider mapping stats'; + setError(errorMessage); + return null; + } finally { + setLoading(false); + } + }, []); + + return { + // State + loading, + error, + lastJobResult, + + // Symbol operations + syncQMSymbols, + syncProviderSymbols, + + // Exchange operations + syncQMExchanges, + syncAllExchanges, + + // Provider mapping operations + syncQMProviderMappings, + syncIBExchanges, + + // Maintenance operations + clearPostgreSQLData, + + // Status and stats operations + getSyncStatus, + getExchangeStats, + getProviderMappingStats, + }; +} \ No newline at end of file diff --git a/apps/stock/web-app/src/features/pipeline/index.ts b/apps/stock/web-app/src/features/pipeline/index.ts new file mode 100644 index 0000000..c4040e8 --- /dev/null +++ b/apps/stock/web-app/src/features/pipeline/index.ts @@ -0,0 +1,3 @@ +export { PipelinePage } from './PipelinePage'; +export * from './hooks/usePipeline'; +export * from './types'; \ No newline at end of file diff --git a/apps/stock/web-app/src/features/pipeline/services/pipelineApi.ts b/apps/stock/web-app/src/features/pipeline/services/pipelineApi.ts new file mode 100644 index 0000000..949a43d --- /dev/null +++ b/apps/stock/web-app/src/features/pipeline/services/pipelineApi.ts @@ -0,0 +1,82 @@ +import type { + DataClearType, + PipelineJobResult, + PipelineStatsResult, +} from '../types'; + +const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || 'http://localhost:2003'; + +class PipelineApiService { + private async request( + endpoint: string, + options?: RequestInit + ): Promise { + const url = `${API_BASE_URL}/pipeline${endpoint}`; + + const response = await fetch(url, { + headers: { + 'Content-Type': 'application/json', + ...options?.headers, + }, + ...options, + }); + + const data = await response.json(); + + if (!response.ok) { + throw new Error(data.error || `HTTP ${response.status}: ${response.statusText}`); + } + + return data; + } + + // Symbol sync operations + async syncQMSymbols(): Promise { + return this.request('/symbols', { method: 'POST' }); + } + + async syncProviderSymbols(provider: string): Promise { + return this.request(`/symbols/${provider}`, { method: 'POST' }); + } + + // Exchange sync operations + async syncQMExchanges(): Promise { + return this.request('/exchanges', { method: 'POST' }); + } + + async syncAllExchanges(clearFirst: boolean = false): Promise { + const params = clearFirst ? '?clear=true' : ''; + return this.request(`/exchanges/all${params}`, { method: 'POST' }); + } + + // Provider mapping sync operations + async syncQMProviderMappings(): Promise { + return this.request('/provider-mappings/qm', { method: 'POST' }); + } + + async syncIBExchanges(): Promise { + return this.request('/provider-mappings/ib', { method: 'POST' }); + } + + // Status and maintenance operations + async getSyncStatus(): Promise { + return this.request('/status'); + } + + async clearPostgreSQLData(dataType: DataClearType = 'all'): Promise { + const params = `?type=${dataType}`; + return this.request(`/clear/postgresql${params}`, { method: 'POST' }); + } + + // Statistics operations + async getExchangeStats(): Promise { + return this.request('/stats/exchanges'); + } + + async getProviderMappingStats(): Promise { + return this.request('/stats/provider-mappings'); + } +} + +// Export singleton instance +export const pipelineApi = new PipelineApiService(); \ No newline at end of file diff --git a/apps/stock/web-app/src/features/pipeline/types/index.ts b/apps/stock/web-app/src/features/pipeline/types/index.ts new file mode 100644 index 0000000..b5a0f78 --- /dev/null +++ b/apps/stock/web-app/src/features/pipeline/types/index.ts @@ -0,0 +1,58 @@ +// Pipeline API types + +export interface PipelineJobResult { + success: boolean; + jobId?: string; + message?: string; + error?: string; + data?: any; +} + +export interface PipelineStatsResult { + success: boolean; + data?: any; + error?: string; +} + +export interface ExchangeStats { + totalExchanges: number; + activeExchanges: number; + totalProviderMappings: number; + activeProviderMappings: number; + verifiedProviderMappings: number; + providers: string[]; +} + +export interface ProviderMappingStats { + totalMappings: number; + activeMappings: number; + verifiedMappings: number; + autoMappedCount: number; + mappingsByProvider: Record; + coveragePercentage: number; +} + +export interface SyncStatus { + lastSync?: { + symbols?: string; + exchanges?: string; + providerMappings?: string; + }; + pendingJobs?: number; + activeJobs?: number; + completedJobs?: number; + failedJobs?: number; +} + +export type DataClearType = 'exchanges' | 'provider_mappings' | 'all'; + +export interface PipelineOperation { + id: string; + name: string; + description: string; + endpoint: string; + method: 'GET' | 'POST'; + category: 'sync' | 'stats' | 'maintenance'; + dangerous?: boolean; + params?: Record; +} \ No newline at end of file diff --git a/apps/stock/web-app/src/lib/constants.ts b/apps/stock/web-app/src/lib/constants.ts index f1478d5..f1428d8 100644 --- a/apps/stock/web-app/src/lib/constants.ts +++ b/apps/stock/web-app/src/lib/constants.ts @@ -5,13 +5,31 @@ import { DocumentTextIcon, HomeIcon, PresentationChartLineIcon, + ServerStackIcon, + CircleStackIcon, + ChartPieIcon, } from '@heroicons/react/24/outline'; -export const navigation = [ +export interface NavigationItem { + name: string; + href?: string; + icon: any; + children?: NavigationItem[]; +} + +export const navigation: NavigationItem[] = [ { name: 'Dashboard', href: '/dashboard', icon: HomeIcon }, { name: 'Exchanges', href: '/exchanges', icon: BuildingLibraryIcon }, { name: 'Portfolio', href: '/portfolio', icon: ChartBarIcon }, { name: 'Strategies', href: '/strategies', icon: DocumentTextIcon }, { name: 'Analytics', href: '/analytics', icon: PresentationChartLineIcon }, + { + name: 'System', + icon: ServerStackIcon, + children: [ + { name: 'Monitoring', href: '/system/monitoring', icon: ChartPieIcon }, + { name: 'Pipeline', href: '/system/pipeline', icon: CircleStackIcon }, + ] + }, { name: 'Settings', href: '/settings', icon: CogIcon }, ]; diff --git a/apps/stock/web-app/src/lib/constants/navigation.ts b/apps/stock/web-app/src/lib/constants/navigation.ts index 3bd3353..1351bee 100644 --- a/apps/stock/web-app/src/lib/constants/navigation.ts +++ b/apps/stock/web-app/src/lib/constants/navigation.ts @@ -5,6 +5,7 @@ import { CurrencyDollarIcon, DocumentTextIcon, HomeIcon, + ServerIcon, } from '@heroicons/react/24/outline'; export const navigation = [ @@ -38,6 +39,12 @@ export const navigation = [ icon: DocumentTextIcon, current: false, }, + { + name: 'System', + href: '/system/monitoring', + icon: ServerIcon, + current: false, + }, { name: 'Settings', href: '/settings', diff --git a/apps/stock/web-app/tsconfig.json b/apps/stock/web-app/tsconfig.json index 0c1ef41..145593e 100644 --- a/apps/stock/web-app/tsconfig.json +++ b/apps/stock/web-app/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../../tsconfig.json", "compilerOptions": { "target": "ES2020", "useDefineForClassFields": true,