renaming services to more suitable names

apps/data-pipeline/src/index.ts

@@ -0,0 +1,267 @@
+// Framework imports
+import { initializeServiceConfig } from '@stock-bot/config';
+import { Hono } from 'hono';
+import { cors } from 'hono/cors';
+// Library imports
+import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
+import { connectMongoDB } from '@stock-bot/mongodb-client';
+import { connectPostgreSQL } from '@stock-bot/postgres-client';
+import { QueueManager, type QueueManagerConfig } from '@stock-bot/queue';
+import { Shutdown } from '@stock-bot/shutdown';
+// Local imports
+import { enhancedSyncRoutes, healthRoutes, statsRoutes, syncRoutes } from './routes';
+
+const config = initializeServiceConfig();
+console.log('Data Sync Service Configuration:', JSON.stringify(config, null, 2));
+const serviceConfig = config.service;
+const databaseConfig = config.database;
+const queueConfig = config.queue;
+
+if (config.log) {
+  setLoggerConfig({
+    logLevel: config.log.level,
+    logConsole: true,
+    logFile: false,
+    environment: config.environment,
+    hideObject: config.log.hideObject,
+  });
+}
+
+// Create logger AFTER config is set
+const logger = getLogger('data-pipeline');
+
+const app = new Hono();
+
+// Add CORS middleware
+app.use(
+  '*',
+  cors({
+    origin: '*',
+    allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'],
+    allowHeaders: ['Content-Type', 'Authorization'],
+    credentials: false,
+  })
+);
+const PORT = serviceConfig.port;
+let server: ReturnType<typeof Bun.serve> | null = null;
+// Singleton clients are managed in libraries
+let queueManager: QueueManager | null = null;
+
+// Initialize shutdown manager
+const shutdown = Shutdown.getInstance({ timeout: 15000 });
+
+// Mount routes
+app.route('/health', healthRoutes);
+app.route('/sync', syncRoutes);
+app.route('/sync', enhancedSyncRoutes);
+app.route('/sync/stats', statsRoutes);
+
+// Initialize services
+async function initializeServices() {
+  logger.info('Initializing data sync service...');
+
+  try {
+    // Initialize MongoDB client singleton
+    logger.debug('Connecting to MongoDB...');
+    const mongoConfig = databaseConfig.mongodb;
+    await connectMongoDB({
+      uri: mongoConfig.uri,
+      database: mongoConfig.database,
+      host: mongoConfig.host || 'localhost',
+      port: mongoConfig.port || 27017,
+      timeouts: {
+        connectTimeout: 30000,
+        socketTimeout: 30000,
+        serverSelectionTimeout: 5000,
+      },
+    });
+    logger.info('MongoDB connected');
+
+    // Initialize PostgreSQL client singleton
+    logger.debug('Connecting to PostgreSQL...');
+    const pgConfig = databaseConfig.postgres;
+    await connectPostgreSQL({
+      host: pgConfig.host,
+      port: pgConfig.port,
+      database: pgConfig.database,
+      username: pgConfig.user,
+      password: pgConfig.password,
+      poolSettings: {
+        min: 2,
+        max: pgConfig.poolSize || 10,
+        idleTimeoutMillis: pgConfig.idleTimeout || 30000,
+      },
+    });
+    logger.info('PostgreSQL connected');
+
+    // Initialize queue system (with delayed worker start)
+    logger.debug('Initializing queue system...');
+    const queueManagerConfig: QueueManagerConfig = {
+      redis: queueConfig?.redis || {
+        host: 'localhost',
+        port: 6379,
+        db: 1,
+      },
+      defaultQueueOptions: {
+        defaultJobOptions: queueConfig?.defaultJobOptions || {
+          attempts: 3,
+          backoff: {
+            type: 'exponential',
+            delay: 1000,
+          },
+          removeOnComplete: 10,
+          removeOnFail: 5,
+        },
+        workers: 2,
+        concurrency: 1,
+        enableMetrics: true,
+        enableDLQ: true,
+      },
+      enableScheduledJobs: true,
+      delayWorkerStart: true,  // Prevent workers from starting until all singletons are ready
+    };
+
+    queueManager = QueueManager.getOrInitialize(queueManagerConfig);
+    logger.info('Queue system initialized');
+
+    // Initialize handlers (register handlers and scheduled jobs)
+    logger.debug('Initializing sync handlers...');
+    const { initializeExchangesHandler } = await import('./handlers/exchanges/exchanges.handler');
+    const { initializeSymbolsHandler } = await import('./handlers/symbols/symbols.handler');
+    
+    initializeExchangesHandler();
+    initializeSymbolsHandler();
+    
+    logger.info('Sync handlers initialized');
+
+    // Create scheduled jobs from registered handlers
+    logger.debug('Creating scheduled jobs from registered handlers...');
+    const { handlerRegistry } = await import('@stock-bot/queue');
+    const allHandlers = handlerRegistry.getAllHandlers();
+
+    let totalScheduledJobs = 0;
+    for (const [handlerName, config] of allHandlers) {
+      if (config.scheduledJobs && config.scheduledJobs.length > 0) {
+        const queue = queueManager.getQueue(handlerName);
+
+        for (const scheduledJob of config.scheduledJobs) {
+          // Include handler and operation info in job data
+          const jobData = {
+            handler: handlerName,
+            operation: scheduledJob.operation,
+            payload: scheduledJob.payload || {},
+          };
+
+          // Build job options from scheduled job config
+          const jobOptions = {
+            priority: scheduledJob.priority,
+            delay: scheduledJob.delay,
+            repeat: {
+              immediately: scheduledJob.immediately,
+            },
+          };
+
+          await queue.addScheduledJob(
+            scheduledJob.operation,
+            jobData,
+            scheduledJob.cronPattern,
+            jobOptions
+          );
+          totalScheduledJobs++;
+          logger.debug('Scheduled job created', {
+            handler: handlerName,
+            operation: scheduledJob.operation,
+            cronPattern: scheduledJob.cronPattern,
+            immediately: scheduledJob.immediately,
+            priority: scheduledJob.priority,
+          });
+        }
+      }
+    }
+    logger.info('Scheduled jobs created', { totalJobs: totalScheduledJobs });
+
+    // Now that all singletons are initialized and jobs are scheduled, start the workers
+    logger.debug('Starting queue workers...');
+    queueManager.startAllWorkers();
+    logger.info('Queue workers started');
+
+    logger.info('All services initialized successfully');
+  } catch (error) {
+    logger.error('Failed to initialize services', { error });
+    throw error;
+  }
+}
+
+// Start server
+async function startServer() {
+  await initializeServices();
+
+  server = Bun.serve({
+    port: PORT,
+    fetch: app.fetch,
+    development: config.environment === 'development',
+  });
+
+  logger.info(`Data Sync Service started on port ${PORT}`);
+}
+
+// Register shutdown handlers with priorities
+// Priority 1: Queue system (highest priority)
+shutdown.onShutdownHigh(async () => {
+  logger.info('Shutting down queue system...');
+  try {
+    if (queueManager) {
+      await queueManager.shutdown();
+    }
+    logger.info('Queue system shut down');
+  } catch (error) {
+    logger.error('Error shutting down queue system', { error });
+  }
+}, 'Queue System');
+
+// Priority 1: HTTP Server (high priority)
+shutdown.onShutdownHigh(async () => {
+  if (server) {
+    logger.info('Stopping HTTP server...');
+    try {
+      server.stop();
+      logger.info('HTTP server stopped');
+    } catch (error) {
+      logger.error('Error stopping HTTP server', { error });
+    }
+  }
+}, 'HTTP Server');
+
+// Priority 2: Database connections (medium priority)
+shutdown.onShutdownMedium(async () => {
+  logger.info('Disconnecting from databases...');
+  try {
+    const { disconnectMongoDB } = await import('@stock-bot/mongodb-client');
+    const { disconnectPostgreSQL } = await import('@stock-bot/postgres-client');
+
+    await disconnectMongoDB();
+    await disconnectPostgreSQL();
+    logger.info('Database connections closed');
+  } catch (error) {
+    logger.error('Error closing database connections', { error });
+  }
+}, 'Databases');
+
+// Priority 3: Logger shutdown (lowest priority - runs last)
+shutdown.onShutdownLow(async () => {
+  try {
+    logger.info('Shutting down loggers...');
+    await shutdownLoggers();
+    // Don't log after shutdown
+  } catch {
+    // Silently ignore logger shutdown errors
+  }
+}, 'Loggers');
+
+// Start the service
+startServer().catch(error => {
+  logger.fatal('Failed to start data sync service', { error });
+  process.exit(1);
+});
+
+logger.info('Data sync service startup initiated');