initial data-ingestion refactor

apps/data-ingestion/src/index.ts

@@ -4,13 +4,13 @@ import { Hono } from 'hono';
 import { cors } from 'hono/cors';
 // Library imports
 import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
-import { connectMongoDB } from '@stock-bot/mongodb-client';
-import { connectPostgreSQL } from '@stock-bot/postgres-client';
 import { QueueManager, type QueueManagerConfig } from '@stock-bot/queue';
 import { Shutdown } from '@stock-bot/shutdown';
 import { ProxyManager } from '@stock-bot/utils';
+import type { ServiceContainer } from '@stock-bot/connection-factory';
 // Local imports
-import { exchangeRoutes, healthRoutes, queueRoutes } from './routes';
+import { setupServiceContainer } from './setup/database-setup';
+import { createRoutes } from './routes/create-routes';
 
 const config = initializeServiceConfig();
 console.log('Data Service Configuration:', JSON.stringify(config, null, 2));
@@ -31,68 +31,42 @@ if (config.log) {
 // Create logger AFTER config is set
 const logger = getLogger('data-ingestion');
 
-const app = new Hono();
-
-// Add CORS middleware
-app.use(
-  '*',
-  cors({
-    origin: '*',
-    allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'],
-    allowHeaders: ['Content-Type', 'Authorization'],
-    credentials: false,
-  })
-);
 const PORT = serviceConfig.port;
 let server: ReturnType<typeof Bun.serve> | null = null;
-// Singleton clients are managed in libraries
+let serviceContainer: ServiceContainer | null = null;
 let queueManager: QueueManager | null = null;
+let app: Hono | null = null;
 
 // Initialize shutdown manager
 const shutdown = Shutdown.getInstance({ timeout: 15000 });
 
-// Mount routes
-app.route('/health', healthRoutes);
-app.route('/api/exchanges', exchangeRoutes);
-app.route('/api/queue', queueRoutes);
-
 // Initialize services
 async function initializeServices() {
-  logger.info('Initializing data service...');
+  logger.info('Initializing data-ingestion service...');
 
   try {
-    // Initialize MongoDB client singleton
-    logger.debug('Connecting to MongoDB...');
-    const mongoConfig = databaseConfig.mongodb;
-    await connectMongoDB({
-      uri: mongoConfig.uri,
-      database: mongoConfig.database,
-      host: mongoConfig.host || 'localhost',
-      port: mongoConfig.port || 27017,
-      timeouts: {
-        connectTimeout: 30000,
-        socketTimeout: 30000,
-        serverSelectionTimeout: 5000,
-      },
-    });
-    logger.info('MongoDB connected');
-
-    // Initialize PostgreSQL client singleton
-    logger.debug('Connecting to PostgreSQL...');
-    const pgConfig = databaseConfig.postgres;
-    await connectPostgreSQL({
-      host: pgConfig.host,
-      port: pgConfig.port,
-      database: pgConfig.database,
-      username: pgConfig.user,
-      password: pgConfig.password,
-      poolSettings: {
-        min: 2,
-        max: pgConfig.poolSize || 10,
-        idleTimeoutMillis: pgConfig.idleTimeout || 30000,
-      },
-    });
-    logger.info('PostgreSQL connected');
+    // Initialize service container with connection pools
+    logger.debug('Setting up service container with connection pools...');
+    serviceContainer = await setupServiceContainer();
+    logger.info('Service container initialized with connection pools');
+    
+    // Create app with routes that have access to the container
+    app = new Hono();
+    
+    // Add CORS middleware
+    app.use(
+      '*',
+      cors({
+        origin: '*',
+        allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'],
+        allowHeaders: ['Content-Type', 'Authorization'],
+        credentials: false,
+      })
+    );
+    
+    // Create and mount routes with container
+    const routes = createRoutes(serviceContainer);
+    app.route('/', routes);
 
     // Initialize queue system (with delayed worker start)
     logger.debug('Initializing queue system...');
@@ -136,12 +110,13 @@ async function initializeServices() {
     const { initializeProxyProvider } = await import('./handlers/proxy/proxy.handler');
     const { initializeQMProvider } = await import('./handlers/qm/qm.handler');
     
-    initializeWebShareProvider();
-    initializeIBProvider();
-    initializeProxyProvider();
-    initializeQMProvider();
+    // Pass service container to handlers
+    initializeWebShareProvider(serviceContainer);
+    initializeIBProvider(serviceContainer);
+    initializeProxyProvider(serviceContainer);
+    initializeQMProvider(serviceContainer);
     
-    logger.info('Data handlers initialized');
+    logger.info('Data handlers initialized with service container');
 
     // Create scheduled jobs from registered handlers
     logger.debug('Creating scheduled jobs from registered handlers...');
@@ -205,13 +180,17 @@ async function initializeServices() {
 async function startServer() {
   await initializeServices();
 
+  if (!app) {
+    throw new Error('App not initialized');
+  }
+
   server = Bun.serve({
     port: PORT,
     fetch: app.fetch,
     development: config.environment === 'development',
   });
 
-  logger.info(`Data Service started on port ${PORT}`);
+  logger.info(`Data-ingestion service started on port ${PORT}`);
 }
 
 // Register shutdown handlers with priorities
@@ -241,20 +220,18 @@ shutdown.onShutdownHigh(async () => {
   }
 }, 'HTTP Server');
 
-// Priority 2: Database connections (medium priority)
+// Priority 2: Service container and connections (medium priority)
 shutdown.onShutdownMedium(async () => {
-  logger.info('Disconnecting from databases...');
+  logger.info('Disposing service container and connections...');
   try {
-    const { disconnectMongoDB } = await import('@stock-bot/mongodb-client');
-    const { disconnectPostgreSQL } = await import('@stock-bot/postgres-client');
-
-    await disconnectMongoDB();
-    await disconnectPostgreSQL();
-    logger.info('Database connections closed');
+    if (serviceContainer) {
+      await serviceContainer.dispose();
+      logger.info('Service container disposed, all connections closed');
+    }
   } catch (error) {
-    logger.error('Error closing database connections', { error });
+    logger.error('Error disposing service container', { error });
   }
-}, 'Databases');
+}, 'Service Container');
 
 // Priority 3: Logger shutdown (lowest priority - runs last)
 shutdown.onShutdownLow(async () => {