removed singletop pattern from queue manager

apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts

@@ -3,12 +3,16 @@
  */
 import { OperationContext } from '@stock-bot/di';
 import type { ProxyInfo } from '@stock-bot/proxy';
-import { QueueManager } from '@stock-bot/queue';
+import type { IServiceContainer } from '@stock-bot/handlers';
 
-export async function queueProxyFetch(): Promise<string> {
+export async function queueProxyFetch(container: IServiceContainer): Promise<string> {
   const ctx = OperationContext.create('proxy', 'queue-fetch');
 
-  const queueManager = QueueManager.getInstance();
+  const queueManager = container.queue;
+  if (!queueManager) {
+    throw new Error('Queue manager not available');
+  }
+  
   const queue = queueManager.getQueue('proxy');
   const job = await queue.add('proxy-fetch', {
     handler: 'proxy',
@@ -22,10 +26,14 @@ export async function queueProxyFetch(): Promise<string> {
   return jobId;
 }
 
-export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
+export async function queueProxyCheck(proxies: ProxyInfo[], container: IServiceContainer): Promise<string> {
   const ctx = OperationContext.create('proxy', 'queue-check');
 
-  const queueManager = QueueManager.getInstance();
+  const queueManager = container.queue;
+  if (!queueManager) {
+    throw new Error('Queue manager not available');
+  }
+  
   const queue = queueManager.getQueue('proxy');
   const job = await queue.add('proxy-check', {
     handler: 'proxy',
@@ -37,4 +45,4 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
   const jobId = job.id || 'unknown';
   ctx.logger.info('Proxy check job queued', { jobId, count: proxies.length });
   return jobId;
-}
+}

apps/data-ingestion/src/handlers/proxy/proxy.handler.ts

@@ -35,6 +35,10 @@ export function initializeProxyProvider(_container: ServiceContainer) {
           return { processed: 0, successful: 0 };
         }
 
+        // Get QueueManager instance - we have to use getInstance for now until handlers get container access
+        const { QueueManager } = await import('@stock-bot/queue');
+        const queueManager = QueueManager.getInstance();
+        
         // Batch process the proxies through check-proxy operation
         const batchResult = await processItems(proxies, 'proxy', {
           handler: 'proxy',
@@ -47,7 +51,7 @@ export function initializeProxyProvider(_container: ServiceContainer) {
           ttl: 30000, // 30 second timeout per proxy check
           removeOnComplete: 5,
           removeOnFail: 3,
-        });
+        }, queueManager);
 
         handlerLogger.info('Batch proxy validation completed', {
           totalProxies: proxies.length,

apps/data-ingestion/src/routes/create-routes.ts

@@ -6,7 +6,7 @@ import { Hono } from 'hono';
 import type { IServiceContainer } from '@stock-bot/handlers';
 import { exchangeRoutes } from './exchange.routes';
 import { healthRoutes } from './health.routes';
-import { queueRoutes } from './queue.routes';
+import { createQueueRoutes } from './queue.routes';
 
 /**
  * Creates all routes with access to type-safe services
@@ -17,9 +17,9 @@ export function createRoutes(services: IServiceContainer): Hono {
   // Mount routes that don't need services
   app.route('/health', healthRoutes);
 
-  // Mount routes that need services (will be updated to use services)
+  // Mount routes that need services
   app.route('/api/exchanges', exchangeRoutes);
-  app.route('/api/queue', queueRoutes);
+  app.route('/api/queue', createQueueRoutes(services));
 
   // Store services in app context for handlers that need it
   app.use('*', async (c, next) => {

apps/data-ingestion/src/routes/market-data.routes.ts

@@ -3,119 +3,140 @@
  */
 import { Hono } from 'hono';
 import { getLogger } from '@stock-bot/logger';
-import { processItems, QueueManager } from '@stock-bot/queue';
+import { processItems } from '@stock-bot/queue';
+import type { IServiceContainer } from '@stock-bot/handlers';
 
 const logger = getLogger('market-data-routes');
 
-export const marketDataRoutes = new Hono();
+export function createMarketDataRoutes(container: IServiceContainer) {
+  const marketDataRoutes = new Hono();
 
-// Market data endpoints
-marketDataRoutes.get('/api/live/:symbol', async c => {
-  const symbol = c.req.param('symbol');
-  logger.info('Live data request', { symbol });
+  // Market data endpoints
+  marketDataRoutes.get('/api/live/:symbol', async c => {
+    const symbol = c.req.param('symbol');
+    logger.info('Live data request', { symbol });
 
-  try {
-    // Queue job for live data using Yahoo provider
-    const queueManager = QueueManager.getInstance();
-    const queue = queueManager.getQueue('yahoo-finance');
-    const job = await queue.add('live-data', {
-      handler: 'yahoo-finance',
-      operation: 'live-data',
-      payload: { symbol },
-    });
-    return c.json({
-      status: 'success',
-      message: 'Live data job queued',
-      jobId: job.id,
-      symbol,
-    });
-  } catch (error) {
-    logger.error('Failed to queue live data job', { symbol, error });
-    return c.json({ status: 'error', message: 'Failed to queue live data job' }, 500);
-  }
-});
-
-marketDataRoutes.get('/api/historical/:symbol', async c => {
-  const symbol = c.req.param('symbol');
-  const from = c.req.query('from');
-  const to = c.req.query('to');
-
-  logger.info('Historical data request', { symbol, from, to });
-
-  try {
-    const fromDate = from ? new Date(from) : new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days ago
-    const toDate = to ? new Date(to) : new Date(); // Now
-
-    // Queue job for historical data using Yahoo provider
-    const queueManager = QueueManager.getInstance();
-    const queue = queueManager.getQueue('yahoo-finance');
-    const job = await queue.add('historical-data', {
-      handler: 'yahoo-finance',
-      operation: 'historical-data',
-      payload: {
+    try {
+      // Queue job for live data using Yahoo provider
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const queue = queueManager.getQueue('yahoo-finance');
+      const job = await queue.add('live-data', {
+        handler: 'yahoo-finance',
+        operation: 'live-data',
+        payload: { symbol },
+      });
+      return c.json({
+        status: 'success',
+        message: 'Live data job queued',
+        jobId: job.id,
         symbol,
-        from: fromDate.toISOString(),
-        to: toDate.toISOString(),
-      },
-    });
-
-    return c.json({
-      status: 'success',
-      message: 'Historical data job queued',
-      jobId: job.id,
-      symbol,
-      from: fromDate,
-      to: toDate,
-    });
-  } catch (error) {
-    logger.error('Failed to queue historical data job', { symbol, from, to, error });
-    return c.json({ status: 'error', message: 'Failed to queue historical data job' }, 500);
-  }
-});
-
-// Batch processing endpoint using new queue system
-marketDataRoutes.post('/api/process-symbols', async c => {
-  try {
-    const {
-      symbols,
-      provider = 'ib',
-      operation = 'fetch-session',
-      useBatching = true,
-      totalDelayHours = 0.0083, // ~30 seconds (30/3600 hours)
-      batchSize = 10,
-    } = await c.req.json();
-
-    if (!symbols || !Array.isArray(symbols) || symbols.length === 0) {
-      return c.json({ status: 'error', message: 'Invalid symbols array' }, 400);
+      });
+    } catch (error) {
+      logger.error('Failed to queue live data job', { symbol, error });
+      return c.json({ status: 'error', message: 'Failed to queue live data job' }, 500);
     }
+  });
 
-    logger.info('Batch processing symbols', {
-      count: symbols.length,
-      provider,
-      operation,
-      useBatching,
-    });
+  marketDataRoutes.get('/api/historical/:symbol', async c => {
+    const symbol = c.req.param('symbol');
+    const from = c.req.query('from');
+    const to = c.req.query('to');
 
-    const result = await processItems(symbols, provider, {
-      handler: provider,
-      operation,
-      totalDelayHours,
-      useBatching,
-      batchSize,
-      priority: 2,
-      retries: 2,
-      removeOnComplete: 5,
-      removeOnFail: 10,
-    });
+    logger.info('Historical data request', { symbol, from, to });
 
-    return c.json({
-      status: 'success',
-      message: 'Batch processing initiated',
-      result,
-      symbols: symbols.length,
-    });
-  } catch (error) {
-    logger.error('Failed to process symbols batch', { error });
-    return c.json({ status: 'error', message: 'Failed to process symbols batch' }, 500);
-  }
-});
+    try {
+      const fromDate = from ? new Date(from) : new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days ago
+      const toDate = to ? new Date(to) : new Date(); // Now
+
+      // Queue job for historical data using Yahoo provider
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const queue = queueManager.getQueue('yahoo-finance');
+      const job = await queue.add('historical-data', {
+        handler: 'yahoo-finance',
+        operation: 'historical-data',
+        payload: {
+          symbol,
+          from: fromDate.toISOString(),
+          to: toDate.toISOString(),
+        },
+      });
+
+      return c.json({
+        status: 'success',
+        message: 'Historical data job queued',
+        jobId: job.id,
+        symbol,
+        from: fromDate,
+        to: toDate,
+      });
+    } catch (error) {
+      logger.error('Failed to queue historical data job', { symbol, from, to, error });
+      return c.json({ status: 'error', message: 'Failed to queue historical data job' }, 500);
+    }
+  });
+
+  // Batch processing endpoint using new queue system
+  marketDataRoutes.post('/api/process-symbols', async c => {
+    try {
+      const {
+        symbols,
+        provider = 'ib',
+        operation = 'fetch-session',
+        useBatching = true,
+        totalDelayHours = 0.0083, // ~30 seconds (30/3600 hours)
+        batchSize = 10,
+      } = await c.req.json();
+
+      if (!symbols || !Array.isArray(symbols) || symbols.length === 0) {
+        return c.json({ status: 'error', message: 'Invalid symbols array' }, 400);
+      }
+
+      logger.info('Batch processing symbols', {
+        count: symbols.length,
+        provider,
+        operation,
+        useBatching,
+      });
+
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const result = await processItems(symbols, provider, {
+        handler: provider,
+        operation,
+        totalDelayHours,
+        useBatching,
+        batchSize,
+        priority: 2,
+        retries: 2,
+        removeOnComplete: 5,
+        removeOnFail: 10,
+      }, queueManager);
+
+      return c.json({
+        status: 'success',
+        message: 'Batch processing initiated',
+        result,
+        symbols: symbols.length,
+      });
+    } catch (error) {
+      logger.error('Failed to process symbols batch', { error });
+      return c.json({ status: 'error', message: 'Failed to process symbols batch' }, 500);
+    }
+  });
+
+  return marketDataRoutes;
+}
+
+// Legacy export for backward compatibility
+export const marketDataRoutes = createMarketDataRoutes({} as IServiceContainer);

apps/data-ingestion/src/routes/queue.routes.ts

@@ -1,25 +1,35 @@
 import { Hono } from 'hono';
 import { getLogger } from '@stock-bot/logger';
-import { QueueManager } from '@stock-bot/queue';
+import type { IServiceContainer } from '@stock-bot/handlers';
 
 const logger = getLogger('queue-routes');
-const queue = new Hono();
 
-// Queue status endpoint
-queue.get('/status', async c => {
-  try {
-    const queueManager = QueueManager.getInstance();
-    const globalStats = await queueManager.getGlobalStats();
+export function createQueueRoutes(container: IServiceContainer) {
+  const queue = new Hono();
 
-    return c.json({
-      status: 'success',
-      data: globalStats,
-      message: 'Queue status retrieved successfully',
-    });
-  } catch (error) {
-    logger.error('Failed to get queue status', { error });
-    return c.json({ status: 'error', message: 'Failed to get queue status' }, 500);
-  }
-});
+  // Queue status endpoint
+  queue.get('/status', async c => {
+    try {
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const globalStats = await queueManager.getGlobalStats();
 
-export { queue as queueRoutes };
+      return c.json({
+        status: 'success',
+        data: globalStats,
+        message: 'Queue status retrieved successfully',
+      });
+    } catch (error) {
+      logger.error('Failed to get queue status', { error });
+      return c.json({ status: 'error', message: 'Failed to get queue status' }, 500);
+    }
+  });
+
+  return queue;
+}
+
+// Legacy export for backward compatibility
+export const queueRoutes = createQueueRoutes({} as IServiceContainer);