huge refactor to remove depenencie hell and add typesafe container

apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts

@@ -95,10 +95,14 @@ export async function processIndividualSymbol(
         await this.mongodb.batchUpsert('ceoShorts', shortData.positions, ['id']);
       }
 
-      await this.scheduleOperation('process-individual-symbol', {
-        ceoId: ceoId,
-        timestamp: latestSpielTime,
-      }, {priority: 0});
+      await this.scheduleOperation(
+        'process-individual-symbol',
+        {
+          ceoId: ceoId,
+          timestamp: latestSpielTime,
+        },
+        { priority: 0 }
+      );
     }
 
     this.logger.info(

apps/stock/data-ingestion/src/handlers/ceo/actions/update-unique-symbols.action.ts

@@ -31,10 +31,14 @@ export async function updateUniqueSymbols(
     let scheduledJobs = 0;
     for (const symbol of uniqueSymbols) {
       // Schedule a job to process this individual symbol
-      await this.scheduleOperation('process-individual-symbol', {
-        ceoId: symbol.ceoId,
-        symbol: symbol.symbol,
-      }, {priority: 10 });
+      await this.scheduleOperation(
+        'process-individual-symbol',
+        {
+          ceoId: symbol.ceoId,
+          symbol: symbol.symbol,
+        },
+        { priority: 10 }
+      );
       scheduledJobs++;
 
       // Add small delay to avoid overwhelming the queue

apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts

@@ -1,6 +1,6 @@
 import type { IServiceContainer } from '@stock-bot/handlers';
-import { fetchSession } from './fetch-session.action';
 import { fetchExchanges } from './fetch-exchanges.action';
+import { fetchSession } from './fetch-session.action';
 import { fetchSymbols } from './fetch-symbols.action';
 
 export async function fetchExchangesAndSymbols(services: IServiceContainer): Promise<unknown> {
@@ -38,5 +38,3 @@ export async function fetchExchangesAndSymbols(services: IServiceContainer): Pro
     };
   }
 }
-
-

apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges.action.ts

@@ -1,4 +1,4 @@
-import type { IServiceContainer } from '@stock-bot/handlers';
+import type { IServiceContainer } from '@stock-bot/types';
 import { IB_CONFIG } from '../shared/config';
 import { fetchSession } from './fetch-session.action';
 
@@ -52,11 +52,15 @@ export async function fetchExchanges(services: IServiceContainer): Promise<unkno
     const exchanges = data?.exchanges || [];
     services.logger.info('✅ Exchange data fetched successfully');
 
-    services.logger.info('Saving IB exchanges to MongoDB...');
-    await services.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']);
-    services.logger.info('✅ Exchange IB data saved to MongoDB:', {
-      count: exchanges.length,
-    });
+    if (services.mongodb) {
+      services.logger.info('Saving IB exchanges to MongoDB...');
+      await services.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']);
+      services.logger.info('✅ Exchange IB data saved to MongoDB:', {
+        count: exchanges.length,
+      });
+    } else {
+      services.logger.warn('MongoDB service not available, skipping data persistence');
+    }
 
     return exchanges;
   } catch (error) {
@@ -64,5 +68,3 @@ export async function fetchExchanges(services: IServiceContainer): Promise<unkno
     return null;
   }
 }
-
-

apps/stock/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts

@@ -2,7 +2,9 @@ import { Browser } from '@stock-bot/browser';
 import type { IServiceContainer } from '@stock-bot/handlers';
 import { IB_CONFIG } from '../shared/config';
 
-export async function fetchSession(services: IServiceContainer): Promise<Record<string, string> | undefined> {
+export async function fetchSession(
+  services: IServiceContainer
+): Promise<Record<string, string> | undefined> {
   try {
     await Browser.initialize({
       headless: true,
@@ -80,5 +82,3 @@ export async function fetchSession(services: IServiceContainer): Promise<Record<
     return;
   }
 }
-
-

apps/stock/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts

@@ -115,5 +115,3 @@ export async function fetchSymbols(services: IServiceContainer): Promise<unknown
     return null;
   }
 }
-
-

apps/stock/data-ingestion/src/handlers/ib/actions/index.ts

@@ -2,4 +2,3 @@ export { fetchSession } from './fetch-session.action';
 export { fetchExchanges } from './fetch-exchanges.action';
 export { fetchSymbols } from './fetch-symbols.action';
 export { fetchExchangesAndSymbols } from './fetch-exchanges-and-symbols.action';
-

apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts

@@ -8,7 +8,7 @@ import {
 import { fetchExchanges, fetchExchangesAndSymbols, fetchSession, fetchSymbols } from './actions';
 
 @Handler('ib')
-class IbHandler extends BaseHandler {
+export class IbHandler extends BaseHandler {
   constructor(services: IServiceContainer) {
     super(services);
   }
@@ -38,5 +38,3 @@ class IbHandler extends BaseHandler {
     return fetchExchangesAndSymbols(this);
   }
 }
-
-

apps/stock/data-ingestion/src/handlers/ib/shared/config.ts

@@ -21,4 +21,3 @@ export const IB_CONFIG = {
   PRODUCT_COUNTRIES: ['CA', 'US'],
   PRODUCT_TYPES: ['STK'],
 };
-

apps/stock/data-ingestion/src/handlers/index.ts

@@ -1,60 +1,48 @@
 /**
- * Handler auto-registration
- * Automatically discovers and registers all handlers
+ * Handler initialization for data-ingestion service
+ * Uses explicit imports for bundling compatibility
  */
 
-import type { IServiceContainer } from '@stock-bot/handlers';
-import { autoRegisterHandlers } from '@stock-bot/handlers';
 import { getLogger } from '@stock-bot/logger';
-// Import handlers for bundling (ensures they're included in the build)
-import './ceo/ceo.handler';
-import './ib/ib.handler';
-import './qm/qm.handler';
-import './webshare/webshare.handler';
+import type { IServiceContainer } from '@stock-bot/types';
+// Import handlers explicitly for bundling (ensures they're included in the build)
+// These imports trigger the decorator metadata to be set
+import { CeoHandler } from './ceo/ceo.handler';
+import { IbHandler } from './ib/ib.handler';
+import { QMHandler } from './qm/qm.handler';
+import { WebShareHandler } from './webshare/webshare.handler';
 
 // Add more handler imports as needed
 
 const logger = getLogger('handler-init');
 
 /**
- * Initialize and register all handlers automatically
+ * Initialize and register all handlers
+ * Note: The actual registration is now handled by the HandlerScanner in the DI container
+ * This function is kept for backward compatibility and explicit handler imports
  */
 export async function initializeAllHandlers(serviceContainer: IServiceContainer): Promise<void> {
   try {
-    // Auto-register all handlers in this directory
-    const result = await autoRegisterHandlers(__dirname, serviceContainer, {
-      pattern: '.handler.',
-      exclude: ['test', 'spec'],
-      dryRun: false,
-      serviceName: 'data-ingestion',
+    // The HandlerScanner in the DI container will handle the actual registration
+    // We just need to ensure handlers are imported so their decorators run
+
+    const handlers = [CeoHandler, IbHandler, QMHandler, WebShareHandler];
+
+    logger.info('Handler imports loaded', {
+      count: handlers.length,
+      handlers: handlers.map(h => (h as any).__handlerName || h.name),
     });
 
-    logger.info('Handler auto-registration complete', {
-      registered: result.registered,
-      failed: result.failed,
-    });
-
-    if (result.failed.length > 0) {
-      logger.error('Some handlers failed to register', { failed: result.failed });
+    // If the container has a handler scanner, we can manually register these
+    const scanner = (serviceContainer as any).handlerScanner;
+    if (scanner?.registerHandlerClass) {
+      for (const HandlerClass of handlers) {
+        scanner.registerHandlerClass(HandlerClass, { serviceName: 'data-ingestion' });
+      }
+      logger.info('Handlers registered with scanner');
     }
   } catch (error) {
-    logger.error('Handler auto-registration failed', { error });
-    // Fall back to manual registration
-    await manualHandlerRegistration(serviceContainer);
-  }
-}
-
-/**
- * Manual fallback registration
- */
-async function manualHandlerRegistration(_serviceContainer: IServiceContainer): Promise<void> {
-  logger.warn('Falling back to manual handler registration');
-
-  try {
-
-    logger.info('Manual handler registration complete');
-  } catch (error) {
-    logger.error('Manual handler registration failed', { error });
+    logger.error('Handler initialization failed', { error });
     throw error;
   }
 }

apps/stock/data-ingestion/src/handlers/qm/actions/exchanges.action.ts

@@ -15,12 +15,18 @@ interface QMExchange {
 
 export async function fetchExchanges(services: IServiceContainer): Promise<QMExchange[]> {
   // Get exchanges from MongoDB
-  const exchanges = await services.mongodb.collection<QMExchange>('qm_exchanges').find({}).toArray();
+  const exchanges = await services.mongodb
+    .collection<QMExchange>('qm_exchanges')
+    .find({})
+    .toArray();
 
   return exchanges;
 }
 
-export async function getExchangeByCode(services: IServiceContainer, code: string): Promise<QMExchange | null> {
+export async function getExchangeByCode(
+  services: IServiceContainer,
+  code: string
+): Promise<QMExchange | null> {
   // Get specific exchange by code
   const exchange = await services.mongodb.collection<QMExchange>('qm_exchanges').findOne({ code });
 

apps/stock/data-ingestion/src/handlers/qm/actions/symbols.action.ts

@@ -16,12 +16,19 @@ interface QMSymbol {
 
 export async function searchSymbols(services: IServiceContainer): Promise<QMSymbol[]> {
   // Get symbols from MongoDB
-  const symbols = await services.mongodb.collection<QMSymbol>('qm_symbols').find({}).limit(50).toArray();
+  const symbols = await services.mongodb
+    .collection<QMSymbol>('qm_symbols')
+    .find({})
+    .limit(50)
+    .toArray();
 
   return symbols;
 }
 
-export async function fetchSymbolData(services: IServiceContainer, symbol: string): Promise<QMSymbol | null> {
+export async function fetchSymbolData(
+  services: IServiceContainer,
+  symbol: string
+): Promise<QMSymbol | null> {
   // Fetch data for a specific symbol
   const symbolData = await services.mongodb.collection<QMSymbol>('qm_symbols').findOne({ symbol });
 

apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts

@@ -1,7 +1,7 @@
 import { BaseHandler, Handler, type IServiceContainer } from '@stock-bot/handlers';
 
 @Handler('qm')
-class QMHandler extends BaseHandler {
+export class QMHandler extends BaseHandler {
   constructor(services: IServiceContainer) {
     super(services); // Handler name read from @Handler decorator
   }

apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts

@@ -4,17 +4,18 @@ import {
   Operation,
   QueueSchedule,
   type ExecutionContext,
-  type IServiceContainer
+  type IServiceContainer,
 } from '@stock-bot/handlers';
 
 @Handler('webshare')
-class WebShareHandler extends BaseHandler {
+export class WebShareHandler extends BaseHandler {
   constructor(services: IServiceContainer) {
     super(services);
   }
 
   @Operation('fetch-proxies')
-  @QueueSchedule('0 */6 * * *', { // every 6 hours
+  @QueueSchedule('0 */6 * * *', {
+    // every 6 hours
     priority: 3,
     immediately: false, // Don't run immediately since ProxyManager fetches on startup
     description: 'Refresh proxies from WebShare API',

apps/stock/data-ingestion/src/index.ts

@@ -3,15 +3,12 @@
  * Simplified entry point using ServiceApplication framework
  */
 
-import { initializeStockConfig, type StockAppConfig } from '@stock-bot/stock-config';
-import {
-  ServiceApplication,
-} from '@stock-bot/di';
+import { ServiceApplication } from '@stock-bot/di';
 import { getLogger } from '@stock-bot/logger';
-
+import { initializeStockConfig, type StockAppConfig } from '@stock-bot/stock-config';
+import { createRoutes } from './routes/create-routes';
 // Local imports
 import { initializeAllHandlers } from './handlers';
-import { createRoutes } from './routes/create-routes';
 
 // Initialize configuration with service-specific overrides
 const config = initializeStockConfig('dataIngestion');
@@ -44,7 +41,7 @@ const app = new ServiceApplication(
   },
   {
     // Lifecycle hooks if needed
-    onStarted: (_port) => {
+    onStarted: _port => {
       const logger = getLogger('data-ingestion');
       logger.info('Data ingestion service startup initiated with ServiceApplication framework');
     },
@@ -54,7 +51,7 @@ const app = new ServiceApplication(
 // Container factory function
 async function createContainer(config: StockAppConfig) {
   const { ServiceContainerBuilder } = await import('@stock-bot/di');
-  
+
   const container = await new ServiceContainerBuilder()
     .withConfig(config)
     .withOptions({
@@ -67,14 +64,13 @@ async function createContainer(config: StockAppConfig) {
       enableProxy: true, // Data ingestion needs proxy for rate limiting
     })
     .build(); // This automatically initializes services
-    
+
   return container;
 }
 
-
 // Start the service
 app.start(createContainer, createRoutes, initializeAllHandlers).catch(error => {
   const logger = getLogger('data-ingestion');
   logger.fatal('Failed to start data service', { error });
   process.exit(1);
-});
+});

apps/stock/data-ingestion/src/routes/market-data.routes.ts

@@ -2,9 +2,9 @@
  * Market data routes
  */
 import { Hono } from 'hono';
+import type { IServiceContainer } from '@stock-bot/handlers';
 import { getLogger } from '@stock-bot/logger';
 import { processItems } from '@stock-bot/queue';
-import type { IServiceContainer } from '@stock-bot/handlers';
 
 const logger = getLogger('market-data-routes');
 
@@ -22,7 +22,7 @@ export function createMarketDataRoutes(container: IServiceContainer) {
       if (!queueManager) {
         return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
       }
-      
+
       const queue = queueManager.getQueue('yahoo-finance');
       const job = await queue.add('live-data', {
         handler: 'yahoo-finance',
@@ -57,7 +57,7 @@ export function createMarketDataRoutes(container: IServiceContainer) {
       if (!queueManager) {
         return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
       }
-      
+
       const queue = queueManager.getQueue('yahoo-finance');
       const job = await queue.add('historical-data', {
         handler: 'yahoo-finance',
@@ -110,18 +110,23 @@ export function createMarketDataRoutes(container: IServiceContainer) {
       if (!queueManager) {
         return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
       }
-      
-      const result = await processItems(symbols, provider, {
-        handler: provider,
-        operation,
-        totalDelayHours,
-        useBatching,
-        batchSize,
-        priority: 2,
-        retries: 2,
-        removeOnComplete: 5,
-        removeOnFail: 10,
-      }, queueManager);
+
+      const result = await processItems(
+        symbols,
+        provider,
+        {
+          handler: provider,
+          operation,
+          totalDelayHours,
+          useBatching,
+          batchSize,
+          priority: 2,
+          retries: 2,
+          removeOnComplete: 5,
+          removeOnFail: 10,
+        },
+        queueManager
+      );
 
       return c.json({
         status: 'success',
@@ -139,4 +144,4 @@ export function createMarketDataRoutes(container: IServiceContainer) {
 }
 
 // Legacy export for backward compatibility
-export const marketDataRoutes = createMarketDataRoutes({} as IServiceContainer);
+export const marketDataRoutes = createMarketDataRoutes({} as IServiceContainer);

apps/stock/data-ingestion/src/routes/queue.routes.ts

@@ -1,6 +1,6 @@
 import { Hono } from 'hono';
-import { getLogger } from '@stock-bot/logger';
 import type { IServiceContainer } from '@stock-bot/handlers';
+import { getLogger } from '@stock-bot/logger';
 
 const logger = getLogger('queue-routes');
 
@@ -14,7 +14,7 @@ export function createQueueRoutes(container: IServiceContainer) {
       if (!queueManager) {
         return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
       }
-      
+
       const globalStats = await queueManager.getGlobalStats();
 
       return c.json({
@@ -29,4 +29,4 @@ export function createQueueRoutes(container: IServiceContainer) {
   });
 
   return queue;
-}
+}