moved folders around

apps/data-ingestion/src/handlers/ib/operations/exchanges.operations.ts

@@ -1,7 +1,7 @@
 /**
  * IB Exchanges Operations - Fetching exchange data from IB API
  */
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 import type { ServiceContainer } from '@stock-bot/connection-factory';
 
 import { IB_CONFIG } from '../shared/config';

apps/data-ingestion/src/handlers/ib/operations/session.operations.ts

@@ -2,8 +2,8 @@
  * IB Session Operations - Browser automation for session headers
  */
 import { Browser } from '@stock-bot/browser';
-import { OperationContext } from '@stock-bot/utils';
-import type { ServiceContainer } from '@stock-bot/connection-factory';
+import { OperationContext } from '@stock-bot/di';
+import type { ServiceContainer } from '@stock-bot/di';
 
 import { IB_CONFIG } from '../shared/config';
 

apps/data-ingestion/src/handlers/ib/operations/symbols.operations.ts

@@ -1,7 +1,7 @@
 /**
  * IB Symbols Operations - Fetching symbol data from IB API
  */
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 import type { ServiceContainer } from '@stock-bot/connection-factory';
 
 import { IB_CONFIG } from '../shared/config';

apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts

@@ -2,7 +2,7 @@
  * Proxy Check Operations - Checking proxy functionality
  */
 import { HttpClient, ProxyInfo } from '@stock-bot/http';
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 
 import { PROXY_CONFIG } from '../shared/config';
 import { ProxyStatsManager } from '../shared/proxy-manager';

apps/data-ingestion/src/handlers/proxy/operations/fetch.operations.ts

@@ -2,7 +2,7 @@
  * Proxy Fetch Operations - Fetching proxies from sources
  */
 import { HttpClient, ProxyInfo } from '@stock-bot/http';
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 
 import { PROXY_CONFIG } from '../shared/config';
 import { ProxyStatsManager } from '../shared/proxy-manager';

apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts

@@ -2,7 +2,7 @@
  * Proxy Query Operations - Getting active proxies from cache
  */
 import { ProxyInfo } from '@stock-bot/http';
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 
 import { PROXY_CONFIG } from '../shared/config';
 

apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts

@@ -3,7 +3,7 @@
  */
 import { ProxyInfo } from '@stock-bot/http';
 import { QueueManager } from '@stock-bot/queue';
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 
 export async function queueProxyFetch(): Promise<string> {
   const ctx = OperationContext.create('proxy', 'queue-fetch');

apps/data-ingestion/src/handlers/proxy/shared/proxy-manager.ts

@@ -1,56 +1,6 @@
 /**
- * Proxy Stats Manager - Singleton for managing proxy statistics
+ * Proxy Manager - Simplified without stats tracking
  */
-import type { ProxySource } from './types';
-import { PROXY_CONFIG } from './config';
 
-export class ProxyStatsManager {
-  private static instance: ProxyStatsManager | null = null;
-  private proxyStats: ProxySource[] = [];
-
-  private constructor() {
-    this.resetStats();
-  }
-
-  static getInstance(): ProxyStatsManager {
-    if (!ProxyStatsManager.instance) {
-      ProxyStatsManager.instance = new ProxyStatsManager();
-    }
-    return ProxyStatsManager.instance;
-  }
-
-  resetStats(): void {
-    this.proxyStats = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
-      id: source.id,
-      total: 0,
-      working: 0,
-      lastChecked: new Date(),
-      protocol: source.protocol,
-      url: source.url,
-    }));
-  }
-
-  getStats(): ProxySource[] {
-    return [...this.proxyStats];
-  }
-
-  updateSourceStats(sourceId: string, success: boolean): ProxySource | undefined {
-    const source = this.proxyStats.find(s => s.id === sourceId);
-    if (source) {
-      if (typeof source.working !== 'number') {
-        source.working = 0;
-      }
-      if (typeof source.total !== 'number') {
-        source.total = 0;
-      }
-      source.total += 1;
-      if (success) {
-        source.working += 1;
-      }
-      source.percentWorking = (source.working / source.total) * 100;
-      source.lastChecked = new Date();
-      return source;
-    }
-    return undefined;
-  }
-}
+// This file is kept for compatibility but ProxyStatsManager has been removed
+// All proxy management is now handled by the global ProxyManager in @stock-bot/utils

apps/data-ingestion/src/handlers/qm/operations/exchanges.operations.ts

@@ -2,7 +2,7 @@
  * QM Exchanges Operations - Exchange fetching functionality
  */
 
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 import type { ServiceContainer } from '@stock-bot/connection-factory';
 
 import { initializeQMResources } from './session.operations';

apps/data-ingestion/src/handlers/qm/operations/session.operations.ts

@@ -2,9 +2,9 @@
  * QM Session Operations - Session creation and management
  */
 
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 import { isShutdownSignalReceived } from '@stock-bot/shutdown';
-import { getRandomProxy } from '@stock-bot/utils';
+import { getRandomProxy } from '@stock-bot/di';
 import type { ServiceContainer } from '@stock-bot/connection-factory';
 
 import { QMSessionManager } from '../shared/session-manager';

apps/data-ingestion/src/handlers/qm/operations/spider.operations.ts

@@ -2,7 +2,7 @@
  * QM Spider Operations - Symbol spider search functionality
  */
 
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 import { QueueManager } from '@stock-bot/queue';
 
 import { QMSessionManager } from '../shared/session-manager';

apps/data-ingestion/src/handlers/qm/operations/symbols.operations.ts

@@ -2,9 +2,9 @@
  * QM Symbols Operations - Symbol fetching and API interactions
  */
 
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 import { getRandomProxy } from '@stock-bot/utils';
-import type { ServiceContainer } from '@stock-bot/connection-factory';
+import type { ServiceContainer } from '@stock-bot/di';
 
 import { QMSessionManager } from '../shared/session-manager';
 import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG } from '../shared/config';

apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts

@@ -2,7 +2,7 @@
  * WebShare Fetch Operations - API integration
  */
 import { type ProxyInfo } from '@stock-bot/http';
-import { OperationContext } from '@stock-bot/utils';
+import { OperationContext } from '@stock-bot/di';
 
 import { WEBSHARE_CONFIG } from '../shared/config';
 

apps/data-ingestion/src/handlers/webshare/webshare.handler.ts

@@ -8,7 +8,7 @@ import {
   type HandlerConfigWithSchedule,
 } from '@stock-bot/queue';
 import { updateProxies } from '@stock-bot/utils';
-import type { ServiceContainer } from '@stock-bot/connection-factory';
+import type { ServiceContainer } from '@stock-bot/di';
 
 const logger = getLogger('webshare-provider');
 

apps/data-ingestion/src/index.ts

@@ -4,10 +4,10 @@ import { Hono } from 'hono';
 import { cors } from 'hono/cors';
 // Library imports
 import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
-import { QueueManager, type QueueManagerConfig } from '@stock-bot/queue';
+import type { QueueManager } from '@stock-bot/queue';
 import { Shutdown } from '@stock-bot/shutdown';
-import { ProxyManager } from '@stock-bot/utils';
-import type { ServiceContainer } from '@stock-bot/connection-factory';
+import { ProxyManager } from '@stock-bot/di';
+import type { ServiceContainer } from '@stock-bot/di';
 // Local imports
 import { setupServiceContainer } from './setup/database-setup';
 import { createRoutes } from './routes/create-routes';
@@ -68,35 +68,10 @@ async function initializeServices() {
     const routes = createRoutes(serviceContainer);
     app.route('/', routes);
 
-    // Initialize queue system (with delayed worker start)
-    logger.debug('Initializing queue system...');
-    const queueManagerConfig: QueueManagerConfig = {
-      redis: queueConfig?.redis || {
-        host: 'localhost',
-        port: 6379,
-        db: 1,
-      },
-      defaultQueueOptions: {
-        defaultJobOptions: queueConfig?.defaultJobOptions || {
-          attempts: 3,
-          backoff: {
-            type: 'exponential',
-            delay: 1000,
-          },
-          removeOnComplete: 10,
-          removeOnFail: 5,
-        },
-        workers: 2,
-        concurrency: 1,
-        enableMetrics: true,
-        enableDLQ: true,
-      },
-      enableScheduledJobs: true,
-      delayWorkerStart: true,  // Prevent workers from starting until all singletons are ready
-    };
-
-    queueManager = QueueManager.getOrInitialize(queueManagerConfig);
-    logger.info('Queue system initialized');
+    // Get queue manager from service container
+    logger.debug('Getting queue manager from service container...');
+    queueManager = await serviceContainer.resolveAsync<QueueManager>('queue');
+    logger.info('Queue system resolved from container');
 
     // Initialize proxy manager
     logger.debug('Initializing proxy manager...');

apps/data-ingestion/src/setup/database-setup.ts

@@ -1,11 +1,11 @@
-import { getDatabaseConfig } from '@stock-bot/config';
+import { getDatabaseConfig, getQueueConfig } from '@stock-bot/config';
 import { getLogger } from '@stock-bot/logger';
 import { 
   ConnectionFactory, 
   ServiceContainer,
   PoolSizeCalculator 
-} from '@stock-bot/connection-factory';
-import type { ConnectionFactoryConfig, DynamicPoolConfig } from '@stock-bot/mongodb-client';
+} from '@stock-bot/di';
+import type { ConnectionFactoryConfig, DynamicPoolConfig } from '@stock-bot/mongodb';
 
 const logger = getLogger('database-setup');
 
@@ -27,6 +27,9 @@ export function createConnectionFactory(): ConnectionFactory {
       },
       cache: {
         poolSize: 20,
+      },
+      queue: {
+        poolSize: 1, // QueueManager is a singleton
       }
     }
   };
@@ -42,6 +45,7 @@ export async function setupServiceContainer(): Promise<ServiceContainer> {
   
   const connectionFactory = createConnectionFactory();
   const dbConfig = getDatabaseConfig();
+  const queueConfig = getQueueConfig();
   
   // Create base container
   const container = new ServiceContainer('data-ingestion');
@@ -130,6 +134,45 @@ export async function setupServiceContainer(): Promise<ServiceContainer> {
     singleton: true,
   });
   
+  // Register QueueManager
+  container.register({
+    name: 'queue',
+    factory: () => {
+      const pool = connectionFactory.createQueue({
+        name: 'default',
+        config: {
+          redis: queueConfig?.redis || {
+            host: dbConfig.dragonfly.host,
+            port: dbConfig.dragonfly.port,
+            db: dbConfig.dragonfly.db || 1,
+          },
+          defaultQueueOptions: {
+            defaultJobOptions: queueConfig?.defaultJobOptions || {
+              attempts: 3,
+              backoff: {
+                type: 'exponential',
+                delay: 1000,
+              },
+              removeOnComplete: 10,
+              removeOnFail: 5,
+            },
+            workers: 2,
+            concurrency: 1,
+            enableMetrics: true,
+            enableDLQ: true,
+          },
+          enableScheduledJobs: true,
+          delayWorkerStart: true,
+        }
+      });
+      return pool.client;
+    },
+    singleton: true,
+    dispose: async (queueManager) => {
+      await queueManager.shutdown();
+    }
+  });
+  
   // Register the connection factory itself for pool management
   container.register({
     name: 'connectionFactory',

apps/data-ingestion/src/setup/dynamic-pool-example.ts

@@ -1,116 +0,0 @@
-import { getDatabaseConfig } from '@stock-bot/config';
-import { getLogger } from '@stock-bot/logger';
-import { createMongoDBClient, createPostgreSQLClient } from '@stock-bot/connection-factory';
-import type { DynamicPoolConfig } from '@stock-bot/mongodb-client';
-
-const logger = getLogger('dynamic-pool-example');
-
-/**
- * Example of setting up dynamic pool sizing for high-load scenarios
- */
-export async function setupDynamicPools() {
-  const dbConfig = getDatabaseConfig();
-  
-  // Dynamic pool configuration for batch processing
-  const dynamicConfig: DynamicPoolConfig = {
-    enabled: true,
-    minSize: 5,
-    maxSize: 100,
-    scaleUpThreshold: 70,    // Scale up when 70% of connections are in use
-    scaleDownThreshold: 30,  // Scale down when only 30% are in use
-    scaleUpIncrement: 10,     // Add 10 connections at a time
-    scaleDownIncrement: 5,    // Remove 5 connections at a time
-    evaluationInterval: 10000 // Check every 10 seconds
-  };
-  
-  // Create MongoDB client with dynamic pooling
-  const mongoClient = createMongoDBClient({
-    uri: dbConfig.mongodb.uri,
-    database: dbConfig.mongodb.database,
-    poolSettings: {
-      minPoolSize: dynamicConfig.minSize,
-      maxPoolSize: dynamicConfig.maxSize,
-    }
-  }, {
-    onConnect: () => logger.info('MongoDB connected with dynamic pooling'),
-    onError: (error) => logger.error('MongoDB pool error', { error }),
-  });
-  
-  await mongoClient.connect();
-  mongoClient.setDynamicPoolConfig(dynamicConfig);
-  
-  // Create PostgreSQL client with dynamic pooling
-  const pgClient = createPostgreSQLClient({
-    host: dbConfig.postgresql.host,
-    port: dbConfig.postgresql.port,
-    database: dbConfig.postgresql.database,
-    username: dbConfig.postgresql.user,
-    password: dbConfig.postgresql.password,
-    poolSettings: {
-      min: dynamicConfig.minSize,
-      max: dynamicConfig.maxSize,
-    }
-  }, undefined, {
-    onConnect: () => logger.info('PostgreSQL connected with dynamic pooling'),
-    onError: (error) => logger.error('PostgreSQL pool error', { error }),
-  });
-  
-  await pgClient.connect();
-  pgClient.setDynamicPoolConfig(dynamicConfig);
-  
-  // Monitor pool metrics
-  setInterval(() => {
-    const mongoMetrics = mongoClient.getPoolMetrics();
-    const pgMetrics = pgClient.getPoolMetrics();
-    
-    logger.info('Pool metrics', {
-      mongodb: {
-        total: mongoMetrics.totalConnections,
-        active: mongoMetrics.activeConnections,
-        idle: mongoMetrics.idleConnections,
-        waiting: mongoMetrics.waitingRequests,
-      },
-      postgresql: {
-        total: pgMetrics.totalConnections,
-        active: pgMetrics.activeConnections,
-        idle: pgMetrics.idleConnections,
-        waiting: pgMetrics.waitingRequests,
-      }
-    });
-  }, 30000); // Log metrics every 30 seconds
-  
-  return { mongoClient, pgClient };
-}
-
-/**
- * Example of adaptive pool sizing based on time of day
- */
-export function getTimeBasedPoolConfig(): DynamicPoolConfig {
-  const hour = new Date().getHours();
-  
-  // High load hours (9 AM - 5 PM)
-  if (hour >= 9 && hour <= 17) {
-    return {
-      enabled: true,
-      minSize: 10,
-      maxSize: 150,
-      scaleUpThreshold: 60,
-      scaleDownThreshold: 20,
-      scaleUpIncrement: 20,
-      scaleDownIncrement: 10,
-      evaluationInterval: 5000 // More frequent checks during peak
-    };
-  }
-  
-  // Low load hours (night time)
-  return {
-    enabled: true,
-    minSize: 2,
-    maxSize: 50,
-    scaleUpThreshold: 80,
-    scaleDownThreshold: 40,
-    scaleUpIncrement: 5,
-    scaleDownIncrement: 2,
-    evaluationInterval: 30000 // Less frequent checks at night
-  };
-}

apps/data-ingestion/src/utils/symbol-search.util.ts

@@ -1,5 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { sleep } from '@stock-bot/utils';
+import { sleep } from '@stock-bot/di';
 
 const logger = getLogger('symbol-search-util');