refactored monorepo for more projects

2025-06-22 23:48:01 -04:00 · 2025-06-22 23:48:01 -04:00 · 9492f1b15e
commit 9492f1b15e
parent 4632c174dc
180 changed files with 1438 additions and 424 deletions
--- a/apps/stock/data-ingestion/AWILIX-MIGRATION.md
+++ b/apps/stock/data-ingestion/AWILIX-MIGRATION.md
@ -0,0 +1,85 @@
+# Awilix DI Container Migration Guide
+
+This guide explains how to use the new Awilix dependency injection container in the data-ingestion service.
+
+## Overview
+
+The Awilix container provides proper dependency injection for decoupled libraries, allowing them to be reused in other projects without stock-bot specific dependencies.
+
+## Current Implementation
+
+The data-ingestion service now uses a hybrid approach:
+1. Awilix container for ProxyManager and other decoupled services
+2. Legacy service factory for backward compatibility
+
+## Usage Example
+
+```typescript
+// Create Awilix container
+const awilixConfig = {
+  redis: {
+    host: config.database.dragonfly.host,
+    port: config.database.dragonfly.port,
+    db: config.database.dragonfly.db,
+  },
+  mongodb: {
+    uri: config.database.mongodb.uri,
+    database: config.database.mongodb.database,
+  },
+  postgres: {
+    host: config.database.postgres.host,
+    port: config.database.postgres.port,
+    database: config.database.postgres.database,
+    user: config.database.postgres.user,
+    password: config.database.postgres.password,
+  },
+  proxy: {
+    cachePrefix: 'proxy:',
+    ttl: 3600,
+  },
+};
+
+const container = createServiceContainer(awilixConfig);
+await initializeServices(container);
+
+// Access services from container
+const proxyManager = container.resolve('proxyManager');
+const cache = container.resolve('cache');
+```
+
+## Handler Integration
+
+Handlers receive services through the enhanced service container:
+
+```typescript
+// Create service adapter with proxy from Awilix
+const serviceContainerWithProxy = createServiceAdapter(services);
+Object.defineProperty(serviceContainerWithProxy, 'proxy', {
+  get: () => container.resolve('proxyManager'),
+  enumerable: true,
+  configurable: true
+});
+
+// Handlers can now access proxy service
+class MyHandler extends BaseHandler {
+  async myOperation() {
+    const proxy = this.proxy.getRandomProxy();
+    // Use proxy...
+  }
+}
+```
+
+## Benefits
+
+1. **Decoupled Libraries**: Libraries no longer depend on @stock-bot/config
+2. **Reusability**: Libraries can be used in other projects
+3. **Testability**: Easy to mock dependencies for testing
+4. **Type Safety**: Full TypeScript support with Awilix
+
+## Next Steps
+
+To fully migrate to Awilix:
+1. Update HTTP library to accept dependencies via constructor
+2. Update Queue library to accept Redis config via constructor
+3. Create actual MongoDB, PostgreSQL, and QuestDB clients in the container
+4. Remove legacy service factory once all services are migrated
--- a/apps/stock/data-ingestion/package.json
+++ b/apps/stock/data-ingestion/package.json
@ -0,0 +1,32 @@
+{
+  "name": "@stock-bot/data-ingestion",
+  "version": "1.0.0",
+  "description": "Market data ingestion from multiple providers with proxy support and rate limiting",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "dev": "bun --watch src/index.ts",
+    "build": "bun build src/index.ts --outdir dist --target node --external chromium-bidi --external electron --external playwright --external playwright-core",
+    "start": "bun dist/index.js",
+    "test": "bun test",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@stock-bot/cache": "*",
+    "@stock-bot/config": "*",
+    "@stock-bot/stock-config": "*",
+    "@stock-bot/di": "*",
+    "@stock-bot/handlers": "*",
+    "@stock-bot/logger": "*",
+    "@stock-bot/mongodb": "*",
+    "@stock-bot/postgres": "*",
+    "@stock-bot/questdb": "*",
+    "@stock-bot/queue": "*",
+    "@stock-bot/shutdown": "*",
+    "@stock-bot/utils": "*",
+    "hono": "^4.0.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.0.0"
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/ceo/actions/index.ts
+++ b/apps/stock/data-ingestion/src/handlers/ceo/actions/index.ts
@ -0,0 +1,3 @@
+export { updateCeoChannels } from './update-ceo-channels.action';
+export { updateUniqueSymbols } from './update-unique-symbols.action';
+export { processIndividualSymbol } from './process-individual-symbol.action';
--- a/apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts
@ -0,0 +1,117 @@
+import { getRandomUserAgent } from '@stock-bot/utils';
+import type { CeoHandler } from '../ceo.handler';
+
+export async function processIndividualSymbol(
+  this: CeoHandler,
+  payload: any,
+  _context: any
+): Promise<unknown> {
+  const { ceoId, symbol, timestamp } = payload;
+  const proxy = this.proxy?.getProxy();
+  if (!proxy) {
+    this.logger.warn('No proxy available for processing individual CEO symbol');
+    return;
+  }
+
+  this.logger.debug('Processing individual CEO symbol', {
+    ceoId,
+    timestamp,
+  });
+  try {
+    // Fetch detailed information for the individual symbol
+    const response = await this.http.get(
+      `https://api.ceo.ca/api/get_spiels?channel=${ceoId}&load_more=top` +
+        (timestamp ? `&until=${timestamp}` : ''),
+      {
+        proxy: proxy,
+        headers: {
+          'User-Agent': getRandomUserAgent(),
+        },
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error(`Failed to fetch details for ceoId ${ceoId}: ${response.statusText}`);
+    }
+
+    const data = await response.json();
+
+    const spielCount = data.spiels.length;
+    if (spielCount === 0) {
+      this.logger.warn(`No spiels found for ceoId ${ceoId}`);
+      return null; // No data to process
+    }
+    const latestSpielTime = data.spiels[0]?.timestamp;
+    const posts = data.spiels.map((spiel: any) => ({
+      ceoId,
+      spiel: spiel.spiel,
+      spielReplyToId: spiel.spiel_reply_to_id,
+      spielReplyTo: spiel.spiel_reply_to,
+      spielReplyToName: spiel.spiel_reply_to_name,
+      spielReplyToEdited: spiel.spiel_reply_to_edited,
+      userId: spiel.user_id,
+      name: spiel.name,
+      timestamp: spiel.timestamp,
+      spielId: spiel.spiel_id,
+      color: spiel.color,
+      parentId: spiel.parent_id,
+      publicId: spiel.public_id,
+      parentChannel: spiel.parent_channel,
+      parentTimestamp: spiel.parent_timestamp,
+      votes: spiel.votes,
+      editable: spiel.editable,
+      edited: spiel.edited,
+      featured: spiel.featured,
+      verified: spiel.verified,
+      fake: spiel.fake,
+      bot: spiel.bot,
+      voted: spiel.voted,
+      flagged: spiel.flagged,
+      ownSpiel: spiel.own_spiel,
+      score: spiel.score,
+      savedId: spiel.saved_id,
+      savedTimestamp: spiel.saved_timestamp,
+      poll: spiel.poll,
+      votedInPoll: spiel.voted_in_poll,
+    }));
+
+    await this.mongodb.batchUpsert('ceoPosts', posts, ['spielId']);
+    this.logger.info(`Fetched ${spielCount} spiels for ceoId ${ceoId}`);
+
+    // Update Shorts
+    const shortRes = await this.http.get(
+      `https://api.ceo.ca/api/short_positions/one?symbol=${symbol}`,
+      {
+        proxy: proxy,
+        headers: {
+          'User-Agent': getRandomUserAgent(),
+        },
+      }
+    );
+
+    if (shortRes.ok) {
+      const shortData = await shortRes.json();
+      if (shortData && shortData.positions) {
+        await this.mongodb.batchUpsert('ceoShorts', shortData.positions, ['id']);
+      }
+
+      await this.scheduleOperation('process-individual-symbol', {
+        ceoId: ceoId,
+        timestamp: latestSpielTime,
+      });
+    }
+
+    this.logger.info(
+      `Successfully processed channel ${ceoId} and added channel ${ceoId} at timestamp ${latestSpielTime}`
+    );
+
+    return { ceoId, spielCount, timestamp };
+  } catch (error) {
+    this.logger.error('Failed to process individual symbol', {
+      error,
+      ceoId,
+      timestamp,
+    });
+    throw error;
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/ceo/actions/update-ceo-channels.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/ceo/actions/update-ceo-channels.action.ts
@ -0,0 +1,72 @@
+import { getRandomUserAgent } from '@stock-bot/utils';
+import type { CeoHandler } from '../ceo.handler';
+
+export async function updateCeoChannels(
+  this: CeoHandler,
+  payload: number | undefined
+): Promise<unknown> {
+  const proxy = this.proxy?.getProxy();
+  if (!proxy) {
+    this.logger.warn('No proxy available for CEO channels update');
+    return;
+  }
+  let page;
+  if (payload === undefined) {
+    page = 1;
+  } else {
+    page = payload;
+  }
+
+  this.logger.info(`Fetching CEO channels for page ${page} with proxy ${proxy}`);
+  const response = await this.http.get(
+    'https://api.ceo.ca/api/home?exchange=all&sort_by=symbol&sector=All&tab=companies&page=' + page,
+    {
+      proxy: proxy,
+      headers: {
+        'User-Agent': getRandomUserAgent(),
+      },
+    }
+  );
+  const results = await response.json();
+  const channels = results.channel_categories[0].channels;
+  const totalChannels = results.channel_categories[0].total_channels;
+  const totalPages = Math.ceil(totalChannels / channels.length);
+  const exchanges: { exchange: string; countryCode: string }[] = [];
+  const symbols = channels.map((channel: any) => {
+    // check if exchange is in the exchanges array object
+    if (!exchanges.find((e: any) => e.exchange === channel.exchange)) {
+      exchanges.push({
+        exchange: channel.exchange,
+        countryCode: 'CA',
+      });
+    }
+    const details = channel.company_details || {};
+    return {
+      symbol: channel.symbol,
+      exchange: channel.exchange,
+      name: channel.title,
+      type: channel.type,
+      ceoId: channel.channel,
+      marketCap: details.market_cap,
+      volumeRatio: details.volume_ratio,
+      avgVolume: details.avg_volume,
+      stockType: details.stock_type,
+      issueType: details.issue_type,
+      sharesOutstanding: details.shares_outstanding,
+      float: details.float,
+    };
+  });
+
+  await this.mongodb.batchUpsert('ceoSymbols', symbols, ['symbol', 'exchange']);
+  await this.mongodb.batchUpsert('ceoExchanges', exchanges, ['exchange']);
+
+  if (page === 1) {
+    for (let i = 2; i <= totalPages; i++) {
+      this.logger.info(`Scheduling page ${i} of ${totalPages} for CEO channels`);
+      await this.scheduleOperation('update-ceo-channels', i);
+    }
+  }
+
+  this.logger.info(`Fetched CEO channels for page ${page}/${totalPages}`);
+  return { page, totalPages };
+}
--- a/apps/stock/data-ingestion/src/handlers/ceo/actions/update-unique-symbols.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/ceo/actions/update-unique-symbols.action.ts
@ -0,0 +1,71 @@
+import type { CeoHandler } from '../ceo.handler';
+
+export async function updateUniqueSymbols(
+  this: CeoHandler,
+  _payload: unknown,
+  _context: any
+): Promise<unknown> {
+  this.logger.info('Starting update to get unique CEO symbols by ceoId');
+
+  try {
+    // Get unique ceoId values from the ceoSymbols collection
+    const uniqueCeoIds = await this.mongodb.collection('ceoSymbols').distinct('ceoId');
+
+    this.logger.info(`Found ${uniqueCeoIds.length} unique CEO IDs`);
+
+    // Get detailed records for each unique ceoId (latest/first record)
+    const uniqueSymbols = [];
+    for (const ceoId of uniqueCeoIds) {
+      const symbol = await this.mongodb
+        .collection('ceoSymbols')
+        .findOne({ ceoId }, { sort: { _id: -1 } }); // Get latest record
+
+      if (symbol) {
+        uniqueSymbols.push(symbol);
+      }
+    }
+
+    this.logger.info(`Retrieved ${uniqueSymbols.length} unique symbol records`);
+
+    // Schedule individual jobs for each unique symbol
+    let scheduledJobs = 0;
+    for (const symbol of uniqueSymbols) {
+      // Schedule a job to process this individual symbol
+      await this.scheduleOperation('process-individual-symbol', {
+        ceoId: symbol.ceoId,
+        symbol: symbol.symbol,
+      });
+      scheduledJobs++;
+
+      // Add small delay to avoid overwhelming the queue
+      if (scheduledJobs % 10 === 0) {
+        this.logger.debug(`Scheduled ${scheduledJobs} jobs so far`);
+      }
+    }
+
+    this.logger.info(`Successfully scheduled ${scheduledJobs} individual symbol update jobs`);
+
+    // Cache the results for monitoring
+    await this.cacheSet(
+      'unique-symbols-last-run',
+      {
+        timestamp: new Date().toISOString(),
+        totalUniqueIds: uniqueCeoIds.length,
+        totalRecords: uniqueSymbols.length,
+        scheduledJobs,
+      },
+      1800
+    ); // Cache for 30 minutes
+
+    return {
+      success: true,
+      uniqueCeoIds: uniqueCeoIds.length,
+      uniqueRecords: uniqueSymbols.length,
+      scheduledJobs,
+      timestamp: new Date().toISOString(),
+    };
+  } catch (error) {
+    this.logger.error('Failed to update unique CEO symbols', { error });
+    throw error;
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/ceo/ceo.handler.ts
+++ b/apps/stock/data-ingestion/src/handlers/ceo/ceo.handler.ts
@ -0,0 +1,34 @@
+import {
+  BaseHandler,
+  Handler,
+  Operation,
+  ScheduledOperation,
+  type IServiceContainer,
+} from '@stock-bot/handlers';
+import { processIndividualSymbol, updateCeoChannels, updateUniqueSymbols } from './actions';
+
+@Handler('ceo')
+// @Disabled()
+export class CeoHandler extends BaseHandler {
+  constructor(services: IServiceContainer) {
+    super(services); // Handler name read from @Handler decorator
+  }
+
+  @ScheduledOperation('update-ceo-channels', '0 */15 * * *', {
+    priority: 7,
+    immediately: false,
+    description: 'Get all CEO symbols and exchanges',
+  })
+  updateCeoChannels = updateCeoChannels;
+
+  @Operation('update-unique-symbols')
+  @ScheduledOperation('process-unique-symbols', '0 0 1 * *', {
+    priority: 5,
+    immediately: false,
+    description: 'Process unique CEO symbols and schedule individual jobs',
+  })
+  updateUniqueSymbols = updateUniqueSymbols;
+
+  @Operation('process-individual-symbol')
+  processIndividualSymbol = processIndividualSymbol;
+}
--- a/apps/stock/data-ingestion/src/handlers/example/example.handler.ts
+++ b/apps/stock/data-ingestion/src/handlers/example/example.handler.ts
@ -0,0 +1,94 @@
+/**
+ * Example Handler - Demonstrates ergonomic handler patterns
+ * Shows inline operations, service helpers, and scheduled operations
+ */
+
+import {
+  BaseHandler,
+  Handler,
+  Operation,
+  ScheduledOperation,
+  type ExecutionContext,
+  type IServiceContainer,
+} from '@stock-bot/handlers';
+
+@Handler('example')
+export class ExampleHandler extends BaseHandler {
+  constructor(services: IServiceContainer) {
+    super(services);
+  }
+
+  /**
+   * Simple inline operation - no separate action file needed
+   */
+  @Operation('get-stats')
+  async getStats(): Promise<{ total: number; active: number; cached: boolean }> {
+    // Use collection helper for cleaner MongoDB access
+    const total = await this.collection('items').countDocuments();
+    const active = await this.collection('items').countDocuments({ status: 'active' });
+
+    // Use cache helpers with automatic prefixing
+    const cached = await this.cacheGet<number>('last-total');
+    await this.cacheSet('last-total', total, 300); // 5 minutes
+
+    // Use log helper with automatic handler context
+    this.log('info', 'Stats retrieved', { total, active });
+
+    return { total, active, cached: cached !== null };
+  }
+
+  /**
+   * Scheduled operation using combined decorator
+   */
+  @ScheduledOperation('cleanup-old-items', '0 2 * * *', {
+    priority: 5,
+    description: 'Clean up items older than 30 days',
+  })
+  async cleanupOldItems(): Promise<{ deleted: number }> {
+    const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
+
+    const result = await this.collection('items').deleteMany({
+      createdAt: { $lt: thirtyDaysAgo },
+    });
+
+    this.log('info', 'Cleanup completed', { deleted: result.deletedCount });
+
+    // Schedule a follow-up task
+    await this.scheduleIn('generate-report', { type: 'cleanup' }, 60); // 1 minute
+
+    return { deleted: result.deletedCount };
+  }
+
+  /**
+   * Operation that uses proxy service
+   */
+  @Operation('fetch-external-data')
+  async fetchExternalData(input: { url: string }): Promise<{ data: any }> {
+    const proxyUrl = this.proxy.getProxy();
+
+    if (!proxyUrl) {
+      throw new Error('No proxy available');
+    }
+
+    // Use HTTP client with proxy
+    const response = await this.http.get(input.url, {
+      proxy: proxyUrl,
+      timeout: 10000,
+    });
+
+    // Cache the result
+    await this.cacheSet(`external:${input.url}`, response.data, 3600);
+
+    return { data: response.data };
+  }
+
+  /**
+   * Complex operation that still uses action file
+   */
+  @Operation('process-batch')
+  async processBatch(input: any, context: ExecutionContext): Promise<unknown> {
+    // For complex operations, still use action files
+    const { processBatch } = await import('./actions/batch.action');
+    return processBatch(this, input);
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts
@ -0,0 +1,38 @@
+import type { IbHandler } from '../ib.handler';
+
+export async function fetchExchangesAndSymbols(this: IbHandler): Promise<unknown> {
+  this.logger.info('Starting IB exchanges and symbols fetch job');
+
+  try {
+    // Fetch session headers first
+    const sessionHeaders = await this.fetchSession();
+    if (!sessionHeaders) {
+      this.logger.error('Failed to get session headers for IB job');
+      return { success: false, error: 'No session headers' };
+    }
+
+    this.logger.info('Session headers obtained, fetching exchanges...');
+
+    // Fetch exchanges
+    const exchanges = await this.fetchExchanges();
+    this.logger.info('Fetched exchanges from IB', { count: exchanges?.length || 0 });
+
+    // Fetch symbols
+    this.logger.info('Fetching symbols...');
+    const symbols = await this.fetchSymbols();
+    this.logger.info('Fetched symbols from IB', { count: symbols?.length || 0 });
+
+    return {
+      success: true,
+      exchangesCount: exchanges?.length || 0,
+      symbolsCount: symbols?.length || 0,
+    };
+  } catch (error) {
+    this.logger.error('Failed to fetch IB exchanges and symbols', { error });
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : 'Unknown error',
+    };
+  }
+}
+
--- a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges.action.ts
@ -0,0 +1,66 @@
+import type { IbHandler } from '../ib.handler';
+import { IB_CONFIG } from '../shared/config';
+
+export async function fetchExchanges(this: IbHandler): Promise<unknown[] | null> {
+  try {
+    // First get session headers
+    const sessionHeaders = await this.fetchSession();
+    if (!sessionHeaders) {
+      throw new Error('Failed to get session headers');
+    }
+
+    this.logger.info('🔍 Fetching exchanges with session headers...');
+
+    // The URL for the exchange data API
+    const exchangeUrl = IB_CONFIG.BASE_URL + IB_CONFIG.EXCHANGE_API;
+
+    // Prepare headers - include all session headers plus any additional ones
+    const requestHeaders = {
+      ...sessionHeaders,
+      Accept: 'application/json, text/plain, */*',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Cache-Control': 'no-cache',
+      Pragma: 'no-cache',
+      'Sec-Fetch-Dest': 'empty',
+      'Sec-Fetch-Mode': 'cors',
+      'Sec-Fetch-Site': 'same-origin',
+      'X-Requested-With': 'XMLHttpRequest',
+    };
+
+    this.logger.info('📤 Making request to exchange API...', {
+      url: exchangeUrl,
+      headerCount: Object.keys(requestHeaders).length,
+    });
+
+    // Use fetch with proxy configuration
+    const response = await fetch(exchangeUrl, {
+      method: 'GET',
+      headers: requestHeaders,
+      proxy: IB_CONFIG.DEFAULT_PROXY,
+    });
+
+    if (!response.ok) {
+      this.logger.error('❌ Exchange API request failed', {
+        status: response.status,
+        statusText: response.statusText,
+      });
+      return null;
+    }
+
+    const data = await response.json();
+    const exchanges = data?.exchanges || [];
+    this.logger.info('✅ Exchange data fetched successfully');
+
+    this.logger.info('Saving IB exchanges to MongoDB...');
+    await this.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']);
+    this.logger.info('✅ Exchange IB data saved to MongoDB:', {
+      count: exchanges.length,
+    });
+
+    return exchanges;
+  } catch (error) {
+    this.logger.error('❌ Failed to fetch exchanges', { error });
+    return null;
+  }
+}
+
--- a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts
@ -0,0 +1,83 @@
+import { Browser } from '@stock-bot/browser';
+import type { IbHandler } from '../ib.handler';
+import { IB_CONFIG } from '../shared/config';
+
+export async function fetchSession(this: IbHandler): Promise<Record<string, string> | undefined> {
+  try {
+    await Browser.initialize({
+      headless: true,
+      timeout: IB_CONFIG.BROWSER_TIMEOUT,
+      blockResources: false,
+    });
+    this.logger.info('✅ Browser initialized');
+
+    const { page } = await Browser.createPageWithProxy(
+      IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_PAGE,
+      IB_CONFIG.DEFAULT_PROXY
+    );
+    this.logger.info('✅ Page created with proxy');
+
+    const headersPromise = new Promise<Record<string, string> | undefined>(resolve => {
+      let resolved = false;
+
+      page.onNetworkEvent(event => {
+        if (event.url.includes('/webrest/search/product-types/summary')) {
+          if (event.type === 'request') {
+            try {
+              resolve(event.headers);
+            } catch (e) {
+              resolve(undefined);
+              this.logger.debug('Raw Summary Response error', { error: (e as Error).message });
+            }
+          }
+        }
+      });
+
+      // Timeout fallback
+      setTimeout(() => {
+        if (!resolved) {
+          resolved = true;
+          this.logger.warn('Timeout waiting for headers');
+          resolve(undefined);
+        }
+      }, IB_CONFIG.HEADERS_TIMEOUT);
+    });
+
+    this.logger.info('⏳ Waiting for page load...');
+    await page.waitForLoadState('domcontentloaded', { timeout: IB_CONFIG.PAGE_LOAD_TIMEOUT });
+    this.logger.info('✅ Page loaded');
+
+    //Products tabs
+    this.logger.info('🔍 Looking for Products tab...');
+    const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
+    await productsTab.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT });
+    this.logger.info('✅ Found Products tab');
+    this.logger.info('🖱️ Clicking Products tab...');
+    await productsTab.click();
+    this.logger.info('✅ Products tab clicked');
+
+    // New Products Checkbox
+    this.logger.info('🔍 Looking for "New Products Only" radio button...');
+    const radioButton = page.locator('span.checkbox-text:has-text("New Products Only")');
+    await radioButton.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT });
+    this.logger.info(`🎯 Found "New Products Only" radio button`);
+    await radioButton.first().click();
+    this.logger.info('✅ "New Products Only" radio button clicked');
+
+    // Wait for and return headers immediately when captured
+    this.logger.info('⏳ Waiting for headers to be captured...');
+    const headers = await headersPromise;
+    page.close();
+    if (headers) {
+      this.logger.info('✅ Headers captured successfully');
+    } else {
+      this.logger.warn('⚠️ No headers were captured');
+    }
+
+    return headers;
+  } catch (error) {
+    this.logger.error('Failed to fetch IB symbol summary', { error });
+    return;
+  }
+}
+
--- a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts
@ -0,0 +1,117 @@
+import type { IbHandler } from '../ib.handler';
+import { IB_CONFIG } from '../shared/config';
+
+export async function fetchSymbols(this: IbHandler): Promise<unknown[] | null> {
+  try {
+    // First get session headers
+    const sessionHeaders = await this.fetchSession();
+    if (!sessionHeaders) {
+      throw new Error('Failed to get session headers');
+    }
+
+    this.logger.info('🔍 Fetching symbols with session headers...');
+
+    // Prepare headers - include all session headers plus any additional ones
+    const requestHeaders = {
+      ...sessionHeaders,
+      Accept: 'application/json, text/plain, */*',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Cache-Control': 'no-cache',
+      Pragma: 'no-cache',
+      'Sec-Fetch-Dest': 'empty',
+      'Sec-Fetch-Mode': 'cors',
+      'Sec-Fetch-Site': 'same-origin',
+      'X-Requested-With': 'XMLHttpRequest',
+    };
+
+    const requestBody = {
+      domain: 'com',
+      newProduct: 'all',
+      pageNumber: 1,
+      pageSize: 100,
+      productCountry: IB_CONFIG.PRODUCT_COUNTRIES,
+      productSymbol: '',
+      productType: IB_CONFIG.PRODUCT_TYPES,
+      sortDirection: 'asc',
+      sortField: 'symbol',
+    };
+
+    // Get Summary
+    const summaryResponse = await fetch(IB_CONFIG.BASE_URL + IB_CONFIG.SUMMARY_API, {
+      method: 'POST',
+      headers: requestHeaders,
+      proxy: IB_CONFIG.DEFAULT_PROXY,
+      body: JSON.stringify(requestBody),
+    });
+
+    if (!summaryResponse.ok) {
+      this.logger.error('❌ Summary API request failed', {
+        status: summaryResponse.status,
+        statusText: summaryResponse.statusText,
+      });
+      return null;
+    }
+
+    const summaryData = await summaryResponse.json();
+    this.logger.info('✅ IB Summary data fetched successfully', {
+      totalCount: summaryData[0].totalCount,
+    });
+
+    const symbols = [];
+    requestBody.pageSize = IB_CONFIG.PAGE_SIZE;
+    const pageCount = Math.ceil(summaryData[0].totalCount / IB_CONFIG.PAGE_SIZE) || 0;
+    this.logger.info('Fetching Symbols for IB', { pageCount });
+
+    const symbolPromises = [];
+    for (let page = 1; page <= pageCount; page++) {
+      requestBody.pageNumber = page;
+
+      // Fetch symbols for the current page
+      const symbolsResponse = fetch(IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_API, {
+        method: 'POST',
+        headers: requestHeaders,
+        proxy: IB_CONFIG.DEFAULT_PROXY,
+        body: JSON.stringify(requestBody),
+      });
+      symbolPromises.push(symbolsResponse);
+    }
+
+    const responses = await Promise.all(symbolPromises);
+    for (const response of responses) {
+      if (!response.ok) {
+        this.logger.error('❌ Symbols API request failed', {
+          status: response.status,
+          statusText: response.statusText,
+        });
+        return null;
+      }
+      const data = await response.json();
+      const symJson = data?.products || [];
+      if (symJson && symJson.length > 0) {
+        symbols.push(...symJson);
+      } else {
+        this.logger.warn('⚠️ No symbols found in response');
+        continue;
+      }
+    }
+
+    if (symbols.length === 0) {
+      this.logger.warn('⚠️ No symbols fetched from IB');
+      return null;
+    }
+
+    this.logger.info('✅ IB symbols fetched successfully, saving to DB...', {
+      totalSymbols: symbols.length,
+    });
+    await this.mongodb.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
+    this.logger.info('Saved IB symbols to DB', {
+      totalSymbols: symbols.length,
+    });
+
+    return symbols;
+  } catch (error) {
+    this.logger.error('❌ Failed to fetch symbols', { error });
+    return null;
+  }
+}
+
--- a/apps/stock/data-ingestion/src/handlers/ib/actions/index.ts
+++ b/apps/stock/data-ingestion/src/handlers/ib/actions/index.ts
@ -0,0 +1,5 @@
+export { fetchSession } from './fetch-session.action';
+export { fetchExchanges } from './fetch-exchanges.action';
+export { fetchSymbols } from './fetch-symbols.action';
+export { fetchExchangesAndSymbols } from './fetch-exchanges-and-symbols.action';
+
--- a/apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts
+++ b/apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts
@ -0,0 +1,33 @@
+import {
+  BaseHandler,
+  Handler,
+  Operation,
+  ScheduledOperation,
+  type IServiceContainer,
+} from '@stock-bot/handlers';
+import { fetchExchanges, fetchExchangesAndSymbols, fetchSession, fetchSymbols } from './actions';
+
+@Handler('ib')
+export class IbHandler extends BaseHandler {
+  constructor(services: IServiceContainer) {
+    super(services);
+  }
+
+  @Operation('fetch-session')
+  fetchSession = fetchSession;
+
+  @Operation('fetch-exchanges')
+  fetchExchanges = fetchExchanges;
+
+  @Operation('fetch-symbols')
+  fetchSymbols = fetchSymbols;
+
+  @Operation('ib-exchanges-and-symbols')
+  @ScheduledOperation('ib-exchanges-and-symbols', '0 0 * * 0', {
+    priority: 5,
+    description: 'Fetch and update IB exchanges and symbols data',
+    immediately: false,
+  })
+  fetchExchangesAndSymbols = fetchExchangesAndSymbols;
+}
+
--- a/apps/stock/data-ingestion/src/handlers/ib/shared/config.ts
+++ b/apps/stock/data-ingestion/src/handlers/ib/shared/config.ts
@ -0,0 +1,24 @@
+/**
+ * Interactive Brokers Configuration Constants
+ */
+
+export const IB_CONFIG = {
+  BASE_URL: 'https://www.interactivebrokers.com',
+  PRODUCTS_PAGE: '/en/trading/products-exchanges.php#/',
+  EXCHANGE_API: '/webrest/exchanges',
+  SUMMARY_API: '/webrest/search/product-types/summary',
+  PRODUCTS_API: '/webrest/search/products-by-filters',
+
+  // Browser configuration
+  BROWSER_TIMEOUT: 10000,
+  PAGE_LOAD_TIMEOUT: 20000,
+  ELEMENT_TIMEOUT: 5000,
+  HEADERS_TIMEOUT: 30000,
+
+  // API configuration
+  DEFAULT_PROXY: 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80',
+  PAGE_SIZE: 500,
+  PRODUCT_COUNTRIES: ['CA', 'US'],
+  PRODUCT_TYPES: ['STK'],
+};
+
--- a/apps/stock/data-ingestion/src/handlers/index.ts
+++ b/apps/stock/data-ingestion/src/handlers/index.ts
@ -0,0 +1,60 @@
+/**
+ * Handler auto-registration
+ * Automatically discovers and registers all handlers
+ */
+
+import type { IServiceContainer } from '@stock-bot/handlers';
+import { autoRegisterHandlers } from '@stock-bot/handlers';
+import { getLogger } from '@stock-bot/logger';
+// Import handlers for bundling (ensures they're included in the build)
+import './ceo/ceo.handler';
+import './ib/ib.handler';
+import './proxy/proxy.handler';
+import './qm/qm.handler';
+import './webshare/webshare.handler';
+
+// Add more handler imports as needed
+
+const logger = getLogger('handler-init');
+
+/**
+ * Initialize and register all handlers automatically
+ */
+export async function initializeAllHandlers(serviceContainer: IServiceContainer): Promise<void> {
+  try {
+    // Auto-register all handlers in this directory
+    const result = await autoRegisterHandlers(__dirname, serviceContainer, {
+      pattern: '.handler.',
+      exclude: ['test', 'spec'],
+      dryRun: false,
+    });
+
+    logger.info('Handler auto-registration complete', {
+      registered: result.registered,
+      failed: result.failed,
+    });
+
+    if (result.failed.length > 0) {
+      logger.error('Some handlers failed to register', { failed: result.failed });
+    }
+  } catch (error) {
+    logger.error('Handler auto-registration failed', { error });
+    // Fall back to manual registration
+    await manualHandlerRegistration(serviceContainer);
+  }
+}
+
+/**
+ * Manual fallback registration
+ */
+async function manualHandlerRegistration(_serviceContainer: any): Promise<void> {
+  logger.warn('Falling back to manual handler registration');
+
+  try {
+
+    logger.info('Manual handler registration complete');
+  } catch (error) {
+    logger.error('Manual handler registration failed', { error });
+    throw error;
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/proxy/operations/check.operations.ts
+++ b/apps/stock/data-ingestion/src/handlers/proxy/operations/check.operations.ts
@ -0,0 +1,176 @@
+/**
+ * Proxy Check Operations - Checking proxy functionality
+ */
+import { OperationContext } from '@stock-bot/di';
+import { getLogger } from '@stock-bot/logger';
+import type { ProxyInfo } from '@stock-bot/proxy';
+import { fetch } from '@stock-bot/utils';
+import { PROXY_CONFIG } from '../shared/config';
+
+/**
+ * Check if a proxy is working
+ */
+export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
+  const ctx = {
+    logger: getLogger('proxy-check'),
+    resolve: <T>(_name: string) => {
+      throw new Error(`Service container not available for proxy operations`);
+    },
+  } as any;
+
+  let success = false;
+  ctx.logger.debug(`Checking Proxy:`, {
+    protocol: proxy.protocol,
+    host: proxy.host,
+    port: proxy.port,
+  });
+
+  try {
+    // Test the proxy using fetch with proxy support
+    const proxyUrl =
+      proxy.username && proxy.password
+        ? `${proxy.protocol}://${encodeURIComponent(proxy.username)}:${encodeURIComponent(proxy.password)}@${proxy.host}:${proxy.port}`
+        : `${proxy.protocol}://${proxy.host}:${proxy.port}`;
+
+    const response = await fetch(PROXY_CONFIG.CHECK_URL, {
+      proxy: proxyUrl,
+      signal: AbortSignal.timeout(PROXY_CONFIG.CHECK_TIMEOUT),
+      logger: ctx.logger,
+    } as any);
+
+    const data = await response.text();
+
+    const isWorking = response.ok;
+    const result: ProxyInfo = {
+      ...proxy,
+      isWorking,
+      lastChecked: new Date(),
+    };
+
+    if (isWorking && !data.includes(PROXY_CONFIG.CHECK_IP)) {
+      success = true;
+      await updateProxyInCache(result, true, ctx);
+    } else {
+      await updateProxyInCache(result, false, ctx);
+    }
+
+    if (proxy.source) {
+      updateProxyStats(proxy.source, success, ctx);
+    }
+
+    ctx.logger.debug('Proxy check completed', {
+      host: proxy.host,
+      port: proxy.port,
+      isWorking,
+    });
+
+    return result;
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    const result: ProxyInfo = {
+      ...proxy,
+      isWorking: false,
+      error: errorMessage,
+      lastChecked: new Date(),
+    };
+
+    // Update cache for failed proxy (increment total, don't update TTL)
+    await updateProxyInCache(result, false, ctx);
+
+    if (proxy.source) {
+      updateProxyStats(proxy.source, success, ctx);
+    }
+
+    ctx.logger.debug('Proxy check failed', {
+      host: proxy.host,
+      port: proxy.port,
+      error: errorMessage,
+    });
+
+    return result;
+  }
+}
+
+/**
+ * Update proxy data in cache with working/total stats and average response time
+ */
+async function updateProxyInCache(
+  proxy: ProxyInfo,
+  isWorking: boolean,
+  ctx: OperationContext
+): Promise<void> {
+  const _cacheKey = `${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`;
+
+  try {
+    // For now, skip cache operations without service container
+    // TODO: Pass service container to operations
+    const existing: ProxyInfo | null = null;
+
+    // For failed proxies, only update if they already exist
+    if (!isWorking && !existing) {
+      ctx.logger.debug('Proxy not in cache, skipping failed update', {
+        proxy: `${proxy.host}:${proxy.port}`,
+      });
+      return;
+    }
+
+    // Calculate new average response time if we have a response time
+    let newAverageResponseTime = existing?.averageResponseTime;
+    if (proxy.responseTime !== undefined) {
+      const existingAvg = existing?.averageResponseTime || 0;
+      const existingTotal = existing?.total || 0;
+
+      // Calculate weighted average: (existing_avg * existing_count + new_response) / (existing_count + 1)
+      newAverageResponseTime =
+        existingTotal > 0
+          ? (existingAvg * existingTotal + proxy.responseTime) / (existingTotal + 1)
+          : proxy.responseTime;
+    }
+
+    // Build updated proxy data
+    const updated = {
+      ...existing,
+      ...proxy, // Keep latest proxy info
+      total: (existing?.total || 0) + 1,
+      working: isWorking ? (existing?.working || 0) + 1 : existing?.working || 0,
+      isWorking,
+      lastChecked: new Date(),
+      // Add firstSeen only for new entries
+      ...(existing ? {} : { firstSeen: new Date() }),
+      // Update average response time if we calculated a new one
+      ...(newAverageResponseTime !== undefined
+        ? { averageResponseTime: newAverageResponseTime }
+        : {}),
+    };
+
+    // Calculate success rate
+    updated.successRate = updated.total > 0 ? (updated.working / updated.total) * 100 : 0;
+
+    // Save to cache: reset TTL for working proxies, keep existing TTL for failed ones
+    const _cacheOptions = isWorking ? { ttl: PROXY_CONFIG.CACHE_TTL } : undefined;
+    // Skip cache operations without service container
+    // TODO: Pass service container to operations
+
+    ctx.logger.debug(`Updated ${isWorking ? 'working' : 'failed'} proxy in cache`, {
+      proxy: `${proxy.host}:${proxy.port}`,
+      working: updated.working,
+      total: updated.total,
+      successRate: updated.successRate.toFixed(1) + '%',
+      avgResponseTime: updated.averageResponseTime
+        ? `${updated.averageResponseTime.toFixed(0)}ms`
+        : 'N/A',
+    });
+  } catch (error) {
+    ctx.logger.error('Failed to update proxy in cache', {
+      proxy: `${proxy.host}:${proxy.port}`,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+function updateProxyStats(sourceId: string, success: boolean, ctx: OperationContext) {
+  // Stats are now handled by the global ProxyManager
+  ctx.logger.debug('Proxy check result', { sourceId, success });
+
+  // TODO: Integrate with global ProxyManager stats if needed
+}
--- a/apps/stock/data-ingestion/src/handlers/proxy/operations/fetch.operations.ts
+++ b/apps/stock/data-ingestion/src/handlers/proxy/operations/fetch.operations.ts
@ -0,0 +1,104 @@
+/**
+ * Proxy Fetch Operations - Fetching proxies from sources
+ */
+import type { ProxyInfo } from '@stock-bot/proxy';
+import { OperationContext } from '@stock-bot/di';
+import { getLogger } from '@stock-bot/logger';
+import { fetch } from '@stock-bot/utils';
+
+import { PROXY_CONFIG } from '../shared/config';
+import type { ProxySource } from '../shared/types';
+
+export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
+  const ctx = {
+    logger: getLogger('proxy-fetch')
+  } as any;
+  
+  ctx.logger.info('Starting proxy fetch from sources');
+  
+  const fetchPromises = PROXY_CONFIG.PROXY_SOURCES.map(source => fetchProxiesFromSource(source, ctx));
+  const results = await Promise.all(fetchPromises);
+  let allProxies: ProxyInfo[] = results.flat();
+  allProxies = removeDuplicateProxies(allProxies);
+  
+  ctx.logger.info('Fetched proxies from all sources', { total: allProxies.length });
+  return allProxies;
+}
+
+export async function fetchProxiesFromSource(source: ProxySource, ctx?: OperationContext): Promise<ProxyInfo[]> {
+  if (!ctx) {
+    ctx = OperationContext.create('proxy', 'fetch-source');
+  }
+  
+  const allProxies: ProxyInfo[] = [];
+
+  try {
+    ctx.logger.info(`Fetching proxies from ${source.url}`);
+
+    const response = await fetch(source.url, {
+      signal: AbortSignal.timeout(10000),
+      logger: ctx.logger
+    } as any);
+
+    if (!response.ok) {
+      ctx.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
+      return [];
+    }
+
+    const text = await response.text();
+    const lines = text.split('\n').filter((line: string) => line.trim());
+
+    for (const line of lines) {
+      let trimmed = line.trim();
+      trimmed = cleanProxyUrl(trimmed);
+      if (!trimmed || trimmed.startsWith('#')) {
+        continue;
+      }
+
+      // Parse formats like \"host:port\" or \"host:port:user:pass\"
+      const parts = trimmed.split(':');
+      if (parts.length >= 2) {
+        const proxy: ProxyInfo = {
+          source: source.id,
+          protocol: source.protocol as 'http' | 'https',
+          host: parts[0],
+          port: parseInt(parts[1]),
+        };
+
+        if (!isNaN(proxy.port) && proxy.host) {
+          allProxies.push(proxy);
+        }
+      }
+    }
+
+    ctx.logger.info(`Parsed ${allProxies.length} proxies from ${source.url}`);
+  } catch (error) {
+    ctx.logger.error(`Error fetching proxies from ${source.url}`, error);
+    return [];
+  }
+
+  return allProxies;
+}
+
+// Utility functions
+function cleanProxyUrl(url: string): string {
+  return url
+    .replace(/^https?:\/\//, '')
+    .replace(/^0+/, '')
+    .replace(/:0+(\d)/g, ':$1');
+}
+
+function removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
+  const seen = new Set<string>();
+  const unique: ProxyInfo[] = [];
+
+  for (const proxy of proxies) {
+    const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
+    if (!seen.has(key)) {
+      seen.add(key);
+      unique.push(proxy);
+    }
+  }
+
+  return unique;
+}
--- a/apps/stock/data-ingestion/src/handlers/proxy/operations/query.operations.ts
+++ b/apps/stock/data-ingestion/src/handlers/proxy/operations/query.operations.ts
@ -0,0 +1,81 @@
+/**
+ * Proxy Query Operations - Getting active proxies from cache
+ */
+import { OperationContext } from '@stock-bot/di';
+import type { ProxyInfo } from '@stock-bot/proxy';
+import { PROXY_CONFIG } from '../shared/config';
+
+/**
+ * Get a random active proxy from the cache
+ * @param protocol - Optional protocol filter ('http' | 'https' | 'socks4' | 'socks5')
+ * @param minSuccessRate - Minimum success rate percentage (default: 50)
+ * @returns A random working proxy or null if none found
+ */
+export async function getRandomActiveProxy(
+  protocol?: 'http' | 'https' | 'socks4' | 'socks5',
+  minSuccessRate: number = 50
+): Promise<ProxyInfo | null> {
+  const ctx = OperationContext.create('proxy', 'get-random');
+
+  try {
+    // Get all active proxy keys from cache
+    const pattern = protocol
+      ? `${PROXY_CONFIG.CACHE_KEY}:${protocol}://*`
+      : `${PROXY_CONFIG.CACHE_KEY}:*`;
+
+    const keys = await ctx.cache.keys(pattern);
+
+    if (keys.length === 0) {
+      ctx.logger.debug('No active proxies found in cache', { pattern });
+      return null;
+    }
+
+    // Shuffle the keys for randomness
+    const shuffledKeys = keys.sort(() => Math.random() - 0.5);
+
+    // Find a working proxy that meets the criteria
+    for (const key of shuffledKeys) {
+      try {
+        const proxyData: ProxyInfo | null = await ctx.cache.get(key);
+
+        if (
+          proxyData &&
+          proxyData.isWorking &&
+          (!proxyData.successRate || proxyData.successRate >= minSuccessRate)
+        ) {
+          ctx.logger.debug('Random active proxy selected', {
+            proxy: `${proxyData.host}:${proxyData.port}`,
+            protocol: proxyData.protocol,
+            successRate: proxyData.successRate?.toFixed(1) + '%',
+            avgResponseTime: proxyData.averageResponseTime
+              ? `${proxyData.averageResponseTime.toFixed(0)}ms`
+              : 'N/A',
+          });
+
+          return proxyData;
+        }
+      } catch (error) {
+        ctx.logger.debug('Error reading proxy from cache', {
+          key,
+          error: (error as Error).message,
+        });
+        continue;
+      }
+    }
+
+    ctx.logger.debug('No working proxies found meeting criteria', {
+      protocol,
+      minSuccessRate,
+      keysChecked: shuffledKeys.length,
+    });
+
+    return null;
+  } catch (error) {
+    ctx.logger.error('Error getting random active proxy', {
+      error: error instanceof Error ? error.message : String(error),
+      protocol,
+      minSuccessRate,
+    });
+    return null;
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/proxy/operations/queue.operations.ts
+++ b/apps/stock/data-ingestion/src/handlers/proxy/operations/queue.operations.ts
@ -0,0 +1,48 @@
+/**
+ * Proxy Queue Operations - Queueing proxy operations
+ */
+import { OperationContext } from '@stock-bot/di';
+import type { ProxyInfo } from '@stock-bot/proxy';
+import type { IServiceContainer } from '@stock-bot/handlers';
+
+export async function queueProxyFetch(container: IServiceContainer): Promise<string> {
+  const ctx = OperationContext.create('proxy', 'queue-fetch');
+
+  const queueManager = container.queue;
+  if (!queueManager) {
+    throw new Error('Queue manager not available');
+  }
+  
+  const queue = queueManager.getQueue('proxy');
+  const job = await queue.add('proxy-fetch', {
+    handler: 'proxy',
+    operation: 'fetch-and-check',
+    payload: {},
+    priority: 5,
+  });
+
+  const jobId = job.id || 'unknown';
+  ctx.logger.info('Proxy fetch job queued', { jobId });
+  return jobId;
+}
+
+export async function queueProxyCheck(proxies: ProxyInfo[], container: IServiceContainer): Promise<string> {
+  const ctx = OperationContext.create('proxy', 'queue-check');
+
+  const queueManager = container.queue;
+  if (!queueManager) {
+    throw new Error('Queue manager not available');
+  }
+  
+  const queue = queueManager.getQueue('proxy');
+  const job = await queue.add('proxy-check', {
+    handler: 'proxy',
+    operation: 'check-specific',
+    payload: { proxies },
+    priority: 3,
+  });
+
+  const jobId = job.id || 'unknown';
+  ctx.logger.info('Proxy check job queued', { jobId, count: proxies.length });
+  return jobId;
+}
--- a/apps/stock/data-ingestion/src/handlers/proxy/proxy.handler.ts
+++ b/apps/stock/data-ingestion/src/handlers/proxy/proxy.handler.ts
@ -0,0 +1,86 @@
+import {
+  BaseHandler,
+  Handler,
+  Operation,
+  ScheduledOperation,
+  type IServiceContainer,
+} from '@stock-bot/handlers';
+import type { ProxyInfo } from '@stock-bot/proxy';
+import { processItems } from '@stock-bot/queue';
+import { fetchProxiesFromSources } from './operations/fetch.operations';
+import { checkProxy } from './operations/check.operations';
+
+@Handler('proxy')
+export class ProxyHandler extends BaseHandler {
+  constructor(services: IServiceContainer) {
+    super(services);
+  }
+
+  @Operation('fetch-from-sources')
+  @ScheduledOperation('proxy-fetch-and-check', '0 0 * * 0', {
+    priority: 0,
+    description: 'Fetch and validate proxy list from sources',
+    // immediately: true, // Don't run immediately during startup to avoid conflicts
+  })
+  async fetchFromSources(): Promise<{
+    processed: number;
+    jobsCreated: number;
+    batchesCreated?: number;
+    mode: string;
+  }> {
+    // Fetch proxies from all configured sources
+    this.logger.info('Processing fetch proxies from sources request');
+    
+    const proxies = await fetchProxiesFromSources();
+    this.logger.info('Fetched proxies from sources', { count: proxies.length });
+
+    if (proxies.length === 0) {
+      this.logger.warn('No proxies fetched from sources');
+      return { processed: 0, jobsCreated: 0, mode: 'direct' };
+    }
+
+    // Get QueueManager from service container
+    const queueManager = this.queue;
+    if (!queueManager) {
+      throw new Error('Queue manager not available');
+    }
+    
+    // Batch process the proxies through check-proxy operation
+    const batchResult = await processItems(proxies, 'proxy', {
+      handler: 'proxy',
+      operation: 'check-proxy',
+      totalDelayHours: 0.083, // 5 minutes (5/60 hours)
+      batchSize: 50, // Process 50 proxies per batch
+      priority: 3,
+      useBatching: true,
+      retries: 1,
+      ttl: 30000, // 30 second timeout per proxy check
+      removeOnComplete: 5,
+      removeOnFail: 3,
+    }, queueManager);
+
+    this.logger.info('Batch proxy validation completed', {
+      totalProxies: proxies.length,
+      jobsCreated: batchResult.jobsCreated,
+      mode: batchResult.mode,
+      batchesCreated: batchResult.batchesCreated,
+      duration: `${batchResult.duration}ms`,
+    });
+
+    return {
+      processed: proxies.length,
+      jobsCreated: batchResult.jobsCreated,
+      batchesCreated: batchResult.batchesCreated,
+      mode: batchResult.mode,
+    };
+  }
+
+  @Operation('check-proxy')
+  async checkProxyOperation(payload: ProxyInfo): Promise<unknown> {
+    // payload is now the raw proxy info object
+    this.logger.debug('Processing proxy check request', {
+      proxy: `${payload.host}:${payload.port}`,
+    });
+    return checkProxy(payload);
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/proxy/shared/config.ts
+++ b/apps/stock/data-ingestion/src/handlers/proxy/shared/config.ts
@ -0,0 +1,140 @@
+/**
+ * Proxy Configuration Constants
+ */
+
+export const PROXY_CONFIG = {
+  CACHE_KEY: 'active',
+  CACHE_STATS_KEY: 'stats',
+  CACHE_TTL: 86400, // 24 hours
+  CHECK_TIMEOUT: 7000,
+  CHECK_IP: '99.246.102.205',
+  CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
+  PROXY_SOURCES: [
+    {
+      id: 'prxchk',
+      url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'casals',
+      url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',
+      protocol: 'http',
+    },
+    {
+      id: 'sunny9577',
+      url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'themiralay',
+      url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'casa-ls',
+      url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',
+      protocol: 'http',
+    },
+    {
+      id: 'databay',
+      url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'speedx',
+      url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'monosans',
+      url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'murong',
+      url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'vakhov-fresh',
+      url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'kangproxy',
+      url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'gfpcom',
+      url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'dpangestuw',
+      url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'gitrecon',
+      url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'vakhov-master',
+      url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'breaking-tech',
+      url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'ercindedeoglu',
+      url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',
+      protocol: 'http',
+    },
+    {
+      id: 'tuanminpay',
+      url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',
+      protocol: 'http',
+    },
+
+    {
+      id: 'r00tee-https',
+      url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',
+      protocol: 'https',
+    },
+    {
+      id: 'ercindedeoglu-https',
+      url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',
+      protocol: 'https',
+    },
+    {
+      id: 'vakhov-fresh-https',
+      url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt',
+      protocol: 'https',
+    },
+    {
+      id: 'databay-https',
+      url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',
+      protocol: 'https',
+    },
+    {
+      id: 'kangproxy-https',
+      url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',
+      protocol: 'https',
+    },
+    {
+      id: 'zloi-user-https',
+      url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',
+      protocol: 'https',
+    },
+    {
+      id: 'gfpcom-https',
+      url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',
+      protocol: 'https',
+    },
+  ],
+};
--- a/apps/stock/data-ingestion/src/handlers/proxy/shared/types.ts
+++ b/apps/stock/data-ingestion/src/handlers/proxy/shared/types.ts
@ -0,0 +1,13 @@
+/**
+ * Proxy Shared Types
+ */
+
+export interface ProxySource {
+  id: string;
+  url: string;
+  protocol: string;
+  working?: number; // Optional, used for stats
+  total?: number; // Optional, used for stats
+  percentWorking?: number; // Optional, used for stats
+  lastChecked?: Date; // Optional, used for stats
+}
--- a/apps/stock/data-ingestion/src/handlers/qm/actions/exchanges.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/actions/exchanges.action.ts
@ -0,0 +1,19 @@
+/**
+ * QM Exchanges Operations - Simple exchange data fetching
+ */
+
+import type { IServiceContainer } from '@stock-bot/handlers';
+
+export async function fetchExchanges(services: IServiceContainer): Promise<any[]> {
+  // Get exchanges from MongoDB
+  const exchanges = await services.mongodb.collection('qm_exchanges').find({}).toArray();
+
+  return exchanges;
+}
+
+export async function getExchangeByCode(services: IServiceContainer, code: string): Promise<any> {
+  // Get specific exchange by code
+  const exchange = await services.mongodb.collection('qm_exchanges').findOne({ code });
+
+  return exchange;
+}
--- a/apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts
@ -0,0 +1,72 @@
+/**
+ * QM Session Actions - Session management and creation
+ */
+
+import { BaseHandler } from '@stock-bot/core/handlers';
+import { QM_SESSION_IDS, SESSION_CONFIG } from '../shared/config';
+import { QMSessionManager } from '../shared/session-manager';
+
+/**
+ * Check existing sessions and queue creation jobs for needed sessions
+ */
+export async function checkSessions(handler: BaseHandler): Promise<{
+  cleaned: number;
+  queued: number;
+  message: string;
+}> {
+  const sessionManager = QMSessionManager.getInstance();
+  const cleanedCount = sessionManager.cleanupFailedSessions();
+  // Check which session IDs need more sessions and queue creation jobs
+  let queuedCount = 0;
+  for (const [sessionType, sessionId] of Object.entries(QM_SESSION_IDS)) {
+    console.log(`Checking session ID: ${sessionId}`);
+    if (sessionManager.needsMoreSessions(sessionId)) {
+      const currentCount = sessionManager.getSessions(sessionId).length;
+      const neededSessions = SESSION_CONFIG.MAX_SESSIONS - currentCount;
+      for (let i = 0; i < neededSessions; i++) {
+        await handler.scheduleOperation('create-session', { sessionId, sessionType });
+        handler.logger.info(`Queued job to create session for ${sessionType}`);
+        queuedCount++;
+      }
+    }
+  }
+
+  return {
+    cleaned: cleanedCount,
+    queued: queuedCount,
+    message: `Session check completed: cleaned ${cleanedCount}, queued ${queuedCount}`,
+  };
+}
+
+/**
+ * Create a single session for a specific session ID
+ */
+export async function createSingleSession(
+  handler: BaseHandler,
+  input: any
+): Promise<{ sessionId: string; status: string; sessionType: string }> {
+  const { sessionId, sessionType } = input || {};
+  const sessionManager = QMSessionManager.getInstance();
+
+  // Get proxy from proxy service
+  const proxyString = handler.proxy.getProxy();
+
+  // const session = {
+  //   proxy: proxyString || 'http://proxy:8080',
+  //   headers: sessionManager.getQmHeaders(),
+  //   successfulCalls: 0,
+  //   failedCalls: 0,
+  //   lastUsed: new Date()
+  // };
+
+  handler.logger.info(`Creating session for ${sessionType}`);
+
+  // Add session to manager
+  // sessionManager.addSession(sessionType, session);
+
+  return {
+    sessionId: sessionType,
+    status: 'created',
+    sessionType,
+  };
+}
--- a/apps/stock/data-ingestion/src/handlers/qm/actions/spider.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/actions/spider.action.ts
@ -0,0 +1,33 @@
+/**
+ * QM Spider Operations - Simple symbol discovery
+ */
+
+import type { IServiceContainer } from '@stock-bot/handlers';
+import type { SymbolSpiderJob } from '../shared/types';
+
+export async function spiderSymbolSearch(
+  services: IServiceContainer,
+  config: SymbolSpiderJob
+): Promise<{ foundSymbols: number; depth: number }> {
+  // Simple spider implementation
+  // TODO: Implement actual API calls to discover symbols
+
+  // For now, just return mock results
+  const foundSymbols = Math.floor(Math.random() * 10) + 1;
+
+  return {
+    foundSymbols,
+    depth: config.depth,
+  };
+}
+
+export async function queueSymbolDiscovery(
+  services: IServiceContainer,
+  searchTerms: string[]
+): Promise<void> {
+  // Queue symbol discovery jobs
+  for (const term of searchTerms) {
+    // TODO: Queue actual discovery jobs
+    await services.cache.set(`discovery:${term}`, { queued: true }, 3600);
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/qm/actions/symbols.action.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/actions/symbols.action.ts
@ -0,0 +1,19 @@
+/**
+ * QM Symbols Operations - Simple symbol fetching
+ */
+
+import type { IServiceContainer } from '@stock-bot/handlers';
+
+export async function searchSymbols(services: IServiceContainer): Promise<any[]> {
+  // Get symbols from MongoDB
+  const symbols = await services.mongodb.collection('qm_symbols').find({}).limit(50).toArray();
+
+  return symbols;
+}
+
+export async function fetchSymbolData(services: IServiceContainer, symbol: string): Promise<any> {
+  // Fetch data for a specific symbol
+  const symbolData = await services.mongodb.collection('qm_symbols').findOne({ symbol });
+
+  return symbolData;
+}
--- a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts
@ -0,0 +1,103 @@
+import { BaseHandler, Handler, type IServiceContainer } from '@stock-bot/handlers';
+
+@Handler('qm')
+export class QMHandler extends BaseHandler {
+  constructor(services: IServiceContainer) {
+    super(services); // Handler name read from @Handler decorator
+  }
+
+  // @Operation('check-sessions')
+  // @QueueSchedule('0 */15 * * *', {
+  //   priority: 7,
+  //   immediately: true,
+  //   description: 'Check and maintain QM sessions'
+  // })
+  // async checkSessions(input: unknown, context: ExecutionContext): Promise<unknown> {
+  //   // Call the session maintenance action
+  //   const { checkSessions } = await import('./actions/session.action');
+  //   return await checkSessions(this);
+  // }
+
+  // @Operation('create-session')
+  // async createSession(input: unknown, context: ExecutionContext): Promise<unknown> {
+  //   // Call the individual session creation action
+  //   const { createSingleSession } = await import('./actions/session.action');
+  //   return await createSingleSession(this, input);
+  // }
+
+  // @Operation('search-symbols')
+  // async searchSymbols(_input: unknown, _context: ExecutionContext): Promise<unknown> {
+  //   this.logger.info('Searching QM symbols with new DI pattern...');
+  //   try {
+  //     // Check existing symbols in MongoDB
+  //     const symbolsCollection = this.mongodb.collection('qm_symbols');
+  //     const symbols = await symbolsCollection.find({}).limit(100).toArray();
+
+  //     this.logger.info('QM symbol search completed', { count: symbols.length });
+
+  //     if (symbols && symbols.length > 0) {
+  //       // Cache result for performance
+  //       await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800);
+
+  //       return {
+  //         success: true,
+  //         message: 'QM symbol search completed successfully',
+  //         count: symbols.length,
+  //         symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
+  //       };
+  //     } else {
+  //       // No symbols found - this is expected initially
+  //       this.logger.info('No QM symbols found in database yet');
+  //       return {
+  //         success: true,
+  //         message: 'No symbols found yet - database is empty',
+  //         count: 0,
+  //       };
+  //     }
+
+  //   } catch (error) {
+  //     this.logger.error('Failed to search QM symbols', { error });
+  //     throw error;
+  //   }
+  // }
+
+  // @Operation('spider-symbol-search')
+  // @QueueSchedule('0 0 * * 0', {
+  //   priority: 10,
+  //   immediately: false,
+  //   description: 'Comprehensive symbol search using QM API'
+  // })
+  // async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise<unknown> {
+  //   // Set default payload for scheduled runs
+  //   const jobPayload: SymbolSpiderJob = payload || {
+  //     prefix: null,
+  //     depth: 1,
+  //     source: 'qm',
+  //     maxDepth: 4
+  //   };
+
+  //   this.logger.info('Starting QM spider symbol search', { payload: jobPayload });
+
+  //   // Store spider job info in cache (temporary data)
+  //   const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`;
+  //   const spiderResult = {
+  //     payload: jobPayload,
+  //     startTime: new Date().toISOString(),
+  //     status: 'started',
+  //     jobId: spiderJobId
+  //   };
+
+  //   // Store in cache with 1 hour TTL (temporary data)
+  //   await this.cache.set(spiderJobId, spiderResult, 3600);
+  //   this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 });
+
+  //   // Schedule follow-up processing if needed
+  //   await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, { delay: 5000 });
+
+  //   return {
+  //     success: true,
+  //     message: 'QM spider search initiated',
+  //     spiderJobId
+  //   };
+  // }
+}
--- a/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts
@ -0,0 +1,38 @@
+/**
+ * Shared configuration for QM operations
+ */
+
+// QM Session IDs for different endpoints
+export const QM_SESSION_IDS = {
+  LOOKUP: 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6', // lookup endpoint
+  // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
+  // cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
+  // '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
+  // '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
+  // a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
+  // a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
+  // b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
+  // '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
+  // '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
+  // '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
+  // d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
+  // d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
+  // '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
+  // Add other session IDs as needed
+} as const;
+
+// QM API Configuration
+export const QM_CONFIG = {
+  BASE_URL: 'https://app.quotemedia.com',
+  AUTH_PATH: '/auth/g/authenticate/dataTool/v0/500',
+  LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json',
+} as const;
+
+// Session management settings
+export const SESSION_CONFIG = {
+  MIN_SESSIONS: 5,
+  MAX_SESSIONS: 10,
+  MAX_FAILED_CALLS: 10,
+  SESSION_TIMEOUT: 10000, // 10 seconds
+  API_TIMEOUT: 15000, // 15 seconds
+} as const;
--- a/apps/stock/data-ingestion/src/handlers/qm/shared/session-manager.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/shared/session-manager.ts
@ -0,0 +1,156 @@
+/**
+ * QM Session Manager - Centralized session state management
+ */
+
+import { getRandomUserAgent } from '@stock-bot/utils';
+import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
+import type { QMSession } from './types';
+
+export class QMSessionManager {
+  private static instance: QMSessionManager | null = null;
+  private sessionCache: Record<string, QMSession[]> = {};
+  private isInitialized = false;
+
+  private constructor() {
+    // Initialize session cache with known session IDs
+    Object.values(QM_SESSION_IDS).forEach(sessionId => {
+      this.sessionCache[sessionId] = [];
+    });
+  }
+
+  static getInstance(): QMSessionManager {
+    if (!QMSessionManager.instance) {
+      QMSessionManager.instance = new QMSessionManager();
+    }
+    return QMSessionManager.instance;
+  }
+
+  /**
+   * Get a random session for the given session ID
+   */
+  getSession(sessionId: string): QMSession | null {
+    const sessions = this.sessionCache[sessionId];
+    if (!sessions || sessions.length === 0) {
+      return null;
+    }
+
+    // Filter out sessions with excessive failures
+    const validSessions = sessions.filter(
+      session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+    );
+    if (validSessions.length === 0) {
+      return null;
+    }
+
+    return validSessions[Math.floor(Math.random() * validSessions.length)];
+  }
+
+  /**
+   * Add a session to the cache
+   */
+  addSession(sessionId: string, session: QMSession): void {
+    if (!this.sessionCache[sessionId]) {
+      this.sessionCache[sessionId] = [];
+    }
+    this.sessionCache[sessionId].push(session);
+  }
+
+  /**
+   * Get all sessions for a session ID
+   */
+  getSessions(sessionId: string): QMSession[] {
+    return this.sessionCache[sessionId] || [];
+  }
+
+  /**
+   * Get session count for all session IDs
+   */
+  getSessionCount(): number {
+    return Object.values(this.sessionCache).reduce((total, sessions) => total + sessions.length, 0);
+  }
+
+  /**
+   * Clean up failed sessions
+   */
+  cleanupFailedSessions(): number {
+    let removedCount = 0;
+
+    Object.keys(this.sessionCache).forEach(sessionId => {
+      const initialCount = this.sessionCache[sessionId].length;
+      this.sessionCache[sessionId] = this.sessionCache[sessionId].filter(
+        session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+      );
+      removedCount += initialCount - this.sessionCache[sessionId].length;
+    });
+
+    return removedCount;
+  }
+
+  getQmHeaders(): Record<string, string> {
+    return {
+      'User-Agent': getRandomUserAgent(),
+      Accept: '*/*',
+      'Accept-Language': 'en',
+      'Sec-Fetch-Mode': 'cors',
+      Origin: 'https://www.quotemedia.com',
+      Referer: 'https://www.quotemedia.com/',
+    };
+  }
+
+  /**
+   * Check if more sessions are needed for a session ID
+   */
+  needsMoreSessions(sessionId: string): boolean {
+    const sessions = this.sessionCache[sessionId] || [];
+    const validSessions = sessions.filter(
+      session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+    );
+    return validSessions.length < SESSION_CONFIG.MIN_SESSIONS;
+  }
+
+  /**
+   * Check if session ID is at capacity
+   */
+  isAtCapacity(sessionId: string): boolean {
+    const sessions = this.sessionCache[sessionId] || [];
+    return sessions.length >= SESSION_CONFIG.MAX_SESSIONS;
+  }
+
+  /**
+   * Get session cache statistics
+   */
+  getStats() {
+    const stats: Record<string, { total: number; valid: number; failed: number }> = {};
+
+    Object.entries(this.sessionCache).forEach(([sessionId, sessions]) => {
+      const validSessions = sessions.filter(
+        session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+      );
+      const failedSessions = sessions.filter(
+        session => session.failedCalls > SESSION_CONFIG.MAX_FAILED_CALLS
+      );
+
+      stats[sessionId] = {
+        total: sessions.length,
+        valid: validSessions.length,
+        failed: failedSessions.length,
+      };
+    });
+
+    return stats;
+  }
+
+  /**
+   * Mark manager as initialized
+   */
+  setInitialized(initialized: boolean = true): void {
+    this.isInitialized = initialized;
+  }
+
+  /**
+   * Check if manager is initialized
+   */
+  getInitialized(): boolean {
+    return this.isInitialized;
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/qm/shared/types.ts
+++ b/apps/stock/data-ingestion/src/handlers/qm/shared/types.ts
@ -0,0 +1,32 @@
+/**
+ * Shared types for QM operations
+ */
+
+export interface QMSession {
+  proxy: string;
+  headers: Record<string, string>;
+  successfulCalls: number;
+  failedCalls: number;
+  lastUsed: Date;
+}
+
+export interface SymbolSpiderJob {
+  prefix: string | null; // null = root job (A-Z)
+  depth: number; // 1=A, 2=AA, 3=AAA, etc.
+  source: string; // 'qm'
+  maxDepth?: number; // optional max depth limit
+}
+
+export interface Exchange {
+  exchange: string;
+  exchangeCode: string;
+  exchangeShortName: string;
+  countryCode: string;
+  source: string;
+}
+
+export interface SpiderResult {
+  success: boolean;
+  symbolsFound: number;
+  jobsCreated: number;
+}
--- a/apps/stock/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts
+++ b/apps/stock/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts
@ -0,0 +1,102 @@
+/**
+ * WebShare Fetch Operations - API integration
+ */
+import { OperationContext } from '@stock-bot/di';
+import type { ProxyInfo } from '@stock-bot/proxy';
+import { WEBSHARE_CONFIG } from '../shared/config';
+
+/**
+ * Fetch proxies from WebShare API and convert to ProxyInfo format
+ */
+export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
+  const ctx = OperationContext.create('webshare', 'fetch-proxies');
+
+  try {
+    // Get configuration from stock config system - ensure it's initialized
+    const { getStockConfig, initializeStockConfig } = await import('@stock-bot/stock-config');
+    
+    // Try to get existing config, or initialize if needed
+    let config;
+    try {
+      config = getStockConfig();
+    } catch (error) {
+      // Config not initialized yet, initialize it
+      config = initializeStockConfig('dataIngestion');
+    }
+
+    const apiKey = config.webshare?.apiKey;
+    const apiUrl = config.webshare?.apiUrl;
+    
+    ctx.logger.debug('WebShare config loaded', {
+      hasConfig: !!config,
+      hasWebshare: !!config.webshare,
+      webshareConfig: config.webshare,
+      apiKeyLength: apiKey?.length || 0,
+      apiUrl: apiUrl,
+      envApiKey: process.env.WEBSHARE_API_KEY ? 'SET' : 'NOT_SET',
+    });
+
+    if (!apiKey || !apiUrl) {
+      ctx.logger.error('Missing WebShare configuration', {
+        hasApiKey: !!apiKey,
+        hasApiUrl: !!apiUrl,
+        apiKeyValue: apiKey ? `${apiKey.substring(0, 5)}...` : 'NOT_SET',
+      });
+      return [];
+    }
+
+    ctx.logger.info('Fetching proxies from WebShare API', { apiUrl });
+
+    const response = await fetch(
+      `${apiUrl}proxy/list/?mode=${WEBSHARE_CONFIG.DEFAULT_MODE}&page=${WEBSHARE_CONFIG.DEFAULT_PAGE}&page_size=${WEBSHARE_CONFIG.DEFAULT_PAGE_SIZE}`,
+      {
+        method: 'GET',
+        headers: {
+          Authorization: `Token ${apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        signal: AbortSignal.timeout(WEBSHARE_CONFIG.TIMEOUT),
+      }
+    );
+
+    if (!response.ok) {
+      ctx.logger.error('WebShare API request failed', {
+        status: response.status,
+        statusText: response.statusText,
+      });
+      return [];
+    }
+
+    const data = await response.json();
+
+    if (!data.results || !Array.isArray(data.results)) {
+      ctx.logger.error('Invalid response format from WebShare API', { data });
+      return [];
+    }
+
+    // Transform proxy data to ProxyInfo format
+    const proxies: ProxyInfo[] = data.results.map(
+      (proxy: { username: string; password: string; proxy_address: string; port: number }) => ({
+        source: 'webshare',
+        protocol: 'http' as const,
+        host: proxy.proxy_address,
+        port: proxy.port,
+        username: proxy.username,
+        password: proxy.password,
+        isWorking: true, // WebShare provides working proxies
+        firstSeen: new Date(),
+        lastChecked: new Date(),
+      })
+    );
+
+    ctx.logger.info('Successfully fetched proxies from WebShare', {
+      count: proxies.length,
+      total: data.count || proxies.length,
+    });
+
+    return proxies;
+  } catch (error) {
+    ctx.logger.error('Failed to fetch proxies from WebShare', { error });
+    return [];
+  }
+}
--- a/apps/stock/data-ingestion/src/handlers/webshare/shared/config.ts
+++ b/apps/stock/data-ingestion/src/handlers/webshare/shared/config.ts
@ -0,0 +1,10 @@
+/**
+ * WebShare Configuration Constants
+ */
+
+export const WEBSHARE_CONFIG = {
+  DEFAULT_PAGE_SIZE: 100,
+  DEFAULT_MODE: 'direct',
+  DEFAULT_PAGE: 1,
+  TIMEOUT: 10000,
+};
--- a/apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts
+++ b/apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts
@ -0,0 +1,73 @@
+import {
+  BaseHandler,
+  Handler,
+  Operation,
+  QueueSchedule,
+  type ExecutionContext,
+  type IServiceContainer,
+} from '@stock-bot/handlers';
+
+@Handler('webshare')
+export class WebShareHandler extends BaseHandler {
+  constructor(services: IServiceContainer) {
+    super(services);
+  }
+
+  @Operation('fetch-proxies')
+  @QueueSchedule('0 */6 * * *', {
+    priority: 3,
+    immediately: true,
+    description: 'Fetch fresh proxies from WebShare API',
+  })
+  async fetchProxies(_input: unknown, _context: ExecutionContext): Promise<unknown> {
+    this.logger.info('Fetching proxies from WebShare API');
+
+    try {
+      const { fetchWebShareProxies } = await import('./operations/fetch.operations');
+      const proxies = await fetchWebShareProxies();
+
+      if (proxies.length > 0) {
+        // Update the centralized proxy manager using the injected service
+        if (!this.proxy) {
+          this.logger.warn('Proxy manager is not initialized, cannot update proxies');
+          return {
+            success: false,
+            proxiesUpdated: 0,
+            error: 'Proxy manager not initialized',
+          };
+        }
+        await this.proxy.updateProxies(proxies);
+
+        this.logger.info('Updated proxy manager with WebShare proxies', {
+          count: proxies.length,
+          workingCount: proxies.filter(p => p.isWorking !== false).length,
+        });
+
+        // Cache proxy stats for monitoring using handler's cache methods
+        await this.cacheSet('proxy-count', proxies.length, 3600);
+        await this.cacheSet(
+          'working-count',
+          proxies.filter(p => p.isWorking !== false).length,
+          3600
+        );
+        await this.cacheSet('last-fetch', new Date().toISOString(), 1800);
+
+        return {
+          success: true,
+          proxiesUpdated: proxies.length,
+          workingProxies: proxies.filter(p => p.isWorking !== false).length,
+        };
+      } else {
+        this.logger.warn('No proxies fetched from WebShare API');
+        return {
+          success: false,
+          proxiesUpdated: 0,
+          error: 'No proxies returned from API',
+        };
+      }
+    } catch (error) {
+      this.logger.error('Failed to fetch and update proxies', { error });
+      throw error;
+    }
+  }
+}
--- a/apps/stock/data-ingestion/src/index.ts
+++ b/apps/stock/data-ingestion/src/index.ts
@ -0,0 +1,77 @@
+/**
+ * Data Ingestion Service
+ * Simplified entry point using ServiceApplication framework
+ */
+
+import { initializeStockConfig } from '@stock-bot/stock-config';
+import {
+  ServiceApplication,
+} from '@stock-bot/di';
+import { getLogger } from '@stock-bot/logger';
+
+// Local imports
+import { initializeAllHandlers } from './handlers';
+import { createRoutes } from './routes/create-routes';
+
+// Initialize configuration with service-specific overrides
+const config = initializeStockConfig('dataIngestion');
+console.log('Data Ingestion Service Configuration:', JSON.stringify(config, null, 2));
+
+// Create service application
+const app = new ServiceApplication(
+  config,
+  {
+    serviceName: 'data-ingestion',
+    enableHandlers: true,
+    enableScheduledJobs: true,
+    corsConfig: {
+      origin: '*',
+      allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'],
+      allowHeaders: ['Content-Type', 'Authorization'],
+      credentials: false,
+    },
+    serviceMetadata: {
+      version: '1.0.0',
+      description: 'Market data ingestion from multiple providers',
+      endpoints: {
+        health: '/health',
+        handlers: '/api/handlers',
+      },
+    },
+  },
+  {
+    // Lifecycle hooks if needed
+    onStarted: (port) => {
+      const logger = getLogger('data-ingestion');
+      logger.info('Data ingestion service startup initiated with ServiceApplication framework');
+    },
+  }
+);
+
+// Container factory function
+async function createContainer(config: any) {
+  const { ServiceContainerBuilder } = await import('@stock-bot/di');
+  
+  const container = await new ServiceContainerBuilder()
+    .withConfig(config)
+    .withOptions({
+      enableQuestDB: false, // Data ingestion doesn't need QuestDB yet
+      enableMongoDB: true,
+      enablePostgres: config.database?.postgres?.enabled ?? false,
+      enableCache: true,
+      enableQueue: true,
+      enableBrowser: true, // Data ingestion needs browser for web scraping
+      enableProxy: true, // Data ingestion needs proxy for rate limiting
+    })
+    .build(); // This automatically initializes services
+    
+  return container;
+}
+
+
+// Start the service
+app.start(createContainer, createRoutes, initializeAllHandlers).catch(error => {
+  const logger = getLogger('data-ingestion');
+  logger.fatal('Failed to start data service', { error });
+  process.exit(1);
+});
--- a/apps/stock/data-ingestion/src/routes/create-routes.ts
+++ b/apps/stock/data-ingestion/src/routes/create-routes.ts
@ -0,0 +1,74 @@
+/**
+ * Routes creation with improved DI pattern
+ */
+
+import { Hono } from 'hono';
+import type { IServiceContainer } from '@stock-bot/handlers';
+import { exchangeRoutes } from './exchange.routes';
+import { healthRoutes } from './health.routes';
+import { createQueueRoutes } from './queue.routes';
+
+/**
+ * Creates all routes with access to type-safe services
+ */
+export function createRoutes(services: IServiceContainer): Hono {
+  const app = new Hono();
+
+  // Mount routes that don't need services
+  app.route('/health', healthRoutes);
+
+  // Mount routes that need services
+  app.route('/api/exchanges', exchangeRoutes);
+  app.route('/api/queue', createQueueRoutes(services));
+
+  // Store services in app context for handlers that need it
+  app.use('*', async (c, next) => {
+    c.set('services', services);
+    await next();
+  });
+
+  // Add a new endpoint to test the improved DI
+  app.get('/api/di-test', async c => {
+    try {
+      const services = c.get('services') as IServiceContainer;
+
+      // Test MongoDB connection
+      const mongoStats = services.mongodb?.getPoolMetrics?.() || {
+        status: services.mongodb ? 'connected' : 'disabled',
+      };
+
+      // Test PostgreSQL connection
+      const pgConnected = services.postgres?.connected || false;
+
+      // Test cache
+      const cacheReady = services.cache?.isReady() || false;
+
+      // Test queue
+      const queueStats = services.queue?.getGlobalStats() || { status: 'disabled' };
+
+      return c.json({
+        success: true,
+        message: 'Improved DI pattern is working!',
+        services: {
+          mongodb: mongoStats,
+          postgres: { connected: pgConnected },
+          cache: { ready: cacheReady },
+          queue: queueStats,
+        },
+        timestamp: new Date().toISOString(),
+      });
+    } catch (error) {
+      const services = c.get('services') as IServiceContainer;
+      services.logger.error('DI test endpoint failed', { error });
+      return c.json(
+        {
+          success: false,
+          error: error instanceof Error ? error.message : String(error),
+        },
+        500
+      );
+    }
+  });
+
+  return app;
+}
--- a/apps/stock/data-ingestion/src/routes/exchange.routes.ts
+++ b/apps/stock/data-ingestion/src/routes/exchange.routes.ts
@ -0,0 +1,22 @@
+import { Hono } from 'hono';
+import { getLogger } from '@stock-bot/logger';
+
+const logger = getLogger('exchange-routes');
+const exchange = new Hono();
+
+// Get all exchanges
+exchange.get('/', async c => {
+  try {
+    // TODO: Implement exchange listing from database
+    return c.json({
+      status: 'success',
+      data: [],
+      message: 'Exchange endpoints will be implemented with database integration',
+    });
+  } catch (error) {
+    logger.error('Failed to get exchanges', { error });
+    return c.json({ status: 'error', message: 'Failed to get exchanges' }, 500);
+  }
+});
+
+export { exchange as exchangeRoutes };
--- a/apps/stock/data-ingestion/src/routes/health.routes.ts
+++ b/apps/stock/data-ingestion/src/routes/health.routes.ts
@ -0,0 +1,14 @@
+import { Hono } from 'hono';
+
+const health = new Hono();
+
+// Health check endpoint
+health.get('/', c => {
+  return c.json({
+    status: 'healthy',
+    service: 'data-ingestion',
+    timestamp: new Date().toISOString(),
+  });
+});
+
+export { health as healthRoutes };
--- a/apps/stock/data-ingestion/src/routes/index.ts
+++ b/apps/stock/data-ingestion/src/routes/index.ts
@ -0,0 +1,6 @@
+/**
+ * Routes index - exports all route modules
+ */
+export { exchangeRoutes } from './exchange.routes';
+export { healthRoutes } from './health.routes';
+export { queueRoutes } from './queue.routes';
--- a/apps/stock/data-ingestion/src/routes/market-data.routes.ts
+++ b/apps/stock/data-ingestion/src/routes/market-data.routes.ts
@ -0,0 +1,142 @@
+/**
+ * Market data routes
+ */
+import { Hono } from 'hono';
+import { getLogger } from '@stock-bot/logger';
+import { processItems } from '@stock-bot/queue';
+import type { IServiceContainer } from '@stock-bot/handlers';
+
+const logger = getLogger('market-data-routes');
+
+export function createMarketDataRoutes(container: IServiceContainer) {
+  const marketDataRoutes = new Hono();
+
+  // Market data endpoints
+  marketDataRoutes.get('/api/live/:symbol', async c => {
+    const symbol = c.req.param('symbol');
+    logger.info('Live data request', { symbol });
+
+    try {
+      // Queue job for live data using Yahoo provider
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const queue = queueManager.getQueue('yahoo-finance');
+      const job = await queue.add('live-data', {
+        handler: 'yahoo-finance',
+        operation: 'live-data',
+        payload: { symbol },
+      });
+      return c.json({
+        status: 'success',
+        message: 'Live data job queued',
+        jobId: job.id,
+        symbol,
+      });
+    } catch (error) {
+      logger.error('Failed to queue live data job', { symbol, error });
+      return c.json({ status: 'error', message: 'Failed to queue live data job' }, 500);
+    }
+  });
+
+  marketDataRoutes.get('/api/historical/:symbol', async c => {
+    const symbol = c.req.param('symbol');
+    const from = c.req.query('from');
+    const to = c.req.query('to');
+
+    logger.info('Historical data request', { symbol, from, to });
+
+    try {
+      const fromDate = from ? new Date(from) : new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days ago
+      const toDate = to ? new Date(to) : new Date(); // Now
+
+      // Queue job for historical data using Yahoo provider
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const queue = queueManager.getQueue('yahoo-finance');
+      const job = await queue.add('historical-data', {
+        handler: 'yahoo-finance',
+        operation: 'historical-data',
+        payload: {
+          symbol,
+          from: fromDate.toISOString(),
+          to: toDate.toISOString(),
+        },
+      });
+
+      return c.json({
+        status: 'success',
+        message: 'Historical data job queued',
+        jobId: job.id,
+        symbol,
+        from: fromDate,
+        to: toDate,
+      });
+    } catch (error) {
+      logger.error('Failed to queue historical data job', { symbol, from, to, error });
+      return c.json({ status: 'error', message: 'Failed to queue historical data job' }, 500);
+    }
+  });
+
+  // Batch processing endpoint using new queue system
+  marketDataRoutes.post('/api/process-symbols', async c => {
+    try {
+      const {
+        symbols,
+        provider = 'ib',
+        operation = 'fetch-session',
+        useBatching = true,
+        totalDelayHours = 0.0083, // ~30 seconds (30/3600 hours)
+        batchSize = 10,
+      } = await c.req.json();
+
+      if (!symbols || !Array.isArray(symbols) || symbols.length === 0) {
+        return c.json({ status: 'error', message: 'Invalid symbols array' }, 400);
+      }
+
+      logger.info('Batch processing symbols', {
+        count: symbols.length,
+        provider,
+        operation,
+        useBatching,
+      });
+
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const result = await processItems(symbols, provider, {
+        handler: provider,
+        operation,
+        totalDelayHours,
+        useBatching,
+        batchSize,
+        priority: 2,
+        retries: 2,
+        removeOnComplete: 5,
+        removeOnFail: 10,
+      }, queueManager);
+
+      return c.json({
+        status: 'success',
+        message: 'Batch processing initiated',
+        result,
+        symbols: symbols.length,
+      });
+    } catch (error) {
+      logger.error('Failed to process symbols batch', { error });
+      return c.json({ status: 'error', message: 'Failed to process symbols batch' }, 500);
+    }
+  });
+
+  return marketDataRoutes;
+}
+
+// Legacy export for backward compatibility
+export const marketDataRoutes = createMarketDataRoutes({} as IServiceContainer);
--- a/apps/stock/data-ingestion/src/routes/queue.routes.ts
+++ b/apps/stock/data-ingestion/src/routes/queue.routes.ts
@ -0,0 +1,35 @@
+import { Hono } from 'hono';
+import { getLogger } from '@stock-bot/logger';
+import type { IServiceContainer } from '@stock-bot/handlers';
+
+const logger = getLogger('queue-routes');
+
+export function createQueueRoutes(container: IServiceContainer) {
+  const queue = new Hono();
+
+  // Queue status endpoint
+  queue.get('/status', async c => {
+    try {
+      const queueManager = container.queue;
+      if (!queueManager) {
+        return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
+      }
+      
+      const globalStats = await queueManager.getGlobalStats();
+
+      return c.json({
+        status: 'success',
+        data: globalStats,
+        message: 'Queue status retrieved successfully',
+      });
+    } catch (error) {
+      logger.error('Failed to get queue status', { error });
+      return c.json({ status: 'error', message: 'Failed to get queue status' }, 500);
+    }
+  });
+
+  return queue;
+}
+
+// Legacy export for backward compatibility
+export const queueRoutes = createQueueRoutes({} as IServiceContainer);
--- a/apps/stock/data-ingestion/src/types/exchange.types.ts
+++ b/apps/stock/data-ingestion/src/types/exchange.types.ts
@ -0,0 +1,40 @@
+/**
+ * Type definitions for exchange data structures
+ */
+
+export interface IBExchange {
+  id: string;
+  country_code: string;
+  name: string;
+  code?: string;
+  exchange_code?: string;
+  currency?: string;
+  timezone?: string;
+  _id?: unknown; // MongoDB ObjectId
+  // Add other properties as needed
+}
+
+export interface MasterExchangeData {
+  id: string;
+  code: string;
+  name: string;
+  country: string;
+  currency: string;
+  // Add other properties as needed
+}
+
+export interface QMSymbol {
+  symbol: string;
+  exchange?: string;
+  name?: string;
+  type?: string;
+  // Add other properties as needed
+}
+
+export interface IBSymbol {
+  symbol: string;
+  exchange: string;
+  name?: string;
+  currency?: string;
+  // Add other properties as needed
+}
--- a/apps/stock/data-ingestion/src/types/job-payloads.ts
+++ b/apps/stock/data-ingestion/src/types/job-payloads.ts
@ -0,0 +1,93 @@
+/**
+ * Type definitions for all job payloads across data service providers
+ */
+
+// Common result types
+export interface JobResult {
+  success: boolean;
+  message: string;
+}
+
+export interface CountableJobResult extends JobResult {
+  count: number;
+}
+
+// QM Provider Types
+export interface SymbolSpiderJob {
+  prefix: string | null; // null = root job (A-Z)
+  depth: number; // 1=A, 2=AA, 3=AAA, etc.
+  source: string; // 'qm'
+  maxDepth?: number; // optional max depth limit
+}
+
+export interface CreateSessionsResult extends JobResult {
+  // No additional fields needed
+}
+
+export interface SearchSymbolsResult extends CountableJobResult {
+  symbols?: unknown[]; // First 10 symbols as sample
+}
+
+export interface SpiderSymbolSearchResult extends JobResult {
+  symbolsFound: number;
+  newSymbolsAdded: number;
+  duplicatesSkipped: number;
+  errors: number;
+  depth: number;
+  prefix: string | null;
+}
+
+// IB Provider Types
+export interface FetchSessionResult extends JobResult {
+  sessionData?: Record<string, string>;
+}
+
+export interface FetchExchangesResult extends CountableJobResult {
+  exchanges?: unknown[];
+}
+
+export interface FetchSymbolsResult extends CountableJobResult {
+  symbols?: unknown[];
+}
+
+export interface IBExchangesAndSymbolsResult extends JobResult {
+  exchanges: FetchExchangesResult;
+  symbols: FetchSymbolsResult;
+}
+
+// Proxy Provider Types
+export interface ProxyInfo {
+  host: string;
+  port: number;
+  protocol: 'http' | 'https' | 'socks4' | 'socks5';
+  username?: string;
+  password?: string;
+  country?: string;
+  city?: string;
+  isValid?: boolean;
+  lastChecked?: Date;
+}
+
+export interface FetchProxiesFromSourcesResult extends CountableJobResult {
+  proxies?: ProxyInfo[];
+  sources: string[];
+}
+
+export interface CheckProxyResult extends JobResult {
+  proxy: ProxyInfo;
+  responseTime?: number;
+  error?: string;
+}
+
+// WebShare Provider Types
+export interface FetchWebShareProxiesResult extends CountableJobResult {
+  proxies?: ProxyInfo[];
+  activeProxies: number;
+  totalQuota: number;
+  remainingQuota: number;
+}
+
+// No payload job types (for operations that don't need input)
+export interface NoPayload {
+  // Empty interface for operations that don't need payload
+}
--- a/apps/stock/data-ingestion/src/utils/symbol-search.util.ts
+++ b/apps/stock/data-ingestion/src/utils/symbol-search.util.ts
@ -0,0 +1,109 @@
+import { sleep } from '@stock-bot/di';
+import { getLogger } from '@stock-bot/logger';
+
+const logger = getLogger('symbol-search-util');
+
+export interface SearchFunction {
+  (query: string): Promise<string[]>;
+}
+
+export class SymbolSearchUtil {
+  private threshold: number;
+  private searchFunction: SearchFunction;
+  private maxDepth: number;
+  private delay: number;
+
+  constructor(
+    searchFunction: SearchFunction,
+    threshold: number = 50,
+    maxDepth: number = 4,
+    delay: number = 100
+  ) {
+    this.searchFunction = searchFunction;
+    this.threshold = threshold;
+    this.maxDepth = maxDepth;
+    this.delay = delay;
+  }
+
+  async searchAllSymbols(): Promise<string[]> {
+    logger.info('Starting comprehensive symbol search...');
+    const allSymbols: string[] = [];
+
+    // Start with single letters A-Z
+    for (let i = 0; i < 26; i++) {
+      const singleLetter = String.fromCharCode(65 + i);
+
+      try {
+        const symbols = await this.searchRecursive(singleLetter, 1);
+        allSymbols.push(...symbols);
+
+        // Add delay between top-level searches
+        if (this.delay > 0) {
+          await sleep(this.delay);
+        }
+      } catch (error) {
+        logger.error(`Failed to search for "${singleLetter}":`, error);
+        // Continue with next letter
+      }
+    }
+
+    // Remove duplicates
+    const uniqueSymbols = [...new Set(allSymbols)];
+    logger.info(`Symbol search completed. Found ${uniqueSymbols.length} unique symbols`);
+    return uniqueSymbols;
+  }
+
+  private async searchRecursive(prefix: string, depth: number): Promise<string[]> {
+    try {
+      const symbols = await this.searchFunction(prefix);
+
+      logger.debug(`Query "${prefix}" returned ${symbols.length} symbols`);
+
+      // If we're at max depth or results are under threshold, return the symbols
+      if (depth >= this.maxDepth || symbols.length < this.threshold) {
+        logger.info(`Added ${symbols.length} symbols from query: ${prefix}`);
+        return symbols;
+      }
+
+      // If we have too many results, go deeper
+      logger.info(
+        `Query "${prefix}" returned ${symbols.length} results (>= ${this.threshold}), going deeper...`
+      );
+      const allSymbols: string[] = [];
+
+      for (let i = 0; i < 26; i++) {
+        const nextQuery = prefix + String.fromCharCode(65 + i);
+
+        try {
+          const deeperSymbols = await this.searchRecursive(nextQuery, depth + 1);
+          allSymbols.push(...deeperSymbols);
+
+          // Add delay between recursive calls
+          if (this.delay > 0 && depth < 3) {
+            // Only delay for first few levels
+            await sleep(this.delay);
+          }
+        } catch (error) {
+          logger.error(`Failed recursive search for "${nextQuery}":`, error);
+          // Continue with next combination
+        }
+      }
+
+      return allSymbols;
+    } catch (error) {
+      logger.error(`Error in recursive search for "${prefix}":`, error);
+      return [];
+    }
+  }
+
+  // Static method for one-off searches
+  static async search(
+    searchFunction: SearchFunction,
+    threshold: number = 50,
+    maxDepth: number = 4,
+    delay: number = 100
+  ): Promise<string[]> {
+    const util = new SymbolSearchUtil(searchFunction, threshold, maxDepth, delay);
+    return util.searchAllSymbols();
+  }
+}
--- a/apps/stock/data-ingestion/test-ceo-operations.ts
+++ b/apps/stock/data-ingestion/test-ceo-operations.ts
@ -0,0 +1,103 @@
+#!/usr/bin/env bun
+
+/**
+ * Test script for CEO handler operations
+ */
+import { initializeServiceConfig } from '@stock-bot/config';
+import { createServiceContainer, initializeServices } from '@stock-bot/di';
+import { getLogger } from '@stock-bot/logger';
+
+const logger = getLogger('test-ceo-operations');
+
+async function testCeoOperations() {
+  logger.info('Testing CEO handler operations...');
+
+  try {
+    // Initialize config
+    const config = initializeServiceConfig();
+
+    // Create Awilix container
+    const awilixConfig = {
+      redis: {
+        host: config.database.dragonfly.host,
+        port: config.database.dragonfly.port,
+        db: config.database.dragonfly.db,
+      },
+      mongodb: {
+        uri: config.database.mongodb.uri,
+        database: config.database.mongodb.database,
+      },
+      postgres: {
+        host: config.database.postgres.host,
+        port: config.database.postgres.port,
+        database: config.database.postgres.database,
+        user: config.database.postgres.user,
+        password: config.database.postgres.password,
+      },
+      questdb: {
+        enabled: false,
+        host: config.database.questdb.host,
+        httpPort: config.database.questdb.httpPort,
+        pgPort: config.database.questdb.pgPort,
+        influxPort: config.database.questdb.ilpPort,
+        database: config.database.questdb.database,
+      },
+    };
+
+    const container = createServiceContainer(awilixConfig);
+    await initializeServices(container);
+
+    const serviceContainer = container.resolve('serviceContainer');
+
+    // Import and create CEO handler
+    const { CeoHandler } = await import('./src/handlers/ceo/ceo.handler');
+    const ceoHandler = new CeoHandler(serviceContainer);
+
+    // Test 1: Check if there are any CEO symbols in the database
+    logger.info('Checking for existing CEO symbols...');
+    const collection = serviceContainer.mongodb.collection('ceoSymbols');
+    const count = await collection.countDocuments();
+    logger.info(`Found ${count} CEO symbols in database`);
+
+    if (count > 0) {
+      // Test 2: Run process-unique-symbols operation
+      logger.info('Testing process-unique-symbols operation...');
+      const result = await ceoHandler.updateUniqueSymbols(undefined, {});
+      logger.info('Process unique symbols result:', result);
+
+      // Test 3: Test individual symbol processing
+      logger.info('Testing process-individual-symbol operation...');
+      const sampleSymbol = await collection.findOne({});
+      if (sampleSymbol) {
+        const individualResult = await ceoHandler.processIndividualSymbol(
+          {
+            ceoId: sampleSymbol.ceoId,
+            symbol: sampleSymbol.symbol,
+            exchange: sampleSymbol.exchange,
+            name: sampleSymbol.name,
+          },
+          {}
+        );
+        logger.info('Process individual symbol result:', individualResult);
+      }
+    } else {
+      logger.warn('No CEO symbols found. Run the service to populate data first.');
+    }
+
+    // Clean up
+    await serviceContainer.mongodb.disconnect();
+    await serviceContainer.postgres.disconnect();
+    if (serviceContainer.cache) {
+      await serviceContainer.cache.disconnect();
+    }
+
+    logger.info('Test completed successfully!');
+    process.exit(0);
+  } catch (error) {
+    logger.error('Test failed:', error);
+    process.exit(1);
+  }
+}
+
+// Run the test
+testCeoOperations();
--- a/apps/stock/data-ingestion/tsconfig.json
+++ b/apps/stock/data-ingestion/tsconfig.json
@ -0,0 +1,18 @@
+{
+  "extends": "../../tsconfig.app.json",
+  "references": [
+    { "path": "../../libs/core/types" },
+    { "path": "../../libs/core/config" },
+    { "path": "../../libs/core/logger" },
+    { "path": "../../libs/core/di" },
+    { "path": "../../libs/core/handlers" },
+    { "path": "../../libs/data/cache" },
+    { "path": "../../libs/data/mongodb" },
+    { "path": "../../libs/data/postgres" },
+    { "path": "../../libs/data/questdb" },
+    { "path": "../../libs/services/queue" },
+    { "path": "../../libs/services/shutdown" },
+    { "path": "../../libs/utils" },
+    { "path": "../config" }
+  ]
+}