work on ib and cleanup

apps/data-service/src/index.ts

@@ -5,6 +5,7 @@ import { Hono } from 'hono';
 import { Browser } from '@stock-bot/browser';
 import { loadEnvVariables } from '@stock-bot/config';
 import { getLogger, shutdownLoggers } from '@stock-bot/logger';
+import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client';
 import { Shutdown } from '@stock-bot/shutdown';
 import { initializeIBResources } from './providers/ib.tasks';
 import { initializeProxyResources } from './providers/proxy.tasks';
@@ -18,7 +19,7 @@ loadEnvVariables();
 const app = new Hono();
 const logger = getLogger('data-service');
 const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002');
-let server: any = null;
+let server: ReturnType<typeof Bun.serve> | null = null;
 
 // Initialize shutdown manager with 15 second timeout
 const shutdown = Shutdown.getInstance({ timeout: 15000 });
@@ -35,6 +36,11 @@ async function initializeServices() {
   logger.info('Initializing data service...');
 
   try {
+    // Initialize MongoDB client first
+    logger.info('Starting MongoDB client initialization...');
+    await connectMongoDB();
+    logger.info('MongoDB client initialized');
+
     // Initialize browser resources
     logger.info('Starting browser resources initialization...');
     await Browser.initialize();
@@ -122,6 +128,18 @@ shutdown.onShutdown(async () => {
   }
 });
 
+// Add MongoDB shutdown handler
+shutdown.onShutdown(async () => {
+  logger.info('Shutting down MongoDB client...');
+  try {
+    await disconnectMongoDB();
+    logger.info('MongoDB client shut down successfully');
+  } catch (error) {
+    logger.error('Error shutting down MongoDB client', { error });
+    // Don't throw here to allow other shutdown handlers to complete
+  }
+});
+
 // Add logger shutdown handler (should be last)
 shutdown.onShutdown(async () => {
   try {

apps/data-service/src/providers/ib.provider.ts

@@ -6,31 +6,36 @@ const logger = getLogger('ib-provider');
 export const ibProvider: ProviderConfig = {
   name: 'ib',
   operations: {
-    'ib-basics': async () => {
+    'ib-exchanges-and-symbols': async () => {
       const { ibTasks } = await import('./ib.tasks');
       logger.info('Fetching symbol summary from IB');
       const sessionHeaders = await ibTasks.fetchSession();
-      logger.info('Fetched symbol summary from IB', {
-        sessionHeaders,
-      });
+      logger.info('Fetched symbol summary from IB');
 
-      // Get Exchanges
-      logger.info('Fetching exchanges from IB');
-      const exchanges = await ibTasks.fetchExchanges(sessionHeaders);
-      logger.info('Fetched exchanges from IB', { exchanges });
-      // return total;
+      if (sessionHeaders) {
+        logger.info('Fetching exchanges from IB');
+        const exchanges = await ibTasks.fetchExchanges(sessionHeaders);
+        logger.info('Fetched exchanges from IB', { count: exchanges.lenght });
+
+        // do the same as above but for symbols
+        logger.info('Fetching symbols from IB');
+        const symbols = await ibTasks.fetchSymbols(sessionHeaders);
+        logger.info('Fetched symbols from IB', { symbols });
+
+        return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length };
+      }
     },
   },
 
   scheduledJobs: [
     {
-      type: 'ib-basics',
-      operation: 'ib-basics',
+      type: 'ib-exchanges-and-symbols',
+      operation: 'ib-exchanges-and-symbols',
       payload: {},
       // should remove and just run at the same time so app restarts dont keeping adding same jobs
-      cronPattern: '*/2 * * * *',
+      cronPattern: '0 0 * * 0',
       priority: 5,
-      immediately: true, // Don't run immediately during startup to avoid conflicts
+      // immediately: true, // Don't run immediately during startup to avoid conflicts
       description: 'Fetch and validate proxy list from sources',
     },
   ],

apps/data-service/src/providers/ib.tasks.ts

@@ -1,12 +1,13 @@
 import { Browser } from '@stock-bot/browser';
 import { getLogger } from '@stock-bot/logger';
+import { getMongoDBClient } from '@stock-bot/mongodb-client';
 
 // Shared instances (module-scoped, not global)
 let isInitialized = false; // Track if resources are initialized
 let logger: ReturnType<typeof getLogger>;
 // let cache: CacheProvider;
 
-export async function initializeIBResources(waitForCache = false): Promise<void> {
+export async function initializeIBResources(): Promise<void> {
   // Skip if already initialized
   if (isInitialized) {
     return;
@@ -93,7 +94,7 @@ export async function fetchSession(): Promise<Record<string, string> | undefined
     // Wait for and return headers immediately when captured
     logger.info('⏳ Waiting for headers to be captured...');
     const headers = await headersPromise;
-
+    page.close();
     if (headers) {
       logger.info('✅ Headers captured successfully');
     } else {
@@ -151,19 +152,156 @@ export async function fetchExchanges(sessionHeaders: Record<string, string>): Pr
     }
 
     const data = await response.json();
+    const exchanges = data?.exchanges || [];
+    logger.info('✅ Exchange data fetched successfully');
 
-    logger.info('✅ Exchange data fetched successfully', {
-      dataKeys: Object.keys(data || {}),
-      dataSize: JSON.stringify(data).length,
+    logger.info('Saving IB exchanges to MongoDB...');
+    const client = getMongoDBClient();
+    await client.batchUpsert('ib_exchanges', exchanges, ['id', 'country_code']);
+    logger.info('✅ Exchange IB data saved to MongoDB:', {
+      count: exchanges.length,
     });
 
-    return data;
+    return exchanges;
   } catch (error) {
     logger.error('❌ Failed to fetch exchanges', { error });
     return null;
   }
 }
+
+// Fetch symbols from IB using the session headers
+export async function fetchSymbols(sessionHeaders: Record<string, string>): Promise<any> {
+  try {
+    logger.info('🔍 Fetching symbols with session headers...');
+    // Configure the proxy
+    const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80';
+    // Prepare headers - include all session headers plus any additional ones
+    const requestHeaders = {
+      ...sessionHeaders,
+      Accept: 'application/json, text/plain, */*',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Cache-Control': 'no-cache',
+      Pragma: 'no-cache',
+      'Sec-Fetch-Dest': 'empty',
+      'Sec-Fetch-Mode': 'cors',
+      'Sec-Fetch-Site': 'same-origin',
+      'X-Requested-With': 'XMLHttpRequest',
+    };
+
+    const requestBody = {
+      domain: 'com',
+      newProduct: 'all',
+      pageNumber: 1,
+      pageSize: 100,
+      productCountry: ['CA', 'US'],
+      productSymbol: '',
+      productType: ['STK'],
+      sortDirection: 'asc',
+      sortField: 'symbol',
+    };
+
+    // Get Summary
+    const summaryResponse = await fetch(
+      'https://www.interactivebrokers.com/webrest/search/product-types/summary',
+      {
+        method: 'POST',
+        headers: requestHeaders,
+        proxy: proxyUrl,
+        body: JSON.stringify(requestBody),
+      }
+    );
+
+    if (!summaryResponse.ok) {
+      logger.error('❌ Summary API request failed', {
+        status: summaryResponse.status,
+        statusText: summaryResponse.statusText,
+      });
+      return null;
+    }
+
+    const summaryData = await summaryResponse.json();
+    logger.info('✅ IB Summary data fetched successfully', {
+      totalCount: summaryData[0].totalCount,
+    });
+
+    const symbols = [];
+    requestBody.pageSize = 500;
+    const pageCount = Math.ceil(summaryData[0].totalCount / 500) || 0;
+    logger.info('Fetching Symbols for IB', { pageCount });
+    const symbolPromises = [];
+    for (let page = 1; page <= pageCount; page++) {
+      requestBody.pageNumber = page;
+
+      // Fetch symbols for the current page
+      const symbolsResponse = fetch(
+        'https://www.interactivebrokers.com/webrest/search/products-by-filters',
+        {
+          method: 'POST',
+          headers: requestHeaders,
+          proxy: proxyUrl,
+          body: JSON.stringify(requestBody),
+        }
+      );
+      symbolPromises.push(symbolsResponse);
+    }
+    const responses = await Promise.all(symbolPromises);
+    for (const response of responses) {
+      if (!response.ok) {
+        logger.error('❌ Symbols API request failed', {
+          status: response.status,
+          statusText: response.statusText,
+        });
+        return null;
+      }
+      const data = await response.json();
+      const symJson = data?.products || [];
+      if (symJson && symJson.length > 0) {
+        symbols.push(...symJson);
+      } else {
+        logger.warn('⚠️ No symbols found in response');
+        continue;
+      }
+    }
+    if (symbols.length === 0) {
+      logger.warn('⚠️ No symbols fetched from IB');
+      return null;
+    }
+
+    logger.info('✅ IB symbols fetched successfully, saving to DB...', {
+      totalSymbols: symbols.length,
+    });
+    const client = getMongoDBClient();
+    await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
+    logger.info('Saved IB symbols to DB', {
+      totalSymbols: symbols.length,
+    });
+    // logger.info('📤 Making request to exchange API...', {
+    //   url: exchangeUrl,
+    //   headerCount: Object.keys(requestHeaders).length,
+    // });
+
+    // // Use fetch with proxy configuration
+    // const response = await fetch(exchangeUrl, {
+    //   method: 'GET',
+    //   headers: requestHeaders,
+    //   proxy: proxyUrl,
+    // });
+
+    // if (!response.ok) {
+    //   logger.error('❌ Exchange API request failed', {
+    //     status: response.status,
+    //     statusText: response.statusText,
+    //   });
+    //   return null;
+    // }
+  } catch (error) {
+    logger.error('❌ Failed to fetch symbols', { error });
+    return null;
+  }
+}
+
 export const ibTasks = {
+  fetchSymbols,
   fetchSession,
   fetchExchanges,
 };