work on ib and cleanup

apps/data-service/src/providers/ib.tasks.ts

@@ -1,12 +1,13 @@
 import { Browser } from '@stock-bot/browser';
 import { getLogger } from '@stock-bot/logger';
+import { getMongoDBClient } from '@stock-bot/mongodb-client';
 
 // Shared instances (module-scoped, not global)
 let isInitialized = false; // Track if resources are initialized
 let logger: ReturnType<typeof getLogger>;
 // let cache: CacheProvider;
 
-export async function initializeIBResources(waitForCache = false): Promise<void> {
+export async function initializeIBResources(): Promise<void> {
   // Skip if already initialized
   if (isInitialized) {
     return;
@@ -93,7 +94,7 @@ export async function fetchSession(): Promise<Record<string, string> | undefined
     // Wait for and return headers immediately when captured
     logger.info('⏳ Waiting for headers to be captured...');
     const headers = await headersPromise;
-
+    page.close();
     if (headers) {
       logger.info('✅ Headers captured successfully');
     } else {
@@ -151,19 +152,156 @@ export async function fetchExchanges(sessionHeaders: Record<string, string>): Pr
     }
 
     const data = await response.json();
+    const exchanges = data?.exchanges || [];
+    logger.info('✅ Exchange data fetched successfully');
 
-    logger.info('✅ Exchange data fetched successfully', {
-      dataKeys: Object.keys(data || {}),
-      dataSize: JSON.stringify(data).length,
+    logger.info('Saving IB exchanges to MongoDB...');
+    const client = getMongoDBClient();
+    await client.batchUpsert('ib_exchanges', exchanges, ['id', 'country_code']);
+    logger.info('✅ Exchange IB data saved to MongoDB:', {
+      count: exchanges.length,
     });
 
-    return data;
+    return exchanges;
   } catch (error) {
     logger.error('❌ Failed to fetch exchanges', { error });
     return null;
   }
 }
+
+// Fetch symbols from IB using the session headers
+export async function fetchSymbols(sessionHeaders: Record<string, string>): Promise<any> {
+  try {
+    logger.info('🔍 Fetching symbols with session headers...');
+    // Configure the proxy
+    const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80';
+    // Prepare headers - include all session headers plus any additional ones
+    const requestHeaders = {
+      ...sessionHeaders,
+      Accept: 'application/json, text/plain, */*',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Cache-Control': 'no-cache',
+      Pragma: 'no-cache',
+      'Sec-Fetch-Dest': 'empty',
+      'Sec-Fetch-Mode': 'cors',
+      'Sec-Fetch-Site': 'same-origin',
+      'X-Requested-With': 'XMLHttpRequest',
+    };
+
+    const requestBody = {
+      domain: 'com',
+      newProduct: 'all',
+      pageNumber: 1,
+      pageSize: 100,
+      productCountry: ['CA', 'US'],
+      productSymbol: '',
+      productType: ['STK'],
+      sortDirection: 'asc',
+      sortField: 'symbol',
+    };
+
+    // Get Summary
+    const summaryResponse = await fetch(
+      'https://www.interactivebrokers.com/webrest/search/product-types/summary',
+      {
+        method: 'POST',
+        headers: requestHeaders,
+        proxy: proxyUrl,
+        body: JSON.stringify(requestBody),
+      }
+    );
+
+    if (!summaryResponse.ok) {
+      logger.error('❌ Summary API request failed', {
+        status: summaryResponse.status,
+        statusText: summaryResponse.statusText,
+      });
+      return null;
+    }
+
+    const summaryData = await summaryResponse.json();
+    logger.info('✅ IB Summary data fetched successfully', {
+      totalCount: summaryData[0].totalCount,
+    });
+
+    const symbols = [];
+    requestBody.pageSize = 500;
+    const pageCount = Math.ceil(summaryData[0].totalCount / 500) || 0;
+    logger.info('Fetching Symbols for IB', { pageCount });
+    const symbolPromises = [];
+    for (let page = 1; page <= pageCount; page++) {
+      requestBody.pageNumber = page;
+
+      // Fetch symbols for the current page
+      const symbolsResponse = fetch(
+        'https://www.interactivebrokers.com/webrest/search/products-by-filters',
+        {
+          method: 'POST',
+          headers: requestHeaders,
+          proxy: proxyUrl,
+          body: JSON.stringify(requestBody),
+        }
+      );
+      symbolPromises.push(symbolsResponse);
+    }
+    const responses = await Promise.all(symbolPromises);
+    for (const response of responses) {
+      if (!response.ok) {
+        logger.error('❌ Symbols API request failed', {
+          status: response.status,
+          statusText: response.statusText,
+        });
+        return null;
+      }
+      const data = await response.json();
+      const symJson = data?.products || [];
+      if (symJson && symJson.length > 0) {
+        symbols.push(...symJson);
+      } else {
+        logger.warn('⚠️ No symbols found in response');
+        continue;
+      }
+    }
+    if (symbols.length === 0) {
+      logger.warn('⚠️ No symbols fetched from IB');
+      return null;
+    }
+
+    logger.info('✅ IB symbols fetched successfully, saving to DB...', {
+      totalSymbols: symbols.length,
+    });
+    const client = getMongoDBClient();
+    await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
+    logger.info('Saved IB symbols to DB', {
+      totalSymbols: symbols.length,
+    });
+    // logger.info('📤 Making request to exchange API...', {
+    //   url: exchangeUrl,
+    //   headerCount: Object.keys(requestHeaders).length,
+    // });
+
+    // // Use fetch with proxy configuration
+    // const response = await fetch(exchangeUrl, {
+    //   method: 'GET',
+    //   headers: requestHeaders,
+    //   proxy: proxyUrl,
+    // });
+
+    // if (!response.ok) {
+    //   logger.error('❌ Exchange API request failed', {
+    //     status: response.status,
+    //     statusText: response.statusText,
+    //   });
+    //   return null;
+    // }
+  } catch (error) {
+    logger.error('❌ Failed to fetch symbols', { error });
+    return null;
+  }
+}
+
 export const ibTasks = {
+  fetchSymbols,
   fetchSession,
   fetchExchanges,
 };