initial symbols done, not liking the outcome, gonna switch to queue based approach

apps/data-service/src/providers/ib.tasks.ts

@@ -157,7 +157,7 @@ export async function fetchExchanges(sessionHeaders: Record<string, string>): Pr
 
     logger.info('Saving IB exchanges to MongoDB...');
     const client = getMongoDBClient();
-    await client.batchUpsert('ib_exchanges', exchanges, ['id', 'country_code']);
+    await client.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']);
     logger.info('✅ Exchange IB data saved to MongoDB:', {
       count: exchanges.length,
     });

apps/data-service/src/providers/qm.provider.ts

@@ -55,9 +55,10 @@ export function initializeQMProvider() {
         type: 'search-symbols',
         operation: 'search-symbols',
         payload: {},
-        cronPattern: '*/1 * * * *', // Every minute
+        cronPattern: '0 0 * * 0', // Every minute
         priority: 10,
-        immediately: false,
+        immediately: true,
+        delay: 100000, // Delay to allow sessions to be ready
         description: 'Comprehensive symbol search using QM API',
       },
     ],

apps/data-service/src/providers/qm.tasks.ts

@@ -1,5 +1,6 @@
 import { getRandomUserAgent } from '@stock-bot/http';
 import { getLogger } from '@stock-bot/logger';
+import { getMongoDBClient } from '@stock-bot/mongodb-client';
 import { SymbolSearchUtil } from '../utils/symbol-search.util';
 import { getProxy } from './webshare.provider';
 
@@ -15,6 +16,7 @@ export interface QMSession {
   failedCalls: number;
   lastUsed: Date;
 }
+
 function getQmHeaders(): Record<string, string> {
   return {
     'User-Agent': getRandomUserAgent(),
@@ -71,7 +73,7 @@ export async function createSessions(): Promise<void> {
         );
       }
 
-      while (sessionCache[sessionId].length < 5) {
+      while (sessionCache[sessionId].length < 50) {
         logger.info(`Creating new session for ${sessionId}`);
         const proxy = getProxy();
         if (proxy === null) {
@@ -151,11 +153,31 @@ async function searchQMSymbolsAPI(query: string): Promise<string[]> {
     }
 
     const symbols = await response.json();
+    const client = getMongoDBClient();
+    await client.batchUpsert('qmSymbols', symbols, ['symbol', 'exchange']);
+    const exchanges: any[] = [];
+    for (const symbol of symbols) {
+      if (!exchanges.some(ex => ex.exchange === symbol.exchange)) {
+        exchanges.push({
+          exchange: symbol.exchange,
+          exchangeCode: symbol.exchangeCode,
+          exchangeShortName: symbol.exchangeShortName,
+          countryCode: symbol.countryCode,
+        });
+      }
+    }
+    await client.batchUpsert('qmExchanges', exchanges, ['exchange']);
+    session.successfulCalls++;
+    session.lastUsed = new Date();
 
     logger.info(`QM API returned ${symbols.length} symbols for query: ${query}`);
     return symbols;
   } catch (error) {
     logger.error(`Error searching QM symbols for query "${query}":`, error);
+    if (session) {
+      session.failedCalls++;
+      session.lastUsed = new Date();
+    }
     throw error;
   }
 }