starting to add qm sessions and symbols

apps/data-service/src/providers/qm.tasks.ts

@@ -0,0 +1,220 @@
+import { getRandomUserAgent } from '@stock-bot/http';
+import { getLogger } from '@stock-bot/logger';
+import { SymbolSearchUtil } from '../utils/symbol-search.util';
+import { getProxy } from './webshare.provider';
+
+// Shared instances (module-scoped, not global)
+let isInitialized = false; // Track if resources are initialized
+let logger: ReturnType<typeof getLogger>;
+// let cache: CacheProvider;
+
+export interface QMSession {
+  proxy: string;
+  headers: Record<string, string>;
+  successfulCalls: number;
+  failedCalls: number;
+  lastUsed: Date;
+}
+function getQmHeaders(): Record<string, string> {
+  return {
+    'User-Agent': getRandomUserAgent(),
+    Accept: '*/*',
+    'Accept-Language': 'en',
+    'Sec-Fetch-Mode': 'cors',
+    Origin: 'https://www.quotemedia.com',
+    Referer: 'https://www.quotemedia.com/',
+  };
+}
+
+const sessionCache: Record<string, QMSession[]> = {
+  // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
+  // cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
+  // '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
+  // '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
+  // a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
+  // a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
+  // b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
+  dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6: [], //fceb3c4bdd // lookup
+  // '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
+  // '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
+  // '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
+  // d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
+  // d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
+  // '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
+};
+
+export async function initializeQMResources(): Promise<void> {
+  // Skip if already initialized
+  if (isInitialized) {
+    return;
+  }
+  logger = getLogger('qm-tasks');
+  isInitialized = true;
+}
+
+export async function createSessions(): Promise<void> {
+  try {
+    //for each session, check array length, if less than 5, create new session
+    if (!isInitialized) {
+      await initializeQMResources();
+    }
+    logger.info('Creating QM sessions...');
+    for (const [sessionId, sessionArray] of Object.entries(sessionCache)) {
+      // remove any sessions with failedCalls > 10
+      // const filteredArray = sessionArray.filter(session => session.failedCalls <= 10);
+      // sessionCache[sessionId] = filteredArray;
+      // if sessionArray is empty or has less than 5 sessions, create a new session
+      while (sessionArray.length < 2) {
+        logger.info(`Creating new session for ${sessionId}`);
+        const proxy = getProxy();
+        if (proxy === null) {
+          logger.error('No proxy available for QM session creation');
+          break; // Skip session creation if no proxy is available
+        }
+        const newSession: QMSession = {
+          proxy: proxy, // Placeholder, should be set to a valid proxy
+          headers: getQmHeaders(),
+          successfulCalls: 0,
+          failedCalls: 0,
+          lastUsed: new Date(),
+        };
+        const sessionResponse = await fetch(
+          `https://app.quotemedia.com/auth/g/authenticate/dataTool/v0/500/${sessionId}`,
+          {
+            method: 'GET',
+            proxy: newSession.proxy,
+            headers: newSession.headers,
+          }
+        );
+
+        logger.debug('Session response received', {
+          status: sessionResponse.status,
+          sessionId,
+        });
+        if (!sessionResponse.ok) {
+          logger.error('Failed to create QM session', {
+            sessionId,
+            status: sessionResponse.status,
+            statusText: sessionResponse.statusText,
+          });
+          continue; // Skip this session if creation failed
+        }
+        const sessionData = await sessionResponse.json();
+        logger.info('QM session created successfully', {
+          sessionId,
+          sessionData,
+        });
+        newSession.headers['Datatool-Token'] = sessionData.token;
+        console.log(newSession.headers);
+        sessionArray.push(newSession);
+      }
+    }
+    return undefined;
+  } catch (error) {
+    logger.error('❌ Failed to fetch QM session', { error });
+    return undefined;
+  }
+}
+
+// API call function to search symbols via QM
+async function searchQMSymbolsAPI(query: string): Promise<string[]> {
+  const proxy = getProxy();
+
+  if (!proxy) {
+    throw new Error('No proxy available for QM API call');
+  }
+  const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6'; // Use the session ID for symbol lookup
+  const session =
+    sessionCache[sessionId][Math.floor(Math.random() * sessionCache[sessionId].length)]; // lookup session
+  if (!session) {
+    throw new Error(`No active session found for QM API with ID: ${sessionId}`);
+  }
+  try {
+    // QM lookup endpoint for symbol search
+    const apiUrl = `https://app.quotemedia.com/datatool/lookup.json?marketType=equity&pathName=%2Fdemo%2Fportal%2Fcompany-summary.php&q=${encodeURIComponent(query)}&qmodTool=SmartSymbolLookup&searchType=symbol&showFree=false&showHisa=false&webmasterId=500`;
+
+    const response = await fetch(apiUrl, {
+      method: 'GET',
+      headers: session.headers,
+      proxy: session.proxy,
+    });
+
+    if (!response.ok) {
+      throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
+    }
+
+    const symbols = await response.json();
+
+    logger.info(`QM API returned ${symbols.length} symbols for query: ${query}`);
+    return symbols;
+  } catch (error) {
+    logger.error(`Error searching QM symbols for query "${query}":`, error);
+    throw error;
+  }
+}
+
+export async function fetchSymbols(): Promise<unknown[] | null> {
+  try {
+    if (!isInitialized) {
+      await initializeQMResources();
+    }
+    const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6'; // Use the session ID for symbol lookup
+
+    const currentSessions = sessionCache[sessionId] || [];
+    if (currentSessions.length === 0) {
+      logger.info('No sessions found, creating sessions first...');
+      await createSessions();
+
+      // Wait a bit for sessions to be ready
+      await new Promise(resolve => setTimeout(resolve, 2000));
+
+      const newSessions = sessionCache[sessionId] || [];
+      if (newSessions.length === 0) {
+        throw new Error('Failed to create sessions before symbol search');
+      }
+      logger.info(`Created ${newSessions.length} sessions for symbol search`);
+    }
+
+    logger.info('🔄 Starting QM symbols fetch...');
+
+    // Create search function that uses our QM API
+    const searchFunction = async (query: string): Promise<string[]> => {
+      return await searchQMSymbolsAPI(query);
+    };
+
+    // Use the utility to perform comprehensive search
+    const symbols = await SymbolSearchUtil.search(
+      searchFunction,
+      50, // threshold
+      4, // max depth (A -> AA -> AAA -> AAAA)
+      200 // delay between requests in ms
+    );
+
+    logger.info(`QM symbols fetch completed. Found ${symbols.length} total symbols`);
+    return symbols;
+  } catch (error) {
+    logger.error('❌ Failed to fetch QM symbols', { error });
+    return null;
+  }
+}
+
+export async function fetchExchanges(): Promise<unknown[] | null> {
+  try {
+    if (!isInitialized) {
+      await initializeQMResources();
+    }
+
+    logger.info('🔄 QM exchanges fetch - not implemented yet');
+    // TODO: Implement QM exchanges fetching logic
+    return null;
+  } catch (error) {
+    logger.error('❌ Failed to fetch QM exchanges', { error });
+    return null;
+  }
+}
+
+export const qmTasks = {
+  createSessions,
+  fetchSymbols,
+  fetchExchanges,
+};