refactored monorepo for more projects

apps/stock/data-ingestion/src/handlers/qm/shared/config.ts

@@ -0,0 +1,38 @@
+/**
+ * Shared configuration for QM operations
+ */
+
+// QM Session IDs for different endpoints
+export const QM_SESSION_IDS = {
+  LOOKUP: 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6', // lookup endpoint
+  // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
+  // cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
+  // '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
+  // '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
+  // a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
+  // a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
+  // b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
+  // '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
+  // '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
+  // '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
+  // d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
+  // d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
+  // '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
+  // Add other session IDs as needed
+} as const;
+
+// QM API Configuration
+export const QM_CONFIG = {
+  BASE_URL: 'https://app.quotemedia.com',
+  AUTH_PATH: '/auth/g/authenticate/dataTool/v0/500',
+  LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json',
+} as const;
+
+// Session management settings
+export const SESSION_CONFIG = {
+  MIN_SESSIONS: 5,
+  MAX_SESSIONS: 10,
+  MAX_FAILED_CALLS: 10,
+  SESSION_TIMEOUT: 10000, // 10 seconds
+  API_TIMEOUT: 15000, // 15 seconds
+} as const;

apps/stock/data-ingestion/src/handlers/qm/shared/session-manager.ts

@@ -0,0 +1,156 @@
+/**
+ * QM Session Manager - Centralized session state management
+ */
+
+import { getRandomUserAgent } from '@stock-bot/utils';
+import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
+import type { QMSession } from './types';
+
+export class QMSessionManager {
+  private static instance: QMSessionManager | null = null;
+  private sessionCache: Record<string, QMSession[]> = {};
+  private isInitialized = false;
+
+  private constructor() {
+    // Initialize session cache with known session IDs
+    Object.values(QM_SESSION_IDS).forEach(sessionId => {
+      this.sessionCache[sessionId] = [];
+    });
+  }
+
+  static getInstance(): QMSessionManager {
+    if (!QMSessionManager.instance) {
+      QMSessionManager.instance = new QMSessionManager();
+    }
+    return QMSessionManager.instance;
+  }
+
+  /**
+   * Get a random session for the given session ID
+   */
+  getSession(sessionId: string): QMSession | null {
+    const sessions = this.sessionCache[sessionId];
+    if (!sessions || sessions.length === 0) {
+      return null;
+    }
+
+    // Filter out sessions with excessive failures
+    const validSessions = sessions.filter(
+      session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+    );
+    if (validSessions.length === 0) {
+      return null;
+    }
+
+    return validSessions[Math.floor(Math.random() * validSessions.length)];
+  }
+
+  /**
+   * Add a session to the cache
+   */
+  addSession(sessionId: string, session: QMSession): void {
+    if (!this.sessionCache[sessionId]) {
+      this.sessionCache[sessionId] = [];
+    }
+    this.sessionCache[sessionId].push(session);
+  }
+
+  /**
+   * Get all sessions for a session ID
+   */
+  getSessions(sessionId: string): QMSession[] {
+    return this.sessionCache[sessionId] || [];
+  }
+
+  /**
+   * Get session count for all session IDs
+   */
+  getSessionCount(): number {
+    return Object.values(this.sessionCache).reduce((total, sessions) => total + sessions.length, 0);
+  }
+
+  /**
+   * Clean up failed sessions
+   */
+  cleanupFailedSessions(): number {
+    let removedCount = 0;
+
+    Object.keys(this.sessionCache).forEach(sessionId => {
+      const initialCount = this.sessionCache[sessionId].length;
+      this.sessionCache[sessionId] = this.sessionCache[sessionId].filter(
+        session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+      );
+      removedCount += initialCount - this.sessionCache[sessionId].length;
+    });
+
+    return removedCount;
+  }
+
+  getQmHeaders(): Record<string, string> {
+    return {
+      'User-Agent': getRandomUserAgent(),
+      Accept: '*/*',
+      'Accept-Language': 'en',
+      'Sec-Fetch-Mode': 'cors',
+      Origin: 'https://www.quotemedia.com',
+      Referer: 'https://www.quotemedia.com/',
+    };
+  }
+
+  /**
+   * Check if more sessions are needed for a session ID
+   */
+  needsMoreSessions(sessionId: string): boolean {
+    const sessions = this.sessionCache[sessionId] || [];
+    const validSessions = sessions.filter(
+      session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+    );
+    return validSessions.length < SESSION_CONFIG.MIN_SESSIONS;
+  }
+
+  /**
+   * Check if session ID is at capacity
+   */
+  isAtCapacity(sessionId: string): boolean {
+    const sessions = this.sessionCache[sessionId] || [];
+    return sessions.length >= SESSION_CONFIG.MAX_SESSIONS;
+  }
+
+  /**
+   * Get session cache statistics
+   */
+  getStats() {
+    const stats: Record<string, { total: number; valid: number; failed: number }> = {};
+
+    Object.entries(this.sessionCache).forEach(([sessionId, sessions]) => {
+      const validSessions = sessions.filter(
+        session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
+      );
+      const failedSessions = sessions.filter(
+        session => session.failedCalls > SESSION_CONFIG.MAX_FAILED_CALLS
+      );
+
+      stats[sessionId] = {
+        total: sessions.length,
+        valid: validSessions.length,
+        failed: failedSessions.length,
+      };
+    });
+
+    return stats;
+  }
+
+  /**
+   * Mark manager as initialized
+   */
+  setInitialized(initialized: boolean = true): void {
+    this.isInitialized = initialized;
+  }
+
+  /**
+   * Check if manager is initialized
+   */
+  getInitialized(): boolean {
+    return this.isInitialized;
+  }
+}

apps/stock/data-ingestion/src/handlers/qm/shared/types.ts

@@ -0,0 +1,32 @@
+/**
+ * Shared types for QM operations
+ */
+
+export interface QMSession {
+  proxy: string;
+  headers: Record<string, string>;
+  successfulCalls: number;
+  failedCalls: number;
+  lastUsed: Date;
+}
+
+export interface SymbolSpiderJob {
+  prefix: string | null; // null = root job (A-Z)
+  depth: number; // 1=A, 2=AA, 3=AAA, etc.
+  source: string; // 'qm'
+  maxDepth?: number; // optional max depth limit
+}
+
+export interface Exchange {
+  exchange: string;
+  exchangeCode: string;
+  exchangeShortName: string;
+  countryCode: string;
+  source: string;
+}
+
+export interface SpiderResult {
+  success: boolean;
+  symbolsFound: number;
+  jobsCreated: number;
+}