fixed proxy started working on new qm

apps/stock/data-ingestion/src/handlers/qm/shared/config.ts

@@ -24,15 +24,15 @@ export const QM_SESSION_IDS = {
 // QM API Configuration
 export const QM_CONFIG = {
   BASE_URL: 'https://app.quotemedia.com',
-  AUTH_PATH: '/auth/g/authenticate/dataTool/v0/500',
+  SESSION_PATH: '/auth/g/authenticate/dataTool/v0/500',
   LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json',
 } as const;
 
 // Session management settings
 export const SESSION_CONFIG = {
-  MIN_SESSIONS: 5,
-  MAX_SESSIONS: 10,
-  MAX_FAILED_CALLS: 10,
-  SESSION_TIMEOUT: 10000, // 10 seconds
-  API_TIMEOUT: 15000, // 15 seconds
+  MIN_SESSIONS: 15,
+  MAX_SESSIONS: 50,
+  MAX_FAILED_CALLS: 3,
+  SESSION_TIMEOUT: 5000, // 10 seconds
+  API_TIMEOUT: 30000, // 15 seconds
 } as const;

apps/stock/data-ingestion/src/handlers/qm/shared/session-manager.ts

@@ -2,14 +2,16 @@
  * QM Session Manager - Centralized session state management
  */
 
+import type { CacheProvider } from '@stock-bot/cache';
 import { getRandomUserAgent } from '@stock-bot/utils';
 import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
-import type { QMSession } from './types';
+import type { CachedSession, QMSession } from './types';
 
 export class QMSessionManager {
   private static instance: QMSessionManager | null = null;
   private sessionCache: Record<string, QMSession[]> = {};
   private isInitialized = false;
+  private cacheProvider: CacheProvider | null = null;
 
   private constructor() {
     // Initialize session cache with known session IDs
@@ -25,6 +27,13 @@ export class QMSessionManager {
     return QMSessionManager.instance;
   }
 
+  /**
+   * Set the cache provider for persistence
+   */
+  setCacheProvider(cache: CacheProvider): void {
+    this.cacheProvider = cache;
+  }
+
   /**
    * Get a random session for the given session ID
    */
@@ -153,4 +162,108 @@ export class QMSessionManager {
   getInitialized(): boolean {
     return this.isInitialized;
   }
+
+  /**
+   * Load sessions from cache
+   */
+  async loadFromCache(): Promise<void> {
+    if (!this.cacheProvider) {
+      return;
+    }
+
+    try {
+      // Load sessions for each session type
+      for (const [sessionType, sessionId] of Object.entries(QM_SESSION_IDS)) {
+        const listKey = `qm:sessions:${sessionType.toLowerCase()}:list`;
+        const sessionIds = await this.cacheProvider.get<string[]>(listKey);
+
+        if (sessionIds && Array.isArray(sessionIds)) {
+          const sessions: QMSession[] = [];
+
+          for (const id of sessionIds) {
+            const sessionKey = `qm:sessions:${sessionType.toLowerCase()}:${id}`;
+            const cachedSession = await this.cacheProvider.get<CachedSession>(sessionKey);
+
+            if (cachedSession) {
+              sessions.push({
+                proxy: cachedSession.proxy,
+                headers: cachedSession.headers,
+                successfulCalls: cachedSession.successfulCalls,
+                failedCalls: cachedSession.failedCalls,
+                lastUsed: new Date(cachedSession.lastUsed),
+              });
+            }
+          }
+
+          this.sessionCache[sessionId] = sessions;
+        }
+      }
+    } catch (error) {
+      console.error('Failed to load sessions from cache:', error);
+    }
+  }
+
+  /**
+   * Sync sessions to cache
+   */
+  async syncToCache(): Promise<void> {
+    if (!this.cacheProvider) {
+      return;
+    }
+
+    try {
+      for (const [sessionType, sessionId] of Object.entries(QM_SESSION_IDS)) {
+        const sessions = this.sessionCache[sessionId] || [];
+        const sessionIds: string[] = [];
+
+        // Store each session
+        for (let i = 0; i < sessions.length; i++) {
+          const session = sessions[i];
+          const id = `${sessionType.toLowerCase()}_${i}_${Date.now()}`;
+          const sessionKey = `qm:sessions:${sessionType.toLowerCase()}:${id}`;
+
+          const cachedSession: CachedSession = {
+            ...session,
+            id,
+            sessionType,
+          };
+
+          await this.cacheProvider.set(sessionKey, cachedSession, 86400); // 24 hour TTL
+          sessionIds.push(id);
+        }
+
+        // Store the list of session IDs
+        const listKey = `qm:sessions:${sessionType.toLowerCase()}:list`;
+        await this.cacheProvider.set(listKey, sessionIds, 86400);
+      }
+
+      // Store stats
+      const statsKey = 'qm:sessions:stats';
+      await this.cacheProvider.set(statsKey, this.getStats(), 3600);
+    } catch (error) {
+      console.error('Failed to sync sessions to cache:', error);
+    }
+  }
+
+  /**
+   * Increment failed calls for a session
+   */
+  async incrementFailedCalls(sessionId: string, session: QMSession): Promise<void> {
+    session.failedCalls++;
+    session.lastUsed = new Date();
+
+    // Sync to cache after update
+    await this.syncToCache();
+  }
+
+  /**
+   * Increment successful calls for a session
+   */
+  async incrementSuccessfulCalls(sessionId: string, session: QMSession): Promise<void> {
+    session.successfulCalls++;
+    session.lastUsed = new Date();
+
+    // Sync to cache after update
+    await this.syncToCache();
+  }
 }

apps/stock/data-ingestion/src/handlers/qm/shared/types.ts

@@ -30,3 +30,22 @@ export interface SpiderResult {
   symbolsFound: number;
   jobsCreated: number;
 }
+
+export interface QMSessionStats {
+  sessionType: string;
+  total: number;
+  valid: number;
+  failed: number;
+  lastUpdate: Date;
+}
+
+export interface QMAuthResponse {
+  success: boolean;
+  cookies?: string[];
+  error?: string;
+}
+
+export interface CachedSession extends QMSession {
+  id: string;
+  sessionType: string;
+}