diff --git a/apps/data-ingestion/src/handlers/qm/actions/session.action.ts b/apps/data-ingestion/src/handlers/qm/actions/session.action.ts index c1badd7..8c6eb70 100644 --- a/apps/data-ingestion/src/handlers/qm/actions/session.action.ts +++ b/apps/data-ingestion/src/handlers/qm/actions/session.action.ts @@ -2,14 +2,14 @@ * QM Session Actions - Session management and creation */ -import type { IServiceContainer } from '@stock-bot/handlers'; +import { BaseHandler } from '@stock-bot/core/handlers'; import { QM_SESSION_IDS, SESSION_CONFIG } from '../shared/config'; import { QMSessionManager } from '../shared/session-manager'; /** * Check existing sessions and queue creation jobs for needed sessions */ -export async function checkSessions(services: IServiceContainer): Promise<{ +export async function checkSessions(handler: BaseHandler): Promise<{ cleaned: number; queued: number; message: string; @@ -19,12 +19,13 @@ export async function checkSessions(services: IServiceContainer): Promise<{ // Check which session IDs need more sessions and queue creation jobs let queuedCount = 0; for (const sessionId of Object.values(QM_SESSION_IDS)) { + console.log(`Checking session ID: ${sessionId}`); if (sessionManager.needsMoreSessions(sessionId)) { const currentCount = sessionManager.getSessions(sessionId).length; const neededSessions = SESSION_CONFIG.MAX_SESSIONS - currentCount; for (let i = 0; i < neededSessions; i++) { - await services.queue.getQueue('qm').add('create-session', { sessionId }); - services.logger.log(`Queued job to create session for ${sessionId}`); + await handler.scheduleOperation('create-session', { sessionId }); + handler.services.logger.log(`Queued job to create session for ${sessionId}`); queuedCount++; } } @@ -41,29 +42,17 @@ export async function checkSessions(services: IServiceContainer): Promise<{ * Create a single session for a specific session ID */ export async function createSingleSession( - services: IServiceContainer, + handler: BaseHandler, input: any ): Promise<{ sessionId: string; status: string; sessionType: string }> { const { sessionId: sessionType = 'default' } = input || {}; const sessionManager = QMSessionManager.getInstance(); - // Check if we're at capacity for this session type - if (sessionManager.isAtCapacity(sessionType)) { - return { - sessionId: '', - status: 'skipped', - sessionType, - }; - } - // TODO: Get actual proxy and headers from proxy service const session = { - proxy: 'http://proxy:8080', // Placeholder - headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; QMBot/1.0)', - 'Accept': 'application/json' - }, + // proxy: handler.services.getRandomProxy(), + headers: sessionManager.getQmHeaders(), successfulCalls: 0, failedCalls: 0, lastUsed: new Date() diff --git a/apps/data-ingestion/src/handlers/qm/qm.handler.ts b/apps/data-ingestion/src/handlers/qm/qm.handler.ts index 07db41d..ab11902 100644 --- a/apps/data-ingestion/src/handlers/qm/qm.handler.ts +++ b/apps/data-ingestion/src/handlers/qm/qm.handler.ts @@ -6,7 +6,6 @@ import { type ExecutionContext, type IServiceContainer } from '@stock-bot/handlers'; -import type { SymbolSpiderJob } from './shared/types'; @Handler('qm') export class QMHandler extends BaseHandler { @@ -23,90 +22,90 @@ export class QMHandler extends BaseHandler { async checkSessions(input: unknown, context: ExecutionContext): Promise { // Call the session maintenance action const { checkSessions } = await import('./actions/session.action'); - return await checkSessions(this.services); + return await checkSessions(this); } @Operation('create-session') async createSession(input: unknown, context: ExecutionContext): Promise { // Call the individual session creation action const { createSingleSession } = await import('./actions/session.action'); - return await createSingleSession(this.services, input); + return await createSingleSession(this, input); } - @Operation('search-symbols') - async searchSymbols(_input: unknown, _context: ExecutionContext): Promise { - this.logger.info('Searching QM symbols with new DI pattern...'); - try { - // Check existing symbols in MongoDB - const symbolsCollection = this.mongodb.collection('qm_symbols'); - const symbols = await symbolsCollection.find({}).limit(100).toArray(); + // @Operation('search-symbols') + // async searchSymbols(_input: unknown, _context: ExecutionContext): Promise { + // this.logger.info('Searching QM symbols with new DI pattern...'); + // try { + // // Check existing symbols in MongoDB + // const symbolsCollection = this.mongodb.collection('qm_symbols'); + // const symbols = await symbolsCollection.find({}).limit(100).toArray(); - this.logger.info('QM symbol search completed', { count: symbols.length }); + // this.logger.info('QM symbol search completed', { count: symbols.length }); - if (symbols && symbols.length > 0) { - // Cache result for performance - await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800); + // if (symbols && symbols.length > 0) { + // // Cache result for performance + // await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800); - return { - success: true, - message: 'QM symbol search completed successfully', - count: symbols.length, - symbols: symbols.slice(0, 10), // Return first 10 symbols as sample - }; - } else { - // No symbols found - this is expected initially - this.logger.info('No QM symbols found in database yet'); - return { - success: true, - message: 'No symbols found yet - database is empty', - count: 0, - }; - } + // return { + // success: true, + // message: 'QM symbol search completed successfully', + // count: symbols.length, + // symbols: symbols.slice(0, 10), // Return first 10 symbols as sample + // }; + // } else { + // // No symbols found - this is expected initially + // this.logger.info('No QM symbols found in database yet'); + // return { + // success: true, + // message: 'No symbols found yet - database is empty', + // count: 0, + // }; + // } - } catch (error) { - this.logger.error('Failed to search QM symbols', { error }); - throw error; - } - } + // } catch (error) { + // this.logger.error('Failed to search QM symbols', { error }); + // throw error; + // } + // } - @Operation('spider-symbol-search') - @QueueSchedule('0 0 * * 0', { - priority: 10, - immediately: false, - description: 'Comprehensive symbol search using QM API' - }) - async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise { - // Set default payload for scheduled runs - const jobPayload: SymbolSpiderJob = payload || { - prefix: null, - depth: 1, - source: 'qm', - maxDepth: 4 - }; + // @Operation('spider-symbol-search') + // @QueueSchedule('0 0 * * 0', { + // priority: 10, + // immediately: false, + // description: 'Comprehensive symbol search using QM API' + // }) + // async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise { + // // Set default payload for scheduled runs + // const jobPayload: SymbolSpiderJob = payload || { + // prefix: null, + // depth: 1, + // source: 'qm', + // maxDepth: 4 + // }; - this.logger.info('Starting QM spider symbol search', { payload: jobPayload }); + // this.logger.info('Starting QM spider symbol search', { payload: jobPayload }); - // Store spider job info in cache (temporary data) - const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`; - const spiderResult = { - payload: jobPayload, - startTime: new Date().toISOString(), - status: 'started', - jobId: spiderJobId - }; + // // Store spider job info in cache (temporary data) + // const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`; + // const spiderResult = { + // payload: jobPayload, + // startTime: new Date().toISOString(), + // status: 'started', + // jobId: spiderJobId + // }; - // Store in cache with 1 hour TTL (temporary data) - await this.cache.set(spiderJobId, spiderResult, 3600); - this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 }); + // // Store in cache with 1 hour TTL (temporary data) + // await this.cache.set(spiderJobId, spiderResult, 3600); + // this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 }); - // Schedule follow-up processing if needed - await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, 5000); + // // Schedule follow-up processing if needed + // await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, 5000); - return { - success: true, - message: 'QM spider search initiated', - spiderJobId - }; - } + // return { + // success: true, + // message: 'QM spider search initiated', + // spiderJobId + // }; + // } } diff --git a/apps/data-ingestion/src/handlers/qm/shared/config.ts b/apps/data-ingestion/src/handlers/qm/shared/config.ts index f54deda..4b5212e 100644 --- a/apps/data-ingestion/src/handlers/qm/shared/config.ts +++ b/apps/data-ingestion/src/handlers/qm/shared/config.ts @@ -2,7 +2,6 @@ * Shared configuration for QM operations */ -import { getRandomUserAgent } from '@stock-bot/http'; // QM Session IDs for different endpoints export const QM_SESSION_IDS = { @@ -28,8 +27,6 @@ export const QM_CONFIG = { BASE_URL: 'https://app.quotemedia.com', AUTH_PATH: '/auth/g/authenticate/dataTool/v0/500', LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json', - ORIGIN: 'https://www.quotemedia.com', - REFERER: 'https://www.quotemedia.com/', } as const; // Session management settings @@ -39,18 +36,4 @@ export const SESSION_CONFIG = { MAX_FAILED_CALLS: 10, SESSION_TIMEOUT: 10000, // 10 seconds API_TIMEOUT: 15000, // 15 seconds -} as const; - -/** - * Generate standard QM headers - */ -export function getQmHeaders(): Record { - return { - 'User-Agent': getRandomUserAgent(), - Accept: '*/*', - 'Accept-Language': 'en', - 'Sec-Fetch-Mode': 'cors', - Origin: QM_CONFIG.ORIGIN, - Referer: QM_CONFIG.REFERER, - }; -} \ No newline at end of file +} as const; \ No newline at end of file diff --git a/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts b/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts index b274e7c..8bbfbce 100644 --- a/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts +++ b/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts @@ -2,8 +2,9 @@ * QM Session Manager - Centralized session state management */ -import type { QMSession } from './types'; +import { getRandomUserAgent } from '@stock-bot/services/http'; import { QM_SESSION_IDS, SESSION_CONFIG } from './config'; +import type { QMSession } from './types'; export class QMSessionManager { private static instance: QMSessionManager | null = null; @@ -83,6 +84,17 @@ export class QMSessionManager { return removedCount; } + getQmHeaders(): Record { + return { + 'User-Agent': getRandomUserAgent(), + Accept: '*/*', + 'Accept-Language': 'en', + 'Sec-Fetch-Mode': 'cors', + Origin: 'https://www.quotemedia.com', + Referer: 'https://www.quotemedia.com/', + }; + } + /** * Check if more sessions are needed for a session ID */ diff --git a/libs/core/handlers/src/base/BaseHandler.ts b/libs/core/handlers/src/base/BaseHandler.ts index 9afae79..b2de06c 100644 --- a/libs/core/handlers/src/base/BaseHandler.ts +++ b/libs/core/handlers/src/base/BaseHandler.ts @@ -1,7 +1,7 @@ import { getLogger } from '@stock-bot/logger'; -import type { IHandler, ExecutionContext } from '../types/types'; +import { createJobHandler, handlerRegistry, type HandlerConfigWithSchedule } from '@stock-bot/types'; import type { IServiceContainer } from '../types/service-container'; -import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/types'; +import type { ExecutionContext, IHandler } from '../types/types'; /** * Abstract base class for all handlers with improved DI @@ -11,7 +11,7 @@ export abstract class BaseHandler implements IHandler { protected readonly logger; private handlerName: string; - constructor(protected readonly services: IServiceContainer, handlerName?: string) { + constructor(readonly services: IServiceContainer, handlerName?: string) { this.logger = getLogger(this.constructor.name); // Read handler name from decorator first, then fallback to parameter or class name const constructor = this.constructor as any; @@ -65,10 +65,7 @@ export abstract class BaseHandler implements IHandler { return await method.call(this, input, context); } - /** - * Queue helper methods - now type-safe and direct - */ - protected async scheduleOperation(operation: string, payload: unknown, delay?: number): Promise { + async scheduleOperation(operation: string, payload: unknown, delay?: number): Promise { const queue = this.services.queue.getQueue(this.handlerName); const jobData = { handler: this.handlerName,