From 34671ea42709830b06382d49956c3dd84d90109a Mon Sep 17 00:00:00 2001 From: Boki Date: Fri, 27 Jun 2025 21:14:14 -0400 Subject: [PATCH] work on qm symbols --- apps/stock/config/config/default.json | 4 +- .../src/handlers/qm/actions/index.ts | 3 +- .../src/handlers/qm/actions/symbol.action.ts | 145 ++++++++---------- .../src/handlers/qm/qm.handler.ts | 11 +- 4 files changed, 68 insertions(+), 95 deletions(-) diff --git a/apps/stock/config/config/default.json b/apps/stock/config/config/default.json index 4462f86..34374b8 100644 --- a/apps/stock/config/config/default.json +++ b/apps/stock/config/config/default.json @@ -86,8 +86,8 @@ "type": "exponential", "delay": 1000 }, - "removeOnComplete": 50000, - "removeOnFail": 50000, + "removeOnComplete": 100, + "removeOnFail": 100, "timeout": 300000 } }, diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts index a0e6f13..e3f1cdb 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts @@ -3,4 +3,5 @@ */ export { checkSessions, createSession } from './session.action'; -export { spiderSymbolSearch, searchSymbols } from './symbol.action'; +export { searchSymbols, spiderSymbol } from './symbol.action'; + diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts index 467f1c7..a943a8a 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts @@ -3,6 +3,8 @@ */ import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers'; +import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config'; +import { QMSessionManager } from '../shared/session-manager'; import type { Exchange, SymbolSpiderJob } from '../shared/types'; /** @@ -10,7 +12,7 @@ import type { Exchange, SymbolSpiderJob } from '../shared/types'; * Root job (no prefix) creates A-Z jobs * Each job searches its prefix and creates child jobs if needed */ -export async function spiderSymbolSearch( +export async function spiderSymbol( this: BaseHandler, input: SymbolSpiderJob, _context: ExecutionContext @@ -22,7 +24,6 @@ export async function spiderSymbolSearch( const { prefix, depth = 0, maxDepth = 4 } = input || {}; this.logger.info('Spider symbol search', { prefix, depth, maxDepth }); - console.log('Spider symbol search', { prefix, depth, maxDepth }); if (!prefix) { // Root job - create A-Z jobs @@ -59,15 +60,7 @@ export async function spiderSymbolSearch( }; } - // Store symbols in MongoDB - const processedSymbols = symbols.map(symbol => ({ - ...symbol, - spiderPrefix: prefix, - spiderDepth: depth, - discoveredAt: new Date() - })); - - await this.mongodb.batchUpsert('qm_symbols', processedSymbols, ['qmSearchCode']); + await this.mongodb.batchUpsert('qm_symbols', symbols, ['qmSearchCode']); this.logger.info('Stored symbols from spider search', { prefix, @@ -101,7 +94,7 @@ export async function spiderSymbolSearch( // Only create child jobs if we found a significant number of symbols // This prevents excessive branching on sparse results - if (symbols.length >= 10) { + if (symbols.length >= 50) { for (let i = 0; i < 26; i++) { const nextPrefix = prefix + String.fromCharCode(65 + i); await this.scheduleOperation('spider-symbols', { @@ -110,8 +103,7 @@ export async function spiderSymbolSearch( source: 'qm', maxDepth }, { - delay: (i * 2000) + 10000, // Start after 10s, stagger by 2s - priority: Math.max(1, 5 - depth) // Lower priority for deeper searches + priority: Math.max(1, 5 - depth) }); jobsCreated++; } @@ -149,90 +141,73 @@ export async function searchSymbols( const { query } = input; this.logger.debug('Searching QM symbols', { query }); - return [] - // const sessionManager = QMSessionManager.getInstance(); - // sessionManager.initialize(this.cache, this.logger); + const sessionManager = QMSessionManager.getInstance(); + sessionManager.initialize(this.cache, this.logger); - // // Get a session - // const sessionId = QM_SESSION_IDS.LOOKUP; - // const session = await sessionManager.getSession(sessionId); + // Get a session + const sessionId = QM_SESSION_IDS.LOOKUP; + const session = await sessionManager.getSession(sessionId); - // if (!session || !session.uuid) { - // throw new Error(`No active session found for QM LOOKUP`); - // } + if (!session || !session.uuid) { + throw new Error(`No active session found for QM LOOKUP`); + } - // try { - // // Check cache first - // const cacheKey = `qm:symbol-search:${query}`; - // const cachedResult = await this.cache?.get(cacheKey); - // if (cachedResult) { - // this.logger.trace('Using cached symbol search result', { query }); - // return cachedResult as any[]; - // } - - // // Build API request - // const searchParams = new URLSearchParams({ - // marketType: 'equity', - // pathName: '/demo/portal/company-summary.php', - // q: query, - // qmodTool: 'SmartSymbolLookup', - // searchType: 'symbol', - // showFree: 'false', - // showHisa: 'false', - // webmasterId: '500' - // }); + try { + // Build API request + const searchParams = new URLSearchParams({ + marketType: 'equity', + pathName: '/demo/portal/company-summary.php', + q: query, + qmodTool: 'SmartSymbolLookup', + searchType: 'symbol', + showFree: 'false', + showHisa: 'false', + webmasterId: '500' + }); - // const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`; + const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`; - // const response = await fetch(apiUrl, { - // method: 'GET', - // headers: session.headers, - // proxy: session.proxy, - // signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT), - // }); + const response = await fetch(apiUrl, { + method: 'GET', + headers: session.headers, + proxy: session.proxy, + }); - // if (!response.ok) { - // throw new Error(`QM API request failed: ${response.status} ${response.statusText}`); - // } + if (!response.ok) { + throw new Error(`QM API request failed: ${response.status} ${response.statusText}`); + } - // const symbols = await response.json(); + const symbols = await response.json(); - // // Update session success stats - // await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid); + // Update session success stats + await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid); - // // Process symbol data - // const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({ - // ...symbol, - // qmSearchCode: symbol.symbol || '', - // symbol: (symbol.symbol as string)?.split(':')[0] || '', - // searchQuery: query, - // fetchedAt: new Date() - // })) : []; + // Process symbol data + const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({ + ...symbol, + qmSearchCode: symbol.symbol || '', + symbol: (symbol.symbol as string)?.split(':')[0] || '', + })) : []; - // // Cache the result - // if (processedSymbols.length > 0) { - // await this.cache?.set(cacheKey, processedSymbols, 1800); // 30 minutes - // } + this.logger.info('QM API returned symbols', { + query, + count: processedSymbols.length + }); - // this.logger.info('QM API returned symbols', { - // query, - // count: processedSymbols.length - // }); + return processedSymbols; - // return processedSymbols; + } catch (error) { + // Update session failure stats + if (session.uuid) { + await sessionManager.incrementFailedCalls(sessionId, session.uuid); + } - // } catch (error) { - // // Update session failure stats - // if (session.uuid) { - // await sessionManager.incrementFailedCalls(sessionId, session.uuid); - // } - - // this.logger.error('Error searching QM symbols', { - // query, - // error: error instanceof Error ? error.message : 'Unknown error' - // }); + this.logger.error('Error searching QM symbols', { + query, + error: error instanceof Error ? error.message : 'Unknown error' + }); - // throw error; - // } + throw error; + } } \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts index aa3178f..6e9e7ac 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts @@ -4,7 +4,7 @@ import { Operation, ScheduledOperation, } from '@stock-bot/handlers'; -import { checkSessions, createSession, searchSymbols, spiderSymbolSearch } from './actions'; +import { checkSessions, createSession, searchSymbols, spiderSymbol } from './actions'; @Handler('qm') export class QMHandler extends BaseHandler { @@ -22,15 +22,12 @@ export class QMHandler extends BaseHandler { @Operation('create-session') createSession = createSession; - @ScheduledOperation('spider-symbol-search', '* * * * *', { - priority: 8, + @ScheduledOperation('spider-symbols', '* * * * *', { + priority: 9, immediately: false, description: 'Weekly comprehensive symbol search using QM API spider - runs every Saturday at midnight' }) - spiderSymbolSchedule = spiderSymbolSearch; - - @Operation('spider-symbols') - spiderSymbolsJob = spiderSymbolSearch; + spiderSymbol = spiderSymbol; @Operation('search-symbols') searchSymbols = searchSymbols;