diff --git a/apps/stock/config/config/default.json b/apps/stock/config/config/default.json index b664a20..4462f86 100644 --- a/apps/stock/config/config/default.json +++ b/apps/stock/config/config/default.json @@ -86,8 +86,8 @@ "type": "exponential", "delay": 1000 }, - "removeOnComplete": 100, - "removeOnFail": 50, + "removeOnComplete": 50000, + "removeOnFail": 50000, "timeout": 300000 } }, diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts index bc8834a..a0e6f13 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/index.ts @@ -3,3 +3,4 @@ */ export { checkSessions, createSession } from './session.action'; +export { spiderSymbolSearch, searchSymbols } from './symbol.action'; diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/spider.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/spider.action.ts deleted file mode 100644 index 2e694c5..0000000 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/spider.action.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * QM Spider Operations - Simple symbol discovery - */ - -import type { IServiceContainer } from '@stock-bot/handlers'; -import type { SymbolSpiderJob } from '../shared/types'; - -export async function spiderSymbolSearch( - services: IServiceContainer, - config: SymbolSpiderJob -): Promise<{ foundSymbols: number; depth: number }> { - // Simple spider implementation - // TODO: Implement actual API calls to discover symbols - - // For now, just return mock results - const foundSymbols = Math.floor(Math.random() * 10) + 1; - - return { - foundSymbols, - depth: config.depth, - }; -} - -export async function queueSymbolDiscovery( - services: IServiceContainer, - searchTerms: string[] -): Promise { - // Queue symbol discovery jobs - for (const term of searchTerms) { - // TODO: Queue actual discovery jobs - await services.cache.set(`discovery:${term}`, { queued: true }, 3600); - } -} diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts new file mode 100644 index 0000000..467f1c7 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts @@ -0,0 +1,238 @@ +/** + * QM Symbol Actions - Symbol search and spider operations + */ + +import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers'; +import type { Exchange, SymbolSpiderJob } from '../shared/types'; + +/** + * Spider search for symbols - recursively searches QM API + * Root job (no prefix) creates A-Z jobs + * Each job searches its prefix and creates child jobs if needed + */ +export async function spiderSymbolSearch( + this: BaseHandler, + input: SymbolSpiderJob, + _context: ExecutionContext +): Promise<{ + message: string; + symbolsFound?: number; + jobsCreated?: number; +}> { + const { prefix, depth = 0, maxDepth = 4 } = input || {}; + + this.logger.info('Spider symbol search', { prefix, depth, maxDepth }); + console.log('Spider symbol search', { prefix, depth, maxDepth }); + + if (!prefix) { + // Root job - create A-Z jobs + let jobsCreated = 0; + for (let i = 0; i < 26; i++) { + const letter = String.fromCharCode(65 + i); // A-Z + await this.scheduleOperation('spider-symbols', { + prefix: letter, + depth: 1, + source: 'qm', + maxDepth + }, { + priority: 5 + }); + jobsCreated++; + } + + this.logger.info('Created root spider jobs', { jobsCreated }); + return { + message: `Queued ${jobsCreated} root jobs (A-Z)`, + jobsCreated + }; + } + + try { + // Search current prefix + const symbols = await searchSymbols.call(this, { query: prefix }); + + if (!symbols || symbols.length === 0) { + this.logger.debug('No symbols found for prefix', { prefix }); + return { + message: `No symbols found for prefix: ${prefix}`, + symbolsFound: 0 + }; + } + + // Store symbols in MongoDB + const processedSymbols = symbols.map(symbol => ({ + ...symbol, + spiderPrefix: prefix, + spiderDepth: depth, + discoveredAt: new Date() + })); + + await this.mongodb.batchUpsert('qm_symbols', processedSymbols, ['qmSearchCode']); + + this.logger.info('Stored symbols from spider search', { + prefix, + count: symbols.length + }); + + // Extract and store unique exchanges + const exchanges: Exchange[] = []; + for (const symbol of symbols) { + if (symbol.exchange && !exchanges.some(ex => ex.exchange === symbol.exchange)) { + exchanges.push({ + exchange: symbol.exchange, + exchangeCode: symbol.exchangeCode || '', + exchangeShortName: symbol.exchangeShortName || '', + countryCode: symbol.countryCode || '', + source: 'qm', + }); + } + } + + if (exchanges.length > 0) { + await this.mongodb.batchUpsert('qm_exchanges', exchanges, ['exchange']); + this.logger.debug('Stored exchanges from spider search', { + count: exchanges.length + }); + } + + // If not at max depth and we found symbols, create child jobs + if (depth < maxDepth && symbols.length > 0) { + let jobsCreated = 0; + + // Only create child jobs if we found a significant number of symbols + // This prevents excessive branching on sparse results + if (symbols.length >= 10) { + for (let i = 0; i < 26; i++) { + const nextPrefix = prefix + String.fromCharCode(65 + i); + await this.scheduleOperation('spider-symbols', { + prefix: nextPrefix, + depth: depth + 1, + source: 'qm', + maxDepth + }, { + delay: (i * 2000) + 10000, // Start after 10s, stagger by 2s + priority: Math.max(1, 5 - depth) // Lower priority for deeper searches + }); + jobsCreated++; + } + } + + return { + message: `Found ${symbols.length} symbols for ${prefix}, queued ${jobsCreated} child jobs`, + symbolsFound: symbols.length, + jobsCreated + }; + } + + return { + message: `Found ${symbols.length} symbols for ${prefix} (at max depth or too few results)`, + symbolsFound: symbols.length + }; + + } catch (error) { + this.logger.error('Spider search failed', { prefix, error }); + return { + message: `Spider search failed for prefix: ${prefix}`, + symbolsFound: 0 + }; + } +} + +/** + * Search QM symbols API directly + */ +export async function searchSymbols( + this: BaseHandler, + input: { query: string }, + _context?: ExecutionContext +): Promise { + const { query } = input; + + this.logger.debug('Searching QM symbols', { query }); + return [] + + // const sessionManager = QMSessionManager.getInstance(); + // sessionManager.initialize(this.cache, this.logger); + + // // Get a session + // const sessionId = QM_SESSION_IDS.LOOKUP; + // const session = await sessionManager.getSession(sessionId); + + // if (!session || !session.uuid) { + // throw new Error(`No active session found for QM LOOKUP`); + // } + + // try { + // // Check cache first + // const cacheKey = `qm:symbol-search:${query}`; + // const cachedResult = await this.cache?.get(cacheKey); + // if (cachedResult) { + // this.logger.trace('Using cached symbol search result', { query }); + // return cachedResult as any[]; + // } + + // // Build API request + // const searchParams = new URLSearchParams({ + // marketType: 'equity', + // pathName: '/demo/portal/company-summary.php', + // q: query, + // qmodTool: 'SmartSymbolLookup', + // searchType: 'symbol', + // showFree: 'false', + // showHisa: 'false', + // webmasterId: '500' + // }); + + // const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`; + + // const response = await fetch(apiUrl, { + // method: 'GET', + // headers: session.headers, + // proxy: session.proxy, + // signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT), + // }); + + // if (!response.ok) { + // throw new Error(`QM API request failed: ${response.status} ${response.statusText}`); + // } + + // const symbols = await response.json(); + + // // Update session success stats + // await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid); + + // // Process symbol data + // const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({ + // ...symbol, + // qmSearchCode: symbol.symbol || '', + // symbol: (symbol.symbol as string)?.split(':')[0] || '', + // searchQuery: query, + // fetchedAt: new Date() + // })) : []; + + // // Cache the result + // if (processedSymbols.length > 0) { + // await this.cache?.set(cacheKey, processedSymbols, 1800); // 30 minutes + // } + + // this.logger.info('QM API returned symbols', { + // query, + // count: processedSymbols.length + // }); + + // return processedSymbols; + + // } catch (error) { + // // Update session failure stats + // if (session.uuid) { + // await sessionManager.incrementFailedCalls(sessionId, session.uuid); + // } + + // this.logger.error('Error searching QM symbols', { + // query, + // error: error instanceof Error ? error.message : 'Unknown error' + // }); + + // throw error; + // } +} \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/symbols.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/symbols.action.ts deleted file mode 100644 index f0def4b..0000000 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/symbols.action.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** - * QM Symbols Operations - Simple symbol fetching - */ - -import type { IServiceContainer } from '@stock-bot/handlers'; - -interface QMSymbol { - _id?: string; - symbol: string; - name: string; - exchange: string; - type?: string; - sector?: string; - industry?: string; -} - -export async function searchSymbols(services: IServiceContainer): Promise { - // Get symbols from MongoDB - const symbols = await services.mongodb - .collection('qm_symbols') - .find({}) - .limit(50) - .toArray(); - - return symbols; -} - -export async function fetchSymbolData( - services: IServiceContainer, - symbol: string -): Promise { - // Fetch data for a specific symbol - const symbolData = await services.mongodb.collection('qm_symbols').findOne({ symbol }); - - return symbolData; -} diff --git a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts index 0998d93..aa3178f 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts @@ -4,7 +4,7 @@ import { Operation, ScheduledOperation, } from '@stock-bot/handlers'; -import { checkSessions, createSession } from './actions'; +import { checkSessions, createSession, searchSymbols, spiderSymbolSearch } from './actions'; @Handler('qm') export class QMHandler extends BaseHandler { @@ -14,7 +14,7 @@ export class QMHandler extends BaseHandler { @ScheduledOperation('check-sessions', '*/2 * * * *', { priority: 8, - immediately: true, + immediately: false, description: 'Check and maintain QM sessions every 2 minutes', }) checkSessions = checkSessions; @@ -22,79 +22,16 @@ export class QMHandler extends BaseHandler { @Operation('create-session') createSession = createSession; - // @Operation('search-symbols') - // async searchSymbols(_input: unknown, _context: ExecutionContext): Promise { - // this.logger.info('Searching QM symbols with new DI pattern...'); - // try { - // // Check existing symbols in MongoDB - // const symbolsCollection = this.mongodb.collection('qm_symbols'); - // const symbols = await symbolsCollection.find({}).limit(100).toArray(); + @ScheduledOperation('spider-symbol-search', '* * * * *', { + priority: 8, + immediately: false, + description: 'Weekly comprehensive symbol search using QM API spider - runs every Saturday at midnight' + }) + spiderSymbolSchedule = spiderSymbolSearch; - // this.logger.info('QM symbol search completed', { count: symbols.length }); + @Operation('spider-symbols') + spiderSymbolsJob = spiderSymbolSearch; - // if (symbols && symbols.length > 0) { - // // Cache result for performance - // await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800); - - // return { - // success: true, - // message: 'QM symbol search completed successfully', - // count: symbols.length, - // symbols: symbols.slice(0, 10), // Return first 10 symbols as sample - // }; - // } else { - // // No symbols found - this is expected initially - // this.logger.info('No QM symbols found in database yet'); - // return { - // success: true, - // message: 'No symbols found yet - database is empty', - // count: 0, - // }; - // } - - // } catch (error) { - // this.logger.error('Failed to search QM symbols', { error }); - // throw error; - // } - // } - - // @Operation('spider-symbol-search') - // @QueueSchedule('0 0 * * 0', { - // priority: 10, - // immediately: false, - // description: 'Comprehensive symbol search using QM API' - // }) - // async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise { - // // Set default payload for scheduled runs - // const jobPayload: SymbolSpiderJob = payload || { - // prefix: null, - // depth: 1, - // source: 'qm', - // maxDepth: 4 - // }; - - // this.logger.info('Starting QM spider symbol search', { payload: jobPayload }); - - // // Store spider job info in cache (temporary data) - // const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`; - // const spiderResult = { - // payload: jobPayload, - // startTime: new Date().toISOString(), - // status: 'started', - // jobId: spiderJobId - // }; - - // // Store in cache with 1 hour TTL (temporary data) - // await this.cache.set(spiderJobId, spiderResult, 3600); - // this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 }); - - // // Schedule follow-up processing if needed - // await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, { delay: 5000 }); - - // return { - // success: true, - // message: 'QM spider search initiated', - // spiderJobId - // }; - // } + @Operation('search-symbols') + searchSymbols = searchSymbols; } diff --git a/apps/stock/data-ingestion/test-spider.md b/apps/stock/data-ingestion/test-spider.md new file mode 100644 index 0000000..0c671e1 --- /dev/null +++ b/apps/stock/data-ingestion/test-spider.md @@ -0,0 +1,80 @@ +# QM Spider Symbol Search Test + +## How the Spider Search Works + +The spider search is a recursive symbol discovery system that explores the QM API systematically: + +### 1. Root Job (Weekly Schedule) +```typescript +// Triggered every Saturday at midnight +@ScheduledOperation('spider-symbol-search', '0 0 * * 6', {...}) + +// Creates 26 jobs: A, B, C, ... Z +input: { prefix: null, depth: 0, maxDepth: 4 } +``` + +### 2. Level 1 - Single Letters (A-Z) +```typescript +// Each job searches for symbols starting with that letter +input: { prefix: 'A', depth: 1, maxDepth: 4 } + +// If 10+ symbols found, creates: AA, AB, AC, ... AZ +``` + +### 3. Level 2 - Two Letters (AA-ZZ) +```typescript +// Searches for symbols starting with two letters +input: { prefix: 'AA', depth: 2, maxDepth: 4 } + +// If 10+ symbols found, creates: AAA, AAB, AAC, ... AAZ +``` + +### 4. Level 3 & 4 - Deeper Search +Continues until `maxDepth` is reached or fewer than 10 symbols are found. + +## Testing the Spider + +### Manual Test - Root Job +```bash +# Trigger spider search root job +curl -X POST http://localhost:3000/api/handlers/qm/operations/spider-symbol-search \ + -H "Content-Type: application/json" \ + -d '{}' +``` + +### Manual Test - Specific Prefix +```bash +# Search for symbols starting with "APP" +curl -X POST http://localhost:3000/api/handlers/qm/operations/spider-symbol-search \ + -H "Content-Type: application/json" \ + -d '{"prefix": "APP", "depth": 3, "maxDepth": 4}' +``` + +### Direct Symbol Search +```bash +# Search for specific symbols +curl -X POST http://localhost:3000/api/handlers/qm/operations/search-symbols \ + -H "Content-Type: application/json" \ + -d '{"query": "AAPL"}' +``` + +## Optimization Features + +1. **Intelligent Branching**: Only creates child jobs if 10+ symbols found +2. **Priority Management**: Deeper searches get lower priority +3. **Staggered Execution**: Jobs are delayed to avoid API rate limits +4. **Session Management**: Uses QM sessions with failure tracking +5. **Caching**: Results cached for 30 minutes to avoid duplicate API calls + +## MongoDB Collections + +- `qm_symbols`: Stores discovered symbols with metadata +- `qm_exchanges`: Stores unique exchanges found during searches + +## Monitoring + +Check logs for: +- "Spider symbol search" - Job execution +- "Created root spider jobs" - Initial A-Z creation +- "Stored symbols from spider search" - Successful symbol storage +- "Found X symbols for Y, queued Z child jobs" - Branching decisions \ No newline at end of file