boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

work on qm symbols

Browse source
This commit is contained in:
Boki 2025-06-27 21:14:14 -04:00
parent 1d62343051
commit 34671ea427
4 changed files with 68 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

4
apps/stock/config/config/default.json
View file

@ -86,8 +86,8 @@
"type": "exponential", "type": "exponential",
"delay": 1000 "delay": 1000
}, },
"removeOnComplete": 50000, "removeOnComplete": 100,
"removeOnFail": 50000, "removeOnFail": 100,
"timeout": 300000 "timeout": 300000
} }
}, },

3
apps/stock/data-ingestion/src/handlers/qm/actions/index.ts
View file

@ -3,4 +3,5 @@
*/ */
export { checkSessions, createSession } from './session.action'; export { checkSessions, createSession } from './session.action';
export { spiderSymbolSearch, searchSymbols } from './symbol.action'; export { searchSymbols, spiderSymbol } from './symbol.action';

145
apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts
View file

@ -3,6 +3,8 @@
*/ */
import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers'; import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMSessionManager } from '../shared/session-manager';
import type { Exchange, SymbolSpiderJob } from '../shared/types'; import type { Exchange, SymbolSpiderJob } from '../shared/types';
/** /**
@ -10,7 +12,7 @@ import type { Exchange, SymbolSpiderJob } from '../shared/types';
* Root job (no prefix) creates A-Z jobs * Root job (no prefix) creates A-Z jobs
* Each job searches its prefix and creates child jobs if needed * Each job searches its prefix and creates child jobs if needed
*/ */
export async function spiderSymbolSearch( export async function spiderSymbol(
this: BaseHandler, this: BaseHandler,
input: SymbolSpiderJob, input: SymbolSpiderJob,
_context: ExecutionContext _context: ExecutionContext
@ -22,7 +24,6 @@ export async function spiderSymbolSearch(
const { prefix, depth = 0, maxDepth = 4 } = input || {}; const { prefix, depth = 0, maxDepth = 4 } = input || {};
this.logger.info('Spider symbol search', { prefix, depth, maxDepth }); this.logger.info('Spider symbol search', { prefix, depth, maxDepth });
console.log('Spider symbol search', { prefix, depth, maxDepth });
if (!prefix) { if (!prefix) {
// Root job - create A-Z jobs // Root job - create A-Z jobs
@ -59,15 +60,7 @@ export async function spiderSymbolSearch(
}; };
} }
// Store symbols in MongoDB await this.mongodb.batchUpsert('qm_symbols', symbols, ['qmSearchCode']);
const processedSymbols = symbols.map(symbol => ({
...symbol,
spiderPrefix: prefix,
spiderDepth: depth,
discoveredAt: new Date()
}));
await this.mongodb.batchUpsert('qm_symbols', processedSymbols, ['qmSearchCode']);
this.logger.info('Stored symbols from spider search', { this.logger.info('Stored symbols from spider search', {
prefix, prefix,
@ -101,7 +94,7 @@ export async function spiderSymbolSearch(
// Only create child jobs if we found a significant number of symbols // Only create child jobs if we found a significant number of symbols
// This prevents excessive branching on sparse results // This prevents excessive branching on sparse results
if (symbols.length >= 10) { if (symbols.length >= 50) {
for (let i = 0; i < 26; i++) { for (let i = 0; i < 26; i++) {
const nextPrefix = prefix + String.fromCharCode(65 + i); const nextPrefix = prefix + String.fromCharCode(65 + i);
await this.scheduleOperation('spider-symbols', { await this.scheduleOperation('spider-symbols', {
@ -110,8 +103,7 @@ export async function spiderSymbolSearch(
source: 'qm', source: 'qm',
maxDepth maxDepth
}, { }, {
delay: (i * 2000) + 10000, // Start after 10s, stagger by 2s priority: Math.max(1, 5 - depth)
priority: Math.max(1, 5 - depth) // Lower priority for deeper searches
}); });
jobsCreated++; jobsCreated++;
} }
@ -149,90 +141,73 @@ export async function searchSymbols(
const { query } = input; const { query } = input;
this.logger.debug('Searching QM symbols', { query }); this.logger.debug('Searching QM symbols', { query });
return []
// const sessionManager = QMSessionManager.getInstance(); const sessionManager = QMSessionManager.getInstance();
// sessionManager.initialize(this.cache, this.logger); sessionManager.initialize(this.cache, this.logger);
// // Get a session // Get a session
// const sessionId = QM_SESSION_IDS.LOOKUP; const sessionId = QM_SESSION_IDS.LOOKUP;
// const session = await sessionManager.getSession(sessionId); const session = await sessionManager.getSession(sessionId);
// if (!session || !session.uuid) { if (!session || !session.uuid) {
// throw new Error(`No active session found for QM LOOKUP`); throw new Error(`No active session found for QM LOOKUP`);
// } }
// try { try {
// // Check cache first // Build API request
// const cacheKey = `qm:symbol-search:${query}`; const searchParams = new URLSearchParams({
// const cachedResult = await this.cache?.get(cacheKey); marketType: 'equity',
// if (cachedResult) { pathName: '/demo/portal/company-summary.php',
// this.logger.trace('Using cached symbol search result', { query }); q: query,
// return cachedResult as any[]; qmodTool: 'SmartSymbolLookup',
// } searchType: 'symbol',
showFree: 'false',
showHisa: 'false',
webmasterId: '500'
});
// // Build API request const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`;
// const searchParams = new URLSearchParams({
// marketType: 'equity',
// pathName: '/demo/portal/company-summary.php',
// q: query,
// qmodTool: 'SmartSymbolLookup',
// searchType: 'symbol',
// showFree: 'false',
// showHisa: 'false',
// webmasterId: '500'
// });
// const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`; const response = await fetch(apiUrl, {
method: 'GET',
headers: session.headers,
proxy: session.proxy,
});
// const response = await fetch(apiUrl, { if (!response.ok) {
// method: 'GET', throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
// headers: session.headers, }
// proxy: session.proxy,
// signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT),
// });
// if (!response.ok) { const symbols = await response.json();
// throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
// }
// const symbols = await response.json(); // Update session success stats
await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
// // Update session success stats // Process symbol data
// await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid); const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({
...symbol,
qmSearchCode: symbol.symbol || '',
symbol: (symbol.symbol as string)?.split(':')[0] || '',
})) : [];
// // Process symbol data this.logger.info('QM API returned symbols', {
// const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({ query,
// ...symbol, count: processedSymbols.length
// qmSearchCode: symbol.symbol || '', });
// symbol: (symbol.symbol as string)?.split(':')[0] || '',
// searchQuery: query,
// fetchedAt: new Date()
// })) : [];
// // Cache the result return processedSymbols;
// if (processedSymbols.length > 0) {
// await this.cache?.set(cacheKey, processedSymbols, 1800); // 30 minutes
// }
// this.logger.info('QM API returned symbols', { } catch (error) {
// query, // Update session failure stats
// count: processedSymbols.length if (session.uuid) {
// }); await sessionManager.incrementFailedCalls(sessionId, session.uuid);
}
// return processedSymbols; this.logger.error('Error searching QM symbols', {
query,
error: error instanceof Error ? error.message : 'Unknown error'
});
// } catch (error) { throw error;
// // Update session failure stats }
// if (session.uuid) {
// await sessionManager.incrementFailedCalls(sessionId, session.uuid);
// }
// this.logger.error('Error searching QM symbols', {
// query,
// error: error instanceof Error ? error.message : 'Unknown error'
// });
// throw error;
// }
} }

11
apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts
View file

@ -4,7 +4,7 @@ import {
Operation, Operation,
ScheduledOperation, ScheduledOperation,
} from '@stock-bot/handlers'; } from '@stock-bot/handlers';
import { checkSessions, createSession, searchSymbols, spiderSymbolSearch } from './actions'; import { checkSessions, createSession, searchSymbols, spiderSymbol } from './actions';
@Handler('qm') @Handler('qm')
export class QMHandler extends BaseHandler { export class QMHandler extends BaseHandler {
@ -22,15 +22,12 @@ export class QMHandler extends BaseHandler {
@Operation('create-session') @Operation('create-session')
createSession = createSession; createSession = createSession;
@ScheduledOperation('spider-symbol-search', '* * * * *', { @ScheduledOperation('spider-symbols', '* * * * *', {
priority: 8, priority: 9,
immediately: false, immediately: false,
description: 'Weekly comprehensive symbol search using QM API spider - runs every Saturday at midnight' description: 'Weekly comprehensive symbol search using QM API spider - runs every Saturday at midnight'
}) })
spiderSymbolSchedule = spiderSymbolSearch; spiderSymbol = spiderSymbol;
@Operation('spider-symbols')
spiderSymbolsJob = spiderSymbolSearch;
@Operation('search-symbols') @Operation('search-symbols')
searchSymbols = searchSymbols; searchSymbols = searchSymbols;