boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

work on qm symbols

Browse source
This commit is contained in:
Boki 2025-06-27 21:14:14 -04:00
parent 1d62343051
commit 34671ea427
4 changed files with 68 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

4
apps/stock/config/config/default.json
View file

@ -86,8 +86,8 @@
"type": "exponential",
"delay": 1000
},
"removeOnComplete": 50000,
"removeOnFail": 50000,
"removeOnComplete": 100,
"removeOnFail": 100,
"timeout": 300000
}
},

3
apps/stock/data-ingestion/src/handlers/qm/actions/index.ts
View file

@ -3,4 +3,5 @@
*/
export { checkSessions, createSession } from './session.action';
export { spiderSymbolSearch, searchSymbols } from './symbol.action';
export { searchSymbols, spiderSymbol } from './symbol.action';

145
apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts
View file

@ -3,6 +3,8 @@
*/
import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMSessionManager } from '../shared/session-manager';
import type { Exchange, SymbolSpiderJob } from '../shared/types';
/**
@ -10,7 +12,7 @@ import type { Exchange, SymbolSpiderJob } from '../shared/types';
* Root job (no prefix) creates A-Z jobs
* Each job searches its prefix and creates child jobs if needed
*/
export async function spiderSymbolSearch(
export async function spiderSymbol(
this: BaseHandler,
input: SymbolSpiderJob,
_context: ExecutionContext
@ -22,7 +24,6 @@ export async function spiderSymbolSearch(
const { prefix, depth = 0, maxDepth = 4 } = input || {};
this.logger.info('Spider symbol search', { prefix, depth, maxDepth });
console.log('Spider symbol search', { prefix, depth, maxDepth });
if (!prefix) {
// Root job - create A-Z jobs
@ -59,15 +60,7 @@ export async function spiderSymbolSearch(
};
}
// Store symbols in MongoDB
const processedSymbols = symbols.map(symbol => ({
...symbol,
spiderPrefix: prefix,
spiderDepth: depth,
discoveredAt: new Date()
}));
await this.mongodb.batchUpsert('qm_symbols', processedSymbols, ['qmSearchCode']);
await this.mongodb.batchUpsert('qm_symbols', symbols, ['qmSearchCode']);
this.logger.info('Stored symbols from spider search', {
prefix,
@ -101,7 +94,7 @@ export async function spiderSymbolSearch(
// Only create child jobs if we found a significant number of symbols
// This prevents excessive branching on sparse results
if (symbols.length >= 10) {
if (symbols.length >= 50) {
for (let i = 0; i < 26; i++) {
const nextPrefix = prefix + String.fromCharCode(65 + i);
await this.scheduleOperation('spider-symbols', {
@ -110,8 +103,7 @@ export async function spiderSymbolSearch(
source: 'qm',
maxDepth
}, {
delay: (i * 2000) + 10000, // Start after 10s, stagger by 2s
priority: Math.max(1, 5 - depth) // Lower priority for deeper searches
priority: Math.max(1, 5 - depth)
});
jobsCreated++;
}
@ -149,90 +141,73 @@ export async function searchSymbols(
const { query } = input;
this.logger.debug('Searching QM symbols', { query });
return []
// const sessionManager = QMSessionManager.getInstance();
// sessionManager.initialize(this.cache, this.logger);
const sessionManager = QMSessionManager.getInstance();
sessionManager.initialize(this.cache, this.logger);
// // Get a session
// const sessionId = QM_SESSION_IDS.LOOKUP;
// const session = await sessionManager.getSession(sessionId);
// Get a session
const sessionId = QM_SESSION_IDS.LOOKUP;
const session = await sessionManager.getSession(sessionId);
// if (!session || !session.uuid) {
// throw new Error(`No active session found for QM LOOKUP`);
// }
if (!session || !session.uuid) {
throw new Error(`No active session found for QM LOOKUP`);
}
// try {
// // Check cache first
// const cacheKey = `qm:symbol-search:${query}`;
// const cachedResult = await this.cache?.get(cacheKey);
// if (cachedResult) {
// this.logger.trace('Using cached symbol search result', { query });
// return cachedResult as any[];
// }
try {
// Build API request
const searchParams = new URLSearchParams({
marketType: 'equity',
pathName: '/demo/portal/company-summary.php',
q: query,
qmodTool: 'SmartSymbolLookup',
searchType: 'symbol',
showFree: 'false',
showHisa: 'false',
webmasterId: '500'
});
// // Build API request
// const searchParams = new URLSearchParams({
// marketType: 'equity',
// pathName: '/demo/portal/company-summary.php',
// q: query,
// qmodTool: 'SmartSymbolLookup',
// searchType: 'symbol',
// showFree: 'false',
// showHisa: 'false',
// webmasterId: '500'
// });
const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`;
// const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`;
const response = await fetch(apiUrl, {
method: 'GET',
headers: session.headers,
proxy: session.proxy,
});
// const response = await fetch(apiUrl, {
// method: 'GET',
// headers: session.headers,
// proxy: session.proxy,
// signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT),
// });
if (!response.ok) {
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
}
// if (!response.ok) {
// throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
// }
const symbols = await response.json();
// const symbols = await response.json();
// Update session success stats
await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
// // Update session success stats
// await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
// Process symbol data
const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({
...symbol,
qmSearchCode: symbol.symbol || '',
symbol: (symbol.symbol as string)?.split(':')[0] || '',
})) : [];
// // Process symbol data
// const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({
// ...symbol,
// qmSearchCode: symbol.symbol || '',
// symbol: (symbol.symbol as string)?.split(':')[0] || '',
// searchQuery: query,
// fetchedAt: new Date()
// })) : [];
this.logger.info('QM API returned symbols', {
query,
count: processedSymbols.length
});
// // Cache the result
// if (processedSymbols.length > 0) {
// await this.cache?.set(cacheKey, processedSymbols, 1800); // 30 minutes
// }
return processedSymbols;
// this.logger.info('QM API returned symbols', {
// query,
// count: processedSymbols.length
// });
} catch (error) {
// Update session failure stats
if (session.uuid) {
await sessionManager.incrementFailedCalls(sessionId, session.uuid);
}
// return processedSymbols;
this.logger.error('Error searching QM symbols', {
query,
error: error instanceof Error ? error.message : 'Unknown error'
});
// } catch (error) {
// // Update session failure stats
// if (session.uuid) {
// await sessionManager.incrementFailedCalls(sessionId, session.uuid);
// }
// this.logger.error('Error searching QM symbols', {
// query,
// error: error instanceof Error ? error.message : 'Unknown error'
// });
// throw error;
// }
throw error;
}
}

11
apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts
View file

@ -4,7 +4,7 @@ import {
Operation,
ScheduledOperation,
} from '@stock-bot/handlers';
import { checkSessions, createSession, searchSymbols, spiderSymbolSearch } from './actions';
import { checkSessions, createSession, searchSymbols, spiderSymbol } from './actions';
@Handler('qm')
export class QMHandler extends BaseHandler {
@ -22,15 +22,12 @@ export class QMHandler extends BaseHandler {
@Operation('create-session')
createSession = createSession;
@ScheduledOperation('spider-symbol-search', '* * * * *', {
priority: 8,
@ScheduledOperation('spider-symbols', '* * * * *', {
priority: 9,
immediately: false,
description: 'Weekly comprehensive symbol search using QM API spider - runs every Saturday at midnight'
})
spiderSymbolSchedule = spiderSymbolSearch;
@Operation('spider-symbols')
spiderSymbolsJob = spiderSymbolSearch;
spiderSymbol = spiderSymbol;
@Operation('search-symbols')
searchSymbols = searchSymbols;