boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

qm fully refactored and ready for more

Browse source
This commit is contained in:
Boki 2025-06-21 10:49:38 -04:00
parent ab0b7a5385
commit 24fda247aa
5 changed files with 36 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

7
apps/data-service/src/handlers/qm/operations/exchanges.operations.ts
View file

@ -3,12 +3,11 @@
*/
import { OperationContext } from '@stock-bot/utils';
import type { Logger } from '@stock-bot/logger';
import { initializeQMResources } from './session.operations';
export async function fetchExchanges(parentLogger?: Logger): Promise<unknown[] | null> {
const ctx = OperationContext.create('qm', 'exchanges', parentLogger);
export async function fetchExchanges(): Promise<unknown[] | null> {
const ctx = OperationContext.create('qm', 'exchanges');
try {
// Ensure resources are initialized
@ -16,7 +15,7 @@ export async function fetchExchanges(parentLogger?: Logger): Promise<unknown[] |
const sessionManager = QMSessionManager.getInstance();
if (!sessionManager.getInitialized()) {
await initializeQMResources(parentLogger);
await initializeQMResources();
}
ctx.logger.info('QM exchanges fetch - not implemented yet');

11
apps/data-service/src/handlers/qm/operations/session.operations.ts
View file

@ -5,14 +5,13 @@
import { OperationContext } from '@stock-bot/utils';
import { isShutdownSignalReceived } from '@stock-bot/shutdown';
import { getRandomProxy } from '@stock-bot/utils';
import type { Logger } from '@stock-bot/logger';
import { QMSessionManager } from '../shared/session-manager';
import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG, getQmHeaders } from '../shared/config';
import type { QMSession } from '../shared/types';
export async function createSessions(parentLogger?: Logger): Promise<void> {
const ctx = OperationContext.create('qm', 'session', parentLogger);
export async function createSessions(): Promise<void> {
const ctx = OperationContext.create('qm', 'session');
try {
ctx.logger.info('Creating QM sessions...');
@ -22,7 +21,7 @@ export async function createSessions(parentLogger?: Logger): Promise<void> {
// Check if already initialized
if (!sessionManager.getInitialized()) {
await initializeQMResources(parentLogger);
await initializeQMResources();
}
// Clean up failed sessions first
@ -162,8 +161,8 @@ async function createSingleSession(
}
}
export async function initializeQMResources(parentLogger?: Logger): Promise<void> {
const ctx = OperationContext.create('qm', 'init', parentLogger);
export async function initializeQMResources(): Promise<void> {
const ctx = OperationContext.create('qm', 'init');
// Check if already initialized
const alreadyInitialized = await ctx.cache.get('initialized');

36
apps/data-service/src/handlers/qm/operations/spider.operations.ts
View file

@ -4,7 +4,6 @@
import { OperationContext } from '@stock-bot/utils';
import { QueueManager } from '@stock-bot/queue';
import type { Logger } from '@stock-bot/logger';
import { QMSessionManager } from '../shared/session-manager';
import { QM_SESSION_IDS } from '../shared/config';
@ -13,10 +12,9 @@ import { initializeQMResources } from './session.operations';
import { searchQMSymbolsAPI } from './symbols.operations';
export async function spiderSymbolSearch(
payload: SymbolSpiderJob,
parentLogger?: Logger
payload: SymbolSpiderJob
): Promise<SpiderResult> {
const ctx = OperationContext.create('qm', 'spider', parentLogger);
const ctx = OperationContext.create('qm', 'spider');
try {
const { prefix, depth, source = 'qm', maxDepth = 4 } = payload;
@ -39,7 +37,7 @@ export async function spiderSymbolSearch(
// Ensure resources are initialized
const sessionManager = QMSessionManager.getInstance();
if (!sessionManager.getInitialized()) {
await initializeQMResources(parentLogger);
await initializeQMResources();
}
let result: SpiderResult;
@ -55,16 +53,20 @@ export async function spiderSymbolSearch(
// Cache the result
await ctx.cache.set(cacheKey, result, { ttl: 3600 });
// Store spider operation metrics in PostgreSQL
if (ctx.postgres) {
try {
await ctx.postgres.query(
'INSERT INTO spider_stats (handler, operation, prefix, depth, symbols_found, jobs_created, search_time) VALUES ($1, $2, $3, $4, $5, $6, $7)',
['qm', 'spider', prefix || 'ROOT', depth, result.symbolsFound, result.jobsCreated, new Date()]
);
} catch (error) {
ctx.logger.warn('Failed to store spider stats in PostgreSQL', { error });
}
// Store spider operation metrics in cache instead of PostgreSQL for now
try {
const statsKey = `spider-stats:${prefix || 'ROOT'}:${depth}:${Date.now()}`;
await ctx.cache.set(statsKey, {
handler: 'qm',
operation: 'spider',
prefix: prefix || 'ROOT',
depth,
symbolsFound: result.symbolsFound,
jobsCreated: result.jobsCreated,
searchTime: new Date().toISOString()
}, { ttl: 86400 }); // Keep for 24 hours
} catch (error) {
ctx.logger.debug('Failed to store spider stats in cache', { error });
}
ctx.logger.info('Spider search completed', {
@ -162,14 +164,14 @@ async function searchAndSpawnJobs(
if (!lookupSession) {
ctx.logger.info('No lookup sessions available, creating sessions first...');
const { createSessions } = await import('./session.operations');
await createSessions(ctx.logger);
await createSessions();
// Wait a bit for session creation
await new Promise(resolve => setTimeout(resolve, 1000));
}
// Search for symbols with this prefix
const symbols = await searchQMSymbolsAPI(prefix, ctx.logger);
const symbols = await searchQMSymbolsAPI(prefix);
const symbolCount = symbols.length;
ctx.logger.info(`Prefix "${prefix}" returned ${symbolCount} symbols`);

13
apps/data-service/src/handlers/qm/operations/symbols.operations.ts
View file

@ -4,7 +4,6 @@
import { OperationContext } from '@stock-bot/utils';
import { getRandomProxy } from '@stock-bot/utils';
import type { Logger } from '@stock-bot/logger';
import { QMSessionManager } from '../shared/session-manager';
import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG } from '../shared/config';
@ -12,13 +11,13 @@ import type { SymbolSpiderJob, Exchange } from '../shared/types';
import { initializeQMResources } from './session.operations';
import { spiderSymbolSearch } from './spider.operations';
export async function fetchSymbols(parentLogger?: Logger): Promise<unknown[] | null> {
const ctx = OperationContext.create('qm', 'symbols', parentLogger);
export async function fetchSymbols(): Promise<unknown[] | null> {
const ctx = OperationContext.create('qm', 'symbols');
try {
const sessionManager = QMSessionManager.getInstance();
if (!sessionManager.getInitialized()) {
await initializeQMResources(parentLogger);
await initializeQMResources();
}
ctx.logger.info('Starting QM spider-based symbol search...');
@ -37,7 +36,7 @@ export async function fetchSymbols(parentLogger?: Logger): Promise<unknown[] | n
maxDepth: 4,
};
const result = await spiderSymbolSearch(rootJob, parentLogger);
const result = await spiderSymbolSearch(rootJob);
if (result.success) {
// Cache successful fetch info
@ -61,8 +60,8 @@ export async function fetchSymbols(parentLogger?: Logger): Promise<unknown[] | n
}
}
export async function searchQMSymbolsAPI(query: string, parentLogger?: Logger): Promise<any[]> {
const ctx = OperationContext.create('qm', 'api-search', parentLogger);
export async function searchQMSymbolsAPI(query: string): Promise<any[]> {
const ctx = OperationContext.create('qm', 'api-search');
const proxyInfo = await getRandomProxy();
if (!proxyInfo) {

17
apps/data-service/src/handlers/qm/qm.handler.ts
View file

@ -16,19 +16,15 @@ export function initializeQMProvider() {
name: 'qm',
operations: {
'create-sessions': createJobHandler(async () => {
handlerLogger.debug('Creating QM sessions...');
const { createSessions } = await import('./operations/session.operations');
await createSessions(handlerLogger);
handlerLogger.debug('QM sessions created successfully');
await createSessions();
return { success: true, message: 'QM sessions created successfully' };
}),
'search-symbols': createJobHandler(async () => {
handlerLogger.info('Starting QM symbol search...');
const { fetchSymbols } = await import('./operations/symbols.operations');
const symbols = await fetchSymbols(handlerLogger);
const symbols = await fetchSymbols();
if (symbols && symbols.length > 0) {
handlerLogger.info('QM symbol search completed successfully', { count: symbols.length });
return {
success: true,
message: 'QM symbol search completed successfully',
@ -36,7 +32,6 @@ export function initializeQMProvider() {
symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
};
} else {
handlerLogger.warn('QM symbol search returned no results');
return {
success: false,
message: 'No symbols found',
@ -45,14 +40,8 @@ export function initializeQMProvider() {
}
}),
'spider-symbol-search': createJobHandler(async (payload: SymbolSpiderJob) => {
handlerLogger.debug('Processing spider symbol search job', { payload });
const { spiderSymbolSearch } = await import('./operations/spider.operations');
const result = await spiderSymbolSearch(payload, handlerLogger);
handlerLogger.debug('Spider search job completed', {
success: result.success,
symbolsFound: result.symbolsFound,
});
const result = await spiderSymbolSearch(payload);
return result;
}),