standartized OperationTracker. need to test it all out now

This commit is contained in:
Boki 2025-07-01 11:15:33 -04:00
parent f78558224f
commit 680b5fd2ae
21 changed files with 2112 additions and 752 deletions

View file

@ -5,23 +5,8 @@
import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMOperationTracker } from '../shared/operation-tracker';
import { QMSessionManager } from '../shared/session-manager';
// Cache tracker instance
let operationTracker: QMOperationTracker | null = null;
/**
* Get or initialize the operation tracker
*/
async function getOperationTracker(handler: QMHandler): Promise<QMOperationTracker> {
if (!operationTracker) {
const { initializeQMOperations } = await import('../shared/operation-registry');
operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
}
return operationTracker;
}
/**
* Update events (dividends, splits, earnings) for a single symbol
* Single API call returns all three data types
@ -77,7 +62,6 @@ export async function updateEvents(
headers: session.headers,
proxy: session.proxy,
});
const tracker = await getOperationTracker(this);
if (!response.ok) {
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
@ -86,7 +70,7 @@ export async function updateEvents(
const corporateData = await response.json();
const results = corporateData.results;
if (typeof results.error === 'object') {
await tracker.updateSymbolOperation(qmSearchCode, 'events_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'events_update', {
status: 'success',
});
throw new Error(`Invalid response structure from QM API for ${qmSearchCode}`);
@ -151,7 +135,7 @@ export async function updateEvents(
// Update tracking for events
const updateTime = new Date();
await tracker.updateSymbolOperation(qmSearchCode, 'events_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'events_update', {
status: 'success',
lastRecordDate: updateTime,
recordCount: dividendCount + splitCount + earningsCount
@ -187,9 +171,8 @@ export async function updateEvents(
});
// Track failure for events
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'events_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'events_update', {
status: 'failure'
});
@ -217,13 +200,12 @@ export async function scheduleEventsUpdates(
errors: number;
}> {
const { limit = 100000, forceUpdate = false } = input;
const tracker = await getOperationTracker(this);
this.logger.info('Scheduling events updates', { limit, forceUpdate });
try {
// Get symbols that need events updates
const staleSymbols = await tracker.getStaleSymbols('events_update', {
const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'events_update', {
minHoursSinceRun: forceUpdate ? 0 : 24 * 7, // Weekly
limit
});

View file

@ -5,23 +5,8 @@
import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMOperationTracker } from '../shared/operation-tracker';
import { QMSessionManager } from '../shared/session-manager';
// Cache tracker instance
let operationTracker: QMOperationTracker | null = null;
/**
* Get or initialize the operation tracker
*/
async function getOperationTracker(handler: QMHandler): Promise<QMOperationTracker> {
if (!operationTracker) {
const { initializeQMOperations } = await import('../shared/operation-registry');
operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
}
return operationTracker;
}
/**
* Update filings for a single symbol
*/
@ -97,8 +82,7 @@ export async function updateFilings(
);
// Update symbol to track last filings update
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'filings_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'filings_update', {
status: 'success',
lastRecordDate: new Date(),
recordCount: filingsData.length
@ -117,8 +101,7 @@ export async function updateFilings(
};
} else {
// Some symbols may not have filings (non-US companies, etc)
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'filings_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'filings_update', {
status: 'success',
lastRecordDate: new Date(),
recordCount: 0
@ -145,8 +128,7 @@ export async function updateFilings(
});
// Track failure
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'filings_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'filings_update', {
status: 'failure'
});
@ -174,13 +156,12 @@ export async function scheduleFilingsUpdates(
errors: number;
}> {
const { limit = 100, forceUpdate = false } = input;
const tracker = await getOperationTracker(this);
this.logger.info('Scheduling filings updates', { limit, forceUpdate });
try {
// Get symbols that need updating
const staleSymbols = await tracker.getStaleSymbols('filings_update', {
const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'filings_update', {
minHoursSinceRun: forceUpdate ? 0 : 24, // Daily for filings
limit
});

View file

@ -5,23 +5,8 @@
import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMOperationTracker } from '../shared/operation-tracker';
import { QMSessionManager } from '../shared/session-manager';
// Cache tracker instance
let operationTracker: QMOperationTracker | null = null;
/**
* Get or initialize the operation tracker
*/
async function getOperationTracker(handler: QMHandler): Promise<QMOperationTracker> {
if (!operationTracker) {
const { initializeQMOperations } = await import('../shared/operation-registry');
operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
}
return operationTracker;
}
/**
* Update financials for a single symbol
*/
@ -105,8 +90,7 @@ export async function updateFinancials(
);
// Update symbol to track last financials update
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, `financials_${reportType === 'Q'? 'quarterly' : 'annual'}_update`, {
await this.operationRegistry.updateOperation('qm', qmSearchCode, `financials_update_${reportType === 'Q'? 'quarterly' : 'annual'}`, {
status: 'success',
lastRecordDate: new Date(),
recordCount: reports.length
@ -144,8 +128,7 @@ export async function updateFinancials(
});
// Track failure
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'financials_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, `financials_update_${reportType === 'Q'? 'quarterly' : 'annual'}`, {
status: 'failure',
});
@ -173,17 +156,16 @@ export async function scheduleFinancialsUpdates(
errors: number;
}> {
const { limit = 100000, forceUpdate = false } = input;
const tracker = await getOperationTracker(this);
this.logger.info('Scheduling financials updates', { limit, forceUpdate });
try {
// Get symbols that need updating for both quarterly and annual
const staleSymbolsQ = await tracker.getStaleSymbols('financials_quarterly_update', {
const staleSymbolsQ = await this.operationRegistry.getStaleSymbols('qm', 'financials_update_quarterly', {
minHoursSinceRun: forceUpdate ? 0 : 24 * 7, // Weekly by default
limit
});
const staleSymbolsA = await tracker.getStaleSymbols('financials_annual_update', {
const staleSymbolsA = await this.operationRegistry.getStaleSymbols('qm', 'financials_update_annual', {
minHoursSinceRun: forceUpdate ? 0 : 24 * 7, // Weekly by default
limit
});

View file

@ -5,23 +5,8 @@
import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMOperationTracker } from '../shared/operation-tracker';
import { QMSessionManager } from '../shared/session-manager';
// Cache tracker instance
let operationTracker: QMOperationTracker | null = null;
/**
* Get or initialize the operation tracker
*/
async function getOperationTracker(handler: QMHandler): Promise<QMOperationTracker> {
if (!operationTracker) {
const { initializeQMOperations } = await import('../shared/operation-registry');
operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
}
return operationTracker;
}
/**
* Update intraday bars for a single symbol
* This handles both initial crawl and incremental updates
@ -107,8 +92,7 @@ export async function updateIntradayBars(
);
// Update operation tracking
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'intraday_bars', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'intraday_bars', {
status: 'success',
lastRecordDate: targetDate,
recordCount: barsData.length
@ -134,8 +118,7 @@ export async function updateIntradayBars(
this.logger.info('No intraday data for date', { symbol, date: targetDate });
// Still update operation tracking as successful (no data is a valid result)
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'intraday_bars', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'intraday_bars', {
status: 'success',
lastRecordDate: targetDate,
recordCount: 0
@ -164,8 +147,7 @@ export async function updateIntradayBars(
});
// Update operation tracking for failure
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'intraday_bars', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'intraday_bars', {
status: 'failure'
});
@ -196,7 +178,6 @@ export async function scheduleIntradayUpdates(
errors: number;
}> {
const { limit = 50, mode = 'update', forceUpdate = false } = input;
const tracker = await getOperationTracker(this);
this.logger.info('Scheduling intraday updates', { limit, mode, forceUpdate });
@ -205,12 +186,12 @@ export async function scheduleIntradayUpdates(
if (mode === 'crawl') {
// Get symbols that need historical crawl
symbolsToProcess = await tracker.getSymbolsForIntradayCrawl('intraday_bars', {
symbolsToProcess = await this.operationRegistry.getSymbolsForCrawl('qm', 'intraday_bars', {
limit
});
} else {
// Get symbols that need regular updates
const staleSymbols = await tracker.getStaleSymbols('intraday_bars', {
const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'intraday_bars', {
minHoursSinceRun: forceUpdate ? 0 : 1, // Hourly updates
limit
});

View file

@ -5,23 +5,8 @@
import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMOperationTracker } from '../shared/operation-tracker';
import { QMSessionManager } from '../shared/session-manager';
// Cache tracker instance
let operationTracker: QMOperationTracker | null = null;
/**
* Get or initialize the operation tracker
*/
async function getOperationTracker(handler: QMHandler): Promise<QMOperationTracker> {
if (!operationTracker) {
const { initializeQMOperations } = await import('../shared/operation-registry');
operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
}
return operationTracker;
}
/**
* Update daily prices for a single symbol
*/
@ -87,13 +72,12 @@ export async function updatePrices(
// Update session success stats
await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
// Update symbol to track last price update
const tracker = await getOperationTracker(this);
const priceData = responseData.results?.history[0].eoddata || [];
if(!priceData || priceData.length === 0) {
this.logger.warn(`No price data found for symbol ${qmSearchCode}`);
await tracker.updateSymbolOperation(qmSearchCode, 'price_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'price_update', {
status: 'success',
recordCount: priceData.length
});
@ -128,7 +112,7 @@ export async function updatePrices(
);
await tracker.updateSymbolOperation(qmSearchCode, 'price_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'price_update', {
status: 'success',
lastRecordDate: latestDate,
recordCount: priceData.length
@ -170,8 +154,7 @@ export async function updatePrices(
});
// Track failure
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'price_update', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'price_update', {
status: 'failure'
});
@ -199,13 +182,12 @@ export async function schedulePriceUpdates(
errors: number;
}> {
const { limit = 50000, forceUpdate = false } = input;
const tracker = await getOperationTracker(this);
this.logger.info('Scheduling price updates', { limit, forceUpdate });
try {
// Get symbols that need updating
const staleSymbols = await tracker.getStaleSymbols('price_update', {
const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'price_update', {
minHoursSinceRun: forceUpdate ? 0 : 24, // Daily updates
limit
});

View file

@ -5,23 +5,8 @@
import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMOperationTracker } from '../shared/operation-tracker';
import { QMSessionManager } from '../shared/session-manager';
// Cache tracker instance
let operationTracker: QMOperationTracker | null = null;
/**
* Get or initialize the operation tracker
*/
async function getOperationTracker(handler: QMHandler): Promise<QMOperationTracker> {
if (!operationTracker) {
const { initializeQMOperations } = await import('../shared/operation-registry');
operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
}
return operationTracker;
}
/**
* Update symbol info for a single symbol
* This is a simple API fetch operation - no tracking logic here
@ -98,8 +83,7 @@ export async function updateSymbolInfo(
);
// Update operation tracking
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'symbol_info', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'symbol_info', {
status: 'success',
lastRecordDate: new Date()
});
@ -134,8 +118,7 @@ export async function updateSymbolInfo(
});
// Update operation tracking for failure
const tracker = await getOperationTracker(this);
await tracker.updateSymbolOperation(qmSearchCode, 'symbol_info', {
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'symbol_info', {
status: 'failure'
});
@ -164,13 +147,12 @@ export async function scheduleSymbolInfoUpdates(
errors: number;
}> {
const { limit = 100000, forceUpdate = false } = input;
const tracker = await getOperationTracker(this);
this.logger.info('Scheduling symbol info updates', { limit, forceUpdate });
try {
// Get symbols that need updating
const staleSymbols = await tracker.getStaleSymbols('symbol_info', {
const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'symbol_info', {
minHoursSinceRun: forceUpdate ? 0 : 24 * 7, // Weekly by default
limit
});

View file

@ -6,6 +6,8 @@ import {
ScheduledOperation,
} from '@stock-bot/handlers';
import type { DataIngestionServices } from '../../types';
import type { OperationRegistry } from '../../shared/operation-manager';
import { createQMOperationRegistry } from './shared/operation-provider';
import {
checkSessions,
createSession,
@ -27,16 +29,22 @@ import {
updatePrices,
updateSymbolInfo
} from './actions';
import { initializeQMOperations } from './shared/operation-registry';
@Handler('qm')
export class QMHandler extends BaseHandler<DataIngestionServices> {
public operationRegistry: OperationRegistry;
constructor(services: any) {
super(services); // Handler name read from @Handler decorator
// Initialize operations after super() so services are available
initializeQMOperations(this.mongodb, this.logger).catch(error => {
this.logger.error('Failed to initialize QM operations', { error });
});
// Initialize operation registry with QM provider
createQMOperationRegistry(this.mongodb, this.logger)
.then(registry => {
this.operationRegistry = registry;
})
.catch(error => {
this.logger.error('Failed to initialize QM operations', { error });
});
}
/**
@ -168,4 +176,4 @@ export class QMHandler extends BaseHandler<DataIngestionServices> {
description: 'Check for symbols needing intraday updates every 30 minutes'
})
scheduleIntradayUpdates = scheduleIntradayUpdates;
}
}

View file

@ -3,4 +3,4 @@ export * from './session-manager';
export * from './session-manager-redis';
export * from './session-manager-wrapper';
export * from './types';
export * from './operation-tracker';
export * from './operation-provider';

View file

@ -0,0 +1,94 @@
/**
* QM Operation Provider - Defines operations for QuoteMedia data source
*/
import { BaseOperationProvider, OperationRegistry, type OperationConfig, type ProviderConfig } from '../../../shared/operation-manager';
/**
* QM operation definitions
*/
export const QM_OPERATIONS: OperationConfig[] = [
// Symbol metadata
{
name: 'symbol_info',
type: 'standard',
description: 'Update symbol metadata',
defaultStaleHours: 24 * 7 // Weekly
},
// Price data
{
name: 'price_update',
type: 'standard',
description: 'Update daily price data',
defaultStaleHours: 24
},
{
name: 'intraday_bars',
type: 'intraday_crawl',
description: 'Crawl intraday price bars from today backwards',
requiresFinishedFlag: true,
defaultStaleHours: 1 // Check every hour for new data
},
// Fundamental data
{
name: 'financials_update_quarterly',
type: 'standard',
description: 'Update quarterly financial statements',
defaultStaleHours: 24 * 7 // Weekly
},
{
name: 'financials_update_annual',
type: 'standard',
description: 'Update annual financial statements',
defaultStaleHours: 24 * 7 // Weekly
},
// Corporate actions
{
name: 'events_update',
type: 'standard',
description: 'Update events (earnings, dividends, splits)',
defaultStaleHours: 24 * 7 // Weekly
},
// Filings
{
name: 'filings_update',
type: 'standard',
description: 'Update SEC filings',
defaultStaleHours: 24 // Daily
}
];
/**
* QM Operation Provider
*/
export class QMOperationProvider extends BaseOperationProvider {
getProviderConfig(): ProviderConfig {
return {
name: 'qm',
collectionName: 'qmSymbols',
symbolField: 'qmSearchCode',
description: 'QuoteMedia data provider'
};
}
getOperations(): OperationConfig[] {
return QM_OPERATIONS;
}
}
/**
* Create and initialize QM operation registry
*/
export async function createQMOperationRegistry(
mongodb: any,
logger: any
): Promise<OperationRegistry> {
const registry = new OperationRegistry({ mongodb, logger });
const provider = new QMOperationProvider({ mongodb, logger });
await registry.registerProvider(provider);
return registry;
}

View file

@ -1,110 +0,0 @@
/**
* QM Operation Registry - Define and register all QM operations
*/
import type { Logger, MongoDBClient } from '@stock-bot/types';
import { QMOperationTracker } from './operation-tracker';
import type { QMOperationConfig } from './types';
// Define all QM operations
export const QM_OPERATIONS: QMOperationConfig[] = [
// Price data operations
{
name: 'symbol_info',
type: 'standard',
description: 'Update symbol metadata',
defaultStaleHours: 24 * 7 // Weekly
},
{
name: 'price_update',
type: 'standard',
description: 'Update daily price data',
defaultStaleHours: 24
},
{
name: 'intraday_bars',
type: 'intraday_crawl',
description: 'Crawl intraday price bars from today backwards',
requiresFinishedFlag: true,
defaultStaleHours: 1 // Check every hour for new data
},
// Fundamental data operations
{
name: 'financials_update_quarterly',
type: 'standard',
description: 'Update quarterly financial statements',
defaultStaleHours: 24 * 7 // Weekly
},
{
name: 'financials_update_annual',
type: 'standard',
description: 'Update annual financial statements',
defaultStaleHours: 24 * 7 // Weekly
},
// Corporate actions - fetched together in one API call
{
name: 'events_update',
type: 'standard',
description: 'Update events (earnings, dividends, splits)',
defaultStaleHours: 24 * 7 // Weekly
},
// News and filings
{
name: 'filings_update',
type: 'standard',
description: 'Update SEC filings',
defaultStaleHours: 24 // Daily
},
// {
// name: 'news_update',
// type: 'standard',
// description: 'Update news articles',
// defaultStaleHours: 6 // Every 6 hours
// },
// // Options data
// {
// name: 'options_chain',
// type: 'standard',
// description: 'Update options chain data',
// defaultStaleHours: 1 // Hourly during market hours
// }
];
/**
* Initialize operation tracker with all registered operations
*/
export async function initializeQMOperations(
mongodb: MongoDBClient,
logger: Logger
): Promise<QMOperationTracker> {
logger.info('Initializing QM operations tracker');
const tracker = new QMOperationTracker(mongodb, logger);
// Register all operations
for (const operation of QM_OPERATIONS) {
try {
await tracker.registerOperation(operation);
logger.debug(`Registered operation: ${operation.name}`);
} catch (error) {
logger.error(`Failed to register operation: ${operation.name}`, { error });
throw error;
}
}
logger.info('QM operations tracker initialized', {
operationCount: QM_OPERATIONS.length
});
return tracker;
}
/**
* Get operation configuration by name
*/
export function getOperationConfig(name: string): QMOperationConfig | undefined {
return QM_OPERATIONS.find(op => op.name === name);
}

View file

@ -1,458 +0,0 @@
/**
* QM Operation Tracker - Tracks operation execution times and states for symbols
* Supports dynamic operation registration with auto-indexing
*/
import type { Logger, MongoDBClient } from '@stock-bot/types';
import type { IntradayCrawlSymbol, QMOperationConfig } from './types';
export class QMOperationTracker {
private registeredOperations: Map<string, QMOperationConfig> = new Map();
private indexesCreated: Set<string> = new Set();
private mongodb: MongoDBClient;
private logger: Logger;
private readonly collectionName = 'qmSymbols';
constructor(mongodb: MongoDBClient, logger: Logger) {
this.mongodb = mongodb;
this.logger = logger;
}
/**
* Register a new operation type with auto-indexing
*/
async registerOperation(config: QMOperationConfig): Promise<void> {
this.logger.info('Registering QM operation', { operation: config.name, type: config.type });
this.registeredOperations.set(config.name, config);
// Auto-create indexes for this operation
await this.createOperationIndexes(config.name);
this.logger.debug('Operation registered successfully', { operation: config.name });
}
/**
* Create indexes for efficient operation queries
*/
private async createOperationIndexes(operationName: string): Promise<void> {
if (this.indexesCreated.has(operationName)) {
this.logger.debug('Indexes already created for operation', { operation: operationName });
return;
}
try {
const indexes = [
// Index for finding stale symbols
{ [`operations.${operationName}.lastRunAt`]: 1, qmSearchCode: 1 },
// Index for finding by last record date
{ [`operations.${operationName}.lastRecordDate`]: 1, qmSearchCode: 1 },
];
// Add crawl state index for intraday operations
const config = this.registeredOperations.get(operationName);
if (config?.type === 'intraday_crawl') {
indexes.push({ [`operations.${operationName}.crawlState.finished`]: 1, qmSearchCode: 1 });
}
for (const indexSpec of indexes) {
const collection = this.mongodb.collection(this.collectionName);
await collection.createIndex(indexSpec, {
background: true,
name: `op_${operationName}_${Object.keys(indexSpec).join('_')}`
});
}
this.indexesCreated.add(operationName);
this.logger.info('Created indexes for operation', {
operation: operationName,
indexCount: indexes.length
});
} catch (error) {
this.logger.error('Failed to create indexes for operation', {
operation: operationName,
error
});
throw error;
}
}
/**
* Update symbol operation status
*/
async updateSymbolOperation(
qmSearchCode: string,
operationName: string,
data: {
status: 'success' | 'failure' | 'partial';
lastRecordDate?: Date;
recordCount?: number;
crawlState?: {
finished?: boolean;
oldestDateReached?: Date;
};
}
): Promise<void> {
const update: any = {
$set: {
[`operations.${operationName}.lastRunAt`]: new Date(),
[`operations.${operationName}.status`]: data.status,
updated_at: new Date()
}
};
// Only update lastSuccessAt on successful operations
if (data.status === 'success') {
update.$set[`operations.${operationName}.lastSuccessAt`] = new Date();
}
if (data.lastRecordDate) {
update.$set[`operations.${operationName}.lastRecordDate`] = data.lastRecordDate;
}
if (data.recordCount !== undefined) {
update.$set[`operations.${operationName}.recordCount`] = data.recordCount;
}
if (data.crawlState) {
update.$set[`operations.${operationName}.crawlState`] = {
...data.crawlState,
lastCrawlDirection: data.crawlState.finished ? 'forward' : 'backward'
};
}
await this.mongodb.updateOne(this.collectionName, { qmSearchCode }, update);
this.logger.debug('Updated symbol operation', {
qmSearchCode,
operation: operationName,
status: data.status
});
}
/**
* Bulk update symbol operations for performance
*/
async bulkUpdateSymbolOperations(
updates: Array<{
qmSearchCode: string;
operation: string;
data: {
status: 'success' | 'failure' | 'partial';
lastRecordDate?: Date;
recordCount?: number;
crawlState?: any;
};
}>
): Promise<void> {
if (updates.length === 0) {return;}
const bulkOps = updates.map(({ qmSearchCode, operation, data }) => {
const update: any = {
$set: {
[`operations.${operation}.lastRunAt`]: new Date(),
[`operations.${operation}.status`]: data.status,
updated_at: new Date()
}
};
// Only update lastSuccessAt on successful operations
if (data.status === 'success') {
update.$set[`operations.${operation}.lastSuccessAt`] = new Date();
}
if (data.lastRecordDate) {
update.$set[`operations.${operation}.lastRecordDate`] = data.lastRecordDate;
}
if (data.recordCount !== undefined) {
update.$set[`operations.${operation}.recordCount`] = data.recordCount;
}
if (data.crawlState) {
update.$set[`operations.${operation}.crawlState`] = {
...data.crawlState,
lastCrawlDirection: data.crawlState.finished ? 'forward' : 'backward'
};
}
return {
updateOne: {
filter: { qmSearchCode },
update
}
};
});
const collection = this.mongodb.collection(this.collectionName);
const result = await collection.bulkWrite(bulkOps as any, { ordered: false });
this.logger.debug('Bulk updated symbol operations', {
totalUpdates: updates.length,
modified: result.modifiedCount,
operations: Array.from(new Set(updates.map(u => u.operation)))
});
}
/**
* Get symbols that need processing for an operation
*/
async getStaleSymbols(
operationName: string,
options: {
notRunSince?: Date;
minHoursSinceRun?: number;
limit?: number;
excludeSymbols?: string[];
} = {}
): Promise<string[]> {
const { limit = 1000, excludeSymbols = [] } = options;
const cutoffDate = options.notRunSince || (() => {
const date = new Date();
const hours = options.minHoursSinceRun ||
this.registeredOperations.get(operationName)?.defaultStaleHours || 24;
date.setHours(date.getHours() - hours);
return date;
})();
const filter: any = {
active: { $ne: false }, // Only active symbols (active: true or active doesn't exist)
$or: [
{ [`operations.${operationName}.lastSuccessAt`]: { $lt: cutoffDate } },
{ [`operations.${operationName}.lastSuccessAt`]: { $exists: false } },
{ [`operations.${operationName}`]: { $exists: false } }
]
};
if (excludeSymbols.length > 0) {
filter.qmSearchCode = { $nin: excludeSymbols };
}
const symbols = await this.mongodb.find(this.collectionName, filter, {
limit,
projection: { qmSearchCode: 1 },
sort: { [`operations.${operationName}.lastSuccessAt`]: 1 } // Oldest successful run first
});
return symbols.map(s => s.qmSearchCode);
}
/**
* Get symbols for intraday crawling
*/
async getSymbolsForIntradayCrawl(
operationName: string,
options: {
limit?: number;
includeFinished?: boolean;
} = {}
): Promise<IntradayCrawlSymbol[]> {
const { limit = 100, includeFinished = false } = options;
const filter: any = {
active: { $ne: false } // Only active symbols
};
if (!includeFinished) {
filter[`operations.${operationName}.crawlState.finished`] = { $ne: true };
}
const symbols = await this.mongodb.find(this.collectionName, filter, {
limit,
projection: {
qmSearchCode: 1,
[`operations.${operationName}`]: 1
},
sort: {
// Prioritize symbols that haven't been crawled yet
[`operations.${operationName}.lastRunAt`]: 1
}
});
return symbols.map(s => ({
qmSearchCode: s.qmSearchCode,
lastRecordDate: s.operations?.[operationName]?.lastRecordDate,
crawlState: s.operations?.[operationName]?.crawlState
}));
}
/**
* Mark intraday crawl as finished
*/
async markCrawlFinished(
qmSearchCode: string,
operationName: string,
oldestDateReached: Date
): Promise<void> {
await this.updateSymbolOperation(qmSearchCode, operationName, {
status: 'success',
crawlState: {
finished: true,
oldestDateReached
}
});
this.logger.info('Marked crawl as finished', {
qmSearchCode,
operation: operationName,
oldestDateReached
});
}
/**
* Get symbols that need data updates based on last record date
*/
async getSymbolsNeedingUpdate(
operationName: string,
options: {
lastRecordBefore?: Date;
neverRun?: boolean;
limit?: number;
} = {}
): Promise<Array<{ qmSearchCode: string; lastRecordDate?: Date }>> {
const { limit = 500 } = options;
const filter: any = {
active: { $ne: false } // Only active symbols
};
if (options.neverRun) {
filter[`operations.${operationName}`] = { $exists: false };
} else if (options.lastRecordBefore) {
filter.$or = [
{ [`operations.${operationName}.lastRecordDate`]: { $lt: options.lastRecordBefore } },
{ [`operations.${operationName}`]: { $exists: false } }
];
}
const symbols = await this.mongodb.find(this.collectionName, filter, {
limit,
projection: {
qmSearchCode: 1,
[`operations.${operationName}.lastRecordDate`]: 1
},
sort: { [`operations.${operationName}.lastRecordDate`]: 1 } // Oldest data first
});
return symbols.map(s => ({
qmSearchCode: s.qmSearchCode,
lastRecordDate: s.operations?.[operationName]?.lastRecordDate
}));
}
/**
* Get operation statistics
*/
async getOperationStats(operationName: string): Promise<{
totalSymbols: number;
processedSymbols: number;
staleSymbols: number;
successfulSymbols: number;
failedSymbols: number;
finishedCrawls?: number;
avgRecordsPerSymbol?: number;
}> {
const collection = this.mongodb.collection(this.collectionName);
const total = await collection.countDocuments({});
const processed = await collection.countDocuments({
[`operations.${operationName}`]: { $exists: true }
});
const successful = await collection.countDocuments({
[`operations.${operationName}.status`]: 'success'
});
const failed = await collection.countDocuments({
[`operations.${operationName}.status`]: 'failure'
});
const staleDate = new Date();
staleDate.setHours(staleDate.getHours() - (
this.registeredOperations.get(operationName)?.defaultStaleHours || 24
));
const stale = await collection.countDocuments({
$or: [
{ [`operations.${operationName}.lastRunAt`]: { $lt: staleDate } },
{ [`operations.${operationName}`]: { $exists: false } }
]
});
const result: any = {
totalSymbols: total,
processedSymbols: processed,
staleSymbols: stale,
successfulSymbols: successful,
failedSymbols: failed
};
// Additional stats for crawl operations
if (this.registeredOperations.get(operationName)?.type === 'intraday_crawl') {
result.finishedCrawls = await collection.countDocuments({
[`operations.${operationName}.crawlState.finished`]: true
});
}
// Calculate average records per symbol
const aggregation = await collection.aggregate([
{
$match: {
[`operations.${operationName}.recordCount`]: { $exists: true }
}
},
{
$group: {
_id: null,
avgRecords: { $avg: `$operations.${operationName}.recordCount` }
}
}
]).toArray();
if (aggregation.length > 0) {
result.avgRecordsPerSymbol = Math.round(aggregation[0].avgRecords);
}
return result;
}
/**
* Get all registered operations
*/
getRegisteredOperations(): QMOperationConfig[] {
return Array.from(this.registeredOperations.values());
}
/**
* Helper: Get symbols for price update
*/
async getSymbolsForPriceUpdate(limit = 1000): Promise<string[]> {
return this.getStaleSymbols('price_update', {
minHoursSinceRun: 24,
limit
});
}
/**
* Helper: Get symbols with outdated financials
*/
async getSymbolsWithOldFinancials(limit = 100): Promise<Array<{ qmSearchCode: string; lastRecordDate?: Date }>> {
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - 90); // 90 days old
return this.getSymbolsNeedingUpdate('financials_update', {
lastRecordBefore: cutoffDate,
limit
});
}
/**
* Helper: Get unprocessed symbols for an operation
*/
async getUnprocessedSymbols(operation: string, limit = 500): Promise<string[]> {
const symbols = await this.getSymbolsNeedingUpdate(operation, {
neverRun: true,
limit
});
return symbols.map(s => s.qmSearchCode);
}
}

View file

@ -1,5 +1,5 @@
/**
* Shared types for QM operations
* Shared types for QM handler
*/
export interface QMSession {
@ -58,49 +58,8 @@ export interface CachedSession {
sessionType: string;
}
/**
* Operation tracking types
*/
export interface QMOperationConfig {
name: string;
type: 'standard' | 'intraday_crawl';
description?: string;
defaultStaleHours?: number;
requiresFinishedFlag?: boolean;
}
export interface QMSymbolOperationStatus {
symbol: string;
qmSearchCode: string;
operations: {
[operationName: string]: {
lastRunAt: Date;
lastRecordDate?: Date;
status: 'success' | 'failure' | 'partial';
recordCount?: number;
// For intraday crawling operations
crawlState?: {
finished: boolean;
oldestDateReached?: Date;
lastCrawlDirection?: 'forward' | 'backward';
};
};
};
updated_at: Date;
}
export interface IntradayCrawlSymbol {
qmSearchCode: string;
lastRecordDate?: Date;
crawlState?: {
finished: boolean;
oldestDateReached?: Date;
lastCrawlDirection?: 'forward' | 'backward';
};
}
export interface ExchangeStats {
symbolCount: number;
totalMarketCap: number;
avgMarketCap: number;
}
}