standartized OperationTracker. need to test it all out now
This commit is contained in:
parent
f78558224f
commit
680b5fd2ae
21 changed files with 2112 additions and 752 deletions
|
|
@ -3,4 +3,4 @@ export * from './session-manager';
|
|||
export * from './session-manager-redis';
|
||||
export * from './session-manager-wrapper';
|
||||
export * from './types';
|
||||
export * from './operation-tracker';
|
||||
export * from './operation-provider';
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* QM Operation Provider - Defines operations for QuoteMedia data source
|
||||
*/
|
||||
|
||||
import { BaseOperationProvider, OperationRegistry, type OperationConfig, type ProviderConfig } from '../../../shared/operation-manager';
|
||||
|
||||
/**
|
||||
* QM operation definitions
|
||||
*/
|
||||
export const QM_OPERATIONS: OperationConfig[] = [
|
||||
// Symbol metadata
|
||||
{
|
||||
name: 'symbol_info',
|
||||
type: 'standard',
|
||||
description: 'Update symbol metadata',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
|
||||
// Price data
|
||||
{
|
||||
name: 'price_update',
|
||||
type: 'standard',
|
||||
description: 'Update daily price data',
|
||||
defaultStaleHours: 24
|
||||
},
|
||||
{
|
||||
name: 'intraday_bars',
|
||||
type: 'intraday_crawl',
|
||||
description: 'Crawl intraday price bars from today backwards',
|
||||
requiresFinishedFlag: true,
|
||||
defaultStaleHours: 1 // Check every hour for new data
|
||||
},
|
||||
|
||||
// Fundamental data
|
||||
{
|
||||
name: 'financials_update_quarterly',
|
||||
type: 'standard',
|
||||
description: 'Update quarterly financial statements',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
{
|
||||
name: 'financials_update_annual',
|
||||
type: 'standard',
|
||||
description: 'Update annual financial statements',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
|
||||
// Corporate actions
|
||||
{
|
||||
name: 'events_update',
|
||||
type: 'standard',
|
||||
description: 'Update events (earnings, dividends, splits)',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
|
||||
// Filings
|
||||
{
|
||||
name: 'filings_update',
|
||||
type: 'standard',
|
||||
description: 'Update SEC filings',
|
||||
defaultStaleHours: 24 // Daily
|
||||
}
|
||||
];
|
||||
|
||||
/**
|
||||
* QM Operation Provider
|
||||
*/
|
||||
export class QMOperationProvider extends BaseOperationProvider {
|
||||
getProviderConfig(): ProviderConfig {
|
||||
return {
|
||||
name: 'qm',
|
||||
collectionName: 'qmSymbols',
|
||||
symbolField: 'qmSearchCode',
|
||||
description: 'QuoteMedia data provider'
|
||||
};
|
||||
}
|
||||
|
||||
getOperations(): OperationConfig[] {
|
||||
return QM_OPERATIONS;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and initialize QM operation registry
|
||||
*/
|
||||
export async function createQMOperationRegistry(
|
||||
mongodb: any,
|
||||
logger: any
|
||||
): Promise<OperationRegistry> {
|
||||
const registry = new OperationRegistry({ mongodb, logger });
|
||||
const provider = new QMOperationProvider({ mongodb, logger });
|
||||
await registry.registerProvider(provider);
|
||||
return registry;
|
||||
}
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
/**
|
||||
* QM Operation Registry - Define and register all QM operations
|
||||
*/
|
||||
|
||||
import type { Logger, MongoDBClient } from '@stock-bot/types';
|
||||
import { QMOperationTracker } from './operation-tracker';
|
||||
import type { QMOperationConfig } from './types';
|
||||
|
||||
// Define all QM operations
|
||||
export const QM_OPERATIONS: QMOperationConfig[] = [
|
||||
// Price data operations
|
||||
{
|
||||
name: 'symbol_info',
|
||||
type: 'standard',
|
||||
description: 'Update symbol metadata',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
{
|
||||
name: 'price_update',
|
||||
type: 'standard',
|
||||
description: 'Update daily price data',
|
||||
defaultStaleHours: 24
|
||||
},
|
||||
{
|
||||
name: 'intraday_bars',
|
||||
type: 'intraday_crawl',
|
||||
description: 'Crawl intraday price bars from today backwards',
|
||||
requiresFinishedFlag: true,
|
||||
defaultStaleHours: 1 // Check every hour for new data
|
||||
},
|
||||
|
||||
// Fundamental data operations
|
||||
{
|
||||
name: 'financials_update_quarterly',
|
||||
type: 'standard',
|
||||
description: 'Update quarterly financial statements',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
{
|
||||
name: 'financials_update_annual',
|
||||
type: 'standard',
|
||||
description: 'Update annual financial statements',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
// Corporate actions - fetched together in one API call
|
||||
{
|
||||
name: 'events_update',
|
||||
type: 'standard',
|
||||
description: 'Update events (earnings, dividends, splits)',
|
||||
defaultStaleHours: 24 * 7 // Weekly
|
||||
},
|
||||
|
||||
// News and filings
|
||||
{
|
||||
name: 'filings_update',
|
||||
type: 'standard',
|
||||
description: 'Update SEC filings',
|
||||
defaultStaleHours: 24 // Daily
|
||||
},
|
||||
// {
|
||||
// name: 'news_update',
|
||||
// type: 'standard',
|
||||
// description: 'Update news articles',
|
||||
// defaultStaleHours: 6 // Every 6 hours
|
||||
// },
|
||||
|
||||
// // Options data
|
||||
// {
|
||||
// name: 'options_chain',
|
||||
// type: 'standard',
|
||||
// description: 'Update options chain data',
|
||||
// defaultStaleHours: 1 // Hourly during market hours
|
||||
// }
|
||||
];
|
||||
|
||||
/**
|
||||
* Initialize operation tracker with all registered operations
|
||||
*/
|
||||
export async function initializeQMOperations(
|
||||
mongodb: MongoDBClient,
|
||||
logger: Logger
|
||||
): Promise<QMOperationTracker> {
|
||||
logger.info('Initializing QM operations tracker');
|
||||
|
||||
const tracker = new QMOperationTracker(mongodb, logger);
|
||||
|
||||
// Register all operations
|
||||
for (const operation of QM_OPERATIONS) {
|
||||
try {
|
||||
await tracker.registerOperation(operation);
|
||||
logger.debug(`Registered operation: ${operation.name}`);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to register operation: ${operation.name}`, { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('QM operations tracker initialized', {
|
||||
operationCount: QM_OPERATIONS.length
|
||||
});
|
||||
|
||||
return tracker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get operation configuration by name
|
||||
*/
|
||||
export function getOperationConfig(name: string): QMOperationConfig | undefined {
|
||||
return QM_OPERATIONS.find(op => op.name === name);
|
||||
}
|
||||
|
|
@ -1,458 +0,0 @@
|
|||
/**
|
||||
* QM Operation Tracker - Tracks operation execution times and states for symbols
|
||||
* Supports dynamic operation registration with auto-indexing
|
||||
*/
|
||||
|
||||
import type { Logger, MongoDBClient } from '@stock-bot/types';
|
||||
import type { IntradayCrawlSymbol, QMOperationConfig } from './types';
|
||||
|
||||
export class QMOperationTracker {
|
||||
private registeredOperations: Map<string, QMOperationConfig> = new Map();
|
||||
private indexesCreated: Set<string> = new Set();
|
||||
private mongodb: MongoDBClient;
|
||||
private logger: Logger;
|
||||
private readonly collectionName = 'qmSymbols';
|
||||
|
||||
constructor(mongodb: MongoDBClient, logger: Logger) {
|
||||
this.mongodb = mongodb;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new operation type with auto-indexing
|
||||
*/
|
||||
async registerOperation(config: QMOperationConfig): Promise<void> {
|
||||
this.logger.info('Registering QM operation', { operation: config.name, type: config.type });
|
||||
|
||||
this.registeredOperations.set(config.name, config);
|
||||
|
||||
// Auto-create indexes for this operation
|
||||
await this.createOperationIndexes(config.name);
|
||||
|
||||
this.logger.debug('Operation registered successfully', { operation: config.name });
|
||||
}
|
||||
|
||||
/**
|
||||
* Create indexes for efficient operation queries
|
||||
*/
|
||||
private async createOperationIndexes(operationName: string): Promise<void> {
|
||||
if (this.indexesCreated.has(operationName)) {
|
||||
this.logger.debug('Indexes already created for operation', { operation: operationName });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const indexes = [
|
||||
// Index for finding stale symbols
|
||||
{ [`operations.${operationName}.lastRunAt`]: 1, qmSearchCode: 1 },
|
||||
// Index for finding by last record date
|
||||
{ [`operations.${operationName}.lastRecordDate`]: 1, qmSearchCode: 1 },
|
||||
];
|
||||
|
||||
// Add crawl state index for intraday operations
|
||||
const config = this.registeredOperations.get(operationName);
|
||||
if (config?.type === 'intraday_crawl') {
|
||||
indexes.push({ [`operations.${operationName}.crawlState.finished`]: 1, qmSearchCode: 1 });
|
||||
}
|
||||
|
||||
for (const indexSpec of indexes) {
|
||||
const collection = this.mongodb.collection(this.collectionName);
|
||||
await collection.createIndex(indexSpec, {
|
||||
background: true,
|
||||
name: `op_${operationName}_${Object.keys(indexSpec).join('_')}`
|
||||
});
|
||||
}
|
||||
|
||||
this.indexesCreated.add(operationName);
|
||||
this.logger.info('Created indexes for operation', {
|
||||
operation: operationName,
|
||||
indexCount: indexes.length
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to create indexes for operation', {
|
||||
operation: operationName,
|
||||
error
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update symbol operation status
|
||||
*/
|
||||
async updateSymbolOperation(
|
||||
qmSearchCode: string,
|
||||
operationName: string,
|
||||
data: {
|
||||
status: 'success' | 'failure' | 'partial';
|
||||
lastRecordDate?: Date;
|
||||
recordCount?: number;
|
||||
crawlState?: {
|
||||
finished?: boolean;
|
||||
oldestDateReached?: Date;
|
||||
};
|
||||
}
|
||||
): Promise<void> {
|
||||
const update: any = {
|
||||
$set: {
|
||||
[`operations.${operationName}.lastRunAt`]: new Date(),
|
||||
[`operations.${operationName}.status`]: data.status,
|
||||
updated_at: new Date()
|
||||
}
|
||||
};
|
||||
|
||||
// Only update lastSuccessAt on successful operations
|
||||
if (data.status === 'success') {
|
||||
update.$set[`operations.${operationName}.lastSuccessAt`] = new Date();
|
||||
}
|
||||
|
||||
if (data.lastRecordDate) {
|
||||
update.$set[`operations.${operationName}.lastRecordDate`] = data.lastRecordDate;
|
||||
}
|
||||
|
||||
if (data.recordCount !== undefined) {
|
||||
update.$set[`operations.${operationName}.recordCount`] = data.recordCount;
|
||||
}
|
||||
|
||||
if (data.crawlState) {
|
||||
update.$set[`operations.${operationName}.crawlState`] = {
|
||||
...data.crawlState,
|
||||
lastCrawlDirection: data.crawlState.finished ? 'forward' : 'backward'
|
||||
};
|
||||
}
|
||||
|
||||
await this.mongodb.updateOne(this.collectionName, { qmSearchCode }, update);
|
||||
|
||||
this.logger.debug('Updated symbol operation', {
|
||||
qmSearchCode,
|
||||
operation: operationName,
|
||||
status: data.status
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk update symbol operations for performance
|
||||
*/
|
||||
async bulkUpdateSymbolOperations(
|
||||
updates: Array<{
|
||||
qmSearchCode: string;
|
||||
operation: string;
|
||||
data: {
|
||||
status: 'success' | 'failure' | 'partial';
|
||||
lastRecordDate?: Date;
|
||||
recordCount?: number;
|
||||
crawlState?: any;
|
||||
};
|
||||
}>
|
||||
): Promise<void> {
|
||||
if (updates.length === 0) {return;}
|
||||
|
||||
const bulkOps = updates.map(({ qmSearchCode, operation, data }) => {
|
||||
const update: any = {
|
||||
$set: {
|
||||
[`operations.${operation}.lastRunAt`]: new Date(),
|
||||
[`operations.${operation}.status`]: data.status,
|
||||
updated_at: new Date()
|
||||
}
|
||||
};
|
||||
|
||||
// Only update lastSuccessAt on successful operations
|
||||
if (data.status === 'success') {
|
||||
update.$set[`operations.${operation}.lastSuccessAt`] = new Date();
|
||||
}
|
||||
|
||||
if (data.lastRecordDate) {
|
||||
update.$set[`operations.${operation}.lastRecordDate`] = data.lastRecordDate;
|
||||
}
|
||||
|
||||
if (data.recordCount !== undefined) {
|
||||
update.$set[`operations.${operation}.recordCount`] = data.recordCount;
|
||||
}
|
||||
|
||||
if (data.crawlState) {
|
||||
update.$set[`operations.${operation}.crawlState`] = {
|
||||
...data.crawlState,
|
||||
lastCrawlDirection: data.crawlState.finished ? 'forward' : 'backward'
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
updateOne: {
|
||||
filter: { qmSearchCode },
|
||||
update
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
const collection = this.mongodb.collection(this.collectionName);
|
||||
const result = await collection.bulkWrite(bulkOps as any, { ordered: false });
|
||||
|
||||
this.logger.debug('Bulk updated symbol operations', {
|
||||
totalUpdates: updates.length,
|
||||
modified: result.modifiedCount,
|
||||
operations: Array.from(new Set(updates.map(u => u.operation)))
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get symbols that need processing for an operation
|
||||
*/
|
||||
async getStaleSymbols(
|
||||
operationName: string,
|
||||
options: {
|
||||
notRunSince?: Date;
|
||||
minHoursSinceRun?: number;
|
||||
limit?: number;
|
||||
excludeSymbols?: string[];
|
||||
} = {}
|
||||
): Promise<string[]> {
|
||||
const { limit = 1000, excludeSymbols = [] } = options;
|
||||
|
||||
const cutoffDate = options.notRunSince || (() => {
|
||||
const date = new Date();
|
||||
const hours = options.minHoursSinceRun ||
|
||||
this.registeredOperations.get(operationName)?.defaultStaleHours || 24;
|
||||
date.setHours(date.getHours() - hours);
|
||||
return date;
|
||||
})();
|
||||
|
||||
const filter: any = {
|
||||
active: { $ne: false }, // Only active symbols (active: true or active doesn't exist)
|
||||
$or: [
|
||||
{ [`operations.${operationName}.lastSuccessAt`]: { $lt: cutoffDate } },
|
||||
{ [`operations.${operationName}.lastSuccessAt`]: { $exists: false } },
|
||||
{ [`operations.${operationName}`]: { $exists: false } }
|
||||
]
|
||||
};
|
||||
|
||||
if (excludeSymbols.length > 0) {
|
||||
filter.qmSearchCode = { $nin: excludeSymbols };
|
||||
}
|
||||
|
||||
const symbols = await this.mongodb.find(this.collectionName, filter, {
|
||||
limit,
|
||||
projection: { qmSearchCode: 1 },
|
||||
sort: { [`operations.${operationName}.lastSuccessAt`]: 1 } // Oldest successful run first
|
||||
});
|
||||
|
||||
return symbols.map(s => s.qmSearchCode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get symbols for intraday crawling
|
||||
*/
|
||||
async getSymbolsForIntradayCrawl(
|
||||
operationName: string,
|
||||
options: {
|
||||
limit?: number;
|
||||
includeFinished?: boolean;
|
||||
} = {}
|
||||
): Promise<IntradayCrawlSymbol[]> {
|
||||
const { limit = 100, includeFinished = false } = options;
|
||||
|
||||
const filter: any = {
|
||||
active: { $ne: false } // Only active symbols
|
||||
};
|
||||
if (!includeFinished) {
|
||||
filter[`operations.${operationName}.crawlState.finished`] = { $ne: true };
|
||||
}
|
||||
|
||||
const symbols = await this.mongodb.find(this.collectionName, filter, {
|
||||
limit,
|
||||
projection: {
|
||||
qmSearchCode: 1,
|
||||
[`operations.${operationName}`]: 1
|
||||
},
|
||||
sort: {
|
||||
// Prioritize symbols that haven't been crawled yet
|
||||
[`operations.${operationName}.lastRunAt`]: 1
|
||||
}
|
||||
});
|
||||
|
||||
return symbols.map(s => ({
|
||||
qmSearchCode: s.qmSearchCode,
|
||||
lastRecordDate: s.operations?.[operationName]?.lastRecordDate,
|
||||
crawlState: s.operations?.[operationName]?.crawlState
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark intraday crawl as finished
|
||||
*/
|
||||
async markCrawlFinished(
|
||||
qmSearchCode: string,
|
||||
operationName: string,
|
||||
oldestDateReached: Date
|
||||
): Promise<void> {
|
||||
await this.updateSymbolOperation(qmSearchCode, operationName, {
|
||||
status: 'success',
|
||||
crawlState: {
|
||||
finished: true,
|
||||
oldestDateReached
|
||||
}
|
||||
});
|
||||
|
||||
this.logger.info('Marked crawl as finished', {
|
||||
qmSearchCode,
|
||||
operation: operationName,
|
||||
oldestDateReached
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get symbols that need data updates based on last record date
|
||||
*/
|
||||
async getSymbolsNeedingUpdate(
|
||||
operationName: string,
|
||||
options: {
|
||||
lastRecordBefore?: Date;
|
||||
neverRun?: boolean;
|
||||
limit?: number;
|
||||
} = {}
|
||||
): Promise<Array<{ qmSearchCode: string; lastRecordDate?: Date }>> {
|
||||
const { limit = 500 } = options;
|
||||
const filter: any = {
|
||||
active: { $ne: false } // Only active symbols
|
||||
};
|
||||
|
||||
if (options.neverRun) {
|
||||
filter[`operations.${operationName}`] = { $exists: false };
|
||||
} else if (options.lastRecordBefore) {
|
||||
filter.$or = [
|
||||
{ [`operations.${operationName}.lastRecordDate`]: { $lt: options.lastRecordBefore } },
|
||||
{ [`operations.${operationName}`]: { $exists: false } }
|
||||
];
|
||||
}
|
||||
|
||||
const symbols = await this.mongodb.find(this.collectionName, filter, {
|
||||
limit,
|
||||
projection: {
|
||||
qmSearchCode: 1,
|
||||
[`operations.${operationName}.lastRecordDate`]: 1
|
||||
},
|
||||
sort: { [`operations.${operationName}.lastRecordDate`]: 1 } // Oldest data first
|
||||
});
|
||||
|
||||
return symbols.map(s => ({
|
||||
qmSearchCode: s.qmSearchCode,
|
||||
lastRecordDate: s.operations?.[operationName]?.lastRecordDate
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get operation statistics
|
||||
*/
|
||||
async getOperationStats(operationName: string): Promise<{
|
||||
totalSymbols: number;
|
||||
processedSymbols: number;
|
||||
staleSymbols: number;
|
||||
successfulSymbols: number;
|
||||
failedSymbols: number;
|
||||
finishedCrawls?: number;
|
||||
avgRecordsPerSymbol?: number;
|
||||
}> {
|
||||
const collection = this.mongodb.collection(this.collectionName);
|
||||
const total = await collection.countDocuments({});
|
||||
|
||||
const processed = await collection.countDocuments({
|
||||
[`operations.${operationName}`]: { $exists: true }
|
||||
});
|
||||
|
||||
const successful = await collection.countDocuments({
|
||||
[`operations.${operationName}.status`]: 'success'
|
||||
});
|
||||
|
||||
const failed = await collection.countDocuments({
|
||||
[`operations.${operationName}.status`]: 'failure'
|
||||
});
|
||||
|
||||
const staleDate = new Date();
|
||||
staleDate.setHours(staleDate.getHours() - (
|
||||
this.registeredOperations.get(operationName)?.defaultStaleHours || 24
|
||||
));
|
||||
|
||||
const stale = await collection.countDocuments({
|
||||
$or: [
|
||||
{ [`operations.${operationName}.lastRunAt`]: { $lt: staleDate } },
|
||||
{ [`operations.${operationName}`]: { $exists: false } }
|
||||
]
|
||||
});
|
||||
|
||||
const result: any = {
|
||||
totalSymbols: total,
|
||||
processedSymbols: processed,
|
||||
staleSymbols: stale,
|
||||
successfulSymbols: successful,
|
||||
failedSymbols: failed
|
||||
};
|
||||
|
||||
// Additional stats for crawl operations
|
||||
if (this.registeredOperations.get(operationName)?.type === 'intraday_crawl') {
|
||||
result.finishedCrawls = await collection.countDocuments({
|
||||
[`operations.${operationName}.crawlState.finished`]: true
|
||||
});
|
||||
}
|
||||
|
||||
// Calculate average records per symbol
|
||||
const aggregation = await collection.aggregate([
|
||||
{
|
||||
$match: {
|
||||
[`operations.${operationName}.recordCount`]: { $exists: true }
|
||||
}
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: null,
|
||||
avgRecords: { $avg: `$operations.${operationName}.recordCount` }
|
||||
}
|
||||
}
|
||||
]).toArray();
|
||||
|
||||
if (aggregation.length > 0) {
|
||||
result.avgRecordsPerSymbol = Math.round(aggregation[0].avgRecords);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered operations
|
||||
*/
|
||||
getRegisteredOperations(): QMOperationConfig[] {
|
||||
return Array.from(this.registeredOperations.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Get symbols for price update
|
||||
*/
|
||||
async getSymbolsForPriceUpdate(limit = 1000): Promise<string[]> {
|
||||
return this.getStaleSymbols('price_update', {
|
||||
minHoursSinceRun: 24,
|
||||
limit
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Get symbols with outdated financials
|
||||
*/
|
||||
async getSymbolsWithOldFinancials(limit = 100): Promise<Array<{ qmSearchCode: string; lastRecordDate?: Date }>> {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - 90); // 90 days old
|
||||
|
||||
return this.getSymbolsNeedingUpdate('financials_update', {
|
||||
lastRecordBefore: cutoffDate,
|
||||
limit
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Get unprocessed symbols for an operation
|
||||
*/
|
||||
async getUnprocessedSymbols(operation: string, limit = 500): Promise<string[]> {
|
||||
const symbols = await this.getSymbolsNeedingUpdate(operation, {
|
||||
neverRun: true,
|
||||
limit
|
||||
});
|
||||
return symbols.map(s => s.qmSearchCode);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Shared types for QM operations
|
||||
* Shared types for QM handler
|
||||
*/
|
||||
|
||||
export interface QMSession {
|
||||
|
|
@ -58,49 +58,8 @@ export interface CachedSession {
|
|||
sessionType: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Operation tracking types
|
||||
*/
|
||||
export interface QMOperationConfig {
|
||||
name: string;
|
||||
type: 'standard' | 'intraday_crawl';
|
||||
description?: string;
|
||||
defaultStaleHours?: number;
|
||||
requiresFinishedFlag?: boolean;
|
||||
}
|
||||
|
||||
export interface QMSymbolOperationStatus {
|
||||
symbol: string;
|
||||
qmSearchCode: string;
|
||||
operations: {
|
||||
[operationName: string]: {
|
||||
lastRunAt: Date;
|
||||
lastRecordDate?: Date;
|
||||
status: 'success' | 'failure' | 'partial';
|
||||
recordCount?: number;
|
||||
// For intraday crawling operations
|
||||
crawlState?: {
|
||||
finished: boolean;
|
||||
oldestDateReached?: Date;
|
||||
lastCrawlDirection?: 'forward' | 'backward';
|
||||
};
|
||||
};
|
||||
};
|
||||
updated_at: Date;
|
||||
}
|
||||
|
||||
export interface IntradayCrawlSymbol {
|
||||
qmSearchCode: string;
|
||||
lastRecordDate?: Date;
|
||||
crawlState?: {
|
||||
finished: boolean;
|
||||
oldestDateReached?: Date;
|
||||
lastCrawlDirection?: 'forward' | 'backward';
|
||||
};
|
||||
}
|
||||
|
||||
export interface ExchangeStats {
|
||||
symbolCount: number;
|
||||
totalMarketCap: number;
|
||||
avgMarketCap: number;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue