boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

qm scaffolding done

Browse source
This commit is contained in:
Boki 2025-06-28 20:48:17 -04:00
parent 736b86e66a
commit c799962f05
11 changed files with 1693 additions and 336 deletions
Download patch file Download diff file Expand all files Collapse all files

82
apps/stock/data-ingestion/src/handlers/qm/shared/operation-registry.ts
View file

@ -2,8 +2,7 @@
* QM Operation Registry - Define and register all QM operations
*/
import type { MongoDBClient } from '@stock-bot/mongodb';
import type { Logger } from '@stock-bot/types';
import type { MongoDBClient, Logger } from '@stock-bot/types';
import { QMOperationTracker } from './operation-tracker';
import type { QMOperationConfig } from './types';
@ -16,53 +15,42 @@ export const QM_OPERATIONS: QMOperationConfig[] = [
description: 'Update symbol metadata',
defaultStaleHours: 24 * 7 // Weekly
},
// {
// name: 'price_update',
// type: 'standard',
// description: 'Update daily price data',
// defaultStaleHours: 24
// },
// {
// name: 'intraday_bars',
// type: 'intraday_crawl',
// description: 'Crawl intraday price bars from today backwards',
// requiresFinishedFlag: true,
// defaultStaleHours: 1 // Check every hour for new data
// },
{
name: 'price_update',
type: 'standard',
description: 'Update daily price data',
defaultStaleHours: 24
},
{
name: 'intraday_bars',
type: 'intraday_crawl',
description: 'Crawl intraday price bars from today backwards',
requiresFinishedFlag: true,
defaultStaleHours: 1 // Check every hour for new data
},
// // Fundamental data operations
// {
// name: 'financials_update',
// type: 'standard',
// description: 'Update financial statements',
// defaultStaleHours: 24 * 7 // Weekly
// },
// {
// name: 'earnings_update',-
// type: 'standard',
// description: 'Update earnings data',
// defaultStaleHours: 24 * 7 // Weekly
// },
// {
// name: 'dividends_update',
// type: 'standard',
// description: 'Update dividend history',
// defaultStaleHours: 24 * 7 // Weekly
// },
// {
// name: 'splits_update',
// type: 'standard',
// description: 'Update stock split history',
// defaultStaleHours: 24 * 30 // Monthly
// },
// Fundamental data operations
{
name: 'financials_update',
type: 'standard',
description: 'Update financial statements',
defaultStaleHours: 24 * 7 // Weekly
},
// Corporate actions - fetched together in one API call
{
name: 'corporate_actions_update',
type: 'standard',
description: 'Update corporate actions (earnings, dividends, splits)',
defaultStaleHours: 24 * 7 // Weekly
},
// // News and filings
// {
// name: 'filings_update',
// type: 'standard',
// description: 'Update SEC filings',
// defaultStaleHours: 24 // Daily
// },
// News and filings
{
name: 'filings_update',
type: 'standard',
description: 'Update SEC filings',
defaultStaleHours: 24 // Daily
},
// {
// name: 'news_update',
// type: 'standard',

25
apps/stock/data-ingestion/src/handlers/qm/shared/operation-tracker.ts
View file

@ -3,8 +3,7 @@
* Supports dynamic operation registration with auto-indexing
*/
import type { MongoDBClient } from '@stock-bot/mongodb';
import type { Logger } from '@stock-bot/types';
import type { MongoDBClient, Logger } from '@stock-bot/types';
import type { IntradayCrawlSymbol, QMOperationConfig } from './types';
export class QMOperationTracker {
@ -57,7 +56,8 @@ export class QMOperationTracker {
}
for (const indexSpec of indexes) {
await this.mongodb.createIndex(this.collectionName, indexSpec, {
const collection = this.mongodb.collection(this.collectionName);
await collection.createIndex(indexSpec, {
background: true,
name: `op_${operationName}_${Object.keys(indexSpec).join('_')}`
});
@ -174,7 +174,7 @@ export class QMOperationTracker {
};
});
const collection = this.mongodb.getCollection(this.collectionName);
const collection = this.mongodb.collection(this.collectionName);
const result = await collection.bulkWrite(bulkOps as any, { ordered: false });
this.logger.debug('Bulk updated symbol operations', {
@ -335,17 +335,18 @@ export class QMOperationTracker {
finishedCrawls?: number;
avgRecordsPerSymbol?: number;
}> {
const total = await this.mongodb.countDocuments(this.collectionName);
const collection = this.mongodb.collection(this.collectionName);
const total = await collection.countDocuments({});
const processed = await this.mongodb.countDocuments(this.collectionName, {
const processed = await collection.countDocuments({
[`operations.${operationName}`]: { $exists: true }
});
const successful = await this.mongodb.countDocuments(this.collectionName, {
const successful = await collection.countDocuments({
[`operations.${operationName}.status`]: 'success'
});
const failed = await this.mongodb.countDocuments(this.collectionName, {
const failed = await collection.countDocuments({
[`operations.${operationName}.status`]: 'failure'
});
@ -354,7 +355,7 @@ export class QMOperationTracker {
this.registeredOperations.get(operationName)?.defaultStaleHours || 24
));
const stale = await this.mongodb.countDocuments(this.collectionName, {
const stale = await collection.countDocuments({
$or: [
{ [`operations.${operationName}.lastRunAt`]: { $lt: staleDate } },
{ [`operations.${operationName}`]: { $exists: false } }
@ -371,13 +372,13 @@ export class QMOperationTracker {
// Additional stats for crawl operations
if (this.registeredOperations.get(operationName)?.type === 'intraday_crawl') {
result.finishedCrawls = await this.mongodb.countDocuments(this.collectionName, {
result.finishedCrawls = await collection.countDocuments({
[`operations.${operationName}.crawlState.finished`]: true
});
}
// Calculate average records per symbol
const aggregation = await this.mongodb.aggregate(this.collectionName, [
const aggregation = await collection.aggregate([
{
$match: {
[`operations.${operationName}.recordCount`]: { $exists: true }
@ -389,7 +390,7 @@ export class QMOperationTracker {
avgRecords: { $avg: `$operations.${operationName}.recordCount` }
}
}
]);
]).toArray();
if (aggregation.length > 0) {
result.avgRecordsPerSymbol = Math.round(aggregation[0].avgRecords);