boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

qm scaffolding done

Browse source
This commit is contained in:
Boki 2025-06-28 20:48:17 -04:00
parent 736b86e66a
commit c799962f05
11 changed files with 1693 additions and 336 deletions
Download patch file Download diff file Expand all files Collapse all files

25
apps/stock/data-ingestion/src/handlers/qm/shared/operation-tracker.ts
View file

@ -3,8 +3,7 @@
* Supports dynamic operation registration with auto-indexing
*/
import type { MongoDBClient } from '@stock-bot/mongodb';
import type { Logger } from '@stock-bot/types';
import type { MongoDBClient, Logger } from '@stock-bot/types';
import type { IntradayCrawlSymbol, QMOperationConfig } from './types';
export class QMOperationTracker {
@ -57,7 +56,8 @@ export class QMOperationTracker {
}
for (const indexSpec of indexes) {
await this.mongodb.createIndex(this.collectionName, indexSpec, {
const collection = this.mongodb.collection(this.collectionName);
await collection.createIndex(indexSpec, {
background: true,
name: `op_${operationName}_${Object.keys(indexSpec).join('_')}`
});
@ -174,7 +174,7 @@ export class QMOperationTracker {
};
});
const collection = this.mongodb.getCollection(this.collectionName);
const collection = this.mongodb.collection(this.collectionName);
const result = await collection.bulkWrite(bulkOps as any, { ordered: false });
this.logger.debug('Bulk updated symbol operations', {
@ -335,17 +335,18 @@ export class QMOperationTracker {
finishedCrawls?: number;
avgRecordsPerSymbol?: number;
}> {
const total = await this.mongodb.countDocuments(this.collectionName);
const collection = this.mongodb.collection(this.collectionName);
const total = await collection.countDocuments({});
const processed = await this.mongodb.countDocuments(this.collectionName, {
const processed = await collection.countDocuments({
[`operations.${operationName}`]: { $exists: true }
});
const successful = await this.mongodb.countDocuments(this.collectionName, {
const successful = await collection.countDocuments({
[`operations.${operationName}.status`]: 'success'
});
const failed = await this.mongodb.countDocuments(this.collectionName, {
const failed = await collection.countDocuments({
[`operations.${operationName}.status`]: 'failure'
});
@ -354,7 +355,7 @@ export class QMOperationTracker {
this.registeredOperations.get(operationName)?.defaultStaleHours || 24
));
const stale = await this.mongodb.countDocuments(this.collectionName, {
const stale = await collection.countDocuments({
$or: [
{ [`operations.${operationName}.lastRunAt`]: { $lt: staleDate } },
{ [`operations.${operationName}`]: { $exists: false } }
@ -371,13 +372,13 @@ export class QMOperationTracker {
// Additional stats for crawl operations
if (this.registeredOperations.get(operationName)?.type === 'intraday_crawl') {
result.finishedCrawls = await this.mongodb.countDocuments(this.collectionName, {
result.finishedCrawls = await collection.countDocuments({
[`operations.${operationName}.crawlState.finished`]: true
});
}
// Calculate average records per symbol
const aggregation = await this.mongodb.aggregate(this.collectionName, [
const aggregation = await collection.aggregate([
{
$match: {
[`operations.${operationName}.recordCount`]: { $exists: true }
@ -389,7 +390,7 @@ export class QMOperationTracker {
avgRecords: { $avg: `$operations.${operationName}.recordCount` }
}
}
]);
]).toArray();
if (aggregation.length > 0) {
result.avgRecordsPerSymbol = Math.round(aggregation[0].avgRecords);