boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

added deduplication and exchange stats

Browse source
This commit is contained in:
Boki 2025-06-29 14:48:40 -04:00
parent 133f37e755
commit 100efb575f
7 changed files with 309 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

4
apps/stock/data-ingestion/src/handlers/qm/shared/config.ts
View file

@ -42,8 +42,8 @@ export const QM_CONFIG = {
// Session management settings
export const SESSION_CONFIG = {
MIN_SESSIONS: 100,
MAX_SESSIONS: 100,
MIN_SESSIONS: 2,
MAX_SESSIONS: 2,
MAX_FAILED_CALLS: 3,
SESSION_TIMEOUT: 5000, // 10 seconds
API_TIMEOUT: 30000, // 15 seconds

9
apps/stock/data-ingestion/src/handlers/qm/shared/operation-tracker.ts
View file

@ -217,6 +217,7 @@ export class QMOperationTracker {
})();
const filter: any = {
active: { $ne: false }, // Only active symbols (active: true or active doesn't exist)
$or: [
{ [`operations.${operationName}.lastSuccessAt`]: { $lt: cutoffDate } },
{ [`operations.${operationName}.lastSuccessAt`]: { $exists: false } },
@ -249,7 +250,9 @@ export class QMOperationTracker {
): Promise<IntradayCrawlSymbol[]> {
const { limit = 100, includeFinished = false } = options;
const filter: any = {};
const filter: any = {
active: { $ne: false } // Only active symbols
};
if (!includeFinished) {
filter[`operations.${operationName}.crawlState.finished`] = { $ne: true };
}
@ -308,7 +311,9 @@ export class QMOperationTracker {
} = {}
): Promise<Array<{ qmSearchCode: string; lastRecordDate?: Date }>> {
const { limit = 500 } = options;
const filter: any = {};
const filter: any = {
active: { $ne: false } // Only active symbols
};
if (options.neverRun) {
filter[`operations.${operationName}`] = { $exists: false };

6
apps/stock/data-ingestion/src/handlers/qm/shared/types.ts
View file

@ -91,3 +91,9 @@ export interface IntradayCrawlSymbol {
lastCrawlDirection?: 'forward' | 'backward';
};
}
export interface ExchangeStats {
symbolCount: number;
totalMarketCap: number;
avgMarketCap: number;
}