/** * Test script to verify intraday crawl data flow */ import { OperationTracker, OperationRegistry } from '../src/shared/operation-manager'; import type { DataIngestionServices } from '../src/types'; async function testIntradayDataFlow() { console.log('=== Testing Intraday Data Flow ===\n'); // Mock services const mockServices: DataIngestionServices = { mongodb: { collection: (name: string) => ({ find: () => ({ toArray: async () => { console.log(`Mock: Query collection ${name}`); if (name === 'qmSymbols') { return [{ symbol: 'X', symbolId: 123456, qmSearchCode: 'X:NYSE', exchange: 'NYSE', operations: { intraday_bars: { lastRunAt: new Date('2024-01-01'), lastSuccessAt: new Date('2024-01-01'), status: 'success', crawlState: { finished: false, oldestDateReached: new Date('2023-01-01'), newestDateReached: new Date('2024-01-01'), lastProcessedDate: new Date('2023-06-15'), totalDaysProcessed: 180 } } } }]; } return []; } }), createIndex: async () => ({ ok: 1 }) }), find: async (collection: string, filter: any, options?: any) => { console.log(`Mock: Direct find on ${collection}`, { filter, options }); return [{ symbol: 'X', qmSearchCode: 'X:NYSE', exchange: 'NYSE', operations: { intraday_bars: { crawlState: { finished: false, oldestDateReached: new Date('2023-01-01'), newestDateReached: new Date('2024-01-01') } } } }]; } } as any, logger: { info: (msg: string, data?: any) => console.log(`[INFO] ${msg}`, JSON.stringify(data, null, 2)), error: (msg: string, data?: any) => console.error(`[ERROR] ${msg}`, data || ''), warn: (msg: string, data?: any) => console.warn(`[WARN] ${msg}`, data || ''), debug: (msg: string, data?: any) => console.debug(`[DEBUG] ${msg}`, data || '') } } as DataIngestionServices; // Create registry and provider const registry = new OperationRegistry(mockServices as any); // Mock provider const mockProvider = { getProviderConfig: () => ({ name: 'qm', collectionName: 'qmSymbols', symbolField: 'qmSearchCode' }), getOperations: () => [{ name: 'intraday_bars', type: 'intraday_crawl' as const, defaultStaleHours: 1 }], validateOperation: () => true, getOperation: () => ({ name: 'intraday_bars', type: 'intraday_crawl' }), getDefaultStaleHours: () => 1, initialize: async () => {}, beforeOperationUpdate: async () => {}, afterOperationUpdate: async () => {} }; const tracker = new OperationTracker(mockServices as any, mockProvider as any); await tracker.initialize(); // Test 1: Get symbols for intraday crawl console.log('\nTest 1: Get symbols for intraday crawl with symbol filter'); const symbols = await tracker.getSymbolsForIntradayCrawl('intraday_bars', { limit: 10, targetOldestDate: new Date('2020-01-01'), includeNewDataGaps: true, symbolFilter: { symbol: 'X' } }); console.log(`\nFound ${symbols.length} symbols:`); symbols.forEach(sym => { console.log({ symbol: sym.symbol, qmSearchCode: sym.qmSearchCode, exchange: sym.exchange, gaps: sym.gaps, crawlState: sym.operationStatus?.crawlState }); }); // Test 2: Verify data preservation console.log('\n\nTest 2: Verify data is preserved through mapping'); const testSymbol = symbols[0]; if (testSymbol) { console.log('Original data:'); console.log({ symbol: testSymbol.symbol, hasGaps: !!testSymbol.gaps, gaps: testSymbol.gaps, hasOperationStatus: !!testSymbol.operationStatus, crawlState: testSymbol.operationStatus?.crawlState }); // Simulate mapping const mapped = { ...testSymbol, symbol: 'X', exchange: 'NYSE', qmSearchCode: 'X:NYSE', gaps: testSymbol.gaps, operationStatus: testSymbol.operationStatus }; console.log('\nAfter mapping:'); console.log({ symbol: mapped.symbol, exchange: mapped.exchange, qmSearchCode: mapped.qmSearchCode, hasGaps: !!mapped.gaps, gaps: mapped.gaps, hasOperationStatus: !!mapped.operationStatus, crawlState: mapped.operationStatus?.crawlState }); } console.log('\n=== Tests Complete ==='); } // Run tests testIntradayDataFlow().catch(console.error);