finished intra-day crawl

apps/stock/data-ingestion/test/test-intraday-flow.ts

@@ -0,0 +1,156 @@
+/**
+ * Test script to verify intraday crawl data flow
+ */
+
+import { OperationTracker, OperationRegistry } from '../src/shared/operation-manager';
+import type { DataIngestionServices } from '../src/types';
+
+async function testIntradayDataFlow() {
+  console.log('=== Testing Intraday Data Flow ===\n');
+
+  // Mock services
+  const mockServices: DataIngestionServices = {
+    mongodb: {
+      collection: (name: string) => ({
+        find: () => ({
+          toArray: async () => {
+            console.log(`Mock: Query collection ${name}`);
+            if (name === 'qmSymbols') {
+              return [{
+                symbol: 'X',
+                symbolId: 123456,
+                qmSearchCode: 'X:NYSE',
+                exchange: 'NYSE',
+                operations: {
+                  intraday_bars: {
+                    lastRunAt: new Date('2024-01-01'),
+                    lastSuccessAt: new Date('2024-01-01'),
+                    status: 'success',
+                    crawlState: {
+                      finished: false,
+                      oldestDateReached: new Date('2023-01-01'),
+                      newestDateReached: new Date('2024-01-01'),
+                      lastProcessedDate: new Date('2023-06-15'),
+                      totalDaysProcessed: 180
+                    }
+                  }
+                }
+              }];
+            }
+            return [];
+          }
+        }),
+        createIndex: async () => ({ ok: 1 })
+      }),
+      find: async (collection: string, filter: any, options?: any) => {
+        console.log(`Mock: Direct find on ${collection}`, { filter, options });
+        return [{
+          symbol: 'X',
+          qmSearchCode: 'X:NYSE',
+          exchange: 'NYSE',
+          operations: {
+            intraday_bars: {
+              crawlState: {
+                finished: false,
+                oldestDateReached: new Date('2023-01-01'),
+                newestDateReached: new Date('2024-01-01')
+              }
+            }
+          }
+        }];
+      }
+    } as any,
+    logger: {
+      info: (msg: string, data?: any) => console.log(`[INFO] ${msg}`, JSON.stringify(data, null, 2)),
+      error: (msg: string, data?: any) => console.error(`[ERROR] ${msg}`, data || ''),
+      warn: (msg: string, data?: any) => console.warn(`[WARN] ${msg}`, data || ''),
+      debug: (msg: string, data?: any) => console.debug(`[DEBUG] ${msg}`, data || '')
+    }
+  } as DataIngestionServices;
+
+  // Create registry and provider
+  const registry = new OperationRegistry(mockServices as any);
+  
+  // Mock provider
+  const mockProvider = {
+    getProviderConfig: () => ({
+      name: 'qm',
+      collectionName: 'qmSymbols',
+      symbolField: 'qmSearchCode'
+    }),
+    getOperations: () => [{
+      name: 'intraday_bars',
+      type: 'intraday_crawl' as const,
+      defaultStaleHours: 1
+    }],
+    validateOperation: () => true,
+    getOperation: () => ({ name: 'intraday_bars', type: 'intraday_crawl' }),
+    getDefaultStaleHours: () => 1,
+    initialize: async () => {},
+    beforeOperationUpdate: async () => {},
+    afterOperationUpdate: async () => {}
+  };
+
+  const tracker = new OperationTracker(mockServices as any, mockProvider as any);
+  await tracker.initialize();
+
+  // Test 1: Get symbols for intraday crawl
+  console.log('\nTest 1: Get symbols for intraday crawl with symbol filter');
+  const symbols = await tracker.getSymbolsForIntradayCrawl('intraday_bars', {
+    limit: 10,
+    targetOldestDate: new Date('2020-01-01'),
+    includeNewDataGaps: true,
+    symbolFilter: { symbol: 'X' }
+  });
+
+  console.log(`\nFound ${symbols.length} symbols:`);
+  symbols.forEach(sym => {
+    console.log({
+      symbol: sym.symbol,
+      qmSearchCode: sym.qmSearchCode,
+      exchange: sym.exchange,
+      gaps: sym.gaps,
+      crawlState: sym.operationStatus?.crawlState
+    });
+  });
+
+  // Test 2: Verify data preservation
+  console.log('\n\nTest 2: Verify data is preserved through mapping');
+  const testSymbol = symbols[0];
+  if (testSymbol) {
+    console.log('Original data:');
+    console.log({
+      symbol: testSymbol.symbol,
+      hasGaps: !!testSymbol.gaps,
+      gaps: testSymbol.gaps,
+      hasOperationStatus: !!testSymbol.operationStatus,
+      crawlState: testSymbol.operationStatus?.crawlState
+    });
+
+    // Simulate mapping
+    const mapped = {
+      ...testSymbol,
+      symbol: 'X',
+      exchange: 'NYSE',
+      qmSearchCode: 'X:NYSE',
+      gaps: testSymbol.gaps,
+      operationStatus: testSymbol.operationStatus
+    };
+
+    console.log('\nAfter mapping:');
+    console.log({
+      symbol: mapped.symbol,
+      exchange: mapped.exchange,
+      qmSearchCode: mapped.qmSearchCode,
+      hasGaps: !!mapped.gaps,
+      gaps: mapped.gaps,
+      hasOperationStatus: !!mapped.operationStatus,
+      crawlState: mapped.operationStatus?.crawlState
+    });
+  }
+
+  console.log('\n=== Tests Complete ===');
+}
+
+// Run tests
+testIntradayDataFlow().catch(console.error);