stock-bot/apps/stock/data-ingestion/test/test-intraday-flow.ts

/**
 * Test script to verify intraday crawl data flow
 */

import { OperationTracker, OperationRegistry } from '../src/shared/operation-manager';
import type { DataIngestionServices } from '../src/types';

async function testIntradayDataFlow() {
  console.log('=== Testing Intraday Data Flow ===\n');

  // Mock services
  const mockServices: DataIngestionServices = {
    mongodb: {
      collection: (name: string) => ({
        find: () => ({
          toArray: async () => {
            console.log(`Mock: Query collection ${name}`);
            if (name === 'qmSymbols') {
              return [{
                symbol: 'X',
                symbolId: 123456,
                qmSearchCode: 'X:NYSE',
                exchange: 'NYSE',
                operations: {
                  intraday_bars: {
                    lastRunAt: new Date('2024-01-01'),
                    lastSuccessAt: new Date('2024-01-01'),
                    status: 'success',
                    crawlState: {
                      finished: false,
                      oldestDateReached: new Date('2023-01-01'),
                      newestDateReached: new Date('2024-01-01'),
                      lastProcessedDate: new Date('2023-06-15'),
                      totalDaysProcessed: 180
                    }
                  }
                }
              }];
            }
            return [];
          }
        }),
        createIndex: async () => ({ ok: 1 })
      }),
      find: async (collection: string, filter: any, options?: any) => {
        console.log(`Mock: Direct find on ${collection}`, { filter, options });
        return [{
          symbol: 'X',
          qmSearchCode: 'X:NYSE',
          exchange: 'NYSE',
          operations: {
            intraday_bars: {
              crawlState: {
                finished: false,
                oldestDateReached: new Date('2023-01-01'),
                newestDateReached: new Date('2024-01-01')
              }
            }
          }
        }];
      }
    } as any,
    logger: {
      info: (msg: string, data?: any) => console.log(`[INFO] ${msg}`, JSON.stringify(data, null, 2)),
      error: (msg: string, data?: any) => console.error(`[ERROR] ${msg}`, data || ''),
      warn: (msg: string, data?: any) => console.warn(`[WARN] ${msg}`, data || ''),
      debug: (msg: string, data?: any) => console.debug(`[DEBUG] ${msg}`, data || '')
    }
  } as DataIngestionServices;

  // Create registry and provider
  const registry = new OperationRegistry(mockServices as any);
  
  // Mock provider
  const mockProvider = {
    getProviderConfig: () => ({
      name: 'qm',
      collectionName: 'qmSymbols',
      symbolField: 'qmSearchCode'
    }),
    getOperations: () => [{
      name: 'intraday_bars',
      type: 'intraday_crawl' as const,
      defaultStaleHours: 1
    }],
    validateOperation: () => true,
    getOperation: () => ({ name: 'intraday_bars', type: 'intraday_crawl' }),
    getDefaultStaleHours: () => 1,
    initialize: async () => {},
    beforeOperationUpdate: async () => {},
    afterOperationUpdate: async () => {}
  };

  const tracker = new OperationTracker(mockServices as any, mockProvider as any);
  await tracker.initialize();

  // Test 1: Get symbols for intraday crawl
  console.log('\nTest 1: Get symbols for intraday crawl with symbol filter');
  const symbols = await tracker.getSymbolsForIntradayCrawl('intraday_bars', {
    limit: 10,
    targetOldestDate: new Date('2020-01-01'),
    includeNewDataGaps: true,
    symbolFilter: { symbol: 'X' }
  });

  console.log(`\nFound ${symbols.length} symbols:`);
  symbols.forEach(sym => {
    console.log({
      symbol: sym.symbol,
      qmSearchCode: sym.qmSearchCode,
      exchange: sym.exchange,
      gaps: sym.gaps,
      crawlState: sym.operationStatus?.crawlState
    });
  });

  // Test 2: Verify data preservation
  console.log('\n\nTest 2: Verify data is preserved through mapping');
  const testSymbol = symbols[0];
  if (testSymbol) {
    console.log('Original data:');
    console.log({
      symbol: testSymbol.symbol,
      hasGaps: !!testSymbol.gaps,
      gaps: testSymbol.gaps,
      hasOperationStatus: !!testSymbol.operationStatus,
      crawlState: testSymbol.operationStatus?.crawlState
    });

    // Simulate mapping
    const mapped = {
      ...testSymbol,
      symbol: 'X',
      exchange: 'NYSE',
      qmSearchCode: 'X:NYSE',
      gaps: testSymbol.gaps,
      operationStatus: testSymbol.operationStatus
    };

    console.log('\nAfter mapping:');
    console.log({
      symbol: mapped.symbol,
      exchange: mapped.exchange,
      qmSearchCode: mapped.qmSearchCode,
      hasGaps: !!mapped.gaps,
      gaps: mapped.gaps,
      hasOperationStatus: !!mapped.operationStatus,
      crawlState: mapped.operationStatus?.crawlState
    });
  }

  console.log('\n=== Tests Complete ===');
}

// Run tests
testIntradayDataFlow().catch(console.error);