work on refactoring operation tracker

apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts

@@ -1,8 +1,7 @@
-import type { BaseHandler } from '@stock-bot/handlers';
-import type { DataIngestionServices } from '../../../types';
+
+import type { CrawlState } from '../../../shared/operation-manager/types';
 import type { EodHandler } from '../eod.handler';
 import { EOD_CONFIG } from '../shared';
-import { getEodExchangeSuffix } from '../shared/utils';
 
 interface FetchIntradayInput {
   symbol: string;
@@ -20,14 +19,7 @@ interface CrawlIntradayInput {
   country?: string;
 }
 
-interface CrawlState {
-  finished: boolean;
-  oldestDateReached?: Date;
-  newestDateReached?: Date;
-  lastProcessedDate?: Date;
-  totalRecordsProcessed?: number;
-  totalBatchesProcessed?: number;
-}
+// CrawlState is imported from operation-manager types
 
 // Max days per interval based on EOD limits
 const MAX_DAYS_PER_INTERVAL = {
@@ -44,37 +36,34 @@ export async function scheduleIntradayCrawl(
   try {
     logger.info('Scheduling intraday crawl jobs');
     
-    // Get Canadian exchanges for now
-    const canadianExchanges = ['TO', 'V', 'CN', 'NEO'];
-    
     // Use OperationTracker to find symbols needing intraday crawl
     const intervals: Array<'1m' | '5m' | '1h'> = ['1m', '5m', '1h'];
-    const operationNames = ['intraday_1m', 'intraday_5m', 'intraday_1h'];
+    const operationNames: string[] = ['intraday_1m', 'intraday_5m', 'intraday_1h'];
     
-    let allSymbolsForCrawl: any[] = [];
+    const allSymbolsForCrawl: any[] = [];
     
     // Get symbols needing crawl for each interval
     for (let i = 0; i < intervals.length; i++) {
       const interval = intervals[i];
-      const operationName = operationNames[i];
+      const operationName = operationNames[i]!; // Non-null assertion since we know the array has 3 elements
       
-      const allSymbolsForInterval = await this.operationRegistry.getSymbolsForIntradayCrawl('eod', operationName, {
-        limit: 500 // Get more symbols to filter from
+      const symbolsForInterval = await this.operationRegistry.getStaleSymbols('eod', operationName, {
+        limit: 100 // Limit per interval
       });
       
-      // Filter for Canadian exchanges and non-delisted symbols
-      const symbolsForInterval = allSymbolsForInterval.filter(item => 
-        canadianExchanges.includes(item.symbol.Exchange) && 
+      // Filter out delisted symbols
+      const activeSymbols = symbolsForInterval.filter(item => 
         item.symbol.delisted === false
-      ).slice(0, 100);
+      );
       
       // Add interval info to each symbol
-      symbolsForInterval.forEach(item => {
+      activeSymbols.forEach(item => {
         allSymbolsForCrawl.push({
           symbol: item.symbol,
           interval: interval,
           operationName: operationName,
-          crawlState: item.crawlState
+          lastRun: item.lastRun,
+          lastSuccess: item.lastSuccess
         });
       });
     }
@@ -88,10 +77,11 @@ export async function scheduleIntradayCrawl(
       count: allSymbolsForCrawl.length,
       samples: allSymbolsForCrawl.slice(0, 5).map(s => ({
         symbol: s.symbol.Code,
-        exchange: s.symbol.Exchange,
+        exchange: s.symbol.eodExchange || s.symbol.Exchange,
         name: s.symbol.Name,
         interval: s.interval,
-        crawlState: s.crawlState
+        lastRun: s.lastRun,
+        lastSuccess: s.lastSuccess
       }))
     });
     
@@ -139,17 +129,20 @@ export async function crawlIntraday(
   try {
     logger.info(`Starting intraday crawl for ${symbol}.${exchange} - ${interval}`);
     
-    // Get current crawl state
+    // Get symbol to check if it exists
     const symbolDoc = await this.mongodb.collection('eodSymbols').findOne({
       Code: symbol,
-      Exchange: exchange
+      eodExchange: exchange
     });
     
     if (!symbolDoc) {
       throw new Error(`Symbol ${symbol}.${exchange} not found`);
     }
     
-    const crawlState: CrawlState = symbolDoc.intradayState?.[interval] || {
+    // Get operation status from tracker
+    const operationName = `intraday_${interval}`;
+    const operationStatus = symbolDoc.operations?.[operationName];
+    const crawlState: CrawlState = operationStatus?.crawlState || {
       finished: false
     };
     
@@ -181,9 +174,9 @@ export async function crawlIntraday(
     // Update crawl state
     const newState: CrawlState = {
       ...crawlState,
+      finished: false,
       lastProcessedDate: fromDate,
-      totalRecordsProcessed: (crawlState.totalRecordsProcessed || 0) + result.recordsSaved,
-      totalBatchesProcessed: (crawlState.totalBatchesProcessed || 0) + 1
+      totalDaysProcessed: (crawlState.totalDaysProcessed || 0) + 1
     };
     
     // Set oldest date reached
@@ -200,23 +193,20 @@ export async function crawlIntraday(
     if (result.recordsSaved === 0) {
       newState.finished = true;
       logger.info(`Intraday crawl finished for ${symbol}.${exchange} - ${interval}`, { 
-        totalRecords: newState.totalRecordsProcessed,
+        symbol,
+        exchange,
+        interval,
         oldestDate: newState.oldestDateReached,
         newestDate: newState.newestDateReached,
-        batches: newState.totalBatchesProcessed
       });
     }
     
-    // Update symbol with new crawl state
-    await this.mongodb.collection('eodSymbols').updateOne(
-      { Code: symbol, Exchange: exchange },
-      { 
-        $set: { 
-          [`intradayState.${interval}`]: newState,
-          lastIntradayUpdate: new Date()
-        } 
-      }
-    );
+    // Update operation tracker with crawl state
+    await this.operationRegistry.updateOperation('eod', symbol, operationName, {
+      status: newState.finished ? 'success' : 'partial',
+      recordCount: result.recordsSaved,
+      crawlState: newState
+    });
     
     // If not finished, schedule next batch
     if (!newState.finished) {
@@ -271,7 +261,7 @@ export async function fetchIntraday(
     if (!symbolCountry) {
       const symbolDoc = await this.mongodb.collection('eodSymbols').findOne({
         Code: symbol,
-        Exchange: exchange
+        eodExchange: exchange
       });
       
       if (!symbolDoc) {
@@ -287,10 +277,8 @@ export async function fetchIntraday(
     }
     
     // Build URL
-    // Use utility function to handle US symbols and EUFUND special case
-    const exchangeSuffix = getEodExchangeSuffix(exchange, symbolCountry);
-    
-    const url = new URL(`https://eodhd.com/api/intraday/${symbol}.${exchangeSuffix}`);
+    // Note: 'exchange' parameter here is already the eodExchange from scheduling
+    const url = new URL(`https://eodhd.com/api/intraday/${symbol}.${exchange}`);
     url.searchParams.append('api_token', apiKey);
     url.searchParams.append('fmt', 'json');
     url.searchParams.append('interval', interval);