intraday test

apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts

@@ -48,12 +48,15 @@ export async function scheduleIntradayCrawl(
       const operationName = operationNames[i]!; // Non-null assertion since we know the array has 3 elements
       
       const symbolsForInterval = await this.operationRegistry.getStaleSymbols('eod', operationName, {
-        limit: 100 // Limit per interval
+        limit: 1000, // Get more to filter
+        symbolFilter: { symbol: 'AAPL' } // Filter for AAPL only
       });
       
-      // Filter out delisted symbols
+      // Filter out delisted symbols and ensure we get AAPL with US exchange
       const activeSymbols = symbolsForInterval.filter(item => 
-        item.symbol.delisted === false
+        item.symbol.delisted === false && 
+        item.symbol.Code === 'AAPL' && 
+        (item.symbol.eodExchange === 'US' || item.symbol.Exchange === 'US')
       );
       
       // Add interval info to each symbol
@@ -75,13 +78,18 @@ export async function scheduleIntradayCrawl(
     
     logger.info(`Found ${allSymbolsForCrawl.length} symbol/interval combinations needing intraday data`, {
       count: allSymbolsForCrawl.length,
+      byInterval: {
+        '1m': allSymbolsForCrawl.filter(s => s.interval === '1m').length,
+        '5m': allSymbolsForCrawl.filter(s => s.interval === '5m').length,
+        '1h': allSymbolsForCrawl.filter(s => s.interval === '1h').length
+      },
       samples: allSymbolsForCrawl.slice(0, 5).map(s => ({
         symbol: s.symbol.Code,
         exchange: s.symbol.eodExchange || s.symbol.Exchange,
         name: s.symbol.Name,
         interval: s.interval,
-        lastRun: s.lastRun,
-        lastSuccess: s.lastSuccess
+        lastRun: s.lastRun ? new Date(s.lastRun).toISOString() : 'never',
+        lastSuccess: s.lastSuccess ? new Date(s.lastSuccess).toISOString() : 'never'
       }))
     });
     
@@ -97,6 +105,7 @@ export async function scheduleIntradayCrawl(
         interval,
         country: symbol.Country
       }, {
+        priority: 5, // Initial crawl jobs get priority 5 (lower priority)
         attempts: 3,
         backoff: {
           type: 'exponential',
@@ -127,7 +136,12 @@ export async function crawlIntraday(
   const { symbol, exchange, interval, country } = input;
   
   try {
-    logger.info(`Starting intraday crawl for ${symbol}.${exchange} - ${interval}`);
+    logger.info(`Starting intraday crawl for ${symbol}.${exchange} - ${interval}`, {
+      symbol,
+      exchange,
+      interval,
+      country
+    });
     
     // Get symbol to check if it exists
     const symbolDoc = await this.mongodb.collection('eodSymbols').findOne({
@@ -161,6 +175,19 @@ export async function crawlIntraday(
     fromDate = new Date(toDate);
     fromDate.setDate(fromDate.getDate() - maxDays + 1);
     
+    logger.info(`Fetching intraday batch for ${symbol}.${exchange} - ${interval} from ${fromDate.toISOString().split('T')[0]} to ${toDate.toISOString().split('T')[0]}`, {
+      symbol,
+      exchange,
+      interval,
+      fromDate: fromDate.toISOString(),
+      toDate: toDate.toISOString(),
+      maxDays,
+      crawlState: {
+        lastProcessedDate: crawlState.lastProcessedDate,
+        totalDaysProcessed: crawlState.totalDaysProcessed || 0
+      }
+    });
+    
     // Fetch data for this batch
     const result = await fetchIntraday.call(this, {
       symbol,
@@ -192,12 +219,14 @@ export async function crawlIntraday(
     // Check if we're finished (no data returned means we've reached the end)
     if (result.recordsSaved === 0) {
       newState.finished = true;
-      logger.info(`Intraday crawl finished for ${symbol}.${exchange} - ${interval}`, { 
+      logger.info(`Intraday crawl finished for ${symbol}.${exchange} - ${interval} (${newState.oldestDateReached?.toISOString().split('T')[0]} to ${newState.newestDateReached?.toISOString().split('T')[0]})`, { 
         symbol,
         exchange,
         interval,
-        oldestDate: newState.oldestDateReached,
-        newestDate: newState.newestDateReached,
+        oldestDate: newState.oldestDateReached?.toISOString(),
+        newestDate: newState.newestDateReached?.toISOString(),
+        totalDaysProcessed: newState.totalDaysProcessed,
+        noDataReturned: true
       });
     }
     
@@ -216,6 +245,7 @@ export async function crawlIntraday(
         interval,
         country
       }, {
+        priority: 3, // Continuation jobs get higher priority (3) than initial jobs (5)
         attempts: 3,
         backoff: {
           type: 'exponential',
@@ -224,11 +254,14 @@ export async function crawlIntraday(
         delay: 5000 // Wait 5 seconds before next batch
       });
       
-      logger.info('Scheduled next intraday batch', { 
+      logger.info(`Scheduled next intraday batch for ${symbol}.${exchange} - ${interval}`, { 
         symbol, 
         exchange, 
         interval,
-        nextFromDate: fromDate.toISOString()
+        currentBatchFrom: fromDate.toISOString(),
+        currentBatchTo: toDate.toISOString(),
+        recordsSaved: result.recordsSaved,
+        totalDaysProcessed: newState.totalDaysProcessed
       });
     }
     
@@ -252,8 +285,13 @@ export async function fetchIntraday(
   
   try {
     logger.info(`Fetching intraday data for ${symbol}.${exchange} - ${interval}`, { 
+      symbol,
+      exchange,
+      interval,
       from: fromDate?.toISOString().split('T')[0],
-      to: toDate?.toISOString().split('T')[0]
+      to: toDate?.toISOString().split('T')[0],
+      country,
+      url: `https://eodhd.com/api/intraday/${symbol}.${exchange}`
     });
     
     // Get country if not provided