initial qm operation tracker

apps/stock/data-ingestion/src/handlers/qm/actions/index.ts

@@ -4,4 +4,5 @@
 
 export { checkSessions, createSession } from './session.action';
 export { searchSymbols, spiderSymbol } from './symbol.action';
+export { updatePrices, updateIntradayBars, getOperationStats } from './price.action';
 

apps/stock/data-ingestion/src/handlers/qm/actions/price.action.ts

@@ -0,0 +1,297 @@
+/**
+ * QM Price Actions - Price data updates with operation tracking
+ */
+
+import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
+import { QMOperationTracker } from '../shared/operation-tracker';
+
+// Cache tracker instance
+let operationTracker: QMOperationTracker | null = null;
+
+/**
+ * Get or initialize the operation tracker
+ */
+async function getOperationTracker(handler: BaseHandler): Promise<QMOperationTracker> {
+  if (!operationTracker) {
+    const { initializeQMOperations } = await import('../shared/operation-registry');
+    operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
+  }
+  return operationTracker;
+}
+
+/**
+ * Update daily price data for stale symbols
+ */
+export async function updatePrices(
+  this: BaseHandler,
+  input: {
+    limit?: number;
+    symbols?: string[];
+  } = {},
+  _context?: ExecutionContext
+): Promise<{
+  message: string;
+  symbolsUpdated: number;
+  errors: number;
+}> {
+  const { limit = 100, symbols } = input;
+  const tracker = await getOperationTracker(this);
+  
+  this.logger.info('Starting price update operation', { limit, specificSymbols: symbols?.length });
+
+  try {
+    // Get symbols that need updating
+    let symbolsToUpdate: string[];
+    
+    if (symbols && symbols.length > 0) {
+      // Update specific symbols
+      symbolsToUpdate = symbols;
+    } else {
+      // Get stale symbols
+      symbolsToUpdate = await tracker.getStaleSymbols('price_update', {
+        minHoursSinceRun: 24,
+        limit
+      });
+    }
+
+    if (symbolsToUpdate.length === 0) {
+      this.logger.info('No symbols need price updates');
+      return {
+        message: 'No symbols need price updates',
+        symbolsUpdated: 0,
+        errors: 0
+      };
+    }
+
+    this.logger.info(`Found ${symbolsToUpdate.length} symbols for price update`);
+
+    let updated = 0;
+    let errors = 0;
+    const updateResults = [];
+
+    // Process symbols (in real implementation, you'd fetch prices from QM API)
+    for (const symbol of symbolsToUpdate) {
+      try {
+        // TODO: Actual price fetching logic here
+        // const prices = await fetchPricesFromQM(symbol);
+        
+        // For now, simulate the update
+        const mockPrices = {
+          symbol,
+          lastPrice: Math.random() * 1000,
+          volume: Math.floor(Math.random() * 1000000),
+          date: new Date()
+        };
+
+        // Track the operation
+        updateResults.push({
+          symbol,
+          operation: 'price_update',
+          data: {
+            status: 'success' as const,
+            lastRecordDate: mockPrices.date,
+            recordCount: 1
+          }
+        });
+        
+        updated++;
+      } catch (error) {
+        this.logger.error(`Failed to update prices for ${symbol}`, { error });
+        
+        updateResults.push({
+          symbol,
+          operation: 'price_update',
+          data: {
+            status: 'failure' as const
+          }
+        });
+        
+        errors++;
+      }
+    }
+
+    // Bulk update operation tracking
+    if (updateResults.length > 0) {
+      await tracker.bulkUpdateSymbolOperations(updateResults);
+    }
+
+    this.logger.info('Price update operation completed', {
+      symbolsUpdated: updated,
+      errors,
+      total: symbolsToUpdate.length
+    });
+
+    return {
+      message: `Updated prices for ${updated} symbols`,
+      symbolsUpdated: updated,
+      errors
+    };
+  } catch (error) {
+    this.logger.error('Price update operation failed', { error });
+    throw error;
+  }
+}
+
+/**
+ * Update intraday price bars - crawls backwards until no more data
+ */
+export async function updateIntradayBars(
+  this: BaseHandler,
+  input: {
+    symbol?: string;
+    limit?: number;
+  } = {},
+  _context?: ExecutionContext
+): Promise<{
+  message: string;
+  symbol: string;
+  barsCollected: number;
+  crawlFinished: boolean;
+}> {
+  const { symbol, limit = 1 } = input;
+  const tracker = await getOperationTracker(this);
+  
+  try {
+    // Get symbols for intraday crawl
+    let symbolData;
+    if (symbol) {
+      symbolData = await tracker.getSymbolsForIntradayCrawl('intraday_bars', {
+        limit: 1
+      }).then(symbols => symbols.find(s => s.symbol === symbol));
+    } else {
+      const symbols = await tracker.getSymbolsForIntradayCrawl('intraday_bars', {
+        limit
+      });
+      symbolData = symbols[0];
+    }
+
+    if (!symbolData) {
+      return {
+        message: 'No symbols available for intraday crawl',
+        symbol: '',
+        barsCollected: 0,
+        crawlFinished: false
+      };
+    }
+
+    this.logger.info('Processing intraday bars', {
+      symbol: symbolData.symbol,
+      crawlState: symbolData.crawlState
+    });
+
+    let barsCollected = 0;
+    let crawlFinished = false;
+
+    if (symbolData.crawlState?.finished) {
+      // Already finished initial crawl, just update from last record
+      this.logger.debug('Symbol already crawled, updating from last record', {
+        symbol: symbolData.symbol,
+        lastRecord: symbolData.lastRecordDate
+      });
+
+      // TODO: Fetch bars from lastRecordDate to now
+      const newBars = 10; // Mock data
+      
+      await tracker.updateSymbolOperation(symbolData.symbol, 'intraday_bars', {
+        status: 'success',
+        lastRecordDate: new Date(),
+        recordCount: (symbolData.crawlState as any).recordCount + newBars
+      });
+
+      return {
+        message: `Updated ${newBars} new bars for ${symbolData.symbol}`,
+        symbol: symbolData.symbol,
+        barsCollected: newBars,
+        crawlFinished: true
+      };
+    }
+
+    // Initial crawl - go backwards until no data
+    let currentDate = new Date();
+    let oldestDate = currentDate;
+    let totalBars = 0;
+    let consecutiveEmptyDays = 0;
+    const maxEmptyDays = 5; // Stop after 5 consecutive days with no data
+
+    while (consecutiveEmptyDays < maxEmptyDays) {
+      // TODO: Actual bar fetching logic
+      // const bars = await fetchIntradayBars(symbolData.symbol, currentDate);
+      
+      // Mock data - simulate decreasing data as we go back
+      const bars = currentDate > new Date('2020-01-01') ? Math.floor(Math.random() * 100) : 0;
+      
+      if (bars === 0) {
+        consecutiveEmptyDays++;
+      } else {
+        consecutiveEmptyDays = 0;
+        totalBars += bars;
+        oldestDate = new Date(currentDate);
+      }
+
+      // Update progress
+      await tracker.updateSymbolOperation(symbolData.symbol, 'intraday_bars', {
+        status: 'partial',
+        lastRecordDate: new Date(),
+        recordCount: totalBars,
+        crawlState: {
+          finished: false,
+          oldestDateReached: oldestDate
+        }
+      });
+
+      // Move to previous day
+      currentDate.setDate(currentDate.getDate() - 1);
+      
+      // Limit crawl for this execution
+      if (totalBars > 1000) {
+        this.logger.info('Reached bar limit for this execution', {
+          symbol: symbolData.symbol,
+          barsCollected: totalBars
+        });
+        break;
+      }
+    }
+
+    // Check if we finished the crawl
+    if (consecutiveEmptyDays >= maxEmptyDays) {
+      crawlFinished = true;
+      await tracker.markCrawlFinished(symbolData.symbol, 'intraday_bars', oldestDate);
+      
+      this.logger.info('Completed initial crawl for symbol', {
+        symbol: symbolData.symbol,
+        totalBars,
+        oldestDate
+      });
+    }
+
+    return {
+      message: `Collected ${totalBars} bars for ${symbolData.symbol}`,
+      symbol: symbolData.symbol,
+      barsCollected: totalBars,
+      crawlFinished
+    };
+  } catch (error) {
+    this.logger.error('Intraday bars update failed', { error });
+    throw error;
+  }
+}
+
+/**
+ * Get operation statistics
+ */
+export async function getOperationStats(
+  this: BaseHandler,
+  input: {
+    operation: string;
+  },
+  _context?: ExecutionContext
+): Promise<any> {
+  const tracker = await getOperationTracker(this);
+  
+  const stats = await tracker.getOperationStats(input.operation);
+  
+  return {
+    operation: input.operation,
+    ...stats
+  };
+}

apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts

@@ -5,6 +5,8 @@
 import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
 import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
 import { QMSessionManager } from '../shared/session-manager';
+import { QMOperationTracker } from '../shared/operation-tracker';
+import { initializeQMOperations } from '../shared/operation-registry';
 import type { Exchange, SymbolSpiderJob } from '../shared/types';
 
 /**