qm scaffolding done

apps/stock/data-ingestion/src/handlers/qm/actions/filings.action.ts

@@ -0,0 +1,245 @@
+/**
+ * QM Filings Actions - Fetch and update SEC filings
+ */
+
+import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
+import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
+import { QMSessionManager } from '../shared/session-manager';
+import { QMOperationTracker } from '../shared/operation-tracker';
+
+// Cache tracker instance
+let operationTracker: QMOperationTracker | null = null;
+
+/**
+ * Get or initialize the operation tracker
+ */
+async function getOperationTracker(handler: BaseHandler): Promise<QMOperationTracker> {
+  if (!operationTracker) {
+    const { initializeQMOperations } = await import('../shared/operation-registry');
+    operationTracker = await initializeQMOperations(handler.mongodb, handler.logger);
+  }
+  return operationTracker;
+}
+
+/**
+ * Update filings for a single symbol
+ */
+export async function updateFilings(
+  this: BaseHandler,
+  input: {
+    symbol: string;
+    symbolId: number;
+  },
+  _context?: ExecutionContext
+): Promise<{
+  success: boolean;
+  symbol: string;
+  message: string;
+  data?: any;
+}> {
+  const { symbol, symbolId } = input;
+  
+  this.logger.info('Fetching filings', { symbol, symbolId });
+
+  const sessionManager = QMSessionManager.getInstance();
+  sessionManager.initialize(this.cache, this.logger);
+
+  // Get a session - you'll need to add the appropriate session ID for filings
+  const sessionId = QM_SESSION_IDS.LOOKUP; // TODO: Update with correct session ID
+  const session = await sessionManager.getSession(sessionId);
+  
+  if (!session || !session.uuid) {
+    throw new Error(`No active session found for QM filings`);
+  }
+
+  try {
+    // Build API request for filings
+    const searchParams = new URLSearchParams({
+      symbol: symbol,
+      symbolId: symbolId.toString(),
+      qmodTool: 'Filings',
+      webmasterId: '500',
+      limit: '50' // Get recent filings
+    });
+    
+    // TODO: Update with correct filings endpoint
+    const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/filings.json?${searchParams.toString()}`;
+
+    const response = await fetch(apiUrl, {
+      method: 'GET',
+      headers: session.headers,
+      proxy: session.proxy,
+    });
+
+    if (!response.ok) {
+      throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
+    }
+
+    const filingsData = await response.json();
+    
+    // Update session success stats
+    await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
+
+    // Process and store filings data
+    if (filingsData && filingsData.length > 0) {
+      // Store filings in a separate collection
+      await this.mongodb.batchUpsert(
+        'qmFilings',
+        filingsData.map((filing: any) => ({
+          ...filing,
+          symbol,
+          symbolId,
+          updated_at: new Date()
+        })),
+        ['symbol', 'filingDate', 'formType', 'accessionNumber'] // Unique keys
+      );
+
+      // Update symbol to track last filings update
+      const tracker = await getOperationTracker(this);
+      await tracker.updateSymbolOperation(symbol, 'filings_update', {
+        status: 'success',
+        lastRecordDate: new Date(),
+        recordCount: filingsData.length
+      });
+
+      this.logger.info('Filings updated successfully', { 
+        symbol,
+        filingsCount: filingsData.length
+      });
+
+      return {
+        success: true,
+        symbol,
+        message: `Filings updated for ${symbol}`,
+        data: { count: filingsData.length }
+      };
+    } else {
+      // Some symbols may not have filings (non-US companies, etc)
+      const tracker = await getOperationTracker(this);
+      await tracker.updateSymbolOperation(symbol, 'filings_update', {
+        status: 'success',
+        lastRecordDate: new Date(),
+        recordCount: 0
+      });
+
+      this.logger.info('No filings found for symbol', { symbol });
+      return {
+        success: true,
+        symbol,
+        message: `No filings found for ${symbol}`,
+        data: { count: 0 }
+      };
+    }
+
+  } catch (error) {
+    // Update session failure stats
+    if (session.uuid) {
+      await sessionManager.incrementFailedCalls(sessionId, session.uuid);
+    }
+
+    this.logger.error('Error fetching filings', { 
+      symbol,
+      error: error instanceof Error ? error.message : 'Unknown error'
+    });
+    
+    // Track failure
+    const tracker = await getOperationTracker(this);
+    await tracker.updateSymbolOperation(symbol, 'filings_update', {
+      status: 'failure'
+    });
+    
+    return {
+      success: false,
+      symbol,
+      message: `Failed to fetch filings: ${error instanceof Error ? error.message : 'Unknown error'}`
+    };
+  }
+}
+
+/**
+ * Schedule filings updates for symbols that need refreshing
+ */
+export async function scheduleFilingsUpdates(
+  this: BaseHandler,
+  input: {
+    limit?: number;
+    forceUpdate?: boolean;
+  } = {},
+  _context?: ExecutionContext
+): Promise<{
+  message: string;
+  symbolsQueued: number;
+  errors: number;
+}> {
+  const { limit = 100, forceUpdate = false } = input;
+  const tracker = await getOperationTracker(this);
+  
+  this.logger.info('Scheduling filings updates', { limit, forceUpdate });
+
+  try {
+    // Get symbols that need updating
+    const staleSymbols = await tracker.getStaleSymbols('filings_update', {
+      minHoursSinceRun: forceUpdate ? 0 : 24, // Daily for filings
+      limit
+    });
+
+    if (staleSymbols.length === 0) {
+      this.logger.info('No symbols need filings updates');
+      return {
+        message: 'No symbols need filings updates',
+        symbolsQueued: 0,
+        errors: 0
+      };
+    }
+
+    this.logger.info(`Found ${staleSymbols.length} symbols needing filings updates`);
+
+    // Get full symbol data to include symbolId
+    const symbolDocs = await this.mongodb.find('qmSymbols', {
+      symbol: { $in: staleSymbols }
+    }, {
+      projection: { symbol: 1, symbolId: 1 }
+    });
+
+    let queued = 0;
+    let errors = 0;
+
+    // Schedule individual update jobs for each symbol
+    for (const doc of symbolDocs) {
+      try {
+        if (!doc.symbolId) {
+          this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`);
+          continue;
+        }
+
+        await this.scheduleOperation('update-filings', {
+          symbol: doc.symbol,
+          symbolId: doc.symbolId
+        }, {
+          priority: 5, // Lower priority than financial data
+          delay: queued * 2000 // 2 seconds between jobs
+        });
+        
+        queued++;
+      } catch (error) {
+        this.logger.error(`Failed to schedule filings update for ${doc.symbol}`, { error });
+        errors++;
+      }
+    }
+
+    this.logger.info('Filings update scheduling completed', {
+      symbolsQueued: queued,
+      errors,
+      total: staleSymbols.length
+    });
+
+    return {
+      message: `Scheduled filings updates for ${queued} symbols`,
+      symbolsQueued: queued,
+      errors
+    };
+  } catch (error) {
+    this.logger.error('Filings scheduling failed', { error });
+    throw error;
+  }
+}