work on qm filings

apps/stock/data-ingestion/src/handlers/qm/actions/insiders.action.ts

@@ -0,0 +1,286 @@
+/**
+ * QM Insiders Actions - Fetch and update insider trading data
+ */
+
+import type { ExecutionContext } from '@stock-bot/handlers';
+import type { QMHandler } from '../qm.handler';
+import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
+import { QMSessionManager } from '../shared/session-manager';
+
+/**
+ * Update insider transactions for a single symbol
+ */
+export async function updateInsiders(
+  this: QMHandler,
+  input: {
+    symbol: string;
+    symbolId: number;
+    qmSearchCode: string;
+    lookbackDays?: number;
+  },
+  _context?: ExecutionContext
+): Promise<{
+  success: boolean;
+  symbol: string;
+  message: string;
+  data?: any;
+}> {
+  const { symbol, symbolId, qmSearchCode, lookbackDays = 365 } = input;
+  
+  this.logger.info('Fetching insider transactions', { symbol, symbolId, lookbackDays });
+
+  const sessionManager = QMSessionManager.getInstance();
+  await sessionManager.initialize(this.cache, this.logger);
+
+  const sessionId = QM_SESSION_IDS.LOOKUP;
+  const session = await sessionManager.getSession(sessionId);
+  
+  if (!session || !session.uuid) {
+    throw new Error(`No active session found for QM insiders`);
+  }
+
+  try {
+    // Calculate date range
+    const endDate = new Date();
+    const startDate = new Date();
+    startDate.setDate(startDate.getDate() - lookbackDays);
+
+    // Build API request for insider transactions
+    const searchParams = new URLSearchParams({
+      symbol: symbol,
+      symbolId: symbolId.toString(),
+      qmodTool: 'InsiderActivity',
+      webmasterId: '500',
+      startDate: startDate.toISOString().split('T')[0],
+      endDate: endDate.toISOString().split('T')[0],
+      includeOptions: 'true',
+      pageSize: '100'
+    } as Record<string, string>);
+    
+    const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/insiders.json?${searchParams.toString()}`;
+
+    const response = await fetch(apiUrl, {
+      method: 'GET',
+      headers: session.headers,
+      proxy: session.proxy,
+    });
+
+    if (!response.ok) {
+      throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
+    }
+
+    const insiderData = await response.json();
+    
+    // Update session success stats
+    await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
+
+    // Process and store insider data
+    if (insiderData && insiderData.transactions && insiderData.transactions.length > 0) {
+      const processedTransactions = insiderData.transactions.map((transaction: any) => ({
+        symbol,
+        symbolId,
+        transactionDate: new Date(transaction.transactionDate),
+        filingDate: new Date(transaction.filingDate),
+        insiderName: transaction.insiderName,
+        insiderTitle: transaction.insiderTitle || 'Unknown',
+        transactionType: transaction.transactionType,
+        shares: parseFloat(transaction.shares) || 0,
+        pricePerShare: parseFloat(transaction.pricePerShare) || 0,
+        totalValue: parseFloat(transaction.totalValue) || 0,
+        sharesOwned: parseFloat(transaction.sharesOwned) || 0,
+        ownershipType: transaction.ownershipType || 'Direct',
+        formType: transaction.formType || 'Form 4',
+        transactionCode: transaction.transactionCode,
+        updated_at: new Date()
+      }));
+
+      // Store in MongoDB
+      await this.mongodb.batchUpsert(
+        'qmInsiders',
+        processedTransactions,
+        ['symbol', 'transactionDate', 'insiderName', 'transactionType'] // Unique keys
+      );
+
+      // Calculate summary statistics
+      const totalBuys = processedTransactions.filter((t: any) => 
+        t.transactionType === 'Buy' || t.transactionType === 'Purchase'
+      ).length;
+      
+      const totalSells = processedTransactions.filter((t: any) => 
+        t.transactionType === 'Sell' || t.transactionType === 'Sale'
+      ).length;
+
+      const totalBuyValue = processedTransactions
+        .filter((t: any) => t.transactionType === 'Buy' || t.transactionType === 'Purchase')
+        .reduce((sum: number, t: any) => sum + t.totalValue, 0);
+
+      const totalSellValue = processedTransactions
+        .filter((t: any) => t.transactionType === 'Sell' || t.transactionType === 'Sale')
+        .reduce((sum: number, t: any) => sum + t.totalValue, 0);
+
+      // Update operation tracking
+      await this.operationRegistry.updateOperation('qm', qmSearchCode, 'insiders_update', {
+        status: 'success',
+        lastRecordDate: endDate,
+        recordCount: processedTransactions.length,
+        metadata: {
+          totalBuys,
+          totalSells,
+          totalBuyValue,
+          totalSellValue,
+          netValue: totalBuyValue - totalSellValue,
+          uniqueInsiders: new Set(processedTransactions.map((t: any) => t.insiderName)).size
+        }
+      });
+
+      this.logger.info('Insider transactions updated successfully', { 
+        symbol,
+        transactionCount: processedTransactions.length,
+        totalBuys,
+        totalSells,
+        netValue: totalBuyValue - totalSellValue
+      });
+
+      return {
+        success: true,
+        symbol,
+        message: `Updated ${processedTransactions.length} insider transactions`,
+        data: { 
+          count: processedTransactions.length,
+          totalBuys,
+          totalSells,
+          totalBuyValue,
+          totalSellValue,
+          netValue: totalBuyValue - totalSellValue
+        }
+      };
+    } else {
+      // No insider data
+      this.logger.info('No insider transactions found', { symbol });
+      
+      await this.operationRegistry.updateOperation('qm', qmSearchCode, 'insiders_update', {
+        status: 'success',
+        lastRecordDate: endDate,
+        recordCount: 0
+      });
+      
+      return {
+        success: true,
+        symbol,
+        message: 'No insider transactions found',
+        data: { count: 0 }
+      };
+    }
+
+  } catch (error) {
+    // Update session failure stats
+    if (session.uuid) {
+      await sessionManager.incrementFailedCalls(sessionId, session.uuid);
+    }
+
+    this.logger.error('Error fetching insider transactions', { 
+      symbol,
+      error: error instanceof Error ? error.message : 'Unknown error'
+    });
+    
+    // Update operation tracking for failure
+    await this.operationRegistry.updateOperation('qm', qmSearchCode, 'insiders_update', {
+      status: 'failure',
+      error: error instanceof Error ? error.message : 'Unknown error'
+    });
+    
+    return {
+      success: false,
+      symbol,
+      message: `Failed to fetch insider transactions: ${error instanceof Error ? error.message : 'Unknown error'}`
+    };
+  }
+}
+
+/**
+ * Schedule insider updates for symbols
+ */
+export async function scheduleInsidersUpdates(
+  this: QMHandler,
+  input: {
+    limit?: number;
+    minHoursSinceRun?: number;
+    forceUpdate?: boolean;
+  } = {},
+  _context?: ExecutionContext
+): Promise<{
+  message: string;
+  symbolsQueued: number;
+  errors: number;
+}> {
+  const { limit = 100, minHoursSinceRun = 24 * 7, forceUpdate = false } = input;
+  
+  this.logger.info('Scheduling insider updates', { limit, minHoursSinceRun, forceUpdate });
+
+  try {
+    // Get symbols that need insider updates
+    const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'insiders_update', {
+      minHoursSinceRun: forceUpdate ? 0 : minHoursSinceRun,
+      limit
+    });
+
+    if (staleSymbols.length === 0) {
+      this.logger.info('No symbols need insider updates');
+      return {
+        message: 'No symbols need insider updates',
+        symbolsQueued: 0,
+        errors: 0
+      };
+    }
+
+    // Get full symbol data
+    const symbolsToProcess = await this.mongodb.find('qmSymbols', {
+      qmSearchCode: { $in: staleSymbols }
+    }, {
+      projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 }
+    });
+
+    this.logger.info(`Found ${symbolsToProcess.length} symbols for insider updates`);
+
+    let symbolsQueued = 0;
+    let errors = 0;
+
+    // Schedule update jobs
+    for (const doc of symbolsToProcess) {
+      try {
+        if (!doc.symbolId) {
+          this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`);
+          continue;
+        }
+
+        await this.scheduleOperation('update-insiders', {
+          symbol: doc.symbol,
+          symbolId: doc.symbolId,
+          qmSearchCode: doc.qmSearchCode
+        }, {
+          priority: 5, // Medium priority
+          delay: symbolsQueued * 1000 // 1 second between jobs
+        });
+        
+        symbolsQueued++;
+      } catch (error) {
+        this.logger.error(`Failed to schedule insider update for ${doc.symbol}`, { error });
+        errors++;
+      }
+    }
+
+    this.logger.info('Insider update scheduling completed', {
+      symbolsQueued,
+      errors
+    });
+
+    return {
+      message: `Scheduled insider updates for ${symbolsQueued} symbols`,
+      symbolsQueued,
+      errors
+    };
+  } catch (error) {
+    this.logger.error('Insider scheduling failed', { error });
+    throw error;
+  }
+}