stock-bot/apps/stock/data-ingestion/src/handlers/qm/actions/insiders.action.ts

/**
 * QM Insiders Actions - Fetch and update insider trading data
 */

import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMSessionManager } from '../shared/session-manager';

/**
 * Update insider transactions for a single symbol
 */
export async function updateInsiders(
  this: QMHandler,
  input: {
    symbol: string;
    symbolId: number;
    qmSearchCode: string;
    lookbackDays?: number;
  },
  _context?: ExecutionContext
): Promise<{
  success: boolean;
  symbol: string;
  message: string;
  data?: any;
}> {
  const { symbol, symbolId, qmSearchCode, lookbackDays = 365 } = input;
  
  this.logger.info('Fetching insider transactions', { symbol, symbolId, lookbackDays });

  const sessionManager = QMSessionManager.getInstance();
  await sessionManager.initialize(this.cache, this.logger);

  const sessionId = QM_SESSION_IDS.LOOKUP;
  const session = await sessionManager.getSession(sessionId);
  
  if (!session || !session.uuid) {
    throw new Error(`No active session found for QM insiders`);
  }

  try {
    // Calculate date range
    const endDate = new Date();
    const startDate = new Date();
    startDate.setDate(startDate.getDate() - lookbackDays);

    // Build API request for insider transactions
    const searchParams = new URLSearchParams({
      symbol: symbol,
      symbolId: symbolId.toString(),
      qmodTool: 'InsiderActivity',
      webmasterId: '500',
      startDate: startDate.toISOString().split('T')[0],
      endDate: endDate.toISOString().split('T')[0],
      includeOptions: 'true',
      pageSize: '100'
    } as Record<string, string>);
    
    const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/insiders.json?${searchParams.toString()}`;

    const response = await fetch(apiUrl, {
      method: 'GET',
      headers: session.headers,
      proxy: session.proxy,
    });

    if (!response.ok) {
      throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
    }

    const insiderData = await response.json();
    
    // Update session success stats
    await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);

    // Process and store insider data
    if (insiderData && insiderData.transactions && insiderData.transactions.length > 0) {
      const processedTransactions = insiderData.transactions.map((transaction: any) => ({
        symbol,
        symbolId,
        transactionDate: new Date(transaction.transactionDate),
        filingDate: new Date(transaction.filingDate),
        insiderName: transaction.insiderName,
        insiderTitle: transaction.insiderTitle || 'Unknown',
        transactionType: transaction.transactionType,
        shares: parseFloat(transaction.shares) || 0,
        pricePerShare: parseFloat(transaction.pricePerShare) || 0,
        totalValue: parseFloat(transaction.totalValue) || 0,
        sharesOwned: parseFloat(transaction.sharesOwned) || 0,
        ownershipType: transaction.ownershipType || 'Direct',
        formType: transaction.formType || 'Form 4',
        transactionCode: transaction.transactionCode,
        updated_at: new Date()
      }));

      // Store in MongoDB
      await this.mongodb.batchUpsert(
        'qmInsiders',
        processedTransactions,
        ['symbol', 'transactionDate', 'insiderName', 'transactionType'] // Unique keys
      );

      // Calculate summary statistics
      const totalBuys = processedTransactions.filter((t: any) => 
        t.transactionType === 'Buy' || t.transactionType === 'Purchase'
      ).length;
      
      const totalSells = processedTransactions.filter((t: any) => 
        t.transactionType === 'Sell' || t.transactionType === 'Sale'
      ).length;

      const totalBuyValue = processedTransactions
        .filter((t: any) => t.transactionType === 'Buy' || t.transactionType === 'Purchase')
        .reduce((sum: number, t: any) => sum + t.totalValue, 0);

      const totalSellValue = processedTransactions
        .filter((t: any) => t.transactionType === 'Sell' || t.transactionType === 'Sale')
        .reduce((sum: number, t: any) => sum + t.totalValue, 0);

      // Update operation tracking
      await this.operationRegistry.updateOperation('qm', qmSearchCode, 'insiders_update', {
        status: 'success',
        lastRecordDate: endDate,
        recordCount: processedTransactions.length,
        metadata: {
          totalBuys,
          totalSells,
          totalBuyValue,
          totalSellValue,
          netValue: totalBuyValue - totalSellValue,
          uniqueInsiders: new Set(processedTransactions.map((t: any) => t.insiderName)).size
        }
      });

      this.logger.info('Insider transactions updated successfully', { 
        symbol,
        transactionCount: processedTransactions.length,
        totalBuys,
        totalSells,
        netValue: totalBuyValue - totalSellValue
      });

      return {
        success: true,
        symbol,
        message: `Updated ${processedTransactions.length} insider transactions`,
        data: { 
          count: processedTransactions.length,
          totalBuys,
          totalSells,
          totalBuyValue,
          totalSellValue,
          netValue: totalBuyValue - totalSellValue
        }
      };
    } else {
      // No insider data
      this.logger.info('No insider transactions found', { symbol });
      
      await this.operationRegistry.updateOperation('qm', qmSearchCode, 'insiders_update', {
        status: 'success',
        lastRecordDate: endDate,
        recordCount: 0
      });
      
      return {
        success: true,
        symbol,
        message: 'No insider transactions found',
        data: { count: 0 }
      };
    }

  } catch (error) {
    // Update session failure stats
    if (session.uuid) {
      await sessionManager.incrementFailedCalls(sessionId, session.uuid);
    }

    this.logger.error('Error fetching insider transactions', { 
      symbol,
      error: error instanceof Error ? error.message : 'Unknown error'
    });
    
    // Update operation tracking for failure
    await this.operationRegistry.updateOperation('qm', qmSearchCode, 'insiders_update', {
      status: 'failure',
      error: error instanceof Error ? error.message : 'Unknown error'
    });
    
    return {
      success: false,
      symbol,
      message: `Failed to fetch insider transactions: ${error instanceof Error ? error.message : 'Unknown error'}`
    };
  }
}

/**
 * Schedule insider updates for symbols
 */
export async function scheduleInsidersUpdates(
  this: QMHandler,
  input: {
    limit?: number;
    minHoursSinceRun?: number;
    forceUpdate?: boolean;
  } = {},
  _context?: ExecutionContext
): Promise<{
  message: string;
  symbolsQueued: number;
  errors: number;
}> {
  const { limit = 100, minHoursSinceRun = 24 * 7, forceUpdate = false } = input;
  
  this.logger.info('Scheduling insider updates', { limit, minHoursSinceRun, forceUpdate });

  try {
    // Get symbols that need insider updates
    const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'insiders_update', {
      minHoursSinceRun: forceUpdate ? 0 : minHoursSinceRun,
      limit
    });

    if (staleSymbols.length === 0) {
      this.logger.info('No symbols need insider updates');
      return {
        message: 'No symbols need insider updates',
        symbolsQueued: 0,
        errors: 0
      };
    }

    // Get full symbol data
    const symbolsToProcess = await this.mongodb.find('qmSymbols', {
      qmSearchCode: { $in: staleSymbols }
    }, {
      projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 }
    });

    this.logger.info(`Found ${symbolsToProcess.length} symbols for insider updates`);

    let symbolsQueued = 0;
    let errors = 0;

    // Schedule update jobs
    for (const doc of symbolsToProcess) {
      try {
        if (!doc.symbolId) {
          this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`);
          continue;
        }

        await this.scheduleOperation('update-insiders', {
          symbol: doc.symbol,
          symbolId: doc.symbolId,
          qmSearchCode: doc.qmSearchCode
        }, {
          priority: 5, // Medium priority
          delay: symbolsQueued * 1000 // 1 second between jobs
        });
        
        symbolsQueued++;
      } catch (error) {
        this.logger.error(`Failed to schedule insider update for ${doc.symbol}`, { error });
        errors++;
      }
    }

    this.logger.info('Insider update scheduling completed', {
      symbolsQueued,
      errors
    });

    return {
      message: `Scheduled insider updates for ${symbolsQueued} symbols`,
      symbolsQueued,
      errors
    };
  } catch (error) {
    this.logger.error('Insider scheduling failed', { error });
    throw error;
  }
}