renaming services to more suitable names

apps/data-pipeline/src/handlers/symbols/operations/index.ts

@@ -0,0 +1,9 @@
+import { syncQMSymbols } from './qm-symbols.operations';
+import { syncSymbolsFromProvider } from './sync-symbols-from-provider.operations';
+import { getSyncStatus } from './sync-status.operations';
+
+export const symbolOperations = {
+  syncQMSymbols,
+  syncSymbolsFromProvider,
+  getSyncStatus,
+};

apps/data-pipeline/src/handlers/symbols/operations/qm-symbols.operations.ts

@@ -0,0 +1,168 @@
+import { getLogger } from '@stock-bot/logger';
+import { getMongoDBClient } from '@stock-bot/mongodb-client';
+import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import type { JobPayload } from '../../../types/job-payloads';
+
+const logger = getLogger('sync-qm-symbols');
+
+export async function syncQMSymbols(payload: JobPayload): Promise<{ processed: number; created: number; updated: number }> {
+  logger.info('Starting QM symbols sync...');
+
+  try {
+    const mongoClient = getMongoDBClient();
+    const postgresClient = getPostgreSQLClient();
+
+    // 1. Get all QM symbols from MongoDB
+    const qmSymbols = await mongoClient.find('qmSymbols', {});
+    logger.info(`Found ${qmSymbols.length} QM symbols to process`);
+
+    let created = 0;
+    let updated = 0;
+
+    for (const symbol of qmSymbols) {
+      try {
+        // 2. Resolve exchange
+        const exchangeId = await resolveExchange(symbol.exchangeCode || symbol.exchange, postgresClient);
+
+        if (!exchangeId) {
+          logger.warn('Unknown exchange, skipping symbol', {
+            symbol: symbol.symbol,
+            exchange: symbol.exchangeCode || symbol.exchange,
+          });
+          continue;
+        }
+
+        // 3. Check if symbol exists
+        const existingSymbol = await findSymbol(symbol.symbol, exchangeId, postgresClient);
+
+        if (existingSymbol) {
+          // Update existing
+          await updateSymbol(existingSymbol.id, symbol, postgresClient);
+          await upsertProviderMapping(existingSymbol.id, 'qm', symbol, postgresClient);
+          updated++;
+        } else {
+          // Create new
+          const newSymbolId = await createSymbol(symbol, exchangeId, postgresClient);
+          await upsertProviderMapping(newSymbolId, 'qm', symbol, postgresClient);
+          created++;
+        }
+      } catch (error) {
+        logger.error('Failed to process symbol', { error, symbol: symbol.symbol });
+      }
+    }
+
+    // 4. Update sync status
+    await updateSyncStatus('qm', 'symbols', qmSymbols.length, postgresClient);
+
+    const result = { processed: qmSymbols.length, created, updated };
+    logger.info('QM symbols sync completed', result);
+    return result;
+  } catch (error) {
+    logger.error('QM symbols sync failed', { error });
+    throw error;
+  }
+}
+
+// Helper functions
+async function resolveExchange(exchangeCode: string, postgresClient: any): Promise<string | null> {
+  if (!exchangeCode) return null;
+
+  // Simple mapping - expand this as needed
+  const exchangeMap: Record<string, string> = {
+    NASDAQ: 'NASDAQ',
+    NYSE: 'NYSE',
+    TSX: 'TSX',
+    TSE: 'TSX', // TSE maps to TSX
+    LSE: 'LSE',
+    CME: 'CME',
+  };
+
+  const normalizedCode = exchangeMap[exchangeCode.toUpperCase()];
+  if (!normalizedCode) {
+    return null;
+  }
+
+  const query = 'SELECT id FROM exchanges WHERE code = $1';
+  const result = await postgresClient.query(query, [normalizedCode]);
+  return result.rows[0]?.id || null;
+}
+
+async function findSymbol(symbol: string, exchangeId: string, postgresClient: any): Promise<any> {
+  const query = 'SELECT * FROM symbols WHERE symbol = $1 AND exchange_id = $2';
+  const result = await postgresClient.query(query, [symbol, exchangeId]);
+  return result.rows[0] || null;
+}
+
+async function createSymbol(qmSymbol: any, exchangeId: string, postgresClient: any): Promise<string> {
+  const query = `
+    INSERT INTO symbols (symbol, exchange_id, company_name, country, currency)
+    VALUES ($1, $2, $3, $4, $5)
+    RETURNING id
+  `;
+
+  const result = await postgresClient.query(query, [
+    qmSymbol.symbol,
+    exchangeId,
+    qmSymbol.companyName || qmSymbol.name,
+    qmSymbol.countryCode || 'US',
+    qmSymbol.currency || 'USD',
+  ]);
+
+  return result.rows[0].id;
+}
+
+async function updateSymbol(symbolId: string, qmSymbol: any, postgresClient: any): Promise<void> {
+  const query = `
+    UPDATE symbols 
+    SET company_name = COALESCE($2, company_name),
+        country = COALESCE($3, country),
+        currency = COALESCE($4, currency),
+        updated_at = NOW()
+    WHERE id = $1
+  `;
+
+  await postgresClient.query(query, [
+    symbolId,
+    qmSymbol.companyName || qmSymbol.name,
+    qmSymbol.countryCode,
+    qmSymbol.currency,
+  ]);
+}
+
+async function upsertProviderMapping(
+  symbolId: string,
+  provider: string,
+  qmSymbol: any,
+  postgresClient: any
+): Promise<void> {
+  const query = `
+    INSERT INTO provider_mappings 
+    (symbol_id, provider, provider_symbol, provider_exchange, last_seen)
+    VALUES ($1, $2, $3, $4, NOW())
+    ON CONFLICT (provider, provider_symbol)
+    DO UPDATE SET 
+      symbol_id = EXCLUDED.symbol_id,
+      provider_exchange = EXCLUDED.provider_exchange,
+      last_seen = NOW()
+  `;
+
+  await postgresClient.query(query, [
+    symbolId,
+    provider,
+    qmSymbol.qmSearchCode || qmSymbol.symbol,
+    qmSymbol.exchangeCode || qmSymbol.exchange,
+  ]);
+}
+
+async function updateSyncStatus(provider: string, dataType: string, count: number, postgresClient: any): Promise<void> {
+  const query = `
+    UPDATE sync_status 
+    SET last_sync_at = NOW(),
+        last_sync_count = $3,
+        sync_errors = NULL,
+        updated_at = NOW()
+    WHERE provider = $1 AND data_type = $2
+  `;
+
+  await postgresClient.query(query, [provider, dataType, count]);
+}

apps/data-pipeline/src/handlers/symbols/operations/sync-status.operations.ts

@@ -0,0 +1,21 @@
+import { getLogger } from '@stock-bot/logger';
+import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import type { JobPayload } from '../../../types/job-payloads';
+
+const logger = getLogger('sync-status');
+
+export async function getSyncStatus(payload: JobPayload): Promise<Record<string, unknown>[]> {
+  logger.info('Getting sync status...');
+
+  try {
+    const postgresClient = getPostgreSQLClient();
+    const query = 'SELECT * FROM sync_status ORDER BY provider, data_type';
+    const result = await postgresClient.query(query);
+    
+    logger.info(`Retrieved sync status for ${result.rows.length} entries`);
+    return result.rows;
+  } catch (error) {
+    logger.error('Failed to get sync status', { error });
+    throw error;
+  }
+}

apps/data-pipeline/src/handlers/symbols/operations/sync-symbols-from-provider.operations.ts

@@ -0,0 +1,216 @@
+import { getLogger } from '@stock-bot/logger';
+import { getMongoDBClient } from "@stock-bot/mongodb-client";
+import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import type { JobPayload, SyncResult } from '../../../types/job-payloads';
+
+const logger = getLogger('enhanced-sync-symbols-from-provider');
+
+export async function syncSymbolsFromProvider(payload: JobPayload): Promise<SyncResult> {
+  const provider = payload.provider;
+  const clearFirst = payload.clearFirst || false;
+  
+  if (!provider) {
+    throw new Error('Provider is required in payload');
+  }
+
+  logger.info(`Starting ${provider} symbols sync...`, { clearFirst });
+
+  const result: SyncResult = {
+    processed: 0,
+    created: 0,
+    updated: 0,
+    skipped: 0,
+    errors: 0,
+  };
+
+  try {
+    const mongoClient = getMongoDBClient();
+    const postgresClient = getPostgreSQLClient();
+
+    // Clear existing data if requested (only symbols and mappings, keep exchanges)
+    if (clearFirst) {
+      await postgresClient.query('BEGIN');
+      await postgresClient.query('DELETE FROM provider_mappings');
+      await postgresClient.query('DELETE FROM symbols');
+      await postgresClient.query('COMMIT');
+      logger.info('Cleared existing symbols and mappings before sync');
+    }
+
+    // Start transaction
+    await postgresClient.query('BEGIN');
+
+    let symbols: Record<string, unknown>[] = [];
+
+    // Get symbols based on provider
+    const db = mongoClient.getDatabase();
+    switch (provider.toLowerCase()) {
+      case 'qm':
+        symbols = await db.collection('qmSymbols').find({}).toArray();
+        break;
+      case 'eod':
+        symbols = await db.collection('eodSymbols').find({}).toArray();
+        break;
+      case 'ib':
+        symbols = await db.collection('ibSymbols').find({}).toArray();
+        break;
+      default:
+        throw new Error(`Unsupported provider: ${provider}`);
+    }
+
+    logger.info(`Found ${symbols.length} ${provider} symbols to process`);
+    result.processed = symbols.length;
+
+    for (const symbol of symbols) {
+      try {
+        await processSingleSymbol(symbol, provider, result);
+      } catch (error) {
+        logger.error('Failed to process symbol', {
+          error,
+          symbol: symbol.symbol || symbol.code,
+          provider,
+        });
+        result.errors++;
+      }
+    }
+
+    // Update sync status
+    await updateSyncStatus(provider, 'symbols', result.processed, postgresClient);
+
+    await postgresClient.query('COMMIT');
+
+    logger.info(`${provider} symbols sync completed`, result);
+    return result;
+  } catch (error) {
+    const postgresClient = getPostgreSQLClient();
+    await postgresClient.query('ROLLBACK');
+    logger.error(`${provider} symbols sync failed`, { error });
+    throw error;
+  }
+}
+
+async function processSingleSymbol(symbol: any, provider: string, result: SyncResult): Promise<void> {
+  const symbolCode = symbol.symbol || symbol.code;
+  const exchangeCode = symbol.exchangeCode || symbol.exchange || symbol.exchange_id;
+
+  if (!symbolCode || !exchangeCode) {
+    result.skipped++;
+    return;
+  }
+
+  // Find active provider exchange mapping
+  const providerMapping = await findActiveProviderExchangeMapping(provider, exchangeCode);
+
+  if (!providerMapping) {
+    result.skipped++;
+    return;
+  }
+
+  // Check if symbol exists
+  const existingSymbol = await findSymbolByCodeAndExchange(
+    symbolCode,
+    providerMapping.master_exchange_id
+  );
+
+  if (existingSymbol) {
+    await updateSymbol(existingSymbol.id, symbol);
+    await upsertProviderMapping(existingSymbol.id, provider, symbol);
+    result.updated++;
+  } else {
+    const newSymbolId = await createSymbol(symbol, providerMapping.master_exchange_id);
+    await upsertProviderMapping(newSymbolId, provider, symbol);
+    result.created++;
+  }
+}
+
+async function findActiveProviderExchangeMapping(provider: string, providerExchangeCode: string): Promise<any> {
+  const postgresClient = getPostgreSQLClient();
+  const query = `
+    SELECT pem.*, e.code as master_exchange_code 
+    FROM provider_exchange_mappings pem
+    JOIN exchanges e ON pem.master_exchange_id = e.id
+    WHERE pem.provider = $1 AND pem.provider_exchange_code = $2 AND pem.active = true
+  `;
+  const result = await postgresClient.query(query, [provider, providerExchangeCode]);
+  return result.rows[0] || null;
+}
+
+async function findSymbolByCodeAndExchange(symbol: string, exchangeId: string): Promise<any> {
+  const postgresClient = getPostgreSQLClient();
+  const query = 'SELECT * FROM symbols WHERE symbol = $1 AND exchange_id = $2';
+  const result = await postgresClient.query(query, [symbol, exchangeId]);
+  return result.rows[0] || null;
+}
+
+async function createSymbol(symbol: any, exchangeId: string): Promise<string> {
+  const postgresClient = getPostgreSQLClient();
+  const query = `
+    INSERT INTO symbols (symbol, exchange_id, company_name, country, currency)
+    VALUES ($1, $2, $3, $4, $5)
+    RETURNING id
+  `;
+
+  const result = await postgresClient.query(query, [
+    symbol.symbol || symbol.code,
+    exchangeId,
+    symbol.companyName || symbol.name || symbol.company_name,
+    symbol.countryCode || symbol.country_code || 'US',
+    symbol.currency || 'USD',
+  ]);
+
+  return result.rows[0].id;
+}
+
+async function updateSymbol(symbolId: string, symbol: any): Promise<void> {
+  const postgresClient = getPostgreSQLClient();
+  const query = `
+    UPDATE symbols 
+    SET company_name = COALESCE($2, company_name),
+        country = COALESCE($3, country),
+        currency = COALESCE($4, currency),
+        updated_at = NOW()
+    WHERE id = $1
+  `;
+
+  await postgresClient.query(query, [
+    symbolId,
+    symbol.companyName || symbol.name || symbol.company_name,
+    symbol.countryCode || symbol.country_code,
+    symbol.currency,
+  ]);
+}
+
+async function upsertProviderMapping(symbolId: string, provider: string, symbol: any): Promise<void> {
+  const postgresClient = getPostgreSQLClient();
+  const query = `
+    INSERT INTO provider_mappings 
+    (symbol_id, provider, provider_symbol, provider_exchange, last_seen)
+    VALUES ($1, $2, $3, $4, NOW())
+    ON CONFLICT (provider, provider_symbol)
+    DO UPDATE SET 
+      symbol_id = EXCLUDED.symbol_id,
+      provider_exchange = EXCLUDED.provider_exchange,
+      last_seen = NOW()
+  `;
+
+  await postgresClient.query(query, [
+    symbolId,
+    provider,
+    symbol.qmSearchCode || symbol.symbol || symbol.code,
+    symbol.exchangeCode || symbol.exchange || symbol.exchange_id,
+  ]);
+}
+
+async function updateSyncStatus(provider: string, dataType: string, count: number, postgresClient: any): Promise<void> {
+  const query = `
+    INSERT INTO sync_status (provider, data_type, last_sync_at, last_sync_count, sync_errors)
+    VALUES ($1, $2, NOW(), $3, NULL)
+    ON CONFLICT (provider, data_type)
+    DO UPDATE SET 
+      last_sync_at = NOW(),
+      last_sync_count = EXCLUDED.last_sync_count,
+      sync_errors = NULL,
+      updated_at = NOW()
+  `;
+
+  await postgresClient.query(query, [provider, dataType, count]);
+}

apps/data-pipeline/src/handlers/symbols/symbols.handler.ts

@@ -0,0 +1,41 @@
+import { getLogger } from '@stock-bot/logger';
+import { handlerRegistry, type HandlerConfig, type ScheduledJobConfig } from '@stock-bot/queue';
+import { symbolOperations } from './operations';
+
+const logger = getLogger('symbols-handler');
+
+const HANDLER_NAME = 'symbols';
+
+const symbolsHandlerConfig: HandlerConfig = {
+  concurrency: 1,
+  maxAttempts: 3,
+  scheduledJobs: [
+    {
+      operation: 'sync-qm-symbols',
+      cronPattern: '0 2 * * *', // Daily at 2 AM
+      payload: {},
+      priority: 5,
+      immediately: false,
+    } as ScheduledJobConfig,
+    {
+      operation: 'sync-symbols-qm',
+      cronPattern: '0 4 * * *', // Daily at 4 AM
+      payload: { provider: 'qm', clearFirst: false },
+      priority: 5,
+      immediately: false,
+    } as ScheduledJobConfig,
+  ],
+  operations: {
+    'sync-qm-symbols': symbolOperations.syncQMSymbols,
+    'sync-symbols-qm': symbolOperations.syncSymbolsFromProvider,
+    'sync-symbols-eod': symbolOperations.syncSymbolsFromProvider,
+    'sync-symbols-ib': symbolOperations.syncSymbolsFromProvider,
+    'sync-status': symbolOperations.getSyncStatus,
+  },
+};
+
+export function initializeSymbolsHandler(): void {
+  logger.info('Registering symbols handler...');
+  handlerRegistry.registerHandler(HANDLER_NAME, symbolsHandlerConfig);
+  logger.info('Symbols handler registered successfully');
+}