initial setup, its a mess

apps/data-service/src/services/queue.service.ts

@@ -1,9 +1,10 @@
 import { Queue, Worker, QueueEvents } from 'bullmq';
 import { Logger } from '@stock-bot/logger';
+import { providerRegistry } from './provider-registry.service';
 
 export interface JobData {
-  type: 'proxy-fetch' | 'proxy-check' | 'market-data' | 'historical-data';
-  service: 'proxy' | 'market-data' | 'analytics';
+  type: string;
+  service: string;
   provider: string;
   operation: string;
   payload: any;
@@ -29,6 +30,9 @@ export class QueueService {
 
     this.logger.info('Initializing queue service...');
 
+    // Register all providers first
+    await this.registerProviders();
+
     const connection = {
       host: process.env.DRAGONFLY_HOST || 'localhost',
       port: parseInt(process.env.DRAGONFLY_PORT || '6379'),
@@ -42,65 +46,85 @@ export class QueueService {
         connection,
         concurrency: 10
       });
-      this.queueEvents = new QueueEvents('data-service-queue', { connection });
-
-      // Test connection
+      this.queueEvents = new QueueEvents('data-service-queue', { connection });      // Test connection
       await this.queue.waitUntilReady();
       await this.worker.waitUntilReady();
       await this.queueEvents.waitUntilReady();
 
       this.setupEventListeners();
-      this.setupScheduledTasks();
-
+      
       this.isInitialized = true;
       this.logger.info('Queue service initialized successfully');
+      
+      await this.setupScheduledTasks();
     } catch (error) {
       this.logger.error('Failed to initialize queue service', { error });
       throw error;
     }
   }
 
-  private async processJob(job: any) {
-    const { type, service, provider, operation, payload }: JobData = job.data;
+  private async registerProviders() {
+    this.logger.info('Registering providers...');
     
-    this.logger.info('Processing job', { id: job.id, type, service, provider, operation });
-
     try {
-      switch (type) {
-        case 'proxy-fetch':
-          return await this.handleProxyFetch(payload);
-        case 'proxy-check':
-          return await this.handleProxyCheck(payload);
-        case 'market-data':
-          return await this.handleMarketData(payload);
-        case 'historical-data':
-          return await this.handleHistoricalData(payload);
-        default:
-          throw new Error(`Unknown job type: ${type}`);
-      }
+      // Import and register all providers
+      const { proxyProvider } = await import('../providers/proxy.provider');
+      const { quotemediaProvider } = await import('../providers/quotemedia.provider');
+      const { yahooProvider } = await import('../providers/yahoo.provider');
+
+      providerRegistry.registerProvider(proxyProvider);
+      providerRegistry.registerProvider(quotemediaProvider);
+      providerRegistry.registerProvider(yahooProvider);
+
+      this.logger.info('All providers registered successfully');
     } catch (error) {
-      this.logger.error('Job failed', { id: job.id, type, error });
+      this.logger.error('Failed to register providers', { error });
       throw error;
     }
   }
 
-  private async handleProxyFetch(payload: any) {
-    const { proxyService } = await import('./proxy.service');
-    return await proxyService.fetchProxiesFromSources();
-  }
+  private async processJob(job: any) {
+    const { service, provider, operation, payload }: JobData = job.data;
+    
+    this.logger.info('Processing job', { 
+      id: job.id, 
+      service, 
+      provider, 
+      operation,
+      payloadKeys: Object.keys(payload || {})
+    });
 
-  private async handleProxyCheck(payload: { proxies: any[] }) {
-    const { proxyService } = await import('./proxy.service');
-    return await proxyService.checkProxies(payload.proxies);
-  }
-  private async handleMarketData(payload: { symbol: string }) {
-    const { marketDataProvider } = await import('../providers/market-data.provider.js');
-    return await marketDataProvider.getLiveData(payload.symbol);
-  }
+    try {
+      // Get handler from registry
+      const handler = providerRegistry.getHandler(service, provider, operation);
+      
+      if (!handler) {
+        throw new Error(`No handler found for ${service}:${provider}:${operation}`);
+      }
 
-  private async handleHistoricalData(payload: { symbol: string; from: Date; to: Date; interval: string }) {
-    const { marketDataProvider } = await import('../providers/market-data.provider.js');
-    return await marketDataProvider.getHistoricalData(payload.symbol, payload.from, payload.to, payload.interval);
+      // Execute the handler
+      const result = await handler(payload);
+      
+      this.logger.info('Job completed successfully', { 
+        id: job.id, 
+        service, 
+        provider, 
+        operation 
+      });
+      
+      return result;
+
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      this.logger.error('Job failed', { 
+        id: job.id, 
+        service, 
+        provider, 
+        operation, 
+        error: errorMessage 
+      });
+      throw error;
+    }
   }
 
   private setupEventListeners() {
@@ -117,27 +141,50 @@ export class QueueService {
     });
   }
 
-  private setupScheduledTasks() {
-    // Market data refresh every minute
-    this.addRecurringJob({
-      type: 'market-data',
-      service: 'market-data',
-      provider: 'unified-data',
-      operation: 'refresh-cache',
-      payload: { symbols: ['AAPL', 'GOOGL', 'MSFT'] }
-    }, '*/1 * * * *');
+  private async setupScheduledTasks() {
+    try {
+      // Market data refresh every minute using Yahoo Finance
+      await this.addRecurringJob({
+        type: 'market-data-refresh',
+        service: 'market-data',
+        provider: 'yahoo-finance',
+        operation: 'live-data',
+        payload: { symbol: 'AAPL' }
+      }, '*/1 * * * *');
 
-    // Proxy check every 15 minutes  
-    this.addRecurringJob({
-      type: 'proxy-fetch',
-      service: 'proxy',
-      provider: 'proxy-service',
-      operation: 'fetch-and-check',
-      payload: {}
-    }, '*/15 * * * *');
+      // Market data refresh using QuoteMedia every 2 minutes
+      await this.addRecurringJob({
+        type: 'market-data-quotemedia',
+        service: 'market-data',
+        provider: 'quotemedia',
+        operation: 'batch-quotes',
+        payload: { symbols: ['GOOGL', 'MSFT', 'TSLA'] }
+      }, '*/2 * * * *');
 
-    this.logger.info('Scheduled tasks configured');
+      // Proxy fetch every 15 minutes  
+      await this.addRecurringJob({
+        type: 'proxy-maintenance',
+        service: 'proxy',
+        provider: 'proxy-service',
+        operation: 'fetch-and-check',
+        payload: {}
+      }, '*/15 * * * *');
+
+      // Proxy cleanup daily at 2 AM
+      await this.addRecurringJob({
+        type: 'proxy-cleanup',
+        service: 'proxy',
+        provider: 'proxy-service',
+        operation: 'cleanup-old-data',
+        payload: { daysToKeep: 7 }
+      }, '0 2 * * *');
+
+      this.logger.info('Scheduled tasks configured');
+    } catch (error) {
+      this.logger.error('Failed to setup scheduled tasks', error);
+    }
   }
+
   async addJob(jobData: JobData, options?: any) {
     if (!this.isInitialized) {
       throw new Error('Queue service not initialized. Call initialize() first.');
@@ -185,7 +232,9 @@ export class QueueService {
       failed: failed.length,
       delayed: delayed.length
     };
-  }  async getQueueStatus() {
+  }
+
+  async getQueueStatus() {
     if (!this.isInitialized) {
       throw new Error('Queue service not initialized. Call initialize() first.');
     }
@@ -209,11 +258,14 @@ export class QueueService {
   }
 
   getRegisteredProviders() {
-    return [
-      { name: 'proxy-service', type: 'proxy', operations: ['fetch-and-check', 'check-specific'] },
-      { name: 'market-data-provider', type: 'market-data', operations: ['live-data', 'historical-data'] }
-    ];
+    return providerRegistry.getProviders().map(({ key, config }) => ({
+      key,
+      name: config.name,
+      service: config.service,
+      operations: Object.keys(config.operations)
+    }));
   }
+
   async shutdown() {
     if (!this.isInitialized) {
       this.logger.warn('Queue service not initialized, nothing to shutdown');