added new queue lib with batch processor and provider

libs/queue/examples/migration-example.ts

@@ -0,0 +1,211 @@
+// Migration example from existing QueueService to new QueueManager
+// OLD WAY (using existing QueueService)
+/*
+import { QueueService } from '../services/queue.service';
+import { providerRegistry } from '../services/provider-registry.service';
+import { processItems, initializeBatchCache } from '../utils/batch-helpers';
+
+class OldDataService {
+  private queueService: QueueService;
+
+  constructor() {
+    this.queueService = new QueueService();
+  }
+
+  async initialize() {
+    // Register providers
+    providerRegistry.register('market-data', {
+      'live-data': async (payload) => {
+        // Handle live data
+      },
+    });
+
+    await this.queueService.initialize();
+  }
+
+  async processSymbols(symbols: string[]) {
+    return processSymbols(symbols, this.queueService, {
+      operation: 'live-data',
+      service: 'market-data',
+      provider: 'yahoo',
+      totalDelayMs: 300000,
+    });
+  }
+}
+*/
+
+// NEW WAY (using @stock-bot/queue)
+import { initializeBatchCache, processItems, QueueManager } from '@stock-bot/queue';
+
+class NewDataService {
+  private queueManager: QueueManager;
+
+  constructor() {
+    this.queueManager = new QueueManager({
+      queueName: 'data-service-queue',
+      workers: 5,
+      concurrency: 20,
+    });
+  }
+
+  async initialize() {
+    // Register providers using the new API
+    this.queueManager.registerProvider('market-data', {
+      'live-data': async payload => {
+        // payload is now the raw symbol string
+        console.log('Processing live data for:', payload);
+        // Handle live data - same logic as before
+        return {
+          symbol: payload,
+          price: Math.random() * 1000,
+          timestamp: new Date().toISOString(),
+        };
+      },
+
+      'historical-data': async payload => {
+        // payload is now the raw symbol string
+        console.log('Processing historical data for:', payload);
+        // Handle historical data
+        return {
+          symbol: payload,
+          data: Array.from({ length: 100 }, (_, i) => ({
+            date: new Date(Date.now() - i * 86400000).toISOString(),
+            price: Math.random() * 1000,
+          })),
+        };
+      },
+    });
+
+    this.queueManager.registerProvider('analytics', {
+      'calculate-indicators': async payload => {
+        // payload is now the raw symbol string
+        console.log('Calculating indicators for:', payload);
+        // Calculate technical indicators
+        return {
+          symbol: payload,
+          indicators: {
+            sma20: Math.random() * 1000,
+            rsi: Math.random() * 100,
+            macd: Math.random() * 10,
+          },
+        };
+      },
+    });
+
+    await this.queueManager.initialize();
+    await initializeBatchCache(this.queueManager);
+  }
+
+  // Method that works exactly like before
+  async processSymbols(symbols: string[]) {
+    return processItems(symbols, this.queueManager, {
+      operation: 'live-data',
+      provider: 'market-data', // Note: provider name in the new system
+      totalDelayMs: 300000,
+      useBatching: false,
+      priority: 1,
+    });
+  }
+
+  // New method showcasing batch processing
+  async processSymbolsBatch(symbols: string[]) {
+    return processItems(symbols, this.queueManager, {
+      totalDelayMs: 300000,
+      useBatching: true,
+      batchSize: 50,
+      priority: 1,
+      provider: 'market-data',
+      operation: 'live-data',
+    });
+  }
+
+  // Analytics processing
+  async processAnalytics(symbols: string[]) {
+    return processItems(symbols, this.queueManager, {
+      totalDelayMs: 180000, // 3 minutes
+      useBatching: true,
+      batchSize: 20,
+      priority: 2,
+      provider: 'analytics',
+      operation: 'calculate-indicators',
+    });
+  }
+
+  async getQueueStats() {
+    return this.queueManager.getStats();
+  }
+
+  async shutdown() {
+    await this.queueManager.shutdown();
+  }
+}
+
+// Example usage
+async function migrationExample() {
+  console.log('=== Migration Example ===');
+
+  const dataService = new NewDataService();
+  await dataService.initialize();
+
+  const symbols = ['AAPL', 'GOOGL', 'MSFT', 'TSLA'];
+
+  // Test symbol processing (works like before)
+  console.log('Processing symbols (direct)...');
+  const directResult = await dataService.processSymbols(symbols.slice(0, 2));
+  console.log('Direct result:', directResult);
+
+  // Test batch processing (new capability)
+  console.log('Processing symbols (batch)...');
+  const batchResult = await dataService.processSymbolsBatch(symbols);
+  console.log('Batch result:', batchResult);
+
+  // Test analytics processing
+  console.log('Processing analytics...');
+  const analyticsResult = await dataService.processAnalytics(symbols);
+  console.log('Analytics result:', analyticsResult);
+
+  // Monitor progress
+  setInterval(async () => {
+    const stats = await dataService.getQueueStats();
+    console.log('Queue stats:', stats);
+
+    if (stats.waiting === 0 && stats.active === 0) {
+      console.log('All jobs complete!');
+      await dataService.shutdown();
+      process.exit(0);
+    }
+  }, 3000);
+}
+
+// Key Migration Steps:
+/*
+1. IMPORTS:
+   - Replace: import { QueueService } from '../services/queue.service'
+   - With: import { QueueManager } from '@stock-bot/queue'
+
+2. PROVIDER REGISTRATION:
+   - Replace: providerRegistry.register(...)
+   - With: queueManager.registerProvider(...)
+
+3. INITIALIZATION:
+   - Replace: await queueService.initialize()
+   - With: await queueManager.initialize() + await initializeBatchCache()
+
+4. BATCH HELPERS:
+   - Replace: import { processItems } from '../utils/batch-helpers'
+   - With: import { processItems } from '@stock-bot/queue'
+
+5. JOB PARAMETERS:
+   - totalDelayHours → totalDelayMs (convert hours to milliseconds)
+   - Ensure provider names match registered providers
+
+6. CONFIGURATION:
+   - Use QueueConfig interface for type safety
+   - Environment variables work the same way
+*/
+
+if (require.main === module) {
+  migrationExample().catch(console.error);
+}
+
+export { migrationExample, NewDataService };