added routes and simplified batch processor

apps/data-service/src/examples/batch-processing-examples.ts

@@ -0,0 +1,117 @@
+/**
+ * Example usage of the new functional batch processing approach
+ */
+
+import { processItems, processSymbols, processProxies, processBatchJob } from '../utils/batch-helpers';
+import { queueManager } from '../services/queue.service';
+
+// Example 1: Process a list of symbols for live data
+export async function exampleSymbolProcessing() {
+  const symbols = ['AAPL', 'GOOGL', 'MSFT', 'TSLA', 'AMZN'];
+  
+  const result = await processSymbols(symbols, queueManager, {
+    operation: 'live-data',
+    service: 'market-data', 
+    provider: 'yahoo',
+    totalDelayMs: 60000, // 1 minute total
+    useBatching: false,  // Process directly
+    priority: 1
+  });
+  
+  console.log('Symbol processing result:', result);
+  // Output: { jobsCreated: 5, mode: 'direct', totalItems: 5, duration: 1234 }
+}
+
+// Example 2: Process proxies in batches
+export async function exampleProxyProcessing() {
+  const proxies = [
+    { host: '1.1.1.1', port: 8080 },
+    { host: '2.2.2.2', port: 3128 },
+    // ... more proxies
+  ];
+  
+  const result = await processProxies(proxies, queueManager, {
+    totalDelayMs: 3600000, // 1 hour total
+    useBatching: true,     // Use batch mode
+    batchSize: 100,        // 100 proxies per batch
+    priority: 2
+  });
+  
+  console.log('Proxy processing result:', result);
+  // Output: { jobsCreated: 10, mode: 'batch', totalItems: 1000, batchesCreated: 10, duration: 2345 }
+}
+
+// Example 3: Custom processing with generic function
+export async function exampleCustomProcessing() {
+  const customData = [
+    { id: 1, name: 'Item 1' },
+    { id: 2, name: 'Item 2' },
+    { id: 3, name: 'Item 3' }
+  ];
+  
+  const result = await processItems(
+    customData,
+    (item, index) => ({
+      // Transform each item for processing
+      itemId: item.id,
+      itemName: item.name,
+      processIndex: index,
+      timestamp: new Date().toISOString()
+    }),
+    queueManager,
+    {
+      totalDelayMs: 30000,  // 30 seconds total
+      useBatching: false,   // Direct processing
+      priority: 1,
+      retries: 3
+    }
+  );
+  
+  console.log('Custom processing result:', result);
+}
+
+// Example 4: Batch job processor (used by workers)
+export async function exampleBatchJobProcessor(jobData: any) {
+  // This would be called by a BullMQ worker when processing batch jobs
+  const result = await processBatchJob(jobData, queueManager);
+  
+  console.log('Batch job processed:', result);
+  // Output: { batchIndex: 0, itemsProcessed: 100, jobsCreated: 100 }
+  
+  return result;
+}
+
+// Comparison: Old vs New approach
+
+// OLD COMPLEX WAY:
+/*
+const batchProcessor = new BatchProcessor(queueManager);
+await batchProcessor.initialize();
+await batchProcessor.processItems({
+  items: symbols,
+  batchSize: 200,
+  totalDelayMs: 3600000,
+  jobNamePrefix: 'yahoo-live',
+  operation: 'live-data',
+  service: 'data-service', 
+  provider: 'yahoo',
+  priority: 2,
+  createJobData: (symbol, index) => ({ symbol }),
+  useBatching: true,
+  removeOnComplete: 5,
+  removeOnFail: 3
+});
+*/
+
+// NEW SIMPLE WAY:
+/*
+await processSymbols(symbols, queueManager, {
+  operation: 'live-data',
+  service: 'data-service',
+  provider: 'yahoo', 
+  totalDelayMs: 3600000,
+  useBatching: true,
+  batchSize: 200,
+  priority: 2
+});
+*/