/** * Example usage of the new functional batch processing approach */ import { processItems, processSymbols, processProxies, processBatchJob } from '../utils/batch-helpers'; import { queueManager } from '../services/queue.service'; // Example 1: Process a list of symbols for live data export async function exampleSymbolProcessing() { const symbols = ['AAPL', 'GOOGL', 'MSFT', 'TSLA', 'AMZN']; const result = await processSymbols(symbols, queueManager, { operation: 'live-data', service: 'market-data', provider: 'yahoo', totalDelayMs: 60000, // 1 minute total useBatching: false, // Process directly priority: 1 }); console.log('Symbol processing result:', result); // Output: { jobsCreated: 5, mode: 'direct', totalItems: 5, duration: 1234 } } // Example 2: Process proxies in batches export async function exampleProxyProcessing() { const proxies = [ { host: '1.1.1.1', port: 8080 }, { host: '2.2.2.2', port: 3128 }, // ... more proxies ]; const result = await processProxies(proxies, queueManager, { totalDelayMs: 3600000, // 1 hour total useBatching: true, // Use batch mode batchSize: 100, // 100 proxies per batch priority: 2 }); console.log('Proxy processing result:', result); // Output: { jobsCreated: 10, mode: 'batch', totalItems: 1000, batchesCreated: 10, duration: 2345 } } // Example 3: Custom processing with generic function export async function exampleCustomProcessing() { const customData = [ { id: 1, name: 'Item 1' }, { id: 2, name: 'Item 2' }, { id: 3, name: 'Item 3' } ]; const result = await processItems( customData, (item, index) => ({ // Transform each item for processing itemId: item.id, itemName: item.name, processIndex: index, timestamp: new Date().toISOString() }), queueManager, { totalDelayMs: 30000, // 30 seconds total useBatching: false, // Direct processing priority: 1, retries: 3 } ); console.log('Custom processing result:', result); } // Example 4: Batch job processor (used by workers) export async function exampleBatchJobProcessor(jobData: any) { // This would be called by a BullMQ worker when processing batch jobs const result = await processBatchJob(jobData, queueManager); console.log('Batch job processed:', result); // Output: { batchIndex: 0, itemsProcessed: 100, jobsCreated: 100 } return result; } // Example: Simple functional approach /* await processSymbols(symbols, queueManager, { operation: 'live-data', service: 'data-service', provider: 'yahoo', totalDelayMs: 3600000, useBatching: true, batchSize: 200, priority: 2 }); */