added new queue lib with batch processor and provider

libs/queue/examples/batch-processing.ts

@@ -0,0 +1,200 @@
+import { initializeBatchCache, processItems, QueueManager } from '@stock-bot/queue';
+
+async function batchProcessingExample() {
+  console.log('=== Batch Processing Example ===');
+
+  // Initialize queue manager
+  const queueManager = new QueueManager({
+    queueName: 'batch-example-queue',
+    workers: 2,
+    concurrency: 5,
+  });
+
+  // Register data processing provider
+  queueManager.registerProvider('data-processor', {
+    'process-item': async payload => {
+      console.log(`Processing item: ${JSON.stringify(payload)}`);
+      // Simulate processing time
+      await new Promise(resolve => setTimeout(resolve, 200));
+      return { processed: true, originalData: payload };
+    },
+
+    'analyze-symbol': async payload => {
+      // payload is now the raw symbol string
+      console.log(`Analyzing symbol: ${payload}`);
+      // Simulate analysis
+      await new Promise(resolve => setTimeout(resolve, 150));
+      return {
+        symbol: payload,
+        analysis: {
+          trend: Math.random() > 0.5 ? 'up' : 'down',
+          confidence: Math.random(),
+          timestamp: new Date().toISOString(),
+        },
+      };
+    },
+  });
+
+  await queueManager.initialize();
+  await initializeBatchCache(queueManager);
+
+  // Example 1: Direct processing (each item = separate job)
+  console.log('\n--- Direct Processing Example ---');
+  const directResult = await processItems(
+    [1, 2, 3, 4, 5], // Just pass the array directly!
+    queueManager,
+    {
+      totalDelayMs: 15000, // 15 seconds total
+      useBatching: false, // Direct mode
+      priority: 2,
+      provider: 'data-processor',
+      operation: 'process-item',
+    }
+  );
+
+  console.log('Direct processing result:', directResult);
+
+  // Example 2: Batch processing (groups of items)
+  console.log('\n--- Batch Processing Example ---');
+  const batchData = Array.from({ length: 25 }, (_, i) => ({
+    id: i + 1,
+    value: Math.random() * 100,
+    category: i % 3 === 0 ? 'A' : i % 3 === 1 ? 'B' : 'C',
+  }));
+
+  const batchResult = await processItems(batchData, queueManager, {
+    totalDelayMs: 20000, // 20 seconds total
+    useBatching: true, // Batch mode
+    batchSize: 5, // 5 items per batch
+    priority: 1,
+    provider: 'data-processor',
+    operation: 'process-item',
+  });
+
+  console.log('Batch processing result:', batchResult);
+
+  // Example 3: Symbol processing (using processItems)
+  console.log('\n--- Symbol Processing Example ---');
+  const symbols = ['AAPL', 'GOOGL', 'MSFT', 'TSLA', 'AMZN', 'META', 'NFLX'];
+
+  const symbolResult = await processItems(symbols, queueManager, {
+    operation: 'analyze-symbol',
+    provider: 'data-processor',
+    totalDelayMs: 25000, // 25 seconds total
+    useBatching: true,
+    batchSize: 3,
+    priority: 1,
+  });
+
+  console.log('Symbol processing result:', symbolResult);
+
+  // Example 4: Large dataset with optimal batching
+  console.log('\n--- Large Dataset Example ---');
+  const largeDataset = Array.from({ length: 1000 }, (_, i) => ({
+    id: i + 1,
+    data: `item-${i + 1}`,
+    random: Math.random(),
+  }));
+
+  const largeResult = await processItems(largeDataset, queueManager, {
+    totalDelayMs: 60000, // 1 minute total
+    useBatching: true,
+    batchSize: 50, // 50 items per batch
+    priority: 3,
+    provider: 'data-processor',
+    operation: 'process-item',
+    retries: 2,
+    removeOnComplete: 5,
+    removeOnFail: 10,
+  });
+
+  console.log('Large dataset result:', largeResult);
+
+  // Monitor queue progress
+  console.log('\n--- Monitoring Queue ---');
+  const monitorInterval = setInterval(async () => {
+    const stats = await queueManager.getStats();
+    console.log('Queue stats:', {
+      waiting: stats.waiting,
+      active: stats.active,
+      completed: stats.completed,
+      failed: stats.failed,
+    });
+
+    // Stop monitoring when queue is mostly empty
+    if (stats.waiting === 0 && stats.active === 0) {
+      clearInterval(monitorInterval);
+      console.log('Queue processing complete!');
+
+      setTimeout(async () => {
+        await queueManager.shutdown();
+        console.log('Shutdown complete');
+      }, 2000);
+    }
+  }, 5000);
+}
+
+// Utility function to compare processing modes
+async function compareProcessingModes() {
+  console.log('\n=== Processing Mode Comparison ===');
+
+  const queueManager = new QueueManager({
+    queueName: 'comparison-queue',
+    workers: 2,
+    concurrency: 10,
+  });
+
+  queueManager.registerProvider('test', {
+    process: async payload => {
+      await new Promise(resolve => setTimeout(resolve, 100));
+      return { processed: true, originalData: payload };
+    },
+  });
+
+  await queueManager.initialize();
+  await initializeBatchCache(queueManager);
+
+  const testData = Array.from({ length: 20 }, (_, i) => ({ id: i + 1 }));
+
+  // Test direct mode
+  console.log('Testing direct mode...');
+  const directStart = Date.now();
+  const directResult = await processItems(testData, queueManager, {
+    totalDelayMs: 10000,
+    useBatching: false,
+    provider: 'test',
+    operation: 'process',
+  });
+  console.log('Direct mode:', {
+    ...directResult,
+    actualDuration: Date.now() - directStart,
+  });
+
+  // Test batch mode
+  console.log('Testing batch mode...');
+  const batchStart = Date.now();
+  const batchResult = await processItems(testData, queueManager, {
+    totalDelayMs: 10000,
+    useBatching: true,
+    batchSize: 5,
+    provider: 'test',
+    operation: 'process',
+  });
+  console.log('Batch mode:', {
+    ...batchResult,
+    actualDuration: Date.now() - batchStart,
+  });
+
+  setTimeout(async () => {
+    await queueManager.shutdown();
+  }, 15000);
+}
+
+// Run examples
+if (require.main === module) {
+  batchProcessingExample()
+    .then(() => compareProcessingModes())
+    .catch(console.error);
+}
+
+export { batchProcessingExample, compareProcessingModes };