switching to generic queue lib

apps/data-service/src/routes/market-data.routes.ts

@@ -3,7 +3,7 @@
  */
 import { Hono } from 'hono';
 import { getLogger } from '@stock-bot/logger';
-import { queueManager } from '../services/queue.service';
+import { queueManager } from '../services/queue-manager.service';
 
 const logger = getLogger('market-data-routes');
 
@@ -16,9 +16,8 @@ marketDataRoutes.get('/api/live/:symbol', async c => {
 
   try {
     // Queue job for live data using Yahoo provider
-    const job = await queueManager.addJob({
+    const job = await queueManager.addJob('market-data-live', {
       type: 'market-data-live',
-      service: 'market-data',
       provider: 'yahoo-finance',
       operation: 'live-data',
       payload: { symbol },
@@ -47,9 +46,8 @@ marketDataRoutes.get('/api/historical/:symbol', async c => {
     const toDate = to ? new Date(to) : new Date(); // Now
 
     // Queue job for historical data using Yahoo provider
-    const job = await queueManager.addJob({
+    const job = await queueManager.addJob('market-data-historical', {
       type: 'market-data-historical',
-      service: 'market-data',
       provider: 'yahoo-finance',
       operation: 'historical-data',
       payload: {
@@ -72,3 +70,50 @@ marketDataRoutes.get('/api/historical/:symbol', async c => {
     return c.json({ status: 'error', message: 'Failed to queue historical data job' }, 500);
   }
 });
+
+// Batch processing endpoint using new queue system
+marketDataRoutes.post('/api/process-symbols', async c => {
+  try {
+    const {
+      symbols,
+      provider = 'ib',
+      operation = 'fetch-session',
+      useBatching = true,
+      totalDelayMs = 30000,
+      batchSize = 10,
+    } = await c.req.json();
+
+    if (!symbols || !Array.isArray(symbols) || symbols.length === 0) {
+      return c.json({ status: 'error', message: 'Invalid symbols array' }, 400);
+    }
+
+    logger.info('Batch processing symbols', {
+      count: symbols.length,
+      provider,
+      operation,
+      useBatching,
+    });
+
+    const result = await queueManager.processSymbols(symbols, {
+      totalDelayMs,
+      useBatching,
+      batchSize,
+      priority: 2,
+      provider,
+      operation,
+      retries: 2,
+      removeOnComplete: 5,
+      removeOnFail: 10,
+    });
+
+    return c.json({
+      status: 'success',
+      message: 'Batch processing initiated',
+      result,
+      symbols: symbols.length,
+    });
+  } catch (error) {
+    logger.error('Failed to process symbols batch', { error });
+    return c.json({ status: 'error', message: 'Failed to process symbols batch' }, 500);
+  }
+});

apps/data-service/src/routes/queue.routes.ts

@@ -3,7 +3,7 @@
  */
 import { Hono } from 'hono';
 import { getLogger } from '@stock-bot/logger';
-import { queueManager } from '../services/queue.service';
+import { queueManager } from '../services/queue-manager.service';
 
 const logger = getLogger('queue-routes');
 
@@ -12,8 +12,8 @@ export const queueRoutes = new Hono();
 // Queue management endpoints
 queueRoutes.get('/api/queue/status', async c => {
   try {
-    const status = await queueManager.getQueueStatus();
-    return c.json({ status: 'success', data: status });
+    const stats = await queueManager.getStats();
+    return c.json({ status: 'success', data: stats });
   } catch (error) {
     logger.error('Failed to get queue status', { error });
     return c.json({ status: 'error', message: 'Failed to get queue status' }, 500);
@@ -22,8 +22,8 @@ queueRoutes.get('/api/queue/status', async c => {
 
 queueRoutes.post('/api/queue/job', async c => {
   try {
-    const jobData = await c.req.json();
-    const job = await queueManager.addJob(jobData);
+    const { name, data, options } = await c.req.json();
+    const job = await queueManager.addJob(name, data, options);
     return c.json({ status: 'success', jobId: job.id });
   } catch (error) {
     logger.error('Failed to add job', { error });
@@ -34,9 +34,9 @@ queueRoutes.post('/api/queue/job', async c => {
 // Provider registry endpoints
 queueRoutes.get('/api/providers', async c => {
   try {
-    const { providerRegistry } = await import('../services/provider-registry.service');
-    const providers = providerRegistry.getProviders();
-    return c.json({ status: 'success', providers });
+    const { providerRegistry } = await import('@stock-bot/queue');
+    const configs = providerRegistry.getProviderConfigs();
+    return c.json({ status: 'success', providers: configs });
   } catch (error) {
     logger.error('Failed to get providers', { error });
     return c.json({ status: 'error', message: 'Failed to get providers' }, 500);
@@ -46,7 +46,7 @@ queueRoutes.get('/api/providers', async c => {
 // Add new endpoint to see scheduled jobs
 queueRoutes.get('/api/scheduled-jobs', async c => {
   try {
-    const { providerRegistry } = await import('../services/provider-registry.service');
+    const { providerRegistry } = await import('@stock-bot/queue');
     const jobs = providerRegistry.getAllScheduledJobs();
     return c.json({
       status: 'success',
@@ -59,13 +59,14 @@ queueRoutes.get('/api/scheduled-jobs', async c => {
   }
 });
 
-queueRoutes.post('/api/queue/drain', async c => {
+queueRoutes.post('/api/queue/clean', async c => {
   try {
-    await queueManager.drainQueue();
-    const status = await queueManager.getQueueStatus();
-    return c.json({ status: 'success', message: 'Queue drained', queueStatus: status });
+    const { grace = 60000 } = await c.req.json(); // Default 1 minute
+    await queueManager.clean(grace);
+    const stats = await queueManager.getStats();
+    return c.json({ status: 'success', message: 'Queue cleaned', queueStats: stats });
   } catch (error) {
-    logger.error('Failed to drain queue', { error });
-    return c.json({ status: 'error', message: 'Failed to drain queue' }, 500);
+    logger.error('Failed to clean queue', { error });
+    return c.json({ status: 'error', message: 'Failed to clean queue' }, 500);
   }
 });