added routes and simplified batch processor

apps/data-service/src/routes/health.routes.ts

@@ -0,0 +1,20 @@
+/**
+ * Health check routes
+ */
+import { Hono } from 'hono';
+import { queueManager } from '../services/queue.service';
+
+export const healthRoutes = new Hono();
+
+// Health check endpoint
+healthRoutes.get('/health', (c) => {
+  return c.json({ 
+    service: 'data-service', 
+    status: 'healthy',
+    timestamp: new Date().toISOString(),
+    queue: {
+      status: 'running',
+      workers: queueManager.getWorkerCount()
+    }
+  });
+});

apps/data-service/src/routes/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Routes index - exports all route modules
+ */
+export { healthRoutes } from './health.routes';
+export { queueRoutes } from './queue.routes';
+export { marketDataRoutes } from './market-data.routes';
+export { proxyRoutes } from './proxy.routes';
+export { testRoutes } from './test.routes';

apps/data-service/src/routes/market-data.routes.ts

@@ -0,0 +1,74 @@
+/**
+ * Market data routes
+ */
+import { Hono } from 'hono';
+import { getLogger } from '@stock-bot/logger';
+import { queueManager } from '../services/queue.service';
+
+const logger = getLogger('market-data-routes');
+
+export const marketDataRoutes = new Hono();
+
+// Market data endpoints
+marketDataRoutes.get('/api/live/:symbol', async (c) => {
+  const symbol = c.req.param('symbol');
+  logger.info('Live data request', { symbol });
+  
+  try {
+    // Queue job for live data using Yahoo provider
+    const job = await queueManager.addJob({
+      type: 'market-data-live',
+      service: 'market-data',
+      provider: 'yahoo-finance',
+      operation: 'live-data',
+      payload: { symbol }
+    });
+    return c.json({ 
+      status: 'success', 
+      message: 'Live data job queued',
+      jobId: job.id,
+      symbol 
+    });
+  } catch (error) {
+    logger.error('Failed to queue live data job', { symbol, error });
+    return c.json({ status: 'error', message: 'Failed to queue live data job' }, 500);
+  }
+});
+
+marketDataRoutes.get('/api/historical/:symbol', async (c) => {
+  const symbol = c.req.param('symbol');
+  const from = c.req.query('from');
+  const to = c.req.query('to');
+  
+  logger.info('Historical data request', { symbol, from, to });
+  
+  try {
+    const fromDate = from ? new Date(from) : new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days ago
+    const toDate = to ? new Date(to) : new Date(); // Now
+    
+    // Queue job for historical data using Yahoo provider
+    const job = await queueManager.addJob({
+      type: 'market-data-historical',
+      service: 'market-data',
+      provider: 'yahoo-finance',
+      operation: 'historical-data',
+      payload: { 
+        symbol, 
+        from: fromDate.toISOString(), 
+        to: toDate.toISOString() 
+      }
+    });
+    
+    return c.json({ 
+      status: 'success', 
+      message: 'Historical data job queued',
+      jobId: job.id,
+      symbol, 
+      from: fromDate, 
+      to: toDate 
+    });
+  } catch (error) {
+    logger.error('Failed to queue historical data job', { symbol, from, to, error });
+    return c.json({ status: 'error', message: 'Failed to queue historical data job' }, 500);
+  }
+});

apps/data-service/src/routes/proxy.routes.ts

@@ -0,0 +1,79 @@
+/**
+ * Proxy management routes
+ */
+import { Hono } from 'hono';
+import { getLogger } from '@stock-bot/logger';
+import { queueManager } from '../services/queue.service';
+
+const logger = getLogger('proxy-routes');
+
+export const proxyRoutes = new Hono();
+
+// Proxy management endpoints
+proxyRoutes.post('/api/proxy/fetch', async (c) => {
+  try {
+    const job = await queueManager.addJob({
+      type: 'proxy-fetch',
+      service: 'proxy',
+      provider: 'proxy-service',
+      operation: 'fetch-and-check',
+      payload: {},
+      priority: 5
+    });
+    
+    return c.json({ 
+      status: 'success', 
+      jobId: job.id, 
+      message: 'Proxy fetch job queued' 
+    });
+  } catch (error) {
+    logger.error('Failed to queue proxy fetch', { error });
+    return c.json({ status: 'error', message: 'Failed to queue proxy fetch' }, 500);
+  }
+});
+
+proxyRoutes.post('/api/proxy/check', async (c) => {
+  try {
+    const { proxies } = await c.req.json();
+    const job = await queueManager.addJob({
+      type: 'proxy-check',
+      service: 'proxy',
+      provider: 'proxy-service',
+      operation: 'check-specific',
+      payload: { proxies },
+      priority: 8
+    });
+    
+    return c.json({ 
+      status: 'success', 
+      jobId: job.id, 
+      message: `Proxy check job queued for ${proxies.length} proxies` 
+    });
+  } catch (error) {
+    logger.error('Failed to queue proxy check', { error });
+    return c.json({ status: 'error', message: 'Failed to queue proxy check' }, 500);
+  }
+});
+
+// Get proxy stats via queue
+proxyRoutes.get('/api/proxy/stats', async (c) => {
+  try {
+    const job = await queueManager.addJob({
+      type: 'proxy-stats',
+      service: 'proxy',
+      provider: 'proxy-service',
+      operation: 'get-stats',
+      payload: {},
+      priority: 3
+    });
+    
+    return c.json({ 
+      status: 'success', 
+      jobId: job.id, 
+      message: 'Proxy stats job queued' 
+    });
+  } catch (error) {
+    logger.error('Failed to queue proxy stats', { error });
+    return c.json({ status: 'error', message: 'Failed to queue proxy stats' }, 500);
+  }
+});

apps/data-service/src/routes/queue.routes.ts

@@ -0,0 +1,58 @@
+/**
+ * Queue management routes
+ */
+import { Hono } from 'hono';
+import { getLogger } from '@stock-bot/logger';
+import { queueManager } from '../services/queue.service';
+
+const logger = getLogger('queue-routes');
+
+export const queueRoutes = new Hono();
+
+// Queue management endpoints
+queueRoutes.get('/api/queue/status', async (c) => {
+  try {
+    const status = await queueManager.getQueueStatus();
+    return c.json({ status: 'success', data: status });
+  } catch (error) {
+    logger.error('Failed to get queue status', { error });
+    return c.json({ status: 'error', message: 'Failed to get queue status' }, 500);
+  }
+});
+
+queueRoutes.post('/api/queue/job', async (c) => {
+  try {
+    const jobData = await c.req.json();
+    const job = await queueManager.addJob(jobData);
+    return c.json({ status: 'success', jobId: job.id });
+  } catch (error) {
+    logger.error('Failed to add job', { error });
+    return c.json({ status: 'error', message: 'Failed to add job' }, 500);
+  }
+});
+
+// Provider registry endpoints
+queueRoutes.get('/api/providers', async (c) => {
+  try {
+    const providers = queueManager.getRegisteredProviders();
+    return c.json({ status: 'success', providers });
+  } catch (error) {
+    logger.error('Failed to get providers', { error });
+    return c.json({ status: 'error', message: 'Failed to get providers' }, 500);
+  }
+});
+
+// Add new endpoint to see scheduled jobs
+queueRoutes.get('/api/scheduled-jobs', async (c) => {
+  try {
+    const jobs = queueManager.getScheduledJobsInfo();
+    return c.json({ 
+      status: 'success', 
+      count: jobs.length,
+      jobs 
+    });
+  } catch (error) {
+    logger.error('Failed to get scheduled jobs info', { error });
+    return c.json({ status: 'error', message: 'Failed to get scheduled jobs' }, 500);
+  }
+});

apps/data-service/src/routes/test.routes.ts

@@ -0,0 +1,77 @@
+/**
+ * Test and development routes for batch processing
+ */
+import { Hono } from 'hono';
+import { getLogger } from '@stock-bot/logger';
+import { queueManager } from '../services/queue.service';
+
+const logger = getLogger('test-routes');
+
+export const testRoutes = new Hono();
+
+// Test endpoint for new functional batch processing
+testRoutes.post('/api/test/batch-symbols', async (c) => {
+  try {
+    const { symbols, useBatching = false, totalDelayMs = 60000 } = await c.req.json();
+    const { processSymbols } = await import('../utils/batch-helpers');
+    
+    if (!symbols || !Array.isArray(symbols)) {
+      return c.json({ status: 'error', message: 'symbols array is required' }, 400);
+    }
+
+    const result = await processSymbols(symbols, queueManager, {
+      operation: 'live-data',
+      service: 'test',
+      provider: 'test-provider',
+      totalDelayMs,
+      useBatching,
+      batchSize: 10,
+      priority: 1
+    });
+
+    return c.json({ 
+      status: 'success', 
+      message: 'Batch processing started',
+      result 
+    });
+  } catch (error) {
+    logger.error('Failed to start batch symbol processing', { error });
+    return c.json({ status: 'error', message: 'Failed to start batch processing' }, 500);
+  }
+});
+
+testRoutes.post('/api/test/batch-custom', async (c) => {
+  try {
+    const { items, useBatching = false, totalDelayMs = 30000 } = await c.req.json();
+    const { processItems } = await import('../utils/batch-helpers');
+    
+    if (!items || !Array.isArray(items)) {
+      return c.json({ status: 'error', message: 'items array is required' }, 400);
+    }
+
+    const result = await processItems(
+      items,
+      (item, index) => ({ 
+        originalItem: item, 
+        processIndex: index,
+        timestamp: new Date().toISOString()
+      }),
+      queueManager,
+      {
+        totalDelayMs,
+        useBatching,
+        batchSize: 5,
+        priority: 1
+      }
+    );
+
+    return c.json({ 
+      status: 'success', 
+      message: 'Custom batch processing started',
+      result 
+    });
+  } catch (error) {
+    logger.error('Failed to start custom batch processing', { error });
+    return c.json({ status: 'error', message: 'Failed to start custom batch processing' }, 500);
+  }
+});