fixed up worker counts

apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-short.action.ts

@@ -0,0 +1,55 @@
+
+import { getRandomUserAgent } from '@stock-bot/utils';
+import type { CeoHandler } from '../ceo.handler';
+
+export async function processIndividualSymbol(
+  this: CeoHandler,
+  payload: any,
+  _context: any
+): Promise<unknown> {
+  const { ceoId, symbol, timestamp } = payload;
+  const proxy = this.proxy?.getProxy();
+  if (!proxy) {
+    this.logger.warn('No proxy available for processing individual CEO symbol');
+    return;
+  }
+
+  this.logger.debug(`Processing individual CEO symbol for ${symbol}`, {
+    ceoId,
+    timestamp,
+  });
+  try {
+    // Update Shorts
+    const response = await this.http.get(
+      `https://api.ceo.ca/api/short_positions/one?symbol=${symbol}`,
+      {
+        proxy: proxy,
+        headers: {
+          'User-Agent': getRandomUserAgent(),
+        },
+      }
+    );
+    let shortCount = 0;
+
+    if (response.ok) {
+      const shortData = await response.json();
+      if (shortData && shortData.positions) {
+        shortCount = shortData.positions.length;
+        await this.mongodb.batchUpsert('ceoShorts', shortData.positions, ['id']);
+      }
+    }
+
+    this.logger.info(
+      `Successfully processed CEO symbol ${symbol} shorts and found ${shortCount} positions`,
+    );
+
+    return { ceoId, shortCount, timestamp };
+  } catch (error) {
+    this.logger.error(`Failed to process individual symbol ${symbol}`, {
+      error,
+      ceoId,
+      timestamp,
+    });
+    throw error;
+  }
+}

apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts

@@ -39,6 +39,7 @@ export async function processIndividualSymbol(
     const spielCount = data.spiels.length;
     if (spielCount === 0) {
       this.logger.warn(`No spiels found for ceoId ${ceoId}`);
+      await this.mongodb.updateMany('ceoChannels', { ceoId }, { $set: { lastSpielTime: timestamp, finished: true } });
       return null; // No data to process
     }
     const latestSpielTime = data.spiels[0]?.timestamp;
@@ -76,35 +77,18 @@ export async function processIndividualSymbol(
     }));
 
     await this.mongodb.batchUpsert('ceoPosts', posts, ['spielId']);
+    await this.mongodb.updateMany('ceoChannels', { ceoId }, { $set: { lastSpielTime: latestSpielTime } });
     this.logger.info(`Fetched ${spielCount} spiels for ceoId ${ceoId}`);
 
-    // Update Shorts
-    const shortRes = await this.http.get(
-      `https://api.ceo.ca/api/short_positions/one?symbol=${symbol}`,
+    await this.scheduleOperation(
+      'process-individual-symbol',
       {
-        proxy: proxy,
-        headers: {
-          'User-Agent': getRandomUserAgent(),
-        },
-      }
+        ceoId: ceoId,
+        timestamp: latestSpielTime,
+      },
+      { priority: 0 }
     );
 
-    if (shortRes.ok) {
-      const shortData = await shortRes.json();
-      if (shortData && shortData.positions) {
-        await this.mongodb.batchUpsert('ceoShorts', shortData.positions, ['id']);
-      }
-
-      await this.scheduleOperation(
-        'process-individual-symbol',
-        {
-          ceoId: ceoId,
-          timestamp: latestSpielTime,
-        },
-        { priority: 0 }
-      );
-    }
-
     this.logger.info(
       `Successfully processed channel ${ceoId} and added channel ${ceoId} at timestamp ${latestSpielTime}`
     );

apps/stock/web-api/src/services/monitoring.service.ts

@@ -100,6 +100,16 @@ export class MonitoringService {
         symbols: 'data-pipeline',
       };
 
+      // Worker configuration per queue (from service configs)
+      const workerConfig: Record<string, { count: number; concurrency: number }> = {
+        qm: { count: 1, concurrency: 2 },
+        ib: { count: 1, concurrency: 1 },
+        ceo: { count: 1, concurrency: 2 },
+        webshare: { count: 1, concurrency: 1 },
+        exchanges: { count: 1, concurrency: 1 },
+        symbols: { count: 1, concurrency: 2 },
+      };
+
       const queueNames = Object.keys(handlerMapping);
       this.logger.debug('Using known queue names', { count: queueNames.length, names: queueNames });
 
@@ -124,6 +134,10 @@ export class MonitoringService {
           // Get stats directly from BullMQ
           const queueStats = await this.getQueueStatsForBullQueue(bullQueue, handlerName);
 
+          // Get actual worker count from BullMQ
+          const actualWorkerCount = await this.getActiveWorkerCountFromBullMQ(bullQueue);
+          const configuredWorkers = workerConfig[handlerName] || { count: 0, concurrency: 1 };
+
           stats.push({
             name: handlerName,
             service: serviceName,
@@ -131,8 +145,8 @@ export class MonitoringService {
             connected: true,
             jobs: queueStats,
             workers: {
-              count: 0,
-              concurrency: 1,
+              count: actualWorkerCount,
+              concurrency: configuredWorkers.concurrency,
             },
           });
 
@@ -790,4 +804,30 @@ export class MonitoringService {
       percentage: (usedMem / totalMem) * 100,
     };
   }
+
+  /**
+   * Get active worker count from Redis
+   */
+  /**
+   * Get active worker count using BullMQ's built-in tracking
+   */
+  private async getActiveWorkerCountFromBullMQ(bullQueue: any): Promise<number> {
+    try {
+      // Use BullMQ's built-in getWorkers method
+      if (bullQueue.getWorkers && typeof bullQueue.getWorkers === 'function') {
+        const workers = await bullQueue.getWorkers();
+        return workers.length;
+      }
+      
+      // Fallback to getWorkersCount if available
+      if (bullQueue.getWorkersCount && typeof bullQueue.getWorkersCount === 'function') {
+        return await bullQueue.getWorkersCount();
+      }
+      
+      return 0;
+    } catch (error) {
+      this.logger.debug('Failed to get active worker count from BullMQ', { error });
+      return 0;
+    }
+  }
 }