huge refactor done

libs/core/queue/src/queue-manager.ts

@@ -1,11 +1,18 @@
+import { Queue as BullQueue, type Job } from 'bullmq';
 import { createCache } from '@stock-bot/cache';
 import type { CacheProvider } from '@stock-bot/cache';
+import type { HandlerRegistry } from '@stock-bot/handler-registry';
+import { getLogger } from '@stock-bot/logger';
 import { Queue, type QueueWorkerConfig } from './queue';
 import { QueueRateLimiter } from './rate-limiter';
+import { getFullQueueName, parseQueueName } from './service-utils';
 import type {
   GlobalStats,
+  JobData,
+  JobOptions,
   QueueManagerConfig,
   QueueOptions,
+  QueueRoute,
   QueueStats,
   RateLimitRule,
 } from './types';
@@ -22,8 +29,8 @@ interface Logger {
 }
 
 /**
- * QueueManager provides unified queue and cache management
- * Main entry point for all queue operations with getQueue() method
+ * QueueManager provides unified queue and cache management with service discovery
+ * Handles both local and cross-service queue operations
  */
 export class QueueManager {
   private queues = new Map<string, Queue>();
@@ -34,10 +41,29 @@ export class QueueManager {
   private shutdownPromise: Promise<void> | null = null;
   private config: QueueManagerConfig;
   private readonly logger: Logger;
+  
+  // Service discovery features
+  private serviceName?: string;
+  private queueRoutes = new Map<string, QueueRoute>();
+  private producerQueues = new Map<string, BullQueue>(); // For cross-service sending
+  private handlerRegistry?: HandlerRegistry;
 
-  constructor(config: QueueManagerConfig, logger?: Logger) {
+  constructor(config: QueueManagerConfig, handlerRegistry?: HandlerRegistry, logger?: Logger) {
+    // Always use DB 0 for queues if service name is provided
+    if (config.serviceName) {
+      config = {
+        ...config,
+        redis: {
+          ...config.redis,
+          db: 0, // All queues in DB 0 for cross-service communication
+        },
+      };
+    }
+    
     this.config = config;
-    this.logger = logger || console;
+    this.serviceName = config.serviceName;
+    this.handlerRegistry = handlerRegistry;
+    this.logger = logger || getLogger('QueueManager');
     this.redisConnection = getRedisConnection(config.redis);
 
     // Initialize rate limiter if rules are provided
@@ -50,16 +76,58 @@ export class QueueManager {
       });
     }
 
+    // Auto-discover routes if enabled and registry provided
+    if (config.serviceName && config.autoDiscoverHandlers !== false && handlerRegistry) {
+      this.discoverQueueRoutes();
+    }
+    
     this.logger.info('QueueManager initialized', {
       redis: `${config.redis.host}:${config.redis.port}`,
+      service: this.serviceName,
+      discoveredRoutes: this.queueRoutes.size,
+      hasRegistry: !!handlerRegistry,
     });
   }
 
   /**
    * Get or create a queue - unified method that handles both scenarios
    * This is the main method for accessing queues
+   * If serviceName is configured, automatically handles namespacing
    */
   getQueue(queueName: string, options: QueueOptions = {}): Queue {
+    let fullQueueName = queueName;
+    let isOwnQueue = true;
+    
+    // Handle service namespacing if service name is configured
+    if (this.serviceName) {
+      const parsed = parseQueueName(queueName);
+      
+      if (parsed) {
+        // Already in service:handler format
+        fullQueueName = queueName;
+        isOwnQueue = parsed.service === this.serviceName;
+      } else {
+        // Just handler name, assume it's for current service
+        fullQueueName = getFullQueueName(this.serviceName, queueName);
+        isOwnQueue = true;
+      }
+      
+      // For cross-service queues, create without workers (producer-only)
+      if (!isOwnQueue) {
+        options = {
+          ...options,
+          workers: 0, // No workers for other services' queues
+        };
+      } else {
+        // For own service queues, include handler registry
+        options = {
+          ...options,
+          handlerRegistry: this.handlerRegistry
+        };
+      }
+      
+      queueName = fullQueueName;
+    }
     // Return existing queue if it exists
     if (this.queues.has(queueName)) {
       const existingQueue = this.queues.get(queueName);
@@ -83,6 +151,7 @@ export class QueueManager {
       workers,
       concurrency,
       startWorker: workers > 0 && !this.config.delayWorkerStart,
+      handlerRegistry: options.handlerRegistry || this.handlerRegistry,
     };
 
     const queue = new Queue(
@@ -112,8 +181,13 @@ export class QueueManager {
 
     this.logger.info('Queue created with batch cache', {
       queueName,
+      originalQueueName: options.handlerRegistry ? 'has-handler-registry' : 'no-handler-registry',
       workers: workers,
       concurrency: concurrency,
+      handlerRegistryProvided: !!this.handlerRegistry,
+      willStartWorkers: workers > 0 && !this.config.delayWorkerStart,
+      isOwnQueue,
+      serviceName: this.serviceName,
     });
 
     return queue;
@@ -411,18 +485,42 @@ export class QueueManager {
     }
 
     let workersStarted = 0;
-    for (const queue of this.queues.values()) {
+    const queues = this.queues;
+    
+    this.logger.info(`Starting workers for ${queues.size} queues: ${Array.from(queues.keys()).join(', ')} (service: ${this.serviceName})`);
+    
+    for (const [queueName, queue] of queues) {
+      // If we have a service name, check if this queue belongs to us
+      if (this.serviceName) {
+        const parsed = parseQueueName(queueName);
+        // Skip if not our service's queue
+        if (parsed && parsed.service !== this.serviceName) {
+          this.logger.trace('Skipping workers for cross-service queue', {
+            queueName,
+            ownerService: parsed.service,
+            currentService: this.serviceName,
+          });
+          continue;
+        }
+      }
+      
       const workerCount = this.config.defaultQueueOptions?.workers || 1;
       const concurrency = this.config.defaultQueueOptions?.concurrency || 1;
 
       if (workerCount > 0) {
         queue.startWorkersManually(workerCount, concurrency);
         workersStarted++;
+        this.logger.debug('Started workers for queue', {
+          queueName,
+          workers: workerCount,
+          concurrency,
+        });
       }
     }
 
-    this.logger.info('All workers started', {
-      totalQueues: this.queues.size,
+    this.logger.info('Service workers started', {
+      service: this.serviceName || 'default',
+      totalQueues: queues.size,
       queuesWithWorkers: workersStarted,
       delayWorkerStart: this.config.delayWorkerStart,
     });
@@ -449,4 +547,169 @@ export class QueueManager {
   getConfig(): Readonly<QueueManagerConfig> {
     return { ...this.config };
   }
+  
+  /**
+   * Send a job to any queue (local or remote)
+   * This is the main method for cross-service communication
+   */
+  async send(
+    targetQueue: string,
+    operation: string,
+    payload: unknown,
+    options: JobOptions = {}
+  ): Promise<Job> {
+    if (!this.serviceName) {
+      // If no service name, just use regular queue
+      const queue = this.getQueue(targetQueue);
+      return queue.add(operation, { handler: targetQueue, operation, payload }, options);
+    }
+    
+    // Resolve the target queue
+    const route = this.resolveQueueRoute(targetQueue);
+    if (!route) {
+      throw new Error(`Unknown queue: ${targetQueue}`);
+    }
+
+    // Validate operation if we have metadata
+    if (route.operations && !route.operations.includes(operation)) {
+      this.logger.warn('Operation not found in handler metadata', {
+        handler: route.handler,
+        operation,
+        availableOperations: route.operations,
+      });
+    }
+
+    // Use a producer queue for cross-service sending
+    const producerQueue = this.getProducerQueue(route.fullName);
+    
+    const jobData: JobData = {
+      handler: route.handler,
+      operation,
+      payload,
+    };
+
+    this.logger.trace('Sending job to queue', {
+      targetQueue: route.fullName,
+      handler: route.handler,
+      operation,
+      fromService: this.serviceName,
+      toService: route.service,
+    });
+
+    return producerQueue.add(operation, jobData, options);
+  }
+  
+  /**
+   * Resolve a queue name to a route
+   */
+  private resolveQueueRoute(queueName: string): QueueRoute | null {
+    // Check if it's a full queue name with service prefix
+    const parsed = parseQueueName(queueName);
+    if (parsed) {
+      // Try to find in discovered routes by handler name
+      const route = this.queueRoutes.get(parsed.handler);
+      if (route && route.service === parsed.service) {
+        return route;
+      }
+      // Create a route on the fly
+      return {
+        fullName: queueName,
+        service: parsed.service,
+        handler: parsed.handler,
+        db: 0, // All queues in DB 0
+      };
+    }
+
+    // Check if it's just a handler name in our routes
+    const route = this.queueRoutes.get(queueName);
+    if (route) {
+      return route;
+    }
+
+    // Try to find in handler registry
+    const ownerService = this.handlerRegistry?.getHandlerService(queueName);
+    if (ownerService) {
+      return {
+        fullName: getFullQueueName(ownerService, queueName),
+        service: ownerService,
+        handler: queueName,
+        db: 0, // All queues in DB 0
+      };
+    }
+
+    return null;
+  }
+  
+  /**
+   * Get a producer queue for sending to other services
+   */
+  private getProducerQueue(queueName: string): BullQueue {
+    if (!this.producerQueues.has(queueName)) {
+      const queue = new BullQueue(queueName, {
+        connection: this.redisConnection,
+        defaultJobOptions: this.config.defaultQueueOptions?.defaultJobOptions,
+      });
+      this.producerQueues.set(queueName, queue);
+    }
+    return this.producerQueues.get(queueName)!;
+  }
+  
+  /**
+   * Discover all available queue routes from handler registry
+   */
+  private discoverQueueRoutes(): void {
+    if (!this.handlerRegistry) {
+      this.logger.warn('No handler registry provided, skipping route discovery');
+      return;
+    }
+
+    try {
+      const handlers = this.handlerRegistry.getAllMetadata();
+      for (const [handlerName, metadata] of handlers) {
+        // Get the service that registered this handler
+        const ownerService = metadata.service;
+        if (ownerService) {
+          const fullName = getFullQueueName(ownerService, handlerName);
+
+          this.queueRoutes.set(handlerName, {
+            fullName,
+            service: ownerService,
+            handler: handlerName,
+            db: 0, // All queues in DB 0
+            operations: metadata.operations.map((op: any) => op.name),
+          });
+
+          this.logger.trace('Discovered queue route', {
+            handler: handlerName,
+            service: ownerService,
+            operations: metadata.operations.length,
+          });
+        }
+      }
+
+      // Also discover handlers registered by the current service
+      if (this.serviceName) {
+        const myHandlers = this.handlerRegistry.getServiceHandlers(this.serviceName);
+        for (const metadata of myHandlers) {
+          const handlerName = metadata.name;
+          if (!this.queueRoutes.has(handlerName)) {
+            const fullName = getFullQueueName(this.serviceName, handlerName);
+            this.queueRoutes.set(handlerName, {
+              fullName,
+              service: this.serviceName,
+              handler: handlerName,
+              db: 0, // All queues in DB 0
+            });
+          }
+        }
+      }
+
+      this.logger.info('Queue routes discovered', {
+        totalRoutes: this.queueRoutes.size,
+        routes: Array.from(this.queueRoutes.keys()),
+      });
+    } catch (error) {
+      this.logger.error('Failed to discover queue routes', { error });
+    }
+  }
 }