added a smart queue manager and moved proxy logic to proxy manager to make handler just schedule a call to it

libs/services/queue/src/smart-queue-manager.ts

@@ -0,0 +1,301 @@
+import { Queue as BullQueue, type Job } from 'bullmq';
+import IoRedis from 'ioredis';
+import { handlerRegistry } from '@stock-bot/types';
+import { getLogger, type Logger } from '@stock-bot/logger';
+import { QueueManager } from './queue-manager';
+import { Queue } from './queue';
+import type { 
+  SmartQueueConfig, 
+  QueueRoute, 
+  JobData, 
+  JobOptions,
+  RedisConfig 
+} from './types';
+import { 
+  SERVICE_REGISTRY, 
+  getServiceConfig, 
+  findServiceForHandler,
+  getFullQueueName,
+  parseQueueName,
+  type ServiceConfig
+} from './service-registry';
+import { getRedisConnection } from './utils';
+
+/**
+ * Smart Queue Manager with automatic service discovery and routing
+ * Handles cross-service communication seamlessly
+ */
+export class SmartQueueManager extends QueueManager {
+  private serviceName: string;
+  private serviceConfig: ServiceConfig;
+  private queueRoutes = new Map<string, QueueRoute>();
+  private connections = new Map<number, any>(); // Redis connections by DB
+  private producerQueues = new Map<string, BullQueue>(); // For cross-service sending
+  private _logger: Logger;
+
+  constructor(config: SmartQueueConfig, logger?: Logger) {
+    // Get service config
+    const serviceConfig = getServiceConfig(config.serviceName);
+    if (!serviceConfig) {
+      throw new Error(`Unknown service: ${config.serviceName}`);
+    }
+
+    // Update Redis config to use service's DB
+    const modifiedConfig = {
+      ...config,
+      redis: {
+        ...config.redis,
+        db: serviceConfig.db,
+      },
+    };
+
+    super(modifiedConfig, logger);
+    
+    this.serviceName = config.serviceName;
+    this.serviceConfig = serviceConfig;
+    this._logger = logger || getLogger('SmartQueueManager');
+
+    // Auto-discover routes if enabled
+    if (config.autoDiscoverHandlers !== false) {
+      this.discoverQueueRoutes();
+    }
+
+    this._logger.info('SmartQueueManager initialized', {
+      service: this.serviceName,
+      db: serviceConfig.db,
+      handlers: serviceConfig.handlers,
+      producerOnly: serviceConfig.producerOnly,
+    });
+  }
+
+  /**
+   * Discover all available queue routes from handler registry
+   */
+  private discoverQueueRoutes(): void {
+    // Discover from handler registry if available
+    try {
+      const handlers = handlerRegistry.getAllHandlers();
+      for (const [handlerName, handlerConfig] of handlers) {
+        // Find which service owns this handler
+        const ownerService = findServiceForHandler(handlerName);
+        if (ownerService) {
+          const ownerConfig = getServiceConfig(ownerService)!;
+          const fullName = getFullQueueName(ownerService, handlerName);
+          
+          this.queueRoutes.set(handlerName, {
+            fullName,
+            service: ownerService,
+            handler: handlerName,
+            db: ownerConfig.db,
+            operations: Object.keys(handlerConfig.operations || {}),
+          });
+
+          this._logger.trace('Discovered queue route', {
+            handler: handlerName,
+            service: ownerService,
+            db: ownerConfig.db,
+          });
+        }
+      }
+    } catch (error) {
+      this._logger.warn('Handler registry not available, using static configuration', { error });
+    }
+
+    // Also add routes from static configuration
+    Object.entries(SERVICE_REGISTRY).forEach(([serviceName, config]) => {
+      if (config.handlers) {
+        config.handlers.forEach(handlerName => {
+          if (!this.queueRoutes.has(handlerName)) {
+            const fullName = getFullQueueName(serviceName, handlerName);
+            this.queueRoutes.set(handlerName, {
+              fullName,
+              service: serviceName,
+              handler: handlerName,
+              db: config.db,
+            });
+          }
+        });
+      }
+    });
+  }
+
+  /**
+   * Get or create a Redis connection for a specific DB
+   */
+  private getConnection(db: number): any {
+    if (!this.connections.has(db)) {
+      const redisConfig: RedisConfig = {
+        ...this.getRedisConfig(),
+        db,
+      };
+      const connection = getRedisConnection(redisConfig);
+      this.connections.set(db, connection);
+      this._logger.debug('Created Redis connection', { db });
+    }
+    return this.connections.get(db);
+  }
+
+  /**
+   * Get a queue for the current service (for processing)
+   * Overrides parent to use namespaced queue names
+   */
+  override getQueue(queueName: string, options = {}): Queue {
+    // For local queues, use the service namespace
+    const fullQueueName = getFullQueueName(this.serviceName, queueName);
+    return super.getQueue(fullQueueName, options);
+  }
+
+  /**
+   * Send a job to any queue (local or remote)
+   * This is the main method for cross-service communication
+   */
+  async send(
+    targetQueue: string,
+    operation: string,
+    payload: unknown,
+    options: JobOptions = {}
+  ): Promise<Job> {
+    // Resolve the target queue
+    const route = this.resolveQueueRoute(targetQueue);
+    if (!route) {
+      throw new Error(`Unknown queue: ${targetQueue}`);
+    }
+
+    // Validate operation if we have metadata
+    if (route.operations && !route.operations.includes(operation)) {
+      this._logger.warn('Operation not found in handler metadata', {
+        queue: targetQueue,
+        operation,
+        available: route.operations,
+      });
+    }
+
+    // Get or create producer queue for the target
+    const producerQueue = this.getProducerQueue(route);
+
+    // Create job data
+    const jobData: JobData = {
+      handler: route.handler,
+      operation,
+      payload,
+    };
+
+    // Send the job
+    const job = await producerQueue.add(operation, jobData, options);
+
+    this._logger.debug('Job sent to queue', {
+      from: this.serviceName,
+      to: route.service,
+      queue: route.handler,
+      operation,
+      jobId: job.id,
+    });
+
+    return job;
+  }
+
+  /**
+   * Alias for send() with more explicit name
+   */
+  async sendTo(
+    targetService: string,
+    handler: string,
+    operation: string,
+    payload: unknown,
+    options: JobOptions = {}
+  ): Promise<Job> {
+    const fullQueueName = `${targetService}:${handler}`;
+    return this.send(fullQueueName, operation, payload, options);
+  }
+
+  /**
+   * Resolve a queue name to a route
+   */
+  private resolveQueueRoute(queueName: string): QueueRoute | null {
+    // Check if it's a full name (service:handler)
+    const parsed = parseQueueName(queueName);
+    if (parsed) {
+      const config = getServiceConfig(parsed.service);
+      if (config) {
+        return {
+          fullName: queueName,
+          service: parsed.service,
+          handler: parsed.handler,
+          db: config.db,
+        };
+      }
+    }
+
+    // Check if it's just a handler name
+    const route = this.queueRoutes.get(queueName);
+    if (route) {
+      return route;
+    }
+
+    // Try to find in static config
+    const ownerService = findServiceForHandler(queueName);
+    if (ownerService) {
+      const config = getServiceConfig(ownerService)!;
+      return {
+        fullName: getFullQueueName(ownerService, queueName),
+        service: ownerService,
+        handler: queueName,
+        db: config.db,
+      };
+    }
+
+    return null;
+  }
+
+  /**
+   * Get or create a producer queue for cross-service communication
+   */
+  private getProducerQueue(route: QueueRoute): BullQueue {
+    if (!this.producerQueues.has(route.fullName)) {
+      const connection = this.getConnection(route.db);
+      const queue = new BullQueue(route.fullName, {
+        connection,
+        defaultJobOptions: this.getConfig().defaultQueueOptions?.defaultJobOptions || {},
+      });
+      this.producerQueues.set(route.fullName, queue);
+    }
+    return this.producerQueues.get(route.fullName)!;
+  }
+
+  /**
+   * Get statistics for all queues across all services
+   */
+  async getAllStats(): Promise<Record<string, any>> {
+    const stats: Record<string, any> = {};
+
+    // Get stats for local queues
+    stats[this.serviceName] = await this.getGlobalStats();
+
+    // Get stats for other services if we have access
+    // This would require additional implementation
+
+    return stats;
+  }
+
+  /**
+   * Graceful shutdown
+   */
+  override async shutdown(): Promise<void> {
+    // Close producer queues
+    for (const [name, queue] of this.producerQueues) {
+      await queue.close();
+      this._logger.debug('Closed producer queue', { queue: name });
+    }
+
+    // Close additional connections
+    for (const [db, connection] of this.connections) {
+      if (db !== this.serviceConfig.db) { // Don't close our main connection
+        connection.disconnect();
+        this._logger.debug('Closed Redis connection', { db });
+      }
+    }
+
+    // Call parent shutdown
+    await super.shutdown();
+  }
+}