restructured libs to be more aligned with core components
This commit is contained in:
parent
947b1d748d
commit
0d1be9e3cb
50 changed files with 73 additions and 67 deletions
349
libs/core/queue/src/smart-queue-manager.ts
Normal file
349
libs/core/queue/src/smart-queue-manager.ts
Normal file
|
|
@ -0,0 +1,349 @@
|
|||
import { Queue as BullQueue, type Job } from 'bullmq';
|
||||
import { handlerRegistry } from '@stock-bot/handlers';
|
||||
import { getLogger, type Logger } from '@stock-bot/logger';
|
||||
import { QueueManager } from './queue-manager';
|
||||
import { Queue } from './queue';
|
||||
import type {
|
||||
SmartQueueConfig,
|
||||
QueueRoute,
|
||||
JobData,
|
||||
JobOptions,
|
||||
RedisConfig
|
||||
} from './types';
|
||||
import {
|
||||
SERVICE_REGISTRY,
|
||||
getServiceConfig,
|
||||
findServiceForHandler,
|
||||
getFullQueueName,
|
||||
parseQueueName,
|
||||
type ServiceConfig
|
||||
} from './service-registry';
|
||||
import { getRedisConnection } from './utils';
|
||||
|
||||
/**
|
||||
* Smart Queue Manager with automatic service discovery and routing
|
||||
* Handles cross-service communication seamlessly
|
||||
*/
|
||||
export class SmartQueueManager extends QueueManager {
|
||||
private serviceName: string;
|
||||
private serviceConfig: ServiceConfig;
|
||||
private queueRoutes = new Map<string, QueueRoute>();
|
||||
private connections = new Map<number, any>(); // Redis connections by DB
|
||||
private producerQueues = new Map<string, BullQueue>(); // For cross-service sending
|
||||
private _logger: Logger;
|
||||
|
||||
constructor(config: SmartQueueConfig, logger?: Logger) {
|
||||
// Get service config
|
||||
const serviceConfig = getServiceConfig(config.serviceName);
|
||||
if (!serviceConfig) {
|
||||
throw new Error(`Unknown service: ${config.serviceName}`);
|
||||
}
|
||||
|
||||
// Update Redis config to use service's DB
|
||||
const modifiedConfig = {
|
||||
...config,
|
||||
redis: {
|
||||
...config.redis,
|
||||
db: serviceConfig.db,
|
||||
},
|
||||
};
|
||||
|
||||
super(modifiedConfig, logger);
|
||||
|
||||
this.serviceName = config.serviceName;
|
||||
this.serviceConfig = serviceConfig;
|
||||
this._logger = logger || getLogger('SmartQueueManager');
|
||||
|
||||
// Auto-discover routes if enabled
|
||||
if (config.autoDiscoverHandlers !== false) {
|
||||
this.discoverQueueRoutes();
|
||||
}
|
||||
|
||||
this._logger.info('SmartQueueManager initialized', {
|
||||
service: this.serviceName,
|
||||
db: serviceConfig.db,
|
||||
handlers: serviceConfig.handlers,
|
||||
producerOnly: serviceConfig.producerOnly,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover all available queue routes from handler registry
|
||||
*/
|
||||
private discoverQueueRoutes(): void {
|
||||
// Discover from handler registry if available
|
||||
try {
|
||||
const handlers = handlerRegistry.getAllHandlers();
|
||||
for (const [handlerName, handlerConfig] of handlers) {
|
||||
// Find which service owns this handler
|
||||
const ownerService = findServiceForHandler(handlerName);
|
||||
if (ownerService) {
|
||||
const ownerConfig = getServiceConfig(ownerService)!;
|
||||
const fullName = getFullQueueName(ownerService, handlerName);
|
||||
|
||||
this.queueRoutes.set(handlerName, {
|
||||
fullName,
|
||||
service: ownerService,
|
||||
handler: handlerName,
|
||||
db: ownerConfig.db,
|
||||
operations: Object.keys(handlerConfig.operations || {}),
|
||||
});
|
||||
|
||||
this._logger.trace('Discovered queue route', {
|
||||
handler: handlerName,
|
||||
service: ownerService,
|
||||
db: ownerConfig.db,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this._logger.warn('Handler registry not available, using static configuration', { error });
|
||||
}
|
||||
|
||||
// Also add routes from static configuration
|
||||
Object.entries(SERVICE_REGISTRY).forEach(([serviceName, config]) => {
|
||||
if (config.handlers) {
|
||||
config.handlers.forEach(handlerName => {
|
||||
if (!this.queueRoutes.has(handlerName)) {
|
||||
const fullName = getFullQueueName(serviceName, handlerName);
|
||||
this.queueRoutes.set(handlerName, {
|
||||
fullName,
|
||||
service: serviceName,
|
||||
handler: handlerName,
|
||||
db: config.db,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a Redis connection for a specific DB
|
||||
*/
|
||||
private getConnection(db: number): any {
|
||||
if (!this.connections.has(db)) {
|
||||
const redisConfig: RedisConfig = {
|
||||
...this.getRedisConfig(),
|
||||
db,
|
||||
};
|
||||
const connection = getRedisConnection(redisConfig);
|
||||
this.connections.set(db, connection);
|
||||
this._logger.debug('Created Redis connection', { db });
|
||||
}
|
||||
return this.connections.get(db);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a queue for the current service (for processing)
|
||||
* Overrides parent to use namespaced queue names
|
||||
*/
|
||||
override getQueue(queueName: string, options = {}): Queue {
|
||||
// For local queues, use the service namespace
|
||||
const fullQueueName = getFullQueueName(this.serviceName, queueName);
|
||||
return super.getQueue(fullQueueName, options);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Send a job to any queue (local or remote)
|
||||
* This is the main method for cross-service communication
|
||||
*/
|
||||
async send(
|
||||
targetQueue: string,
|
||||
operation: string,
|
||||
payload: unknown,
|
||||
options: JobOptions = {}
|
||||
): Promise<Job> {
|
||||
// Resolve the target queue
|
||||
const route = this.resolveQueueRoute(targetQueue);
|
||||
if (!route) {
|
||||
throw new Error(`Unknown queue: ${targetQueue}`);
|
||||
}
|
||||
|
||||
// Validate operation if we have metadata
|
||||
if (route.operations && !route.operations.includes(operation)) {
|
||||
this._logger.warn('Operation not found in handler metadata', {
|
||||
queue: targetQueue,
|
||||
operation,
|
||||
available: route.operations,
|
||||
});
|
||||
}
|
||||
|
||||
// Get or create producer queue for the target
|
||||
const producerQueue = this.getProducerQueue(route);
|
||||
|
||||
// Create job data
|
||||
const jobData: JobData = {
|
||||
handler: route.handler,
|
||||
operation,
|
||||
payload,
|
||||
};
|
||||
|
||||
// Send the job
|
||||
const job = await producerQueue.add(operation, jobData, options);
|
||||
|
||||
this._logger.debug('Job sent to queue', {
|
||||
from: this.serviceName,
|
||||
to: route.service,
|
||||
queue: route.handler,
|
||||
operation,
|
||||
jobId: job.id,
|
||||
});
|
||||
|
||||
return job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Alias for send() with more explicit name
|
||||
*/
|
||||
async sendTo(
|
||||
targetService: string,
|
||||
handler: string,
|
||||
operation: string,
|
||||
payload: unknown,
|
||||
options: JobOptions = {}
|
||||
): Promise<Job> {
|
||||
const fullQueueName = `${targetService}:${handler}`;
|
||||
return this.send(fullQueueName, operation, payload, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a queue name to a route
|
||||
*/
|
||||
private resolveQueueRoute(queueName: string): QueueRoute | null {
|
||||
// Check if it's a handler name (which is now the full queue name)
|
||||
const parsed = parseQueueName(queueName);
|
||||
if (parsed) {
|
||||
const config = getServiceConfig(parsed.service);
|
||||
if (config) {
|
||||
return {
|
||||
fullName: queueName,
|
||||
service: parsed.service,
|
||||
handler: parsed.handler,
|
||||
db: config.db,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Check if it's just a handler name
|
||||
const route = this.queueRoutes.get(queueName);
|
||||
if (route) {
|
||||
return route;
|
||||
}
|
||||
|
||||
// Try to find in static config
|
||||
const ownerService = findServiceForHandler(queueName);
|
||||
if (ownerService) {
|
||||
const config = getServiceConfig(ownerService)!;
|
||||
return {
|
||||
fullName: getFullQueueName(ownerService, queueName),
|
||||
service: ownerService,
|
||||
handler: queueName,
|
||||
db: config.db,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a producer queue for cross-service communication
|
||||
*/
|
||||
private getProducerQueue(route: QueueRoute): BullQueue {
|
||||
if (!this.producerQueues.has(route.fullName)) {
|
||||
const connection = this.getConnection(route.db);
|
||||
// Match the queue name format used by workers: {queueName}
|
||||
const queue = new BullQueue(`{${route.fullName}}`, {
|
||||
connection,
|
||||
defaultJobOptions: this.getConfig().defaultQueueOptions?.defaultJobOptions || {},
|
||||
});
|
||||
this.producerQueues.set(route.fullName, queue);
|
||||
}
|
||||
return this.producerQueues.get(route.fullName)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all queues (for monitoring purposes)
|
||||
*/
|
||||
getAllQueues(): Record<string, BullQueue> {
|
||||
const allQueues: Record<string, BullQueue> = {};
|
||||
|
||||
// Get all worker queues using public API
|
||||
const workerQueueNames = this.getQueueNames();
|
||||
for (const name of workerQueueNames) {
|
||||
const queue = this.getQueue(name);
|
||||
if (queue && typeof queue.getBullQueue === 'function') {
|
||||
// Extract the underlying BullMQ queue using the public getter
|
||||
// Use the simple handler name without service prefix for display
|
||||
const parts = name.split(':');
|
||||
const simpleName = parts.length > 1 ? parts[1] : name;
|
||||
if (simpleName) {
|
||||
allQueues[simpleName] = queue.getBullQueue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add producer queues
|
||||
for (const [name, queue] of this.producerQueues) {
|
||||
// Use the simple handler name without service prefix for display
|
||||
const parts = name.split(':');
|
||||
const simpleName = parts.length > 1 ? parts[1] : name;
|
||||
if (simpleName && !allQueues[simpleName]) {
|
||||
allQueues[simpleName] = queue;
|
||||
}
|
||||
}
|
||||
|
||||
// If no queues found, return all registered handlers as BullMQ queues
|
||||
if (Object.keys(allQueues).length === 0) {
|
||||
// Create BullMQ queue instances for known handlers
|
||||
const handlers = ['proxy', 'qm', 'ib', 'ceo', 'webshare', 'exchanges', 'symbols'];
|
||||
for (const handler of handlers) {
|
||||
const connection = this.getConnection(1); // Use default DB
|
||||
allQueues[handler] = new BullQueue(`{${handler}}`, {
|
||||
connection,
|
||||
defaultJobOptions: this.getConfig().defaultQueueOptions?.defaultJobOptions || {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return allQueues;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics for all queues across all services
|
||||
*/
|
||||
async getAllStats(): Promise<Record<string, any>> {
|
||||
const stats: Record<string, any> = {};
|
||||
|
||||
// Get stats for local queues
|
||||
stats[this.serviceName] = await this.getGlobalStats();
|
||||
|
||||
// Get stats for other services if we have access
|
||||
// This would require additional implementation
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Graceful shutdown
|
||||
*/
|
||||
override async shutdown(): Promise<void> {
|
||||
// Close producer queues
|
||||
for (const [name, queue] of this.producerQueues) {
|
||||
await queue.close();
|
||||
this._logger.debug('Closed producer queue', { queue: name });
|
||||
}
|
||||
|
||||
// Close additional connections
|
||||
for (const [db, connection] of this.connections) {
|
||||
if (db !== this.serviceConfig.db) { // Don't close our main connection
|
||||
connection.disconnect();
|
||||
this._logger.debug('Closed Redis connection', { db });
|
||||
}
|
||||
}
|
||||
|
||||
// Call parent shutdown
|
||||
await super.shutdown();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue