boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

handler to auto register and removed service registry, cleaned up queues and cache naming

Browse source
This commit is contained in:
Boki 2025-06-23 21:23:38 -04:00
parent 0d1be9e3cb
commit 34c6c36695
19 changed files with 474 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

214
libs/core/queue/src/smart-queue-manager.ts
View file

@ -10,14 +10,7 @@ import type {
JobOptions,
RedisConfig
} from './types';
import {
SERVICE_REGISTRY,
getServiceConfig,
findServiceForHandler,
getFullQueueName,
parseQueueName,
type ServiceConfig
} from './service-registry';
import { getFullQueueName, parseQueueName } from './service-utils';
import { getRedisConnection } from './utils';
/**
@ -26,32 +19,24 @@ import { getRedisConnection } from './utils';
*/
export class SmartQueueManager extends QueueManager {
private serviceName: string;
private serviceConfig: ServiceConfig;
private queueRoutes = new Map<string, QueueRoute>();
private connections = new Map<number, any>(); // Redis connections by DB
private producerQueues = new Map<string, BullQueue>(); // For cross-service sending
private _logger: Logger;
constructor(config: SmartQueueConfig, logger?: Logger) {
// Get service config
const serviceConfig = getServiceConfig(config.serviceName);
if (!serviceConfig) {
throw new Error(`Unknown service: ${config.serviceName}`);
}
// Update Redis config to use service's DB
// Always use DB 0 for queues (unified queue database)
const modifiedConfig = {
...config,
redis: {
...config.redis,
db: serviceConfig.db,
db: 0, // All queues in DB 0
},
};
super(modifiedConfig, logger);
this.serviceName = config.serviceName;
this.serviceConfig = serviceConfig;
this._logger = logger || getLogger('SmartQueueManager');
// Auto-discover routes if enabled
@ -61,9 +46,7 @@ export class SmartQueueManager extends QueueManager {
this._logger.info('SmartQueueManager initialized', {
service: this.serviceName,
db: serviceConfig.db,
handlers: serviceConfig.handlers,
producerOnly: serviceConfig.producerOnly,
discoveredRoutes: this.queueRoutes.size,
});
}
@ -71,51 +54,56 @@ export class SmartQueueManager extends QueueManager {
* Discover all available queue routes from handler registry
*/
private discoverQueueRoutes(): void {
// Discover from handler registry if available
try {
const handlers = handlerRegistry.getAllHandlers();
for (const [handlerName, handlerConfig] of handlers) {
// Find which service owns this handler
const ownerService = findServiceForHandler(handlerName);
// Get the service that registered this handler
const ownerService = handlerRegistry.getHandlerService(handlerName);
if (ownerService) {
const ownerConfig = getServiceConfig(ownerService)!;
const fullName = getFullQueueName(ownerService, handlerName);
this.queueRoutes.set(handlerName, {
fullName,
service: ownerService,
handler: handlerName,
db: ownerConfig.db,
db: 0, // All queues in DB 0
operations: Object.keys(handlerConfig.operations || {}),
});
this._logger.trace('Discovered queue route', {
handler: handlerName,
service: ownerService,
db: ownerConfig.db,
operations: Object.keys(handlerConfig.operations || {}).length,
});
} else {
this._logger.warn('Handler has no service ownership', { handlerName });
}
}
// Also discover handlers registered by the current service
const myHandlers = handlerRegistry.getServiceHandlers(this.serviceName);
for (const handlerName of myHandlers) {
if (!this.queueRoutes.has(handlerName)) {
const fullName = getFullQueueName(this.serviceName, handlerName);
this.queueRoutes.set(handlerName, {
fullName,
service: this.serviceName,
handler: handlerName,
db: 0, // All queues in DB 0
});
}
}
} catch (error) {
this._logger.warn('Handler registry not available, using static configuration', { error });
}
// Also add routes from static configuration
Object.entries(SERVICE_REGISTRY).forEach(([serviceName, config]) => {
if (config.handlers) {
config.handlers.forEach(handlerName => {
if (!this.queueRoutes.has(handlerName)) {
const fullName = getFullQueueName(serviceName, handlerName);
this.queueRoutes.set(handlerName, {
fullName,
service: serviceName,
handler: handlerName,
db: config.db,
});
}
});
}
});
this._logger.info('Queue routes discovered', {
totalRoutes: this.queueRoutes.size,
routes: Array.from(this.queueRoutes.values()).map(r => ({
handler: r.handler,
service: r.service
})),
});
} catch (error) {
this._logger.error('Failed to discover queue routes', { error });
}
}
/**
@ -136,11 +124,34 @@ export class SmartQueueManager extends QueueManager {
/**
* Get a queue for the current service (for processing)
* Overrides parent to use namespaced queue names
* Overrides parent to use namespaced queue names and ensure service-specific workers
*/
override getQueue(queueName: string, options = {}): Queue {
// For local queues, use the service namespace
const fullQueueName = getFullQueueName(this.serviceName, queueName);
// Check if this is already a full queue name (service:handler format)
const parsed = parseQueueName(queueName);
let fullQueueName: string;
let isOwnQueue: boolean;
if (parsed) {
// Already in service:handler format
fullQueueName = queueName;
isOwnQueue = parsed.service === this.serviceName;
} else {
// Just handler name, assume it's for current service
fullQueueName = getFullQueueName(this.serviceName, queueName);
isOwnQueue = true;
}
// For cross-service queues, create without workers (producer-only)
if (!isOwnQueue) {
return super.getQueue(fullQueueName, {
...options,
workers: 0, // No workers for other services' queues
});
}
// For own service queues, use configured workers
return super.getQueue(fullQueueName, options);
}
@ -212,35 +223,37 @@ export class SmartQueueManager extends QueueManager {
* Resolve a queue name to a route
*/
private resolveQueueRoute(queueName: string): QueueRoute | null {
// Check if it's a handler name (which is now the full queue name)
// Check if it's a full queue name with service prefix
const parsed = parseQueueName(queueName);
if (parsed) {
const config = getServiceConfig(parsed.service);
if (config) {
return {
fullName: queueName,
service: parsed.service,
handler: parsed.handler,
db: config.db,
};
// Try to find in discovered routes by handler name
const route = this.queueRoutes.get(parsed.handler);
if (route && route.service === parsed.service) {
return route;
}
// Create a route on the fly
return {
fullName: queueName,
service: parsed.service,
handler: parsed.handler,
db: 0, // All queues in DB 0
};
}
// Check if it's just a handler name
// Check if it's just a handler name in our routes
const route = this.queueRoutes.get(queueName);
if (route) {
return route;
}
// Try to find in static config
const ownerService = findServiceForHandler(queueName);
// Try to find in handler registry
const ownerService = handlerRegistry.getHandlerService(queueName);
if (ownerService) {
const config = getServiceConfig(ownerService)!;
return {
fullName: getFullQueueName(ownerService, queueName),
service: ownerService,
handler: queueName,
db: config.db,
db: 0, // All queues in DB 0
};
}
@ -253,8 +266,8 @@ export class SmartQueueManager extends QueueManager {
private getProducerQueue(route: QueueRoute): BullQueue {
if (!this.producerQueues.has(route.fullName)) {
const connection = this.getConnection(route.db);
// Match the queue name format used by workers: {queueName}
const queue = new BullQueue(`{${route.fullName}}`, {
// Use the same queue name format as workers
const queue = new BullQueue(route.fullName, {
connection,
defaultJobOptions: this.getConfig().defaultQueueOptions?.defaultJobOptions || {},
});
@ -276,8 +289,8 @@ export class SmartQueueManager extends QueueManager {
if (queue && typeof queue.getBullQueue === 'function') {
// Extract the underlying BullMQ queue using the public getter
// Use the simple handler name without service prefix for display
const parts = name.split(':');
const simpleName = parts.length > 1 ? parts[1] : name;
const parsed = parseQueueName(name);
const simpleName = parsed ? parsed.handler : name;
if (simpleName) {
allQueues[simpleName] = queue.getBullQueue();
}
@ -287,20 +300,18 @@ export class SmartQueueManager extends QueueManager {
// Add producer queues
for (const [name, queue] of this.producerQueues) {
// Use the simple handler name without service prefix for display
const parts = name.split(':');
const simpleName = parts.length > 1 ? parts[1] : name;
const parsed = parseQueueName(name);
const simpleName = parsed ? parsed.handler : name;
if (simpleName && !allQueues[simpleName]) {
allQueues[simpleName] = queue;
}
}
// If no queues found, return all registered handlers as BullMQ queues
// If no queues found, create from discovered routes
if (Object.keys(allQueues).length === 0) {
// Create BullMQ queue instances for known handlers
const handlers = ['proxy', 'qm', 'ib', 'ceo', 'webshare', 'exchanges', 'symbols'];
for (const handler of handlers) {
const connection = this.getConnection(1); // Use default DB
allQueues[handler] = new BullQueue(`{${handler}}`, {
for (const [handlerName, route] of this.queueRoutes) {
const connection = this.getConnection(0); // Use unified queue DB
allQueues[handlerName] = new BullQueue(route.fullName, {
connection,
defaultJobOptions: this.getConfig().defaultQueueOptions?.defaultJobOptions || {},
});
@ -325,6 +336,57 @@ export class SmartQueueManager extends QueueManager {
return stats;
}
/**
* Start workers for all queues belonging to this service
* Overrides parent to ensure only own queues get workers
*/
override startAllWorkers(): void {
if (!this.getConfig().delayWorkerStart) {
this._logger.info(
'startAllWorkers() called but workers already started automatically (delayWorkerStart is false)'
);
return;
}
let workersStarted = 0;
const queues = this.getQueues();
for (const [queueName, queue] of queues) {
// Parse queue name to check if it belongs to this service
const parsed = parseQueueName(queueName);
// Skip if not our service's queue
if (parsed && parsed.service !== this.serviceName) {
this._logger.trace('Skipping workers for cross-service queue', {
queueName,
ownerService: parsed.service,
currentService: this.serviceName,
});
continue;
}
const workerCount = this.getConfig().defaultQueueOptions?.workers || 1;
const concurrency = this.getConfig().defaultQueueOptions?.concurrency || 1;
if (workerCount > 0) {
queue.startWorkersManually(workerCount, concurrency);
workersStarted++;
this._logger.debug('Started workers for queue', {
queueName,
workers: workerCount,
concurrency,
});
}
}
this._logger.info('Service workers started', {
service: this.serviceName,
totalQueues: queues.size,
queuesWithWorkers: workersStarted,
delayWorkerStart: this.getConfig().delayWorkerStart,
});
}
/**
* Graceful shutdown
*/
@ -337,7 +399,7 @@ export class SmartQueueManager extends QueueManager {
// Close additional connections
for (const [db, connection] of this.connections) {
if (db !== this.serviceConfig.db) { // Don't close our main connection
if (db !== 0) { // Don't close our main connection (DB 0 for queues)
connection.disconnect();
this._logger.debug('Closed Redis connection', { db });
}