boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

handler to auto register and removed service registry, cleaned up queues and cache naming

Browse source
This commit is contained in:
Boki 2025-06-23 21:23:38 -04:00
parent 0d1be9e3cb
commit 34c6c36695
19 changed files with 474 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

10
libs/core/queue/src/index.ts
View file

@ -3,13 +3,13 @@ export { Queue } from './queue';
export { QueueManager } from './queue-manager';
export { SmartQueueManager } from './smart-queue-manager';
export { ServiceCache, createServiceCache } from './service-cache';
// Service utilities
export {
SERVICE_REGISTRY,
getServiceConfig,
findServiceForHandler,
normalizeServiceName,
generateCachePrefix,
getFullQueueName,
parseQueueName
} from './service-registry';
} from './service-utils';
// Re-export handler registry and utilities from handlers package
export { handlerRegistry, createJobHandler } from '@stock-bot/handlers';
@ -71,5 +71,3 @@ export type {
} from './types';
// Re-export service registry types
export type { ServiceConfig } from './service-registry';

9
libs/core/queue/src/queue-manager.ts
View file

@ -8,6 +8,7 @@ import type {
QueueOptions,
QueueStats,
RateLimitRule,
RedisConfig,
} from './types';
import { getRedisConnection } from './utils';
@ -173,6 +174,14 @@ export class QueueManager {
this.logger.trace('Batch cache initialized synchronously for queue', { queueName });
}
/**
* Get the queues map (for subclasses)
*/
protected getQueues(): Map<string, Queue> {
return this.queues;
}
/**
* Get statistics for all queues
*/

8
libs/core/queue/src/queue.ts
View file

@ -45,7 +45,7 @@ export class Queue {
const connection = getRedisConnection(redisConfig);
// Initialize BullMQ queue
this.bullQueue = new BullQueue(`{${queueName}}`, {
this.bullQueue = new BullQueue(queueName, {
connection,
defaultJobOptions: {
removeOnComplete: 10,
@ -61,7 +61,7 @@ export class Queue {
// Initialize queue events if workers will be used
if (config.workers && config.workers > 0) {
this.queueEvents = new QueueEvents(`{${queueName}}`, { connection });
this.queueEvents = new QueueEvents(queueName, { connection });
}
// Start workers if requested and not explicitly disabled
@ -278,7 +278,7 @@ export class Queue {
const connection = getRedisConnection(this.redisConfig);
for (let i = 0; i < workerCount; i++) {
const worker = new Worker(`{${this.queueName}}`, this.processJob.bind(this), {
const worker = new Worker(this.queueName, this.processJob.bind(this), {
connection,
concurrency,
maxStalledCount: 3,
@ -378,7 +378,7 @@ export class Queue {
// Initialize queue events if not already done
if (!this.queueEvents) {
const connection = getRedisConnection(this.redisConfig);
this.queueEvents = new QueueEvents(`{${this.queueName}}`, { connection });
this.queueEvents = new QueueEvents(this.queueName, { connection });
}
this.startWorkers(workerCount, concurrency);

30
libs/core/queue/src/service-cache.ts
View file

@ -1,6 +1,6 @@
import { createCache, type CacheProvider, type CacheStats } from '@stock-bot/cache';
import type { RedisConfig } from './types';
import { getServiceConfig } from './service-registry';
import { generateCachePrefix } from './service-utils';
/**
* Service-aware cache that uses the service's Redis DB
@ -16,24 +16,18 @@ export class ServiceCache implements CacheProvider {
isGlobalCache: boolean = false,
logger?: any
) {
// Get service configuration
const serviceConfig = getServiceConfig(serviceName);
if (!serviceConfig && !isGlobalCache) {
throw new Error(`Unknown service: ${serviceName}`);
}
// Determine Redis DB and prefix
let db: number;
let prefix: string;
if (isGlobalCache) {
// Global cache uses db:0
db = 0;
// Global cache uses db:1
db = 1;
prefix = 'stock-bot:shared';
} else {
// Service cache uses service's DB
db = serviceConfig!.db;
prefix = serviceConfig!.cachePrefix;
// Service cache also uses db:1 with service-specific prefix
db = 1;
prefix = generateCachePrefix(serviceName);
}
// Create underlying cache with correct DB
@ -148,6 +142,18 @@ export class ServiceCache implements CacheProvider {
return this.cache.set(key, updated, ttl);
}
/**
* Get a value using a raw Redis key (bypassing the keyPrefix)
* Delegates to the underlying cache's getRaw method if available
*/
async getRaw<T = unknown>(key: string): Promise<T | null> {
if (this.cache.getRaw) {
return this.cache.getRaw<T>(key);
}
// Fallback: if underlying cache doesn't support getRaw, return null
return null;
}
/**
* Get the actual Redis key with prefix
*/

71
libs/core/queue/src/service-registry.ts → libs/core/queue/src/service-registry.deprecated.ts
View file

@ -1,67 +1,54 @@
/**
* Service Registry Configuration
* Maps services to their Redis databases and configurations
*
* @deprecated This static service registry has been replaced by runtime discovery
* using the handler registry. Service ownership is now tracked when handlers are
* registered, eliminating the need for static configuration.
*
* Migration:
* - Service names are auto-discovered from handler registration
* - Cache prefixes are generated using generateCachePrefix()
* - Queue names use getFullQueueName() from service-utils
* - Handler ownership is tracked by handlerRegistry.getHandlerService()
*/
export interface ServiceConfig {
/** Redis database number for this service (used for both queues and cache) */
db: number;
/** Prefix for queue keys (e.g., 'bull:di') */
queuePrefix: string;
/** Prefix for cache keys (e.g., 'cache:di') */
/** Prefix for cache keys (e.g., 'cache:data-ingestion') */
cachePrefix: string;
/** Whether this service only produces jobs (doesn't process them) */
producerOnly?: boolean;
/** List of handlers this service owns (auto-discovered if not provided) */
handlers?: string[];
}
/**
* Central registry of all services and their configurations
* Each service gets one Redis DB for both queues and cache
*
* Database assignments:
* - db:0 = Global shared cache
* - db:1 = data-ingestion (queues + cache)
* - db:2 = data-pipeline (queues + cache)
* - db:3 = web-api (cache only, producer-only for queues)
* - db:0 = All queues (unified queue database)
* - db:1 = Global shared cache + service-specific caches
*/
export const SERVICE_REGISTRY: Record<string, ServiceConfig> = {
'data-ingestion': {
db: 1,
queuePrefix: 'bull:di',
cachePrefix: 'cache:di',
cachePrefix: 'cache:data-ingestion',
handlers: ['ceo', 'qm', 'webshare', 'ib', 'proxy'],
},
'data-pipeline': {
db: 2,
queuePrefix: 'bull:dp',
cachePrefix: 'cache:dp',
cachePrefix: 'cache:data-pipeline',
handlers: ['exchanges', 'symbols'],
},
'web-api': {
db: 3,
queuePrefix: 'bull:api', // Not used since producer-only
cachePrefix: 'cache:api',
producerOnly: true,
cachePrefix: 'cache:web-api',
},
// Add aliases for services with different naming conventions
'webApi': {
db: 3,
queuePrefix: 'bull:api',
cachePrefix: 'cache:api',
producerOnly: true,
cachePrefix: 'cache:web-api',
},
'dataIngestion': {
db: 1,
queuePrefix: 'bull:di',
cachePrefix: 'cache:di',
cachePrefix: 'cache:data-ingestion',
handlers: ['ceo', 'qm', 'webshare', 'ib', 'proxy'],
},
'dataPipeline': {
db: 2,
queuePrefix: 'bull:dp',
cachePrefix: 'cache:dp',
cachePrefix: 'cache:data-pipeline',
handlers: ['exchanges', 'symbols'],
},
};
@ -86,30 +73,26 @@ export function findServiceForHandler(handlerName: string): string | undefined {
}
/**
* Get full queue name - just the handler name since each service has its own Redis DB
* Get full queue name with service namespace
*/
export function getFullQueueName(serviceName: string, handlerName: string): string {
// Just return the handler name since DB isolation provides namespace separation
return handlerName;
// Use {service_handler} format for Dragonfly optimization and BullMQ compatibility
return `{${serviceName}_${handlerName}}`;
}
/**
* Parse a full queue name into service and handler
* Since queue names are just handler names now, we need to find the service from the handler
*/
export function parseQueueName(fullQueueName: string): { service: string; handler: string } | null {
// Queue name is just the handler name now
const handlerName = fullQueueName;
// Match pattern {service_handler}
const match = fullQueueName.match(/^\{([^_]+)_([^}]+)\}$/);
// Find which service owns this handler
const serviceName = findServiceForHandler(handlerName);
if (!serviceName) {
if (!match || !match[1] || !match[2]) {
return null;
}
return {
service: serviceName,
handler: handlerName,
service: match[1],
handler: match[2],
};
}

53
libs/core/queue/src/service-utils.ts Normal file
View file

@ -0,0 +1,53 @@
/**
* Service utilities for name normalization and auto-discovery
*/
/**
* Normalize service name to kebab-case format
* Examples:
* - webApi -> web-api
* - dataIngestion -> data-ingestion
* - data-pipeline -> data-pipeline (unchanged)
*/
export function normalizeServiceName(serviceName: string): string {
// Handle camelCase to kebab-case conversion
const kebabCase = serviceName
.replace(/([a-z])([A-Z])/g, '$1-$2')
.toLowerCase();
return kebabCase;
}
/**
* Generate cache prefix for a service
*/
export function generateCachePrefix(serviceName: string): string {
const normalized = normalizeServiceName(serviceName);
return `cache:${normalized}`;
}
/**
* Generate full queue name with service namespace
*/
export function getFullQueueName(serviceName: string, handlerName: string): string {
const normalized = normalizeServiceName(serviceName);
// Use {service_handler} format for Dragonfly optimization and BullMQ compatibility
return `{${normalized}_${handlerName}}`;
}
/**
* Parse a full queue name into service and handler
*/
export function parseQueueName(fullQueueName: string): { service: string; handler: string } | null {
// Match pattern {service_handler}
const match = fullQueueName.match(/^\{([^_]+)_([^}]+)\}$/);
if (!match || !match[1] || !match[2]) {
return null;
}
return {
service: match[1],
handler: match[2],
};
}

214
libs/core/queue/src/smart-queue-manager.ts
View file

@ -10,14 +10,7 @@ import type {
JobOptions,
RedisConfig
} from './types';
import {
SERVICE_REGISTRY,
getServiceConfig,
findServiceForHandler,
getFullQueueName,
parseQueueName,
type ServiceConfig
} from './service-registry';
import { getFullQueueName, parseQueueName } from './service-utils';
import { getRedisConnection } from './utils';
/**
@ -26,32 +19,24 @@ import { getRedisConnection } from './utils';
*/
export class SmartQueueManager extends QueueManager {
private serviceName: string;
private serviceConfig: ServiceConfig;
private queueRoutes = new Map<string, QueueRoute>();
private connections = new Map<number, any>(); // Redis connections by DB
private producerQueues = new Map<string, BullQueue>(); // For cross-service sending
private _logger: Logger;
constructor(config: SmartQueueConfig, logger?: Logger) {
// Get service config
const serviceConfig = getServiceConfig(config.serviceName);
if (!serviceConfig) {
throw new Error(`Unknown service: ${config.serviceName}`);
}
// Update Redis config to use service's DB
// Always use DB 0 for queues (unified queue database)
const modifiedConfig = {
...config,
redis: {
...config.redis,
db: serviceConfig.db,
db: 0, // All queues in DB 0
},
};
super(modifiedConfig, logger);
this.serviceName = config.serviceName;
this.serviceConfig = serviceConfig;
this._logger = logger || getLogger('SmartQueueManager');
// Auto-discover routes if enabled
@ -61,9 +46,7 @@ export class SmartQueueManager extends QueueManager {
this._logger.info('SmartQueueManager initialized', {
service: this.serviceName,
db: serviceConfig.db,
handlers: serviceConfig.handlers,
producerOnly: serviceConfig.producerOnly,
discoveredRoutes: this.queueRoutes.size,
});
}
@ -71,51 +54,56 @@ export class SmartQueueManager extends QueueManager {
* Discover all available queue routes from handler registry
*/
private discoverQueueRoutes(): void {
// Discover from handler registry if available
try {
const handlers = handlerRegistry.getAllHandlers();
for (const [handlerName, handlerConfig] of handlers) {
// Find which service owns this handler
const ownerService = findServiceForHandler(handlerName);
// Get the service that registered this handler
const ownerService = handlerRegistry.getHandlerService(handlerName);
if (ownerService) {
const ownerConfig = getServiceConfig(ownerService)!;
const fullName = getFullQueueName(ownerService, handlerName);
this.queueRoutes.set(handlerName, {
fullName,
service: ownerService,
handler: handlerName,
db: ownerConfig.db,
db: 0, // All queues in DB 0
operations: Object.keys(handlerConfig.operations || {}),
});
this._logger.trace('Discovered queue route', {
handler: handlerName,
service: ownerService,
db: ownerConfig.db,
operations: Object.keys(handlerConfig.operations || {}).length,
});
} else {
this._logger.warn('Handler has no service ownership', { handlerName });
}
}
// Also discover handlers registered by the current service
const myHandlers = handlerRegistry.getServiceHandlers(this.serviceName);
for (const handlerName of myHandlers) {
if (!this.queueRoutes.has(handlerName)) {
const fullName = getFullQueueName(this.serviceName, handlerName);
this.queueRoutes.set(handlerName, {
fullName,
service: this.serviceName,
handler: handlerName,
db: 0, // All queues in DB 0
});
}
}
} catch (error) {
this._logger.warn('Handler registry not available, using static configuration', { error });
}
// Also add routes from static configuration
Object.entries(SERVICE_REGISTRY).forEach(([serviceName, config]) => {
if (config.handlers) {
config.handlers.forEach(handlerName => {
if (!this.queueRoutes.has(handlerName)) {
const fullName = getFullQueueName(serviceName, handlerName);
this.queueRoutes.set(handlerName, {
fullName,
service: serviceName,
handler: handlerName,
db: config.db,
});
}
});
}
});
this._logger.info('Queue routes discovered', {
totalRoutes: this.queueRoutes.size,
routes: Array.from(this.queueRoutes.values()).map(r => ({
handler: r.handler,
service: r.service
})),
});
} catch (error) {
this._logger.error('Failed to discover queue routes', { error });
}
}
/**
@ -136,11 +124,34 @@ export class SmartQueueManager extends QueueManager {
/**
* Get a queue for the current service (for processing)
* Overrides parent to use namespaced queue names
* Overrides parent to use namespaced queue names and ensure service-specific workers
*/
override getQueue(queueName: string, options = {}): Queue {
// For local queues, use the service namespace
const fullQueueName = getFullQueueName(this.serviceName, queueName);
// Check if this is already a full queue name (service:handler format)
const parsed = parseQueueName(queueName);
let fullQueueName: string;
let isOwnQueue: boolean;
if (parsed) {
// Already in service:handler format
fullQueueName = queueName;
isOwnQueue = parsed.service === this.serviceName;
} else {
// Just handler name, assume it's for current service
fullQueueName = getFullQueueName(this.serviceName, queueName);
isOwnQueue = true;
}
// For cross-service queues, create without workers (producer-only)
if (!isOwnQueue) {
return super.getQueue(fullQueueName, {
...options,
workers: 0, // No workers for other services' queues
});
}
// For own service queues, use configured workers
return super.getQueue(fullQueueName, options);
}
@ -212,35 +223,37 @@ export class SmartQueueManager extends QueueManager {
* Resolve a queue name to a route
*/
private resolveQueueRoute(queueName: string): QueueRoute | null {
// Check if it's a handler name (which is now the full queue name)
// Check if it's a full queue name with service prefix
const parsed = parseQueueName(queueName);
if (parsed) {
const config = getServiceConfig(parsed.service);
if (config) {
return {
fullName: queueName,
service: parsed.service,
handler: parsed.handler,
db: config.db,
};
// Try to find in discovered routes by handler name
const route = this.queueRoutes.get(parsed.handler);
if (route && route.service === parsed.service) {
return route;
}
// Create a route on the fly
return {
fullName: queueName,
service: parsed.service,
handler: parsed.handler,
db: 0, // All queues in DB 0
};
}
// Check if it's just a handler name
// Check if it's just a handler name in our routes
const route = this.queueRoutes.get(queueName);
if (route) {
return route;
}
// Try to find in static config
const ownerService = findServiceForHandler(queueName);
// Try to find in handler registry
const ownerService = handlerRegistry.getHandlerService(queueName);
if (ownerService) {
const config = getServiceConfig(ownerService)!;
return {
fullName: getFullQueueName(ownerService, queueName),
service: ownerService,
handler: queueName,
db: config.db,
db: 0, // All queues in DB 0
};
}
@ -253,8 +266,8 @@ export class SmartQueueManager extends QueueManager {
private getProducerQueue(route: QueueRoute): BullQueue {
if (!this.producerQueues.has(route.fullName)) {
const connection = this.getConnection(route.db);
// Match the queue name format used by workers: {queueName}
const queue = new BullQueue(`{${route.fullName}}`, {
// Use the same queue name format as workers
const queue = new BullQueue(route.fullName, {
connection,
defaultJobOptions: this.getConfig().defaultQueueOptions?.defaultJobOptions || {},
});
@ -276,8 +289,8 @@ export class SmartQueueManager extends QueueManager {
if (queue && typeof queue.getBullQueue === 'function') {
// Extract the underlying BullMQ queue using the public getter
// Use the simple handler name without service prefix for display
const parts = name.split(':');
const simpleName = parts.length > 1 ? parts[1] : name;
const parsed = parseQueueName(name);
const simpleName = parsed ? parsed.handler : name;
if (simpleName) {
allQueues[simpleName] = queue.getBullQueue();
}
@ -287,20 +300,18 @@ export class SmartQueueManager extends QueueManager {
// Add producer queues
for (const [name, queue] of this.producerQueues) {
// Use the simple handler name without service prefix for display
const parts = name.split(':');
const simpleName = parts.length > 1 ? parts[1] : name;
const parsed = parseQueueName(name);
const simpleName = parsed ? parsed.handler : name;
if (simpleName && !allQueues[simpleName]) {
allQueues[simpleName] = queue;
}
}
// If no queues found, return all registered handlers as BullMQ queues
// If no queues found, create from discovered routes
if (Object.keys(allQueues).length === 0) {
// Create BullMQ queue instances for known handlers
const handlers = ['proxy', 'qm', 'ib', 'ceo', 'webshare', 'exchanges', 'symbols'];
for (const handler of handlers) {
const connection = this.getConnection(1); // Use default DB
allQueues[handler] = new BullQueue(`{${handler}}`, {
for (const [handlerName, route] of this.queueRoutes) {
const connection = this.getConnection(0); // Use unified queue DB
allQueues[handlerName] = new BullQueue(route.fullName, {
connection,
defaultJobOptions: this.getConfig().defaultQueueOptions?.defaultJobOptions || {},
});
@ -325,6 +336,57 @@ export class SmartQueueManager extends QueueManager {
return stats;
}
/**
* Start workers for all queues belonging to this service
* Overrides parent to ensure only own queues get workers
*/
override startAllWorkers(): void {
if (!this.getConfig().delayWorkerStart) {
this._logger.info(
'startAllWorkers() called but workers already started automatically (delayWorkerStart is false)'
);
return;
}
let workersStarted = 0;
const queues = this.getQueues();
for (const [queueName, queue] of queues) {
// Parse queue name to check if it belongs to this service
const parsed = parseQueueName(queueName);
// Skip if not our service's queue
if (parsed && parsed.service !== this.serviceName) {
this._logger.trace('Skipping workers for cross-service queue', {
queueName,
ownerService: parsed.service,
currentService: this.serviceName,
});
continue;
}
const workerCount = this.getConfig().defaultQueueOptions?.workers || 1;
const concurrency = this.getConfig().defaultQueueOptions?.concurrency || 1;
if (workerCount > 0) {
queue.startWorkersManually(workerCount, concurrency);
workersStarted++;
this._logger.debug('Started workers for queue', {
queueName,
workers: workerCount,
concurrency,
});
}
}
this._logger.info('Service workers started', {
service: this.serviceName,
totalQueues: queues.size,
queuesWithWorkers: workersStarted,
delayWorkerStart: this.getConfig().delayWorkerStart,
});
}
/**
* Graceful shutdown
*/
@ -337,7 +399,7 @@ export class SmartQueueManager extends QueueManager {
// Close additional connections
for (const [db, connection] of this.connections) {
if (db !== this.serviceConfig.db) { // Don't close our main connection
if (db !== 0) { // Don't close our main connection (DB 0 for queues)
connection.disconnect();
this._logger.debug('Closed Redis connection', { db });
}