queue work
This commit is contained in:
parent
c05a7413dc
commit
d3ef73ae00
9 changed files with 938 additions and 1086 deletions
|
|
@ -1,578 +1,394 @@
|
|||
import { Queue, QueueEvents, Worker, type Job } from 'bullmq';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { DeadLetterQueueHandler } from './dlq-handler';
|
||||
import { handlerRegistry } from './handler-registry';
|
||||
import { QueueMetricsCollector } from './queue-metrics';
|
||||
import { QueueRateLimiter, type RateLimitRule } from './rate-limiter';
|
||||
import type { HandlerConfig, HandlerInitializer, JobData, QueueConfig } from './types';
|
||||
import { QueueRateLimiter } from './rate-limiter';
|
||||
import { Queue, type QueueConfig } from './queue';
|
||||
import { CacheProvider, createCache } from '@stock-bot/cache';
|
||||
import type {
|
||||
QueueManagerConfig,
|
||||
QueueOptions,
|
||||
GlobalStats,
|
||||
QueueStats,
|
||||
RateLimitRule
|
||||
} from './types';
|
||||
import { getRedisConnection } from './utils';
|
||||
|
||||
const logger = getLogger('queue-manager');
|
||||
|
||||
/**
|
||||
* Singleton QueueManager that provides unified queue and cache management
|
||||
* Main entry point for all queue operations with getQueue() method
|
||||
*/
|
||||
export class QueueManager {
|
||||
private queue!: Queue;
|
||||
private workers: Worker[] = [];
|
||||
private queueEvents!: QueueEvents;
|
||||
private config: Required<QueueConfig>;
|
||||
private handlers: HandlerInitializer[];
|
||||
private enableScheduledJobs: boolean;
|
||||
private dlqHandler?: DeadLetterQueueHandler;
|
||||
private metricsCollector?: QueueMetricsCollector;
|
||||
private static instance: QueueManager | null = null;
|
||||
private queues = new Map<string, Queue>();
|
||||
private caches = new Map<string, CacheProvider>();
|
||||
private rateLimiter?: QueueRateLimiter;
|
||||
private redisConnection: any;
|
||||
private isShuttingDown = false;
|
||||
private isInitialized = false;
|
||||
|
||||
private get isInitialized() {
|
||||
return !!this.queue;
|
||||
private constructor(private config: QueueManagerConfig) {
|
||||
this.redisConnection = getRedisConnection(config.redis);
|
||||
|
||||
// Initialize rate limiter if rules are provided
|
||||
if (config.rateLimitRules && config.rateLimitRules.length > 0) {
|
||||
this.rateLimiter = new QueueRateLimiter(this.redisConnection);
|
||||
config.rateLimitRules.forEach(rule => {
|
||||
this.rateLimiter!.addRule(rule);
|
||||
});
|
||||
}
|
||||
|
||||
this.isInitialized = true;
|
||||
logger.info('QueueManager singleton initialized');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the queue name
|
||||
* Get the singleton instance
|
||||
*/
|
||||
get queueName(): string {
|
||||
return this.config.queueName;
|
||||
}
|
||||
|
||||
constructor(config: QueueConfig = {}) {
|
||||
// Enhanced configuration
|
||||
this.handlers = config.handlers || [];
|
||||
this.enableScheduledJobs = config.enableScheduledJobs ?? true;
|
||||
|
||||
// Set default configuration
|
||||
this.config = {
|
||||
workers: config.workers ?? 5,
|
||||
concurrency: config.concurrency ?? 20,
|
||||
redis: {
|
||||
host: config.redis?.host || 'localhost',
|
||||
port: config.redis?.port || 6379,
|
||||
password: config.redis?.password || '',
|
||||
db: config.redis?.db || 0,
|
||||
},
|
||||
queueName: config.queueName || 'default-queue',
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 10,
|
||||
removeOnFail: 5,
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000,
|
||||
},
|
||||
...config.defaultJobOptions,
|
||||
},
|
||||
handlers: this.handlers,
|
||||
enableScheduledJobs: this.enableScheduledJobs,
|
||||
enableRateLimit: config.enableRateLimit || false,
|
||||
globalRateLimit: config.globalRateLimit,
|
||||
enableDLQ: config.enableDLQ || false,
|
||||
dlqConfig: config.dlqConfig,
|
||||
enableMetrics: config.enableMetrics || false,
|
||||
rateLimitRules: config.rateLimitRules || [],
|
||||
};
|
||||
static getInstance(config?: QueueManagerConfig): QueueManager {
|
||||
if (!QueueManager.instance) {
|
||||
if (!config) {
|
||||
throw new Error('QueueManager not initialized. Provide config on first call.');
|
||||
}
|
||||
QueueManager.instance = new QueueManager(config);
|
||||
}
|
||||
return QueueManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the queue manager with enhanced handler and scheduled job support
|
||||
* Initialize the singleton with config
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
logger.warn('Queue manager already initialized');
|
||||
return;
|
||||
static initialize(config: QueueManagerConfig): QueueManager {
|
||||
if (QueueManager.instance) {
|
||||
logger.warn('QueueManager already initialized, returning existing instance');
|
||||
return QueueManager.instance;
|
||||
}
|
||||
QueueManager.instance = new QueueManager(config);
|
||||
return QueueManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the singleton (mainly for testing)
|
||||
*/
|
||||
static async reset(): Promise<void> {
|
||||
if (QueueManager.instance) {
|
||||
await QueueManager.instance.shutdown();
|
||||
QueueManager.instance = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a queue - unified method that handles both scenarios
|
||||
* This is the main method for accessing queues
|
||||
*/
|
||||
getQueue(queueName: string, options: QueueOptions = {}): Queue {
|
||||
// Return existing queue if it exists
|
||||
if (this.queues.has(queueName)) {
|
||||
return this.queues.get(queueName)!;
|
||||
}
|
||||
|
||||
logger.info('Initializing enhanced queue manager...', {
|
||||
queueName: this.config.queueName,
|
||||
workers: this.config.workers,
|
||||
concurrency: this.config.concurrency,
|
||||
handlers: this.handlers.length,
|
||||
enableScheduledJobs: this.enableScheduledJobs,
|
||||
// Create new queue with merged options
|
||||
const mergedOptions = {
|
||||
...this.config.defaultQueueOptions,
|
||||
...options,
|
||||
};
|
||||
|
||||
// Prepare queue configuration
|
||||
const queueConfig: QueueConfig = {
|
||||
workers: mergedOptions.workers,
|
||||
concurrency: mergedOptions.concurrency,
|
||||
startWorker: mergedOptions.workers && mergedOptions.workers > 0,
|
||||
};
|
||||
|
||||
const queue = new Queue(
|
||||
queueName,
|
||||
this.config.redis,
|
||||
mergedOptions.defaultJobOptions || {},
|
||||
queueConfig
|
||||
);
|
||||
|
||||
// Store the queue
|
||||
this.queues.set(queueName, queue);
|
||||
|
||||
// Automatically initialize batch cache for the queue
|
||||
this.initializeBatchCacheSync(queueName);
|
||||
|
||||
// Add queue-specific rate limit rules
|
||||
if (this.rateLimiter && mergedOptions.rateLimitRules) {
|
||||
mergedOptions.rateLimitRules.forEach(rule => {
|
||||
// Ensure queue name is set for queue-specific rules
|
||||
const ruleWithQueue = { ...rule, queueName };
|
||||
this.rateLimiter!.addRule(ruleWithQueue);
|
||||
});
|
||||
}
|
||||
|
||||
logger.info('Queue created with batch cache', {
|
||||
queueName,
|
||||
workers: mergedOptions.workers || 0,
|
||||
concurrency: mergedOptions.concurrency || 1
|
||||
});
|
||||
|
||||
try {
|
||||
// Step 1: Register all handlers
|
||||
await this.registerHandlers();
|
||||
return queue;
|
||||
}
|
||||
|
||||
// Step 2: Initialize core queue infrastructure
|
||||
const connection = this.getConnection();
|
||||
const queueName = `{${this.config.queueName}}`;
|
||||
/**
|
||||
* Check if a queue exists
|
||||
*/
|
||||
hasQueue(queueName: string): boolean {
|
||||
return this.queues.has(queueName);
|
||||
}
|
||||
|
||||
// Initialize queue
|
||||
this.queue = new Queue(queueName, {
|
||||
connection,
|
||||
defaultJobOptions: this.config.defaultJobOptions,
|
||||
/**
|
||||
* Get all queue names
|
||||
*/
|
||||
getQueueNames(): string[] {
|
||||
return Array.from(this.queues.keys());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a cache for a queue
|
||||
*/
|
||||
getCache(queueName: string): CacheProvider {
|
||||
if (!this.caches.has(queueName)) {
|
||||
const cacheProvider = createCache({
|
||||
redisConfig: this.config.redis,
|
||||
keyPrefix: `batch:${queueName}:`,
|
||||
ttl: 86400, // 24 hours default
|
||||
enableMetrics: true,
|
||||
});
|
||||
|
||||
// Initialize queue events
|
||||
this.queueEvents = new QueueEvents(queueName, { connection });
|
||||
|
||||
// Wait for queue to be ready
|
||||
await this.queue.waitUntilReady();
|
||||
|
||||
// Step 3: Initialize DLQ handler if enabled
|
||||
if (this.config.enableDLQ) {
|
||||
this.dlqHandler = new DeadLetterQueueHandler(this.queue, connection, this.config.dlqConfig);
|
||||
}
|
||||
|
||||
// Step 4: Initialize metrics collector if enabled
|
||||
if (this.config.enableMetrics) {
|
||||
this.metricsCollector = new QueueMetricsCollector(this.queue, this.queueEvents);
|
||||
}
|
||||
|
||||
// Step 5: Initialize rate limiter if enabled
|
||||
if (this.config.enableRateLimit && this.config.rateLimitRules) {
|
||||
const redis = await this.getRedisClient();
|
||||
this.rateLimiter = new QueueRateLimiter(redis);
|
||||
|
||||
// Add configured rate limit rules
|
||||
for (const rule of this.config.rateLimitRules) {
|
||||
this.rateLimiter.addRule(rule);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 6: Start workers
|
||||
await this.startWorkers();
|
||||
|
||||
// Step 7: Setup event listeners
|
||||
this.setupEventListeners();
|
||||
|
||||
// Step 8: Batch cache will be initialized by individual Queue instances
|
||||
|
||||
// Step 9: Set up scheduled jobs
|
||||
if (this.enableScheduledJobs) {
|
||||
await this.setupScheduledJobs();
|
||||
}
|
||||
|
||||
logger.info('Enhanced queue manager initialized successfully');
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize enhanced queue manager', { error });
|
||||
throw error;
|
||||
this.caches.set(queueName, cacheProvider);
|
||||
logger.debug('Cache created for queue', { queueName });
|
||||
}
|
||||
return this.caches.get(queueName)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register all configured handlers
|
||||
* Initialize cache for a queue (ensures it's ready)
|
||||
*/
|
||||
private async registerHandlers(): Promise<void> {
|
||||
logger.info('Registering queue handlers...', { count: this.handlers.length });
|
||||
|
||||
// Initialize handlers using the configured handler initializers
|
||||
for (const handlerInitializer of this.handlers) {
|
||||
try {
|
||||
await handlerInitializer();
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize handler', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Now register all handlers from the registry with the queue manager
|
||||
const allHandlers = handlerRegistry.getAllHandlers();
|
||||
for (const [handlerName, config] of allHandlers) {
|
||||
this.registerHandler(handlerName, config.operations);
|
||||
logger.info(`Registered handler: ${handlerName}`);
|
||||
}
|
||||
|
||||
// Log scheduled jobs
|
||||
const scheduledJobs = handlerRegistry.getAllScheduledJobs();
|
||||
logger.info(`Registered ${scheduledJobs.length} scheduled jobs across all handlers`);
|
||||
for (const { handler, job } of scheduledJobs) {
|
||||
logger.info(
|
||||
`Scheduled job: ${handler}.${job.type} - ${job.description} (${job.cronPattern})`
|
||||
);
|
||||
}
|
||||
|
||||
logger.info('All handlers registered successfully');
|
||||
async initializeCache(queueName: string): Promise<void> {
|
||||
const cache = this.getCache(queueName);
|
||||
await cache.waitForReady(10000);
|
||||
logger.info('Cache initialized for queue', { queueName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up scheduled jobs from handler registry
|
||||
* Initialize batch cache synchronously (for automatic initialization)
|
||||
* The cache will be ready for use, but we don't wait for Redis connection
|
||||
*/
|
||||
private async setupScheduledJobs(): Promise<void> {
|
||||
const scheduledJobs = handlerRegistry.getAllScheduledJobs();
|
||||
private initializeBatchCacheSync(queueName: string): void {
|
||||
// Just create the cache - it will connect automatically when first used
|
||||
this.getCache(queueName);
|
||||
logger.debug('Batch cache initialized synchronously for queue', { queueName });
|
||||
}
|
||||
|
||||
if (scheduledJobs.length === 0) {
|
||||
logger.info('No scheduled jobs found');
|
||||
return;
|
||||
/**
|
||||
* Get statistics for all queues
|
||||
*/
|
||||
async getGlobalStats(): Promise<GlobalStats> {
|
||||
const queueStats: Record<string, QueueStats> = {};
|
||||
let totalJobs = 0;
|
||||
let totalWorkers = 0;
|
||||
|
||||
for (const [queueName, queue] of this.queues) {
|
||||
const stats = await queue.getStats();
|
||||
queueStats[queueName] = stats;
|
||||
|
||||
totalJobs += stats.waiting + stats.active + stats.completed + stats.failed + stats.delayed;
|
||||
totalWorkers += stats.workers || 0;
|
||||
}
|
||||
|
||||
logger.info(`Setting up ${scheduledJobs.length} scheduled jobs...`);
|
||||
|
||||
for (const { handler, job } of scheduledJobs) {
|
||||
try {
|
||||
const jobData: JobData = {
|
||||
type: job.type,
|
||||
handler,
|
||||
operation: job.operation,
|
||||
payload: job.payload,
|
||||
priority: job.priority,
|
||||
};
|
||||
|
||||
await this.add(`recurring-${handler}-${job.operation}`, jobData, {
|
||||
repeat: {
|
||||
pattern: job.cronPattern,
|
||||
tz: 'UTC',
|
||||
immediately: job.immediately || false,
|
||||
},
|
||||
delay: job.delay || 0,
|
||||
removeOnComplete: 1,
|
||||
removeOnFail: 1,
|
||||
attempts: 2,
|
||||
backoff: {
|
||||
type: 'fixed',
|
||||
delay: 5000,
|
||||
},
|
||||
});
|
||||
|
||||
logger.info(`Scheduled job registered: ${handler}.${job.type} (${job.cronPattern})`);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to register scheduled job: ${handler}.${job.type}`, { error });
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('Scheduled jobs setup complete');
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a handler with its operations
|
||||
*/
|
||||
registerHandler(handlerName: string, config: HandlerConfig): void {
|
||||
handlerRegistry.register(handlerName, config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a single job to the queue
|
||||
*/
|
||||
async add(name: string, data: JobData, options: Record<string, unknown> = {}): Promise<Job> {
|
||||
this.ensureInitialized();
|
||||
return await this.queue.add(name, data, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add multiple jobs to the queue in bulk
|
||||
*/
|
||||
async addBulk(
|
||||
jobs: Array<{ name: string; data: JobData; opts?: Record<string, unknown> }>
|
||||
): Promise<Job[]> {
|
||||
this.ensureInitialized();
|
||||
return await this.queue.addBulk(jobs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics
|
||||
*/
|
||||
async getStats(): Promise<{
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
}> {
|
||||
this.ensureInitialized();
|
||||
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
||||
this.queue.getWaiting(),
|
||||
this.queue.getActive(),
|
||||
this.queue.getCompleted(),
|
||||
this.queue.getFailed(),
|
||||
this.queue.getDelayed(),
|
||||
]);
|
||||
|
||||
return {
|
||||
waiting: waiting.length,
|
||||
active: active.length,
|
||||
completed: completed.length,
|
||||
failed: failed.length,
|
||||
delayed: delayed.length,
|
||||
queues: queueStats,
|
||||
totalJobs,
|
||||
totalWorkers,
|
||||
uptime: process.uptime(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause the queue
|
||||
* Get statistics for a specific queue
|
||||
*/
|
||||
async pause(): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
await this.queue.pause();
|
||||
logger.info('Queue paused');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume the queue
|
||||
*/
|
||||
async resume(): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
await this.queue.resume();
|
||||
logger.info('Queue resumed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean completed and failed jobs
|
||||
*/
|
||||
async clean(grace: number = 0, limit: number = 100): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
await Promise.all([
|
||||
this.queue.clean(grace, limit, 'completed'),
|
||||
this.queue.clean(grace, limit, 'failed'),
|
||||
]);
|
||||
logger.info('Queue cleaned', { grace, limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the queue name
|
||||
*/
|
||||
getQueueName(): string {
|
||||
return this.config.queueName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the redis configuration
|
||||
*/
|
||||
getRedisConfig(): any {
|
||||
return this.config.redis;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue metrics
|
||||
*/
|
||||
async getMetrics() {
|
||||
if (!this.metricsCollector) {
|
||||
throw new Error('Metrics not enabled. Set enableMetrics: true in config');
|
||||
async getQueueStats(queueName: string): Promise<QueueStats | undefined> {
|
||||
const queue = this.queues.get(queueName);
|
||||
if (!queue) {
|
||||
return undefined;
|
||||
}
|
||||
return this.metricsCollector.collect();
|
||||
return await queue.getStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metrics report
|
||||
*/
|
||||
async getMetricsReport(): Promise<string> {
|
||||
if (!this.metricsCollector) {
|
||||
throw new Error('Metrics not enabled. Set enableMetrics: true in config');
|
||||
}
|
||||
return this.metricsCollector.getReport();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get DLQ stats
|
||||
*/
|
||||
async getDLQStats() {
|
||||
if (!this.dlqHandler) {
|
||||
throw new Error('DLQ not enabled. Set enableDLQ: true in config');
|
||||
}
|
||||
return this.dlqHandler.getStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry jobs from DLQ
|
||||
*/
|
||||
async retryDLQJobs(limit = 10) {
|
||||
if (!this.dlqHandler) {
|
||||
throw new Error('DLQ not enabled. Set enableDLQ: true in config');
|
||||
}
|
||||
return this.dlqHandler.retryDLQJobs(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add rate limit rule
|
||||
* Add a rate limit rule
|
||||
*/
|
||||
addRateLimitRule(rule: RateLimitRule): void {
|
||||
if (!this.rateLimiter) {
|
||||
throw new Error('Rate limiting not enabled. Set enableRateLimit: true in config');
|
||||
this.rateLimiter = new QueueRateLimiter(this.redisConnection);
|
||||
}
|
||||
this.rateLimiter.addRule(rule);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get rate limit status
|
||||
* Check rate limits for a job
|
||||
*/
|
||||
async getRateLimitStatus(handler: string, operation: string) {
|
||||
async checkRateLimit(queueName: string, handler: string, operation: string): Promise<{
|
||||
allowed: boolean;
|
||||
retryAfter?: number;
|
||||
remainingPoints?: number;
|
||||
appliedRule?: RateLimitRule;
|
||||
}> {
|
||||
if (!this.rateLimiter) {
|
||||
throw new Error('Rate limiting not enabled. Set enableRateLimit: true in config');
|
||||
return { allowed: true };
|
||||
}
|
||||
return this.rateLimiter.getStatus(handler, operation);
|
||||
|
||||
return await this.rateLimiter.checkLimit(queueName, handler, operation);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the queue manager
|
||||
* Get rate limit status
|
||||
*/
|
||||
async getRateLimitStatus(queueName: string, handler: string, operation: string) {
|
||||
if (!this.rateLimiter) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
};
|
||||
}
|
||||
|
||||
return await this.rateLimiter.getStatus(queueName, handler, operation);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause all queues
|
||||
*/
|
||||
async pauseAll(): Promise<void> {
|
||||
const pausePromises = Array.from(this.queues.values()).map(queue => queue.pause());
|
||||
await Promise.all(pausePromises);
|
||||
logger.info('All queues paused');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume all queues
|
||||
*/
|
||||
async resumeAll(): Promise<void> {
|
||||
const resumePromises = Array.from(this.queues.values()).map(queue => queue.resume());
|
||||
await Promise.all(resumePromises);
|
||||
logger.info('All queues resumed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause a specific queue
|
||||
*/
|
||||
async pauseQueue(queueName: string): Promise<boolean> {
|
||||
const queue = this.queues.get(queueName);
|
||||
if (!queue) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await queue.pause();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume a specific queue
|
||||
*/
|
||||
async resumeQueue(queueName: string): Promise<boolean> {
|
||||
const queue = this.queues.get(queueName);
|
||||
if (!queue) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await queue.resume();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain all queues
|
||||
*/
|
||||
async drainAll(delayed = false): Promise<void> {
|
||||
const drainPromises = Array.from(this.queues.values()).map(queue => queue.drain(delayed));
|
||||
await Promise.all(drainPromises);
|
||||
logger.info('All queues drained', { delayed });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean all queues
|
||||
*/
|
||||
async cleanAll(
|
||||
grace: number = 0,
|
||||
limit: number = 100,
|
||||
type: 'completed' | 'failed' = 'completed'
|
||||
): Promise<void> {
|
||||
const cleanPromises = Array.from(this.queues.values()).map(queue =>
|
||||
queue.clean(grace, limit, type)
|
||||
);
|
||||
await Promise.all(cleanPromises);
|
||||
logger.info('All queues cleaned', { type, grace, limit });
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Shutdown all queues and workers
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
logger.info('Shutting down queue manager...');
|
||||
|
||||
const shutdownTasks: Promise<void>[] = [];
|
||||
|
||||
try {
|
||||
// Shutdown DLQ handler
|
||||
if (this.dlqHandler) {
|
||||
shutdownTasks.push(
|
||||
this.dlqHandler.shutdown().catch(err =>
|
||||
logger.warn('Error shutting down DLQ handler', { error: err })
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Close workers
|
||||
if (this.workers.length > 0) {
|
||||
shutdownTasks.push(
|
||||
Promise.all(
|
||||
this.workers.map(worker =>
|
||||
worker.close().catch(err =>
|
||||
logger.warn('Error closing worker', { error: err })
|
||||
)
|
||||
)
|
||||
).then(() => {
|
||||
this.workers = [];
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// Close queue events
|
||||
if (this.queueEvents) {
|
||||
shutdownTasks.push(
|
||||
this.queueEvents.close().catch(err =>
|
||||
logger.warn('Error closing queue events', { error: err })
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Close queue
|
||||
if (this.queue) {
|
||||
shutdownTasks.push(
|
||||
this.queue.close().catch(err =>
|
||||
logger.warn('Error closing queue', { error: err })
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Wait for all shutdown tasks with a timeout
|
||||
await Promise.race([
|
||||
Promise.all(shutdownTasks),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Shutdown timeout')), 5000)
|
||||
)
|
||||
]).catch(err => {
|
||||
logger.warn('Some shutdown tasks did not complete cleanly', { error: err });
|
||||
});
|
||||
|
||||
logger.info('Queue manager shutdown complete');
|
||||
} catch (error) {
|
||||
logger.error('Error during queue manager shutdown', { error });
|
||||
// Don't throw in shutdown to avoid hanging tests
|
||||
}
|
||||
}
|
||||
|
||||
private getConnection() {
|
||||
return getRedisConnection(this.config.redis);
|
||||
}
|
||||
|
||||
private async getRedisClient() {
|
||||
// Create a redis client for rate limiting
|
||||
const Redis = require('ioredis');
|
||||
return new Redis(this.getConnection());
|
||||
}
|
||||
|
||||
private async startWorkers(): Promise<void> {
|
||||
const connection = this.getConnection();
|
||||
const queueName = `{${this.config.queueName}}`;
|
||||
|
||||
for (let i = 0; i < this.config.workers; i++) {
|
||||
const worker = new Worker(queueName, this.processJob.bind(this), {
|
||||
connection,
|
||||
concurrency: this.config.concurrency,
|
||||
});
|
||||
|
||||
worker.on('completed', job => {
|
||||
logger.debug('Job completed', {
|
||||
id: job.id,
|
||||
name: job.name,
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
logger.error('Job failed', {
|
||||
id: job?.id,
|
||||
name: job?.name,
|
||||
error: err.message,
|
||||
});
|
||||
});
|
||||
|
||||
this.workers.push(worker);
|
||||
if (this.isShuttingDown) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Started ${this.config.workers} workers`);
|
||||
}
|
||||
|
||||
private async processJob(job: Job) {
|
||||
const { handler, operation, payload }: JobData = job.data;
|
||||
|
||||
logger.info('Processing job', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
payloadKeys: Object.keys(payload || {}),
|
||||
});
|
||||
this.isShuttingDown = true;
|
||||
logger.info('Shutting down QueueManager...');
|
||||
|
||||
try {
|
||||
// Check rate limits if enabled
|
||||
if (this.rateLimiter) {
|
||||
const rateLimit = await this.rateLimiter.checkLimit(handler, operation);
|
||||
if (!rateLimit.allowed) {
|
||||
// Reschedule job with delay
|
||||
const delay = rateLimit.retryAfter || 60000;
|
||||
logger.warn('Job rate limited, rescheduling', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
retryAfter: delay,
|
||||
});
|
||||
|
||||
throw new Error(`Rate limited. Retry after ${delay}ms`);
|
||||
// Close all queues (this now includes workers since they're managed by Queue class)
|
||||
const queueShutdownPromises = Array.from(this.queues.values()).map(async (queue) => {
|
||||
try {
|
||||
await queue.close();
|
||||
} catch (error) {
|
||||
logger.warn('Error closing queue', { error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
// Regular handler lookup
|
||||
const jobHandler = handlerRegistry.getHandler(handler, operation);
|
||||
|
||||
if (!jobHandler) {
|
||||
throw new Error(`No handler found for ${handler}:${operation}`);
|
||||
}
|
||||
|
||||
const result = await jobHandler(payload);
|
||||
|
||||
logger.info('Job completed successfully', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
});
|
||||
|
||||
return result;
|
||||
await Promise.all(queueShutdownPromises);
|
||||
|
||||
// Close all caches
|
||||
const cacheShutdownPromises = Array.from(this.caches.values()).map(async (cache) => {
|
||||
try {
|
||||
// Try different disconnect methods as different cache providers may use different names
|
||||
if (typeof cache.disconnect === 'function') {
|
||||
await cache.disconnect();
|
||||
} else if (typeof cache.close === 'function') {
|
||||
await cache.close();
|
||||
} else if (typeof cache.quit === 'function') {
|
||||
await cache.quit();
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Error closing cache', { error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(cacheShutdownPromises);
|
||||
|
||||
// Clear collections
|
||||
this.queues.clear();
|
||||
this.caches.clear();
|
||||
|
||||
logger.info('QueueManager shutdown complete');
|
||||
} catch (error) {
|
||||
logger.error('Job processing failed', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
|
||||
// Handle DLQ if enabled
|
||||
if (this.dlqHandler && error instanceof Error) {
|
||||
await this.dlqHandler.handleFailedJob(job, error);
|
||||
}
|
||||
|
||||
logger.error('Error during shutdown', { error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private setupEventListeners(): void {
|
||||
this.queueEvents.on('completed', ({ jobId }) => {
|
||||
logger.debug('Job completed event', { jobId });
|
||||
});
|
||||
|
||||
this.queueEvents.on('failed', ({ jobId, failedReason }) => {
|
||||
logger.warn('Job failed event', { jobId, failedReason });
|
||||
});
|
||||
|
||||
this.queueEvents.on('stalled', ({ jobId }) => {
|
||||
logger.warn('Job stalled event', { jobId });
|
||||
});
|
||||
/**
|
||||
* Wait for all queues to be ready
|
||||
*/
|
||||
async waitUntilReady(): Promise<void> {
|
||||
const readyPromises = Array.from(this.queues.values()).map(queue => queue.waitUntilReady());
|
||||
await Promise.all(readyPromises);
|
||||
}
|
||||
|
||||
private ensureInitialized(): void {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('Queue manager not initialized. Call initialize() first.');
|
||||
}
|
||||
/**
|
||||
* Get Redis configuration (for backward compatibility)
|
||||
*/
|
||||
getRedisConfig() {
|
||||
return this.config.redis;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue