format
This commit is contained in:
parent
d858222af7
commit
7d9044ab29
202 changed files with 10755 additions and 10972 deletions
|
|
@ -1,251 +1,249 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import { Queue, type Job } from 'bullmq';
|
||||
import type { DLQConfig, RedisConfig } from './types';
|
||||
import { getRedisConnection } from './utils';
|
||||
|
||||
const logger = getLogger('dlq-handler');
|
||||
|
||||
export class DeadLetterQueueHandler {
|
||||
private dlq: Queue;
|
||||
private config: Required<DLQConfig>;
|
||||
private failureCount = new Map<string, number>();
|
||||
|
||||
constructor(
|
||||
private mainQueue: Queue,
|
||||
connection: RedisConfig,
|
||||
config: DLQConfig = {}
|
||||
) {
|
||||
this.config = {
|
||||
maxRetries: config.maxRetries ?? 3,
|
||||
retryDelay: config.retryDelay ?? 60000, // 1 minute
|
||||
alertThreshold: config.alertThreshold ?? 100,
|
||||
cleanupAge: config.cleanupAge ?? 168, // 7 days
|
||||
};
|
||||
|
||||
// Create DLQ with same name but -dlq suffix
|
||||
const dlqName = `${mainQueue.name}-dlq`;
|
||||
this.dlq = new Queue(dlqName, { connection: getRedisConnection(connection) });
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a failed job - either retry or move to DLQ
|
||||
*/
|
||||
async handleFailedJob(job: Job, error: Error): Promise<void> {
|
||||
const jobKey = `${job.name}:${job.id}`;
|
||||
const currentFailures = (this.failureCount.get(jobKey) || 0) + 1;
|
||||
this.failureCount.set(jobKey, currentFailures);
|
||||
|
||||
logger.warn('Job failed', {
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
attempt: job.attemptsMade,
|
||||
maxAttempts: job.opts.attempts,
|
||||
error: error.message,
|
||||
failureCount: currentFailures,
|
||||
});
|
||||
|
||||
// Check if job should be moved to DLQ
|
||||
if (job.attemptsMade >= (job.opts.attempts || this.config.maxRetries)) {
|
||||
await this.moveToDeadLetterQueue(job, error);
|
||||
this.failureCount.delete(jobKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Move job to dead letter queue
|
||||
*/
|
||||
private async moveToDeadLetterQueue(job: Job, error: Error): Promise<void> {
|
||||
try {
|
||||
const dlqData = {
|
||||
originalJob: {
|
||||
id: job.id,
|
||||
name: job.name,
|
||||
data: job.data,
|
||||
opts: job.opts,
|
||||
attemptsMade: job.attemptsMade,
|
||||
failedReason: job.failedReason,
|
||||
processedOn: job.processedOn,
|
||||
timestamp: job.timestamp,
|
||||
},
|
||||
error: {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
name: error.name,
|
||||
},
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
await this.dlq.add('failed-job', dlqData, {
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 50,
|
||||
});
|
||||
|
||||
logger.error('Job moved to DLQ', {
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
error: error.message,
|
||||
});
|
||||
|
||||
// Check if we need to alert
|
||||
await this.checkAlertThreshold();
|
||||
} catch (dlqError) {
|
||||
logger.error('Failed to move job to DLQ', {
|
||||
jobId: job.id,
|
||||
error: dlqError,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry jobs from DLQ
|
||||
*/
|
||||
async retryDLQJobs(limit = 10): Promise<number> {
|
||||
const jobs = await this.dlq.getCompleted(0, limit);
|
||||
let retriedCount = 0;
|
||||
|
||||
for (const dlqJob of jobs) {
|
||||
try {
|
||||
const { originalJob } = dlqJob.data;
|
||||
|
||||
// Re-add to main queue with delay
|
||||
await this.mainQueue.add(
|
||||
originalJob.name,
|
||||
originalJob.data,
|
||||
{
|
||||
...originalJob.opts,
|
||||
delay: this.config.retryDelay,
|
||||
attempts: this.config.maxRetries,
|
||||
}
|
||||
);
|
||||
|
||||
// Remove from DLQ
|
||||
await dlqJob.remove();
|
||||
retriedCount++;
|
||||
|
||||
logger.info('Job retried from DLQ', {
|
||||
originalJobId: originalJob.id,
|
||||
jobName: originalJob.name,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to retry DLQ job', {
|
||||
dlqJobId: dlqJob.id,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return retriedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get DLQ statistics
|
||||
*/
|
||||
async getStats(): Promise<{
|
||||
total: number;
|
||||
recent: number;
|
||||
byJobName: Record<string, number>;
|
||||
oldestJob: Date | null;
|
||||
}> {
|
||||
const [completed, failed, waiting] = await Promise.all([
|
||||
this.dlq.getCompleted(),
|
||||
this.dlq.getFailed(),
|
||||
this.dlq.getWaiting(),
|
||||
]);
|
||||
|
||||
const allJobs = [...completed, ...failed, ...waiting];
|
||||
const byJobName: Record<string, number> = {};
|
||||
let oldestTimestamp: number | null = null;
|
||||
|
||||
for (const job of allJobs) {
|
||||
const jobName = job.data.originalJob?.name || 'unknown';
|
||||
byJobName[jobName] = (byJobName[jobName] || 0) + 1;
|
||||
|
||||
if (!oldestTimestamp || job.timestamp < oldestTimestamp) {
|
||||
oldestTimestamp = job.timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
// Count recent jobs (last 24 hours)
|
||||
const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000;
|
||||
const recent = allJobs.filter(job => job.timestamp > oneDayAgo).length;
|
||||
|
||||
return {
|
||||
total: allJobs.length,
|
||||
recent,
|
||||
byJobName,
|
||||
oldestJob: oldestTimestamp ? new Date(oldestTimestamp) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old DLQ entries
|
||||
*/
|
||||
async cleanup(): Promise<number> {
|
||||
const ageInMs = this.config.cleanupAge * 60 * 60 * 1000;
|
||||
const cutoffTime = Date.now() - ageInMs;
|
||||
|
||||
const jobs = await this.dlq.getCompleted();
|
||||
let removedCount = 0;
|
||||
|
||||
for (const job of jobs) {
|
||||
if (job.timestamp < cutoffTime) {
|
||||
await job.remove();
|
||||
removedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('DLQ cleanup completed', {
|
||||
removedCount,
|
||||
cleanupAge: `${this.config.cleanupAge} hours`,
|
||||
});
|
||||
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if alert threshold is exceeded
|
||||
*/
|
||||
private async checkAlertThreshold(): Promise<void> {
|
||||
const stats = await this.getStats();
|
||||
|
||||
if (stats.total >= this.config.alertThreshold) {
|
||||
logger.error('DLQ alert threshold exceeded', {
|
||||
threshold: this.config.alertThreshold,
|
||||
currentCount: stats.total,
|
||||
byJobName: stats.byJobName,
|
||||
});
|
||||
// In a real implementation, this would trigger alerts
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get failed jobs for inspection
|
||||
*/
|
||||
async inspectFailedJobs(limit = 10): Promise<Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
data: unknown;
|
||||
error: unknown;
|
||||
failedAt: string;
|
||||
attempts: number;
|
||||
}>> {
|
||||
const jobs = await this.dlq.getCompleted(0, limit);
|
||||
|
||||
return jobs.map(job => ({
|
||||
id: job.data.originalJob.id,
|
||||
name: job.data.originalJob.name,
|
||||
data: job.data.originalJob.data,
|
||||
error: job.data.error,
|
||||
failedAt: job.data.movedToDLQAt,
|
||||
attempts: job.data.originalJob.attemptsMade,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown DLQ handler
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
await this.dlq.close();
|
||||
this.failureCount.clear();
|
||||
}
|
||||
}
|
||||
import { Queue, type Job } from 'bullmq';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { DLQConfig, RedisConfig } from './types';
|
||||
import { getRedisConnection } from './utils';
|
||||
|
||||
const logger = getLogger('dlq-handler');
|
||||
|
||||
export class DeadLetterQueueHandler {
|
||||
private dlq: Queue;
|
||||
private config: Required<DLQConfig>;
|
||||
private failureCount = new Map<string, number>();
|
||||
|
||||
constructor(
|
||||
private mainQueue: Queue,
|
||||
connection: RedisConfig,
|
||||
config: DLQConfig = {}
|
||||
) {
|
||||
this.config = {
|
||||
maxRetries: config.maxRetries ?? 3,
|
||||
retryDelay: config.retryDelay ?? 60000, // 1 minute
|
||||
alertThreshold: config.alertThreshold ?? 100,
|
||||
cleanupAge: config.cleanupAge ?? 168, // 7 days
|
||||
};
|
||||
|
||||
// Create DLQ with same name but -dlq suffix
|
||||
const dlqName = `${mainQueue.name}-dlq`;
|
||||
this.dlq = new Queue(dlqName, { connection: getRedisConnection(connection) });
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a failed job - either retry or move to DLQ
|
||||
*/
|
||||
async handleFailedJob(job: Job, error: Error): Promise<void> {
|
||||
const jobKey = `${job.name}:${job.id}`;
|
||||
const currentFailures = (this.failureCount.get(jobKey) || 0) + 1;
|
||||
this.failureCount.set(jobKey, currentFailures);
|
||||
|
||||
logger.warn('Job failed', {
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
attempt: job.attemptsMade,
|
||||
maxAttempts: job.opts.attempts,
|
||||
error: error.message,
|
||||
failureCount: currentFailures,
|
||||
});
|
||||
|
||||
// Check if job should be moved to DLQ
|
||||
if (job.attemptsMade >= (job.opts.attempts || this.config.maxRetries)) {
|
||||
await this.moveToDeadLetterQueue(job, error);
|
||||
this.failureCount.delete(jobKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Move job to dead letter queue
|
||||
*/
|
||||
private async moveToDeadLetterQueue(job: Job, error: Error): Promise<void> {
|
||||
try {
|
||||
const dlqData = {
|
||||
originalJob: {
|
||||
id: job.id,
|
||||
name: job.name,
|
||||
data: job.data,
|
||||
opts: job.opts,
|
||||
attemptsMade: job.attemptsMade,
|
||||
failedReason: job.failedReason,
|
||||
processedOn: job.processedOn,
|
||||
timestamp: job.timestamp,
|
||||
},
|
||||
error: {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
name: error.name,
|
||||
},
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
await this.dlq.add('failed-job', dlqData, {
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 50,
|
||||
});
|
||||
|
||||
logger.error('Job moved to DLQ', {
|
||||
jobId: job.id,
|
||||
jobName: job.name,
|
||||
error: error.message,
|
||||
});
|
||||
|
||||
// Check if we need to alert
|
||||
await this.checkAlertThreshold();
|
||||
} catch (dlqError) {
|
||||
logger.error('Failed to move job to DLQ', {
|
||||
jobId: job.id,
|
||||
error: dlqError,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry jobs from DLQ
|
||||
*/
|
||||
async retryDLQJobs(limit = 10): Promise<number> {
|
||||
const jobs = await this.dlq.getCompleted(0, limit);
|
||||
let retriedCount = 0;
|
||||
|
||||
for (const dlqJob of jobs) {
|
||||
try {
|
||||
const { originalJob } = dlqJob.data;
|
||||
|
||||
// Re-add to main queue with delay
|
||||
await this.mainQueue.add(originalJob.name, originalJob.data, {
|
||||
...originalJob.opts,
|
||||
delay: this.config.retryDelay,
|
||||
attempts: this.config.maxRetries,
|
||||
});
|
||||
|
||||
// Remove from DLQ
|
||||
await dlqJob.remove();
|
||||
retriedCount++;
|
||||
|
||||
logger.info('Job retried from DLQ', {
|
||||
originalJobId: originalJob.id,
|
||||
jobName: originalJob.name,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to retry DLQ job', {
|
||||
dlqJobId: dlqJob.id,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return retriedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get DLQ statistics
|
||||
*/
|
||||
async getStats(): Promise<{
|
||||
total: number;
|
||||
recent: number;
|
||||
byJobName: Record<string, number>;
|
||||
oldestJob: Date | null;
|
||||
}> {
|
||||
const [completed, failed, waiting] = await Promise.all([
|
||||
this.dlq.getCompleted(),
|
||||
this.dlq.getFailed(),
|
||||
this.dlq.getWaiting(),
|
||||
]);
|
||||
|
||||
const allJobs = [...completed, ...failed, ...waiting];
|
||||
const byJobName: Record<string, number> = {};
|
||||
let oldestTimestamp: number | null = null;
|
||||
|
||||
for (const job of allJobs) {
|
||||
const jobName = job.data.originalJob?.name || 'unknown';
|
||||
byJobName[jobName] = (byJobName[jobName] || 0) + 1;
|
||||
|
||||
if (!oldestTimestamp || job.timestamp < oldestTimestamp) {
|
||||
oldestTimestamp = job.timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
// Count recent jobs (last 24 hours)
|
||||
const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000;
|
||||
const recent = allJobs.filter(job => job.timestamp > oneDayAgo).length;
|
||||
|
||||
return {
|
||||
total: allJobs.length,
|
||||
recent,
|
||||
byJobName,
|
||||
oldestJob: oldestTimestamp ? new Date(oldestTimestamp) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old DLQ entries
|
||||
*/
|
||||
async cleanup(): Promise<number> {
|
||||
const ageInMs = this.config.cleanupAge * 60 * 60 * 1000;
|
||||
const cutoffTime = Date.now() - ageInMs;
|
||||
|
||||
const jobs = await this.dlq.getCompleted();
|
||||
let removedCount = 0;
|
||||
|
||||
for (const job of jobs) {
|
||||
if (job.timestamp < cutoffTime) {
|
||||
await job.remove();
|
||||
removedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('DLQ cleanup completed', {
|
||||
removedCount,
|
||||
cleanupAge: `${this.config.cleanupAge} hours`,
|
||||
});
|
||||
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if alert threshold is exceeded
|
||||
*/
|
||||
private async checkAlertThreshold(): Promise<void> {
|
||||
const stats = await this.getStats();
|
||||
|
||||
if (stats.total >= this.config.alertThreshold) {
|
||||
logger.error('DLQ alert threshold exceeded', {
|
||||
threshold: this.config.alertThreshold,
|
||||
currentCount: stats.total,
|
||||
byJobName: stats.byJobName,
|
||||
});
|
||||
// In a real implementation, this would trigger alerts
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get failed jobs for inspection
|
||||
*/
|
||||
async inspectFailedJobs(limit = 10): Promise<
|
||||
Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
data: unknown;
|
||||
error: unknown;
|
||||
failedAt: string;
|
||||
attempts: number;
|
||||
}>
|
||||
> {
|
||||
const jobs = await this.dlq.getCompleted(0, limit);
|
||||
|
||||
return jobs.map(job => ({
|
||||
id: job.data.originalJob.id,
|
||||
name: job.data.originalJob.name,
|
||||
data: job.data.originalJob.data,
|
||||
error: job.data.error,
|
||||
failedAt: job.data.movedToDLQAt,
|
||||
attempts: job.data.originalJob.attemptsMade,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown DLQ handler
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
await this.dlq.close();
|
||||
this.failureCount.clear();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,34 +26,33 @@ export type {
|
|||
QueueOptions,
|
||||
QueueStats,
|
||||
GlobalStats,
|
||||
|
||||
|
||||
// Batch processing types
|
||||
BatchResult,
|
||||
ProcessOptions,
|
||||
BatchJobData,
|
||||
|
||||
|
||||
// Handler types
|
||||
JobHandler,
|
||||
TypedJobHandler,
|
||||
HandlerConfig,
|
||||
HandlerConfigWithSchedule,
|
||||
HandlerInitializer,
|
||||
|
||||
|
||||
// Configuration types
|
||||
RedisConfig,
|
||||
QueueConfig,
|
||||
QueueManagerConfig,
|
||||
|
||||
|
||||
// Rate limiting types
|
||||
RateLimitConfig,
|
||||
RateLimitRule,
|
||||
|
||||
|
||||
// DLQ types
|
||||
DLQConfig,
|
||||
DLQJobInfo,
|
||||
|
||||
|
||||
// Scheduled job types
|
||||
ScheduledJob,
|
||||
ScheduleConfig,
|
||||
} from './types';
|
||||
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ export class QueueManager {
|
|||
const queueConfig: QueueWorkerConfig = {
|
||||
workers: mergedOptions.workers,
|
||||
concurrency: mergedOptions.concurrency,
|
||||
startWorker: !!mergedOptions.workers && mergedOptions.workers > 0 && !this.config.delayWorkerStart,
|
||||
startWorker:
|
||||
!!mergedOptions.workers && mergedOptions.workers > 0 && !this.config.delayWorkerStart,
|
||||
};
|
||||
|
||||
const queue = new Queue(
|
||||
|
|
@ -443,7 +444,9 @@ export class QueueManager {
|
|||
*/
|
||||
startAllWorkers(): void {
|
||||
if (!this.config.delayWorkerStart) {
|
||||
logger.info('startAllWorkers() called but workers already started automatically (delayWorkerStart is false)');
|
||||
logger.info(
|
||||
'startAllWorkers() called but workers already started automatically (delayWorkerStart is false)'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -451,17 +454,17 @@ export class QueueManager {
|
|||
for (const queue of this.queues.values()) {
|
||||
const workerCount = this.config.defaultQueueOptions?.workers || 1;
|
||||
const concurrency = this.config.defaultQueueOptions?.concurrency || 1;
|
||||
|
||||
|
||||
if (workerCount > 0) {
|
||||
queue.startWorkersManually(workerCount, concurrency);
|
||||
workersStarted++;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('All workers started', {
|
||||
logger.info('All workers started', {
|
||||
totalQueues: this.queues.size,
|
||||
queuesWithWorkers: workersStarted,
|
||||
delayWorkerStart: this.config.delayWorkerStart
|
||||
delayWorkerStart: this.config.delayWorkerStart,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,314 +1,318 @@
|
|||
import { Queue, QueueEvents } from 'bullmq';
|
||||
// import { getLogger } from '@stock-bot/logger';
|
||||
|
||||
// const logger = getLogger('queue-metrics');
|
||||
|
||||
export interface QueueMetrics {
|
||||
// Job counts
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
paused?: number;
|
||||
|
||||
// Performance metrics
|
||||
processingTime: {
|
||||
avg: number;
|
||||
min: number;
|
||||
max: number;
|
||||
p95: number;
|
||||
p99: number;
|
||||
};
|
||||
|
||||
// Throughput
|
||||
throughput: {
|
||||
completedPerMinute: number;
|
||||
failedPerMinute: number;
|
||||
totalPerMinute: number;
|
||||
};
|
||||
|
||||
// Job age
|
||||
oldestWaitingJob: Date | null;
|
||||
|
||||
// Health
|
||||
isHealthy: boolean;
|
||||
healthIssues: string[];
|
||||
}
|
||||
|
||||
export class QueueMetricsCollector {
|
||||
private processingTimes: number[] = [];
|
||||
private completedTimestamps: number[] = [];
|
||||
private failedTimestamps: number[] = [];
|
||||
private jobStartTimes = new Map<string, number>();
|
||||
private readonly maxSamples = 1000;
|
||||
private readonly metricsInterval = 60000; // 1 minute
|
||||
|
||||
constructor(
|
||||
private queue: Queue,
|
||||
private queueEvents: QueueEvents
|
||||
) {
|
||||
this.setupEventListeners();
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup event listeners for metrics collection
|
||||
*/
|
||||
private setupEventListeners(): void {
|
||||
this.queueEvents.on('completed', () => {
|
||||
// Record completion
|
||||
this.completedTimestamps.push(Date.now());
|
||||
this.cleanupOldTimestamps();
|
||||
});
|
||||
|
||||
this.queueEvents.on('failed', () => {
|
||||
// Record failure
|
||||
this.failedTimestamps.push(Date.now());
|
||||
this.cleanupOldTimestamps();
|
||||
});
|
||||
|
||||
// Track processing times
|
||||
this.queueEvents.on('active', ({ jobId }) => {
|
||||
this.jobStartTimes.set(jobId, Date.now());
|
||||
});
|
||||
|
||||
this.queueEvents.on('completed', ({ jobId }) => {
|
||||
const startTime = this.jobStartTimes.get(jobId);
|
||||
if (startTime) {
|
||||
const processingTime = Date.now() - startTime;
|
||||
this.recordProcessingTime(processingTime);
|
||||
this.jobStartTimes.delete(jobId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record processing time
|
||||
*/
|
||||
private recordProcessingTime(time: number): void {
|
||||
this.processingTimes.push(time);
|
||||
|
||||
// Keep only recent samples
|
||||
if (this.processingTimes.length > this.maxSamples) {
|
||||
this.processingTimes = this.processingTimes.slice(-this.maxSamples);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old timestamps
|
||||
*/
|
||||
private cleanupOldTimestamps(): void {
|
||||
const cutoff = Date.now() - this.metricsInterval;
|
||||
|
||||
this.completedTimestamps = this.completedTimestamps.filter(ts => ts > cutoff);
|
||||
this.failedTimestamps = this.failedTimestamps.filter(ts => ts > cutoff);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect current metrics
|
||||
*/
|
||||
async collect(): Promise<QueueMetrics> {
|
||||
// Get job counts
|
||||
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
||||
this.queue.getWaitingCount(),
|
||||
this.queue.getActiveCount(),
|
||||
this.queue.getCompletedCount(),
|
||||
this.queue.getFailedCount(),
|
||||
this.queue.getDelayedCount(),
|
||||
]);
|
||||
|
||||
// BullMQ doesn't have getPausedCount, check if queue is paused
|
||||
const paused = await this.queue.isPaused() ? waiting : 0;
|
||||
|
||||
// Calculate processing time metrics
|
||||
const processingTime = this.calculateProcessingTimeMetrics();
|
||||
|
||||
// Calculate throughput
|
||||
const throughput = this.calculateThroughput();
|
||||
|
||||
// Get oldest waiting job
|
||||
const oldestWaitingJob = await this.getOldestWaitingJob();
|
||||
|
||||
// Check health
|
||||
const { isHealthy, healthIssues } = this.checkHealth({
|
||||
waiting,
|
||||
active,
|
||||
failed,
|
||||
processingTime,
|
||||
});
|
||||
|
||||
return {
|
||||
waiting,
|
||||
active,
|
||||
completed,
|
||||
failed,
|
||||
delayed,
|
||||
paused,
|
||||
processingTime,
|
||||
throughput,
|
||||
oldestWaitingJob,
|
||||
isHealthy,
|
||||
healthIssues,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate processing time metrics
|
||||
*/
|
||||
private calculateProcessingTimeMetrics(): QueueMetrics['processingTime'] {
|
||||
if (this.processingTimes.length === 0) {
|
||||
return { avg: 0, min: 0, max: 0, p95: 0, p99: 0 };
|
||||
}
|
||||
|
||||
const sorted = [...this.processingTimes].sort((a, b) => a - b);
|
||||
const sum = sorted.reduce((acc, val) => acc + val, 0);
|
||||
|
||||
return {
|
||||
avg: sorted.length > 0 ? Math.round(sum / sorted.length) : 0,
|
||||
min: sorted[0] || 0,
|
||||
max: sorted[sorted.length - 1] || 0,
|
||||
p95: sorted[Math.floor(sorted.length * 0.95)] || 0,
|
||||
p99: sorted[Math.floor(sorted.length * 0.99)] || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate throughput metrics
|
||||
*/
|
||||
private calculateThroughput(): QueueMetrics['throughput'] {
|
||||
const now = Date.now();
|
||||
const oneMinuteAgo = now - 60000;
|
||||
|
||||
const completedPerMinute = this.completedTimestamps.filter(ts => ts > oneMinuteAgo).length;
|
||||
const failedPerMinute = this.failedTimestamps.filter(ts => ts > oneMinuteAgo).length;
|
||||
|
||||
return {
|
||||
completedPerMinute,
|
||||
failedPerMinute,
|
||||
totalPerMinute: completedPerMinute + failedPerMinute,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get oldest waiting job
|
||||
*/
|
||||
private async getOldestWaitingJob(): Promise<Date | null> {
|
||||
const waitingJobs = await this.queue.getWaiting(0, 1);
|
||||
|
||||
if (waitingJobs.length > 0) {
|
||||
return new Date(waitingJobs[0].timestamp);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check queue health
|
||||
*/
|
||||
private checkHealth(metrics: {
|
||||
waiting: number;
|
||||
active: number;
|
||||
failed: number;
|
||||
processingTime: QueueMetrics['processingTime'];
|
||||
}): { isHealthy: boolean; healthIssues: string[] } {
|
||||
const issues: string[] = [];
|
||||
|
||||
// Check for high failure rate
|
||||
const failureRate = metrics.failed / (metrics.failed + this.completedTimestamps.length);
|
||||
if (failureRate > 0.1) {
|
||||
issues.push(`High failure rate: ${(failureRate * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// Check for queue backlog
|
||||
if (metrics.waiting > 1000) {
|
||||
issues.push(`Large queue backlog: ${metrics.waiting} jobs waiting`);
|
||||
}
|
||||
|
||||
// Check for slow processing
|
||||
if (metrics.processingTime.avg > 30000) { // 30 seconds
|
||||
issues.push(`Slow average processing time: ${(metrics.processingTime.avg / 1000).toFixed(1)}s`);
|
||||
}
|
||||
|
||||
// Check for stalled active jobs
|
||||
if (metrics.active > 100) {
|
||||
issues.push(`High number of active jobs: ${metrics.active}`);
|
||||
}
|
||||
|
||||
return {
|
||||
isHealthy: issues.length === 0,
|
||||
healthIssues: issues,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get formatted metrics report
|
||||
*/
|
||||
async getReport(): Promise<string> {
|
||||
const metrics = await this.collect();
|
||||
|
||||
return `
|
||||
Queue Metrics Report
|
||||
===================
|
||||
Status: ${metrics.isHealthy ? '✅ Healthy' : '⚠️ Issues Detected'}
|
||||
|
||||
Job Counts:
|
||||
- Waiting: ${metrics.waiting}
|
||||
- Active: ${metrics.active}
|
||||
- Completed: ${metrics.completed}
|
||||
- Failed: ${metrics.failed}
|
||||
- Delayed: ${metrics.delayed}
|
||||
- Paused: ${metrics.paused}
|
||||
|
||||
Performance:
|
||||
- Avg Processing Time: ${(metrics.processingTime.avg / 1000).toFixed(2)}s
|
||||
- Min/Max: ${(metrics.processingTime.min / 1000).toFixed(2)}s / ${(metrics.processingTime.max / 1000).toFixed(2)}s
|
||||
- P95/P99: ${(metrics.processingTime.p95 / 1000).toFixed(2)}s / ${(metrics.processingTime.p99 / 1000).toFixed(2)}s
|
||||
|
||||
Throughput:
|
||||
- Completed/min: ${metrics.throughput.completedPerMinute}
|
||||
- Failed/min: ${metrics.throughput.failedPerMinute}
|
||||
- Total/min: ${metrics.throughput.totalPerMinute}
|
||||
|
||||
${metrics.oldestWaitingJob ? `Oldest Waiting Job: ${metrics.oldestWaitingJob.toISOString()}` : 'No waiting jobs'}
|
||||
|
||||
${metrics.healthIssues.length > 0 ? `\nHealth Issues:\n${metrics.healthIssues.map(issue => `- ${issue}`).join('\n')}` : ''}
|
||||
`.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Export metrics in Prometheus format
|
||||
*/
|
||||
async getPrometheusMetrics(): Promise<string> {
|
||||
const metrics = await this.collect();
|
||||
const queueName = this.queue.name;
|
||||
|
||||
return `
|
||||
# HELP queue_jobs_total Total number of jobs by status
|
||||
# TYPE queue_jobs_total gauge
|
||||
queue_jobs_total{queue="${queueName}",status="waiting"} ${metrics.waiting}
|
||||
queue_jobs_total{queue="${queueName}",status="active"} ${metrics.active}
|
||||
queue_jobs_total{queue="${queueName}",status="completed"} ${metrics.completed}
|
||||
queue_jobs_total{queue="${queueName}",status="failed"} ${metrics.failed}
|
||||
queue_jobs_total{queue="${queueName}",status="delayed"} ${metrics.delayed}
|
||||
queue_jobs_total{queue="${queueName}",status="paused"} ${metrics.paused}
|
||||
|
||||
# HELP queue_processing_time_seconds Job processing time in seconds
|
||||
# TYPE queue_processing_time_seconds summary
|
||||
queue_processing_time_seconds{queue="${queueName}",quantile="0.5"} ${(metrics.processingTime.avg / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds{queue="${queueName}",quantile="0.95"} ${(metrics.processingTime.p95 / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds{queue="${queueName}",quantile="0.99"} ${(metrics.processingTime.p99 / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds_sum{queue="${queueName}"} ${(metrics.processingTime.avg * this.processingTimes.length / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds_count{queue="${queueName}"} ${this.processingTimes.length}
|
||||
|
||||
# HELP queue_throughput_per_minute Jobs processed per minute
|
||||
# TYPE queue_throughput_per_minute gauge
|
||||
queue_throughput_per_minute{queue="${queueName}",status="completed"} ${metrics.throughput.completedPerMinute}
|
||||
queue_throughput_per_minute{queue="${queueName}",status="failed"} ${metrics.throughput.failedPerMinute}
|
||||
queue_throughput_per_minute{queue="${queueName}",status="total"} ${metrics.throughput.totalPerMinute}
|
||||
|
||||
# HELP queue_health Queue health status
|
||||
# TYPE queue_health gauge
|
||||
queue_health{queue="${queueName}"} ${metrics.isHealthy ? 1 : 0}
|
||||
`.trim();
|
||||
}
|
||||
}
|
||||
import { Queue, QueueEvents } from 'bullmq';
|
||||
|
||||
// import { getLogger } from '@stock-bot/logger';
|
||||
|
||||
// const logger = getLogger('queue-metrics');
|
||||
|
||||
export interface QueueMetrics {
|
||||
// Job counts
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
paused?: number;
|
||||
|
||||
// Performance metrics
|
||||
processingTime: {
|
||||
avg: number;
|
||||
min: number;
|
||||
max: number;
|
||||
p95: number;
|
||||
p99: number;
|
||||
};
|
||||
|
||||
// Throughput
|
||||
throughput: {
|
||||
completedPerMinute: number;
|
||||
failedPerMinute: number;
|
||||
totalPerMinute: number;
|
||||
};
|
||||
|
||||
// Job age
|
||||
oldestWaitingJob: Date | null;
|
||||
|
||||
// Health
|
||||
isHealthy: boolean;
|
||||
healthIssues: string[];
|
||||
}
|
||||
|
||||
export class QueueMetricsCollector {
|
||||
private processingTimes: number[] = [];
|
||||
private completedTimestamps: number[] = [];
|
||||
private failedTimestamps: number[] = [];
|
||||
private jobStartTimes = new Map<string, number>();
|
||||
private readonly maxSamples = 1000;
|
||||
private readonly metricsInterval = 60000; // 1 minute
|
||||
|
||||
constructor(
|
||||
private queue: Queue,
|
||||
private queueEvents: QueueEvents
|
||||
) {
|
||||
this.setupEventListeners();
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup event listeners for metrics collection
|
||||
*/
|
||||
private setupEventListeners(): void {
|
||||
this.queueEvents.on('completed', () => {
|
||||
// Record completion
|
||||
this.completedTimestamps.push(Date.now());
|
||||
this.cleanupOldTimestamps();
|
||||
});
|
||||
|
||||
this.queueEvents.on('failed', () => {
|
||||
// Record failure
|
||||
this.failedTimestamps.push(Date.now());
|
||||
this.cleanupOldTimestamps();
|
||||
});
|
||||
|
||||
// Track processing times
|
||||
this.queueEvents.on('active', ({ jobId }) => {
|
||||
this.jobStartTimes.set(jobId, Date.now());
|
||||
});
|
||||
|
||||
this.queueEvents.on('completed', ({ jobId }) => {
|
||||
const startTime = this.jobStartTimes.get(jobId);
|
||||
if (startTime) {
|
||||
const processingTime = Date.now() - startTime;
|
||||
this.recordProcessingTime(processingTime);
|
||||
this.jobStartTimes.delete(jobId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record processing time
|
||||
*/
|
||||
private recordProcessingTime(time: number): void {
|
||||
this.processingTimes.push(time);
|
||||
|
||||
// Keep only recent samples
|
||||
if (this.processingTimes.length > this.maxSamples) {
|
||||
this.processingTimes = this.processingTimes.slice(-this.maxSamples);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old timestamps
|
||||
*/
|
||||
private cleanupOldTimestamps(): void {
|
||||
const cutoff = Date.now() - this.metricsInterval;
|
||||
|
||||
this.completedTimestamps = this.completedTimestamps.filter(ts => ts > cutoff);
|
||||
this.failedTimestamps = this.failedTimestamps.filter(ts => ts > cutoff);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect current metrics
|
||||
*/
|
||||
async collect(): Promise<QueueMetrics> {
|
||||
// Get job counts
|
||||
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
||||
this.queue.getWaitingCount(),
|
||||
this.queue.getActiveCount(),
|
||||
this.queue.getCompletedCount(),
|
||||
this.queue.getFailedCount(),
|
||||
this.queue.getDelayedCount(),
|
||||
]);
|
||||
|
||||
// BullMQ doesn't have getPausedCount, check if queue is paused
|
||||
const paused = (await this.queue.isPaused()) ? waiting : 0;
|
||||
|
||||
// Calculate processing time metrics
|
||||
const processingTime = this.calculateProcessingTimeMetrics();
|
||||
|
||||
// Calculate throughput
|
||||
const throughput = this.calculateThroughput();
|
||||
|
||||
// Get oldest waiting job
|
||||
const oldestWaitingJob = await this.getOldestWaitingJob();
|
||||
|
||||
// Check health
|
||||
const { isHealthy, healthIssues } = this.checkHealth({
|
||||
waiting,
|
||||
active,
|
||||
failed,
|
||||
processingTime,
|
||||
});
|
||||
|
||||
return {
|
||||
waiting,
|
||||
active,
|
||||
completed,
|
||||
failed,
|
||||
delayed,
|
||||
paused,
|
||||
processingTime,
|
||||
throughput,
|
||||
oldestWaitingJob,
|
||||
isHealthy,
|
||||
healthIssues,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate processing time metrics
|
||||
*/
|
||||
private calculateProcessingTimeMetrics(): QueueMetrics['processingTime'] {
|
||||
if (this.processingTimes.length === 0) {
|
||||
return { avg: 0, min: 0, max: 0, p95: 0, p99: 0 };
|
||||
}
|
||||
|
||||
const sorted = [...this.processingTimes].sort((a, b) => a - b);
|
||||
const sum = sorted.reduce((acc, val) => acc + val, 0);
|
||||
|
||||
return {
|
||||
avg: sorted.length > 0 ? Math.round(sum / sorted.length) : 0,
|
||||
min: sorted[0] || 0,
|
||||
max: sorted[sorted.length - 1] || 0,
|
||||
p95: sorted[Math.floor(sorted.length * 0.95)] || 0,
|
||||
p99: sorted[Math.floor(sorted.length * 0.99)] || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate throughput metrics
|
||||
*/
|
||||
private calculateThroughput(): QueueMetrics['throughput'] {
|
||||
const now = Date.now();
|
||||
const oneMinuteAgo = now - 60000;
|
||||
|
||||
const completedPerMinute = this.completedTimestamps.filter(ts => ts > oneMinuteAgo).length;
|
||||
const failedPerMinute = this.failedTimestamps.filter(ts => ts > oneMinuteAgo).length;
|
||||
|
||||
return {
|
||||
completedPerMinute,
|
||||
failedPerMinute,
|
||||
totalPerMinute: completedPerMinute + failedPerMinute,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get oldest waiting job
|
||||
*/
|
||||
private async getOldestWaitingJob(): Promise<Date | null> {
|
||||
const waitingJobs = await this.queue.getWaiting(0, 1);
|
||||
|
||||
if (waitingJobs.length > 0) {
|
||||
return new Date(waitingJobs[0].timestamp);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check queue health
|
||||
*/
|
||||
private checkHealth(metrics: {
|
||||
waiting: number;
|
||||
active: number;
|
||||
failed: number;
|
||||
processingTime: QueueMetrics['processingTime'];
|
||||
}): { isHealthy: boolean; healthIssues: string[] } {
|
||||
const issues: string[] = [];
|
||||
|
||||
// Check for high failure rate
|
||||
const failureRate = metrics.failed / (metrics.failed + this.completedTimestamps.length);
|
||||
if (failureRate > 0.1) {
|
||||
issues.push(`High failure rate: ${(failureRate * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// Check for queue backlog
|
||||
if (metrics.waiting > 1000) {
|
||||
issues.push(`Large queue backlog: ${metrics.waiting} jobs waiting`);
|
||||
}
|
||||
|
||||
// Check for slow processing
|
||||
if (metrics.processingTime.avg > 30000) {
|
||||
// 30 seconds
|
||||
issues.push(
|
||||
`Slow average processing time: ${(metrics.processingTime.avg / 1000).toFixed(1)}s`
|
||||
);
|
||||
}
|
||||
|
||||
// Check for stalled active jobs
|
||||
if (metrics.active > 100) {
|
||||
issues.push(`High number of active jobs: ${metrics.active}`);
|
||||
}
|
||||
|
||||
return {
|
||||
isHealthy: issues.length === 0,
|
||||
healthIssues: issues,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get formatted metrics report
|
||||
*/
|
||||
async getReport(): Promise<string> {
|
||||
const metrics = await this.collect();
|
||||
|
||||
return `
|
||||
Queue Metrics Report
|
||||
===================
|
||||
Status: ${metrics.isHealthy ? '✅ Healthy' : '⚠️ Issues Detected'}
|
||||
|
||||
Job Counts:
|
||||
- Waiting: ${metrics.waiting}
|
||||
- Active: ${metrics.active}
|
||||
- Completed: ${metrics.completed}
|
||||
- Failed: ${metrics.failed}
|
||||
- Delayed: ${metrics.delayed}
|
||||
- Paused: ${metrics.paused}
|
||||
|
||||
Performance:
|
||||
- Avg Processing Time: ${(metrics.processingTime.avg / 1000).toFixed(2)}s
|
||||
- Min/Max: ${(metrics.processingTime.min / 1000).toFixed(2)}s / ${(metrics.processingTime.max / 1000).toFixed(2)}s
|
||||
- P95/P99: ${(metrics.processingTime.p95 / 1000).toFixed(2)}s / ${(metrics.processingTime.p99 / 1000).toFixed(2)}s
|
||||
|
||||
Throughput:
|
||||
- Completed/min: ${metrics.throughput.completedPerMinute}
|
||||
- Failed/min: ${metrics.throughput.failedPerMinute}
|
||||
- Total/min: ${metrics.throughput.totalPerMinute}
|
||||
|
||||
${metrics.oldestWaitingJob ? `Oldest Waiting Job: ${metrics.oldestWaitingJob.toISOString()}` : 'No waiting jobs'}
|
||||
|
||||
${metrics.healthIssues.length > 0 ? `\nHealth Issues:\n${metrics.healthIssues.map(issue => `- ${issue}`).join('\n')}` : ''}
|
||||
`.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Export metrics in Prometheus format
|
||||
*/
|
||||
async getPrometheusMetrics(): Promise<string> {
|
||||
const metrics = await this.collect();
|
||||
const queueName = this.queue.name;
|
||||
|
||||
return `
|
||||
# HELP queue_jobs_total Total number of jobs by status
|
||||
# TYPE queue_jobs_total gauge
|
||||
queue_jobs_total{queue="${queueName}",status="waiting"} ${metrics.waiting}
|
||||
queue_jobs_total{queue="${queueName}",status="active"} ${metrics.active}
|
||||
queue_jobs_total{queue="${queueName}",status="completed"} ${metrics.completed}
|
||||
queue_jobs_total{queue="${queueName}",status="failed"} ${metrics.failed}
|
||||
queue_jobs_total{queue="${queueName}",status="delayed"} ${metrics.delayed}
|
||||
queue_jobs_total{queue="${queueName}",status="paused"} ${metrics.paused}
|
||||
|
||||
# HELP queue_processing_time_seconds Job processing time in seconds
|
||||
# TYPE queue_processing_time_seconds summary
|
||||
queue_processing_time_seconds{queue="${queueName}",quantile="0.5"} ${(metrics.processingTime.avg / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds{queue="${queueName}",quantile="0.95"} ${(metrics.processingTime.p95 / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds{queue="${queueName}",quantile="0.99"} ${(metrics.processingTime.p99 / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds_sum{queue="${queueName}"} ${((metrics.processingTime.avg * this.processingTimes.length) / 1000).toFixed(3)}
|
||||
queue_processing_time_seconds_count{queue="${queueName}"} ${this.processingTimes.length}
|
||||
|
||||
# HELP queue_throughput_per_minute Jobs processed per minute
|
||||
# TYPE queue_throughput_per_minute gauge
|
||||
queue_throughput_per_minute{queue="${queueName}",status="completed"} ${metrics.throughput.completedPerMinute}
|
||||
queue_throughput_per_minute{queue="${queueName}",status="failed"} ${metrics.throughput.failedPerMinute}
|
||||
queue_throughput_per_minute{queue="${queueName}",status="total"} ${metrics.throughput.totalPerMinute}
|
||||
|
||||
# HELP queue_health Queue health status
|
||||
# TYPE queue_health gauge
|
||||
queue_health{queue="${queueName}"} ${metrics.isHealthy ? 1 : 0}
|
||||
`.trim();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,372 +1,372 @@
|
|||
import { Queue as BullQueue, QueueEvents, Worker, type Job } from 'bullmq';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { handlerRegistry } from '@stock-bot/types';
|
||||
import type { JobData, JobOptions, QueueStats, RedisConfig } from './types';
|
||||
import { getRedisConnection } from './utils';
|
||||
|
||||
const logger = getLogger('queue');
|
||||
|
||||
export interface QueueWorkerConfig {
|
||||
workers?: number;
|
||||
concurrency?: number;
|
||||
startWorker?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consolidated Queue class that handles both job operations and optional worker management
|
||||
* Can be used as a simple job queue or with workers for automatic processing
|
||||
*/
|
||||
export class Queue {
|
||||
private bullQueue: BullQueue;
|
||||
private workers: Worker[] = [];
|
||||
private queueEvents?: QueueEvents;
|
||||
private queueName: string;
|
||||
private redisConfig: RedisConfig;
|
||||
|
||||
constructor(
|
||||
queueName: string,
|
||||
redisConfig: RedisConfig,
|
||||
defaultJobOptions: JobOptions = {},
|
||||
config: QueueWorkerConfig = {}
|
||||
) {
|
||||
this.queueName = queueName;
|
||||
this.redisConfig = redisConfig;
|
||||
|
||||
const connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Initialize BullMQ queue
|
||||
this.bullQueue = new BullQueue(`{${queueName}}`, {
|
||||
connection,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 10,
|
||||
removeOnFail: 5,
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000,
|
||||
},
|
||||
...defaultJobOptions,
|
||||
},
|
||||
});
|
||||
|
||||
// Initialize queue events if workers will be used
|
||||
if (config.workers && config.workers > 0) {
|
||||
this.queueEvents = new QueueEvents(`{${queueName}}`, { connection });
|
||||
}
|
||||
|
||||
// Start workers if requested and not explicitly disabled
|
||||
if (config.workers && config.workers > 0 && config.startWorker !== false) {
|
||||
this.startWorkers(config.workers, config.concurrency || 1);
|
||||
}
|
||||
|
||||
logger.trace('Queue created', {
|
||||
queueName,
|
||||
workers: config.workers || 0,
|
||||
concurrency: config.concurrency || 1,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the queue name
|
||||
*/
|
||||
getName(): string {
|
||||
return this.queueName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a single job to the queue
|
||||
*/
|
||||
async add(name: string, data: JobData, options: JobOptions = {}): Promise<Job> {
|
||||
logger.trace('Adding job', { queueName: this.queueName, jobName: name });
|
||||
return await this.bullQueue.add(name, data, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add multiple jobs to the queue in bulk
|
||||
*/
|
||||
async addBulk(jobs: Array<{ name: string; data: JobData; opts?: JobOptions }>): Promise<Job[]> {
|
||||
logger.trace('Adding bulk jobs', {
|
||||
queueName: this.queueName,
|
||||
jobCount: jobs.length,
|
||||
});
|
||||
return await this.bullQueue.addBulk(jobs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a scheduled job with cron-like pattern
|
||||
*/
|
||||
async addScheduledJob(
|
||||
name: string,
|
||||
data: JobData,
|
||||
cronPattern: string,
|
||||
options: JobOptions = {}
|
||||
): Promise<Job> {
|
||||
const scheduledOptions: JobOptions = {
|
||||
...options,
|
||||
repeat: {
|
||||
pattern: cronPattern,
|
||||
// Use job name as repeat key to prevent duplicates
|
||||
key: `${this.queueName}:${name}`,
|
||||
...options.repeat,
|
||||
},
|
||||
};
|
||||
|
||||
logger.info('Adding scheduled job', {
|
||||
queueName: this.queueName,
|
||||
jobName: name,
|
||||
cronPattern,
|
||||
repeatKey: scheduledOptions.repeat?.key,
|
||||
immediately: scheduledOptions.repeat?.immediately,
|
||||
});
|
||||
|
||||
return await this.bullQueue.add(name, data, scheduledOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics
|
||||
*/
|
||||
async getStats(): Promise<QueueStats> {
|
||||
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
||||
this.bullQueue.getWaiting(),
|
||||
this.bullQueue.getActive(),
|
||||
this.bullQueue.getCompleted(),
|
||||
this.bullQueue.getFailed(),
|
||||
this.bullQueue.getDelayed(),
|
||||
]);
|
||||
|
||||
const isPaused = await this.bullQueue.isPaused();
|
||||
|
||||
return {
|
||||
waiting: waiting.length,
|
||||
active: active.length,
|
||||
completed: completed.length,
|
||||
failed: failed.length,
|
||||
delayed: delayed.length,
|
||||
paused: isPaused,
|
||||
workers: this.workers.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific job by ID
|
||||
*/
|
||||
async getJob(jobId: string): Promise<Job | undefined> {
|
||||
return await this.bullQueue.getJob(jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get jobs by state
|
||||
*/
|
||||
async getJobs(
|
||||
states: Array<'waiting' | 'active' | 'completed' | 'failed' | 'delayed'>,
|
||||
start = 0,
|
||||
end = 100
|
||||
): Promise<Job[]> {
|
||||
return await this.bullQueue.getJobs(states, start, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause the queue (stops processing new jobs)
|
||||
*/
|
||||
async pause(): Promise<void> {
|
||||
await this.bullQueue.pause();
|
||||
logger.info('Queue paused', { queueName: this.queueName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume the queue
|
||||
*/
|
||||
async resume(): Promise<void> {
|
||||
await this.bullQueue.resume();
|
||||
logger.info('Queue resumed', { queueName: this.queueName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain the queue (remove all jobs)
|
||||
*/
|
||||
async drain(delayed = false): Promise<void> {
|
||||
await this.bullQueue.drain(delayed);
|
||||
logger.info('Queue drained', { queueName: this.queueName, delayed });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean completed and failed jobs
|
||||
*/
|
||||
async clean(
|
||||
grace: number = 0,
|
||||
limit: number = 100,
|
||||
type: 'completed' | 'failed' = 'completed'
|
||||
): Promise<void> {
|
||||
await this.bullQueue.clean(grace, limit, type);
|
||||
logger.debug('Queue cleaned', { queueName: this.queueName, type, grace, limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until the queue is ready
|
||||
*/
|
||||
async waitUntilReady(): Promise<void> {
|
||||
await this.bullQueue.waitUntilReady();
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the queue (cleanup resources)
|
||||
*/
|
||||
/**
|
||||
* Close the queue (cleanup resources)
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
// Close the queue itself
|
||||
await this.bullQueue.close();
|
||||
logger.info('Queue closed', { queueName: this.queueName });
|
||||
|
||||
// Close queue events
|
||||
if (this.queueEvents) {
|
||||
await this.queueEvents.close();
|
||||
logger.debug('Queue events closed', { queueName: this.queueName });
|
||||
}
|
||||
|
||||
// Close workers first
|
||||
if (this.workers.length > 0) {
|
||||
await Promise.all(
|
||||
this.workers.map(async worker => {
|
||||
return await worker.close();
|
||||
})
|
||||
);
|
||||
this.workers = [];
|
||||
logger.debug('Workers closed', { queueName: this.queueName });
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error closing queue', { queueName: this.queueName, error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start workers for this queue
|
||||
*/
|
||||
private startWorkers(workerCount: number, concurrency: number): void {
|
||||
const connection = getRedisConnection(this.redisConfig);
|
||||
|
||||
for (let i = 0; i < workerCount; i++) {
|
||||
const worker = new Worker(`{${this.queueName}}`, this.processJob.bind(this), {
|
||||
connection,
|
||||
concurrency,
|
||||
maxStalledCount: 3,
|
||||
stalledInterval: 30000,
|
||||
});
|
||||
|
||||
// Setup worker event handlers
|
||||
worker.on('completed', job => {
|
||||
logger.trace('Job completed', {
|
||||
queueName: this.queueName,
|
||||
jobId: job.id,
|
||||
handler: job.data?.handler,
|
||||
operation: job.data?.operation,
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
logger.error('Job failed', {
|
||||
queueName: this.queueName,
|
||||
jobId: job?.id,
|
||||
handler: job?.data?.handler,
|
||||
operation: job?.data?.operation,
|
||||
error: err.message,
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('error', error => {
|
||||
logger.error('Worker error', {
|
||||
queueName: this.queueName,
|
||||
workerId: i,
|
||||
error: error.message,
|
||||
});
|
||||
});
|
||||
|
||||
this.workers.push(worker);
|
||||
}
|
||||
|
||||
logger.info('Workers started', {
|
||||
queueName: this.queueName,
|
||||
workerCount,
|
||||
concurrency,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a job using the handler registry
|
||||
*/
|
||||
private async processJob(job: Job): Promise<unknown> {
|
||||
const { handler, operation, payload }: JobData = job.data;
|
||||
|
||||
logger.trace('Processing job', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
queueName: this.queueName,
|
||||
});
|
||||
|
||||
try {
|
||||
// Look up handler in registry
|
||||
const jobHandler = handlerRegistry.getOperation(handler, operation);
|
||||
|
||||
if (!jobHandler) {
|
||||
throw new Error(`No handler found for ${handler}:${operation}`);
|
||||
}
|
||||
|
||||
const result = await jobHandler(payload);
|
||||
|
||||
logger.trace('Job completed successfully', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
queueName: this.queueName,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error('Job processing failed', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
queueName: this.queueName,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start workers manually (for delayed initialization)
|
||||
*/
|
||||
startWorkersManually(workerCount: number, concurrency: number = 1): void {
|
||||
if (this.workers.length > 0) {
|
||||
logger.warn('Workers already started for queue', { queueName: this.queueName });
|
||||
return;
|
||||
}
|
||||
|
||||
// Initialize queue events if not already done
|
||||
if (!this.queueEvents) {
|
||||
const connection = getRedisConnection(this.redisConfig);
|
||||
this.queueEvents = new QueueEvents(`{${this.queueName}}`, { connection });
|
||||
}
|
||||
|
||||
this.startWorkers(workerCount, concurrency);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of active workers
|
||||
*/
|
||||
getWorkerCount(): number {
|
||||
return this.workers.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the underlying BullMQ queue (for advanced operations)
|
||||
* @deprecated Use direct methods instead
|
||||
*/
|
||||
getBullQueue(): BullQueue {
|
||||
return this.bullQueue;
|
||||
}
|
||||
}
|
||||
import { Queue as BullQueue, QueueEvents, Worker, type Job } from 'bullmq';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { handlerRegistry } from '@stock-bot/types';
|
||||
import type { JobData, JobOptions, QueueStats, RedisConfig } from './types';
|
||||
import { getRedisConnection } from './utils';
|
||||
|
||||
const logger = getLogger('queue');
|
||||
|
||||
export interface QueueWorkerConfig {
|
||||
workers?: number;
|
||||
concurrency?: number;
|
||||
startWorker?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consolidated Queue class that handles both job operations and optional worker management
|
||||
* Can be used as a simple job queue or with workers for automatic processing
|
||||
*/
|
||||
export class Queue {
|
||||
private bullQueue: BullQueue;
|
||||
private workers: Worker[] = [];
|
||||
private queueEvents?: QueueEvents;
|
||||
private queueName: string;
|
||||
private redisConfig: RedisConfig;
|
||||
|
||||
constructor(
|
||||
queueName: string,
|
||||
redisConfig: RedisConfig,
|
||||
defaultJobOptions: JobOptions = {},
|
||||
config: QueueWorkerConfig = {}
|
||||
) {
|
||||
this.queueName = queueName;
|
||||
this.redisConfig = redisConfig;
|
||||
|
||||
const connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Initialize BullMQ queue
|
||||
this.bullQueue = new BullQueue(`{${queueName}}`, {
|
||||
connection,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 10,
|
||||
removeOnFail: 5,
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000,
|
||||
},
|
||||
...defaultJobOptions,
|
||||
},
|
||||
});
|
||||
|
||||
// Initialize queue events if workers will be used
|
||||
if (config.workers && config.workers > 0) {
|
||||
this.queueEvents = new QueueEvents(`{${queueName}}`, { connection });
|
||||
}
|
||||
|
||||
// Start workers if requested and not explicitly disabled
|
||||
if (config.workers && config.workers > 0 && config.startWorker !== false) {
|
||||
this.startWorkers(config.workers, config.concurrency || 1);
|
||||
}
|
||||
|
||||
logger.trace('Queue created', {
|
||||
queueName,
|
||||
workers: config.workers || 0,
|
||||
concurrency: config.concurrency || 1,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the queue name
|
||||
*/
|
||||
getName(): string {
|
||||
return this.queueName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a single job to the queue
|
||||
*/
|
||||
async add(name: string, data: JobData, options: JobOptions = {}): Promise<Job> {
|
||||
logger.trace('Adding job', { queueName: this.queueName, jobName: name });
|
||||
return await this.bullQueue.add(name, data, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add multiple jobs to the queue in bulk
|
||||
*/
|
||||
async addBulk(jobs: Array<{ name: string; data: JobData; opts?: JobOptions }>): Promise<Job[]> {
|
||||
logger.trace('Adding bulk jobs', {
|
||||
queueName: this.queueName,
|
||||
jobCount: jobs.length,
|
||||
});
|
||||
return await this.bullQueue.addBulk(jobs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a scheduled job with cron-like pattern
|
||||
*/
|
||||
async addScheduledJob(
|
||||
name: string,
|
||||
data: JobData,
|
||||
cronPattern: string,
|
||||
options: JobOptions = {}
|
||||
): Promise<Job> {
|
||||
const scheduledOptions: JobOptions = {
|
||||
...options,
|
||||
repeat: {
|
||||
pattern: cronPattern,
|
||||
// Use job name as repeat key to prevent duplicates
|
||||
key: `${this.queueName}:${name}`,
|
||||
...options.repeat,
|
||||
},
|
||||
};
|
||||
|
||||
logger.info('Adding scheduled job', {
|
||||
queueName: this.queueName,
|
||||
jobName: name,
|
||||
cronPattern,
|
||||
repeatKey: scheduledOptions.repeat?.key,
|
||||
immediately: scheduledOptions.repeat?.immediately,
|
||||
});
|
||||
|
||||
return await this.bullQueue.add(name, data, scheduledOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics
|
||||
*/
|
||||
async getStats(): Promise<QueueStats> {
|
||||
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
||||
this.bullQueue.getWaiting(),
|
||||
this.bullQueue.getActive(),
|
||||
this.bullQueue.getCompleted(),
|
||||
this.bullQueue.getFailed(),
|
||||
this.bullQueue.getDelayed(),
|
||||
]);
|
||||
|
||||
const isPaused = await this.bullQueue.isPaused();
|
||||
|
||||
return {
|
||||
waiting: waiting.length,
|
||||
active: active.length,
|
||||
completed: completed.length,
|
||||
failed: failed.length,
|
||||
delayed: delayed.length,
|
||||
paused: isPaused,
|
||||
workers: this.workers.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific job by ID
|
||||
*/
|
||||
async getJob(jobId: string): Promise<Job | undefined> {
|
||||
return await this.bullQueue.getJob(jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get jobs by state
|
||||
*/
|
||||
async getJobs(
|
||||
states: Array<'waiting' | 'active' | 'completed' | 'failed' | 'delayed'>,
|
||||
start = 0,
|
||||
end = 100
|
||||
): Promise<Job[]> {
|
||||
return await this.bullQueue.getJobs(states, start, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause the queue (stops processing new jobs)
|
||||
*/
|
||||
async pause(): Promise<void> {
|
||||
await this.bullQueue.pause();
|
||||
logger.info('Queue paused', { queueName: this.queueName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume the queue
|
||||
*/
|
||||
async resume(): Promise<void> {
|
||||
await this.bullQueue.resume();
|
||||
logger.info('Queue resumed', { queueName: this.queueName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain the queue (remove all jobs)
|
||||
*/
|
||||
async drain(delayed = false): Promise<void> {
|
||||
await this.bullQueue.drain(delayed);
|
||||
logger.info('Queue drained', { queueName: this.queueName, delayed });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean completed and failed jobs
|
||||
*/
|
||||
async clean(
|
||||
grace: number = 0,
|
||||
limit: number = 100,
|
||||
type: 'completed' | 'failed' = 'completed'
|
||||
): Promise<void> {
|
||||
await this.bullQueue.clean(grace, limit, type);
|
||||
logger.debug('Queue cleaned', { queueName: this.queueName, type, grace, limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until the queue is ready
|
||||
*/
|
||||
async waitUntilReady(): Promise<void> {
|
||||
await this.bullQueue.waitUntilReady();
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the queue (cleanup resources)
|
||||
*/
|
||||
/**
|
||||
* Close the queue (cleanup resources)
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
// Close the queue itself
|
||||
await this.bullQueue.close();
|
||||
logger.info('Queue closed', { queueName: this.queueName });
|
||||
|
||||
// Close queue events
|
||||
if (this.queueEvents) {
|
||||
await this.queueEvents.close();
|
||||
logger.debug('Queue events closed', { queueName: this.queueName });
|
||||
}
|
||||
|
||||
// Close workers first
|
||||
if (this.workers.length > 0) {
|
||||
await Promise.all(
|
||||
this.workers.map(async worker => {
|
||||
return await worker.close();
|
||||
})
|
||||
);
|
||||
this.workers = [];
|
||||
logger.debug('Workers closed', { queueName: this.queueName });
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error closing queue', { queueName: this.queueName, error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start workers for this queue
|
||||
*/
|
||||
private startWorkers(workerCount: number, concurrency: number): void {
|
||||
const connection = getRedisConnection(this.redisConfig);
|
||||
|
||||
for (let i = 0; i < workerCount; i++) {
|
||||
const worker = new Worker(`{${this.queueName}}`, this.processJob.bind(this), {
|
||||
connection,
|
||||
concurrency,
|
||||
maxStalledCount: 3,
|
||||
stalledInterval: 30000,
|
||||
});
|
||||
|
||||
// Setup worker event handlers
|
||||
worker.on('completed', job => {
|
||||
logger.trace('Job completed', {
|
||||
queueName: this.queueName,
|
||||
jobId: job.id,
|
||||
handler: job.data?.handler,
|
||||
operation: job.data?.operation,
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
logger.error('Job failed', {
|
||||
queueName: this.queueName,
|
||||
jobId: job?.id,
|
||||
handler: job?.data?.handler,
|
||||
operation: job?.data?.operation,
|
||||
error: err.message,
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('error', error => {
|
||||
logger.error('Worker error', {
|
||||
queueName: this.queueName,
|
||||
workerId: i,
|
||||
error: error.message,
|
||||
});
|
||||
});
|
||||
|
||||
this.workers.push(worker);
|
||||
}
|
||||
|
||||
logger.info('Workers started', {
|
||||
queueName: this.queueName,
|
||||
workerCount,
|
||||
concurrency,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a job using the handler registry
|
||||
*/
|
||||
private async processJob(job: Job): Promise<unknown> {
|
||||
const { handler, operation, payload }: JobData = job.data;
|
||||
|
||||
logger.trace('Processing job', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
queueName: this.queueName,
|
||||
});
|
||||
|
||||
try {
|
||||
// Look up handler in registry
|
||||
const jobHandler = handlerRegistry.getOperation(handler, operation);
|
||||
|
||||
if (!jobHandler) {
|
||||
throw new Error(`No handler found for ${handler}:${operation}`);
|
||||
}
|
||||
|
||||
const result = await jobHandler(payload);
|
||||
|
||||
logger.trace('Job completed successfully', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
queueName: this.queueName,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error('Job processing failed', {
|
||||
id: job.id,
|
||||
handler,
|
||||
operation,
|
||||
queueName: this.queueName,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start workers manually (for delayed initialization)
|
||||
*/
|
||||
startWorkersManually(workerCount: number, concurrency: number = 1): void {
|
||||
if (this.workers.length > 0) {
|
||||
logger.warn('Workers already started for queue', { queueName: this.queueName });
|
||||
return;
|
||||
}
|
||||
|
||||
// Initialize queue events if not already done
|
||||
if (!this.queueEvents) {
|
||||
const connection = getRedisConnection(this.redisConfig);
|
||||
this.queueEvents = new QueueEvents(`{${this.queueName}}`, { connection });
|
||||
}
|
||||
|
||||
this.startWorkers(workerCount, concurrency);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of active workers
|
||||
*/
|
||||
getWorkerCount(): number {
|
||||
return this.workers.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the underlying BullMQ queue (for advanced operations)
|
||||
* @deprecated Use direct methods instead
|
||||
*/
|
||||
getBullQueue(): BullQueue {
|
||||
return this.bullQueue;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,294 +1,327 @@
|
|||
import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { RateLimitConfig as BaseRateLimitConfig, RateLimitRule } from './types';
|
||||
|
||||
const logger = getLogger('rate-limiter');
|
||||
|
||||
// Extend the base config to add rate-limiter specific fields
|
||||
export interface RateLimitConfig extends BaseRateLimitConfig {
|
||||
keyPrefix?: string;
|
||||
}
|
||||
|
||||
export class QueueRateLimiter {
|
||||
private limiters = new Map<string, RateLimiterRedis>();
|
||||
private rules: RateLimitRule[] = [];
|
||||
|
||||
constructor(private redisClient: ReturnType<typeof import('./utils').getRedisConnection>) {}
|
||||
|
||||
/**
|
||||
* Add a rate limit rule
|
||||
*/
|
||||
addRule(rule: RateLimitRule): void {
|
||||
this.rules.push(rule);
|
||||
|
||||
const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation);
|
||||
const limiter = new RateLimiterRedis({
|
||||
storeClient: this.redisClient,
|
||||
keyPrefix: `rl:${key}`,
|
||||
points: rule.config.points,
|
||||
duration: rule.config.duration,
|
||||
blockDuration: rule.config.blockDuration || 0,
|
||||
});
|
||||
|
||||
this.limiters.set(key, limiter);
|
||||
|
||||
logger.info('Rate limit rule added', {
|
||||
level: rule.level,
|
||||
queueName: rule.queueName,
|
||||
handler: rule.handler,
|
||||
operation: rule.operation,
|
||||
points: rule.config.points,
|
||||
duration: rule.config.duration,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a job can be processed based on rate limits
|
||||
* Uses hierarchical precedence: operation > handler > queue > global
|
||||
* The most specific matching rule takes precedence
|
||||
*/
|
||||
async checkLimit(queueName: string, handler: string, operation: string): Promise<{
|
||||
allowed: boolean;
|
||||
retryAfter?: number;
|
||||
remainingPoints?: number;
|
||||
appliedRule?: RateLimitRule;
|
||||
}> {
|
||||
const applicableRule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (!applicableRule) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const key = this.getRuleKey(applicableRule.level, applicableRule.queueName, applicableRule.handler, applicableRule.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
|
||||
if (!limiter) {
|
||||
logger.warn('Rate limiter not found for rule', { key, rule: applicableRule });
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.consumePoint(limiter, this.getConsumerKey(queueName, handler, operation));
|
||||
|
||||
return {
|
||||
...result,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Rate limit check failed', { queueName, handler, operation, error });
|
||||
// On error, allow the request to proceed
|
||||
return { allowed: true };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the most specific rule that applies to this job
|
||||
* Precedence: operation > handler > queue > global
|
||||
*/
|
||||
private getMostSpecificRule(queueName: string, handler: string, operation: string): RateLimitRule | undefined {
|
||||
// 1. Check for operation-specific rule (most specific)
|
||||
let rule = this.rules.find(r =>
|
||||
r.level === 'operation' &&
|
||||
r.queueName === queueName &&
|
||||
r.handler === handler &&
|
||||
r.operation === operation
|
||||
);
|
||||
if (rule) {return rule;}
|
||||
|
||||
// 2. Check for handler-specific rule
|
||||
rule = this.rules.find(r =>
|
||||
r.level === 'handler' &&
|
||||
r.queueName === queueName &&
|
||||
r.handler === handler
|
||||
);
|
||||
if (rule) {return rule;}
|
||||
|
||||
// 3. Check for queue-specific rule
|
||||
rule = this.rules.find(r =>
|
||||
r.level === 'queue' &&
|
||||
r.queueName === queueName
|
||||
);
|
||||
if (rule) {return rule;}
|
||||
|
||||
// 4. Check for global rule (least specific)
|
||||
rule = this.rules.find(r => r.level === 'global');
|
||||
return rule;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume a point from the rate limiter
|
||||
*/
|
||||
private async consumePoint(
|
||||
limiter: RateLimiterRedis,
|
||||
key: string
|
||||
): Promise<{ allowed: boolean; retryAfter?: number; remainingPoints?: number }> {
|
||||
try {
|
||||
const result = await limiter.consume(key);
|
||||
return {
|
||||
allowed: true,
|
||||
remainingPoints: result.remainingPoints,
|
||||
};
|
||||
} catch (rejRes) {
|
||||
if (rejRes instanceof RateLimiterRes) {
|
||||
logger.warn('Rate limit exceeded', {
|
||||
key,
|
||||
retryAfter: rejRes.msBeforeNext,
|
||||
});
|
||||
|
||||
return {
|
||||
allowed: false,
|
||||
retryAfter: rejRes.msBeforeNext,
|
||||
remainingPoints: rejRes.remainingPoints,
|
||||
};
|
||||
}
|
||||
throw rejRes;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get rule key for storing rate limiter
|
||||
*/
|
||||
private getRuleKey(level: string, queueName?: string, handler?: string, operation?: string): string {
|
||||
switch (level) {
|
||||
case 'global':
|
||||
return 'global';
|
||||
case 'queue':
|
||||
return `queue:${queueName}`;
|
||||
case 'handler':
|
||||
return `handler:${queueName}:${handler}`;
|
||||
case 'operation':
|
||||
return `operation:${queueName}:${handler}:${operation}`;
|
||||
default:
|
||||
return level;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get consumer key for rate limiting (what gets counted)
|
||||
*/
|
||||
private getConsumerKey(queueName: string, handler: string, operation: string): string {
|
||||
return `${queueName}:${handler}:${operation}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current rate limit status for a queue/handler/operation
|
||||
*/
|
||||
async getStatus(queueName: string, handler: string, operation: string): Promise<{
|
||||
queueName: string;
|
||||
handler: string;
|
||||
operation: string;
|
||||
appliedRule?: RateLimitRule;
|
||||
limit?: {
|
||||
level: string;
|
||||
points: number;
|
||||
duration: number;
|
||||
remaining: number;
|
||||
resetIn: number;
|
||||
};
|
||||
}> {
|
||||
const applicableRule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (!applicableRule) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
};
|
||||
}
|
||||
|
||||
const key = this.getRuleKey(applicableRule.level, applicableRule.queueName, applicableRule.handler, applicableRule.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
|
||||
if (!limiter) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const consumerKey = this.getConsumerKey(queueName, handler, operation);
|
||||
const result = await limiter.get(consumerKey);
|
||||
|
||||
const limit = {
|
||||
level: applicableRule.level,
|
||||
points: limiter.points,
|
||||
duration: limiter.duration,
|
||||
remaining: result?.remainingPoints ?? limiter.points,
|
||||
resetIn: result?.msBeforeNext ?? 0,
|
||||
};
|
||||
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule: applicableRule,
|
||||
limit,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get rate limit status', { queueName, handler, operation, error });
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset rate limits for a specific consumer
|
||||
*/
|
||||
async reset(queueName: string, handler?: string, operation?: string): Promise<void> {
|
||||
if (handler && operation) {
|
||||
// Reset specific operation
|
||||
const consumerKey = this.getConsumerKey(queueName, handler, operation);
|
||||
const rule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (rule) {
|
||||
const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
if (limiter) {
|
||||
await limiter.delete(consumerKey);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Reset broader scope - this is more complex with the new hierarchy
|
||||
logger.warn('Broad reset not implemented yet', { queueName, handler, operation });
|
||||
}
|
||||
|
||||
logger.info('Rate limits reset', { queueName, handler, operation });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all configured rate limit rules
|
||||
*/
|
||||
getRules(): RateLimitRule[] {
|
||||
return [...this.rules];
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a rate limit rule
|
||||
*/
|
||||
removeRule(level: string, queueName?: string, handler?: string, operation?: string): boolean {
|
||||
const key = this.getRuleKey(level, queueName, handler, operation);
|
||||
const ruleIndex = this.rules.findIndex(r =>
|
||||
r.level === level &&
|
||||
(!queueName || r.queueName === queueName) &&
|
||||
(!handler || r.handler === handler) &&
|
||||
(!operation || r.operation === operation)
|
||||
);
|
||||
|
||||
if (ruleIndex >= 0) {
|
||||
this.rules.splice(ruleIndex, 1);
|
||||
this.limiters.delete(key);
|
||||
|
||||
logger.info('Rate limit rule removed', { level, queueName, handler, operation });
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { RateLimitConfig as BaseRateLimitConfig, RateLimitRule } from './types';
|
||||
|
||||
const logger = getLogger('rate-limiter');
|
||||
|
||||
// Extend the base config to add rate-limiter specific fields
|
||||
export interface RateLimitConfig extends BaseRateLimitConfig {
|
||||
keyPrefix?: string;
|
||||
}
|
||||
|
||||
export class QueueRateLimiter {
|
||||
private limiters = new Map<string, RateLimiterRedis>();
|
||||
private rules: RateLimitRule[] = [];
|
||||
|
||||
constructor(private redisClient: ReturnType<typeof import('./utils').getRedisConnection>) {}
|
||||
|
||||
/**
|
||||
* Add a rate limit rule
|
||||
*/
|
||||
addRule(rule: RateLimitRule): void {
|
||||
this.rules.push(rule);
|
||||
|
||||
const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation);
|
||||
const limiter = new RateLimiterRedis({
|
||||
storeClient: this.redisClient,
|
||||
keyPrefix: `rl:${key}`,
|
||||
points: rule.config.points,
|
||||
duration: rule.config.duration,
|
||||
blockDuration: rule.config.blockDuration || 0,
|
||||
});
|
||||
|
||||
this.limiters.set(key, limiter);
|
||||
|
||||
logger.info('Rate limit rule added', {
|
||||
level: rule.level,
|
||||
queueName: rule.queueName,
|
||||
handler: rule.handler,
|
||||
operation: rule.operation,
|
||||
points: rule.config.points,
|
||||
duration: rule.config.duration,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a job can be processed based on rate limits
|
||||
* Uses hierarchical precedence: operation > handler > queue > global
|
||||
* The most specific matching rule takes precedence
|
||||
*/
|
||||
async checkLimit(
|
||||
queueName: string,
|
||||
handler: string,
|
||||
operation: string
|
||||
): Promise<{
|
||||
allowed: boolean;
|
||||
retryAfter?: number;
|
||||
remainingPoints?: number;
|
||||
appliedRule?: RateLimitRule;
|
||||
}> {
|
||||
const applicableRule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (!applicableRule) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const key = this.getRuleKey(
|
||||
applicableRule.level,
|
||||
applicableRule.queueName,
|
||||
applicableRule.handler,
|
||||
applicableRule.operation
|
||||
);
|
||||
const limiter = this.limiters.get(key);
|
||||
|
||||
if (!limiter) {
|
||||
logger.warn('Rate limiter not found for rule', { key, rule: applicableRule });
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await this.consumePoint(
|
||||
limiter,
|
||||
this.getConsumerKey(queueName, handler, operation)
|
||||
);
|
||||
|
||||
return {
|
||||
...result,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Rate limit check failed', { queueName, handler, operation, error });
|
||||
// On error, allow the request to proceed
|
||||
return { allowed: true };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the most specific rule that applies to this job
|
||||
* Precedence: operation > handler > queue > global
|
||||
*/
|
||||
private getMostSpecificRule(
|
||||
queueName: string,
|
||||
handler: string,
|
||||
operation: string
|
||||
): RateLimitRule | undefined {
|
||||
// 1. Check for operation-specific rule (most specific)
|
||||
let rule = this.rules.find(
|
||||
r =>
|
||||
r.level === 'operation' &&
|
||||
r.queueName === queueName &&
|
||||
r.handler === handler &&
|
||||
r.operation === operation
|
||||
);
|
||||
if (rule) {
|
||||
return rule;
|
||||
}
|
||||
|
||||
// 2. Check for handler-specific rule
|
||||
rule = this.rules.find(
|
||||
r => r.level === 'handler' && r.queueName === queueName && r.handler === handler
|
||||
);
|
||||
if (rule) {
|
||||
return rule;
|
||||
}
|
||||
|
||||
// 3. Check for queue-specific rule
|
||||
rule = this.rules.find(r => r.level === 'queue' && r.queueName === queueName);
|
||||
if (rule) {
|
||||
return rule;
|
||||
}
|
||||
|
||||
// 4. Check for global rule (least specific)
|
||||
rule = this.rules.find(r => r.level === 'global');
|
||||
return rule;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume a point from the rate limiter
|
||||
*/
|
||||
private async consumePoint(
|
||||
limiter: RateLimiterRedis,
|
||||
key: string
|
||||
): Promise<{ allowed: boolean; retryAfter?: number; remainingPoints?: number }> {
|
||||
try {
|
||||
const result = await limiter.consume(key);
|
||||
return {
|
||||
allowed: true,
|
||||
remainingPoints: result.remainingPoints,
|
||||
};
|
||||
} catch (rejRes) {
|
||||
if (rejRes instanceof RateLimiterRes) {
|
||||
logger.warn('Rate limit exceeded', {
|
||||
key,
|
||||
retryAfter: rejRes.msBeforeNext,
|
||||
});
|
||||
|
||||
return {
|
||||
allowed: false,
|
||||
retryAfter: rejRes.msBeforeNext,
|
||||
remainingPoints: rejRes.remainingPoints,
|
||||
};
|
||||
}
|
||||
throw rejRes;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get rule key for storing rate limiter
|
||||
*/
|
||||
private getRuleKey(
|
||||
level: string,
|
||||
queueName?: string,
|
||||
handler?: string,
|
||||
operation?: string
|
||||
): string {
|
||||
switch (level) {
|
||||
case 'global':
|
||||
return 'global';
|
||||
case 'queue':
|
||||
return `queue:${queueName}`;
|
||||
case 'handler':
|
||||
return `handler:${queueName}:${handler}`;
|
||||
case 'operation':
|
||||
return `operation:${queueName}:${handler}:${operation}`;
|
||||
default:
|
||||
return level;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get consumer key for rate limiting (what gets counted)
|
||||
*/
|
||||
private getConsumerKey(queueName: string, handler: string, operation: string): string {
|
||||
return `${queueName}:${handler}:${operation}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current rate limit status for a queue/handler/operation
|
||||
*/
|
||||
async getStatus(
|
||||
queueName: string,
|
||||
handler: string,
|
||||
operation: string
|
||||
): Promise<{
|
||||
queueName: string;
|
||||
handler: string;
|
||||
operation: string;
|
||||
appliedRule?: RateLimitRule;
|
||||
limit?: {
|
||||
level: string;
|
||||
points: number;
|
||||
duration: number;
|
||||
remaining: number;
|
||||
resetIn: number;
|
||||
};
|
||||
}> {
|
||||
const applicableRule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (!applicableRule) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
};
|
||||
}
|
||||
|
||||
const key = this.getRuleKey(
|
||||
applicableRule.level,
|
||||
applicableRule.queueName,
|
||||
applicableRule.handler,
|
||||
applicableRule.operation
|
||||
);
|
||||
const limiter = this.limiters.get(key);
|
||||
|
||||
if (!limiter) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const consumerKey = this.getConsumerKey(queueName, handler, operation);
|
||||
const result = await limiter.get(consumerKey);
|
||||
|
||||
const limit = {
|
||||
level: applicableRule.level,
|
||||
points: limiter.points,
|
||||
duration: limiter.duration,
|
||||
remaining: result?.remainingPoints ?? limiter.points,
|
||||
resetIn: result?.msBeforeNext ?? 0,
|
||||
};
|
||||
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule: applicableRule,
|
||||
limit,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get rate limit status', { queueName, handler, operation, error });
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset rate limits for a specific consumer
|
||||
*/
|
||||
async reset(queueName: string, handler?: string, operation?: string): Promise<void> {
|
||||
if (handler && operation) {
|
||||
// Reset specific operation
|
||||
const consumerKey = this.getConsumerKey(queueName, handler, operation);
|
||||
const rule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (rule) {
|
||||
const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
if (limiter) {
|
||||
await limiter.delete(consumerKey);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Reset broader scope - this is more complex with the new hierarchy
|
||||
logger.warn('Broad reset not implemented yet', { queueName, handler, operation });
|
||||
}
|
||||
|
||||
logger.info('Rate limits reset', { queueName, handler, operation });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all configured rate limit rules
|
||||
*/
|
||||
getRules(): RateLimitRule[] {
|
||||
return [...this.rules];
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a rate limit rule
|
||||
*/
|
||||
removeRule(level: string, queueName?: string, handler?: string, operation?: string): boolean {
|
||||
const key = this.getRuleKey(level, queueName, handler, operation);
|
||||
const ruleIndex = this.rules.findIndex(
|
||||
r =>
|
||||
r.level === level &&
|
||||
(!queueName || r.queueName === queueName) &&
|
||||
(!handler || r.handler === handler) &&
|
||||
(!operation || r.operation === operation)
|
||||
);
|
||||
|
||||
if (ruleIndex >= 0) {
|
||||
this.rules.splice(ruleIndex, 1);
|
||||
this.limiters.delete(key);
|
||||
|
||||
logger.info('Rate limit rule removed', { level, queueName, handler, operation });
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ export interface QueueOptions {
|
|||
enableMetrics?: boolean;
|
||||
enableDLQ?: boolean;
|
||||
enableRateLimit?: boolean;
|
||||
rateLimitRules?: RateLimitRule[]; // Queue-specific rate limit rules
|
||||
rateLimitRules?: RateLimitRule[]; // Queue-specific rate limit rules
|
||||
}
|
||||
|
||||
export interface QueueManagerConfig {
|
||||
|
|
@ -79,8 +79,8 @@ export interface QueueManagerConfig {
|
|||
defaultQueueOptions?: QueueOptions;
|
||||
enableScheduledJobs?: boolean;
|
||||
globalRateLimit?: RateLimitConfig;
|
||||
rateLimitRules?: RateLimitRule[]; // Global rate limit rules
|
||||
delayWorkerStart?: boolean; // If true, workers won't start automatically
|
||||
rateLimitRules?: RateLimitRule[]; // Global rate limit rules
|
||||
delayWorkerStart?: boolean; // If true, workers won't start automatically
|
||||
}
|
||||
|
||||
export interface QueueStats {
|
||||
|
|
@ -118,7 +118,7 @@ export interface BatchJobData {
|
|||
batchIndex: number;
|
||||
totalBatches: number;
|
||||
itemCount: number;
|
||||
totalDelayHours: number; // Total time to distribute all batches
|
||||
totalDelayHours: number; // Total time to distribute all batches
|
||||
}
|
||||
|
||||
export interface HandlerInitializer {
|
||||
|
|
@ -134,9 +134,9 @@ export interface RateLimitConfig {
|
|||
|
||||
export interface RateLimitRule {
|
||||
level: 'global' | 'queue' | 'handler' | 'operation';
|
||||
queueName?: string; // For queue-level limits
|
||||
handler?: string; // For handler-level limits
|
||||
operation?: string; // For operation-level limits (most specific)
|
||||
queueName?: string; // For queue-level limits
|
||||
handler?: string; // For handler-level limits
|
||||
operation?: string; // For operation-level limits (most specific)
|
||||
config: RateLimitConfig;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import type { RedisConfig } from './types';
|
|||
*/
|
||||
export function getRedisConnection(config: RedisConfig) {
|
||||
const isTest = process.env.NODE_ENV === 'test' || process.env['BUNIT'] === '1';
|
||||
|
||||
|
||||
return {
|
||||
host: config.host,
|
||||
port: config.port,
|
||||
|
|
|
|||
|
|
@ -1,355 +1,364 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { QueueManager, Queue, handlerRegistry, processItems } from '../src';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('Batch Processor', () => {
|
||||
let queueManager: QueueManager;
|
||||
let queue: Queue;
|
||||
let queueName: string;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clear handler registry
|
||||
handlerRegistry.clear();
|
||||
|
||||
// Register test handler
|
||||
handlerRegistry.register('batch-test', {
|
||||
'process-item': async (payload) => {
|
||||
return { processed: true, data: payload };
|
||||
},
|
||||
'generic': async (payload) => {
|
||||
return { processed: true, data: payload };
|
||||
},
|
||||
'process-batch-items': async (_batchData) => {
|
||||
// This is called by the batch processor internally
|
||||
return { batchProcessed: true };
|
||||
},
|
||||
});
|
||||
|
||||
// Use unique queue name per test to avoid conflicts
|
||||
queueName = `batch-test-queue-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
|
||||
// Reset and initialize singleton QueueManager for tests
|
||||
await QueueManager.reset();
|
||||
queueManager = QueueManager.initialize({
|
||||
redis: redisConfig,
|
||||
defaultQueueOptions: {
|
||||
workers: 0, // No workers in tests
|
||||
concurrency: 5,
|
||||
},
|
||||
});
|
||||
|
||||
// Get queue using the new getQueue() method (batch cache is now auto-initialized)
|
||||
queue = queueManager.getQueue(queueName);
|
||||
// Note: Batch cache is now automatically initialized when getting the queue
|
||||
|
||||
// Ensure completely clean state - wait for queue to be ready first
|
||||
await queue.getBullQueue().waitUntilReady();
|
||||
|
||||
// Clear all job states
|
||||
await queue.getBullQueue().drain(true);
|
||||
await queue.getBullQueue().clean(0, 1000, 'completed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'failed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'active');
|
||||
await queue.getBullQueue().clean(0, 1000, 'waiting');
|
||||
await queue.getBullQueue().clean(0, 1000, 'delayed');
|
||||
|
||||
// Add a small delay to ensure cleanup is complete
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
// Clean up jobs first
|
||||
if (queue) {
|
||||
try {
|
||||
await queue.getBullQueue().drain(true);
|
||||
await queue.getBullQueue().clean(0, 1000, 'completed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'failed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'active');
|
||||
await queue.getBullQueue().clean(0, 1000, 'waiting');
|
||||
await queue.getBullQueue().clean(0, 1000, 'delayed');
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await queue.close();
|
||||
}
|
||||
|
||||
if (queueManager) {
|
||||
await Promise.race([
|
||||
QueueManager.reset(),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Shutdown timeout')), 3000)
|
||||
)
|
||||
]);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Cleanup error:', error.message);
|
||||
} finally {
|
||||
handlerRegistry.clear();
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
});
|
||||
|
||||
describe('Direct Processing', () => {
|
||||
test('should process items directly without batching', async () => {
|
||||
const items = ['item1', 'item2', 'item3', 'item4', 'item5'];
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001, // 3.6 seconds total
|
||||
useBatching: false,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
priority: 1,
|
||||
});
|
||||
|
||||
expect(result.mode).toBe('direct');
|
||||
expect(result.totalItems).toBe(5);
|
||||
expect(result.jobsCreated).toBe(5);
|
||||
|
||||
// Verify jobs were created - BullMQ has an issue where job ID "1" doesn't show up in state queries
|
||||
// but exists when queried directly, so we need to check both ways
|
||||
const [delayedJobs, waitingJobs, activeJobs, completedJobs, failedJobs, job1] = await Promise.all([
|
||||
queue.getBullQueue().getJobs(['delayed']),
|
||||
queue.getBullQueue().getJobs(['waiting']),
|
||||
queue.getBullQueue().getJobs(['active']),
|
||||
queue.getBullQueue().getJobs(['completed']),
|
||||
queue.getBullQueue().getJobs(['failed']),
|
||||
queue.getBullQueue().getJob('1'), // Job 1 often doesn't show up in state queries
|
||||
]);
|
||||
|
||||
const jobs = [...delayedJobs, ...waitingJobs, ...activeJobs, ...completedJobs, ...failedJobs];
|
||||
const ourJobs = jobs.filter(j => j.name === 'process-item' && j.data.handler === 'batch-test');
|
||||
|
||||
// Include job 1 if we found it directly but it wasn't in the state queries
|
||||
if (job1 && job1.name === 'process-item' && job1.data.handler === 'batch-test' && !ourJobs.find(j => j.id === '1')) {
|
||||
ourJobs.push(job1);
|
||||
}
|
||||
|
||||
expect(ourJobs.length).toBe(5);
|
||||
|
||||
// Check delays are distributed
|
||||
const delays = ourJobs.map(j => j.opts.delay || 0).sort((a, b) => a - b);
|
||||
expect(delays[0]).toBe(0);
|
||||
expect(delays[4]).toBeGreaterThan(delays[0]);
|
||||
});
|
||||
|
||||
test('should process complex objects directly', async () => {
|
||||
const items = [
|
||||
{ id: 1, name: 'Product A', price: 100 },
|
||||
{ id: 2, name: 'Product B', price: 200 },
|
||||
{ id: 3, name: 'Product C', price: 300 },
|
||||
];
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: false,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.jobsCreated).toBe(3);
|
||||
|
||||
// Check job payloads
|
||||
const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']);
|
||||
const ourJobs = jobs.filter(j => j.name === 'process-item' && j.data.handler === 'batch-test');
|
||||
const payloads = ourJobs.map(j => j.data.payload);
|
||||
|
||||
expect(payloads).toContainEqual({ id: 1, name: 'Product A', price: 100 });
|
||||
expect(payloads).toContainEqual({ id: 2, name: 'Product B', price: 200 });
|
||||
expect(payloads).toContainEqual({ id: 3, name: 'Product C', price: 300 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('Batch Processing', () => {
|
||||
test('should process items in batches', async () => {
|
||||
const items = Array.from({ length: 50 }, (_, i) => ({ id: i, value: `item-${i}` }));
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: true,
|
||||
batchSize: 10,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.mode).toBe('batch');
|
||||
expect(result.totalItems).toBe(50);
|
||||
expect(result.batchesCreated).toBe(5); // 50 items / 10 per batch
|
||||
expect(result.jobsCreated).toBe(5); // 5 batch jobs
|
||||
|
||||
// Verify batch jobs were created
|
||||
const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']);
|
||||
const batchJobs = jobs.filter(j => j.name === 'process-batch');
|
||||
expect(batchJobs.length).toBe(5);
|
||||
});
|
||||
|
||||
test('should handle different batch sizes', async () => {
|
||||
const items = Array.from({ length: 23 }, (_, i) => i);
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: true,
|
||||
batchSize: 7,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.batchesCreated).toBe(4); // 23/7 = 3.28, rounded up to 4
|
||||
expect(result.jobsCreated).toBe(4);
|
||||
});
|
||||
|
||||
test('should store batch payloads in cache', async () => {
|
||||
const items = [
|
||||
{ type: 'A', data: 'test1' },
|
||||
{ type: 'B', data: 'test2' },
|
||||
];
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: true,
|
||||
batchSize: 2,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
ttl: 3600, // 1 hour TTL
|
||||
});
|
||||
|
||||
expect(result.jobsCreated).toBe(1);
|
||||
|
||||
// Get the batch job
|
||||
const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']);
|
||||
expect(jobs.length).toBe(1);
|
||||
|
||||
const batchJob = jobs[0];
|
||||
expect(batchJob.data.payload.payloadKey).toBeDefined();
|
||||
expect(batchJob.data.payload.itemCount).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Empty and Edge Cases', () => {
|
||||
test('should handle empty item list', async () => {
|
||||
const result = await processItems([], queueName, {
|
||||
totalDelayHours: 1,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.totalItems).toBe(0);
|
||||
expect(result.jobsCreated).toBe(0);
|
||||
expect(result.duration).toBeDefined();
|
||||
});
|
||||
|
||||
test('should handle single item', async () => {
|
||||
const result = await processItems(['single-item'], queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.totalItems).toBe(1);
|
||||
expect(result.jobsCreated).toBe(1);
|
||||
});
|
||||
|
||||
test('should handle large batch with delays', async () => {
|
||||
const items = Array.from({ length: 100 }, (_, i) => ({ index: i }));
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.01, // 36 seconds total
|
||||
useBatching: true,
|
||||
batchSize: 25,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.batchesCreated).toBe(4); // 100/25
|
||||
expect(result.jobsCreated).toBe(4);
|
||||
|
||||
// Check delays are distributed
|
||||
const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']);
|
||||
const delays = jobs.map(j => j.opts.delay || 0).sort((a, b) => a - b);
|
||||
|
||||
expect(delays[0]).toBe(0); // First batch has no delay
|
||||
expect(delays[3]).toBeGreaterThan(0); // Last batch has delay
|
||||
});
|
||||
});
|
||||
|
||||
describe('Job Options', () => {
|
||||
test('should respect custom job options', async () => {
|
||||
const items = ['a', 'b', 'c'];
|
||||
|
||||
await processItems(items, queueName, {
|
||||
totalDelayHours: 0,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
priority: 5,
|
||||
retries: 10,
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 50,
|
||||
});
|
||||
|
||||
// Check all states including job ID "1" specifically (as it often doesn't show up in state queries)
|
||||
const [waitingJobs, delayedJobs, job1, job2, job3] = await Promise.all([
|
||||
queue.getBullQueue().getJobs(['waiting']),
|
||||
queue.getBullQueue().getJobs(['delayed']),
|
||||
queue.getBullQueue().getJob('1'),
|
||||
queue.getBullQueue().getJob('2'),
|
||||
queue.getBullQueue().getJob('3'),
|
||||
]);
|
||||
|
||||
const jobs = [...waitingJobs, ...delayedJobs];
|
||||
// Add any missing jobs that exist but don't show up in state queries
|
||||
[job1, job2, job3].forEach(job => {
|
||||
if (job && !jobs.find(j => j.id === job.id)) {
|
||||
jobs.push(job);
|
||||
}
|
||||
});
|
||||
|
||||
expect(jobs.length).toBe(3);
|
||||
|
||||
jobs.forEach(job => {
|
||||
expect(job.opts.priority).toBe(5);
|
||||
expect(job.opts.attempts).toBe(10);
|
||||
expect(job.opts.removeOnComplete).toBe(100);
|
||||
expect(job.opts.removeOnFail).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
test('should set handler and operation correctly', async () => {
|
||||
// Register custom handler for this test
|
||||
handlerRegistry.register('custom-handler', {
|
||||
'custom-operation': async (payload) => {
|
||||
return { processed: true, data: payload };
|
||||
},
|
||||
});
|
||||
|
||||
await processItems(['test'], queueName, {
|
||||
totalDelayHours: 0,
|
||||
handler: 'custom-handler',
|
||||
operation: 'custom-operation',
|
||||
});
|
||||
|
||||
const jobs = await queue.getBullQueue().getJobs(['waiting']);
|
||||
expect(jobs.length).toBe(1);
|
||||
expect(jobs[0].data.handler).toBe('custom-handler');
|
||||
expect(jobs[0].data.operation).toBe('custom-operation');
|
||||
});
|
||||
});
|
||||
});
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { handlerRegistry, processItems, Queue, QueueManager } from '../src';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (
|
||||
message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('Batch Processor', () => {
|
||||
let queueManager: QueueManager;
|
||||
let queue: Queue;
|
||||
let queueName: string;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clear handler registry
|
||||
handlerRegistry.clear();
|
||||
|
||||
// Register test handler
|
||||
handlerRegistry.register('batch-test', {
|
||||
'process-item': async payload => {
|
||||
return { processed: true, data: payload };
|
||||
},
|
||||
generic: async payload => {
|
||||
return { processed: true, data: payload };
|
||||
},
|
||||
'process-batch-items': async _batchData => {
|
||||
// This is called by the batch processor internally
|
||||
return { batchProcessed: true };
|
||||
},
|
||||
});
|
||||
|
||||
// Use unique queue name per test to avoid conflicts
|
||||
queueName = `batch-test-queue-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
|
||||
// Reset and initialize singleton QueueManager for tests
|
||||
await QueueManager.reset();
|
||||
queueManager = QueueManager.initialize({
|
||||
redis: redisConfig,
|
||||
defaultQueueOptions: {
|
||||
workers: 0, // No workers in tests
|
||||
concurrency: 5,
|
||||
},
|
||||
});
|
||||
|
||||
// Get queue using the new getQueue() method (batch cache is now auto-initialized)
|
||||
queue = queueManager.getQueue(queueName);
|
||||
// Note: Batch cache is now automatically initialized when getting the queue
|
||||
|
||||
// Ensure completely clean state - wait for queue to be ready first
|
||||
await queue.getBullQueue().waitUntilReady();
|
||||
|
||||
// Clear all job states
|
||||
await queue.getBullQueue().drain(true);
|
||||
await queue.getBullQueue().clean(0, 1000, 'completed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'failed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'active');
|
||||
await queue.getBullQueue().clean(0, 1000, 'waiting');
|
||||
await queue.getBullQueue().clean(0, 1000, 'delayed');
|
||||
|
||||
// Add a small delay to ensure cleanup is complete
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
// Clean up jobs first
|
||||
if (queue) {
|
||||
try {
|
||||
await queue.getBullQueue().drain(true);
|
||||
await queue.getBullQueue().clean(0, 1000, 'completed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'failed');
|
||||
await queue.getBullQueue().clean(0, 1000, 'active');
|
||||
await queue.getBullQueue().clean(0, 1000, 'waiting');
|
||||
await queue.getBullQueue().clean(0, 1000, 'delayed');
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await queue.close();
|
||||
}
|
||||
|
||||
if (queueManager) {
|
||||
await Promise.race([
|
||||
QueueManager.reset(),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error('Shutdown timeout')), 3000)),
|
||||
]);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Cleanup error:', error.message);
|
||||
} finally {
|
||||
handlerRegistry.clear();
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
});
|
||||
|
||||
describe('Direct Processing', () => {
|
||||
test('should process items directly without batching', async () => {
|
||||
const items = ['item1', 'item2', 'item3', 'item4', 'item5'];
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001, // 3.6 seconds total
|
||||
useBatching: false,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
priority: 1,
|
||||
});
|
||||
|
||||
expect(result.mode).toBe('direct');
|
||||
expect(result.totalItems).toBe(5);
|
||||
expect(result.jobsCreated).toBe(5);
|
||||
|
||||
// Verify jobs were created - BullMQ has an issue where job ID "1" doesn't show up in state queries
|
||||
// but exists when queried directly, so we need to check both ways
|
||||
const [delayedJobs, waitingJobs, activeJobs, completedJobs, failedJobs, job1] =
|
||||
await Promise.all([
|
||||
queue.getBullQueue().getJobs(['delayed']),
|
||||
queue.getBullQueue().getJobs(['waiting']),
|
||||
queue.getBullQueue().getJobs(['active']),
|
||||
queue.getBullQueue().getJobs(['completed']),
|
||||
queue.getBullQueue().getJobs(['failed']),
|
||||
queue.getBullQueue().getJob('1'), // Job 1 often doesn't show up in state queries
|
||||
]);
|
||||
|
||||
const jobs = [...delayedJobs, ...waitingJobs, ...activeJobs, ...completedJobs, ...failedJobs];
|
||||
const ourJobs = jobs.filter(
|
||||
j => j.name === 'process-item' && j.data.handler === 'batch-test'
|
||||
);
|
||||
|
||||
// Include job 1 if we found it directly but it wasn't in the state queries
|
||||
if (
|
||||
job1 &&
|
||||
job1.name === 'process-item' &&
|
||||
job1.data.handler === 'batch-test' &&
|
||||
!ourJobs.find(j => j.id === '1')
|
||||
) {
|
||||
ourJobs.push(job1);
|
||||
}
|
||||
|
||||
expect(ourJobs.length).toBe(5);
|
||||
|
||||
// Check delays are distributed
|
||||
const delays = ourJobs.map(j => j.opts.delay || 0).sort((a, b) => a - b);
|
||||
expect(delays[0]).toBe(0);
|
||||
expect(delays[4]).toBeGreaterThan(delays[0]);
|
||||
});
|
||||
|
||||
test('should process complex objects directly', async () => {
|
||||
const items = [
|
||||
{ id: 1, name: 'Product A', price: 100 },
|
||||
{ id: 2, name: 'Product B', price: 200 },
|
||||
{ id: 3, name: 'Product C', price: 300 },
|
||||
];
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: false,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.jobsCreated).toBe(3);
|
||||
|
||||
// Check job payloads
|
||||
const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']);
|
||||
const ourJobs = jobs.filter(
|
||||
j => j.name === 'process-item' && j.data.handler === 'batch-test'
|
||||
);
|
||||
const payloads = ourJobs.map(j => j.data.payload);
|
||||
|
||||
expect(payloads).toContainEqual({ id: 1, name: 'Product A', price: 100 });
|
||||
expect(payloads).toContainEqual({ id: 2, name: 'Product B', price: 200 });
|
||||
expect(payloads).toContainEqual({ id: 3, name: 'Product C', price: 300 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('Batch Processing', () => {
|
||||
test('should process items in batches', async () => {
|
||||
const items = Array.from({ length: 50 }, (_, i) => ({ id: i, value: `item-${i}` }));
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: true,
|
||||
batchSize: 10,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.mode).toBe('batch');
|
||||
expect(result.totalItems).toBe(50);
|
||||
expect(result.batchesCreated).toBe(5); // 50 items / 10 per batch
|
||||
expect(result.jobsCreated).toBe(5); // 5 batch jobs
|
||||
|
||||
// Verify batch jobs were created
|
||||
const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']);
|
||||
const batchJobs = jobs.filter(j => j.name === 'process-batch');
|
||||
expect(batchJobs.length).toBe(5);
|
||||
});
|
||||
|
||||
test('should handle different batch sizes', async () => {
|
||||
const items = Array.from({ length: 23 }, (_, i) => i);
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: true,
|
||||
batchSize: 7,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.batchesCreated).toBe(4); // 23/7 = 3.28, rounded up to 4
|
||||
expect(result.jobsCreated).toBe(4);
|
||||
});
|
||||
|
||||
test('should store batch payloads in cache', async () => {
|
||||
const items = [
|
||||
{ type: 'A', data: 'test1' },
|
||||
{ type: 'B', data: 'test2' },
|
||||
];
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
useBatching: true,
|
||||
batchSize: 2,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
ttl: 3600, // 1 hour TTL
|
||||
});
|
||||
|
||||
expect(result.jobsCreated).toBe(1);
|
||||
|
||||
// Get the batch job
|
||||
const jobs = await queue.getBullQueue().getJobs(['waiting', 'delayed']);
|
||||
expect(jobs.length).toBe(1);
|
||||
|
||||
const batchJob = jobs[0];
|
||||
expect(batchJob.data.payload.payloadKey).toBeDefined();
|
||||
expect(batchJob.data.payload.itemCount).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Empty and Edge Cases', () => {
|
||||
test('should handle empty item list', async () => {
|
||||
const result = await processItems([], queueName, {
|
||||
totalDelayHours: 1,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.totalItems).toBe(0);
|
||||
expect(result.jobsCreated).toBe(0);
|
||||
expect(result.duration).toBeDefined();
|
||||
});
|
||||
|
||||
test('should handle single item', async () => {
|
||||
const result = await processItems(['single-item'], queueName, {
|
||||
totalDelayHours: 0.001,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.totalItems).toBe(1);
|
||||
expect(result.jobsCreated).toBe(1);
|
||||
});
|
||||
|
||||
test('should handle large batch with delays', async () => {
|
||||
const items = Array.from({ length: 100 }, (_, i) => ({ index: i }));
|
||||
|
||||
const result = await processItems(items, queueName, {
|
||||
totalDelayHours: 0.01, // 36 seconds total
|
||||
useBatching: true,
|
||||
batchSize: 25,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
});
|
||||
|
||||
expect(result.batchesCreated).toBe(4); // 100/25
|
||||
expect(result.jobsCreated).toBe(4);
|
||||
|
||||
// Check delays are distributed
|
||||
const jobs = await queue.getBullQueue().getJobs(['delayed', 'waiting']);
|
||||
const delays = jobs.map(j => j.opts.delay || 0).sort((a, b) => a - b);
|
||||
|
||||
expect(delays[0]).toBe(0); // First batch has no delay
|
||||
expect(delays[3]).toBeGreaterThan(0); // Last batch has delay
|
||||
});
|
||||
});
|
||||
|
||||
describe('Job Options', () => {
|
||||
test('should respect custom job options', async () => {
|
||||
const items = ['a', 'b', 'c'];
|
||||
|
||||
await processItems(items, queueName, {
|
||||
totalDelayHours: 0,
|
||||
handler: 'batch-test',
|
||||
operation: 'process-item',
|
||||
priority: 5,
|
||||
retries: 10,
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 50,
|
||||
});
|
||||
|
||||
// Check all states including job ID "1" specifically (as it often doesn't show up in state queries)
|
||||
const [waitingJobs, delayedJobs, job1, job2, job3] = await Promise.all([
|
||||
queue.getBullQueue().getJobs(['waiting']),
|
||||
queue.getBullQueue().getJobs(['delayed']),
|
||||
queue.getBullQueue().getJob('1'),
|
||||
queue.getBullQueue().getJob('2'),
|
||||
queue.getBullQueue().getJob('3'),
|
||||
]);
|
||||
|
||||
const jobs = [...waitingJobs, ...delayedJobs];
|
||||
// Add any missing jobs that exist but don't show up in state queries
|
||||
[job1, job2, job3].forEach(job => {
|
||||
if (job && !jobs.find(j => j.id === job.id)) {
|
||||
jobs.push(job);
|
||||
}
|
||||
});
|
||||
|
||||
expect(jobs.length).toBe(3);
|
||||
|
||||
jobs.forEach(job => {
|
||||
expect(job.opts.priority).toBe(5);
|
||||
expect(job.opts.attempts).toBe(10);
|
||||
expect(job.opts.removeOnComplete).toBe(100);
|
||||
expect(job.opts.removeOnFail).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
test('should set handler and operation correctly', async () => {
|
||||
// Register custom handler for this test
|
||||
handlerRegistry.register('custom-handler', {
|
||||
'custom-operation': async payload => {
|
||||
return { processed: true, data: payload };
|
||||
},
|
||||
});
|
||||
|
||||
await processItems(['test'], queueName, {
|
||||
totalDelayHours: 0,
|
||||
handler: 'custom-handler',
|
||||
operation: 'custom-operation',
|
||||
});
|
||||
|
||||
const jobs = await queue.getBullQueue().getJobs(['waiting']);
|
||||
expect(jobs.length).toBe(1);
|
||||
expect(jobs[0].data.handler).toBe('custom-handler');
|
||||
expect(jobs[0].data.operation).toBe('custom-operation');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,357 +1,379 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { Queue, Worker } from 'bullmq';
|
||||
import { DeadLetterQueueHandler } from '../src/dlq-handler';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('DeadLetterQueueHandler', () => {
|
||||
let mainQueue: Queue;
|
||||
let dlqHandler: DeadLetterQueueHandler;
|
||||
let worker: Worker;
|
||||
let connection: any;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Create main queue
|
||||
mainQueue = new Queue('test-queue', { connection });
|
||||
|
||||
// Create DLQ handler
|
||||
dlqHandler = new DeadLetterQueueHandler(mainQueue, connection, {
|
||||
maxRetries: 3,
|
||||
retryDelay: 100,
|
||||
alertThreshold: 5,
|
||||
cleanupAge: 24,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
if (worker) {
|
||||
await worker.close();
|
||||
}
|
||||
await dlqHandler.shutdown();
|
||||
await mainQueue.close();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Failed Job Handling', () => {
|
||||
test('should move job to DLQ after max retries', async () => {
|
||||
let attemptCount = 0;
|
||||
|
||||
// Create worker that always fails
|
||||
worker = new Worker('test-queue', async () => {
|
||||
attemptCount++;
|
||||
throw new Error('Job failed');
|
||||
}, {
|
||||
connection,
|
||||
autorun: false,
|
||||
});
|
||||
|
||||
// Add job with limited attempts
|
||||
const _job = await mainQueue.add('failing-job', { test: true }, {
|
||||
attempts: 3,
|
||||
backoff: { type: 'fixed', delay: 50 },
|
||||
});
|
||||
|
||||
// Process job manually
|
||||
await worker.run();
|
||||
|
||||
// Wait for retries
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
|
||||
// Job should have failed 3 times
|
||||
expect(attemptCount).toBe(3);
|
||||
|
||||
// Check if job was moved to DLQ
|
||||
const dlqStats = await dlqHandler.getStats();
|
||||
expect(dlqStats.total).toBe(1);
|
||||
expect(dlqStats.byJobName['failing-job']).toBe(1);
|
||||
});
|
||||
|
||||
test('should track failure count correctly', async () => {
|
||||
const job = await mainQueue.add('test-job', { data: 'test' });
|
||||
const error = new Error('Test error');
|
||||
|
||||
// Simulate multiple failures
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
|
||||
// On third failure with max attempts reached, should move to DLQ
|
||||
job.attemptsMade = 3;
|
||||
job.opts.attempts = 3;
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Statistics', () => {
|
||||
test('should provide detailed statistics', async () => {
|
||||
// Add some failed jobs to DLQ
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'job-type-a',
|
||||
data: { test: true },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: { message: 'Error 1' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'job-type-b',
|
||||
data: { test: true },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: { message: 'Error 2' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.recent).toBe(2); // Both are recent
|
||||
expect(Object.keys(stats.byJobName).length).toBe(2);
|
||||
expect(stats.oldestJob).toBeDefined();
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
|
||||
test('should count recent jobs correctly', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add old job (25 hours ago)
|
||||
const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000;
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '1', name: 'old-job' },
|
||||
error: { message: 'Old error' },
|
||||
movedToDLQAt: new Date(oldTimestamp).toISOString(),
|
||||
}, { timestamp: oldTimestamp });
|
||||
|
||||
// Add recent job
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '2', name: 'recent-job' },
|
||||
error: { message: 'Recent error' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.recent).toBe(1); // Only one is recent
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Retry', () => {
|
||||
test('should retry jobs from DLQ', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add failed jobs to DLQ
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'retry-job',
|
||||
data: { retry: true },
|
||||
opts: { priority: 1 },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'retry-job-2',
|
||||
data: { retry: true },
|
||||
opts: {},
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Retry jobs
|
||||
const retriedCount = await dlqHandler.retryDLQJobs(10);
|
||||
expect(retriedCount).toBe(2);
|
||||
|
||||
// Check main queue has the retried jobs
|
||||
const mainQueueJobs = await mainQueue.getWaiting();
|
||||
expect(mainQueueJobs.length).toBe(2);
|
||||
expect(mainQueueJobs[0].name).toBe('retry-job');
|
||||
expect(mainQueueJobs[0].data).toEqual({ retry: true });
|
||||
|
||||
// DLQ should be empty
|
||||
const dlqJobs = await dlq.getCompleted();
|
||||
expect(dlqJobs.length).toBe(0);
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
|
||||
test('should respect retry limit', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add 5 failed jobs
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: `${i}`,
|
||||
name: `job-${i}`,
|
||||
data: { index: i },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// Retry only 3 jobs
|
||||
const retriedCount = await dlqHandler.retryDLQJobs(3);
|
||||
expect(retriedCount).toBe(3);
|
||||
|
||||
// Check counts
|
||||
const mainQueueJobs = await mainQueue.getWaiting();
|
||||
expect(mainQueueJobs.length).toBe(3);
|
||||
|
||||
const remainingDLQ = await dlq.getCompleted();
|
||||
expect(remainingDLQ.length).toBe(2);
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Cleanup', () => {
|
||||
test('should cleanup old DLQ entries', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add old job (25 hours ago)
|
||||
const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000;
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '1', name: 'old-job' },
|
||||
error: { message: 'Old error' },
|
||||
}, { timestamp: oldTimestamp });
|
||||
|
||||
// Add recent job (1 hour ago)
|
||||
const recentTimestamp = Date.now() - 1 * 60 * 60 * 1000;
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '2', name: 'recent-job' },
|
||||
error: { message: 'Recent error' },
|
||||
}, { timestamp: recentTimestamp });
|
||||
|
||||
// Run cleanup (24 hour threshold)
|
||||
const removedCount = await dlqHandler.cleanup();
|
||||
expect(removedCount).toBe(1);
|
||||
|
||||
// Check remaining jobs
|
||||
const remaining = await dlq.getCompleted();
|
||||
expect(remaining.length).toBe(1);
|
||||
expect(remaining[0].data.originalJob.name).toBe('recent-job');
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Failed Job Inspection', () => {
|
||||
test('should inspect failed jobs', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add failed jobs with different error types
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'network-job',
|
||||
data: { url: 'https://api.example.com' },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: {
|
||||
message: 'Network timeout',
|
||||
stack: 'Error: Network timeout\n at ...',
|
||||
name: 'NetworkError',
|
||||
},
|
||||
movedToDLQAt: '2024-01-01T10:00:00Z',
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'parse-job',
|
||||
data: { input: 'invalid-json' },
|
||||
attemptsMade: 2,
|
||||
},
|
||||
error: {
|
||||
message: 'Invalid JSON',
|
||||
stack: 'SyntaxError: Invalid JSON\n at ...',
|
||||
name: 'SyntaxError',
|
||||
},
|
||||
movedToDLQAt: '2024-01-01T11:00:00Z',
|
||||
});
|
||||
|
||||
const failedJobs = await dlqHandler.inspectFailedJobs(10);
|
||||
expect(failedJobs.length).toBe(2);
|
||||
|
||||
expect(failedJobs[0]).toMatchObject({
|
||||
id: '1',
|
||||
name: 'network-job',
|
||||
data: { url: 'https://api.example.com' },
|
||||
error: {
|
||||
message: 'Network timeout',
|
||||
name: 'NetworkError',
|
||||
},
|
||||
failedAt: '2024-01-01T10:00:00Z',
|
||||
attempts: 3,
|
||||
});
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Alert Threshold', () => {
|
||||
test('should detect when alert threshold is exceeded', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add jobs to exceed threshold (5)
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: `${i}`,
|
||||
name: `job-${i}`,
|
||||
data: { index: i },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(6);
|
||||
// In a real implementation, this would trigger alerts
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
});
|
||||
import { Queue, Worker } from 'bullmq';
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { DeadLetterQueueHandler } from '../src/dlq-handler';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (
|
||||
message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('DeadLetterQueueHandler', () => {
|
||||
let mainQueue: Queue;
|
||||
let dlqHandler: DeadLetterQueueHandler;
|
||||
let worker: Worker;
|
||||
let connection: any;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Create main queue
|
||||
mainQueue = new Queue('test-queue', { connection });
|
||||
|
||||
// Create DLQ handler
|
||||
dlqHandler = new DeadLetterQueueHandler(mainQueue, connection, {
|
||||
maxRetries: 3,
|
||||
retryDelay: 100,
|
||||
alertThreshold: 5,
|
||||
cleanupAge: 24,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
if (worker) {
|
||||
await worker.close();
|
||||
}
|
||||
await dlqHandler.shutdown();
|
||||
await mainQueue.close();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Failed Job Handling', () => {
|
||||
test('should move job to DLQ after max retries', async () => {
|
||||
let attemptCount = 0;
|
||||
|
||||
// Create worker that always fails
|
||||
worker = new Worker(
|
||||
'test-queue',
|
||||
async () => {
|
||||
attemptCount++;
|
||||
throw new Error('Job failed');
|
||||
},
|
||||
{
|
||||
connection,
|
||||
autorun: false,
|
||||
}
|
||||
);
|
||||
|
||||
// Add job with limited attempts
|
||||
const _job = await mainQueue.add(
|
||||
'failing-job',
|
||||
{ test: true },
|
||||
{
|
||||
attempts: 3,
|
||||
backoff: { type: 'fixed', delay: 50 },
|
||||
}
|
||||
);
|
||||
|
||||
// Process job manually
|
||||
await worker.run();
|
||||
|
||||
// Wait for retries
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
|
||||
// Job should have failed 3 times
|
||||
expect(attemptCount).toBe(3);
|
||||
|
||||
// Check if job was moved to DLQ
|
||||
const dlqStats = await dlqHandler.getStats();
|
||||
expect(dlqStats.total).toBe(1);
|
||||
expect(dlqStats.byJobName['failing-job']).toBe(1);
|
||||
});
|
||||
|
||||
test('should track failure count correctly', async () => {
|
||||
const job = await mainQueue.add('test-job', { data: 'test' });
|
||||
const error = new Error('Test error');
|
||||
|
||||
// Simulate multiple failures
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
|
||||
// On third failure with max attempts reached, should move to DLQ
|
||||
job.attemptsMade = 3;
|
||||
job.opts.attempts = 3;
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Statistics', () => {
|
||||
test('should provide detailed statistics', async () => {
|
||||
// Add some failed jobs to DLQ
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'job-type-a',
|
||||
data: { test: true },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: { message: 'Error 1' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'job-type-b',
|
||||
data: { test: true },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: { message: 'Error 2' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.recent).toBe(2); // Both are recent
|
||||
expect(Object.keys(stats.byJobName).length).toBe(2);
|
||||
expect(stats.oldestJob).toBeDefined();
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
|
||||
test('should count recent jobs correctly', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add old job (25 hours ago)
|
||||
const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000;
|
||||
await dlq.add(
|
||||
'failed-job',
|
||||
{
|
||||
originalJob: { id: '1', name: 'old-job' },
|
||||
error: { message: 'Old error' },
|
||||
movedToDLQAt: new Date(oldTimestamp).toISOString(),
|
||||
},
|
||||
{ timestamp: oldTimestamp }
|
||||
);
|
||||
|
||||
// Add recent job
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '2', name: 'recent-job' },
|
||||
error: { message: 'Recent error' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.recent).toBe(1); // Only one is recent
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Retry', () => {
|
||||
test('should retry jobs from DLQ', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add failed jobs to DLQ
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'retry-job',
|
||||
data: { retry: true },
|
||||
opts: { priority: 1 },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'retry-job-2',
|
||||
data: { retry: true },
|
||||
opts: {},
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Retry jobs
|
||||
const retriedCount = await dlqHandler.retryDLQJobs(10);
|
||||
expect(retriedCount).toBe(2);
|
||||
|
||||
// Check main queue has the retried jobs
|
||||
const mainQueueJobs = await mainQueue.getWaiting();
|
||||
expect(mainQueueJobs.length).toBe(2);
|
||||
expect(mainQueueJobs[0].name).toBe('retry-job');
|
||||
expect(mainQueueJobs[0].data).toEqual({ retry: true });
|
||||
|
||||
// DLQ should be empty
|
||||
const dlqJobs = await dlq.getCompleted();
|
||||
expect(dlqJobs.length).toBe(0);
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
|
||||
test('should respect retry limit', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add 5 failed jobs
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: `${i}`,
|
||||
name: `job-${i}`,
|
||||
data: { index: i },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// Retry only 3 jobs
|
||||
const retriedCount = await dlqHandler.retryDLQJobs(3);
|
||||
expect(retriedCount).toBe(3);
|
||||
|
||||
// Check counts
|
||||
const mainQueueJobs = await mainQueue.getWaiting();
|
||||
expect(mainQueueJobs.length).toBe(3);
|
||||
|
||||
const remainingDLQ = await dlq.getCompleted();
|
||||
expect(remainingDLQ.length).toBe(2);
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Cleanup', () => {
|
||||
test('should cleanup old DLQ entries', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add old job (25 hours ago)
|
||||
const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000;
|
||||
await dlq.add(
|
||||
'failed-job',
|
||||
{
|
||||
originalJob: { id: '1', name: 'old-job' },
|
||||
error: { message: 'Old error' },
|
||||
},
|
||||
{ timestamp: oldTimestamp }
|
||||
);
|
||||
|
||||
// Add recent job (1 hour ago)
|
||||
const recentTimestamp = Date.now() - 1 * 60 * 60 * 1000;
|
||||
await dlq.add(
|
||||
'failed-job',
|
||||
{
|
||||
originalJob: { id: '2', name: 'recent-job' },
|
||||
error: { message: 'Recent error' },
|
||||
},
|
||||
{ timestamp: recentTimestamp }
|
||||
);
|
||||
|
||||
// Run cleanup (24 hour threshold)
|
||||
const removedCount = await dlqHandler.cleanup();
|
||||
expect(removedCount).toBe(1);
|
||||
|
||||
// Check remaining jobs
|
||||
const remaining = await dlq.getCompleted();
|
||||
expect(remaining.length).toBe(1);
|
||||
expect(remaining[0].data.originalJob.name).toBe('recent-job');
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Failed Job Inspection', () => {
|
||||
test('should inspect failed jobs', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add failed jobs with different error types
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'network-job',
|
||||
data: { url: 'https://api.example.com' },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: {
|
||||
message: 'Network timeout',
|
||||
stack: 'Error: Network timeout\n at ...',
|
||||
name: 'NetworkError',
|
||||
},
|
||||
movedToDLQAt: '2024-01-01T10:00:00Z',
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'parse-job',
|
||||
data: { input: 'invalid-json' },
|
||||
attemptsMade: 2,
|
||||
},
|
||||
error: {
|
||||
message: 'Invalid JSON',
|
||||
stack: 'SyntaxError: Invalid JSON\n at ...',
|
||||
name: 'SyntaxError',
|
||||
},
|
||||
movedToDLQAt: '2024-01-01T11:00:00Z',
|
||||
});
|
||||
|
||||
const failedJobs = await dlqHandler.inspectFailedJobs(10);
|
||||
expect(failedJobs.length).toBe(2);
|
||||
|
||||
expect(failedJobs[0]).toMatchObject({
|
||||
id: '1',
|
||||
name: 'network-job',
|
||||
data: { url: 'https://api.example.com' },
|
||||
error: {
|
||||
message: 'Network timeout',
|
||||
name: 'NetworkError',
|
||||
},
|
||||
failedAt: '2024-01-01T10:00:00Z',
|
||||
attempts: 3,
|
||||
});
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Alert Threshold', () => {
|
||||
test('should detect when alert threshold is exceeded', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add jobs to exceed threshold (5)
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: `${i}`,
|
||||
name: `job-${i}`,
|
||||
data: { index: i },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(6);
|
||||
// In a real implementation, this would trigger alerts
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { QueueManager, handlerRegistry } from '../src';
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { handlerRegistry, QueueManager } from '../src';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
if (
|
||||
message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')
|
||||
) {
|
||||
// Suppress these specific Redis errors in tests
|
||||
return;
|
||||
}
|
||||
|
|
@ -34,9 +36,7 @@ describe('QueueManager Integration Tests', () => {
|
|||
try {
|
||||
await Promise.race([
|
||||
queueManager.shutdown(),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Shutdown timeout')), 3000)
|
||||
)
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error('Shutdown timeout')), 3000)),
|
||||
]);
|
||||
} catch (error) {
|
||||
// Ignore shutdown errors in tests
|
||||
|
|
@ -45,10 +45,10 @@ describe('QueueManager Integration Tests', () => {
|
|||
queueManager = null as any;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Clear handler registry to prevent conflicts
|
||||
handlerRegistry.clear();
|
||||
|
||||
|
||||
// Add delay to allow connections to close
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,371 +1,371 @@
|
|||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { handlerRegistry, QueueManager } from '../src';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('QueueManager', () => {
|
||||
let queueManager: QueueManager;
|
||||
|
||||
// Use local Redis/Dragonfly
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
handlerRegistry.clear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (queueManager) {
|
||||
try {
|
||||
await Promise.race([
|
||||
queueManager.shutdown(),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Shutdown timeout')), 3000)
|
||||
)
|
||||
]);
|
||||
} catch (error) {
|
||||
console.warn('Shutdown error:', error.message);
|
||||
} finally {
|
||||
queueManager = null as any;
|
||||
}
|
||||
}
|
||||
|
||||
handlerRegistry.clear();
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
});
|
||||
|
||||
describe('Basic Operations', () => {
|
||||
test('should initialize queue manager', async () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
concurrency: 5,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
expect(queueManager.queueName).toBe('test-queue');
|
||||
});
|
||||
|
||||
test('should add and process a job', async () => {
|
||||
let processedPayload: any;
|
||||
|
||||
// Register handler
|
||||
handlerRegistry.register('test-handler', {
|
||||
'test-operation': async payload => {
|
||||
processedPayload = payload;
|
||||
return { success: true, data: payload };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Add job
|
||||
const job = await queueManager.add('test-job', {
|
||||
handler: 'test-handler',
|
||||
operation: 'test-operation',
|
||||
payload: { message: 'Hello, Queue!' },
|
||||
});
|
||||
|
||||
expect(job.name).toBe('test-job');
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
expect(processedPayload).toEqual({ message: 'Hello, Queue!' });
|
||||
});
|
||||
|
||||
test('should handle missing handler gracefully', async () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
const job = await queueManager.add('test-job', {
|
||||
handler: 'non-existent',
|
||||
operation: 'test-operation',
|
||||
payload: { test: true },
|
||||
});
|
||||
|
||||
// Wait for job to fail
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const failed = await job.isFailed();
|
||||
expect(failed).toBe(true);
|
||||
});
|
||||
|
||||
test('should add multiple jobs in bulk', async () => {
|
||||
let processedCount = 0;
|
||||
|
||||
handlerRegistry.register('bulk-handler', {
|
||||
process: async _payload => {
|
||||
processedCount++;
|
||||
return { processed: true };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 2,
|
||||
concurrency: 5,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
const jobs = await queueManager.addBulk([
|
||||
{
|
||||
name: 'job1',
|
||||
data: { handler: 'bulk-handler', operation: 'process', payload: { id: 1 } },
|
||||
},
|
||||
{
|
||||
name: 'job2',
|
||||
data: { handler: 'bulk-handler', operation: 'process', payload: { id: 2 } },
|
||||
},
|
||||
{
|
||||
name: 'job3',
|
||||
data: { handler: 'bulk-handler', operation: 'process', payload: { id: 3 } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(jobs.length).toBe(3);
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
expect(processedCount).toBe(3);
|
||||
});
|
||||
|
||||
test('should get queue statistics', async () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 0, // No workers, jobs will stay in waiting
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Add some jobs
|
||||
await queueManager.add('job1', {
|
||||
handler: 'test',
|
||||
operation: 'test',
|
||||
payload: { id: 1 },
|
||||
});
|
||||
|
||||
await queueManager.add('job2', {
|
||||
handler: 'test',
|
||||
operation: 'test',
|
||||
payload: { id: 2 },
|
||||
});
|
||||
|
||||
const stats = await queueManager.getStats();
|
||||
|
||||
expect(stats.waiting).toBe(2);
|
||||
expect(stats.active).toBe(0);
|
||||
expect(stats.completed).toBe(0);
|
||||
expect(stats.failed).toBe(0);
|
||||
});
|
||||
|
||||
test('should pause and resume queue', async () => {
|
||||
let processedCount = 0;
|
||||
|
||||
handlerRegistry.register('pause-test', {
|
||||
process: async () => {
|
||||
processedCount++;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Pause queue
|
||||
await queueManager.pause();
|
||||
|
||||
// Add job while paused
|
||||
await queueManager.add('job1', {
|
||||
handler: 'pause-test',
|
||||
operation: 'process',
|
||||
payload: {},
|
||||
});
|
||||
|
||||
// Wait a bit - job should not be processed
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(processedCount).toBe(0);
|
||||
|
||||
// Resume queue
|
||||
await queueManager.resume();
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(processedCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scheduled Jobs', () => {
|
||||
test('should register and process scheduled jobs', async () => {
|
||||
let executionCount = 0;
|
||||
|
||||
handlerRegistry.registerWithSchedule({
|
||||
name: 'scheduled-handler',
|
||||
operations: {
|
||||
'scheduled-task': async _payload => {
|
||||
executionCount++;
|
||||
return { executed: true, timestamp: Date.now() };
|
||||
},
|
||||
},
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'test-schedule',
|
||||
operation: 'scheduled-task',
|
||||
payload: { test: true },
|
||||
cronPattern: '*/1 * * * * *', // Every second
|
||||
description: 'Test scheduled job',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
enableScheduledJobs: true,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Wait for scheduled job to execute
|
||||
await new Promise(resolve => setTimeout(resolve, 2500));
|
||||
|
||||
expect(executionCount).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
test('should handle job errors with retries', async () => {
|
||||
let attemptCount = 0;
|
||||
|
||||
handlerRegistry.register('retry-handler', {
|
||||
'failing-operation': async () => {
|
||||
attemptCount++;
|
||||
if (attemptCount < 3) {
|
||||
throw new Error(`Attempt ${attemptCount} failed`);
|
||||
}
|
||||
return { success: true };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
defaultJobOptions: {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'fixed',
|
||||
delay: 50,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
const job = await queueManager.add('retry-job', {
|
||||
handler: 'retry-handler',
|
||||
operation: 'failing-operation',
|
||||
payload: {},
|
||||
});
|
||||
|
||||
// Wait for retries
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const completed = await job.isCompleted();
|
||||
expect(completed).toBe(true);
|
||||
expect(attemptCount).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Multiple Handlers', () => {
|
||||
test('should handle multiple handlers with different operations', async () => {
|
||||
const results: any[] = [];
|
||||
|
||||
handlerRegistry.register('handler-a', {
|
||||
'operation-1': async payload => {
|
||||
results.push({ handler: 'a', op: '1', payload });
|
||||
return { handler: 'a', op: '1' };
|
||||
},
|
||||
'operation-2': async payload => {
|
||||
results.push({ handler: 'a', op: '2', payload });
|
||||
return { handler: 'a', op: '2' };
|
||||
},
|
||||
});
|
||||
|
||||
handlerRegistry.register('handler-b', {
|
||||
'operation-1': async payload => {
|
||||
results.push({ handler: 'b', op: '1', payload });
|
||||
return { handler: 'b', op: '1' };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 2,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Add jobs for different handlers
|
||||
await queueManager.addBulk([
|
||||
{
|
||||
name: 'job1',
|
||||
data: { handler: 'handler-a', operation: 'operation-1', payload: { id: 1 } },
|
||||
},
|
||||
{
|
||||
name: 'job2',
|
||||
data: { handler: 'handler-a', operation: 'operation-2', payload: { id: 2 } },
|
||||
},
|
||||
{
|
||||
name: 'job3',
|
||||
data: { handler: 'handler-b', operation: 'operation-1', payload: { id: 3 } },
|
||||
},
|
||||
]);
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
expect(results.length).toBe(3);
|
||||
expect(results).toContainEqual({ handler: 'a', op: '1', payload: { id: 1 } });
|
||||
expect(results).toContainEqual({ handler: 'a', op: '2', payload: { id: 2 } });
|
||||
expect(results).toContainEqual({ handler: 'b', op: '1', payload: { id: 3 } });
|
||||
});
|
||||
});
|
||||
});
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { handlerRegistry, QueueManager } from '../src';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (
|
||||
message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('QueueManager', () => {
|
||||
let queueManager: QueueManager;
|
||||
|
||||
// Use local Redis/Dragonfly
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
handlerRegistry.clear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (queueManager) {
|
||||
try {
|
||||
await Promise.race([
|
||||
queueManager.shutdown(),
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error('Shutdown timeout')), 3000)),
|
||||
]);
|
||||
} catch (error) {
|
||||
console.warn('Shutdown error:', error.message);
|
||||
} finally {
|
||||
queueManager = null as any;
|
||||
}
|
||||
}
|
||||
|
||||
handlerRegistry.clear();
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
});
|
||||
|
||||
describe('Basic Operations', () => {
|
||||
test('should initialize queue manager', async () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
concurrency: 5,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
expect(queueManager.queueName).toBe('test-queue');
|
||||
});
|
||||
|
||||
test('should add and process a job', async () => {
|
||||
let processedPayload: any;
|
||||
|
||||
// Register handler
|
||||
handlerRegistry.register('test-handler', {
|
||||
'test-operation': async payload => {
|
||||
processedPayload = payload;
|
||||
return { success: true, data: payload };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Add job
|
||||
const job = await queueManager.add('test-job', {
|
||||
handler: 'test-handler',
|
||||
operation: 'test-operation',
|
||||
payload: { message: 'Hello, Queue!' },
|
||||
});
|
||||
|
||||
expect(job.name).toBe('test-job');
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
expect(processedPayload).toEqual({ message: 'Hello, Queue!' });
|
||||
});
|
||||
|
||||
test('should handle missing handler gracefully', async () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
const job = await queueManager.add('test-job', {
|
||||
handler: 'non-existent',
|
||||
operation: 'test-operation',
|
||||
payload: { test: true },
|
||||
});
|
||||
|
||||
// Wait for job to fail
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const failed = await job.isFailed();
|
||||
expect(failed).toBe(true);
|
||||
});
|
||||
|
||||
test('should add multiple jobs in bulk', async () => {
|
||||
let processedCount = 0;
|
||||
|
||||
handlerRegistry.register('bulk-handler', {
|
||||
process: async _payload => {
|
||||
processedCount++;
|
||||
return { processed: true };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 2,
|
||||
concurrency: 5,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
const jobs = await queueManager.addBulk([
|
||||
{
|
||||
name: 'job1',
|
||||
data: { handler: 'bulk-handler', operation: 'process', payload: { id: 1 } },
|
||||
},
|
||||
{
|
||||
name: 'job2',
|
||||
data: { handler: 'bulk-handler', operation: 'process', payload: { id: 2 } },
|
||||
},
|
||||
{
|
||||
name: 'job3',
|
||||
data: { handler: 'bulk-handler', operation: 'process', payload: { id: 3 } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(jobs.length).toBe(3);
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
expect(processedCount).toBe(3);
|
||||
});
|
||||
|
||||
test('should get queue statistics', async () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 0, // No workers, jobs will stay in waiting
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Add some jobs
|
||||
await queueManager.add('job1', {
|
||||
handler: 'test',
|
||||
operation: 'test',
|
||||
payload: { id: 1 },
|
||||
});
|
||||
|
||||
await queueManager.add('job2', {
|
||||
handler: 'test',
|
||||
operation: 'test',
|
||||
payload: { id: 2 },
|
||||
});
|
||||
|
||||
const stats = await queueManager.getStats();
|
||||
|
||||
expect(stats.waiting).toBe(2);
|
||||
expect(stats.active).toBe(0);
|
||||
expect(stats.completed).toBe(0);
|
||||
expect(stats.failed).toBe(0);
|
||||
});
|
||||
|
||||
test('should pause and resume queue', async () => {
|
||||
let processedCount = 0;
|
||||
|
||||
handlerRegistry.register('pause-test', {
|
||||
process: async () => {
|
||||
processedCount++;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Pause queue
|
||||
await queueManager.pause();
|
||||
|
||||
// Add job while paused
|
||||
await queueManager.add('job1', {
|
||||
handler: 'pause-test',
|
||||
operation: 'process',
|
||||
payload: {},
|
||||
});
|
||||
|
||||
// Wait a bit - job should not be processed
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(processedCount).toBe(0);
|
||||
|
||||
// Resume queue
|
||||
await queueManager.resume();
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(processedCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Scheduled Jobs', () => {
|
||||
test('should register and process scheduled jobs', async () => {
|
||||
let executionCount = 0;
|
||||
|
||||
handlerRegistry.registerWithSchedule({
|
||||
name: 'scheduled-handler',
|
||||
operations: {
|
||||
'scheduled-task': async _payload => {
|
||||
executionCount++;
|
||||
return { executed: true, timestamp: Date.now() };
|
||||
},
|
||||
},
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'test-schedule',
|
||||
operation: 'scheduled-task',
|
||||
payload: { test: true },
|
||||
cronPattern: '*/1 * * * * *', // Every second
|
||||
description: 'Test scheduled job',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
enableScheduledJobs: true,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Wait for scheduled job to execute
|
||||
await new Promise(resolve => setTimeout(resolve, 2500));
|
||||
|
||||
expect(executionCount).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
test('should handle job errors with retries', async () => {
|
||||
let attemptCount = 0;
|
||||
|
||||
handlerRegistry.register('retry-handler', {
|
||||
'failing-operation': async () => {
|
||||
attemptCount++;
|
||||
if (attemptCount < 3) {
|
||||
throw new Error(`Attempt ${attemptCount} failed`);
|
||||
}
|
||||
return { success: true };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 1,
|
||||
defaultJobOptions: {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'fixed',
|
||||
delay: 50,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
const job = await queueManager.add('retry-job', {
|
||||
handler: 'retry-handler',
|
||||
operation: 'failing-operation',
|
||||
payload: {},
|
||||
});
|
||||
|
||||
// Wait for retries
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const completed = await job.isCompleted();
|
||||
expect(completed).toBe(true);
|
||||
expect(attemptCount).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Multiple Handlers', () => {
|
||||
test('should handle multiple handlers with different operations', async () => {
|
||||
const results: any[] = [];
|
||||
|
||||
handlerRegistry.register('handler-a', {
|
||||
'operation-1': async payload => {
|
||||
results.push({ handler: 'a', op: '1', payload });
|
||||
return { handler: 'a', op: '1' };
|
||||
},
|
||||
'operation-2': async payload => {
|
||||
results.push({ handler: 'a', op: '2', payload });
|
||||
return { handler: 'a', op: '2' };
|
||||
},
|
||||
});
|
||||
|
||||
handlerRegistry.register('handler-b', {
|
||||
'operation-1': async payload => {
|
||||
results.push({ handler: 'b', op: '1', payload });
|
||||
return { handler: 'b', op: '1' };
|
||||
},
|
||||
});
|
||||
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
workers: 2,
|
||||
});
|
||||
|
||||
await queueManager.initialize();
|
||||
|
||||
// Add jobs for different handlers
|
||||
await queueManager.addBulk([
|
||||
{
|
||||
name: 'job1',
|
||||
data: { handler: 'handler-a', operation: 'operation-1', payload: { id: 1 } },
|
||||
},
|
||||
{
|
||||
name: 'job2',
|
||||
data: { handler: 'handler-a', operation: 'operation-2', payload: { id: 2 } },
|
||||
},
|
||||
{
|
||||
name: 'job3',
|
||||
data: { handler: 'handler-b', operation: 'operation-1', payload: { id: 3 } },
|
||||
},
|
||||
]);
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
expect(results.length).toBe(3);
|
||||
expect(results).toContainEqual({ handler: 'a', op: '1', payload: { id: 1 } });
|
||||
expect(results).toContainEqual({ handler: 'a', op: '2', payload: { id: 2 } });
|
||||
expect(results).toContainEqual({ handler: 'b', op: '1', payload: { id: 3 } });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,303 +1,327 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { Queue, QueueEvents, Worker } from 'bullmq';
|
||||
import { QueueMetricsCollector } from '../src/queue-metrics';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('QueueMetricsCollector', () => {
|
||||
let queue: Queue;
|
||||
let queueEvents: QueueEvents;
|
||||
let metricsCollector: QueueMetricsCollector;
|
||||
let worker: Worker;
|
||||
let connection: any;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Create queue and events
|
||||
queue = new Queue('metrics-test-queue', { connection });
|
||||
queueEvents = new QueueEvents('metrics-test-queue', { connection });
|
||||
|
||||
// Create metrics collector
|
||||
metricsCollector = new QueueMetricsCollector(queue, queueEvents);
|
||||
|
||||
// Wait for connections
|
||||
await queue.waitUntilReady();
|
||||
await queueEvents.waitUntilReady();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
if (worker) {
|
||||
await worker.close();
|
||||
}
|
||||
await queueEvents.close();
|
||||
await queue.close();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Job Count Metrics', () => {
|
||||
test('should collect basic job counts', async () => {
|
||||
// Add jobs in different states
|
||||
await queue.add('waiting-job', { test: true });
|
||||
await queue.add('delayed-job', { test: true }, { delay: 60000 });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.waiting).toBe(1);
|
||||
expect(metrics.delayed).toBe(1);
|
||||
expect(metrics.active).toBe(0);
|
||||
expect(metrics.completed).toBe(0);
|
||||
expect(metrics.failed).toBe(0);
|
||||
});
|
||||
|
||||
test('should track completed and failed jobs', async () => {
|
||||
let jobCount = 0;
|
||||
|
||||
// Create worker that alternates between success and failure
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
jobCount++;
|
||||
if (jobCount % 2 === 0) {
|
||||
throw new Error('Test failure');
|
||||
}
|
||||
return { success: true };
|
||||
}, { connection });
|
||||
|
||||
// Add jobs
|
||||
await queue.add('job1', { test: 1 });
|
||||
await queue.add('job2', { test: 2 });
|
||||
await queue.add('job3', { test: 3 });
|
||||
await queue.add('job4', { test: 4 });
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.completed).toBe(2);
|
||||
expect(metrics.failed).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Processing Time Metrics', () => {
|
||||
test('should track processing times', async () => {
|
||||
const processingTimes = [50, 100, 150, 200, 250];
|
||||
let jobIndex = 0;
|
||||
|
||||
// Create worker with variable processing times
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
const delay = processingTimes[jobIndex++] || 100;
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
return { processed: true };
|
||||
}, { connection });
|
||||
|
||||
// Add jobs
|
||||
for (let i = 0; i < processingTimes.length; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.processingTime.avg).toBeGreaterThan(0);
|
||||
expect(metrics.processingTime.min).toBeGreaterThanOrEqual(50);
|
||||
expect(metrics.processingTime.max).toBeLessThanOrEqual(300);
|
||||
expect(metrics.processingTime.p95).toBeGreaterThan(metrics.processingTime.avg);
|
||||
});
|
||||
|
||||
test('should handle empty processing times', async () => {
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.processingTime).toEqual({
|
||||
avg: 0,
|
||||
min: 0,
|
||||
max: 0,
|
||||
p95: 0,
|
||||
p99: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Throughput Metrics', () => {
|
||||
test('should calculate throughput correctly', async () => {
|
||||
// Create fast worker
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
return { success: true };
|
||||
}, { connection, concurrency: 5 });
|
||||
|
||||
// Add multiple jobs
|
||||
const jobPromises = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
jobPromises.push(queue.add(`job${i}`, { index: i }));
|
||||
}
|
||||
await Promise.all(jobPromises);
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.throughput.completedPerMinute).toBeGreaterThan(0);
|
||||
expect(metrics.throughput.totalPerMinute).toBe(
|
||||
metrics.throughput.completedPerMinute + metrics.throughput.failedPerMinute
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Queue Health', () => {
|
||||
test('should report healthy queue', async () => {
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(true);
|
||||
expect(metrics.healthIssues).toEqual([]);
|
||||
});
|
||||
|
||||
test('should detect high failure rate', async () => {
|
||||
// Create worker that always fails
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
throw new Error('Always fails');
|
||||
}, { connection });
|
||||
|
||||
// Add jobs
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
// Wait for failures
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(false);
|
||||
expect(metrics.healthIssues).toContain(
|
||||
expect.stringMatching(/High failure rate/)
|
||||
);
|
||||
});
|
||||
|
||||
test('should detect large queue backlog', async () => {
|
||||
// Add many jobs without workers
|
||||
for (let i = 0; i < 1001; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(false);
|
||||
expect(metrics.healthIssues).toContain(
|
||||
expect.stringMatching(/Large queue backlog/)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Oldest Waiting Job', () => {
|
||||
test('should track oldest waiting job', async () => {
|
||||
const beforeAdd = Date.now();
|
||||
|
||||
// Add jobs with delays
|
||||
await queue.add('old-job', { test: true });
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
await queue.add('new-job', { test: true });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.oldestWaitingJob).toBeDefined();
|
||||
expect(metrics.oldestWaitingJob!.getTime()).toBeGreaterThanOrEqual(beforeAdd);
|
||||
});
|
||||
|
||||
test('should return null when no waiting jobs', async () => {
|
||||
// Create worker that processes immediately
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
return { success: true };
|
||||
}, { connection });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
expect(metrics.oldestWaitingJob).toBe(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Metrics Report', () => {
|
||||
test('should generate formatted report', async () => {
|
||||
// Add some jobs
|
||||
await queue.add('job1', { test: true });
|
||||
await queue.add('job2', { test: true }, { delay: 5000 });
|
||||
|
||||
const report = await metricsCollector.getReport();
|
||||
|
||||
expect(report).toContain('Queue Metrics Report');
|
||||
expect(report).toContain('Status:');
|
||||
expect(report).toContain('Job Counts:');
|
||||
expect(report).toContain('Performance:');
|
||||
expect(report).toContain('Throughput:');
|
||||
expect(report).toContain('Waiting: 1');
|
||||
expect(report).toContain('Delayed: 1');
|
||||
});
|
||||
|
||||
test('should include health issues in report', async () => {
|
||||
// Add many jobs to trigger health issue
|
||||
for (let i = 0; i < 1001; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
const report = await metricsCollector.getReport();
|
||||
|
||||
expect(report).toContain('Issues Detected');
|
||||
expect(report).toContain('Health Issues:');
|
||||
expect(report).toContain('Large queue backlog');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Prometheus Metrics', () => {
|
||||
test('should export metrics in Prometheus format', async () => {
|
||||
// Add some jobs and process them
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
return { success: true };
|
||||
}, { connection });
|
||||
|
||||
await queue.add('job1', { test: true });
|
||||
await queue.add('job2', { test: true });
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
const prometheusMetrics = await metricsCollector.getPrometheusMetrics();
|
||||
|
||||
// Check format
|
||||
expect(prometheusMetrics).toContain('# HELP queue_jobs_total');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_jobs_total gauge');
|
||||
expect(prometheusMetrics).toContain('queue_jobs_total{queue="metrics-test-queue",status="completed"}');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_processing_time_seconds');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_processing_time_seconds summary');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_throughput_per_minute');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_throughput_per_minute gauge');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_health');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_health gauge');
|
||||
});
|
||||
});
|
||||
});
|
||||
import { Queue, QueueEvents, Worker } from 'bullmq';
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { QueueMetricsCollector } from '../src/queue-metrics';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (
|
||||
message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('QueueMetricsCollector', () => {
|
||||
let queue: Queue;
|
||||
let queueEvents: QueueEvents;
|
||||
let metricsCollector: QueueMetricsCollector;
|
||||
let worker: Worker;
|
||||
let connection: any;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Create queue and events
|
||||
queue = new Queue('metrics-test-queue', { connection });
|
||||
queueEvents = new QueueEvents('metrics-test-queue', { connection });
|
||||
|
||||
// Create metrics collector
|
||||
metricsCollector = new QueueMetricsCollector(queue, queueEvents);
|
||||
|
||||
// Wait for connections
|
||||
await queue.waitUntilReady();
|
||||
await queueEvents.waitUntilReady();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
if (worker) {
|
||||
await worker.close();
|
||||
}
|
||||
await queueEvents.close();
|
||||
await queue.close();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Job Count Metrics', () => {
|
||||
test('should collect basic job counts', async () => {
|
||||
// Add jobs in different states
|
||||
await queue.add('waiting-job', { test: true });
|
||||
await queue.add('delayed-job', { test: true }, { delay: 60000 });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.waiting).toBe(1);
|
||||
expect(metrics.delayed).toBe(1);
|
||||
expect(metrics.active).toBe(0);
|
||||
expect(metrics.completed).toBe(0);
|
||||
expect(metrics.failed).toBe(0);
|
||||
});
|
||||
|
||||
test('should track completed and failed jobs', async () => {
|
||||
let jobCount = 0;
|
||||
|
||||
// Create worker that alternates between success and failure
|
||||
worker = new Worker(
|
||||
'metrics-test-queue',
|
||||
async () => {
|
||||
jobCount++;
|
||||
if (jobCount % 2 === 0) {
|
||||
throw new Error('Test failure');
|
||||
}
|
||||
return { success: true };
|
||||
},
|
||||
{ connection }
|
||||
);
|
||||
|
||||
// Add jobs
|
||||
await queue.add('job1', { test: 1 });
|
||||
await queue.add('job2', { test: 2 });
|
||||
await queue.add('job3', { test: 3 });
|
||||
await queue.add('job4', { test: 4 });
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.completed).toBe(2);
|
||||
expect(metrics.failed).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Processing Time Metrics', () => {
|
||||
test('should track processing times', async () => {
|
||||
const processingTimes = [50, 100, 150, 200, 250];
|
||||
let jobIndex = 0;
|
||||
|
||||
// Create worker with variable processing times
|
||||
worker = new Worker(
|
||||
'metrics-test-queue',
|
||||
async () => {
|
||||
const delay = processingTimes[jobIndex++] || 100;
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
return { processed: true };
|
||||
},
|
||||
{ connection }
|
||||
);
|
||||
|
||||
// Add jobs
|
||||
for (let i = 0; i < processingTimes.length; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.processingTime.avg).toBeGreaterThan(0);
|
||||
expect(metrics.processingTime.min).toBeGreaterThanOrEqual(50);
|
||||
expect(metrics.processingTime.max).toBeLessThanOrEqual(300);
|
||||
expect(metrics.processingTime.p95).toBeGreaterThan(metrics.processingTime.avg);
|
||||
});
|
||||
|
||||
test('should handle empty processing times', async () => {
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.processingTime).toEqual({
|
||||
avg: 0,
|
||||
min: 0,
|
||||
max: 0,
|
||||
p95: 0,
|
||||
p99: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Throughput Metrics', () => {
|
||||
test('should calculate throughput correctly', async () => {
|
||||
// Create fast worker
|
||||
worker = new Worker(
|
||||
'metrics-test-queue',
|
||||
async () => {
|
||||
return { success: true };
|
||||
},
|
||||
{ connection, concurrency: 5 }
|
||||
);
|
||||
|
||||
// Add multiple jobs
|
||||
const jobPromises = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
jobPromises.push(queue.add(`job${i}`, { index: i }));
|
||||
}
|
||||
await Promise.all(jobPromises);
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.throughput.completedPerMinute).toBeGreaterThan(0);
|
||||
expect(metrics.throughput.totalPerMinute).toBe(
|
||||
metrics.throughput.completedPerMinute + metrics.throughput.failedPerMinute
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Queue Health', () => {
|
||||
test('should report healthy queue', async () => {
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(true);
|
||||
expect(metrics.healthIssues).toEqual([]);
|
||||
});
|
||||
|
||||
test('should detect high failure rate', async () => {
|
||||
// Create worker that always fails
|
||||
worker = new Worker(
|
||||
'metrics-test-queue',
|
||||
async () => {
|
||||
throw new Error('Always fails');
|
||||
},
|
||||
{ connection }
|
||||
);
|
||||
|
||||
// Add jobs
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
// Wait for failures
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(false);
|
||||
expect(metrics.healthIssues).toContain(expect.stringMatching(/High failure rate/));
|
||||
});
|
||||
|
||||
test('should detect large queue backlog', async () => {
|
||||
// Add many jobs without workers
|
||||
for (let i = 0; i < 1001; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(false);
|
||||
expect(metrics.healthIssues).toContain(expect.stringMatching(/Large queue backlog/));
|
||||
});
|
||||
});
|
||||
|
||||
describe('Oldest Waiting Job', () => {
|
||||
test('should track oldest waiting job', async () => {
|
||||
const beforeAdd = Date.now();
|
||||
|
||||
// Add jobs with delays
|
||||
await queue.add('old-job', { test: true });
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
await queue.add('new-job', { test: true });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.oldestWaitingJob).toBeDefined();
|
||||
expect(metrics.oldestWaitingJob!.getTime()).toBeGreaterThanOrEqual(beforeAdd);
|
||||
});
|
||||
|
||||
test('should return null when no waiting jobs', async () => {
|
||||
// Create worker that processes immediately
|
||||
worker = new Worker(
|
||||
'metrics-test-queue',
|
||||
async () => {
|
||||
return { success: true };
|
||||
},
|
||||
{ connection }
|
||||
);
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
expect(metrics.oldestWaitingJob).toBe(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Metrics Report', () => {
|
||||
test('should generate formatted report', async () => {
|
||||
// Add some jobs
|
||||
await queue.add('job1', { test: true });
|
||||
await queue.add('job2', { test: true }, { delay: 5000 });
|
||||
|
||||
const report = await metricsCollector.getReport();
|
||||
|
||||
expect(report).toContain('Queue Metrics Report');
|
||||
expect(report).toContain('Status:');
|
||||
expect(report).toContain('Job Counts:');
|
||||
expect(report).toContain('Performance:');
|
||||
expect(report).toContain('Throughput:');
|
||||
expect(report).toContain('Waiting: 1');
|
||||
expect(report).toContain('Delayed: 1');
|
||||
});
|
||||
|
||||
test('should include health issues in report', async () => {
|
||||
// Add many jobs to trigger health issue
|
||||
for (let i = 0; i < 1001; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
const report = await metricsCollector.getReport();
|
||||
|
||||
expect(report).toContain('Issues Detected');
|
||||
expect(report).toContain('Health Issues:');
|
||||
expect(report).toContain('Large queue backlog');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Prometheus Metrics', () => {
|
||||
test('should export metrics in Prometheus format', async () => {
|
||||
// Add some jobs and process them
|
||||
worker = new Worker(
|
||||
'metrics-test-queue',
|
||||
async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
return { success: true };
|
||||
},
|
||||
{ connection }
|
||||
);
|
||||
|
||||
await queue.add('job1', { test: true });
|
||||
await queue.add('job2', { test: true });
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
const prometheusMetrics = await metricsCollector.getPrometheusMetrics();
|
||||
|
||||
// Check format
|
||||
expect(prometheusMetrics).toContain('# HELP queue_jobs_total');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_jobs_total gauge');
|
||||
expect(prometheusMetrics).toContain(
|
||||
'queue_jobs_total{queue="metrics-test-queue",status="completed"}'
|
||||
);
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_processing_time_seconds');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_processing_time_seconds summary');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_throughput_per_minute');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_throughput_per_minute gauge');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_health');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_health gauge');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,81 +1,81 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { QueueManager, handlerRegistry } from '../src';
|
||||
|
||||
describe('QueueManager Simple Tests', () => {
|
||||
let queueManager: QueueManager;
|
||||
|
||||
// Assumes Redis is running locally on default port
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
handlerRegistry.clear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (queueManager) {
|
||||
try {
|
||||
await queueManager.shutdown();
|
||||
} catch {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('should create queue manager instance', () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
});
|
||||
|
||||
expect(queueManager.queueName).toBe('test-queue');
|
||||
});
|
||||
|
||||
test('should handle missing Redis gracefully', async () => {
|
||||
// Use a port that's likely not running Redis
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: {
|
||||
host: 'localhost',
|
||||
port: 9999,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(queueManager.initialize()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test('handler registry should work', () => {
|
||||
const testHandler = async (payload: any) => {
|
||||
return { success: true, payload };
|
||||
};
|
||||
|
||||
handlerRegistry.register('test-handler', {
|
||||
'test-op': testHandler,
|
||||
});
|
||||
|
||||
const handler = handlerRegistry.getHandler('test-handler', 'test-op');
|
||||
expect(handler).toBe(testHandler);
|
||||
});
|
||||
|
||||
test('handler registry should return null for missing handler', () => {
|
||||
const handler = handlerRegistry.getHandler('missing', 'op');
|
||||
expect(handler).toBe(null);
|
||||
});
|
||||
|
||||
test('should get handler statistics', () => {
|
||||
handlerRegistry.register('handler1', {
|
||||
'op1': async () => ({}),
|
||||
'op2': async () => ({}),
|
||||
});
|
||||
|
||||
handlerRegistry.register('handler2', {
|
||||
'op1': async () => ({}),
|
||||
});
|
||||
|
||||
const stats = handlerRegistry.getStats();
|
||||
expect(stats.handlers).toBe(2);
|
||||
expect(stats.totalOperations).toBe(3);
|
||||
});
|
||||
});
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { handlerRegistry, QueueManager } from '../src';
|
||||
|
||||
describe('QueueManager Simple Tests', () => {
|
||||
let queueManager: QueueManager;
|
||||
|
||||
// Assumes Redis is running locally on default port
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
handlerRegistry.clear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (queueManager) {
|
||||
try {
|
||||
await queueManager.shutdown();
|
||||
} catch {
|
||||
// Ignore errors during cleanup
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('should create queue manager instance', () => {
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: redisConfig,
|
||||
});
|
||||
|
||||
expect(queueManager.queueName).toBe('test-queue');
|
||||
});
|
||||
|
||||
test('should handle missing Redis gracefully', async () => {
|
||||
// Use a port that's likely not running Redis
|
||||
queueManager = new QueueManager({
|
||||
queueName: 'test-queue',
|
||||
redis: {
|
||||
host: 'localhost',
|
||||
port: 9999,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(queueManager.initialize()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test('handler registry should work', () => {
|
||||
const testHandler = async (payload: any) => {
|
||||
return { success: true, payload };
|
||||
};
|
||||
|
||||
handlerRegistry.register('test-handler', {
|
||||
'test-op': testHandler,
|
||||
});
|
||||
|
||||
const handler = handlerRegistry.getHandler('test-handler', 'test-op');
|
||||
expect(handler).toBe(testHandler);
|
||||
});
|
||||
|
||||
test('handler registry should return null for missing handler', () => {
|
||||
const handler = handlerRegistry.getHandler('missing', 'op');
|
||||
expect(handler).toBe(null);
|
||||
});
|
||||
|
||||
test('should get handler statistics', () => {
|
||||
handlerRegistry.register('handler1', {
|
||||
op1: async () => ({}),
|
||||
op2: async () => ({}),
|
||||
});
|
||||
|
||||
handlerRegistry.register('handler2', {
|
||||
op1: async () => ({}),
|
||||
});
|
||||
|
||||
const stats = handlerRegistry.getStats();
|
||||
expect(stats.handlers).toBe(2);
|
||||
expect(stats.totalOperations).toBe(3);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,309 +1,311 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { QueueRateLimiter } from '../src/rate-limiter';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
import Redis from 'ioredis';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('QueueRateLimiter', () => {
|
||||
let redisClient: Redis;
|
||||
let rateLimiter: QueueRateLimiter;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
// Create Redis client
|
||||
redisClient = new Redis(getRedisConnection(redisConfig));
|
||||
|
||||
// Clear Redis keys for tests
|
||||
try {
|
||||
const keys = await redisClient.keys('rl:*');
|
||||
if (keys.length > 0) {
|
||||
await redisClient.del(...keys);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
rateLimiter = new QueueRateLimiter(redisClient);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (redisClient) {
|
||||
try {
|
||||
await redisClient.quit();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Rate Limit Rules', () => {
|
||||
test('should add and enforce global rate limit', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: {
|
||||
points: 5,
|
||||
duration: 1, // 1 second
|
||||
},
|
||||
});
|
||||
|
||||
// Consume 5 points
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const result = await rateLimiter.checkLimit('any-handler', 'any-operation');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
// 6th request should be blocked
|
||||
const blocked = await rateLimiter.checkLimit('any-handler', 'any-operation');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
expect(blocked.retryAfter).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should add and enforce handler-level rate limit', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'api-handler',
|
||||
config: {
|
||||
points: 3,
|
||||
duration: 1,
|
||||
},
|
||||
});
|
||||
|
||||
// api-handler should be limited
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const result = await rateLimiter.checkLimit('api-handler', 'any-operation');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
const blocked = await rateLimiter.checkLimit('api-handler', 'any-operation');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Other handlers should not be limited
|
||||
const otherHandler = await rateLimiter.checkLimit('other-handler', 'any-operation');
|
||||
expect(otherHandler.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should add and enforce operation-level rate limit', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'operation',
|
||||
handler: 'data-handler',
|
||||
operation: 'fetch-prices',
|
||||
config: {
|
||||
points: 2,
|
||||
duration: 1,
|
||||
},
|
||||
});
|
||||
|
||||
// Specific operation should be limited
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const result = await rateLimiter.checkLimit('data-handler', 'fetch-prices');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
const blocked = await rateLimiter.checkLimit('data-handler', 'fetch-prices');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Other operations on same handler should work
|
||||
const otherOp = await rateLimiter.checkLimit('data-handler', 'fetch-volume');
|
||||
expect(otherOp.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should enforce multiple rate limits (most restrictive wins)', async () => {
|
||||
// Global: 10/sec
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 10, duration: 1 },
|
||||
});
|
||||
|
||||
// Handler: 5/sec
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'test-handler',
|
||||
config: { points: 5, duration: 1 },
|
||||
});
|
||||
|
||||
// Operation: 2/sec
|
||||
rateLimiter.addRule({
|
||||
level: 'operation',
|
||||
handler: 'test-handler',
|
||||
operation: 'test-op',
|
||||
config: { points: 2, duration: 1 },
|
||||
});
|
||||
|
||||
// Should be limited by operation level (most restrictive)
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const result = await rateLimiter.checkLimit('test-handler', 'test-op');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
const blocked = await rateLimiter.checkLimit('test-handler', 'test-op');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Rate Limit Status', () => {
|
||||
test('should get rate limit status', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'status-test',
|
||||
config: { points: 10, duration: 60 },
|
||||
});
|
||||
|
||||
// Consume some points
|
||||
await rateLimiter.checkLimit('status-test', 'operation');
|
||||
await rateLimiter.checkLimit('status-test', 'operation');
|
||||
|
||||
const status = await rateLimiter.getStatus('status-test', 'operation');
|
||||
expect(status.handler).toBe('status-test');
|
||||
expect(status.operation).toBe('operation');
|
||||
expect(status.limits.length).toBe(1);
|
||||
expect(status.limits[0].points).toBe(10);
|
||||
expect(status.limits[0].remaining).toBe(8);
|
||||
});
|
||||
|
||||
test('should show multiple applicable limits in status', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 100, duration: 60 },
|
||||
});
|
||||
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'multi-test',
|
||||
config: { points: 50, duration: 60 },
|
||||
});
|
||||
|
||||
const status = await rateLimiter.getStatus('multi-test', 'operation');
|
||||
expect(status.limits.length).toBe(2);
|
||||
|
||||
const globalLimit = status.limits.find(l => l.level === 'global');
|
||||
const handlerLimit = status.limits.find(l => l.level === 'handler');
|
||||
|
||||
expect(globalLimit?.points).toBe(100);
|
||||
expect(handlerLimit?.points).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Rate Limit Management', () => {
|
||||
test('should reset rate limits', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'reset-test',
|
||||
config: { points: 1, duration: 60 },
|
||||
});
|
||||
|
||||
// Consume the limit
|
||||
await rateLimiter.checkLimit('reset-test', 'operation');
|
||||
const blocked = await rateLimiter.checkLimit('reset-test', 'operation');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Reset limits
|
||||
await rateLimiter.reset('reset-test');
|
||||
|
||||
// Should be allowed again
|
||||
const afterReset = await rateLimiter.checkLimit('reset-test', 'operation');
|
||||
expect(afterReset.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should get all rules', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 100, duration: 60 },
|
||||
});
|
||||
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'test',
|
||||
config: { points: 50, duration: 60 },
|
||||
});
|
||||
|
||||
const rules = rateLimiter.getRules();
|
||||
expect(rules.length).toBe(2);
|
||||
expect(rules[0].level).toBe('global');
|
||||
expect(rules[1].level).toBe('handler');
|
||||
});
|
||||
|
||||
test('should remove specific rule', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'remove-test',
|
||||
config: { points: 1, duration: 1 },
|
||||
});
|
||||
|
||||
// Verify rule exists
|
||||
await rateLimiter.checkLimit('remove-test', 'op');
|
||||
const blocked = await rateLimiter.checkLimit('remove-test', 'op');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Remove rule
|
||||
const removed = rateLimiter.removeRule('handler', 'remove-test');
|
||||
expect(removed).toBe(true);
|
||||
|
||||
// Should not be limited anymore
|
||||
const afterRemove = await rateLimiter.checkLimit('remove-test', 'op');
|
||||
expect(afterRemove.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Block Duration', () => {
|
||||
test('should block for specified duration after limit exceeded', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'block-test',
|
||||
config: {
|
||||
points: 1,
|
||||
duration: 1,
|
||||
blockDuration: 2, // Block for 2 seconds
|
||||
},
|
||||
});
|
||||
|
||||
// Consume limit
|
||||
await rateLimiter.checkLimit('block-test', 'op');
|
||||
|
||||
// Should be blocked
|
||||
const blocked = await rateLimiter.checkLimit('block-test', 'op');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
expect(blocked.retryAfter).toBeGreaterThanOrEqual(1000); // At least 1 second
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
test('should allow requests when rate limiter fails', async () => {
|
||||
// Create a rate limiter with invalid redis client
|
||||
const badRedis = new Redis({
|
||||
host: 'invalid-host',
|
||||
port: 9999,
|
||||
retryStrategy: () => null, // Disable retries
|
||||
});
|
||||
|
||||
const failingLimiter = new QueueRateLimiter(badRedis);
|
||||
|
||||
failingLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 1, duration: 1 },
|
||||
});
|
||||
|
||||
// Should allow even though Redis is not available
|
||||
const result = await failingLimiter.checkLimit('test', 'test');
|
||||
expect(result.allowed).toBe(true);
|
||||
|
||||
badRedis.disconnect();
|
||||
});
|
||||
});
|
||||
});
|
||||
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import Redis from 'ioredis';
|
||||
import { QueueRateLimiter } from '../src/rate-limiter';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (
|
||||
message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('QueueRateLimiter', () => {
|
||||
let redisClient: Redis;
|
||||
let rateLimiter: QueueRateLimiter;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
// Create Redis client
|
||||
redisClient = new Redis(getRedisConnection(redisConfig));
|
||||
|
||||
// Clear Redis keys for tests
|
||||
try {
|
||||
const keys = await redisClient.keys('rl:*');
|
||||
if (keys.length > 0) {
|
||||
await redisClient.del(...keys);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
rateLimiter = new QueueRateLimiter(redisClient);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (redisClient) {
|
||||
try {
|
||||
await redisClient.quit();
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Rate Limit Rules', () => {
|
||||
test('should add and enforce global rate limit', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: {
|
||||
points: 5,
|
||||
duration: 1, // 1 second
|
||||
},
|
||||
});
|
||||
|
||||
// Consume 5 points
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const result = await rateLimiter.checkLimit('any-handler', 'any-operation');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
// 6th request should be blocked
|
||||
const blocked = await rateLimiter.checkLimit('any-handler', 'any-operation');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
expect(blocked.retryAfter).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('should add and enforce handler-level rate limit', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'api-handler',
|
||||
config: {
|
||||
points: 3,
|
||||
duration: 1,
|
||||
},
|
||||
});
|
||||
|
||||
// api-handler should be limited
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const result = await rateLimiter.checkLimit('api-handler', 'any-operation');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
const blocked = await rateLimiter.checkLimit('api-handler', 'any-operation');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Other handlers should not be limited
|
||||
const otherHandler = await rateLimiter.checkLimit('other-handler', 'any-operation');
|
||||
expect(otherHandler.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should add and enforce operation-level rate limit', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'operation',
|
||||
handler: 'data-handler',
|
||||
operation: 'fetch-prices',
|
||||
config: {
|
||||
points: 2,
|
||||
duration: 1,
|
||||
},
|
||||
});
|
||||
|
||||
// Specific operation should be limited
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const result = await rateLimiter.checkLimit('data-handler', 'fetch-prices');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
const blocked = await rateLimiter.checkLimit('data-handler', 'fetch-prices');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Other operations on same handler should work
|
||||
const otherOp = await rateLimiter.checkLimit('data-handler', 'fetch-volume');
|
||||
expect(otherOp.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should enforce multiple rate limits (most restrictive wins)', async () => {
|
||||
// Global: 10/sec
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 10, duration: 1 },
|
||||
});
|
||||
|
||||
// Handler: 5/sec
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'test-handler',
|
||||
config: { points: 5, duration: 1 },
|
||||
});
|
||||
|
||||
// Operation: 2/sec
|
||||
rateLimiter.addRule({
|
||||
level: 'operation',
|
||||
handler: 'test-handler',
|
||||
operation: 'test-op',
|
||||
config: { points: 2, duration: 1 },
|
||||
});
|
||||
|
||||
// Should be limited by operation level (most restrictive)
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const result = await rateLimiter.checkLimit('test-handler', 'test-op');
|
||||
expect(result.allowed).toBe(true);
|
||||
}
|
||||
|
||||
const blocked = await rateLimiter.checkLimit('test-handler', 'test-op');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Rate Limit Status', () => {
|
||||
test('should get rate limit status', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'status-test',
|
||||
config: { points: 10, duration: 60 },
|
||||
});
|
||||
|
||||
// Consume some points
|
||||
await rateLimiter.checkLimit('status-test', 'operation');
|
||||
await rateLimiter.checkLimit('status-test', 'operation');
|
||||
|
||||
const status = await rateLimiter.getStatus('status-test', 'operation');
|
||||
expect(status.handler).toBe('status-test');
|
||||
expect(status.operation).toBe('operation');
|
||||
expect(status.limits.length).toBe(1);
|
||||
expect(status.limits[0].points).toBe(10);
|
||||
expect(status.limits[0].remaining).toBe(8);
|
||||
});
|
||||
|
||||
test('should show multiple applicable limits in status', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 100, duration: 60 },
|
||||
});
|
||||
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'multi-test',
|
||||
config: { points: 50, duration: 60 },
|
||||
});
|
||||
|
||||
const status = await rateLimiter.getStatus('multi-test', 'operation');
|
||||
expect(status.limits.length).toBe(2);
|
||||
|
||||
const globalLimit = status.limits.find(l => l.level === 'global');
|
||||
const handlerLimit = status.limits.find(l => l.level === 'handler');
|
||||
|
||||
expect(globalLimit?.points).toBe(100);
|
||||
expect(handlerLimit?.points).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Rate Limit Management', () => {
|
||||
test('should reset rate limits', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'reset-test',
|
||||
config: { points: 1, duration: 60 },
|
||||
});
|
||||
|
||||
// Consume the limit
|
||||
await rateLimiter.checkLimit('reset-test', 'operation');
|
||||
const blocked = await rateLimiter.checkLimit('reset-test', 'operation');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Reset limits
|
||||
await rateLimiter.reset('reset-test');
|
||||
|
||||
// Should be allowed again
|
||||
const afterReset = await rateLimiter.checkLimit('reset-test', 'operation');
|
||||
expect(afterReset.allowed).toBe(true);
|
||||
});
|
||||
|
||||
test('should get all rules', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 100, duration: 60 },
|
||||
});
|
||||
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'test',
|
||||
config: { points: 50, duration: 60 },
|
||||
});
|
||||
|
||||
const rules = rateLimiter.getRules();
|
||||
expect(rules.length).toBe(2);
|
||||
expect(rules[0].level).toBe('global');
|
||||
expect(rules[1].level).toBe('handler');
|
||||
});
|
||||
|
||||
test('should remove specific rule', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'remove-test',
|
||||
config: { points: 1, duration: 1 },
|
||||
});
|
||||
|
||||
// Verify rule exists
|
||||
await rateLimiter.checkLimit('remove-test', 'op');
|
||||
const blocked = await rateLimiter.checkLimit('remove-test', 'op');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
|
||||
// Remove rule
|
||||
const removed = rateLimiter.removeRule('handler', 'remove-test');
|
||||
expect(removed).toBe(true);
|
||||
|
||||
// Should not be limited anymore
|
||||
const afterRemove = await rateLimiter.checkLimit('remove-test', 'op');
|
||||
expect(afterRemove.allowed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Block Duration', () => {
|
||||
test('should block for specified duration after limit exceeded', async () => {
|
||||
rateLimiter.addRule({
|
||||
level: 'handler',
|
||||
handler: 'block-test',
|
||||
config: {
|
||||
points: 1,
|
||||
duration: 1,
|
||||
blockDuration: 2, // Block for 2 seconds
|
||||
},
|
||||
});
|
||||
|
||||
// Consume limit
|
||||
await rateLimiter.checkLimit('block-test', 'op');
|
||||
|
||||
// Should be blocked
|
||||
const blocked = await rateLimiter.checkLimit('block-test', 'op');
|
||||
expect(blocked.allowed).toBe(false);
|
||||
expect(blocked.retryAfter).toBeGreaterThanOrEqual(1000); // At least 1 second
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
test('should allow requests when rate limiter fails', async () => {
|
||||
// Create a rate limiter with invalid redis client
|
||||
const badRedis = new Redis({
|
||||
host: 'invalid-host',
|
||||
port: 9999,
|
||||
retryStrategy: () => null, // Disable retries
|
||||
});
|
||||
|
||||
const failingLimiter = new QueueRateLimiter(badRedis);
|
||||
|
||||
failingLimiter.addRule({
|
||||
level: 'global',
|
||||
config: { points: 1, duration: 1 },
|
||||
});
|
||||
|
||||
// Should allow even though Redis is not available
|
||||
const result = await failingLimiter.checkLimit('test', 'test');
|
||||
expect(result.allowed).toBe(true);
|
||||
|
||||
badRedis.disconnect();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue