reworked queue lib
This commit is contained in:
parent
629ba2b8d4
commit
c05a7413dc
34 changed files with 3887 additions and 861 deletions
357
libs/queue/test/dlq-handler.test.ts
Normal file
357
libs/queue/test/dlq-handler.test.ts
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { Queue, Worker, Job } from 'bullmq';
|
||||
import { DeadLetterQueueHandler } from '../src/dlq-handler';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('DeadLetterQueueHandler', () => {
|
||||
let mainQueue: Queue;
|
||||
let dlqHandler: DeadLetterQueueHandler;
|
||||
let worker: Worker;
|
||||
let connection: any;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Create main queue
|
||||
mainQueue = new Queue('test-queue', { connection });
|
||||
|
||||
// Create DLQ handler
|
||||
dlqHandler = new DeadLetterQueueHandler(mainQueue, connection, {
|
||||
maxRetries: 3,
|
||||
retryDelay: 100,
|
||||
alertThreshold: 5,
|
||||
cleanupAge: 24,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
if (worker) {
|
||||
await worker.close();
|
||||
}
|
||||
await dlqHandler.shutdown();
|
||||
await mainQueue.close();
|
||||
} catch (error) {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Failed Job Handling', () => {
|
||||
test('should move job to DLQ after max retries', async () => {
|
||||
let attemptCount = 0;
|
||||
|
||||
// Create worker that always fails
|
||||
worker = new Worker('test-queue', async () => {
|
||||
attemptCount++;
|
||||
throw new Error('Job failed');
|
||||
}, {
|
||||
connection,
|
||||
autorun: false,
|
||||
});
|
||||
|
||||
// Add job with limited attempts
|
||||
const job = await mainQueue.add('failing-job', { test: true }, {
|
||||
attempts: 3,
|
||||
backoff: { type: 'fixed', delay: 50 },
|
||||
});
|
||||
|
||||
// Process job manually
|
||||
await worker.run();
|
||||
|
||||
// Wait for retries
|
||||
await new Promise(resolve => setTimeout(resolve, 300));
|
||||
|
||||
// Job should have failed 3 times
|
||||
expect(attemptCount).toBe(3);
|
||||
|
||||
// Check if job was moved to DLQ
|
||||
const dlqStats = await dlqHandler.getStats();
|
||||
expect(dlqStats.total).toBe(1);
|
||||
expect(dlqStats.byJobName['failing-job']).toBe(1);
|
||||
});
|
||||
|
||||
test('should track failure count correctly', async () => {
|
||||
const job = await mainQueue.add('test-job', { data: 'test' });
|
||||
const error = new Error('Test error');
|
||||
|
||||
// Simulate multiple failures
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
|
||||
// On third failure with max attempts reached, should move to DLQ
|
||||
job.attemptsMade = 3;
|
||||
job.opts.attempts = 3;
|
||||
await dlqHandler.handleFailedJob(job, error);
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Statistics', () => {
|
||||
test('should provide detailed statistics', async () => {
|
||||
// Add some failed jobs to DLQ
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'job-type-a',
|
||||
data: { test: true },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: { message: 'Error 1' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'job-type-b',
|
||||
data: { test: true },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: { message: 'Error 2' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.recent).toBe(2); // Both are recent
|
||||
expect(Object.keys(stats.byJobName).length).toBe(2);
|
||||
expect(stats.oldestJob).toBeDefined();
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
|
||||
test('should count recent jobs correctly', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add old job (25 hours ago)
|
||||
const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000;
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '1', name: 'old-job' },
|
||||
error: { message: 'Old error' },
|
||||
movedToDLQAt: new Date(oldTimestamp).toISOString(),
|
||||
}, { timestamp: oldTimestamp });
|
||||
|
||||
// Add recent job
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '2', name: 'recent-job' },
|
||||
error: { message: 'Recent error' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(2);
|
||||
expect(stats.recent).toBe(1); // Only one is recent
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Retry', () => {
|
||||
test('should retry jobs from DLQ', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add failed jobs to DLQ
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'retry-job',
|
||||
data: { retry: true },
|
||||
opts: { priority: 1 },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'retry-job-2',
|
||||
data: { retry: true },
|
||||
opts: {},
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Retry jobs
|
||||
const retriedCount = await dlqHandler.retryDLQJobs(10);
|
||||
expect(retriedCount).toBe(2);
|
||||
|
||||
// Check main queue has the retried jobs
|
||||
const mainQueueJobs = await mainQueue.getWaiting();
|
||||
expect(mainQueueJobs.length).toBe(2);
|
||||
expect(mainQueueJobs[0].name).toBe('retry-job');
|
||||
expect(mainQueueJobs[0].data).toEqual({ retry: true });
|
||||
|
||||
// DLQ should be empty
|
||||
const dlqJobs = await dlq.getCompleted();
|
||||
expect(dlqJobs.length).toBe(0);
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
|
||||
test('should respect retry limit', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add 5 failed jobs
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: `${i}`,
|
||||
name: `job-${i}`,
|
||||
data: { index: i },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
// Retry only 3 jobs
|
||||
const retriedCount = await dlqHandler.retryDLQJobs(3);
|
||||
expect(retriedCount).toBe(3);
|
||||
|
||||
// Check counts
|
||||
const mainQueueJobs = await mainQueue.getWaiting();
|
||||
expect(mainQueueJobs.length).toBe(3);
|
||||
|
||||
const remainingDLQ = await dlq.getCompleted();
|
||||
expect(remainingDLQ.length).toBe(2);
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('DLQ Cleanup', () => {
|
||||
test('should cleanup old DLQ entries', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add old job (25 hours ago)
|
||||
const oldTimestamp = Date.now() - 25 * 60 * 60 * 1000;
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '1', name: 'old-job' },
|
||||
error: { message: 'Old error' },
|
||||
}, { timestamp: oldTimestamp });
|
||||
|
||||
// Add recent job (1 hour ago)
|
||||
const recentTimestamp = Date.now() - 1 * 60 * 60 * 1000;
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: { id: '2', name: 'recent-job' },
|
||||
error: { message: 'Recent error' },
|
||||
}, { timestamp: recentTimestamp });
|
||||
|
||||
// Run cleanup (24 hour threshold)
|
||||
const removedCount = await dlqHandler.cleanup();
|
||||
expect(removedCount).toBe(1);
|
||||
|
||||
// Check remaining jobs
|
||||
const remaining = await dlq.getCompleted();
|
||||
expect(remaining.length).toBe(1);
|
||||
expect(remaining[0].data.originalJob.name).toBe('recent-job');
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Failed Job Inspection', () => {
|
||||
test('should inspect failed jobs', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add failed jobs with different error types
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '1',
|
||||
name: 'network-job',
|
||||
data: { url: 'https://api.example.com' },
|
||||
attemptsMade: 3,
|
||||
},
|
||||
error: {
|
||||
message: 'Network timeout',
|
||||
stack: 'Error: Network timeout\n at ...',
|
||||
name: 'NetworkError',
|
||||
},
|
||||
movedToDLQAt: '2024-01-01T10:00:00Z',
|
||||
});
|
||||
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: '2',
|
||||
name: 'parse-job',
|
||||
data: { input: 'invalid-json' },
|
||||
attemptsMade: 2,
|
||||
},
|
||||
error: {
|
||||
message: 'Invalid JSON',
|
||||
stack: 'SyntaxError: Invalid JSON\n at ...',
|
||||
name: 'SyntaxError',
|
||||
},
|
||||
movedToDLQAt: '2024-01-01T11:00:00Z',
|
||||
});
|
||||
|
||||
const failedJobs = await dlqHandler.inspectFailedJobs(10);
|
||||
expect(failedJobs.length).toBe(2);
|
||||
|
||||
expect(failedJobs[0]).toMatchObject({
|
||||
id: '1',
|
||||
name: 'network-job',
|
||||
data: { url: 'https://api.example.com' },
|
||||
error: {
|
||||
message: 'Network timeout',
|
||||
name: 'NetworkError',
|
||||
},
|
||||
failedAt: '2024-01-01T10:00:00Z',
|
||||
attempts: 3,
|
||||
});
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Alert Threshold', () => {
|
||||
test('should detect when alert threshold is exceeded', async () => {
|
||||
const dlq = new Queue(`test-queue-dlq`, { connection });
|
||||
|
||||
// Add jobs to exceed threshold (5)
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await dlq.add('failed-job', {
|
||||
originalJob: {
|
||||
id: `${i}`,
|
||||
name: `job-${i}`,
|
||||
data: { index: i },
|
||||
},
|
||||
error: { message: 'Failed' },
|
||||
movedToDLQAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
const stats = await dlqHandler.getStats();
|
||||
expect(stats.total).toBe(6);
|
||||
// In a real implementation, this would trigger alerts
|
||||
|
||||
await dlq.close();
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue