reworked queue lib
This commit is contained in:
parent
629ba2b8d4
commit
c05a7413dc
34 changed files with 3887 additions and 861 deletions
303
libs/queue/test/queue-metrics.test.ts
Normal file
303
libs/queue/test/queue-metrics.test.ts
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { Queue, QueueEvents, Worker } from 'bullmq';
|
||||
import { QueueMetricsCollector } from '../src/queue-metrics';
|
||||
import { getRedisConnection } from '../src/utils';
|
||||
|
||||
// Suppress Redis connection errors in tests
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
if (reason && typeof reason === 'object' && 'message' in reason) {
|
||||
const message = (reason as Error).message;
|
||||
if (message.includes('Connection is closed') ||
|
||||
message.includes('Connection is in monitoring mode')) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
});
|
||||
|
||||
describe('QueueMetricsCollector', () => {
|
||||
let queue: Queue;
|
||||
let queueEvents: QueueEvents;
|
||||
let metricsCollector: QueueMetricsCollector;
|
||||
let worker: Worker;
|
||||
let connection: any;
|
||||
|
||||
const redisConfig = {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
password: '',
|
||||
db: 0,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
connection = getRedisConnection(redisConfig);
|
||||
|
||||
// Create queue and events
|
||||
queue = new Queue('metrics-test-queue', { connection });
|
||||
queueEvents = new QueueEvents('metrics-test-queue', { connection });
|
||||
|
||||
// Create metrics collector
|
||||
metricsCollector = new QueueMetricsCollector(queue, queueEvents);
|
||||
|
||||
// Wait for connections
|
||||
await queue.waitUntilReady();
|
||||
await queueEvents.waitUntilReady();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
if (worker) {
|
||||
await worker.close();
|
||||
}
|
||||
await queueEvents.close();
|
||||
await queue.close();
|
||||
} catch (error) {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
});
|
||||
|
||||
describe('Job Count Metrics', () => {
|
||||
test('should collect basic job counts', async () => {
|
||||
// Add jobs in different states
|
||||
await queue.add('waiting-job', { test: true });
|
||||
await queue.add('delayed-job', { test: true }, { delay: 60000 });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.waiting).toBe(1);
|
||||
expect(metrics.delayed).toBe(1);
|
||||
expect(metrics.active).toBe(0);
|
||||
expect(metrics.completed).toBe(0);
|
||||
expect(metrics.failed).toBe(0);
|
||||
});
|
||||
|
||||
test('should track completed and failed jobs', async () => {
|
||||
let jobCount = 0;
|
||||
|
||||
// Create worker that alternates between success and failure
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
jobCount++;
|
||||
if (jobCount % 2 === 0) {
|
||||
throw new Error('Test failure');
|
||||
}
|
||||
return { success: true };
|
||||
}, { connection });
|
||||
|
||||
// Add jobs
|
||||
await queue.add('job1', { test: 1 });
|
||||
await queue.add('job2', { test: 2 });
|
||||
await queue.add('job3', { test: 3 });
|
||||
await queue.add('job4', { test: 4 });
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.completed).toBe(2);
|
||||
expect(metrics.failed).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Processing Time Metrics', () => {
|
||||
test('should track processing times', async () => {
|
||||
const processingTimes = [50, 100, 150, 200, 250];
|
||||
let jobIndex = 0;
|
||||
|
||||
// Create worker with variable processing times
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
const delay = processingTimes[jobIndex++] || 100;
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
return { processed: true };
|
||||
}, { connection });
|
||||
|
||||
// Add jobs
|
||||
for (let i = 0; i < processingTimes.length; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.processingTime.avg).toBeGreaterThan(0);
|
||||
expect(metrics.processingTime.min).toBeGreaterThanOrEqual(50);
|
||||
expect(metrics.processingTime.max).toBeLessThanOrEqual(300);
|
||||
expect(metrics.processingTime.p95).toBeGreaterThan(metrics.processingTime.avg);
|
||||
});
|
||||
|
||||
test('should handle empty processing times', async () => {
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.processingTime).toEqual({
|
||||
avg: 0,
|
||||
min: 0,
|
||||
max: 0,
|
||||
p95: 0,
|
||||
p99: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Throughput Metrics', () => {
|
||||
test('should calculate throughput correctly', async () => {
|
||||
// Create fast worker
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
return { success: true };
|
||||
}, { connection, concurrency: 5 });
|
||||
|
||||
// Add multiple jobs
|
||||
const jobPromises = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
jobPromises.push(queue.add(`job${i}`, { index: i }));
|
||||
}
|
||||
await Promise.all(jobPromises);
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.throughput.completedPerMinute).toBeGreaterThan(0);
|
||||
expect(metrics.throughput.totalPerMinute).toBe(
|
||||
metrics.throughput.completedPerMinute + metrics.throughput.failedPerMinute
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Queue Health', () => {
|
||||
test('should report healthy queue', async () => {
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(true);
|
||||
expect(metrics.healthIssues).toEqual([]);
|
||||
});
|
||||
|
||||
test('should detect high failure rate', async () => {
|
||||
// Create worker that always fails
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
throw new Error('Always fails');
|
||||
}, { connection });
|
||||
|
||||
// Add jobs
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
// Wait for failures
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(false);
|
||||
expect(metrics.healthIssues).toContain(
|
||||
expect.stringMatching(/High failure rate/)
|
||||
);
|
||||
});
|
||||
|
||||
test('should detect large queue backlog', async () => {
|
||||
// Add many jobs without workers
|
||||
for (let i = 0; i < 1001; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.isHealthy).toBe(false);
|
||||
expect(metrics.healthIssues).toContain(
|
||||
expect.stringMatching(/Large queue backlog/)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Oldest Waiting Job', () => {
|
||||
test('should track oldest waiting job', async () => {
|
||||
const beforeAdd = Date.now();
|
||||
|
||||
// Add jobs with delays
|
||||
await queue.add('old-job', { test: true });
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
await queue.add('new-job', { test: true });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
|
||||
expect(metrics.oldestWaitingJob).toBeDefined();
|
||||
expect(metrics.oldestWaitingJob!.getTime()).toBeGreaterThanOrEqual(beforeAdd);
|
||||
});
|
||||
|
||||
test('should return null when no waiting jobs', async () => {
|
||||
// Create worker that processes immediately
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
return { success: true };
|
||||
}, { connection });
|
||||
|
||||
const metrics = await metricsCollector.collect();
|
||||
expect(metrics.oldestWaitingJob).toBe(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Metrics Report', () => {
|
||||
test('should generate formatted report', async () => {
|
||||
// Add some jobs
|
||||
await queue.add('job1', { test: true });
|
||||
await queue.add('job2', { test: true }, { delay: 5000 });
|
||||
|
||||
const report = await metricsCollector.getReport();
|
||||
|
||||
expect(report).toContain('Queue Metrics Report');
|
||||
expect(report).toContain('Status:');
|
||||
expect(report).toContain('Job Counts:');
|
||||
expect(report).toContain('Performance:');
|
||||
expect(report).toContain('Throughput:');
|
||||
expect(report).toContain('Waiting: 1');
|
||||
expect(report).toContain('Delayed: 1');
|
||||
});
|
||||
|
||||
test('should include health issues in report', async () => {
|
||||
// Add many jobs to trigger health issue
|
||||
for (let i = 0; i < 1001; i++) {
|
||||
await queue.add(`job${i}`, { index: i });
|
||||
}
|
||||
|
||||
const report = await metricsCollector.getReport();
|
||||
|
||||
expect(report).toContain('Issues Detected');
|
||||
expect(report).toContain('Health Issues:');
|
||||
expect(report).toContain('Large queue backlog');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Prometheus Metrics', () => {
|
||||
test('should export metrics in Prometheus format', async () => {
|
||||
// Add some jobs and process them
|
||||
worker = new Worker('metrics-test-queue', async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
return { success: true };
|
||||
}, { connection });
|
||||
|
||||
await queue.add('job1', { test: true });
|
||||
await queue.add('job2', { test: true });
|
||||
|
||||
// Wait for processing
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
const prometheusMetrics = await metricsCollector.getPrometheusMetrics();
|
||||
|
||||
// Check format
|
||||
expect(prometheusMetrics).toContain('# HELP queue_jobs_total');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_jobs_total gauge');
|
||||
expect(prometheusMetrics).toContain('queue_jobs_total{queue="metrics-test-queue",status="completed"}');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_processing_time_seconds');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_processing_time_seconds summary');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_throughput_per_minute');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_throughput_per_minute gauge');
|
||||
|
||||
expect(prometheusMetrics).toContain('# HELP queue_health');
|
||||
expect(prometheusMetrics).toContain('# TYPE queue_health gauge');
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue