added new queue lib with batch processor and provider
This commit is contained in:
parent
ddcf94a587
commit
6c548416d1
19 changed files with 1939 additions and 35 deletions
312
libs/queue/src/queue-manager.ts
Normal file
312
libs/queue/src/queue-manager.ts
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
import { Queue, QueueEvents, Worker, type Job } from 'bullmq';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { processBatchJob } from './batch-processor';
|
||||
import { providerRegistry } from './provider-registry';
|
||||
import type { JobData, ProviderConfig, QueueConfig } from './types';
|
||||
|
||||
const logger = getLogger('queue-manager');
|
||||
|
||||
export class QueueManager {
|
||||
private queue!: Queue;
|
||||
private workers: Worker[] = [];
|
||||
private queueEvents!: QueueEvents;
|
||||
private config: Required<QueueConfig>;
|
||||
|
||||
private get isInitialized() {
|
||||
return !!this.queue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the queue name
|
||||
*/
|
||||
get queueName(): string {
|
||||
return this.config.queueName;
|
||||
}
|
||||
|
||||
constructor(config: QueueConfig = {}) {
|
||||
// Set default configuration
|
||||
this.config = {
|
||||
workers: config.workers || parseInt(process.env.WORKER_COUNT || '5'),
|
||||
concurrency: config.concurrency || parseInt(process.env.WORKER_CONCURRENCY || '20'),
|
||||
redis: {
|
||||
host: config.redis?.host || process.env.DRAGONFLY_HOST || 'localhost',
|
||||
port: config.redis?.port || parseInt(process.env.DRAGONFLY_PORT || '6379'),
|
||||
password: config.redis?.password || process.env.DRAGONFLY_PASSWORD,
|
||||
db: config.redis?.db || parseInt(process.env.DRAGONFLY_DB || '0'),
|
||||
},
|
||||
queueName: config.queueName || 'default-queue',
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: 10,
|
||||
removeOnFail: 5,
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000,
|
||||
},
|
||||
...config.defaultJobOptions,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the queue manager
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
logger.warn('Queue manager already initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info('Initializing queue manager...', {
|
||||
queueName: this.config.queueName,
|
||||
workers: this.config.workers,
|
||||
concurrency: this.config.concurrency,
|
||||
});
|
||||
|
||||
try {
|
||||
const connection = this.getConnection();
|
||||
const queueName = `{${this.config.queueName}}`;
|
||||
|
||||
// Initialize queue
|
||||
this.queue = new Queue(queueName, {
|
||||
connection,
|
||||
defaultJobOptions: this.config.defaultJobOptions,
|
||||
});
|
||||
|
||||
// Initialize queue events
|
||||
this.queueEvents = new QueueEvents(queueName, { connection });
|
||||
|
||||
// Start workers
|
||||
await this.startWorkers();
|
||||
|
||||
// Setup event listeners
|
||||
this.setupEventListeners();
|
||||
|
||||
logger.info('Queue manager initialized successfully');
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize queue manager', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a provider with its operations
|
||||
*/
|
||||
registerProvider(providerName: string, config: ProviderConfig): void {
|
||||
providerRegistry.register(providerName, config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a single job to the queue
|
||||
*/
|
||||
async add(name: string, data: JobData, options: any = {}): Promise<Job> {
|
||||
this.ensureInitialized();
|
||||
return await this.queue.add(name, data, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add multiple jobs to the queue in bulk
|
||||
*/
|
||||
async addBulk(jobs: Array<{ name: string; data: JobData; opts?: any }>): Promise<Job[]> {
|
||||
this.ensureInitialized();
|
||||
return await this.queue.addBulk(jobs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics
|
||||
*/
|
||||
async getStats(): Promise<{
|
||||
waiting: number;
|
||||
active: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
delayed: number;
|
||||
}> {
|
||||
this.ensureInitialized();
|
||||
const [waiting, active, completed, failed, delayed] = await Promise.all([
|
||||
this.queue.getWaiting(),
|
||||
this.queue.getActive(),
|
||||
this.queue.getCompleted(),
|
||||
this.queue.getFailed(),
|
||||
this.queue.getDelayed(),
|
||||
]);
|
||||
|
||||
return {
|
||||
waiting: waiting.length,
|
||||
active: active.length,
|
||||
completed: completed.length,
|
||||
failed: failed.length,
|
||||
delayed: delayed.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause the queue
|
||||
*/
|
||||
async pause(): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
await this.queue.pause();
|
||||
logger.info('Queue paused');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume the queue
|
||||
*/
|
||||
async resume(): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
await this.queue.resume();
|
||||
logger.info('Queue resumed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean completed and failed jobs
|
||||
*/
|
||||
async clean(grace: number = 0, limit: number = 100): Promise<void> {
|
||||
this.ensureInitialized();
|
||||
await Promise.all([
|
||||
this.queue.clean(grace, limit, 'completed'),
|
||||
this.queue.clean(grace, limit, 'failed'),
|
||||
]);
|
||||
logger.info('Queue cleaned', { grace, limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the queue name
|
||||
*/
|
||||
getQueueName(): string {
|
||||
return this.config.queueName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the queue manager
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
logger.info('Shutting down queue manager...');
|
||||
|
||||
try {
|
||||
// Close workers
|
||||
await Promise.all(this.workers.map(worker => worker.close()));
|
||||
this.workers = [];
|
||||
|
||||
// Close queue events
|
||||
if (this.queueEvents) {
|
||||
await this.queueEvents.close();
|
||||
}
|
||||
|
||||
// Close queue
|
||||
if (this.queue) {
|
||||
await this.queue.close();
|
||||
}
|
||||
|
||||
logger.info('Queue manager shutdown complete');
|
||||
} catch (error) {
|
||||
logger.error('Error during queue manager shutdown', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private getConnection() {
|
||||
return {
|
||||
host: this.config.redis.host,
|
||||
port: this.config.redis.port,
|
||||
password: this.config.redis.password,
|
||||
db: this.config.redis.db,
|
||||
};
|
||||
}
|
||||
|
||||
private async startWorkers(): Promise<void> {
|
||||
const connection = this.getConnection();
|
||||
const queueName = `{${this.config.queueName}}`;
|
||||
|
||||
for (let i = 0; i < this.config.workers; i++) {
|
||||
const worker = new Worker(queueName, this.processJob.bind(this), {
|
||||
connection,
|
||||
concurrency: this.config.concurrency,
|
||||
});
|
||||
|
||||
worker.on('completed', job => {
|
||||
logger.debug('Job completed', {
|
||||
id: job.id,
|
||||
name: job.name,
|
||||
});
|
||||
});
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
logger.error('Job failed', {
|
||||
id: job?.id,
|
||||
name: job?.name,
|
||||
error: err.message,
|
||||
});
|
||||
});
|
||||
|
||||
this.workers.push(worker);
|
||||
}
|
||||
|
||||
logger.info(`Started ${this.config.workers} workers`);
|
||||
}
|
||||
|
||||
private async processJob(job: Job) {
|
||||
const { provider, operation, payload }: JobData = job.data;
|
||||
|
||||
logger.info('Processing job', {
|
||||
id: job.id,
|
||||
provider,
|
||||
operation,
|
||||
payloadKeys: Object.keys(payload || {}),
|
||||
});
|
||||
|
||||
try {
|
||||
let result;
|
||||
|
||||
if (operation === 'process-batch-items') {
|
||||
// Special handling for batch processing - requires queue manager instance
|
||||
result = await processBatchJob(payload, this);
|
||||
} else {
|
||||
// Regular handler lookup
|
||||
const handler = providerRegistry.getHandler(provider, operation);
|
||||
|
||||
if (!handler) {
|
||||
throw new Error(`No handler found for ${provider}:${operation}`);
|
||||
}
|
||||
|
||||
result = await handler(payload);
|
||||
}
|
||||
|
||||
logger.info('Job completed successfully', {
|
||||
id: job.id,
|
||||
provider,
|
||||
operation,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error('Job processing failed', {
|
||||
id: job.id,
|
||||
provider,
|
||||
operation,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private setupEventListeners(): void {
|
||||
this.queueEvents.on('completed', ({ jobId }) => {
|
||||
logger.debug('Job completed event', { jobId });
|
||||
});
|
||||
|
||||
this.queueEvents.on('failed', ({ jobId, failedReason }) => {
|
||||
logger.warn('Job failed event', { jobId, failedReason });
|
||||
});
|
||||
|
||||
this.queueEvents.on('stalled', ({ jobId }) => {
|
||||
logger.warn('Job stalled event', { jobId });
|
||||
});
|
||||
}
|
||||
|
||||
private ensureInitialized(): void {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('Queue manager not initialized. Call initialize() first.');
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue