restructured libs to be more aligned with core components
This commit is contained in:
parent
947b1d748d
commit
0d1be9e3cb
50 changed files with 73 additions and 67 deletions
444
libs/core/queue/src/queue-manager.ts
Normal file
444
libs/core/queue/src/queue-manager.ts
Normal file
|
|
@ -0,0 +1,444 @@
|
|||
import { createCache } from '@stock-bot/cache';
|
||||
import type { CacheProvider } from '@stock-bot/cache';
|
||||
import { Queue, type QueueWorkerConfig } from './queue';
|
||||
import { QueueRateLimiter } from './rate-limiter';
|
||||
import type {
|
||||
GlobalStats,
|
||||
QueueManagerConfig,
|
||||
QueueOptions,
|
||||
QueueStats,
|
||||
RateLimitRule,
|
||||
} from './types';
|
||||
import { getRedisConnection } from './utils';
|
||||
|
||||
// Logger interface for type safety
|
||||
interface Logger {
|
||||
info(message: string, meta?: Record<string, unknown>): void;
|
||||
error(message: string, meta?: Record<string, unknown>): void;
|
||||
warn(message: string, meta?: Record<string, unknown>): void;
|
||||
debug(message: string, meta?: Record<string, unknown>): void;
|
||||
trace(message: string, meta?: Record<string, unknown>): void;
|
||||
child?(name: string, context?: Record<string, unknown>): Logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* QueueManager provides unified queue and cache management
|
||||
* Main entry point for all queue operations with getQueue() method
|
||||
*/
|
||||
export class QueueManager {
|
||||
private queues = new Map<string, Queue>();
|
||||
private caches = new Map<string, CacheProvider>();
|
||||
private rateLimiter?: QueueRateLimiter;
|
||||
private redisConnection: ReturnType<typeof getRedisConnection>;
|
||||
private isShuttingDown = false;
|
||||
private shutdownPromise: Promise<void> | null = null;
|
||||
private config: QueueManagerConfig;
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(config: QueueManagerConfig, logger?: Logger) {
|
||||
this.config = config;
|
||||
this.logger = logger || console;
|
||||
this.redisConnection = getRedisConnection(config.redis);
|
||||
|
||||
// Initialize rate limiter if rules are provided
|
||||
if (config.rateLimitRules && config.rateLimitRules.length > 0) {
|
||||
this.rateLimiter = new QueueRateLimiter(this.redisConnection, this.logger);
|
||||
config.rateLimitRules.forEach(rule => {
|
||||
if (this.rateLimiter) {
|
||||
this.rateLimiter.addRule(rule);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.info('QueueManager initialized', {
|
||||
redis: `${config.redis.host}:${config.redis.port}`,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a queue - unified method that handles both scenarios
|
||||
* This is the main method for accessing queues
|
||||
*/
|
||||
getQueue(queueName: string, options: QueueOptions = {}): Queue {
|
||||
// Return existing queue if it exists
|
||||
if (this.queues.has(queueName)) {
|
||||
const existingQueue = this.queues.get(queueName);
|
||||
if (existingQueue) {
|
||||
return existingQueue;
|
||||
}
|
||||
}
|
||||
|
||||
// Create new queue with merged options
|
||||
const mergedOptions = {
|
||||
...this.config.defaultQueueOptions,
|
||||
...options,
|
||||
};
|
||||
|
||||
// Prepare queue configuration
|
||||
const workers = mergedOptions.workers ?? this.config.defaultQueueOptions?.workers ?? 1;
|
||||
const concurrency = mergedOptions.concurrency ?? this.config.defaultQueueOptions?.concurrency ?? 1;
|
||||
|
||||
const queueConfig: QueueWorkerConfig = {
|
||||
workers,
|
||||
concurrency,
|
||||
startWorker: workers > 0 && !this.config.delayWorkerStart,
|
||||
};
|
||||
|
||||
const queue = new Queue(
|
||||
queueName,
|
||||
this.config.redis,
|
||||
mergedOptions.defaultJobOptions || {},
|
||||
queueConfig,
|
||||
this.logger
|
||||
);
|
||||
|
||||
// Store the queue
|
||||
this.queues.set(queueName, queue);
|
||||
|
||||
// Automatically initialize batch cache for the queue
|
||||
this.initializeBatchCacheSync(queueName);
|
||||
|
||||
// Add queue-specific rate limit rules
|
||||
if (this.rateLimiter && mergedOptions.rateLimitRules) {
|
||||
mergedOptions.rateLimitRules.forEach(rule => {
|
||||
// Ensure queue name is set for queue-specific rules
|
||||
const ruleWithQueue = { ...rule, queueName };
|
||||
if (this.rateLimiter) {
|
||||
this.rateLimiter.addRule(ruleWithQueue);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.info('Queue created with batch cache', {
|
||||
queueName,
|
||||
workers: workers,
|
||||
concurrency: concurrency,
|
||||
});
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a queue exists
|
||||
*/
|
||||
hasQueue(queueName: string): boolean {
|
||||
return this.queues.has(queueName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all queue names
|
||||
*/
|
||||
getQueueNames(): string[] {
|
||||
return Array.from(this.queues.keys());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a cache for a queue
|
||||
*/
|
||||
getCache(queueName: string): CacheProvider {
|
||||
if (!this.caches.has(queueName)) {
|
||||
const cacheProvider = createCache({
|
||||
redisConfig: this.config.redis,
|
||||
keyPrefix: `batch:${queueName}:`,
|
||||
ttl: 86400, // 24 hours default
|
||||
enableMetrics: true,
|
||||
logger: this.logger,
|
||||
});
|
||||
this.caches.set(queueName, cacheProvider);
|
||||
this.logger.trace('Cache created for queue', { queueName });
|
||||
}
|
||||
const cache = this.caches.get(queueName);
|
||||
if (!cache) {
|
||||
throw new Error(`Expected cache for queue ${queueName} to exist`);
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize cache for a queue (ensures it's ready)
|
||||
*/
|
||||
async initializeCache(queueName: string): Promise<void> {
|
||||
const cache = this.getCache(queueName);
|
||||
await cache.waitForReady(10000);
|
||||
this.logger.info('Cache initialized for queue', { queueName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize batch cache synchronously (for automatic initialization)
|
||||
* The cache will be ready for use, but we don't wait for Redis connection
|
||||
*/
|
||||
private initializeBatchCacheSync(queueName: string): void {
|
||||
// Just create the cache - it will connect automatically when first used
|
||||
this.getCache(queueName);
|
||||
this.logger.trace('Batch cache initialized synchronously for queue', { queueName });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics for all queues
|
||||
*/
|
||||
async getGlobalStats(): Promise<GlobalStats> {
|
||||
const queueStats: Record<string, QueueStats> = {};
|
||||
let totalJobs = 0;
|
||||
let totalWorkers = 0;
|
||||
|
||||
for (const [queueName, queue] of this.queues) {
|
||||
const stats = await queue.getStats();
|
||||
queueStats[queueName] = stats;
|
||||
|
||||
totalJobs += stats.waiting + stats.active + stats.completed + stats.failed + stats.delayed;
|
||||
totalWorkers += stats.workers || 0;
|
||||
}
|
||||
|
||||
return {
|
||||
queues: queueStats,
|
||||
totalJobs,
|
||||
totalWorkers,
|
||||
uptime: process.uptime(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics for a specific queue
|
||||
*/
|
||||
async getQueueStats(queueName: string): Promise<QueueStats | undefined> {
|
||||
const queue = this.queues.get(queueName);
|
||||
if (!queue) {
|
||||
return undefined;
|
||||
}
|
||||
return await queue.getStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a rate limit rule
|
||||
*/
|
||||
addRateLimitRule(rule: RateLimitRule): void {
|
||||
if (!this.rateLimiter) {
|
||||
this.rateLimiter = new QueueRateLimiter(this.redisConnection, this.logger);
|
||||
}
|
||||
this.rateLimiter.addRule(rule);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check rate limits for a job
|
||||
*/
|
||||
async checkRateLimit(
|
||||
queueName: string,
|
||||
handler: string,
|
||||
operation: string
|
||||
): Promise<{
|
||||
allowed: boolean;
|
||||
retryAfter?: number;
|
||||
remainingPoints?: number;
|
||||
appliedRule?: RateLimitRule;
|
||||
}> {
|
||||
if (!this.rateLimiter) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
return await this.rateLimiter.checkLimit(queueName, handler, operation);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get rate limit status
|
||||
*/
|
||||
async getRateLimitStatus(queueName: string, handler: string, operation: string) {
|
||||
if (!this.rateLimiter) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
};
|
||||
}
|
||||
|
||||
return await this.rateLimiter.getStatus(queueName, handler, operation);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause all queues
|
||||
*/
|
||||
async pauseAll(): Promise<void> {
|
||||
const pausePromises = Array.from(this.queues.values()).map(queue => queue.pause());
|
||||
await Promise.all(pausePromises);
|
||||
this.logger.info('All queues paused');
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume all queues
|
||||
*/
|
||||
async resumeAll(): Promise<void> {
|
||||
const resumePromises = Array.from(this.queues.values()).map(queue => queue.resume());
|
||||
await Promise.all(resumePromises);
|
||||
this.logger.info('All queues resumed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Pause a specific queue
|
||||
*/
|
||||
async pauseQueue(queueName: string): Promise<boolean> {
|
||||
const queue = this.queues.get(queueName);
|
||||
if (!queue) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await queue.pause();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume a specific queue
|
||||
*/
|
||||
async resumeQueue(queueName: string): Promise<boolean> {
|
||||
const queue = this.queues.get(queueName);
|
||||
if (!queue) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await queue.resume();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain all queues
|
||||
*/
|
||||
async drainAll(delayed = false): Promise<void> {
|
||||
const drainPromises = Array.from(this.queues.values()).map(queue => queue.drain(delayed));
|
||||
await Promise.all(drainPromises);
|
||||
this.logger.info('All queues drained', { delayed });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean all queues
|
||||
*/
|
||||
async cleanAll(
|
||||
grace: number = 0,
|
||||
limit: number = 100,
|
||||
type: 'completed' | 'failed' = 'completed'
|
||||
): Promise<void> {
|
||||
const cleanPromises = Array.from(this.queues.values()).map(queue =>
|
||||
queue.clean(grace, limit, type)
|
||||
);
|
||||
await Promise.all(cleanPromises);
|
||||
this.logger.info('All queues cleaned', { type, grace, limit });
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown all queues and workers (thread-safe)
|
||||
*/
|
||||
async shutdown(): Promise<void> {
|
||||
// If already shutting down, return the existing promise
|
||||
if (this.shutdownPromise) {
|
||||
return this.shutdownPromise;
|
||||
}
|
||||
|
||||
if (this.isShuttingDown) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isShuttingDown = true;
|
||||
this.logger.info('Shutting down QueueManager...');
|
||||
|
||||
// Create shutdown promise
|
||||
this.shutdownPromise = this.performShutdown();
|
||||
return this.shutdownPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform the actual shutdown
|
||||
*/
|
||||
private async performShutdown(): Promise<void> {
|
||||
try {
|
||||
// Close all queues (this now includes workers since they're managed by Queue class)
|
||||
const queueShutdownPromises = Array.from(this.queues.values()).map(async queue => {
|
||||
try {
|
||||
// Add timeout to queue.close() to prevent hanging
|
||||
await queue.close();
|
||||
// const timeoutPromise = new Promise<never>((_, reject) =>
|
||||
// setTimeout(() => reject(new Error('Queue close timeout')), 100)
|
||||
// );
|
||||
|
||||
// await Promise.race([closePromise, timeoutPromise]);
|
||||
} catch (error) {
|
||||
this.logger.warn('Error closing queue', { error: (error as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(queueShutdownPromises);
|
||||
|
||||
// Close all caches
|
||||
const cacheShutdownPromises = Array.from(this.caches.values()).map(async cache => {
|
||||
try {
|
||||
// Clear cache before shutdown
|
||||
await cache.clear();
|
||||
} catch (error) {
|
||||
this.logger.warn('Error clearing cache', { error: (error as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(cacheShutdownPromises);
|
||||
|
||||
// Clear collections
|
||||
this.queues.clear();
|
||||
this.caches.clear();
|
||||
|
||||
this.logger.info('QueueManager shutdown complete');
|
||||
} catch (error) {
|
||||
this.logger.error('Error during shutdown', { error: (error as Error).message });
|
||||
throw error;
|
||||
} finally {
|
||||
// Reset shutdown state
|
||||
this.shutdownPromise = null;
|
||||
this.isShuttingDown = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start workers for all queues (used when delayWorkerStart is enabled)
|
||||
*/
|
||||
startAllWorkers(): void {
|
||||
if (!this.config.delayWorkerStart) {
|
||||
this.logger.info(
|
||||
'startAllWorkers() called but workers already started automatically (delayWorkerStart is false)'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let workersStarted = 0;
|
||||
for (const queue of this.queues.values()) {
|
||||
const workerCount = this.config.defaultQueueOptions?.workers || 1;
|
||||
const concurrency = this.config.defaultQueueOptions?.concurrency || 1;
|
||||
|
||||
if (workerCount > 0) {
|
||||
queue.startWorkersManually(workerCount, concurrency);
|
||||
workersStarted++;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('All workers started', {
|
||||
totalQueues: this.queues.size,
|
||||
queuesWithWorkers: workersStarted,
|
||||
delayWorkerStart: this.config.delayWorkerStart,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for all queues to be ready
|
||||
*/
|
||||
async waitUntilReady(): Promise<void> {
|
||||
const readyPromises = Array.from(this.queues.values()).map(queue => queue.waitUntilReady());
|
||||
await Promise.all(readyPromises);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Redis configuration (for backward compatibility)
|
||||
*/
|
||||
getRedisConfig() {
|
||||
return this.config.redis;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current configuration
|
||||
*/
|
||||
getConfig(): Readonly<QueueManagerConfig> {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue