boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

removed singletop pattern from queue manager

Browse source
This commit is contained in:
Boki 2025-06-22 19:16:25 -04:00
parent eeb5d1aca2
commit db3aa9c330
12 changed files with 504 additions and 380 deletions
Download patch file Download diff file Expand all files Collapse all files

18
apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts
View file

@ -3,12 +3,16 @@
*/
import { OperationContext } from '@stock-bot/di';
import type { ProxyInfo } from '@stock-bot/proxy';
import { QueueManager } from '@stock-bot/queue';
import type { IServiceContainer } from '@stock-bot/handlers';
export async function queueProxyFetch(): Promise<string> {
export async function queueProxyFetch(container: IServiceContainer): Promise<string> {
const ctx = OperationContext.create('proxy', 'queue-fetch');
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
throw new Error('Queue manager not available');
}
const queue = queueManager.getQueue('proxy');
const job = await queue.add('proxy-fetch', {
handler: 'proxy',
@ -22,10 +26,14 @@ export async function queueProxyFetch(): Promise<string> {
return jobId;
}
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
export async function queueProxyCheck(proxies: ProxyInfo[], container: IServiceContainer): Promise<string> {
const ctx = OperationContext.create('proxy', 'queue-check');
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
throw new Error('Queue manager not available');
}
const queue = queueManager.getQueue('proxy');
const job = await queue.add('proxy-check', {
handler: 'proxy',

6
apps/data-ingestion/src/handlers/proxy/proxy.handler.ts
View file

@ -35,6 +35,10 @@ export function initializeProxyProvider(_container: ServiceContainer) {
return { processed: 0, successful: 0 };
}
// Get QueueManager instance - we have to use getInstance for now until handlers get container access
const { QueueManager } = await import('@stock-bot/queue');
const queueManager = QueueManager.getInstance();
// Batch process the proxies through check-proxy operation
const batchResult = await processItems(proxies, 'proxy', {
handler: 'proxy',
@ -47,7 +51,7 @@ export function initializeProxyProvider(_container: ServiceContainer) {
ttl: 30000, // 30 second timeout per proxy check
removeOnComplete: 5,
removeOnFail: 3,
});
}, queueManager);
handlerLogger.info('Batch proxy validation completed', {
totalProxies: proxies.length,

6
apps/data-ingestion/src/routes/create-routes.ts
View file

@ -6,7 +6,7 @@ import { Hono } from 'hono';
import type { IServiceContainer } from '@stock-bot/handlers';
import { exchangeRoutes } from './exchange.routes';
import { healthRoutes } from './health.routes';
import { queueRoutes } from './queue.routes';
import { createQueueRoutes } from './queue.routes';
/**
* Creates all routes with access to type-safe services
@ -17,9 +17,9 @@ export function createRoutes(services: IServiceContainer): Hono {
// Mount routes that don't need services
app.route('/health', healthRoutes);
// Mount routes that need services (will be updated to use services)
// Mount routes that need services
app.route('/api/exchanges', exchangeRoutes);
app.route('/api/queue', queueRoutes);
app.route('/api/queue', createQueueRoutes(services));
// Store services in app context for handlers that need it
app.use('*', async (c, next) => {

47
apps/data-ingestion/src/routes/market-data.routes.ts
View file

@ -3,20 +3,26 @@
*/
import { Hono } from 'hono';
import { getLogger } from '@stock-bot/logger';
import { processItems, QueueManager } from '@stock-bot/queue';
import { processItems } from '@stock-bot/queue';
import type { IServiceContainer } from '@stock-bot/handlers';
const logger = getLogger('market-data-routes');
export const marketDataRoutes = new Hono();
export function createMarketDataRoutes(container: IServiceContainer) {
const marketDataRoutes = new Hono();
// Market data endpoints
marketDataRoutes.get('/api/live/:symbol', async c => {
// Market data endpoints
marketDataRoutes.get('/api/live/:symbol', async c => {
const symbol = c.req.param('symbol');
logger.info('Live data request', { symbol });
try {
// Queue job for live data using Yahoo provider
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
}
const queue = queueManager.getQueue('yahoo-finance');
const job = await queue.add('live-data', {
handler: 'yahoo-finance',
@ -33,9 +39,9 @@ marketDataRoutes.get('/api/live/:symbol', async c => {
logger.error('Failed to queue live data job', { symbol, error });
return c.json({ status: 'error', message: 'Failed to queue live data job' }, 500);
}
});
});
marketDataRoutes.get('/api/historical/:symbol', async c => {
marketDataRoutes.get('/api/historical/:symbol', async c => {
const symbol = c.req.param('symbol');
const from = c.req.query('from');
const to = c.req.query('to');
@ -47,7 +53,11 @@ marketDataRoutes.get('/api/historical/:symbol', async c => {
const toDate = to ? new Date(to) : new Date(); // Now
// Queue job for historical data using Yahoo provider
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
}
const queue = queueManager.getQueue('yahoo-finance');
const job = await queue.add('historical-data', {
handler: 'yahoo-finance',
@ -71,10 +81,10 @@ marketDataRoutes.get('/api/historical/:symbol', async c => {
logger.error('Failed to queue historical data job', { symbol, from, to, error });
return c.json({ status: 'error', message: 'Failed to queue historical data job' }, 500);
}
});
});
// Batch processing endpoint using new queue system
marketDataRoutes.post('/api/process-symbols', async c => {
// Batch processing endpoint using new queue system
marketDataRoutes.post('/api/process-symbols', async c => {
try {
const {
symbols,
@ -96,6 +106,11 @@ marketDataRoutes.post('/api/process-symbols', async c => {
useBatching,
});
const queueManager = container.queue;
if (!queueManager) {
return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
}
const result = await processItems(symbols, provider, {
handler: provider,
operation,
@ -106,7 +121,7 @@ marketDataRoutes.post('/api/process-symbols', async c => {
retries: 2,
removeOnComplete: 5,
removeOnFail: 10,
});
}, queueManager);
return c.json({
status: 'success',
@ -118,4 +133,10 @@ marketDataRoutes.post('/api/process-symbols', async c => {
logger.error('Failed to process symbols batch', { error });
return c.json({ status: 'error', message: 'Failed to process symbols batch' }, 500);
}
});
});
return marketDataRoutes;
}
// Legacy export for backward compatibility
export const marketDataRoutes = createMarketDataRoutes({} as IServiceContainer);

24
apps/data-ingestion/src/routes/queue.routes.ts
View file

@ -1,14 +1,20 @@
import { Hono } from 'hono';
import { getLogger } from '@stock-bot/logger';
import { QueueManager } from '@stock-bot/queue';
import type { IServiceContainer } from '@stock-bot/handlers';
const logger = getLogger('queue-routes');
const queue = new Hono();
// Queue status endpoint
queue.get('/status', async c => {
export function createQueueRoutes(container: IServiceContainer) {
const queue = new Hono();
// Queue status endpoint
queue.get('/status', async c => {
try {
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ status: 'error', message: 'Queue manager not available' }, 503);
}
const globalStats = await queueManager.getGlobalStats();
return c.json({
@ -20,6 +26,10 @@ queue.get('/status', async c => {
logger.error('Failed to get queue status', { error });
return c.json({ status: 'error', message: 'Failed to get queue status' }, 500);
}
});
});
export { queue as queueRoutes };
return queue;
}
// Legacy export for backward compatibility
export const queueRoutes = createQueueRoutes({} as IServiceContainer);

15
apps/data-pipeline/src/routes/create-routes.ts
View file

@ -4,10 +4,13 @@
*/
import { Hono } from 'hono';
import type { ServiceContainer } from '@stock-bot/di';
import { healthRoutes, syncRoutes, enhancedSyncRoutes, statsRoutes } from './index';
import type { IServiceContainer } from '@stock-bot/handlers';
import { healthRoutes } from './health.routes';
import { createSyncRoutes } from './sync.routes';
import { createEnhancedSyncRoutes } from './enhanced-sync.routes';
import { createStatsRoutes } from './stats.routes';
export function createRoutes(container: ServiceContainer): Hono {
export function createRoutes(container: IServiceContainer): Hono {
const app = new Hono();
// Add container to context for all routes
@ -18,9 +21,9 @@ export function createRoutes(container: ServiceContainer): Hono {
// Mount routes
app.route('/health', healthRoutes);
app.route('/sync', syncRoutes);
app.route('/sync', enhancedSyncRoutes);
app.route('/sync/stats', statsRoutes);
app.route('/sync', createSyncRoutes(container));
app.route('/sync', createEnhancedSyncRoutes(container));
app.route('/sync/stats', createStatsRoutes(container));
return app;
}

130
apps/data-pipeline/src/routes/enhanced-sync.routes.ts
View file

@ -1,15 +1,21 @@
import { Hono } from 'hono';
import { getLogger } from '@stock-bot/logger';
import { QueueManager } from '@stock-bot/queue';
import type { IServiceContainer } from '@stock-bot/handlers';
const logger = getLogger('enhanced-sync-routes');
const enhancedSync = new Hono();
// Enhanced sync endpoints
enhancedSync.post('/exchanges/all', async c => {
export function createEnhancedSyncRoutes(container: IServiceContainer) {
const enhancedSync = new Hono();
// Enhanced sync endpoints
enhancedSync.post('/exchanges/all', async c => {
try {
const clearFirst = c.req.query('clear') === 'true';
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('sync-all-exchanges', {
@ -26,11 +32,15 @@ enhancedSync.post('/exchanges/all', async c => {
500
);
}
});
});
enhancedSync.post('/provider-mappings/qm', async c => {
enhancedSync.post('/provider-mappings/qm', async c => {
try {
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('sync-qm-provider-mappings', {
@ -51,50 +61,94 @@ enhancedSync.post('/provider-mappings/qm', async c => {
500
);
}
});
enhancedSync.post('/symbols/:provider', async c => {
try {
const provider = c.req.param('provider');
const clearFirst = c.req.query('clear') === 'true';
const queueManager = QueueManager.getInstance();
const symbolsQueue = queueManager.getQueue('symbols');
const job = await symbolsQueue.addJob(`sync-symbols-${provider}`, {
handler: 'symbols',
operation: `sync-symbols-${provider}`,
payload: { provider, clearFirst },
});
return c.json({ success: true, jobId: job.id, message: `${provider} symbols sync job queued` });
enhancedSync.post('/provider-mappings/ib', async c => {
try {
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('sync-ib-exchanges', {
handler: 'exchanges',
operation: 'sync-ib-exchanges',
payload: {},
});
return c.json({
success: true,
jobId: job.id,
message: 'IB exchanges sync job queued',
});
} catch (error) {
logger.error('Failed to queue enhanced symbol sync job', { error });
logger.error('Failed to queue IB exchanges sync job', { error });
return c.json(
{ success: false, error: error instanceof Error ? error.message : 'Unknown error' },
500
);
}
});
});
// Enhanced status endpoints
enhancedSync.get('/status/enhanced', async c => {
enhancedSync.get('/status', async c => {
try {
const queueManager = QueueManager.getInstance();
const exchangesQueue = queueManager.getQueue('exchanges');
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const job = await exchangesQueue.addJob('enhanced-sync-status', {
handler: 'exchanges',
operation: 'enhanced-sync-status',
const symbolsQueue = queueManager.getQueue('symbols');
const job = await symbolsQueue.addJob('sync-status', {
handler: 'symbols',
operation: 'sync-status',
payload: {},
});
// Wait for job to complete and return result
const result = await job.waitUntilFinished();
return c.json(result);
return c.json({ success: true, jobId: job.id, message: 'Sync status job queued' });
} catch (error) {
logger.error('Failed to get enhanced sync status', { error });
return c.json({ error: error instanceof Error ? error.message : 'Unknown error' }, 500);
logger.error('Failed to queue sync status job', { error });
return c.json(
{ success: false, error: error instanceof Error ? error.message : 'Unknown error' },
500
);
}
});
});
export { enhancedSync as enhancedSyncRoutes };
enhancedSync.post('/clear/postgresql', async c => {
try {
const dataType = c.req.query('type') as 'exchanges' | 'provider_mappings' | 'all';
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('clear-postgresql-data', {
handler: 'exchanges',
operation: 'clear-postgresql-data',
payload: { dataType: dataType || 'all' },
});
return c.json({
success: true,
jobId: job.id,
message: 'PostgreSQL data clear job queued',
});
} catch (error) {
logger.error('Failed to queue PostgreSQL clear job', { error });
return c.json(
{ success: false, error: error instanceof Error ? error.message : 'Unknown error' },
500
);
}
});
return enhancedSync;
}
// Legacy export for backward compatibility
export const enhancedSyncRoutes = createEnhancedSyncRoutes({} as IServiceContainer);

34
apps/data-pipeline/src/routes/stats.routes.ts
View file

@ -1,14 +1,20 @@
import { Hono } from 'hono';
import { getLogger } from '@stock-bot/logger';
import { QueueManager } from '@stock-bot/queue';
import type { IServiceContainer } from '@stock-bot/handlers';
const logger = getLogger('stats-routes');
const stats = new Hono();
// Statistics endpoints
stats.get('/exchanges', async c => {
export function createStatsRoutes(container: IServiceContainer) {
const stats = new Hono();
// Statistics endpoints
stats.get('/exchanges', async c => {
try {
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ error: 'Queue manager not available' }, 503);
}
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('get-exchange-stats', {
@ -24,11 +30,15 @@ stats.get('/exchanges', async c => {
logger.error('Failed to get exchange stats', { error });
return c.json({ error: error instanceof Error ? error.message : 'Unknown error' }, 500);
}
});
});
stats.get('/provider-mappings', async c => {
stats.get('/provider-mappings', async c => {
try {
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ error: 'Queue manager not available' }, 503);
}
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('get-provider-mapping-stats', {
@ -44,6 +54,10 @@ stats.get('/provider-mappings', async c => {
logger.error('Failed to get provider mapping stats', { error });
return c.json({ error: error instanceof Error ? error.message : 'Unknown error' }, 500);
}
});
});
export { stats as statsRoutes };
return stats;
}
// Legacy export for backward compatibility
export const statsRoutes = createStatsRoutes({} as IServiceContainer);

81
apps/data-pipeline/src/routes/sync.routes.ts
View file

@ -1,14 +1,20 @@
import { Hono } from 'hono';
import { getLogger } from '@stock-bot/logger';
import { QueueManager } from '@stock-bot/queue';
import type { IServiceContainer } from '@stock-bot/handlers';
const logger = getLogger('sync-routes');
const sync = new Hono();
// Manual sync trigger endpoints
sync.post('/symbols', async c => {
export function createSyncRoutes(container: IServiceContainer) {
const sync = new Hono();
// Manual sync trigger endpoints
sync.post('/symbols', async c => {
try {
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const symbolsQueue = queueManager.getQueue('symbols');
const job = await symbolsQueue.addJob('sync-qm-symbols', {
@ -25,11 +31,15 @@ sync.post('/symbols', async c => {
500
);
}
});
});
sync.post('/exchanges', async c => {
sync.post('/exchanges', async c => {
try {
const queueManager = QueueManager.getInstance();
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('sync-qm-exchanges', {
@ -46,51 +56,40 @@ sync.post('/exchanges', async c => {
500
);
}
});
});
// Get sync status
sync.get('/status', async c => {
sync.post('/symbols/:provider', async c => {
try {
const queueManager = QueueManager.getInstance();
const provider = c.req.param('provider');
const queueManager = container.queue;
if (!queueManager) {
return c.json({ success: false, error: 'Queue manager not available' }, 503);
}
const symbolsQueue = queueManager.getQueue('symbols');
const job = await symbolsQueue.addJob('sync-status', {
const job = await symbolsQueue.addJob('sync-symbols-from-provider', {
handler: 'symbols',
operation: 'sync-status',
payload: {},
operation: 'sync-symbols-from-provider',
payload: { provider },
});
// Wait for job to complete and return result
const result = await job.waitUntilFinished();
return c.json(result);
} catch (error) {
logger.error('Failed to get sync status', { error });
return c.json({ error: error instanceof Error ? error.message : 'Unknown error' }, 500);
}
});
// Clear data endpoint
sync.post('/clear', async c => {
try {
const queueManager = QueueManager.getInstance();
const exchangesQueue = queueManager.getQueue('exchanges');
const job = await exchangesQueue.addJob('clear-postgresql-data', {
handler: 'exchanges',
operation: 'clear-postgresql-data',
payload: {},
return c.json({
success: true,
jobId: job.id,
message: `${provider} symbols sync job queued`,
});
// Wait for job to complete and return result
const result = await job.waitUntilFinished();
return c.json({ success: true, result });
} catch (error) {
logger.error('Failed to clear PostgreSQL data', { error });
logger.error('Failed to queue provider symbol sync job', { error });
return c.json(
{ success: false, error: error instanceof Error ? error.message : 'Unknown error' },
500
);
}
});
});
export { sync as syncRoutes };
return sync;
}
// Legacy export for backward compatibility
export const syncRoutes = createSyncRoutes({} as IServiceContainer);

21
libs/core/di/src/awilix-container.ts
View file

@ -188,19 +188,18 @@ export function createServiceContainer(rawConfig: unknown): AwilixContainer<Serv
registrations.questdbClient = asValue(null);
}
// Queue manager - placeholder until decoupled from singleton
registrations.queueManager = asFunction(({ redisConfig, cache, logger }) => {
// Import dynamically to avoid circular dependency
// Queue manager - properly instantiated with DI
registrations.queueManager = asFunction(({ redisConfig, logger }) => {
const { QueueManager } = require('@stock-bot/queue');
// Check if already initialized (singleton pattern)
if (QueueManager.isInitialized()) {
return QueueManager.getInstance();
}
// Initialize if not already done
return QueueManager.initialize({
redis: { host: redisConfig.host, port: redisConfig.port, db: redisConfig.db },
return new QueueManager({
redis: {
host: redisConfig.host,
port: redisConfig.port,
db: redisConfig.db,
password: redisConfig.password,
username: redisConfig.username,
},
enableScheduledJobs: true,
delayWorkerStart: true, // We'll start workers manually
});

44
libs/services/queue/src/batch-processor.ts
View file

@ -11,9 +11,9 @@ const logger = getLogger('batch-processor');
export async function processItems<T>(
items: T[],
queueName: string,
options: ProcessOptions
options: ProcessOptions,
queueManager: QueueManager
): Promise<BatchResult> {
const queueManager = QueueManager.getInstance();
queueManager.getQueue(queueName);
const startTime = Date.now();
@ -35,8 +35,8 @@ export async function processItems<T>(
try {
const result = options.useBatching
? await processBatched(items, queueName, options)
: await processDirect(items, queueName, options);
? await processBatched(items, queueName, options, queueManager)
: await processDirect(items, queueName, options, queueManager);
const duration = Date.now() - startTime;
@ -58,9 +58,9 @@ export async function processItems<T>(
async function processDirect<T>(
items: T[],
queueName: string,
options: ProcessOptions
options: ProcessOptions,
queueManager: QueueManager
): Promise<Omit<BatchResult, 'duration'>> {
const queueManager = QueueManager.getInstance();
queueManager.getQueue(queueName);
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; // Convert hours to milliseconds
const delayPerItem = totalDelayMs / items.length;
@ -87,7 +87,7 @@ async function processDirect<T>(
},
}));
const createdJobs = await addJobsInChunks(queueName, jobs);
const createdJobs = await addJobsInChunks(queueName, jobs, queueManager);
return {
totalItems: items.length,
@ -102,9 +102,9 @@ async function processDirect<T>(
async function processBatched<T>(
items: T[],
queueName: string,
options: ProcessOptions
options: ProcessOptions,
queueManager: QueueManager
): Promise<Omit<BatchResult, 'duration'>> {
const queueManager = QueueManager.getInstance();
queueManager.getQueue(queueName);
const batchSize = options.batchSize || 100;
const batches = createBatches(items, batchSize);
@ -121,7 +121,7 @@ async function processBatched<T>(
const batchJobs = await Promise.all(
batches.map(async (batch, batchIndex) => {
// Just store the items directly - no processing needed
const payloadKey = await storeItems(batch, queueName, options);
const payloadKey = await storeItems(batch, queueName, options, queueManager);
return {
name: 'process-batch',
@ -148,7 +148,7 @@ async function processBatched<T>(
})
);
const createdJobs = await addJobsInChunks(queueName, batchJobs);
const createdJobs = await addJobsInChunks(queueName, batchJobs, queueManager);
return {
totalItems: items.length,
@ -161,8 +161,7 @@ async function processBatched<T>(
/**
* Process a batch job - loads items and creates individual jobs
*/
export async function processBatchJob(jobData: BatchJobData, queueName: string): Promise<unknown> {
const queueManager = QueueManager.getInstance();
export async function processBatchJob(jobData: BatchJobData, queueName: string, queueManager: QueueManager): Promise<unknown> {
queueManager.getQueue(queueName);
const { payloadKey, batchIndex, totalBatches, itemCount, totalDelayHours } = jobData;
@ -174,7 +173,7 @@ export async function processBatchJob(jobData: BatchJobData, queueName: string):
});
try {
const payload = await loadPayload(payloadKey, queueName);
const payload = await loadPayload(payloadKey, queueName, queueManager);
if (!payload || !payload.items || !payload.options) {
logger.error('Invalid payload data', { payloadKey, payload });
throw new Error(`Invalid payload data for key: ${payloadKey}`);
@ -210,10 +209,10 @@ export async function processBatchJob(jobData: BatchJobData, queueName: string):
},
}));
const createdJobs = await addJobsInChunks(queueName, jobs);
const createdJobs = await addJobsInChunks(queueName, jobs, queueManager);
// Cleanup payload after successful processing
await cleanupPayload(payloadKey, queueName);
await cleanupPayload(payloadKey, queueName, queueManager);
return {
batchIndex,
@ -239,9 +238,9 @@ function createBatches<T>(items: T[], batchSize: number): T[][] {
async function storeItems<T>(
items: T[],
queueName: string,
options: ProcessOptions
options: ProcessOptions,
queueManager: QueueManager
): Promise<string> {
const queueManager = QueueManager.getInstance();
const cache = queueManager.getCache(queueName);
const payloadKey = `payload:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`;
@ -265,7 +264,8 @@ async function storeItems<T>(
async function loadPayload<T>(
key: string,
queueName: string
queueName: string,
queueManager: QueueManager
): Promise<{
items: T[];
options: {
@ -276,7 +276,6 @@ async function loadPayload<T>(
operation: string;
};
} | null> {
const queueManager = QueueManager.getInstance();
const cache = queueManager.getCache(queueName);
return (await cache.get(key)) as {
items: T[];
@ -290,8 +289,7 @@ async function loadPayload<T>(
} | null;
}
async function cleanupPayload(key: string, queueName: string): Promise<void> {
const queueManager = QueueManager.getInstance();
async function cleanupPayload(key: string, queueName: string, queueManager: QueueManager): Promise<void> {
const cache = queueManager.getCache(queueName);
await cache.del(key);
}
@ -299,9 +297,9 @@ async function cleanupPayload(key: string, queueName: string): Promise<void> {
async function addJobsInChunks(
queueName: string,
jobs: Array<{ name: string; data: JobData; opts?: Record<string, unknown> }>,
queueManager: QueueManager,
chunkSize = 100
): Promise<unknown[]> {
const queueManager = QueueManager.getInstance();
const queue = queueManager.getQueue(queueName);
const allCreatedJobs = [];

20
libs/services/queue/src/queue-manager.ts
View file

@ -15,7 +15,7 @@ import { getRedisConnection } from './utils';
const logger = getLogger('queue-manager');
/**
* Singleton QueueManager that provides unified queue and cache management
* QueueManager provides unified queue and cache management
* Main entry point for all queue operations with getQueue() method
*/
export class QueueManager {
@ -28,7 +28,7 @@ export class QueueManager {
private shutdownPromise: Promise<void> | null = null;
private config: QueueManagerConfig;
private constructor(config: QueueManagerConfig) {
constructor(config: QueueManagerConfig) {
this.config = config;
this.redisConnection = getRedisConnection(config.redis);
@ -42,16 +42,20 @@ export class QueueManager {
});
}
logger.info('QueueManager singleton initialized', {
logger.info('QueueManager initialized', {
redis: `${config.redis.host}:${config.redis.port}`,
});
}
/**
* @deprecated Use dependency injection instead. This method will be removed in a future version.
* Get the singleton instance
* @throws Error if not initialized - use initialize() first
*/
static getInstance(): QueueManager {
logger.warn(
'QueueManager.getInstance() is deprecated. Please use dependency injection instead.'
);
if (!QueueManager.instance) {
throw new Error('QueueManager not initialized. Call QueueManager.initialize(config) first.');
}
@ -59,10 +63,14 @@ export class QueueManager {
}
/**
* @deprecated Use dependency injection instead. This method will be removed in a future version.
* Initialize the singleton with config
* Must be called before getInstance()
*/
static initialize(config: QueueManagerConfig): QueueManager {
logger.warn(
'QueueManager.initialize() is deprecated. Please use dependency injection instead.'
);
if (QueueManager.instance) {
logger.warn('QueueManager already initialized, returning existing instance');
return QueueManager.instance;
@ -72,10 +80,14 @@ export class QueueManager {
}
/**
* @deprecated Use dependency injection instead. This method will be removed in a future version.
* Get or initialize the singleton
* Convenience method that combines initialize and getInstance
*/
static getOrInitialize(config?: QueueManagerConfig): QueueManager {
logger.warn(
'QueueManager.getOrInitialize() is deprecated. Please use dependency injection instead.'
);
if (QueueManager.instance) {
return QueueManager.instance;
}
@ -91,6 +103,7 @@ export class QueueManager {
}
/**
* @deprecated Use dependency injection instead. This method will be removed in a future version.
* Check if the QueueManager is initialized
*/
static isInitialized(): boolean {
@ -98,6 +111,7 @@ export class QueueManager {
}
/**
* @deprecated Use dependency injection instead. This method will be removed in a future version.
* Reset the singleton (mainly for testing)
*/
static async reset(): Promise<void> {