thing im pretty much done with extracting the queue and making it reususable, maybe just a few more change to be able to making the batch names a bit more specific

apps/data-service/src/index.ts

@@ -6,10 +6,10 @@ import { Browser } from '@stock-bot/browser';
 import { loadEnvVariables } from '@stock-bot/config';
 import { getLogger, shutdownLoggers } from '@stock-bot/logger';
 import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client';
+import { processItems, QueueManager, type QueueConfig } from '@stock-bot/queue';
 import { Shutdown } from '@stock-bot/shutdown';
 import { initializeIBResources } from './providers/ib.tasks';
 import { initializeProxyResources } from './providers/proxy.tasks';
-import { initializeQueueManager, shutdownQueueManager } from './services/queue-manager.service';
 import { healthRoutes, queueRoutes } from './routes';
 
 // Load environment variables
@@ -23,6 +23,41 @@ let server: ReturnType<typeof Bun.serve> | null = null;
 // Initialize shutdown manager with 15 second timeout
 const shutdown = Shutdown.getInstance({ timeout: 15000 });
 
+/**
+ * Create and configure the enhanced queue manager for data service
+ */
+function createDataServiceQueueManager(): QueueManager {
+  const config: QueueConfig = {
+    queueName: 'data-service-queue',
+    workers: parseInt(process.env.WORKER_COUNT || '5'),
+    concurrency: parseInt(process.env.WORKER_CONCURRENCY || '20'),
+    redis: {
+      host: process.env.DRAGONFLY_HOST || 'localhost',
+      port: parseInt(process.env.DRAGONFLY_PORT || '6379'),
+    },
+    providers: [
+      // Import and initialize providers lazily
+      async () => {
+        const { initializeIBProvider } = await import('./providers/ib.provider');
+        return initializeIBProvider();
+      },
+      async () => {
+        const { initializeProxyProvider } = await import('./providers/proxy.provider');
+        return initializeProxyProvider();
+      },
+    ],
+    enableScheduledJobs: true,
+  };
+
+  return new QueueManager(config);
+}
+
+// Create singleton instance
+export const queueManager = createDataServiceQueueManager();
+
+// Export processItems for direct use
+export { processItems };
+
 // Register all routes
 app.route('', healthRoutes);
 app.route('', queueRoutes);
@@ -54,7 +89,7 @@ async function initializeServices() {
 
     // Initialize queue manager (includes batch cache initialization)
     logger.info('Starting queue manager initialization...');
-    await initializeQueueManager();
+    await queueManager.initialize();
     logger.info('Queue manager initialized');
 
     logger.info('All services initialized successfully');
@@ -92,7 +127,7 @@ shutdown.onShutdown(async () => {
 shutdown.onShutdown(async () => {
   logger.info('Shutting down queue manager...');
   try {
-    await shutdownQueueManager();
+    await queueManager.shutdown();
     logger.info('Queue manager shut down successfully');
   } catch (error) {
     logger.error('Error shutting down queue manager', { error });