fully cleaned things up, few more things to go.

libs/queue/src/queue-manager.ts

@@ -2,7 +2,7 @@ import { Queue, QueueEvents, Worker, type Job } from 'bullmq';
 import { getLogger } from '@stock-bot/logger';
 import { processBatchJob } from './batch-processor';
 import { providerRegistry } from './provider-registry';
-import type { JobData, ProviderConfig, QueueConfig } from './types';
+import type { JobData, ProviderConfig, ProviderInitializer, QueueConfig } from './types';
 
 const logger = getLogger('queue-manager');
 
@@ -11,6 +11,8 @@ export class QueueManager {
   private workers: Worker[] = [];
   private queueEvents!: QueueEvents;
   private config: Required<QueueConfig>;
+  private providers: ProviderInitializer[];
+  private enableScheduledJobs: boolean;
 
   private get isInitialized() {
     return !!this.queue;
@@ -24,6 +26,10 @@ export class QueueManager {
   }
 
   constructor(config: QueueConfig = {}) {
+    // Enhanced configuration
+    this.providers = config.providers || [];
+    this.enableScheduledJobs = config.enableScheduledJobs ?? true;
+
     // Set default configuration
     this.config = {
       workers: config.workers || parseInt(process.env.WORKER_COUNT || '5'),
@@ -45,11 +51,13 @@ export class QueueManager {
         },
         ...config.defaultJobOptions,
       },
+      providers: this.providers,
+      enableScheduledJobs: this.enableScheduledJobs,
     };
   }
 
   /**
-   * Initialize the queue manager
+   * Initialize the queue manager with enhanced provider and scheduled job support
    */
   async initialize(): Promise<void> {
     if (this.isInitialized) {
@@ -57,13 +65,19 @@ export class QueueManager {
       return;
     }
 
-    logger.info('Initializing queue manager...', {
+    logger.info('Initializing enhanced queue manager...', {
       queueName: this.config.queueName,
       workers: this.config.workers,
       concurrency: this.config.concurrency,
+      providers: this.providers.length,
+      enableScheduledJobs: this.enableScheduledJobs,
     });
 
     try {
+      // Step 1: Register all providers
+      await this.registerProviders();
+
+      // Step 2: Initialize core queue infrastructure
       const connection = this.getConnection();
       const queueName = `{${this.config.queueName}}`;
 
@@ -76,19 +90,110 @@ export class QueueManager {
       // Initialize queue events
       this.queueEvents = new QueueEvents(queueName, { connection });
 
-      // Start workers
+      // Step 3: Start workers
       await this.startWorkers();
 
-      // Setup event listeners
+      // Step 4: Setup event listeners
       this.setupEventListeners();
 
-      logger.info('Queue manager initialized successfully');
+      // Step 5: Initialize batch cache
+      const { initializeBatchCache } = await import('./batch-processor');
+      await initializeBatchCache(this);
+
+      // Step 6: Set up scheduled jobs
+      if (this.enableScheduledJobs) {
+        await this.setupScheduledJobs();
+      }
+
+      logger.info('Enhanced queue manager initialized successfully');
     } catch (error) {
-      logger.error('Failed to initialize queue manager', { error });
+      logger.error('Failed to initialize enhanced queue manager', { error });
       throw error;
     }
   }
 
+  /**
+   * Register all configured providers
+   */
+  private async registerProviders(): Promise<void> {
+    logger.info('Registering queue providers...', { count: this.providers.length });
+
+    // Initialize providers using the configured provider initializers
+    for (const providerInitializer of this.providers) {
+      try {
+        await providerInitializer();
+      } catch (error) {
+        logger.error('Failed to initialize provider', { error });
+        throw error;
+      }
+    }
+
+    // Now register all providers from the registry with the queue manager
+    const allProviders = providerRegistry.getAllProviders();
+    for (const [providerName, config] of allProviders) {
+      this.registerProvider(providerName, config.operations);
+      logger.info(`Registered provider: ${providerName}`);
+    }
+
+    // Log scheduled jobs
+    const scheduledJobs = providerRegistry.getAllScheduledJobs();
+    logger.info(`Registered ${scheduledJobs.length} scheduled jobs across all providers`);
+    for (const { provider, job } of scheduledJobs) {
+      logger.info(
+        `Scheduled job: ${provider}.${job.type} - ${job.description} (${job.cronPattern})`
+      );
+    }
+
+    logger.info('All providers registered successfully');
+  }
+
+  /**
+   * Set up scheduled jobs from provider registry
+   */
+  private async setupScheduledJobs(): Promise<void> {
+    const scheduledJobs = providerRegistry.getAllScheduledJobs();
+
+    if (scheduledJobs.length === 0) {
+      logger.info('No scheduled jobs found');
+      return;
+    }
+
+    logger.info(`Setting up ${scheduledJobs.length} scheduled jobs...`);
+
+    for (const { provider, job } of scheduledJobs) {
+      try {
+        const jobData: JobData = {
+          type: job.type,
+          provider,
+          operation: job.operation,
+          payload: job.payload,
+          priority: job.priority,
+        };
+
+        await this.add(`recurring-${provider}-${job.operation}`, jobData, {
+          repeat: {
+            pattern: job.cronPattern,
+            tz: 'UTC',
+            immediately: job.immediately || false,
+          },
+          removeOnComplete: 1,
+          removeOnFail: 1,
+          attempts: 2,
+          backoff: {
+            type: 'fixed',
+            delay: 5000,
+          },
+        });
+
+        logger.info(`Scheduled job registered: ${provider}.${job.type} (${job.cronPattern})`);
+      } catch (error) {
+        logger.error(`Failed to register scheduled job: ${provider}.${job.type}`, { error });
+      }
+    }
+
+    logger.info('Scheduled jobs setup complete');
+  }
+
   /**
    * Register a provider with its operations
    */
@@ -99,7 +204,7 @@ export class QueueManager {
   /**
    * Add a single job to the queue
    */
-  async add(name: string, data: JobData, options: any = {}): Promise<Job> {
+  async add(name: string, data: JobData, options: Record<string, unknown> = {}): Promise<Job> {
     this.ensureInitialized();
     return await this.queue.add(name, data, options);
   }
@@ -107,7 +212,9 @@ export class QueueManager {
   /**
    * Add multiple jobs to the queue in bulk
    */
-  async addBulk(jobs: Array<{ name: string; data: JobData; opts?: any }>): Promise<Job[]> {
+  async addBulk(
+    jobs: Array<{ name: string; data: JobData; opts?: Record<string, unknown> }>
+  ): Promise<Job[]> {
     this.ensureInitialized();
     return await this.queue.addBulk(jobs);
   }