trying to get simpler batcher working

apps/data-service/src/utils/batch-helpers.ts

@@ -14,6 +14,10 @@ export interface ProcessOptions {
   ttl?: number;
   removeOnComplete?: number;
   removeOnFail?: number;
+  // Job routing information
+  service?: string;
+  provider?: string;
+  operation?: string;
 }
 
 export interface BatchResult {
@@ -106,9 +110,9 @@ async function processDirect<T>(
     name: 'process-item',
     data: {
       type: 'process-item',
-      service: 'batch-processor',
-      provider: 'direct',
-      operation: 'process-single-item',
+      service: options.service || 'data-service',
+      provider: options.provider || 'generic',
+      operation: options.operation || 'process-item',
       payload: processor(item, index),
       priority: options.priority || 1
     },
@@ -205,18 +209,22 @@ export async function processBatchJob(jobData: any, queue: QueueService): Promis
 
   try {
     const payload = await loadPayload(payloadKey);
+    if (!payload || !payload.items || !payload.processorStr) {
+      logger.error('Invalid payload data', { payloadKey, payload });
+      throw new Error(`Invalid payload data for key: ${payloadKey}`);
+    }
     const { items, processorStr, options } = payload;
     
-    // Deserialize processor function (in production, use safer alternatives)
+    // Deserialize the processor function
     const processor = new Function('return ' + processorStr)();
     
     const jobs = items.map((item: any, index: number) => ({
       name: 'process-item',
       data: {
         type: 'process-item',
-        service: 'batch-processor',
-        provider: 'batch-item',
-        operation: 'process-single-item',
+        service: options.service || 'data-service',
+        provider: options.provider || 'generic',
+        operation: options.operation || 'process-item',
         payload: processor(item, index),
         priority: options.priority || 1
       },
@@ -260,6 +268,10 @@ async function storePayload<T>(
   options: ProcessOptions
 ): Promise<string> {
   const cache = getCache();
+  
+  // Wait for cache to be ready before storing
+  await cache.waitForReady(5000);
+  
   const key = `payload_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
   
   const payload = {
@@ -268,14 +280,24 @@ async function storePayload<T>(
     options: {
       delayPerItem: 1000,
       priority: options.priority || 1,
-      retries: options.retries || 3
+      retries: options.retries || 3,
+      // Store routing information for later use
+      service: options.service || 'data-service',
+      provider: options.provider || 'generic',
+      operation: options.operation || 'process-item'
     },
     createdAt: Date.now()
   };
-
-  await cache.set(key, JSON.stringify(payload), options.ttl || 86400);
   
-  logger.debug('Stored batch payload', { 
+  logger.debug('Storing batch payload', { 
+    key, 
+    itemCount: items.length,
+    cacheReady: cache.isReady()
+  });
+  
+  await cache.set(key, payload, options.ttl || 86400);
+  
+  logger.debug('Stored batch payload successfully', { 
     key, 
     itemCount: items.length 
   });
@@ -285,13 +307,27 @@ async function storePayload<T>(
 
 async function loadPayload(key: string): Promise<any> {
   const cache = getCache();
+  
+  // Wait for cache to be ready before loading
+  await cache.waitForReady(5000);
+  
+  logger.debug('Loading batch payload', { 
+    key, 
+    cacheReady: cache.isReady() 
+  });
+  
   const data = await cache.get(key);
   
   if (!data) {
+    logger.error('Payload not found in cache', { 
+      key, 
+      cacheReady: cache.isReady() 
+    });
     throw new Error(`Payload not found: ${key}`);
   }
 
-  return JSON.parse(data as string);
+  logger.debug('Loaded batch payload successfully', { key });
+  return data;
 }
 
 async function cleanupPayload(key: string): Promise<void> {
@@ -356,7 +392,10 @@ export async function processSymbols(
       totalDelayMs: options.totalDelayMs,
       batchSize: options.batchSize || 100,
       priority: options.priority || 1,
-      useBatching: options.useBatching || false
+      useBatching: options.useBatching || false,
+      service: options.service,
+      provider: options.provider,
+      operation: options.operation
     }
   );
 }
@@ -369,6 +408,9 @@ export async function processProxies(
     useBatching?: boolean;
     batchSize?: number;
     priority?: number;
+    service?: string;
+    provider?: string;
+    operation?: string;
   }
 ): Promise<BatchResult> {
   return processItems(
@@ -383,7 +425,10 @@ export async function processProxies(
       totalDelayMs: options.totalDelayMs,
       batchSize: options.batchSize || 200,
       priority: options.priority || 2,
-      useBatching: options.useBatching || true
+      useBatching: options.useBatching || true,
+      service: options.service || 'data-service',
+      provider: options.provider || 'proxy-service',
+      operation: options.operation || 'check-proxy'
     }
   );
 }