diff --git a/apps/data-service/src/utils/batch-processor.ts b/apps/data-service/src/utils/batch-processor.ts index bcfb87b..61e54c2 100644 --- a/apps/data-service/src/utils/batch-processor.ts +++ b/apps/data-service/src/utils/batch-processor.ts @@ -87,11 +87,10 @@ export class BatchProcessor { type: `${jobNamePrefix}-batch-processing`, service, provider, - operation: `process-${jobNamePrefix}-batch`, - payload: { + operation: `process-${jobNamePrefix}-batch`, payload: { items: batchItems, batchIndex: i, - totalBatch: totalBatches, // Changed to match your property name + total: totalBatches, // Changed to total to match proxy provider batchSize, config: { jobNamePrefix, @@ -145,11 +144,10 @@ export class BatchProcessor { } /** * Process a batch by creating individual item jobs - */ - async processBatch(payload: { + */ async processBatch(payload: { items: T[]; batchIndex: number; - totalBatch: number; // Changed to match common property name + total: number; // Changed to match proxy provider batchSize: number; config: { jobNamePrefix: string; @@ -166,13 +164,12 @@ export class BatchProcessor { jobsCreated: number; jobsFailed: number; }> { - const { items, batchIndex, totalBatch, config } = payload; - + const { items, batchIndex, total, config } = payload; logger.info('Processing batch', { batchIndex, batchSize: items.length, - totalBatch, - progress: `${((batchIndex + 1) / totalBatch * 100).toFixed(2)}%` + total, + progress: `${((batchIndex + 1) / total * 100).toFixed(2)}%` }); // Spread items over a reasonable time period @@ -183,12 +180,12 @@ export class BatchProcessor { // Get user data first const userData = createJobData(item, i); - // Automatically merge with batch info using generic property names + // Automatically merge with batch info using your property names const finalPayload = { ...userData, batchIndex, - itemIndexInBatch: i, // Generic property name - totalBatch, // Generic property name + itemIndex: i, // Changed to match proxy provider + total, // Changed to match proxy provider source: userData.source || 'batch-processing' }; @@ -213,13 +210,12 @@ export class BatchProcessor { try { const jobs = await this.queueManager.queue.addBulk(jobsToCreate); - - logger.info('Batch processing completed', { + logger.info('Batch processing completed', { batchIndex, totalItems: items.length, jobsCreated: jobs.length, batchDelay: '15 minutes', - progress: `${((batchIndex + 1) / totalBatch * 100).toFixed(2)}%` + progress: `${((batchIndex + 1) / total * 100).toFixed(2)}%` }); return {