boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

batch processor changes

Browse source
This commit is contained in:
Bojan Kucera 2025-06-09 00:03:50 -04:00
parent 8b6f6008e4
commit f78fa459d2
1 changed files with 12 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

28
apps/data-service/src/utils/batch-processor.ts
View file

@ -87,11 +87,10 @@ export class BatchProcessor {
type: `${jobNamePrefix}-batch-processing`,
service,
provider,
operation: `process-${jobNamePrefix}-batch`,
payload: {
operation: `process-${jobNamePrefix}-batch`, payload: {
items: batchItems,
batchIndex: i,
totalBatch: totalBatches, // Changed to match your property name
total: totalBatches, // Changed to total to match proxy provider
batchSize,
config: {
jobNamePrefix,
@ -145,11 +144,10 @@ export class BatchProcessor {
}
/**
* Process a batch by creating individual item jobs
*/
async processBatch<T>(payload: {
*/ async processBatch<T>(payload: {
items: T[];
batchIndex: number;
totalBatch: number; // Changed to match common property name
total: number; // Changed to match proxy provider
batchSize: number;
config: {
jobNamePrefix: string;
@ -166,13 +164,12 @@ export class BatchProcessor {
jobsCreated: number;
jobsFailed: number;
}> {
const { items, batchIndex, totalBatch, config } = payload;
const { items, batchIndex, total, config } = payload;
logger.info('Processing batch', {
batchIndex,
batchSize: items.length,
totalBatch,
progress: `${((batchIndex + 1) / totalBatch * 100).toFixed(2)}%`
total,
progress: `${((batchIndex + 1) / total * 100).toFixed(2)}%`
});
// Spread items over a reasonable time period
@ -183,12 +180,12 @@ export class BatchProcessor {
// Get user data first
const userData = createJobData(item, i);
// Automatically merge with batch info using generic property names
// Automatically merge with batch info using your property names
const finalPayload = {
...userData,
batchIndex,
itemIndexInBatch: i, // Generic property name
totalBatch, // Generic property name
itemIndex: i, // Changed to match proxy provider
total, // Changed to match proxy provider
source: userData.source || 'batch-processing'
};
@ -213,13 +210,12 @@ export class BatchProcessor {
try {
const jobs = await this.queueManager.queue.addBulk(jobsToCreate);
logger.info('Batch processing completed', {
logger.info('Batch processing completed', {
batchIndex,
totalItems: items.length,
jobsCreated: jobs.length,
batchDelay: '15 minutes',
progress: `${((batchIndex + 1) / totalBatch * 100).toFixed(2)}%`
progress: `${((batchIndex + 1) / total * 100).toFixed(2)}%`
});
return {