running prettier for cleanup
This commit is contained in:
parent
fe7733aeb5
commit
d85cd58acd
151 changed files with 29158 additions and 27966 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import { CacheProvider, createCache } from '@stock-bot/cache';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { createCache, CacheProvider } from '@stock-bot/cache';
|
||||
import type { QueueService } from '../services/queue.service';
|
||||
|
||||
const logger = getLogger('batch-helpers');
|
||||
|
|
@ -35,7 +35,7 @@ function getCache(): CacheProvider {
|
|||
cacheProvider = createCache({
|
||||
keyPrefix: 'batch:',
|
||||
ttl: 86400, // 24 hours default
|
||||
enableMetrics: true
|
||||
enableMetrics: true,
|
||||
});
|
||||
}
|
||||
return cacheProvider;
|
||||
|
|
@ -62,13 +62,13 @@ export async function processItems<T>(
|
|||
options: ProcessOptions
|
||||
): Promise<BatchResult> {
|
||||
const startTime = Date.now();
|
||||
|
||||
|
||||
if (items.length === 0) {
|
||||
return {
|
||||
jobsCreated: 0,
|
||||
mode: 'direct',
|
||||
totalItems: 0,
|
||||
duration: 0
|
||||
duration: 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -76,23 +76,22 @@ export async function processItems<T>(
|
|||
totalItems: items.length,
|
||||
mode: options.useBatching ? 'batch' : 'direct',
|
||||
batchSize: options.batchSize,
|
||||
totalDelayHours: options.totalDelayHours
|
||||
totalDelayHours: options.totalDelayHours,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = options.useBatching
|
||||
const result = options.useBatching
|
||||
? await processBatched(items, processor, queue, options)
|
||||
: await processDirect(items, processor, queue, options);
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
|
||||
logger.info('Batch processing completed', {
|
||||
...result,
|
||||
duration: `${(duration / 1000).toFixed(1)}s`
|
||||
duration: `${(duration / 1000).toFixed(1)}s`,
|
||||
});
|
||||
|
||||
return { ...result, duration };
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Batch processing failed', error);
|
||||
throw error;
|
||||
|
|
@ -108,13 +107,12 @@ async function processDirect<T>(
|
|||
queue: QueueService,
|
||||
options: ProcessOptions
|
||||
): Promise<Omit<BatchResult, 'duration'>> {
|
||||
|
||||
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000;
|
||||
const delayPerItem = totalDelayMs / items.length;
|
||||
|
||||
|
||||
logger.info('Creating direct jobs', {
|
||||
totalItems: items.length,
|
||||
delayPerItem: `${(delayPerItem / 1000).toFixed(1)}s`
|
||||
delayPerItem: `${(delayPerItem / 1000).toFixed(1)}s`,
|
||||
});
|
||||
|
||||
const jobs = items.map((item, index) => ({
|
||||
|
|
@ -124,23 +122,23 @@ async function processDirect<T>(
|
|||
provider: options.provider || 'generic',
|
||||
operation: options.operation || 'process-item',
|
||||
payload: processor(item, index),
|
||||
priority: options.priority || 1
|
||||
priority: options.priority || 1,
|
||||
},
|
||||
opts: {
|
||||
delay: index * delayPerItem,
|
||||
priority: options.priority || 1,
|
||||
attempts: options.retries || 3,
|
||||
removeOnComplete: options.removeOnComplete || 10,
|
||||
removeOnFail: options.removeOnFail || 5
|
||||
}
|
||||
removeOnFail: options.removeOnFail || 5,
|
||||
},
|
||||
}));
|
||||
|
||||
const createdJobs = await addJobsInChunks(queue, jobs);
|
||||
|
||||
|
||||
return {
|
||||
totalItems: items.length,
|
||||
jobsCreated: createdJobs.length,
|
||||
mode: 'direct'
|
||||
mode: 'direct',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -153,7 +151,6 @@ async function processBatched<T>(
|
|||
queue: QueueService,
|
||||
options: ProcessOptions
|
||||
): Promise<Omit<BatchResult, 'duration'>> {
|
||||
|
||||
const batchSize = options.batchSize || 100;
|
||||
const batches = createBatches(items, batchSize);
|
||||
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000;
|
||||
|
|
@ -163,13 +160,13 @@ async function processBatched<T>(
|
|||
totalItems: items.length,
|
||||
batchSize,
|
||||
totalBatches: batches.length,
|
||||
delayPerBatch: `${(delayPerBatch / 1000 / 60).toFixed(2)} minutes`
|
||||
delayPerBatch: `${(delayPerBatch / 1000 / 60).toFixed(2)} minutes`,
|
||||
});
|
||||
|
||||
const batchJobs = await Promise.all(
|
||||
batches.map(async (batch, batchIndex) => {
|
||||
const payloadKey = await storePayload(batch, processor, options);
|
||||
|
||||
|
||||
return {
|
||||
name: 'process-batch',
|
||||
data: {
|
||||
|
|
@ -180,17 +177,17 @@ async function processBatched<T>(
|
|||
payloadKey,
|
||||
batchIndex,
|
||||
totalBatches: batches.length,
|
||||
itemCount: batch.length
|
||||
itemCount: batch.length,
|
||||
},
|
||||
priority: options.priority || 2
|
||||
priority: options.priority || 2,
|
||||
},
|
||||
opts: {
|
||||
delay: batchIndex * delayPerBatch,
|
||||
priority: options.priority || 2,
|
||||
attempts: options.retries || 3,
|
||||
removeOnComplete: options.removeOnComplete || 10,
|
||||
removeOnFail: options.removeOnFail || 5
|
||||
}
|
||||
removeOnFail: options.removeOnFail || 5,
|
||||
},
|
||||
};
|
||||
})
|
||||
);
|
||||
|
|
@ -201,7 +198,7 @@ async function processBatched<T>(
|
|||
totalItems: items.length,
|
||||
jobsCreated: createdJobs.length,
|
||||
batchesCreated: batches.length,
|
||||
mode: 'batch'
|
||||
mode: 'batch',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -210,11 +207,11 @@ async function processBatched<T>(
|
|||
*/
|
||||
export async function processBatchJob(jobData: any, queue: QueueService): Promise<any> {
|
||||
const { payloadKey, batchIndex, totalBatches, itemCount } = jobData;
|
||||
|
||||
logger.debug('Processing batch job', {
|
||||
batchIndex,
|
||||
totalBatches,
|
||||
itemCount
|
||||
|
||||
logger.debug('Processing batch job', {
|
||||
batchIndex,
|
||||
totalBatches,
|
||||
itemCount,
|
||||
});
|
||||
|
||||
try {
|
||||
|
|
@ -225,7 +222,7 @@ export async function processBatchJob(jobData: any, queue: QueueService): Promis
|
|||
}
|
||||
|
||||
const { items, processorStr, options } = payload;
|
||||
|
||||
|
||||
// Deserialize the processor function
|
||||
const processor = new Function('return ' + processorStr)();
|
||||
|
||||
|
|
@ -236,26 +233,25 @@ export async function processBatchJob(jobData: any, queue: QueueService): Promis
|
|||
provider: options.provider || 'generic',
|
||||
operation: options.operation || 'generic',
|
||||
payload: processor(item, index),
|
||||
priority: options.priority || 1
|
||||
priority: options.priority || 1,
|
||||
},
|
||||
opts: {
|
||||
delay: index * (options.delayPerItem || 1000),
|
||||
priority: options.priority || 1,
|
||||
attempts: options.retries || 3
|
||||
}
|
||||
attempts: options.retries || 3,
|
||||
},
|
||||
}));
|
||||
|
||||
const createdJobs = await addJobsInChunks(queue, jobs);
|
||||
|
||||
|
||||
// Cleanup payload after successful processing
|
||||
await cleanupPayload(payloadKey);
|
||||
|
||||
return {
|
||||
batchIndex,
|
||||
itemsProcessed: items.length,
|
||||
jobsCreated: createdJobs.length
|
||||
jobsCreated: createdJobs.length,
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Batch job processing failed', { batchIndex, error });
|
||||
throw error;
|
||||
|
|
@ -273,20 +269,20 @@ function createBatches<T>(items: T[], batchSize: number): T[][] {
|
|||
}
|
||||
|
||||
async function storePayload<T>(
|
||||
items: T[],
|
||||
items: T[],
|
||||
processor: (item: T, index: number) => any,
|
||||
options: ProcessOptions
|
||||
): Promise<string> {
|
||||
const cache = getCache();
|
||||
|
||||
|
||||
// Create more specific key: batch:provider:operation:payload_timestamp_random
|
||||
const timestamp = Date.now();
|
||||
const randomId = Math.random().toString(36).substr(2, 9);
|
||||
const provider = options.provider || 'generic';
|
||||
const operation = options.operation || 'generic';
|
||||
|
||||
|
||||
const key = `${provider}:${operation}:payload_${timestamp}_${randomId}`;
|
||||
|
||||
|
||||
const payload = {
|
||||
items,
|
||||
processorStr: processor.toString(),
|
||||
|
|
@ -296,33 +292,33 @@ async function storePayload<T>(
|
|||
retries: options.retries || 3,
|
||||
// Store routing information for later use
|
||||
provider: options.provider || 'generic',
|
||||
operation: options.operation || 'generic'
|
||||
operation: options.operation || 'generic',
|
||||
},
|
||||
createdAt: Date.now()
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
|
||||
logger.debug('Storing batch payload', {
|
||||
key,
|
||||
itemCount: items.length
|
||||
|
||||
logger.debug('Storing batch payload', {
|
||||
key,
|
||||
itemCount: items.length,
|
||||
});
|
||||
|
||||
|
||||
await cache.set(key, payload, options.ttl || 86400);
|
||||
|
||||
logger.debug('Stored batch payload successfully', {
|
||||
key,
|
||||
itemCount: items.length
|
||||
|
||||
logger.debug('Stored batch payload successfully', {
|
||||
key,
|
||||
itemCount: items.length,
|
||||
});
|
||||
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
async function loadPayload(key: string): Promise<any> {
|
||||
const cache = getCache();
|
||||
|
||||
|
||||
logger.debug('Loading batch payload', { key });
|
||||
|
||||
|
||||
const data = await cache.get(key);
|
||||
|
||||
|
||||
if (!data) {
|
||||
logger.error('Payload not found in cache', { key });
|
||||
throw new Error(`Payload not found: ${key}`);
|
||||
|
|
@ -344,27 +340,25 @@ async function cleanupPayload(key: string): Promise<void> {
|
|||
|
||||
async function addJobsInChunks(queue: QueueService, jobs: any[], chunkSize = 100): Promise<any[]> {
|
||||
const allCreatedJobs = [];
|
||||
|
||||
|
||||
for (let i = 0; i < jobs.length; i += chunkSize) {
|
||||
const chunk = jobs.slice(i, i + chunkSize);
|
||||
try {
|
||||
const createdJobs = await queue.addBulk(chunk);
|
||||
allCreatedJobs.push(...createdJobs);
|
||||
|
||||
|
||||
// Small delay between chunks to avoid overwhelming Redis
|
||||
if (i + chunkSize < jobs.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to add job chunk', {
|
||||
startIndex: i,
|
||||
chunkSize: chunk.length,
|
||||
error
|
||||
logger.error('Failed to add job chunk', {
|
||||
startIndex: i,
|
||||
chunkSize: chunk.length,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return allCreatedJobs;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue