thing im pretty much done with extracting the queue and making it reususable, maybe just a few more change to be able to making the batch names a bit more specific
This commit is contained in:
parent
ad5e353ec3
commit
e8c90532d5
14 changed files with 117 additions and 93 deletions
|
|
@ -35,7 +35,7 @@ async function debugBatchCleanup() {
|
|||
|
||||
// Process in batches
|
||||
const result = await processItems(items, queueManager, {
|
||||
totalDelayMs: 10000, // 10 seconds total
|
||||
totalDelayHours: 0.0028, // 10 seconds
|
||||
useBatching: true,
|
||||
batchSize: 3, // This will create 3 batches: [3,3,1]
|
||||
priority: 1,
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ async function basicUsageExample() {
|
|||
const symbols = ['GOOGL', 'MSFT', 'TSLA', 'AMZN'];
|
||||
|
||||
const result = await processItems(symbols, queueManager, {
|
||||
totalDelayMs: 30000, // 30 seconds total
|
||||
totalDelayHours: 0.0083, // 30 seconds
|
||||
useBatching: true,
|
||||
batchSize: 2,
|
||||
priority: 1,
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ async function batchProcessingExample() {
|
|||
[1, 2, 3, 4, 5], // Just pass the array directly!
|
||||
queueManager,
|
||||
{
|
||||
totalDelayMs: 15000, // 15 seconds total
|
||||
totalDelayHours: 0.0042, // 15 seconds
|
||||
useBatching: false, // Direct mode
|
||||
priority: 2,
|
||||
provider: 'data-processor',
|
||||
|
|
@ -63,7 +63,7 @@ async function batchProcessingExample() {
|
|||
}));
|
||||
|
||||
const batchResult = await processItems(batchData, queueManager, {
|
||||
totalDelayMs: 20000, // 20 seconds total
|
||||
totalDelayHours: 0.0056, // 20 seconds
|
||||
useBatching: true, // Batch mode
|
||||
batchSize: 5, // 5 items per batch
|
||||
priority: 1,
|
||||
|
|
@ -80,7 +80,7 @@ async function batchProcessingExample() {
|
|||
const symbolResult = await processItems(symbols, queueManager, {
|
||||
operation: 'analyze-symbol',
|
||||
provider: 'data-processor',
|
||||
totalDelayMs: 25000, // 25 seconds total
|
||||
totalDelayHours: 0.0069, // 25 seconds
|
||||
useBatching: true,
|
||||
batchSize: 3,
|
||||
priority: 1,
|
||||
|
|
@ -97,7 +97,7 @@ async function batchProcessingExample() {
|
|||
}));
|
||||
|
||||
const largeResult = await processItems(largeDataset, queueManager, {
|
||||
totalDelayMs: 60000, // 1 minute total
|
||||
totalDelayHours: 0.0167, // 1 minute
|
||||
useBatching: true,
|
||||
batchSize: 50, // 50 items per batch
|
||||
priority: 3,
|
||||
|
|
@ -160,7 +160,7 @@ async function compareProcessingModes() {
|
|||
console.log('Testing direct mode...');
|
||||
const directStart = Date.now();
|
||||
const directResult = await processItems(testData, queueManager, {
|
||||
totalDelayMs: 10000,
|
||||
totalDelayHours: 0.0028, // 10 seconds
|
||||
useBatching: false,
|
||||
provider: 'test',
|
||||
operation: 'process',
|
||||
|
|
@ -174,7 +174,7 @@ async function compareProcessingModes() {
|
|||
console.log('Testing batch mode...');
|
||||
const batchStart = Date.now();
|
||||
const batchResult = await processItems(testData, queueManager, {
|
||||
totalDelayMs: 10000,
|
||||
totalDelayHours: 0.0028, // 10 seconds
|
||||
useBatching: true,
|
||||
batchSize: 5,
|
||||
provider: 'test',
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ export async function processItems<T>(
|
|||
totalItems: items.length,
|
||||
mode: options.useBatching ? 'batch' : 'direct',
|
||||
batchSize: options.batchSize,
|
||||
totalDelayMs: options.totalDelayMs,
|
||||
totalDelayHours: options.totalDelayHours,
|
||||
});
|
||||
|
||||
try {
|
||||
|
|
@ -86,7 +86,8 @@ async function processDirect<T>(
|
|||
queue: QueueManager,
|
||||
options: ProcessOptions
|
||||
): Promise<Omit<BatchResult, 'duration'>> {
|
||||
const delayPerItem = options.totalDelayMs / items.length;
|
||||
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; // Convert hours to milliseconds
|
||||
const delayPerItem = totalDelayMs / items.length;
|
||||
|
||||
logger.info('Creating direct jobs', {
|
||||
totalItems: items.length,
|
||||
|
|
@ -130,7 +131,8 @@ async function processBatched<T>(
|
|||
): Promise<Omit<BatchResult, 'duration'>> {
|
||||
const batchSize = options.batchSize || 100;
|
||||
const batches = createBatches(items, batchSize);
|
||||
const delayPerBatch = options.totalDelayMs / batches.length;
|
||||
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; // Convert hours to milliseconds
|
||||
const delayPerBatch = totalDelayMs / batches.length;
|
||||
|
||||
logger.info('Creating batch jobs', {
|
||||
totalItems: items.length,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ export interface JobData {
|
|||
}
|
||||
|
||||
export interface ProcessOptions {
|
||||
totalDelayMs: number;
|
||||
totalDelayHours: number;
|
||||
batchSize?: number;
|
||||
priority?: number;
|
||||
useBatching?: boolean;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ async function quickTest() {
|
|||
// Verify the processItems function signature
|
||||
const items = [1, 2, 3];
|
||||
const options = {
|
||||
totalDelayMs: 1000,
|
||||
totalDelayHours: 0.0003, // ~1 second
|
||||
useBatching: false,
|
||||
provider: 'test',
|
||||
operation: 'test',
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ async function testSimplifiedAPI() {
|
|||
// Test 1: Simple array of numbers
|
||||
const numbers = [1, 2, 3, 4, 5];
|
||||
const result1 = await processItems(numbers, queueManager, {
|
||||
totalDelayMs: 5000,
|
||||
totalDelayHours: 0.0014, // ~5 seconds (5/3600 hours)
|
||||
useBatching: false,
|
||||
provider: 'test-provider',
|
||||
operation: 'process-item',
|
||||
|
|
@ -46,7 +46,7 @@ async function testSimplifiedAPI() {
|
|||
];
|
||||
|
||||
const result2 = await processItems(objects, queueManager, {
|
||||
totalDelayMs: 5000,
|
||||
totalDelayHours: 0.0014, // ~5 seconds
|
||||
useBatching: true,
|
||||
batchSize: 2,
|
||||
provider: 'test-provider',
|
||||
|
|
@ -59,7 +59,7 @@ async function testSimplifiedAPI() {
|
|||
const symbols = Array.from({ length: 1000 }, (_, i) => `Symbol-${i + 1}`);
|
||||
console.log('📋 Testing with symbols...');
|
||||
const result3 = await processItems(symbols, queueManager, {
|
||||
totalDelayMs: 3000,
|
||||
totalDelayHours: 0.0008, // ~3 seconds
|
||||
useBatching: true,
|
||||
batchSize: 1,
|
||||
provider: 'test-provider',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue