thing im pretty much done with extracting the queue and making it reususable, maybe just a few more change to be able to making the batch names a bit more specific

This commit is contained in:
Boki 2025-06-14 18:22:28 -04:00
parent ad5e353ec3
commit e8c90532d5
14 changed files with 117 additions and 93 deletions

View file

@ -35,7 +35,7 @@ async function debugBatchCleanup() {
// Process in batches
const result = await processItems(items, queueManager, {
totalDelayMs: 10000, // 10 seconds total
totalDelayHours: 0.0028, // 10 seconds
useBatching: true,
batchSize: 3, // This will create 3 batches: [3,3,1]
priority: 1,

View file

@ -54,7 +54,7 @@ async function basicUsageExample() {
const symbols = ['GOOGL', 'MSFT', 'TSLA', 'AMZN'];
const result = await processItems(symbols, queueManager, {
totalDelayMs: 30000, // 30 seconds total
totalDelayHours: 0.0083, // 30 seconds
useBatching: true,
batchSize: 2,
priority: 1,

View file

@ -44,7 +44,7 @@ async function batchProcessingExample() {
[1, 2, 3, 4, 5], // Just pass the array directly!
queueManager,
{
totalDelayMs: 15000, // 15 seconds total
totalDelayHours: 0.0042, // 15 seconds
useBatching: false, // Direct mode
priority: 2,
provider: 'data-processor',
@ -63,7 +63,7 @@ async function batchProcessingExample() {
}));
const batchResult = await processItems(batchData, queueManager, {
totalDelayMs: 20000, // 20 seconds total
totalDelayHours: 0.0056, // 20 seconds
useBatching: true, // Batch mode
batchSize: 5, // 5 items per batch
priority: 1,
@ -80,7 +80,7 @@ async function batchProcessingExample() {
const symbolResult = await processItems(symbols, queueManager, {
operation: 'analyze-symbol',
provider: 'data-processor',
totalDelayMs: 25000, // 25 seconds total
totalDelayHours: 0.0069, // 25 seconds
useBatching: true,
batchSize: 3,
priority: 1,
@ -97,7 +97,7 @@ async function batchProcessingExample() {
}));
const largeResult = await processItems(largeDataset, queueManager, {
totalDelayMs: 60000, // 1 minute total
totalDelayHours: 0.0167, // 1 minute
useBatching: true,
batchSize: 50, // 50 items per batch
priority: 3,
@ -160,7 +160,7 @@ async function compareProcessingModes() {
console.log('Testing direct mode...');
const directStart = Date.now();
const directResult = await processItems(testData, queueManager, {
totalDelayMs: 10000,
totalDelayHours: 0.0028, // 10 seconds
useBatching: false,
provider: 'test',
operation: 'process',
@ -174,7 +174,7 @@ async function compareProcessingModes() {
console.log('Testing batch mode...');
const batchStart = Date.now();
const batchResult = await processItems(testData, queueManager, {
totalDelayMs: 10000,
totalDelayHours: 0.0028, // 10 seconds
useBatching: true,
batchSize: 5,
provider: 'test',

View file

@ -56,7 +56,7 @@ export async function processItems<T>(
totalItems: items.length,
mode: options.useBatching ? 'batch' : 'direct',
batchSize: options.batchSize,
totalDelayMs: options.totalDelayMs,
totalDelayHours: options.totalDelayHours,
});
try {
@ -86,7 +86,8 @@ async function processDirect<T>(
queue: QueueManager,
options: ProcessOptions
): Promise<Omit<BatchResult, 'duration'>> {
const delayPerItem = options.totalDelayMs / items.length;
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; // Convert hours to milliseconds
const delayPerItem = totalDelayMs / items.length;
logger.info('Creating direct jobs', {
totalItems: items.length,
@ -130,7 +131,8 @@ async function processBatched<T>(
): Promise<Omit<BatchResult, 'duration'>> {
const batchSize = options.batchSize || 100;
const batches = createBatches(items, batchSize);
const delayPerBatch = options.totalDelayMs / batches.length;
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; // Convert hours to milliseconds
const delayPerBatch = totalDelayMs / batches.length;
logger.info('Creating batch jobs', {
totalItems: items.length,

View file

@ -8,7 +8,7 @@ export interface JobData {
}
export interface ProcessOptions {
totalDelayMs: number;
totalDelayHours: number;
batchSize?: number;
priority?: number;
useBatching?: boolean;

View file

@ -22,7 +22,7 @@ async function quickTest() {
// Verify the processItems function signature
const items = [1, 2, 3];
const options = {
totalDelayMs: 1000,
totalDelayHours: 0.0003, // ~1 second
useBatching: false,
provider: 'test',
operation: 'test',

View file

@ -30,7 +30,7 @@ async function testSimplifiedAPI() {
// Test 1: Simple array of numbers
const numbers = [1, 2, 3, 4, 5];
const result1 = await processItems(numbers, queueManager, {
totalDelayMs: 5000,
totalDelayHours: 0.0014, // ~5 seconds (5/3600 hours)
useBatching: false,
provider: 'test-provider',
operation: 'process-item',
@ -46,7 +46,7 @@ async function testSimplifiedAPI() {
];
const result2 = await processItems(objects, queueManager, {
totalDelayMs: 5000,
totalDelayHours: 0.0014, // ~5 seconds
useBatching: true,
batchSize: 2,
provider: 'test-provider',
@ -59,7 +59,7 @@ async function testSimplifiedAPI() {
const symbols = Array.from({ length: 1000 }, (_, i) => `Symbol-${i + 1}`);
console.log('📋 Testing with symbols...');
const result3 = await processItems(symbols, queueManager, {
totalDelayMs: 3000,
totalDelayHours: 0.0008, // ~3 seconds
useBatching: true,
batchSize: 1,
provider: 'test-provider',