95 lines
2.7 KiB
TypeScript
95 lines
2.7 KiB
TypeScript
/**
|
|
* Example usage of the new functional batch processing approach
|
|
*/
|
|
|
|
import { processItems, processSymbols, processProxies, processBatchJob } from '../utils/batch-helpers';
|
|
import { queueManager } from '../services/queue.service';
|
|
|
|
// Example 1: Process a list of symbols for live data
|
|
export async function exampleSymbolProcessing() {
|
|
const symbols = ['AAPL', 'GOOGL', 'MSFT', 'TSLA', 'AMZN'];
|
|
|
|
const result = await processSymbols(symbols, queueManager, {
|
|
operation: 'live-data',
|
|
service: 'market-data',
|
|
provider: 'yahoo',
|
|
totalDelayMs: 60000, // 1 minute total
|
|
useBatching: false, // Process directly
|
|
priority: 1
|
|
});
|
|
|
|
console.log('Symbol processing result:', result);
|
|
// Output: { jobsCreated: 5, mode: 'direct', totalItems: 5, duration: 1234 }
|
|
}
|
|
|
|
// Example 2: Process proxies in batches
|
|
export async function exampleProxyProcessing() {
|
|
const proxies = [
|
|
{ host: '1.1.1.1', port: 8080 },
|
|
{ host: '2.2.2.2', port: 3128 },
|
|
// ... more proxies
|
|
];
|
|
|
|
const result = await processProxies(proxies, queueManager, {
|
|
totalDelayMs: 3600000, // 1 hour total
|
|
useBatching: true, // Use batch mode
|
|
batchSize: 100, // 100 proxies per batch
|
|
priority: 2
|
|
});
|
|
|
|
console.log('Proxy processing result:', result);
|
|
// Output: { jobsCreated: 10, mode: 'batch', totalItems: 1000, batchesCreated: 10, duration: 2345 }
|
|
}
|
|
|
|
// Example 3: Custom processing with generic function
|
|
export async function exampleCustomProcessing() {
|
|
const customData = [
|
|
{ id: 1, name: 'Item 1' },
|
|
{ id: 2, name: 'Item 2' },
|
|
{ id: 3, name: 'Item 3' }
|
|
];
|
|
|
|
const result = await processItems(
|
|
customData,
|
|
(item, index) => ({
|
|
// Transform each item for processing
|
|
itemId: item.id,
|
|
itemName: item.name,
|
|
processIndex: index,
|
|
timestamp: new Date().toISOString()
|
|
}),
|
|
queueManager,
|
|
{
|
|
totalDelayMs: 30000, // 30 seconds total
|
|
useBatching: false, // Direct processing
|
|
priority: 1,
|
|
retries: 3
|
|
}
|
|
);
|
|
|
|
console.log('Custom processing result:', result);
|
|
}
|
|
|
|
// Example 4: Batch job processor (used by workers)
|
|
export async function exampleBatchJobProcessor(jobData: any) {
|
|
// This would be called by a BullMQ worker when processing batch jobs
|
|
const result = await processBatchJob(jobData, queueManager);
|
|
|
|
console.log('Batch job processed:', result);
|
|
// Output: { batchIndex: 0, itemsProcessed: 100, jobsCreated: 100 }
|
|
|
|
return result;
|
|
}
|
|
|
|
// Example: Simple functional approach
|
|
/*
|
|
await processSymbols(symbols, queueManager, {
|
|
operation: 'live-data',
|
|
service: 'data-service',
|
|
provider: 'yahoo',
|
|
totalDelayMs: 3600000,
|
|
useBatching: true,
|
|
batchSize: 200,
|
|
priority: 2
|
|
});
|
|
*/
|