boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fixed batcher for proxy

Browse source
This commit is contained in:
Boki 2025-06-15 09:08:28 -04:00
parent e8c90532d5
commit 98d5f43eb8
5 changed files with 15 additions and 565 deletions
Download patch file Download diff file Expand all files Collapse all files

34
apps/data-service/src/providers/proxy.provider.ts
View file

@ -21,7 +21,7 @@ export function initializeProxyProvider() {
logger.info('Processing fetch proxies from sources request');
const { fetchProxiesFromSources } = await import('./proxy.tasks');
const { processItems, queueManager } = await import('../index');
// Fetch all proxies from sources
const proxies = await fetchProxiesFromSources();
logger.info('Fetched proxies from sources', { count: proxies.length });
@ -32,36 +32,32 @@ export function initializeProxyProvider() {
}
// Batch process the proxies through check-proxy operation
const batchResult = await processItems(
proxies,
queueManager,
{
provider: 'proxy',
operation: 'check-proxy',
totalDelayHours: 0.083, // 5 minutes (5/60 hours)
batchSize: 50, // Process 50 proxies per batch
priority: 3,
useBatching: true,
retries: 1,
ttl: 30000, // 30 second timeout per proxy check
removeOnComplete: 5,
removeOnFail: 3,
}
);
const batchResult = await processItems(proxies, queueManager, {
provider: 'proxy',
operation: 'check-proxy',
totalDelayHours: 0.083, // 5 minutes (5/60 hours)
batchSize: 50, // Process 50 proxies per batch
priority: 3,
useBatching: true,
retries: 1,
ttl: 30000, // 30 second timeout per proxy check
removeOnComplete: 5,
removeOnFail: 3,
});
logger.info('Batch proxy validation completed', {
totalProxies: proxies.length,
jobsCreated: batchResult.jobsCreated,
mode: batchResult.mode,
batchesCreated: batchResult.batchesCreated,
duration: `${batchResult.duration}ms`
duration: `${batchResult.duration}ms`,
});
return {
processed: proxies.length,
jobsCreated: batchResult.jobsCreated,
batchesCreated: batchResult.batchesCreated,
mode: batchResult.mode
mode: batchResult.mode,
};
},