boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fixed batching and waiting priority plus cleanup

Browse source
This commit is contained in:
Boki 2025-06-11 12:56:07 -04:00
parent e4d5dba73a
commit 6fb98c69f2
6 changed files with 121 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

61
apps/data-service/src/providers/proxy.provider.ts
View file

@ -37,21 +37,14 @@ export const proxyProvider: ProviderConfig = {
}),
queueManager,
{
totalDelayHours: 0.1, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
totalDelayHours: 12, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
useBatching: process.env.PROXY_DIRECT_MODE !== 'true',
priority: 2,
provider: 'proxy-provider',
operation: 'check-proxy',
}
);
return {
proxiesFetched: result.totalItems,
jobsCreated: result.jobsCreated,
mode: result.mode,
batchesCreated: result.batchesCreated,
processingTimeMs: result.duration,
};
return result;
},
'process-batch-items': async (payload: any) => {
// Process a batch using the simplified batch helpers
@ -77,55 +70,29 @@ export const proxyProvider: ProviderConfig = {
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
isWorking: result.isWorking,
responseTime: result.responseTime,
batchIndex: payload.batchIndex,
});
return {
result,
proxy: payload.proxy,
// Only include batch info if it exists (for batch mode)
...(payload.batchIndex !== undefined && {
batchInfo: {
batchIndex: payload.batchIndex,
itemIndex: payload.itemIndex,
total: payload.total,
source: payload.source,
},
}),
};
return { result, proxy: payload.proxy };
} catch (error) {
logger.warn('Proxy validation failed', {
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
error: error instanceof Error ? error.message : String(error),
batchIndex: payload.batchIndex,
});
return {
result: { isWorking: false, error: String(error) },
proxy: payload.proxy,
// Only include batch info if it exists (for batch mode)
...(payload.batchIndex !== undefined && {
batchInfo: {
batchIndex: payload.batchIndex,
itemIndex: payload.itemIndex,
total: payload.total,
source: payload.source,
},
}),
};
return { result: { isWorking: false, error: String(error) }, proxy: payload.proxy };
}
},
},
scheduledJobs: [
{
type: 'proxy-maintenance',
operation: 'fetch-and-check',
payload: {},
// should remove and just run at the same time so app restarts dont keeping adding same jobs
cronPattern: getEvery24HourCron(),
priority: 5,
immediately: true, // Don't run immediately during startup to avoid conflicts
description: 'Fetch and validate proxy list from sources',
},
// {
// type: 'proxy-maintenance',
// operation: 'fetch-and-check',
// payload: {},
// // should remove and just run at the same time so app restarts dont keeping adding same jobs
// cronPattern: getEvery24HourCron(),
// priority: 5,
// immediately: true, // Don't run immediately during startup to avoid conflicts
// description: 'Fetch and validate proxy list from sources',
// },
],
};