boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

finished proxy-provider - need to test

Browse source
This commit is contained in:
Bojan Kucera 2025-06-09 08:07:45 -04:00
parent f78fa459d2
commit b2817656b3
2 changed files with 228 additions and 312 deletions
Download patch file Download diff file Expand all files Collapse all files

102
apps/data-service/src/providers/proxy.provider.ts
View file

@ -17,96 +17,50 @@ const getEvery24HourCron = (): string => {
export const proxyProvider: ProviderConfig = {
name: 'proxy-service',
service: 'proxy',
operations: { 'fetch-and-check': async (payload: { sources?: string[] }) => {
operations: {
'fetch-and-check': async (payload: { sources?: string[] }) => {
const { proxyService } = await import('./proxy.tasks');
const { queueManager } = await import('../services/queue.service');
await queueManager.drainQueue();
const proxies = await proxyService.fetchProxiesFromSources();
const proxiesCount = proxies.length;
if (proxiesCount === 0) {
logger.info('No proxies fetched, skipping job creation');
if (proxies.length === 0) {
return { proxiesFetched: 0, jobsCreated: 0 };
}
const batchProcessor = new BatchProcessor(queueManager);
// Environment-configurable settings
const targetHours = parseInt(process.env.PROXY_VALIDATION_HOURS || '8');
const batchSize = parseInt(process.env.PROXY_BATCH_SIZE || '200');
const useDirectMode = process.env.PROXY_DIRECT_MODE === 'true';
logger.info('Proxy validation configuration', {
targetHours,
batchSize,
useDirectMode,
totalProxies: proxies.length
// Simplified configuration
const result = await batchProcessor.processItems({
items: proxies,
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '8') * 60 * 60 * 1000,
jobNamePrefix: 'proxy',
operation: 'check-proxy',
service: 'proxy',
provider: 'proxy-service',
priority: 2,
useBatching: process.env.PROXY_DIRECT_MODE !== 'true', // Simple boolean flag
createJobData: (proxy: ProxyInfo) => ({
proxy,
source: 'fetch-and-check'
}),
removeOnComplete: 5,
removeOnFail: 3
});
if (useDirectMode) {
// Direct approach - simpler, creates all jobs immediately
const result = await batchProcessor.createDirectJobs({
items: proxies,
batchSize: 0, // Not used in direct mode
totalDelayMs: targetHours * 60 * 60 * 1000,
jobNamePrefix: 'proxy',
operation: 'check-proxy',
service: 'proxy',
provider: 'proxy-service',
priority: 2,
createJobData: (proxy: ProxyInfo) => ({
proxy,
source: 'fetch-and-check'
}),
removeOnComplete: 5,
removeOnFail: 3
});
return {
proxiesFetched: result.totalItems,
jobsCreated: result.jobsCreated,
mode: 'direct'
};
} else { // Batch approach - creates batch jobs that create individual jobs
const result = await batchProcessor.createBatchJobs({
items: proxies,
batchSize,
totalDelayMs: targetHours * 60 * 60 * 1000,
jobNamePrefix: 'proxy',
operation: 'check-proxy',
service: 'proxy',
provider: 'proxy-service',
priority: 3,
createJobData: (proxy: ProxyInfo) => ({
proxy,
source: 'fetch-and-check'
}),
removeOnComplete: 3,
removeOnFail: 5
});
return {
proxiesFetched: result.totalItems,
batchJobsCreated: result.batchJobsCreated,
totalBatches: result.totalBatches,
estimatedDurationHours: result.estimatedDurationHours,
mode: 'batch'
};
}
return {
proxiesFetched: result.totalItems,
...result
};
},
'process-proxy-batch': async (payload: any) => {
'process-proxy-batch': async (payload: any) => {
// Process a batch of proxies - uses the fetch-and-check JobNamePrefix process-(proxy)-batch
const { queueManager } = await import('../services/queue.service');
const batchProcessor = new BatchProcessor(queueManager);
return await batchProcessor.processBatch(
payload,
(proxy: ProxyInfo, index: number) => ({
proxy,
source: payload.config?.source || 'batch-processing'
})
);
return await batchProcessor.processBatch(payload);
},
'check-proxy': async (payload: {