98 lines
3.3 KiB
TypeScript
98 lines
3.3 KiB
TypeScript
import { ProxyInfo } from 'libs/http/src/types';
|
|
import { getLogger } from '@stock-bot/logger';
|
|
import { ProviderConfig } from '../services/provider-registry.service';
|
|
|
|
// Create logger for this provider
|
|
const logger = getLogger('proxy-provider');
|
|
|
|
// This will run at the same time each day as when the app started
|
|
const getEvery24HourCron = (): string => {
|
|
const now = new Date();
|
|
const hours = now.getHours();
|
|
const minutes = now.getMinutes();
|
|
return `${minutes} ${hours} * * *`; // Every day at startup time
|
|
};
|
|
|
|
export const proxyProvider: ProviderConfig = {
|
|
name: 'proxy-provider',
|
|
operations: {
|
|
'fetch-and-check': async (_payload: { sources?: string[] }) => {
|
|
const { proxyService } = await import('./proxy.tasks');
|
|
const { queueManager } = await import('../services/queue.service');
|
|
const { processItems } = await import('../utils/batch-helpers');
|
|
|
|
const proxies = await proxyService.fetchProxiesFromSources();
|
|
|
|
if (proxies.length === 0) {
|
|
return { proxiesFetched: 0, jobsCreated: 0 };
|
|
}
|
|
|
|
// Use generic function with routing parameters
|
|
const result = await processItems(
|
|
proxies,
|
|
(proxy, index) => ({
|
|
proxy,
|
|
index,
|
|
source: 'batch-processing',
|
|
}),
|
|
queueManager,
|
|
{
|
|
totalDelayHours: 12, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
|
|
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
|
|
useBatching: process.env.PROXY_DIRECT_MODE !== 'true',
|
|
provider: 'proxy-provider',
|
|
operation: 'check-proxy',
|
|
}
|
|
);
|
|
return result;
|
|
},
|
|
'process-batch-items': async (payload: any) => {
|
|
// Process a batch using the simplified batch helpers
|
|
const { processBatchJob } = await import('../utils/batch-helpers');
|
|
const { queueManager } = await import('../services/queue.service');
|
|
|
|
return await processBatchJob(payload, queueManager);
|
|
},
|
|
|
|
'check-proxy': async (payload: {
|
|
proxy: ProxyInfo;
|
|
source?: string;
|
|
batchIndex?: number;
|
|
itemIndex?: number;
|
|
total?: number;
|
|
}) => {
|
|
const { checkProxy } = await import('./proxy.tasks');
|
|
|
|
try {
|
|
const result = await checkProxy(payload.proxy);
|
|
|
|
logger.debug('Proxy validated', {
|
|
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
|
isWorking: result.isWorking,
|
|
responseTime: result.responseTime,
|
|
});
|
|
|
|
return { result, proxy: payload.proxy };
|
|
} catch (error) {
|
|
logger.warn('Proxy validation failed', {
|
|
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
});
|
|
|
|
return { result: { isWorking: false, error: String(error) }, proxy: payload.proxy };
|
|
}
|
|
},
|
|
},
|
|
scheduledJobs: [
|
|
// {
|
|
// type: 'proxy-maintenance',
|
|
// operation: 'fetch-and-check',
|
|
// payload: {},
|
|
// // should remove and just run at the same time so app restarts dont keeping adding same jobs
|
|
// cronPattern: getEvery24HourCron(),
|
|
// priority: 5,
|
|
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
|
// description: 'Fetch and validate proxy list from sources',
|
|
// },
|
|
],
|
|
};
|