boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

added routes and simplified batch processor

Browse source
This commit is contained in:
Boki 2025-06-10 20:59:53 -04:00
parent 0357908b69
commit 4883daa3e2
12 changed files with 1130 additions and 238 deletions
Download patch file Download diff file Expand all files Collapse all files

54
apps/data-service/src/providers/proxy.provider.ts
View file

@ -1,7 +1,6 @@
import { ProxyInfo } from 'libs/http/src/types';
import { ProviderConfig } from '../services/provider-registry.service';
import { getLogger } from '@stock-bot/logger';
import { BatchProcessor } from '../utils/batch-processor';
// Create logger for this provider
const logger = getLogger('proxy-provider');
@ -17,10 +16,10 @@ const getEvery24HourCron = (): string => {
export const proxyProvider: ProviderConfig = {
name: 'proxy-service',
service: 'proxy',
operations: {
'fetch-and-check': async (payload: { sources?: string[] }) => {
operations: { 'fetch-and-check': async (payload: { sources?: string[] }) => {
const { proxyService } = await import('./proxy.tasks');
const { queueManager } = await import('../services/queue.service');
const { processProxies } = await import('../utils/batch-helpers');
const proxies = await proxyService.fetchProxiesFromSources();
@ -28,44 +27,25 @@ export const proxyProvider: ProviderConfig = {
return { proxiesFetched: 0, jobsCreated: 0 };
}
const batchProcessor = new BatchProcessor(queueManager);
// Simplified configuration
const result = await batchProcessor.processItems({
items: proxies,
// Use simplified functional approach
const result = await processProxies(proxies, queueManager, {
totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '4') * 60 * 60 * 1000,
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '4') * 60 * 60 * 1000 ,
jobNamePrefix: 'proxy',
operation: 'check-proxy',
service: 'proxy',
provider: 'proxy-service',
priority: 2,
useBatching: process.env.PROXY_DIRECT_MODE !== 'true', // Simple boolean flag
createJobData: (proxy: ProxyInfo) => ({
proxy,
source: 'fetch-and-check'
}),
removeOnComplete: 5,
removeOnFail: 3
});
return {
useBatching: process.env.PROXY_DIRECT_MODE !== 'true',
priority: 2
}); return {
proxiesFetched: result.totalItems,
...result
jobsCreated: result.jobsCreated,
mode: result.mode,
batchesCreated: result.batchesCreated,
processingTimeMs: result.duration
};
},
'process-proxy-batch': async (payload: any) => {
// Process a batch of proxies - uses the fetch-and-check JobNamePrefix process-(proxy)-batch
}, 'process-proxy-batch': async (payload: any) => {
// Process a batch using the simplified batch helpers
const { processBatchJob } = await import('../utils/batch-helpers');
const { queueManager } = await import('../services/queue.service');
const batchProcessor = new BatchProcessor(queueManager);
return await batchProcessor.processBatch(
payload,
(proxy: ProxyInfo) => ({
proxy,
source: payload.config?.source || 'batch-processing'
})
);
return await processBatchJob(payload, queueManager);
},
'check-proxy': async (payload: {