From 58c5ba12000adca15312bea4c3e3f637540d0699 Mon Sep 17 00:00:00 2001 From: Boki Date: Wed, 11 Jun 2025 08:33:36 -0400 Subject: [PATCH] fixed up delay time --- .../src/providers/proxy.provider.ts | 22 +++++------ .../data-service/src/providers/proxy.tasks.ts | 38 ++++++++++--------- apps/data-service/src/routes/test.routes.ts | 38 ++++++++++++------- apps/data-service/src/utils/batch-helpers.ts | 10 +++-- 4 files changed, 61 insertions(+), 47 deletions(-) diff --git a/apps/data-service/src/providers/proxy.provider.ts b/apps/data-service/src/providers/proxy.provider.ts index a7a44f2..4da7b95 100644 --- a/apps/data-service/src/providers/proxy.provider.ts +++ b/apps/data-service/src/providers/proxy.provider.ts @@ -35,7 +35,7 @@ export const proxyProvider: ProviderConfig = { source: 'batch-processing' }), queueManager, { - totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '7') * 60 * 60 * 1000, + totalDelayHours: 0.1,//parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'), batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'), useBatching: process.env.PROXY_DIRECT_MODE !== 'true', priority: 2, @@ -114,16 +114,16 @@ export const proxyProvider: ProviderConfig = { } }, scheduledJobs: [ - // { - // type: 'proxy-maintenance', - // operation: 'fetch-and-check', - // payload: {}, - // // should remove and just run at the same time so app restarts dont keeping adding same jobs - // cronPattern: getEvery24HourCron(), - // priority: 5, - // immediately: true, // Don't run immediately during startup to avoid conflicts - // description: 'Fetch and validate proxy list from sources' - // } + { + type: 'proxy-maintenance', + operation: 'fetch-and-check', + payload: {}, + // should remove and just run at the same time so app restarts dont keeping adding same jobs + cronPattern: getEvery24HourCron(), + priority: 5, + immediately: true, // Don't run immediately during startup to avoid conflicts + description: 'Fetch and validate proxy list from sources' + } ] }; diff --git a/apps/data-service/src/providers/proxy.tasks.ts b/apps/data-service/src/providers/proxy.tasks.ts index c24fe2d..1c7ea48 100644 --- a/apps/data-service/src/providers/proxy.tasks.ts +++ b/apps/data-service/src/providers/proxy.tasks.ts @@ -26,30 +26,32 @@ const PROXY_CONFIG = { PROXY_SOURCES: [ {id: 'prxchk', url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt', protocol: 'http'}, {id: 'casals', url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http', protocol: 'http'}, - {id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'}, - {id: 'vakhov-fresh', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', protocol: 'http'}, {id: 'sunny9577', url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', protocol: 'http'}, - {id: 'kangproxy', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', protocol: 'http'}, - {id: 'gfpcom', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http'}, - {id: 'dpangestuw', url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', protocol: 'http'}, - {id: 'gitrecon', url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', protocol: 'http'}, {id: 'themiralay', url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt', protocol: 'http'}, - {id: 'vakhov-master', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', protocol: 'http'}, {id: 'casa-ls', url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http', protocol: 'http'}, {id: 'databay', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt', protocol: 'http'}, - {id: 'breaking-tech', url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http'}, {id: 'speedx', url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', protocol: 'http'}, - {id: 'ercindedeoglu', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', protocol: 'http'}, {id: 'monosans', url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', protocol: 'http'}, - {id: 'tuanminpay', url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', protocol: 'http'}, - // {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https' }, - // {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', }, + + // {id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'}, + // {id: 'vakhov-fresh', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', protocol: 'http'}, + // {id: 'kangproxy', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', protocol: 'http'}, + // {id: 'gfpcom', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http'}, + // {id: 'dpangestuw', url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', protocol: 'http'}, + // {id: 'gitrecon', url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', protocol: 'http'}, + // {id: 'vakhov-master', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', protocol: 'http'}, + // {id: 'breaking-tech', url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http'}, + // {id: 'ercindedeoglu', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', protocol: 'http'}, + // {id: 'tuanminpay', url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', protocol: 'http'}, + + // {id: 'r00tee-https', url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt', protocol: 'https'}, + // {id: 'ercindedeoglu-https', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt', protocol: 'https'}, + {id: 'vakhov-fresh-https', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https'}, + // {id: 'databay-https', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt', protocol: 'https'}, + // {id: 'kangproxy-https', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt', protocol: 'https'}, + // {id: 'zloi-user-https', url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt', protocol: 'https'}, + // {id: 'gfpcom-https', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt', protocol: 'https'}, ] }; @@ -289,7 +291,7 @@ export async function checkProxy(proxy: ProxyInfo): Promise { if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) { success = true; - await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL); + await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, { ttl: PROXY_CONFIG.CACHE_TTL }); } else { await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); } diff --git a/apps/data-service/src/routes/test.routes.ts b/apps/data-service/src/routes/test.routes.ts index bd3b4aa..8ebd6c2 100644 --- a/apps/data-service/src/routes/test.routes.ts +++ b/apps/data-service/src/routes/test.routes.ts @@ -12,22 +12,30 @@ export const testRoutes = new Hono(); // Test endpoint for new functional batch processing testRoutes.post('/api/test/batch-symbols', async (c) => { try { - const { symbols, useBatching = false, totalDelayMs = 60000 } = await c.req.json(); - const { processSymbols } = await import('../utils/batch-helpers'); + const { symbols, useBatching = false, totalDelayHours = 1 } = await c.req.json(); + const { processItems } = await import('../utils/batch-helpers'); if (!symbols || !Array.isArray(symbols)) { return c.json({ status: 'error', message: 'symbols array is required' }, 400); } - const result = await processSymbols(symbols, queueManager, { - operation: 'live-data', - service: 'test', - provider: 'test-provider', - totalDelayMs, - useBatching, - batchSize: 10, - priority: 1 - }); + const result = await processItems( + symbols, + (symbol, index) => ({ + symbol, + index, + timestamp: new Date().toISOString() + }), + queueManager, + { + totalDelayHours, + useBatching, + batchSize: 10, + priority: 1, + provider: 'test-provider', + operation: 'live-data' + } + ); return c.json({ status: 'success', @@ -42,7 +50,7 @@ testRoutes.post('/api/test/batch-symbols', async (c) => { testRoutes.post('/api/test/batch-custom', async (c) => { try { - const { items, useBatching = false, totalDelayMs = 30000 } = await c.req.json(); + const { items, useBatching = false, totalDelayHours = 0.5 } = await c.req.json(); const { processItems } = await import('../utils/batch-helpers'); if (!items || !Array.isArray(items)) { @@ -58,10 +66,12 @@ testRoutes.post('/api/test/batch-custom', async (c) => { }), queueManager, { - totalDelayMs, + totalDelayHours, useBatching, batchSize: 5, - priority: 1 + priority: 1, + provider: 'test-provider', + operation: 'custom-test' } ); diff --git a/apps/data-service/src/utils/batch-helpers.ts b/apps/data-service/src/utils/batch-helpers.ts index c441d11..e730dfc 100644 --- a/apps/data-service/src/utils/batch-helpers.ts +++ b/apps/data-service/src/utils/batch-helpers.ts @@ -6,7 +6,7 @@ const logger = getLogger('batch-helpers'); // Simple interfaces export interface ProcessOptions { - totalDelayMs: number; + totalDelayHours: number; batchSize?: number; priority?: number; useBatching?: boolean; @@ -76,7 +76,7 @@ export async function processItems( totalItems: items.length, mode: options.useBatching ? 'batch' : 'direct', batchSize: options.batchSize, - totalDelayHours: (options.totalDelayMs / 1000 / 60 / 60).toFixed(1) + totalDelayHours: options.totalDelayHours }); try { @@ -109,7 +109,8 @@ async function processDirect( options: ProcessOptions ): Promise> { - const delayPerItem = Math.floor(options.totalDelayMs / items.length); + const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; + const delayPerItem = Math.floor(totalDelayMs / items.length); logger.info('Creating direct jobs', { totalItems: items.length, @@ -155,7 +156,8 @@ async function processBatched( const batchSize = options.batchSize || 100; const batches = createBatches(items, batchSize); - const delayPerBatch = Math.floor(options.totalDelayMs / batches.length); + const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; + const delayPerBatch = Math.floor(totalDelayMs / batches.length); logger.info('Creating batch jobs', { totalItems: items.length,