From e4d5dba73af32b1038f3dd500d40cf95d2f55d88 Mon Sep 17 00:00:00 2001 From: Boki Date: Wed, 11 Jun 2025 11:11:47 -0400 Subject: [PATCH] testing --- .../src/providers/proxy.provider.ts | 2 +- .../data-service/src/providers/proxy.tasks.ts | 190 +++++++++--------- .../src/services/queue.service.ts | 9 +- apps/strategy-service/src/cli/index.ts | 2 +- 4 files changed, 103 insertions(+), 100 deletions(-) diff --git a/apps/data-service/src/providers/proxy.provider.ts b/apps/data-service/src/providers/proxy.provider.ts index 168f697..c1ea4a8 100644 --- a/apps/data-service/src/providers/proxy.provider.ts +++ b/apps/data-service/src/providers/proxy.provider.ts @@ -37,7 +37,7 @@ export const proxyProvider: ProviderConfig = { }), queueManager, { - totalDelayHours: 4, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'), + totalDelayHours: 0.1, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'), batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'), useBatching: process.env.PROXY_DIRECT_MODE !== 'true', priority: 2, diff --git a/apps/data-service/src/providers/proxy.tasks.ts b/apps/data-service/src/providers/proxy.tasks.ts index 16cc8ec..92cbc89 100644 --- a/apps/data-service/src/providers/proxy.tasks.ts +++ b/apps/data-service/src/providers/proxy.tasks.ts @@ -54,103 +54,103 @@ const PROXY_CONFIG = { url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt', protocol: 'http', }, - { - id: 'speedx', - url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', - protocol: 'http', - }, - { - id: 'monosans', - url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', - protocol: 'http', - }, + // { + // id: 'speedx', + // url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', + // protocol: 'http', + // }, + // { + // id: 'monosans', + // url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', + // protocol: 'http', + // }, - { - id: 'murong', - url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', - protocol: 'http', - }, - { - id: 'vakhov-fresh', - url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', - protocol: 'http', - }, - { - id: 'kangproxy', - url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', - protocol: 'http', - }, - { - id: 'gfpcom', - url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', - protocol: 'http', - }, - { - id: 'dpangestuw', - url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', - protocol: 'http', - }, - { - id: 'gitrecon', - url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', - protocol: 'http', - }, - { - id: 'vakhov-master', - url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', - protocol: 'http', - }, - { - id: 'breaking-tech', - url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', - protocol: 'http', - }, - { - id: 'ercindedeoglu', - url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', - protocol: 'http', - }, - { - id: 'tuanminpay', - url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', - protocol: 'http', - }, + // { + // id: 'murong', + // url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', + // protocol: 'http', + // }, + // { + // id: 'vakhov-fresh', + // url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', + // protocol: 'http', + // }, + // { + // id: 'kangproxy', + // url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', + // protocol: 'http', + // }, + // { + // id: 'gfpcom', + // url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', + // protocol: 'http', + // }, + // { + // id: 'dpangestuw', + // url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', + // protocol: 'http', + // }, + // { + // id: 'gitrecon', + // url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', + // protocol: 'http', + // }, + // { + // id: 'vakhov-master', + // url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', + // protocol: 'http', + // }, + // { + // id: 'breaking-tech', + // url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', + // protocol: 'http', + // }, + // { + // id: 'ercindedeoglu', + // url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', + // protocol: 'http', + // }, + // { + // id: 'tuanminpay', + // url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', + // protocol: 'http', + // }, - { - id: 'r00tee-https', - url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt', - protocol: 'https', - }, - { - id: 'ercindedeoglu-https', - url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt', - protocol: 'https', - }, - { - id: 'vakhov-fresh-https', - url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', - protocol: 'https', - }, - { - id: 'databay-https', - url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt', - protocol: 'https', - }, - { - id: 'kangproxy-https', - url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt', - protocol: 'https', - }, - { - id: 'zloi-user-https', - url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt', - protocol: 'https', - }, - { - id: 'gfpcom-https', - url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt', - protocol: 'https', - }, + // { + // id: 'r00tee-https', + // url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt', + // protocol: 'https', + // }, + // { + // id: 'ercindedeoglu-https', + // url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt', + // protocol: 'https', + // }, + // { + // id: 'vakhov-fresh-https', + // url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', + // protocol: 'https', + // }, + // { + // id: 'databay-https', + // url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt', + // protocol: 'https', + // }, + // { + // id: 'kangproxy-https', + // url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt', + // protocol: 'https', + // }, + // { + // id: 'zloi-user-https', + // url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt', + // protocol: 'https', + // }, + // { + // id: 'gfpcom-https', + // url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt', + // protocol: 'https', + // }, ], }; diff --git a/apps/data-service/src/services/queue.service.ts b/apps/data-service/src/services/queue.service.ts index e1a1dd2..4885f8d 100644 --- a/apps/data-service/src/services/queue.service.ts +++ b/apps/data-service/src/services/queue.service.ts @@ -9,8 +9,8 @@ export class QueueService { private queueEvents!: QueueEvents; private config = { - workers: parseInt(process.env.WORKER_COUNT || '5'), - concurrency: parseInt(process.env.WORKER_CONCURRENCY || '20'), + workers: 1, //parseInt(process.env.WORKER_COUNT || '5'), + concurrency: 1, //parseInt(process.env.WORKER_CONCURRENCY || '20'), redis: { host: process.env.DRAGONFLY_HOST || 'localhost', port: parseInt(process.env.DRAGONFLY_PORT || '6379'), @@ -45,7 +45,10 @@ export class QueueService { removeOnComplete: 10, removeOnFail: 5, attempts: 3, - backoff: { type: 'exponential', delay: 1000 }, + backoff: { + type: 'exponential', + delay: 1000, + }, }, }); diff --git a/apps/strategy-service/src/cli/index.ts b/apps/strategy-service/src/cli/index.ts index 24ce69f..94182b3 100644 --- a/apps/strategy-service/src/cli/index.ts +++ b/apps/strategy-service/src/cli/index.ts @@ -282,4 +282,4 @@ program // Parse command line arguments program.parse(); -export { runBacktest, listStrategies, validateStrategy }; +export { listStrategies, runBacktest, validateStrategy };