diff --git a/apps/data-service/src/proxy-demo.ts b/apps/data-service/src/proxy-demo.ts index fed20d7..e069047 100644 --- a/apps/data-service/src/proxy-demo.ts +++ b/apps/data-service/src/proxy-demo.ts @@ -10,7 +10,8 @@ async function demonstrateCustomProxySource() { logger.info('🔧 Demonstrating custom proxy source...'); try { - const count = await proxyService.fetchProxiesFromSources(); + await proxyService.fetchProxiesFromSources(); + console.log('🔧 Demonstrating custom proxy source is DONE!'); } catch (error) { logger.error('❌ Custom source scraping failed',{ error: error diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index d40cdee..4ea7e97 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -7,12 +7,13 @@ export class ProxyService { private logger = new Logger('proxy-service'); private cache: CacheProvider = createCache('hybrid'); private httpClient: HttpClient; - private readonly concurrencyLimit = pLimit(500); + private readonly concurrencyLimit = pLimit(10); private readonly CACHE_KEY = 'proxy'; private readonly CACHE_TTL = 86400; // 24 hours private readonly CHECK_TIMEOUT = 5000; private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; private readonly PROXY_SOURCES = [ + // { url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http' }, // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, @@ -22,23 +23,25 @@ export class ProxyService { // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, + {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks5.txt',protocol: 'socks5', }, ] constructor() { diff --git a/docker-compose.yml b/docker-compose.yml index 265e0aa..7f754c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: # Dragonfly - Redis replacement for caching and events dragonfly: @@ -11,9 +9,10 @@ services: - dragonfly - --logtostderr - --cache_mode=true - - --maxmemory=2gb + - --maxmemory=8gb - --proactor_threads=8 - --bind=0.0.0.0 + - --admin_port=6380 volumes: - dragonfly_data:/data restart: unless-stopped