diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index 4fd2c55..d40cdee 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -3,56 +3,42 @@ import createCache, { type CacheProvider } from '@stock-bot/cache'; import { HttpClient, ProxyInfo } from '@stock-bot/http'; import pLimit from 'p-limit'; -export interface ProxyCheckResult { - proxy: ProxyInfo; - isWorking: boolean; - responseTime: number; - error?: string; - checkedAt: Date; -} - -export interface ProxyStats { - total: number; - working: number; - avgResponseTime: number; -} - export class ProxyService { private logger = new Logger('proxy-service'); private cache: CacheProvider = createCache('hybrid'); private httpClient: HttpClient; - private readonly concurrencyLimit = pLimit(1000); + private readonly concurrencyLimit = pLimit(500); private readonly CACHE_KEY = 'proxy'; private readonly CACHE_TTL = 86400; // 24 hours private readonly CHECK_TIMEOUT = 5000; - private readonly CHECK_URL = 'https://httpbin.org/ip'; + private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; private readonly PROXY_SOURCES = [ - {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, - {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, ] constructor() { @@ -69,10 +55,26 @@ export class ProxyService { this.concurrencyLimit(() => this.fetchProxiesFromSource(source)) ) const result = await Promise.all(sources); - this.checkProxies(result.flat()) + const allProxies: ProxyInfo[] = result.flat(); + await this.checkProxies(this.removeDuplicateProxies(allProxies)) return true } + private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] { + const seen = new Set(); + const unique: ProxyInfo[] = []; + + for (const proxy of proxies) { + const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`; + if (!seen.has(key)) { + seen.add(key); + unique.push(proxy); + } + } + + return unique; + } + async fetchProxiesFromSource(source: { url: string; protocol: string }): Promise { const allProxies: ProxyInfo[] = []; @@ -125,9 +127,8 @@ export class ProxyService { /** * Check if a proxy is working */ - async checkProxy(proxy: ProxyInfo): Promise { - const startTime = Date.now(); - console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`); + async checkProxy(proxy: ProxyInfo): Promise { + console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount); try { // Test the proxy @@ -135,19 +136,15 @@ export class ProxyService { proxy, timeout: this.CHECK_TIMEOUT }); - - const responseTime = Date.now() - startTime; + const isWorking = response.status >= 200 && response.status < 300; - const result: ProxyCheckResult = { - proxy, + const result: ProxyInfo = { + ...proxy, isWorking, - responseTime, - checkedAt: new Date() + checkedAt: new Date(), + responseTime: response.responseTime, }; - - // Cache the result - // await this.cache.set(cacheKey, result, this.CACHE_TTL); if (isWorking) { await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL); @@ -159,26 +156,23 @@ export class ProxyService { host: proxy.host, port: proxy.port, isWorking, - responseTime }); return result; } catch (error) { - const responseTime = Date.now() - startTime; const errorMessage = error instanceof Error ? error.message : String(error); - const result: ProxyCheckResult = { - proxy, + const result: ProxyInfo = { + ...proxy, isWorking: false, - responseTime, error: errorMessage, checkedAt: new Date() }; // Cache failed result for shorter time // await this.cache.set(cacheKey, result, 300); // 5 minutes - await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); + // await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); this.logger.debug('Proxy check failed', { host: proxy.host, @@ -193,7 +187,7 @@ export class ProxyService { /** * Check multiple proxies concurrently */ - async checkProxies(proxies: ProxyInfo[]): Promise { + async checkProxies(proxies: ProxyInfo[]): Promise { this.logger.info('Checking proxies', { count: proxies.length }); const checkPromises = proxies.map(proxy => @@ -239,18 +233,6 @@ export class ProxyService { }); } - /** - * Get proxy statistics - */ - async getStats(): Promise { - // Simplified stats - in production you'd track these properly - return { - total: 0, - working: 0, - avgResponseTime: 0 - }; - } - /** * Clear proxy cache */ diff --git a/libs/http/src/client.ts b/libs/http/src/client.ts index 2f312c5..732eb1b 100644 --- a/libs/http/src/client.ts +++ b/libs/http/src/client.ts @@ -7,6 +7,7 @@ import type { import { HttpError } from './types.js'; import { ProxyManager } from './proxy-manager.js'; import got from 'got'; +import { clear } from 'node:console'; export class HttpClient { private readonly config: HttpClientConfig; @@ -43,6 +44,7 @@ export class HttpClient { */ async request(config: RequestConfig): Promise> { const finalConfig = this.mergeConfig(config); + const startTime = Date.now(); this.logger?.debug('Making HTTP request', { method: finalConfig.method, url: finalConfig.url, @@ -55,11 +57,14 @@ export class HttpClient { const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy); const response = await this.makeRequest(finalConfig, useBunFetch); - + const responseTime = Date.now() - startTime; + response.responseTime = responseTime; + this.logger?.debug('HTTP request successful', { method: finalConfig.method, url: finalConfig.url, status: response.status, + responseTime: responseTime, }); return response; @@ -79,28 +84,41 @@ export class HttpClient { private async makeRequest(config: RequestConfig, useBunFetch: boolean): Promise> { const timeout = config.timeout ?? this.config.timeout ?? 30000; const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout); + + // Create timeout promise that rejects with proper error + let timeoutId: NodeJS.Timeout | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + controller.abort(); + reject(new HttpError(`Request timeout after ${timeout}ms`)); + }, timeout); + }); + + // Create request promise (don't await here!) + const requestPromise = useBunFetch + ? this.fetchRequest(config, controller.signal) + : this.gotRequest(config, controller.signal); try { - const response = useBunFetch - ? await this.fetchRequest(config, controller.signal) - : await this.gotRequest(config, controller.signal); - - clearTimeout(timeoutId); - return response; + // Race the promises + const result = await Promise.race([requestPromise, timeoutPromise]); + if (timeoutId) clearTimeout(timeoutId); + return result; } catch (error) { - clearTimeout(timeoutId); - - // Unified timeout error handling - if (controller.signal.aborted) { - throw new HttpError(`Request timeout after ${timeout}ms`); - } - if ((error as any).name === 'TimeoutError') { - throw new HttpError(`Request timeout after ${timeout}ms`); + // If it's our timeout error, handle it + if (error instanceof HttpError && error.message.includes('timeout')) { + this.logger?.warn('Request timed out', { + method: config.method, + url: config.url, + timeout: timeout, + }); + throw error; // Re-throw the timeout error } - throw error; // Re-throw other errors as-is + // Handle other errors (network, parsing, etc.) + throw error; } + } /** * Bun fetch implementation (simplified) @@ -108,9 +126,9 @@ export class HttpClient { private async fetchRequest(config: RequestConfig, signal: AbortSignal): Promise> { try { - // const options = this.buildFetchOptions(config, signal); + const options = this.buildFetchOptions(config, signal); // console.log('Using fetch with proxy:', config.proxy); - const options = config.proxy? { proxy: config.proxy.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal); + // const options = config.proxy? { proxy: config.proxy.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal); this.logger?.debug('Making request with fetch: ', { url: config.url, options }) diff --git a/libs/http/src/types.ts b/libs/http/src/types.ts index a7e4829..783207f 100644 --- a/libs/http/src/types.ts +++ b/libs/http/src/types.ts @@ -32,6 +32,7 @@ export interface HttpResponse { status: number; headers: Record; ok: boolean; + responseTime?: number; } export class HttpError extends Error {