diff --git a/apps/data-service/src/proxy-demo.ts b/apps/data-service/src/proxy-demo.ts index d85ac19..fed20d7 100644 --- a/apps/data-service/src/proxy-demo.ts +++ b/apps/data-service/src/proxy-demo.ts @@ -1,116 +1,19 @@ -import { proxyService, ProxySource } from './services/proxy.service.js'; -import { getLogger, shutdownLoggers } from '@stock-bot/logger'; -import { onShutdown, setShutdownTimeout } from '@stock-bot/shutdown'; +import { proxyService } from './services/proxy.service.js'; +import { getLogger } from '@stock-bot/logger'; // Initialize logger for the demo const logger = getLogger('proxy-demo'); - -/** - * Example usage of the ProxyService with enhanced logging - */ -// async function demonstrateProxyService() { -// console.log('๐Ÿš€ Starting Proxy Service Demo...'); - -// try { -// // 1. Start the proxy refresh job (scrapes proxies every 30 minutes) -// console.log('๐Ÿ“ฅ Starting proxy refresh job...'); -// await proxyService.queueRefreshProxies(30 * 60 * 1000); // 30 minutes - -// // 2. Start health checks (checks working proxies every 15 minutes) -// console.log('๐Ÿ” Starting proxy health checks...'); -// await proxyService.startHealthChecks(15 * 60 * 1000); // 15 minutes - -// // 3. Manually scrape proxies -// console.log('๐ŸŒ Manually scraping proxies...'); -// const scrapedCount = await proxyService.scrapeProxies(); -// console.log(`โœ… Scraped ${scrapedCount} unique proxies`); - -// // 4. Wait a bit for some validation to complete -// await new Promise(resolve => setTimeout(resolve, 5000)); - -// // 5. Get proxy statistics -// console.log('๐Ÿ“Š Getting proxy statistics...'); -// const stats = await proxyService.getProxyStats(); -// console.log('Stats:', { -// total: stats.total, -// working: stats.working, -// failed: stats.failed, -// avgResponseTime: stats.avgResponseTime + 'ms' -// }); - -// // 6. Get a working proxy -// console.log('๐ŸŽฏ Getting a working proxy...'); -// const workingProxy = await proxyService.getWorkingProxy(); -// if (workingProxy) { -// console.log('Working proxy found:', { -// host: workingProxy.host, -// port: workingProxy.port, -// protocol: workingProxy.protocol -// }); - -// // 7. Test the proxy -// console.log('๐Ÿงช Testing proxy...'); -// const testResult = await proxyService.checkProxy(workingProxy); -// console.log('Test result:', { -// isWorking: testResult.isWorking, -// responseTime: testResult.responseTime + 'ms', -// error: testResult.error || 'None' -// }); -// } else { -// console.log('โŒ No working proxies available yet'); -// } - -// // 8. Get multiple working proxies -// console.log('๐Ÿ“‹ Getting multiple working proxies...'); -// const workingProxies = await proxyService.getWorkingProxies(5); -// console.log(`Found ${workingProxies.length} working proxies`); - -// // 9. Example: Using a proxy with HttpClient -// if (workingProxies.length > 0) { -// console.log('๐Ÿ”„ Example: Using proxy with HttpClient...'); -// try { -// const { HttpClient } = await import('@stock-bot/http'); -// const proxyClient = new HttpClient({ -// proxy: workingProxies[0], -// timeout: 10000 -// }); - -// const response = await proxyClient.get('http://httpbin.org/ip'); -// console.log('โœ… Request through proxy successful:', response.data); -// } catch (error) { -// console.log('โŒ Request through proxy failed:', error); -// } -// } - -// console.log('๐ŸŽ‰ Proxy Service Demo completed!'); - -// } catch (error) { -// console.error('โŒ Demo failed:', error); -// } -// } - /** * Example: Custom proxy source with enhanced logging */ async function demonstrateCustomProxySource() { logger.info('๐Ÿ”ง Demonstrating custom proxy source...'); - const customSources : ProxySource[] = [ - { - url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt', - protocol: 'http' - } - ]; try { - const count = await proxyService.scrapeProxies(customSources); - logger.info(`โœ… Scraped ${count} proxies from custom source`, { - count, - sourceCount: customSources.length - }); + const count = await proxyService.fetchProxiesFromSources(); } catch (error) { logger.error('โŒ Custom source scraping failed',{ - error: error as Error, - sourceUrls: customSources.map(s => s.url) + error: error }); } } diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index e231d3e..4fd2c55 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -1,273 +1,169 @@ import { Logger } from '@stock-bot/logger'; import createCache, { type CacheProvider } from '@stock-bot/cache'; -import { HttpClient, HttpClientConfig, ProxyConfig , RequestConfig } from '@stock-bot/http'; +import { HttpClient, ProxyInfo } from '@stock-bot/http'; import pLimit from 'p-limit'; -export interface ProxySource { - url: string; - protocol: 'http' | 'https' | 'socks4' | 'socks5'; - parser?: (content: string) => ProxyConfig[]; -} - -export interface ProxyStats { - total: number; - working: number; - failed: number; - lastCheck: Date; - avgResponseTime: number; -} - export interface ProxyCheckResult { - proxy: ProxyConfig; + proxy: ProxyInfo; isWorking: boolean; responseTime: number; error?: string; checkedAt: Date; } -export interface ProxyData extends ProxyConfig { - addedAt: Date; - lastChecked: string | null; - isWorking: boolean | null; - responseTime: number | null; +export interface ProxyStats { + total: number; + working: number; + avgResponseTime: number; } export class ProxyService { - private logger; - private cache: CacheProvider; + private logger = new Logger('proxy-service'); + private cache: CacheProvider = createCache('hybrid'); private httpClient: HttpClient; - private readonly concurrencyLimit = pLimit(200); - private readonly CACHE_PREFIX = 'proxy:'; - private readonly WORKING_PROXIES_KEY = 'proxy:working'; - private readonly PROXY_STATS_KEY = 'proxy:stats'; - private readonly CHECK_TIMEOUT = 3000; // 10 seconds - private readonly DEFAULT_CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; - private readonly DEFAULT_IP_ADDRESS = '99.246.102.205' - - private readonly defaultSources: ProxySource[] = [ - // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, - // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, - // {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, - ]; + private readonly concurrencyLimit = pLimit(1000); + private readonly CACHE_KEY = 'proxy'; + private readonly CACHE_TTL = 86400; // 24 hours + private readonly CHECK_TIMEOUT = 5000; + private readonly CHECK_URL = 'https://httpbin.org/ip'; + private readonly PROXY_SOURCES = [ + {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, + {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, + {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, + ] constructor() { - this.logger = new Logger('proxy-service'); - - this.cache = createCache('hybrid'); - this.httpClient = new HttpClient({ timeout: this.CHECK_TIMEOUT, }, this.logger); - + this.logger.info('ProxyService initialized'); } + - /** - * Start the proxy refresh job - */ - async queueRefreshProxies(intervalMs: number = 30 * 60 * 1000): Promise { - this.logger.info('Starting proxy refresh job', { intervalMs }); + async fetchProxiesFromSources() : Promise { + const sources = this.PROXY_SOURCES.map(source => + this.concurrencyLimit(() => this.fetchProxiesFromSource(source)) + ) + const result = await Promise.all(sources); + this.checkProxies(result.flat()) + return true + } - // Initial refresh - await this.scrapeProxies(); - // Set up periodic refresh - setInterval(async () => { + async fetchProxiesFromSource(source: { url: string; protocol: string }): Promise { + const allProxies: ProxyInfo[] = []; + try { - await this.scrapeProxies(); + this.logger.info(`Fetching proxies from ${source.url}`); + + const response = await this.httpClient.get(source.url, { + timeout: 10000 + }); + + if (response.status !== 200) { + this.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`); + return [] + } + + const text = response.data; + const lines = text.split('\n').filter((line: string) => line.trim()); + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + + // Parse formats like "host:port" or "host:port:user:pass" + const parts = trimmed.split(':'); + if (parts.length >= 2) { + const proxy: ProxyInfo = { + protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5', + host: parts[0], + port: parseInt(parts[1]) + }; + + if (!isNaN(proxy.port) && proxy.host) { + allProxies.push(proxy); + } + } + } + + this.logger.info(`Parsed ${allProxies.length} proxies from ${source.url}`); + } catch (error) { - this.logger.error('Error in periodic proxy refresh', error); + this.logger.error(`Error fetching proxies from ${source.url}`, error); + return []; } - }, intervalMs); - } - - /** - * Scrape proxies from all sources - */ - async scrapeProxies(sources: ProxySource[] = this.defaultSources): Promise { - this.logger.info('Starting proxy scraping', { sourceCount: sources.length }); - const allProxies: ProxyConfig[] = []; - const scrapingPromises = sources.map(source => this.scrapeFromSource(source)); - - const results = await Promise.allSettled(scrapingPromises); - - results.forEach((result, index) => { - if (result.status === 'fulfilled') { - allProxies.push(...result.value); - this.logger.info('Successfully scraped from source', { - url: sources[index].url, - count: result.value.length - }); - } else { - this.logger.error('Failed to scrape from source', { - url: sources[index].url, - error: result.reason - }); - } - }); - - // Remove duplicates - const uniqueProxies = this.removeDuplicateProxies(allProxies); - - // Store all proxies in cache - // await this.storeProxies(uniqueProxies); - - this.logger.info('Proxy scraping completed', { - total: allProxies.length, - unique: uniqueProxies.length - }); - - // Start validation of new proxies - this.validateProxiesInBackground(uniqueProxies); - return uniqueProxies.length; + this.logger.info(`Total proxies fetched: ${allProxies.length}`); + return allProxies; } - /** - * Scrape proxies from a single source - */ - private async scrapeFromSource(source: ProxySource): Promise { - try { - const response = await this.httpClient.get(source.url); - - if (!response.data || typeof response.data !== 'string') { - throw new Error('Invalid response data'); - } - - const proxies = source.parser ? - source.parser(response.data) : - this.parseHttpProxyList(response.data); - - return proxies.map(proxy => ({ - protocol: source.protocol, - host: proxy.host, - port: proxy.port, - username: proxy.username, - password: proxy.password - })); - - } catch (error) { - this.logger.error('Error scraping from source', { - url: source.url, - error: error - }); - return []; - } - } - - /** - * Parse HTTP proxy list in format "ip:port" - */ - private parseHttpProxyList(content: string): ProxyConfig[] { - const lines = content.split('\n').filter(line => line.trim()); - const proxies: ProxyConfig[] = []; - - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - - const [host, port] = trimmed.split(':'); - if (host && port && this.isValidIP(host) && this.isValidPort(port)) { - proxies.push({ - protocol: 'http', - host: host.trim(), - port: parseInt(port.trim()) - }); - } - } - - return proxies; - } - - /** - * Validate IP address format - */ - private isValidIP(ip: string): boolean { - const ipRegex = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; - return ipRegex.test(ip); - } - - /** - * Validate port number - */ - private isValidPort(port: string): boolean { - const portNum = parseInt(port); - return !isNaN(portNum) && portNum > 0 && portNum <= 65535; - } - - /** - * Remove duplicate proxies based on host:port combination - */ - private removeDuplicateProxies(proxies: ProxyConfig[]): ProxyConfig[] { - const seen = new Set(); - return proxies.filter(proxy => { - const key = `${proxy.host}:${proxy.port}`; - if (seen.has(key)) { - return false; - } - seen.add(key); - return true; - }); - } - + /** * Check if a proxy is working */ - async checkProxy(proxy: ProxyConfig, checkUrl: string = this.DEFAULT_CHECK_URL): Promise { + async checkProxy(proxy: ProxyInfo): Promise { const startTime = Date.now(); + console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`); try { - this.logger.debug('Proxy check initiate request', { - proxy: proxy.host + ':' + proxy.port, + + // Test the proxy + const response = await this.httpClient.get(this.CHECK_URL, { + proxy, + timeout: this.CHECK_TIMEOUT }); - const response = await this.httpClient.get(checkUrl, {proxy: proxy, timeout: this.CHECK_TIMEOUT}); + const responseTime = Date.now() - startTime; - this.logger.debug('Proxy check response', { - proxy: proxy.host + ':' + proxy.port, - }); - if (response.status >= 200 && response.status < 300 && !response.data.contains(this.DEFAULT_IP_ADDRESS)) { - const result: ProxyCheckResult = { - proxy, - isWorking: true, - responseTime, - checkedAt: new Date() - }; + const isWorking = response.status >= 200 && response.status < 300; - // Update cache with working status - await this.updateProxyStatus(proxy, true, responseTime); - - this.logger.debug('Proxy check successful', { - host: proxy.host, - port: proxy.port, - responseTime - }); + const result: ProxyCheckResult = { + proxy, + isWorking, + responseTime, + checkedAt: new Date() + }; - return result; + // Cache the result + // await this.cache.set(cacheKey, result, this.CACHE_TTL); + + if (isWorking) { + await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL); } else { - throw new Error(`HTTP ${response.status}`); + await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); } + this.logger.debug('Proxy check completed', { + host: proxy.host, + port: proxy.port, + isWorking, + responseTime + }); + + return result; + } catch (error) { const responseTime = Date.now() - startTime; const errorMessage = error instanceof Error ? error.message : String(error); @@ -280,72 +176,50 @@ export class ProxyService { checkedAt: new Date() }; - // Update cache with failed status - await this.updateProxyStatus(proxy, false, responseTime); + // Cache failed result for shorter time + // await this.cache.set(cacheKey, result, 300); // 5 minutes + await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); - this.logger.debug('Proxy check failed', { - host: proxy.host, - port: proxy.port, - error: errorMessage + this.logger.debug('Proxy check failed', { + host: proxy.host, + port: proxy.port, + error: errorMessage }); return result; } } - /** - * Update proxy status in cache - */ - private async updateProxyStatus(proxy: ProxyConfig, isWorking: boolean, responseTime: number): Promise { - try { - const key = this.getProxyKey(proxy); - const existingData = await this.cache.get(key); - - if (existingData) { - const data: ProxyData = { - ...existingData, - isWorking, - responseTime, - lastChecked: new Date().toISOString() - }; - - await this.cache.set(key, data, 86400); - // Manage working proxies list - const workingKey = `${this.WORKING_PROXIES_KEY}:${proxy.host}:${proxy.port}`; - - if (isWorking) { - await this.cache.set(workingKey, proxy, 86400); - } else { - await this.cache.del(workingKey); - } - } - } catch (error) { - this.logger.error('Error updating proxy status', error); - } - } /** - * Get a working proxy from cache + * Check multiple proxies concurrently */ - async getWorkingProxy(): Promise { + async checkProxies(proxies: ProxyInfo[]): Promise { + this.logger.info('Checking proxies', { count: proxies.length }); + + const checkPromises = proxies.map(proxy => + this.concurrencyLimit(() => this.checkProxy(proxy)) + ); + + const results = await Promise.all(checkPromises); + const workingCount = results.filter(r => r.isWorking).length; + + this.logger.info('Proxy check completed', { + total: proxies.length, + working: workingCount, + failed: proxies.length - workingCount + }); + + return results; + } + + /** + * Get a random working proxy from cache + */ + async getWorkingProxy(): Promise { try { - // Get all working proxy keys and pick one randomly - const allProxies = await this.getAllProxies(); - const workingProxies = []; - - for (const proxy of allProxies) { - const key = this.getProxyKey(proxy); - const data = await this.cache.get(key); - if (data && data.isWorking) { - workingProxies.push(proxy); - } - } - - if (workingProxies.length > 0) { - const randomIndex = Math.floor(Math.random() * workingProxies.length); - return workingProxies[randomIndex]; - } - - this.logger.warn('No working proxies available'); + // Note: This is a simplified implementation + // In production, you'd want to maintain a working proxies list + this.logger.warn('getWorkingProxy not fully implemented - requires proxy list management'); return null; } catch (error) { this.logger.error('Error getting working proxy', error); @@ -354,166 +228,42 @@ export class ProxyService { } /** - * Get multiple working proxies + * Add proxies to check and cache */ - async getWorkingProxies(count: number = 10): Promise { - try { - const allProxies = await this.getAllProxies(); - const workingProxies: ProxyConfig[] = []; - - for (const proxy of allProxies) { - if (workingProxies.length >= count) break; - - const key = this.getProxyKey(proxy); - const data = await this.cache.get(key); - if (data && data.isWorking) { - workingProxies.push(proxy); - } - } - - return workingProxies; - } catch (error) { - this.logger.error('Error getting working proxies', error); - return []; - } - } - /** - * Get all proxies from cache - */ - async getAllProxies(): Promise { - try { - // Since we can't use keys() directly, we'll need to track proxy keys separately - // For now, we'll implement a simple approach using a known key pattern - const proxies: ProxyConfig[] = []; - - // We'll need to either: - // 1. Maintain a separate index of all proxy keys - // 2. Or use a different approach - // For now, let's return empty array and log a warning - this.logger.warn('getAllProxies not fully implemented - Redis cache provider limitations'); - return proxies; - } catch (error) { - this.logger.error('Error getting all proxies', error); - return []; - } + async addProxies(proxies: ProxyInfo[]): Promise { + this.logger.info('Adding proxies for validation', { count: proxies.length }); + + // Start background validation + this.checkProxies(proxies).catch(error => { + this.logger.error('Error in background proxy validation', error); + }); } /** * Get proxy statistics */ - async getProxyStats(): Promise { - try { - const allProxies = await this.getAllProxies(); - const workingProxies = await this.getWorkingProxies(1000); // Get up to 1000 for stats - - const avgResponseTime = workingProxies.length > 0 - ? workingProxies.reduce((sum, _proxy) => { - // Since responseTime is not in ProxyConfig, we'll calculate differently - return sum + 1000; // placeholder average - }, 0) / workingProxies.length - : 0; - - const stats: ProxyStats = { - total: allProxies.length, - working: workingProxies.length, - failed: allProxies.length - workingProxies.length, - lastCheck: new Date(), - avgResponseTime: Math.round(avgResponseTime) - }; - - // Cache stats for 5 minutes - await this.cache.set(this.PROXY_STATS_KEY, stats, 300); - - return stats; - } catch (error) { - this.logger.error('Error getting proxy stats', error); - return { - total: 0, - working: 0, - failed: 0, - lastCheck: new Date(), - avgResponseTime: 0 - }; - } - } - /** - * Validate proxies in background - */ - private async validateProxiesInBackground(proxies: ProxyConfig[]): Promise { - this.logger.info('Starting background proxy validation', { count: proxies.length }); - - const validationPromises = proxies.map(proxy => - this.concurrencyLimit(() => - this.checkProxy(proxy).catch(error => { - this.logger.error('Error validating proxy', { - host: proxy.host, - port: proxy.port, - error - }); - return null; - }) - ) - ); - - await Promise.allSettled(validationPromises); - this.logger.info('Background proxy validation completed'); - } - /** - * Start periodic proxy health checks - */ - async startHealthChecks(intervalMs: number = 15 * 60 * 1000): Promise { - this.logger.info('Starting periodic proxy health checks', { intervalMs }); - - setInterval(async () => { - try { - const workingProxies = await this.getWorkingProxies(100); // Check up to 100 working proxies - const validationPromises = workingProxies.map(proxy => - this.concurrencyLimit(() => this.checkProxy(proxy)) - ); - - const results = await Promise.allSettled(validationPromises); - const successCount = results.filter(r => - r.status === 'fulfilled' && r.value.isWorking - ).length; - - this.logger.info('Health check completed', { - checked: workingProxies.length, - stillWorking: successCount - }); - } catch (error) { - this.logger.error('Error in health check', error); - } - }, intervalMs); - } - /** - * Clear all proxy data from cache - */ - async clearProxies(): Promise { - try { - // Since we can't use keys() and del() with spread, we'll clear known keys - await this.cache.del(this.PROXY_STATS_KEY); - - // Note: This is a limitation of the current cache provider - // In a full implementation, we'd need to maintain an index of proxy keys - this.logger.info('Cleared proxy stats from cache'); - this.logger.warn('Full proxy data clearing not implemented due to cache provider limitations'); - } catch (error) { - this.logger.error('Error clearing proxy data', error); - } - } - /** - * Get cache key for a proxy - */ - private getProxyKey(proxy: ProxyConfig): string { - return `${this.CACHE_PREFIX}${proxy.host}:${proxy.port}`; + async getStats(): Promise { + // Simplified stats - in production you'd track these properly + return { + total: 0, + working: 0, + avgResponseTime: 0 + }; } /** - * Graceful shutdown + * Clear proxy cache + */ + async clearCache(): Promise { + this.logger.info('Clearing proxy cache'); + // Note: Cache provider limitations - would need proper key tracking + } + + /** + * Shutdown service */ async shutdown(): Promise { this.logger.info('Shutting down ProxyService'); - // The cache and http client will handle their own cleanup } } diff --git a/libs/http/src/proxy-manager.ts b/libs/http/src/proxy-manager.ts index 673badf..4df704d 100644 --- a/libs/http/src/proxy-manager.ts +++ b/libs/http/src/proxy-manager.ts @@ -2,20 +2,20 @@ import got from 'got'; import { SocksProxyAgent } from 'socks-proxy-agent'; import { HttpsProxyAgent } from 'https-proxy-agent'; import { HttpProxyAgent } from 'http-proxy-agent'; -import type { ProxyConfig } from './types.js'; +import type { ProxyInfo } from './types.js'; export class ProxyManager { /** * Determine if we should use Bun fetch (HTTP/HTTPS) or Got (SOCKS) */ - static shouldUseBunFetch(proxy: ProxyConfig): boolean { + static shouldUseBunFetch(proxy: ProxyInfo): boolean { return proxy.protocol === 'http' || proxy.protocol === 'https'; } /** * Create Bun fetch proxy URL for HTTP/HTTPS proxies */ - static createBunProxyUrl(proxy: ProxyConfig): string { + static createBunProxyUrl(proxy: ProxyInfo): string { const { protocol, host, port, username, password } = proxy; if (username && password) { @@ -27,7 +27,7 @@ export class ProxyManager { /** * Create appropriate agent for Got based on proxy type */ - static createGotAgent(proxy: ProxyConfig) { + static createGotAgent(proxy: ProxyInfo) { this.validateConfig(proxy); const proxyUrl = this.buildProxyUrl(proxy); @@ -48,7 +48,7 @@ export class ProxyManager { /** * Create Got instance with proxy configuration */ - static createGotInstance(proxy: ProxyConfig) { + static createGotInstance(proxy: ProxyInfo) { const agent = this.createGotAgent(proxy); return got.extend({ @@ -68,7 +68,7 @@ export class ProxyManager { }); } - private static buildProxyUrl(proxy: ProxyConfig): string { + private static buildProxyUrl(proxy: ProxyInfo): string { const { protocol, host, port, username, password } = proxy; if (username && password) { @@ -80,7 +80,7 @@ export class ProxyManager { /** * Simple proxy config validation */ - static validateConfig(proxy: ProxyConfig): void { + static validateConfig(proxy: ProxyInfo): void { if (!proxy.host || !proxy.port) { throw new Error('Proxy host and port are required'); } diff --git a/libs/http/src/types.ts b/libs/http/src/types.ts index 331e75f..a7e4829 100644 --- a/libs/http/src/types.ts +++ b/libs/http/src/types.ts @@ -1,12 +1,16 @@ // Minimal types for fast HTTP client export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; -export interface ProxyConfig { +export interface ProxyInfo { protocol: 'http' | 'https' | 'socks4' | 'socks5'; host: string; port: number; username?: string; password?: string; + isWorking?: boolean; + responseTime?: number; + error?: string; + checkedAt?: Date; } export interface HttpClientConfig { @@ -20,7 +24,7 @@ export interface RequestConfig { headers?: Record; body?: any; timeout?: number; - proxy?: ProxyConfig; + proxy?: ProxyInfo; } export interface HttpResponse { diff --git a/libs/http/test/http.test.ts b/libs/http/test/http.test.ts index 18068b8..b3db1ed 100644 --- a/libs/http/test/http.test.ts +++ b/libs/http/test/http.test.ts @@ -1,6 +1,6 @@ import { describe, test, expect, beforeEach, beforeAll, afterAll } from 'bun:test'; import { HttpClient, HttpError, ProxyManager } from '../src/index.js'; -import type { ProxyConfig } from '../src/types.js'; +import type { ProxyInfo } from '../src/types.js'; import { MockServer } from './mock-server.js'; // Global mock server instance @@ -119,13 +119,13 @@ describe('HttpClient', () => { describe('ProxyManager', () => { test('should determine when to use Bun fetch', () => { - const httpProxy: ProxyConfig = { + const httpProxy: ProxyInfo = { protocol: 'http', host: 'proxy.example.com', port: 8080 }; - const socksProxy: ProxyConfig = { + const socksProxy: ProxyInfo = { protocol: 'socks5', host: 'proxy.example.com', port: 1080 @@ -136,7 +136,7 @@ describe('ProxyManager', () => { }); test('should create proxy URL for Bun fetch', () => { - const proxy: ProxyConfig = { + const proxy: ProxyInfo = { protocol: 'http', host: 'proxy.example.com', port: 8080, @@ -149,7 +149,7 @@ describe('ProxyManager', () => { }); test('should create proxy URL without credentials', () => { - const proxy: ProxyConfig = { + const proxy: ProxyInfo = { protocol: 'https', host: 'proxy.example.com', port: 8080