diff --git a/apps/data-service/src/proxy-demo.ts b/apps/data-service/src/proxy-demo.ts index d86d797..d85ac19 100644 --- a/apps/data-service/src/proxy-demo.ts +++ b/apps/data-service/src/proxy-demo.ts @@ -97,7 +97,7 @@ async function demonstrateCustomProxySource() { const customSources : ProxySource[] = [ { - url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', + url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt', protocol: 'http' } ]; @@ -114,82 +114,77 @@ async function demonstrateCustomProxySource() { }); } } - -// Export functions for use in other files -export { - // demonstrateProxyService, - demonstrateCustomProxySource -}; - -// Execute the demo with enhanced logging and graceful shutdown -logger.info('🚀 Starting enhanced proxy demo...'); - -// Configure graceful shutdown -setShutdownTimeout(15000); // 15 seconds shutdown timeout - -// Register shutdown handlers -onShutdown(async () => { - logger.info('🔧 Cleaning up proxy service...'); - try { - // Clean up proxy service resources if needed - // await proxyService.shutdown(); - logger.info('✅ Proxy service cleanup completed'); - } catch (error) { - logger.error('❌ Proxy service cleanup failed', error); - } -}); - -onShutdown(async () => { - logger.info('🔧 Shutting down loggers...'); - try { - await shutdownLoggers(); - console.log('✅ Logger shutdown completed'); - } catch (error) { - console.error('❌ Logger shutdown failed:', error); - } -}); - -onShutdown(async () => { - logger.info('🔧 Performing final cleanup...'); - // Any additional cleanup can go here - await new Promise(resolve => setTimeout(resolve, 1000)); // Simulate cleanup work - logger.info('✅ Final cleanup completed'); -}); - -// Demonstrate graceful shutdown by setting up a timer to shutdown after demo -const shutdownTimer = setTimeout(() => { - logger.info('⏰ Demo timeout reached, initiating graceful shutdown...'); - process.kill(process.pid, 'SIGTERM'); -}, 20000); // Shutdown after 20 seconds - -// Clear timer if demo completes early -const clearShutdownTimer = () => { - clearTimeout(shutdownTimer); - logger.info('⏰ Demo completed, canceling automatic shutdown timer'); -}; - demonstrateCustomProxySource() - .then(() => { - logger.info('🎉 Proxy demo completed successfully!'); - clearShutdownTimer(); - - // Demonstrate manual shutdown after a short delay - setTimeout(() => { - logger.info('🔄 Demonstrating manual graceful shutdown in 3 seconds...'); - setTimeout(() => { - process.kill(process.pid, 'SIGTERM'); - }, 3000); - }, 2000); - }) - .catch((error) => { - logger.error('💥 Proxy demo failed', error); - clearShutdownTimer(); - setTimeout(() => process.exit(1), 1000); - }); -// If this file is run directly, execute the demo -// if (import.meta.main) { - // demonstrateProxyService() - // demonstrateCustomProxySource() - // .catch(console.error); -// } +// // Execute the demo with enhanced logging and graceful shutdown +// logger.info('🚀 Starting enhanced proxy demo...'); + +// // Configure graceful shutdown +// setShutdownTimeout(15000); // 15 seconds shutdown timeout + +// // Register shutdown handlers +// onShutdown(async () => { +// logger.info('🔧 Cleaning up proxy service...'); +// try { +// // Clean up proxy service resources if needed +// // await proxyService.shutdown(); +// logger.info('✅ Proxy service cleanup completed'); +// } catch (error) { +// logger.error('❌ Proxy service cleanup failed', error); +// } +// }); + +// onShutdown(async () => { +// logger.info('🔧 Shutting down loggers...'); +// try { +// await shutdownLoggers(); +// console.log('✅ Logger shutdown completed'); +// } catch (error) { +// console.error('❌ Logger shutdown failed:', error); +// } +// }); + +// onShutdown(async () => { +// logger.info('🔧 Performing final cleanup...'); +// // Any additional cleanup can go here +// await new Promise(resolve => setTimeout(resolve, 1000)); // Simulate cleanup work +// logger.info('✅ Final cleanup completed'); +// }); + +// // Demonstrate graceful shutdown by setting up a timer to shutdown after demo +// const shutdownTimer = setTimeout(() => { +// logger.info('⏰ Demo timeout reached, initiating graceful shutdown...'); +// process.kill(process.pid, 'SIGTERM'); +// }, 20000); // Shutdown after 20 seconds + +// // Clear timer if demo completes early +// const clearShutdownTimer = () => { +// clearTimeout(shutdownTimer); +// logger.info('⏰ Demo completed, canceling automatic shutdown timer'); +// }; + +// demonstrateCustomProxySource() +// .then(() => { +// logger.info('🎉 Proxy demo completed successfully!'); +// clearShutdownTimer(); + +// // Demonstrate manual shutdown after a short delay +// setTimeout(() => { +// logger.info('🔄 Demonstrating manual graceful shutdown in 3 seconds...'); +// setTimeout(() => { +// process.kill(process.pid, 'SIGTERM'); +// }, 3000); +// }, 2000); +// }) +// .catch((error) => { +// logger.error('💥 Proxy demo failed', error); +// clearShutdownTimer(); +// setTimeout(() => process.exit(1), 1000); +// }); + +// // If this file is run directly, execute the demo +// // if (import.meta.main) { +// // demonstrateProxyService() +// // demonstrateCustomProxySource() +// // .catch(console.error); +// // } diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index a24801d..bf8001f 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -38,7 +38,7 @@ export class ProxyService { private readonly CACHE_PREFIX = 'proxy:'; private readonly WORKING_PROXIES_KEY = 'proxy:working'; private readonly PROXY_STATS_KEY = 'proxy:stats'; - private readonly CHECK_TIMEOUT = 10000; // 10 seconds + private readonly CHECK_TIMEOUT = 3000; // 10 seconds private readonly DEFAULT_CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; private readonly DEFAULT_IP_ADDRESS = '99.246.102.205' @@ -78,7 +78,7 @@ export class ProxyService { this.httpClient = new HttpClient({ timeout: this.CHECK_TIMEOUT, - }); + }, this.logger); this.logger.info('ProxyService initialized'); } @@ -442,7 +442,6 @@ export class ProxyService { this.logger.info('Starting background proxy validation', { count: proxies.length }); const concurrency = 50; // Process 50 proxies concurrently const chunks = this.chunkArray(proxies, concurrency); - for (const chunk of chunks) { const validationPromises = chunk.map(proxy => this.checkProxy(proxy).catch(error => { @@ -454,9 +453,7 @@ export class ProxyService { return null; }) ); - await Promise.allSettled(validationPromises); - // Small delay between chunks to avoid overwhelming the system await new Promise(resolve => setTimeout(resolve, 1000)); } diff --git a/libs/http/src/client.ts b/libs/http/src/client.ts index 670662b..700b4c8 100644 --- a/libs/http/src/client.ts +++ b/libs/http/src/client.ts @@ -14,7 +14,9 @@ export class HttpClient { constructor(config: HttpClientConfig = {}, logger?: Logger) { this.config = config; - this.logger = logger; + this.logger = logger?.child({ + component: 'http', + }); } // Convenience methods @@ -41,7 +43,6 @@ export class HttpClient { */ async request(config: RequestConfig): Promise> { const finalConfig = this.mergeConfig(config); - this.logger?.debug('Making HTTP request', { method: finalConfig.method, url: finalConfig.url, @@ -79,7 +80,7 @@ export class HttpClient { const timeout = config.timeout ?? this.config.timeout ?? 30000; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeout); - + try { const response = useBunFetch ? await this.fetchRequest(config, controller.signal) @@ -106,7 +107,13 @@ export class HttpClient { */ private async fetchRequest(config: RequestConfig, signal: AbortSignal): Promise> { try { - const options = this.buildFetchOptions(config, signal); + + // const options = this.buildFetchOptions(config, signal); + + const options = config.proxy? { proxy: config.proxy?.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal); + + this.logger?.error('Making request with fetch',{url: config.url, options }) + const response = await fetch(config.url, options); return this.parseFetchResponse(response); @@ -121,6 +128,7 @@ export class HttpClient { private async gotRequest(config: RequestConfig, signal: AbortSignal): Promise> { try { const options = this.buildGotOptions(config, signal); + const response = await got(config.url, options); return this.parseGotResponse(response); @@ -146,6 +154,7 @@ export class HttpClient { signal, }; + // Add body if (config.body && config.method !== 'GET') { if (typeof config.body === 'object') {