working on proxy service

apps/data-service/src/proxy-demo.ts

@@ -97,7 +97,7 @@ async function demonstrateCustomProxySource() {
 
   const customSources : ProxySource[] = [
     {
-      url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',
+      url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt',
       protocol: 'http'
     }
   ];
@@ -114,82 +114,77 @@ async function demonstrateCustomProxySource() {
     });
   }
 }
-
-// Export functions for use in other files
-export {
-  // demonstrateProxyService,
-  demonstrateCustomProxySource
-};
-
-// Execute the demo with enhanced logging and graceful shutdown
-logger.info('🚀 Starting enhanced proxy demo...');
-
-// Configure graceful shutdown
-setShutdownTimeout(15000); // 15 seconds shutdown timeout
-
-// Register shutdown handlers
-onShutdown(async () => {
-  logger.info('🔧 Cleaning up proxy service...');
-  try {
-    // Clean up proxy service resources if needed
-    // await proxyService.shutdown();
-    logger.info('✅ Proxy service cleanup completed');
-  } catch (error) {
-    logger.error('❌ Proxy service cleanup failed', error);
-  }
-});
-
-onShutdown(async () => {
-  logger.info('🔧 Shutting down loggers...');
-  try {
-    await shutdownLoggers();
-    console.log('✅ Logger shutdown completed');
-  } catch (error) {
-    console.error('❌ Logger shutdown failed:', error);
-  }
-});
-
-onShutdown(async () => {
-  logger.info('🔧 Performing final cleanup...');
-  // Any additional cleanup can go here
-  await new Promise(resolve => setTimeout(resolve, 1000)); // Simulate cleanup work
-  logger.info('✅ Final cleanup completed');
-});
-
-// Demonstrate graceful shutdown by setting up a timer to shutdown after demo
-const shutdownTimer = setTimeout(() => {
-  logger.info('⏰ Demo timeout reached, initiating graceful shutdown...');
-  process.kill(process.pid, 'SIGTERM');
-}, 20000); // Shutdown after 20 seconds
-
-// Clear timer if demo completes early
-const clearShutdownTimer = () => {
-  clearTimeout(shutdownTimer);
-  logger.info('⏰ Demo completed, canceling automatic shutdown timer');
-};
-
 demonstrateCustomProxySource()
-  .then(() => {
-    logger.info('🎉 Proxy demo completed successfully!');
-    clearShutdownTimer();
-    
-    // Demonstrate manual shutdown after a short delay
-    setTimeout(() => {
-      logger.info('🔄 Demonstrating manual graceful shutdown in 3 seconds...');
-      setTimeout(() => {
-        process.kill(process.pid, 'SIGTERM');
-      }, 3000);
-    }, 2000);
-  })
-  .catch((error) => {
-    logger.error('💥 Proxy demo failed', error);
-    clearShutdownTimer();
-    setTimeout(() => process.exit(1), 1000);
-  });
 
-// If this file is run directly, execute the demo
+// // Execute the demo with enhanced logging and graceful shutdown
-// if (import.meta.main) {
+// logger.info('🚀 Starting enhanced proxy demo...');
-  // demonstrateProxyService()
+
-    // demonstrateCustomProxySource()
+// // Configure graceful shutdown
-    // .catch(console.error);
+// setShutdownTimeout(15000); // 15 seconds shutdown timeout
-// }
+
+// // Register shutdown handlers
+// onShutdown(async () => {
+//   logger.info('🔧 Cleaning up proxy service...');
+//   try {
+//     // Clean up proxy service resources if needed
+//     // await proxyService.shutdown();
+//     logger.info('✅ Proxy service cleanup completed');
+//   } catch (error) {
+//     logger.error('❌ Proxy service cleanup failed', error);
+//   }
+// });
+
+// onShutdown(async () => {
+//   logger.info('🔧 Shutting down loggers...');
+//   try {
+//     await shutdownLoggers();
+//     console.log('✅ Logger shutdown completed');
+//   } catch (error) {
+//     console.error('❌ Logger shutdown failed:', error);
+//   }
+// });
+
+// onShutdown(async () => {
+//   logger.info('🔧 Performing final cleanup...');
+//   // Any additional cleanup can go here
+//   await new Promise(resolve => setTimeout(resolve, 1000)); // Simulate cleanup work
+//   logger.info('✅ Final cleanup completed');
+// });
+
+// // Demonstrate graceful shutdown by setting up a timer to shutdown after demo
+// const shutdownTimer = setTimeout(() => {
+//   logger.info('⏰ Demo timeout reached, initiating graceful shutdown...');
+//   process.kill(process.pid, 'SIGTERM');
+// }, 20000); // Shutdown after 20 seconds
+
+// // Clear timer if demo completes early
+// const clearShutdownTimer = () => {
+//   clearTimeout(shutdownTimer);
+//   logger.info('⏰ Demo completed, canceling automatic shutdown timer');
+// };
+
+// demonstrateCustomProxySource()
+//   .then(() => {
+//     logger.info('🎉 Proxy demo completed successfully!');
+//     clearShutdownTimer();
+    
+//     // Demonstrate manual shutdown after a short delay
+//     setTimeout(() => {
+//       logger.info('🔄 Demonstrating manual graceful shutdown in 3 seconds...');
+//       setTimeout(() => {
+//         process.kill(process.pid, 'SIGTERM');
+//       }, 3000);
+//     }, 2000);
+//   })
+//   .catch((error) => {
+//     logger.error('💥 Proxy demo failed', error);
+//     clearShutdownTimer();
+//     setTimeout(() => process.exit(1), 1000);
+//   });
+
+// // If this file is run directly, execute the demo
+// // if (import.meta.main) {
+//   // demonstrateProxyService()
+//     // demonstrateCustomProxySource()
+//     // .catch(console.error);
+// // }

apps/data-service/src/services/proxy.service.ts

@@ -38,7 +38,7 @@ export class ProxyService {
   private readonly CACHE_PREFIX = 'proxy:';
   private readonly WORKING_PROXIES_KEY = 'proxy:working';
   private readonly PROXY_STATS_KEY = 'proxy:stats';
-  private readonly CHECK_TIMEOUT = 10000; // 10 seconds
+  private readonly CHECK_TIMEOUT = 3000; // 10 seconds
   private readonly DEFAULT_CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
   private readonly DEFAULT_IP_ADDRESS = '99.246.102.205'
 
@@ -78,7 +78,7 @@ export class ProxyService {
 
     this.httpClient = new HttpClient({
       timeout: this.CHECK_TIMEOUT,
-    });
+    }, this.logger);
 
     this.logger.info('ProxyService initialized');
   }
@@ -442,7 +442,6 @@ export class ProxyService {
     this.logger.info('Starting background proxy validation', { count: proxies.length });
     const concurrency = 50; // Process 50 proxies concurrently
     const chunks = this.chunkArray(proxies, concurrency);
-    
     for (const chunk of chunks) {
       const validationPromises = chunk.map(proxy => 
         this.checkProxy(proxy).catch(error => {
@@ -454,9 +453,7 @@ export class ProxyService {
           return null;
         })
       );
-      
       await Promise.allSettled(validationPromises);
-      
       // Small delay between chunks to avoid overwhelming the system
       await new Promise(resolve => setTimeout(resolve, 1000));
     }

libs/http/src/client.ts

@@ -14,7 +14,9 @@ export class HttpClient {
 
   constructor(config: HttpClientConfig = {}, logger?: Logger) {
     this.config = config;
-    this.logger = logger;
+    this.logger = logger?.child({ 
+        component: 'http',
+      });
   }
 
   // Convenience methods
@@ -41,7 +43,6 @@ export class HttpClient {
    */
   async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
     const finalConfig = this.mergeConfig(config);
-    
     this.logger?.debug('Making HTTP request', { 
       method: finalConfig.method,
       url: finalConfig.url,
@@ -79,7 +80,7 @@ export class HttpClient {
     const timeout = config.timeout ?? this.config.timeout ?? 30000;
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
-
+    
     try {
       const response = useBunFetch 
         ? await this.fetchRequest<T>(config, controller.signal)
@@ -106,7 +107,13 @@ export class HttpClient {
    */
   private async fetchRequest<T>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
     try {
-      const options = this.buildFetchOptions(config, signal);
+      
+      // const options = this.buildFetchOptions(config, signal);
+
+      const options = config.proxy? { proxy: config.proxy?.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal);
+
+      this.logger?.error('Making request with fetch',{url: config.url, options }) 
+
       const response = await fetch(config.url, options);
       
       return this.parseFetchResponse<T>(response);
@@ -121,6 +128,7 @@ export class HttpClient {
   private async gotRequest<T>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
     try {
       const options = this.buildGotOptions(config, signal);
+
       const response = await got(config.url, options);
       
       return this.parseGotResponse<T>(response);
@@ -146,6 +154,7 @@ export class HttpClient {
       signal,
     };
 
+
     // Add body
     if (config.body && config.method !== 'GET') {
       if (typeof config.body === 'object') {