From 2f070d73f9f49f6b5a242af4b8803ade78f23b81 Mon Sep 17 00:00:00 2001 From: Bojan Kucera Date: Sun, 8 Jun 2025 08:47:44 -0400 Subject: [PATCH] work on proxy --- apps/data-service/src/proxy-demo.ts | 76 +------------------ .../src/services/proxy.service.ts | 12 ++- libs/config/src/core.ts | 4 +- libs/config/src/env-utils.ts | 56 +++++++++++++- libs/http/src/client.ts | 7 +- libs/logger/src/logger.ts | 4 +- 6 files changed, 73 insertions(+), 86 deletions(-) diff --git a/apps/data-service/src/proxy-demo.ts b/apps/data-service/src/proxy-demo.ts index e069047..d9019a2 100644 --- a/apps/data-service/src/proxy-demo.ts +++ b/apps/data-service/src/proxy-demo.ts @@ -3,13 +3,16 @@ import { getLogger } from '@stock-bot/logger'; // Initialize logger for the demo const logger = getLogger('proxy-demo'); +console.log('🔧 Starting proxy demo...'); /** * Example: Custom proxy source with enhanced logging */ async function demonstrateCustomProxySource() { + console.log('🔧 Demonstrating!'); logger.info('🔧 Demonstrating custom proxy source...'); try { + console.log('🔧 Demonstrating 1'); await proxyService.fetchProxiesFromSources(); console.log('🔧 Demonstrating custom proxy source is DONE!'); } catch (error) { @@ -19,76 +22,3 @@ async function demonstrateCustomProxySource() { } } demonstrateCustomProxySource() - -// // Execute the demo with enhanced logging and graceful shutdown -// logger.info('🚀 Starting enhanced proxy demo...'); - -// // Configure graceful shutdown -// setShutdownTimeout(15000); // 15 seconds shutdown timeout - -// // Register shutdown handlers -// onShutdown(async () => { -// logger.info('🔧 Cleaning up proxy service...'); -// try { -// // Clean up proxy service resources if needed -// // await proxyService.shutdown(); -// logger.info('✅ Proxy service cleanup completed'); -// } catch (error) { -// logger.error('❌ Proxy service cleanup failed', error); -// } -// }); - -// onShutdown(async () => { -// logger.info('🔧 Shutting down loggers...'); -// try { -// await shutdownLoggers(); -// console.log('✅ Logger shutdown completed'); -// } catch (error) { -// console.error('❌ Logger shutdown failed:', error); -// } -// }); - -// onShutdown(async () => { -// logger.info('🔧 Performing final cleanup...'); -// // Any additional cleanup can go here -// await new Promise(resolve => setTimeout(resolve, 1000)); // Simulate cleanup work -// logger.info('✅ Final cleanup completed'); -// }); - -// // Demonstrate graceful shutdown by setting up a timer to shutdown after demo -// const shutdownTimer = setTimeout(() => { -// logger.info('⏰ Demo timeout reached, initiating graceful shutdown...'); -// process.kill(process.pid, 'SIGTERM'); -// }, 20000); // Shutdown after 20 seconds - -// // Clear timer if demo completes early -// const clearShutdownTimer = () => { -// clearTimeout(shutdownTimer); -// logger.info('⏰ Demo completed, canceling automatic shutdown timer'); -// }; - -// demonstrateCustomProxySource() -// .then(() => { -// logger.info('🎉 Proxy demo completed successfully!'); -// clearShutdownTimer(); - -// // Demonstrate manual shutdown after a short delay -// setTimeout(() => { -// logger.info('🔄 Demonstrating manual graceful shutdown in 3 seconds...'); -// setTimeout(() => { -// process.kill(process.pid, 'SIGTERM'); -// }, 3000); -// }, 2000); -// }) -// .catch((error) => { -// logger.error('💥 Proxy demo failed', error); -// clearShutdownTimer(); -// setTimeout(() => process.exit(1), 1000); -// }); - -// // If this file is run directly, execute the demo -// // if (import.meta.main) { -// // demonstrateProxyService() -// // demonstrateCustomProxySource() -// // .catch(console.error); -// // } diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index c329407..2497957 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -15,6 +15,7 @@ export class ProxyService { private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; private readonly PROXY_SOURCES = [ {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, @@ -146,9 +147,7 @@ export class ProxyService { this.logger.error(`Error fetching proxies from ${source.url}`, error); return []; } - - - this.logger.info(`Total proxies fetched: ${allProxies.length}`); + // this.logger.info(`Total proxies fetched: ${allProxies.length}`); return allProxies; } @@ -164,6 +163,10 @@ export class ProxyService { * Check if a proxy is working */ async checkProxy(proxy: ProxyInfo): Promise { + this.logger.debug('Checking Proxy : ', { + host: proxy.host, + port: proxy.port, + }); // console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount); try { @@ -179,7 +182,8 @@ export class ProxyService { ...proxy, isWorking, checkedAt: new Date(), - responseTime: response.responseTime, }; + responseTime: response.responseTime, + }; // console.log('Proxy check result:', proxy); if (isWorking && JSON.stringify(response.data).includes(this.CHECK_IP)) { await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL); diff --git a/libs/config/src/core.ts b/libs/config/src/core.ts index 9282b38..32718d2 100644 --- a/libs/config/src/core.ts +++ b/libs/config/src/core.ts @@ -29,7 +29,7 @@ export enum Environment { */ export function loadEnvVariables(envOverride?: string): void { const env = envOverride || process.env.NODE_ENV || 'development'; - + console.log(`Current environment: ${env}`); // Order of loading: // 1. .env (base environment variables) // 2. .env.{environment} (environment-specific variables) @@ -51,7 +51,6 @@ export function loadEnvVariables(envOverride?: string): void { */ export function getEnvironment(): Environment { const env = process.env.NODE_ENV?.toLowerCase() || 'development'; - switch (env) { case 'development': return Environment.Development; @@ -64,5 +63,6 @@ export function getEnvironment(): Environment { return Environment.Production; default: return Environment.Development; + } } diff --git a/libs/config/src/env-utils.ts b/libs/config/src/env-utils.ts index 3cd2c8b..7a70739 100644 --- a/libs/config/src/env-utils.ts +++ b/libs/config/src/env-utils.ts @@ -3,9 +3,51 @@ */ import { z } from 'zod'; import { config } from 'dotenv'; +import { join } from 'path'; +import { existsSync } from 'fs'; + +// Function to find and load environment variables +function loadEnvFiles() { + const cwd = process.cwd(); + const possiblePaths = [ + // Current working directory + join(cwd, '.env'), + join(cwd, '.env.local'), + // Root of the workspace (common pattern) + join(cwd, '../../.env'), + join(cwd, '../../../.env'), + // Config library directory + join(__dirname, '../.env'), + join(__dirname, '../../.env'), + join(__dirname, '../../../.env'), + ]; + + // Try to load each possible .env file + for (const envPath of possiblePaths) { + if (existsSync(envPath)) { + console.log(`📄 Loading environment from: ${envPath}`); + config({ path: envPath }); + break; // Use the first .env file found + } + } + + // Also try to load environment-specific files + const environment = process.env.NODE_ENV || 'development'; + const envSpecificPaths = [ + join(cwd, `.env.${environment}`), + join(cwd, `.env.${environment}.local`), + ]; + + for (const envPath of envSpecificPaths) { + if (existsSync(envPath)) { + console.log(`📄 Loading ${environment} environment from: ${envPath}`); + config({ path: envPath, override: false }); // Don't override existing vars + } + } +} // Load environment variables -config(); +loadEnvFiles(); /** * Creates a Zod schema for environment variable validation @@ -34,6 +76,18 @@ export function validateEnv( return result.data; } +/** + * Manually load environment variables from a specific path + */ +export function loadEnv(path?: string) { + if (path) { + console.log(`📄 Manually loading environment from: ${path}`); + config({ path }); + } else { + loadEnvFiles(); + } +} + /** * Helper functions for common validation patterns */ diff --git a/libs/http/src/client.ts b/libs/http/src/client.ts index 401e0de..4b1dfdc 100644 --- a/libs/http/src/client.ts +++ b/libs/http/src/client.ts @@ -44,7 +44,7 @@ export class HttpClient { async request(config: RequestConfig): Promise> { const finalConfig = this.mergeConfig(config); const startTime = Date.now(); - + this.logger?.debug('Making HTTP request', { method: finalConfig.method, url: finalConfig.url, @@ -75,11 +75,10 @@ export class HttpClient { /** * Execute request with timeout handling - no race conditions - */ - private async executeRequest(config: RequestConfig): Promise> { + */ private async executeRequest(config: RequestConfig): Promise> { const timeout = config.timeout ?? this.config.timeout ?? 30000; const controller = new AbortController(); - + // Set up timeout const timeoutId = setTimeout(() => { controller.abort(); diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 39afd55..0505dda 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -14,7 +14,7 @@ import type { LogLevel, LogContext, LogMetadata } from './types'; // Simple cache for logger instances const loggerCache = new Map(); - +console.log('Logger cache initialized: ', loggingConfig.LOG_LEVEL); /** * Create transport configuration */ @@ -25,7 +25,7 @@ function createTransports(serviceName: string): any { if (loggingConfig.LOG_CONSOLE) { targets.push({ target: 'pino-pretty', - level: 'error', // Only show errors on console + level: loggingConfig.LOG_LEVEL, // Only show errors on console options: { colorize: true, translateTime: 'yyyy-mm-dd HH:MM:ss.l',