From afc1843fdb843031b57592ebf06d8a754d7c2a5b Mon Sep 17 00:00:00 2001 From: Bojan Kucera Date: Sun, 8 Jun 2025 07:36:07 -0400 Subject: [PATCH] working on proxy service. getting closer --- .../src/services/proxy.service.ts | 125 +-- libs/config/src/example.ts | 817 ------------------ libs/http/src/client.ts | 22 +- libs/http/src/proxy-manager.ts | 12 +- libs/logger/src/logger.ts | 7 +- 5 files changed, 91 insertions(+), 892 deletions(-) delete mode 100644 libs/config/src/example.ts diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index 9dedc59..f44f9e6 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -7,50 +7,62 @@ export class ProxyService { private logger = new Logger('proxy-service'); private cache: CacheProvider = createCache('hybrid'); private httpClient: HttpClient; - private readonly concurrencyLimit = pLimit(250); + private readonly concurrencyLimit = pLimit(300); private readonly CACHE_KEY = 'proxy'; private readonly CACHE_TTL = 86400; // 24 hours - private readonly CHECK_TIMEOUT = 7000; + private readonly CHECK_TIMEOUT = 10000; private readonly CHECK_IP = '99.246.102.205' private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; private readonly PROXY_SOURCES = [ - // {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http' }, - // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',protocol: 'http', }, + // {url: 'https://github.com/zloi-user/hideip.me/raw/refs/heads/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http' }, + {url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http' }, + {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, + {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, - // {url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' }, - // {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', }, - {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, - {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, - {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks5.txt',protocol: 'socks5', }, + + {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, + {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, + {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, + {url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' }, + {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', }, + {url: 'https://github.com/officialputuid/KangProxy/blob/KangProxy/https/https.txt',protocol: 'https', }, + // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', }, + // {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', }, + // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, + // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', }, + // {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks5.txt',protocol: 'socks5', }, ] constructor() { @@ -62,14 +74,15 @@ export class ProxyService { } - async fetchProxiesFromSources() : Promise { + async fetchProxiesFromSources() : Promise { const sources = this.PROXY_SOURCES.map(source => this.concurrencyLimit(() => this.fetchProxiesFromSource(source)) - ) + ) const result = await Promise.all(sources); - const allProxies: ProxyInfo[] = result.flat(); - await this.checkProxies(this.removeDuplicateProxies(allProxies)) - return true + let allProxies: ProxyInfo[] = result.flat(); + allProxies = this.removeDuplicateProxies(allProxies) + await this.checkProxies(allProxies) + return allProxies.length } private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] { @@ -106,8 +119,10 @@ export class ProxyService { const text = response.data; const lines = text.split('\n').filter((line: string) => line.trim()); + for (const line of lines) { - const trimmed = line.trim(); + let trimmed = line.trim(); + trimmed = this.cleanProxyUrl(trimmed); if (!trimmed || trimmed.startsWith('#')) continue; // Parse formats like "host:port" or "host:port:user:pass" @@ -131,16 +146,25 @@ export class ProxyService { this.logger.error(`Error fetching proxies from ${source.url}`, error); return []; } + this.logger.info(`Total proxies fetched: ${allProxies.length}`); return allProxies; } + private cleanProxyUrl(url: string): string { + // Remove http:// or https:// and any leading zeros from the host part + return url + .replace(/^https?:\/\//, '') // Remove protocol + .replace(/^0+/, '') // Remove leading zeros at start + .replace(/:0+(\d)/g, ':$1'); // Remove leading zeros from port numbers + } + /** * Check if a proxy is working */ async checkProxy(proxy: ProxyInfo): Promise { - console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount); + // console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount); try { // Test the proxy @@ -157,12 +181,13 @@ export class ProxyService { checkedAt: new Date(), responseTime: response.responseTime, }; - - if (isWorking && !response.data.includes('CHECK_IP')) { + // console.log('Proxy check result:', proxy); + if (isWorking && !JSON.stringify(response.data).includes(this.CHECK_IP)) { await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL); - } else { - await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); - } + } + // else { // TODO + // await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); + // } this.logger.debug('Proxy check completed', { host: proxy.host, diff --git a/libs/config/src/example.ts b/libs/config/src/example.ts deleted file mode 100644 index e5d3c0b..0000000 --- a/libs/config/src/example.ts +++ /dev/null @@ -1,817 +0,0 @@ -/** - * Example usage of the Stock Bot configuration library - * - * This file demonstrates how to use the Zod-based configuration - * system for various services in the Stock Bot platform. - */ - -// Import all the configuration modules -import { - // Core utilities - loadEnvVariables, - getEnvironment, - Environment, - ConfigurationError, -} from './core'; - -import { - // PostgreSQL configuration - postgresConfig, - PostgresConfig, - POSTGRES_HOST, - POSTGRES_PORT, - POSTGRES_DATABASE, - POSTGRES_USERNAME, - POSTGRES_PASSWORD, -} from './postgres'; - -import { - // QuestDB configuration - questdbConfig, - QuestDbConfig, - QUESTDB_HOST, - QUESTDB_HTTP_PORT, - QUESTDB_PG_PORT, -} from './questdb'; - -import { - // MongoDB configuration - mongodbConfig, - MongoDbConfig, - MONGODB_HOST, - MONGODB_PORT, - MONGODB_DATABASE, - MONGODB_USERNAME, -} from './mongodb'; - -import { - // Dragonfly configuration - dragonflyConfig, - DragonflyConfig, - DRAGONFLY_HOST, - DRAGONFLY_PORT, - DRAGONFLY_DATABASE, -} from './dragonfly'; - -import { - // Monitoring configuration - prometheusConfig, - grafanaConfig, - PrometheusConfig, - GrafanaConfig, - PROMETHEUS_HOST, - PROMETHEUS_PORT, - GRAFANA_HOST, - GRAFANA_PORT, -} from './monitoring'; - -import { - // Loki configuration - lokiConfig, - LokiConfig, - LOKI_HOST, - LOKI_PORT, - LOKI_SERVICE_LABEL, - LOKI_BATCH_SIZE, -} from './loki'; - -import { - // Logging configuration - loggingConfig, - LoggingConfig, - LOG_LEVEL, - LOG_FORMAT, - LOG_CONSOLE, - LOG_FILE, - LOG_SERVICE_NAME, -} from './logging'; - - -import { - // Risk management configuration - riskConfig, - RiskConfig, - RISK_MAX_POSITION_SIZE, - RISK_DEFAULT_STOP_LOSS, - RISK_CIRCUIT_BREAKER_ENABLED, -} from './risk'; - -import { - // Data provider configuration - dataProvidersConfig, - DataProvidersConfig, - getProviderConfig, - getEnabledProviders, - getDefaultProvider, - DEFAULT_DATA_PROVIDER, - ALPACA_API_KEY, -} from './data-providers'; - -/** - * Example 1: Basic usage - Load environment variables and get configuration - */ -function basicUsageExample() { - console.log('=== Basic Usage Example ==='); - - // Load environment variables (optional - they're loaded automatically) - loadEnvVariables(); - - // Get the current environment - const env = getEnvironment(); - console.log(`Current environment: ${env}`); - // Access individual configuration values - console.log(`Database host: ${POSTGRES_HOST}`); - console.log(`Database port: ${POSTGRES_PORT}`); - console.log(`Log level: ${LOG_LEVEL}`); - - // Access full database config objects - console.log(`Full database config:`, { - host: postgresConfig.POSTGRES_HOST, - port: postgresConfig.POSTGRES_PORT, - name: postgresConfig.POSTGRES_DATABASE, - ssl: postgresConfig.POSTGRES_SSL, - }); -} - -/** - * Example 2: Using configuration in a database connection - */ -async function databaseConnectionExample() { - console.log('=== Database Connection Example ==='); - - try { - // Use the database configuration to create a connection string - const connectionString = `postgresql://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}`; - - console.log('Database connection settings:'); - console.log(`- Host: ${postgresConfig.POSTGRES_HOST}`); - console.log(`- Port: ${postgresConfig.POSTGRES_PORT}`); - console.log(`- Database: ${postgresConfig.POSTGRES_DATABASE}`); - console.log(`- SSL enabled: ${postgresConfig.POSTGRES_SSL}`); - console.log(`- Pool max connections: ${postgresConfig.POSTGRES_POOL_MAX}`); - console.log(`- Query timeout: ${postgresConfig.POSTGRES_QUERY_TIMEOUT}ms`); - - // Example pool configuration - const poolConfig = { - host: postgresConfig.POSTGRES_HOST, - port: postgresConfig.POSTGRES_PORT, - database: postgresConfig.POSTGRES_DATABASE, - user: postgresConfig.POSTGRES_USERNAME, - password: postgresConfig.POSTGRES_PASSWORD, - ssl: postgresConfig.POSTGRES_SSL, - min: postgresConfig.POSTGRES_POOL_MIN, - max: postgresConfig.POSTGRES_POOL_MAX, - idleTimeoutMillis: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT, - }; - - console.log('Pool configuration:', poolConfig); - - } catch (error) { - console.error('Database configuration error:', error); - } -} - - -/** - * Example 3: Logging setup example - */ -function loggingSetupExample() { - console.log('=== Logging Setup Example ==='); - - // Access logging configuration - console.log('Logging settings:'); - console.log(`- Level: ${loggingConfig.LOG_LEVEL}`); - console.log(`- Format: ${loggingConfig.LOG_FORMAT}`); - console.log(`- Console enabled: ${loggingConfig.LOG_CONSOLE}`); - console.log(`- File logging: ${loggingConfig.LOG_FILE}`); - console.log(`- Service name: ${loggingConfig.LOG_SERVICE_NAME}`); - // Example logger configuration - const loggerConfig = { - level: loggingConfig.LOG_LEVEL, - format: loggingConfig.LOG_FORMAT, - transports: [] as any[], - defaultMeta: { - service: loggingConfig.LOG_SERVICE_NAME, - version: loggingConfig.LOG_SERVICE_VERSION, - environment: loggingConfig.LOG_ENVIRONMENT, - }, - }; - - if (loggingConfig.LOG_CONSOLE) { - loggerConfig.transports.push({ - type: 'console', - format: loggingConfig.LOG_FORMAT, - timestamp: loggingConfig.LOG_TIMESTAMP, - }); - } - - if (loggingConfig.LOG_FILE) { - loggerConfig.transports.push({ - type: 'file', - filename: `${loggingConfig.LOG_FILE_PATH}/application.log`, - maxSize: loggingConfig.LOG_FILE_MAX_SIZE, - maxFiles: loggingConfig.LOG_FILE_MAX_FILES, - datePattern: loggingConfig.LOG_FILE_DATE_PATTERN, - }); - } - - // Example Loki transport configuration - if (lokiConfig.LOKI_HOST) { - loggerConfig.transports.push({ - type: 'loki', - host: lokiConfig.LOKI_HOST, - port: lokiConfig.LOKI_PORT, - batchSize: lokiConfig.LOKI_BATCH_SIZE, - labels: { - service: lokiConfig.LOKI_SERVICE_LABEL, - environment: lokiConfig.LOKI_ENVIRONMENT_LABEL, - }, - }); - } - console.log('Logger configuration:', loggerConfig); -} - -/** - * Example 4: Risk management configuration - */ -function riskManagementExample() { - console.log('=== Risk Management Example ==='); - - // Access risk configuration - console.log('Risk management settings:'); - console.log(`- Max position size: ${RISK_MAX_POSITION_SIZE * 100}%`); - console.log(`- Default stop loss: ${RISK_DEFAULT_STOP_LOSS * 100}%`); - console.log(`- Circuit breaker enabled: ${RISK_CIRCUIT_BREAKER_ENABLED}`); - console.log(`- Max leverage: ${riskConfig.RISK_MAX_LEVERAGE}x`); - - // Example risk calculator - function calculatePositionSize(portfolioValue: number, riskPerTrade: number = RISK_DEFAULT_STOP_LOSS) { - const maxPositionValue = portfolioValue * RISK_MAX_POSITION_SIZE; - const riskAmount = portfolioValue * riskPerTrade; - - return { - maxPositionValue, - riskAmount, - maxShares: Math.floor(maxPositionValue / 100), // Assuming $100 per share - }; - } - - const portfolioValue = 100000; // $100k portfolio - const position = calculatePositionSize(portfolioValue); - console.log(`Position sizing for $${portfolioValue} portfolio:`, position); -} - -/** - * Example 5: Data provider configuration - */ -function dataProviderExample() { - console.log('=== Data Provider Example ==='); - - // Get the default provider - const defaultProvider = getDefaultProvider(); - console.log('Default provider:', defaultProvider); - - // Get all enabled providers - const enabledProviders = getEnabledProviders(); - console.log('Enabled providers:', enabledProviders.map(p => p.name)); - - // Get specific provider configuration - try { - const alpacaConfig = getProviderConfig('alpaca'); - console.log('Alpaca configuration:', { - enabled: alpacaConfig.enabled, - baseUrl: alpacaConfig.baseUrl, - hasApiKey: !!alpacaConfig.apiKey, - rateLimit: alpacaConfig.rateLimits, - }); - } catch (error) { - console.error('Error getting Alpaca config:', error); - } - - // Example API client setup - const apiClients = enabledProviders.map(provider => ({ - name: provider.name, - client: { - baseURL: provider.baseUrl, - timeout: dataProvidersConfig.DATA_PROVIDER_TIMEOUT, - retries: dataProvidersConfig.DATA_PROVIDER_RETRIES, - retryDelay: dataProvidersConfig.DATA_PROVIDER_RETRY_DELAY, - headers: provider.apiKey ? { - 'Authorization': `Bearer ${provider.apiKey}` - } : {}, - } - })); - - console.log('API clients configuration:', apiClients); -} - -/** - * Example 6: Environment-specific configuration - */ -function environmentSpecificExample() { - console.log('=== Environment-Specific Example ==='); - - const env = getEnvironment(); - - switch (env) { - case Environment.Development: - console.log('Development environment detected'); - console.log('- Using local database'); - console.log('- Verbose logging enabled'); - console.log('- Paper trading mode'); - break; - - case Environment.Testing: - console.log('Testing environment detected'); - console.log('- Using test database'); - console.log('- Structured logging'); - console.log('- Mock data providers'); - break; - - case Environment.Staging: - console.log('Staging environment detected'); - console.log('- Using staging database'); - console.log('- Production-like settings'); - console.log('- Real data providers (limited)'); - break; - - case Environment.Production: - console.log('Production environment detected'); - console.log('- Using production database'); - console.log('- Optimized logging'); - console.log('- Live trading enabled'); - break; - } - - // Example of environment-specific behavior - const isProduction = env === Environment.Production; - const tradingMode = isProduction ? 'live' : 'paper'; - const logLevel = isProduction ? 'info' : 'debug'; - - console.log(`Trading mode: ${tradingMode}`); - console.log(`Recommended log level: ${logLevel}`); -} - -/** - * Example 7: Configuration validation and error handling - */ -function configurationValidationExample() { - console.log('=== Configuration Validation Example ==='); - - try { - // Check required configurations - if (!ALPACA_API_KEY && DEFAULT_DATA_PROVIDER === 'alpaca') { - throw new ConfigurationError('Alpaca API key is required when using Alpaca as default provider'); - } - - // Validate risk settings - if (RISK_MAX_POSITION_SIZE > 1.0) { - throw new ConfigurationError('Maximum position size cannot exceed 100%'); - } - - if (riskConfig.RISK_DEFAULT_STOP_LOSS > riskConfig.RISK_DEFAULT_TAKE_PROFIT) { - console.warn('Warning: Stop loss is greater than take profit - check your risk settings'); - } - - // Validate database connection settings - if (postgresConfig.POSTGRES_POOL_MAX < postgresConfig.POSTGRES_POOL_MIN) { - throw new ConfigurationError('Database max pool size must be greater than min pool size'); - } - - console.log('✅ All configuration validations passed'); - - } catch (error) { - if (error instanceof ConfigurationError) { - console.error('❌ Configuration error:', error.message); - } else { - console.error('❌ Unexpected error:', error); - } - } -} - -/** - * Example 8: QuestDB time-series database configuration - */ -function questdbConfigurationExample() { - console.log('=== QuestDB Configuration Example ==='); - - // Access QuestDB configuration - console.log('QuestDB settings:'); - console.log(`- Host: ${questdbConfig.QUESTDB_HOST}`); - console.log(`- HTTP port (web console): ${questdbConfig.QUESTDB_HTTP_PORT}`); - console.log(`- PostgreSQL port: ${questdbConfig.QUESTDB_PG_PORT}`); - console.log(`- InfluxDB port: ${questdbConfig.QUESTDB_INFLUX_PORT}`); - console.log(`- TLS enabled: ${questdbConfig.QUESTDB_TLS_ENABLED}`); - - // Example QuestDB client configuration - const questdbClientConfig = { - http: { - host: questdbConfig.QUESTDB_HOST, - port: questdbConfig.QUESTDB_HTTP_PORT, - tls: questdbConfig.QUESTDB_TLS_ENABLED, - timeout: questdbConfig.QUESTDB_REQUEST_TIMEOUT, - }, - postgresql: { - host: questdbConfig.QUESTDB_HOST, - port: questdbConfig.QUESTDB_PG_PORT, - database: questdbConfig.QUESTDB_DEFAULT_DATABASE, - user: questdbConfig.QUESTDB_USER, - password: questdbConfig.QUESTDB_PASSWORD, - }, - influxdb: { - host: questdbConfig.QUESTDB_HOST, - port: questdbConfig.QUESTDB_INFLUX_PORT, - } - }; - - console.log('QuestDB client configuration:', questdbClientConfig); - - // Example time-series table creation - const createTableQuery = ` - CREATE TABLE IF NOT EXISTS ohlcv_data ( - timestamp TIMESTAMP, - symbol SYMBOL, - open DOUBLE, - high DOUBLE, - low DOUBLE, - close DOUBLE, - volume LONG - ) timestamp(timestamp) PARTITION BY DAY; - `; - - console.log('Example table creation query:', createTableQuery); -} - -/** - * Example 9: MongoDB document database configuration - */ -function mongodbConfigurationExample() { - console.log('=== MongoDB Configuration Example ==='); - - // Access MongoDB configuration - console.log('MongoDB settings:'); - console.log(`- Host: ${mongodbConfig.MONGODB_HOST}`); - console.log(`- Port: ${mongodbConfig.MONGODB_PORT}`); - console.log(`- Database: ${mongodbConfig.MONGODB_DATABASE}`); - console.log(`- Username: ${mongodbConfig.MONGODB_USERNAME}`); - console.log(`- TLS enabled: ${mongodbConfig.MONGODB_TLS}`); - console.log(`- Max pool size: ${mongodbConfig.MONGODB_MAX_POOL_SIZE}`); - - // Build connection URI - const buildMongoUri = () => { - if (mongodbConfig.MONGODB_URI) { - return mongodbConfig.MONGODB_URI; - } - - const auth = mongodbConfig.MONGODB_USERNAME && mongodbConfig.MONGODB_PASSWORD - ? `${mongodbConfig.MONGODB_USERNAME}:${mongodbConfig.MONGODB_PASSWORD}@` - : ''; - - const tls = mongodbConfig.MONGODB_TLS ? '?tls=true' : ''; - - return `mongodb://${auth}${mongodbConfig.MONGODB_HOST}:${mongodbConfig.MONGODB_PORT}/${mongodbConfig.MONGODB_DATABASE}${tls}`; - }; - - const mongoUri = buildMongoUri(); - console.log('MongoDB connection URI:', mongoUri.replace(/:[^:@]*@/, ':***@')); // Hide password - - // Example MongoDB client configuration - const mongoClientConfig = { - maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE, - minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE, - maxIdleTimeMS: mongodbConfig.MONGODB_MAX_IDLE_TIME, - connectTimeoutMS: mongodbConfig.MONGODB_CONNECT_TIMEOUT, - socketTimeoutMS: mongodbConfig.MONGODB_SOCKET_TIMEOUT, - serverSelectionTimeoutMS: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT, - retryWrites: mongodbConfig.MONGODB_RETRY_WRITES, - w: mongodbConfig.MONGODB_WRITE_CONCERN, - readPreference: mongodbConfig.MONGODB_READ_PREFERENCE, - }; - - console.log('MongoDB client configuration:', mongoClientConfig); - - // Example collections structure - const collections = [ - 'sentiment_data', // News sentiment analysis - 'market_news', // Raw news articles - 'social_signals', // Social media signals - 'earnings_reports', // Earnings data - 'analyst_ratings', // Analyst recommendations - ]; - - console.log('Example collections:', collections); -} - -/** - * Example 10: Dragonfly (Redis replacement) configuration - */ -function dragonflyConfigurationExample() { - console.log('=== Dragonfly Configuration Example ==='); - - // Access Dragonfly configuration - console.log('Dragonfly settings:'); - console.log(`- Host: ${dragonflyConfig.DRAGONFLY_HOST}`); - console.log(`- Port: ${dragonflyConfig.DRAGONFLY_PORT}`); - console.log(`- Database: ${dragonflyConfig.DRAGONFLY_DATABASE}`); - console.log(`- Cache mode: ${dragonflyConfig.DRAGONFLY_CACHE_MODE}`); - console.log(`- Max memory: ${dragonflyConfig.DRAGONFLY_MAX_MEMORY}`); - console.log(`- Pool size: ${dragonflyConfig.DRAGONFLY_POOL_SIZE}`); - // Example Dragonfly client configuration - const dragonflyClientConfig = { - host: dragonflyConfig.DRAGONFLY_HOST, - port: dragonflyConfig.DRAGONFLY_PORT, - db: dragonflyConfig.DRAGONFLY_DATABASE, - password: dragonflyConfig.DRAGONFLY_PASSWORD || undefined, - username: dragonflyConfig.DRAGONFLY_USERNAME || undefined, - retryDelayOnFailover: dragonflyConfig.DRAGONFLY_RETRY_DELAY, - maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES, - connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT, - commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT, - enableAutoPipelining: true, - }; - - console.log('Dragonfly client configuration:', dragonflyClientConfig); - - // Example cache key patterns - const cachePatterns = { - marketData: 'market:{symbol}:{timeframe}', - indicators: 'indicators:{symbol}:{indicator}:{period}', - positions: 'positions:{account_id}', - orders: 'orders:{order_id}', - rateLimit: 'rate_limit:{provider}:{endpoint}', - sessions: 'session:{user_id}', - }; - - console.log('Example cache key patterns:', cachePatterns); - - // Example TTL configurations - const ttlConfigs = { - marketData: 60, // 1 minute for real-time data - indicators: 300, // 5 minutes for calculated indicators - positions: 30, // 30 seconds for positions - orders: 86400, // 1 day for order history - rateLimit: 3600, // 1 hour for rate limiting - sessions: 1800, // 30 minutes for user sessions - }; - - console.log('Example TTL configurations (seconds):', ttlConfigs); -} - -/** - * Example 11: Monitoring stack configuration (Prometheus, Grafana, Loki) - */ -function monitoringConfigurationExample() { - console.log('=== Monitoring Configuration Example ==='); - - // Prometheus configuration - console.log('Prometheus settings:'); - console.log(`- Host: ${prometheusConfig.PROMETHEUS_HOST}`); - console.log(`- Port: ${prometheusConfig.PROMETHEUS_PORT}`); - console.log(`- Scrape interval: ${prometheusConfig.PROMETHEUS_SCRAPE_INTERVAL}`); - console.log(`- Retention time: ${prometheusConfig.PROMETHEUS_RETENTION_TIME}`); - - // Grafana configuration - console.log('\nGrafana settings:'); - console.log(`- Host: ${grafanaConfig.GRAFANA_HOST}`); - console.log(`- Port: ${grafanaConfig.GRAFANA_PORT}`); - console.log(`- Admin user: ${grafanaConfig.GRAFANA_ADMIN_USER}`); - console.log(`- Allow sign up: ${grafanaConfig.GRAFANA_ALLOW_SIGN_UP}`); - console.log(`- Database type: ${grafanaConfig.GRAFANA_DATABASE_TYPE}`); - - // Loki configuration - console.log('\nLoki settings:'); - console.log(`- Host: ${lokiConfig.LOKI_HOST}`); - console.log(`- Port: ${lokiConfig.LOKI_PORT}`); - console.log(`- Batch size: ${lokiConfig.LOKI_BATCH_SIZE}`); - console.log(`- Retention period: ${lokiConfig.LOKI_RETENTION_PERIOD}`); - - // Example monitoring endpoints - const monitoringEndpoints = { - prometheus: `http://${prometheusConfig.PROMETHEUS_HOST}:${prometheusConfig.PROMETHEUS_PORT}`, - grafana: `http://${grafanaConfig.GRAFANA_HOST}:${grafanaConfig.GRAFANA_PORT}`, - loki: `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`, - }; - - console.log('\nMonitoring endpoints:', monitoringEndpoints); - - // Example metrics configuration - const metricsConfig = { - defaultLabels: { - service: 'stock-bot', - environment: getEnvironment(), - version: process.env.npm_package_version || '1.0.0', - }, - collectDefaultMetrics: true, - prefix: 'stockbot_', - buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60], // Response time buckets in seconds - }; - - console.log('Example metrics configuration:', metricsConfig); -} - -/** - * Example 12: Multi-database service configuration - */ -function multiDatabaseServiceExample() { - console.log('=== Multi-Database Service Example ==='); - - // Complete database configuration for a microservice - const serviceConfig = { - service: { - name: 'market-data-processor', - version: '1.0.0', - environment: getEnvironment(), - }, - - // PostgreSQL for operational data - postgresql: { - host: postgresConfig.POSTGRES_HOST, - port: postgresConfig.POSTGRES_PORT, - database: postgresConfig.POSTGRES_DATABASE, - username: postgresConfig.POSTGRES_USERNAME, - password: postgresConfig.POSTGRES_PASSWORD, - ssl: postgresConfig.POSTGRES_SSL, - pool: { - min: postgresConfig.POSTGRES_POOL_MIN, - max: postgresConfig.POSTGRES_POOL_MAX, - idleTimeout: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT, - }, - }, - - // QuestDB for time-series data - questdb: { - host: questdbConfig.QUESTDB_HOST, - httpPort: questdbConfig.QUESTDB_HTTP_PORT, - pgPort: questdbConfig.QUESTDB_PG_PORT, - database: questdbConfig.QUESTDB_DEFAULT_DATABASE, - timeout: questdbConfig.QUESTDB_REQUEST_TIMEOUT, - }, - - // MongoDB for document storage - mongodb: { - host: mongodbConfig.MONGODB_HOST, - port: mongodbConfig.MONGODB_PORT, - database: mongodbConfig.MONGODB_DATABASE, - username: mongodbConfig.MONGODB_USERNAME, - maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE, - readPreference: mongodbConfig.MONGODB_READ_PREFERENCE, - }, - - // Dragonfly for caching - dragonfly: { - host: dragonflyConfig.DRAGONFLY_HOST, - port: dragonflyConfig.DRAGONFLY_PORT, - database: dragonflyConfig.DRAGONFLY_DATABASE, - poolSize: dragonflyConfig.DRAGONFLY_POOL_SIZE, - commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT, - }, - - // Monitoring - monitoring: { - prometheus: { - pushGateway: `http://${prometheusConfig.PROMETHEUS_HOST}:${prometheusConfig.PROMETHEUS_PORT}`, - scrapeInterval: prometheusConfig.PROMETHEUS_SCRAPE_INTERVAL, - }, loki: { - host: lokiConfig.LOKI_HOST, - port: lokiConfig.LOKI_PORT, - batchSize: lokiConfig.LOKI_BATCH_SIZE, - labels: { - service: 'market-data-processor', - environment: getEnvironment(), - }, - }, - }, - }; - - console.log('Complete service configuration:', JSON.stringify(serviceConfig, null, 2)); - - // Example data flow - const dataFlow = { - ingestion: 'Market data → Dragonfly (cache) → QuestDB (storage)', - processing: 'QuestDB → Analysis → PostgreSQL (results) → MongoDB (metadata)', - serving: 'Dragonfly (cache) ← PostgreSQL/QuestDB ← API requests', - monitoring: 'All services → Prometheus → Grafana dashboards', - logging: 'All services → Loki → Grafana log viewer', - }; - - console.log('\nData flow patterns:', dataFlow); -} - -/** - * Example 8: Creating a service configuration object - */ -function serviceConfigurationExample() { - console.log('=== Service Configuration Example ==='); - - // Example: Market Data Gateway service configuration - const marketDataGatewayConfig = { - service: { - name: 'market-data-gateway', - port: 3001, - environment: getEnvironment(), - }, - database: { - host: postgresConfig.POSTGRES_HOST, - port: postgresConfig.POSTGRES_PORT, - name: postgresConfig.POSTGRES_DATABASE, - ssl: postgresConfig.POSTGRES_SSL, - }, logging: { - level: loggingConfig.LOG_LEVEL, - console: loggingConfig.LOG_CONSOLE, - loki: { - host: lokiConfig.LOKI_HOST, - port: lokiConfig.LOKI_PORT, - labels: { - service: 'market-data-gateway', - environment: getEnvironment(), - } - } - }, - dataProviders: { - default: DEFAULT_DATA_PROVIDER, - enabled: getEnabledProviders(), - timeout: dataProvidersConfig.DATA_PROVIDER_TIMEOUT, - retries: dataProvidersConfig.DATA_PROVIDER_RETRIES, - }, - cache: { - enabled: dataProvidersConfig.DATA_CACHE_ENABLED, - ttl: dataProvidersConfig.DATA_CACHE_TTL, - maxSize: dataProvidersConfig.DATA_CACHE_MAX_SIZE, - } - }; - - console.log('Market Data Gateway configuration:', JSON.stringify(marketDataGatewayConfig, null, 2)); -} - -/** - * Main example runner - */ -function runAllExamples() { - console.log('🚀 Stock Bot Configuration Examples\n'); - - try { - basicUsageExample(); - console.log('\n'); - - databaseConnectionExample(); - console.log('\n'); - - loggingSetupExample(); - console.log('\n'); - - riskManagementExample(); - console.log('\n'); - - dataProviderExample(); - console.log('\n'); - - environmentSpecificExample(); - console.log('\n'); - - configurationValidationExample(); - console.log('\n'); - - questdbConfigurationExample(); - console.log('\n'); - - mongodbConfigurationExample(); - console.log('\n'); - - dragonflyConfigurationExample(); - console.log('\n'); - - monitoringConfigurationExample(); - console.log('\n'); - - multiDatabaseServiceExample(); - console.log('\n'); - - serviceConfigurationExample(); - - } catch (error) { - console.error('Example execution error:', error); - } -} - -// Export the examples for use in other files -export { - basicUsageExample, - databaseConnectionExample, - loggingSetupExample, - riskManagementExample, - dataProviderExample, - environmentSpecificExample, - configurationValidationExample, - questdbConfigurationExample, - mongodbConfigurationExample, - dragonflyConfigurationExample, - monitoringConfigurationExample, - multiDatabaseServiceExample, - serviceConfigurationExample, - runAllExamples, -}; - -// Run examples if this file is executed directly -if (require.main === module) { - runAllExamples(); -} diff --git a/libs/http/src/client.ts b/libs/http/src/client.ts index 0f29f99..57d72f6 100644 --- a/libs/http/src/client.ts +++ b/libs/http/src/client.ts @@ -7,7 +7,7 @@ import type { import { HttpError } from './types.js'; import { ProxyManager } from './proxy-manager.js'; import axios, { type AxiosResponse, AxiosError } from 'axios'; -import { clear } from 'node:console'; +import { loggingConfig } from '@stock-bot/config'; export class HttpClient { private readonly config: HttpClientConfig; @@ -15,8 +15,9 @@ export class HttpClient { constructor(config: HttpClientConfig = {}, logger?: Logger) { this.config = config; - this.logger = logger?.child({ + this.logger = logger?.child({ //TODO fix pino levels component: 'http', + // level: loggingConfig?.LOG_LEVEL || 'info', }); } @@ -68,11 +69,11 @@ export class HttpClient { return response; } catch (error) { - this.logger?.warn('HTTP request failed', { - method: finalConfig.method, - url: finalConfig.url, - error: (error as Error).message, - }); + // this.logger?.warn('HTTP request failed', { + // method: finalConfig.method, + // url: finalConfig.url, + // error: (error as Error).message, + // }); throw error; } } @@ -124,10 +125,10 @@ export class HttpClient { try { const options = this.buildFetchOptions(config, signal); - this.logger?.debug('Making request with fetch: ', { url: config.url, options }) - + this.logger?.debug('Making request with fetch: ', { url: config.url, options }) + // console.log(options) const response = await fetch(config.url, options); - + // console.log('Fetch response:', response.status); return this.parseFetchResponse(response); } catch (error) { throw signal.aborted @@ -201,7 +202,6 @@ export class HttpClient { if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) { (options as any).proxy = ProxyManager.createProxyUrl(config.proxy); } - return options; } /** * Build Axios options (for reference, though we're creating instance in ProxyManager) diff --git a/libs/http/src/proxy-manager.ts b/libs/http/src/proxy-manager.ts index 2a92d8e..301719d 100644 --- a/libs/http/src/proxy-manager.ts +++ b/libs/http/src/proxy-manager.ts @@ -16,9 +16,6 @@ export class ProxyManager { */ static createProxyUrl(proxy: ProxyInfo): string { const { protocol, host, port, username, password } = proxy; - if(protocol.includes('socks')) { - return `${protocol}://${host}:${port}`; - } if (username && password) { return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`; } @@ -32,11 +29,10 @@ export class ProxyManager { this.validateConfig(proxy); const proxyUrl = this.createProxyUrl(proxy); - console.log(`Using proxy url: ${proxyUrl}`); switch (proxy.protocol) { case 'socks4': case 'socks5': - console.log(`Using SOCKS proxy: ${proxyUrl}`); + // console.log(`Using SOCKS proxy: ${proxyUrl}`); return new SocksProxyAgent(proxyUrl); case 'http': return new HttpProxyAgent(proxyUrl); @@ -54,12 +50,6 @@ export class ProxyManager { return { httpAgent: agent, httpsAgent: agent, - // timeout: 30000, - // validateStatus: () => true, // Don't throw errors on HTTP status codes - // maxRedirects: 5, - // headers: { - // 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' - // } }; } /** diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 867542d..9b1dc13 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -200,9 +200,10 @@ export class Logger { * Create child logger with additional context */ child(context: LogContext): Logger { - const childLogger = new Logger((this.pino.bindings() as any).service, { ...this.context, ...context }); - // Use the pino child logger to properly propagate context - childLogger.pino = this.pino.child(context); + // Create child logger that shares the same pino instance with additional context + const childLogger = Object.create(Logger.prototype); + childLogger.context = { ...this.context, ...context }; + childLogger.pino = this.pino.child(context); // Let pino handle level inheritance naturally return childLogger; } }