diff --git a/apps/data-service/src/proxy-demo.ts b/apps/data-service/src/proxy-demo.ts index 9f39e1d..1364578 100644 --- a/apps/data-service/src/proxy-demo.ts +++ b/apps/data-service/src/proxy-demo.ts @@ -92,14 +92,14 @@ async function demonstrateCustomProxySource() { const customSources : ProxySource[] = [ { - url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt', + url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', protocol: 'http' } ]; try { const count = await proxyService.scrapeProxies(customSources); - console.log(`✅ Scraped ${count} SOCKS4 proxies from custom source`); + console.log(`✅ Scraped ${count} proxies from custom source`); } catch (error) { console.error('❌ Custom source scraping failed:', error); } diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index 0c3c979..a9655cf 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -1,6 +1,6 @@ import { createLogger } from '@stock-bot/logger'; import createCache, { type CacheProvider } from '@stock-bot/cache'; -import { HttpClient, ProxyConfig , RequestConfig } from '@stock-bot/http-client'; +import { HttpClient, HttpClientConfig, ProxyConfig , RequestConfig } from '@stock-bot/http-client'; import type { Logger as PinoLogger } from 'pino'; export interface ProxySource { @@ -79,7 +79,6 @@ export class ProxyService { this.httpClient = new HttpClient({ timeout: this.CHECK_TIMEOUT, - retries: 1 }); this.logger.info('ProxyService initialized'); @@ -265,13 +264,11 @@ export class ProxyService { */ async checkProxy(proxy: ProxyConfig, checkUrl: string = this.DEFAULT_CHECK_URL): Promise { const startTime = Date.now(); - try { // Create a new HttpClient instance with the proxy const proxyClient = new HttpClient({ timeout: this.CHECK_TIMEOUT, - retries: 0, proxy: proxy }); diff --git a/libs/http-client/src/client.ts b/libs/http-client/src/client.ts index e69de29..cc0d0c1 100644 --- a/libs/http-client/src/client.ts +++ b/libs/http-client/src/client.ts @@ -0,0 +1,263 @@ +import type { Logger } from '@stock-bot/logger'; +import type { + HttpClientConfig, + RequestConfig, + HttpResponse, +} from './types.js'; +import { HttpError } from './types.js'; +import { ProxyManager } from './proxy-manager.js'; +import { request } from 'undici'; + +export class HttpClient { + private readonly config: HttpClientConfig; + private readonly logger?: Logger; + + constructor(config: HttpClientConfig = {}, logger?: Logger) { + this.config = config; + this.logger = logger; + + // Validate proxy configuration if provided + if (this.config.proxy) { + ProxyManager.validateConfig(this.config.proxy); + } + } + + /** + * Make an HTTP request using hybrid approach: + * - Bun fetch for HTTP/HTTPS proxies (fastest ~150-300ms) + * - Undici for SOCKS proxies (fast ~300-600ms) + */ + async request(config: RequestConfig): Promise> { + const finalConfig = this.mergeConfig(config); + + this.logger?.debug('Making HTTP request', { + method: finalConfig.method, + url: finalConfig.url + }); + + try { + const response = await this.executeRequest(finalConfig); + + this.logger?.debug('HTTP request successful', { + method: finalConfig.method, + url: finalConfig.url, + status: response.status, + }); + + return response; + } catch (error) { + this.logger?.warn('HTTP request failed', { + method: finalConfig.method, + url: finalConfig.url, + error: (error as Error).message, + }); + throw error; + } + } + + /** + * Convenience methods for common HTTP methods + */ + async get(url: string, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'GET', url }); + } + + async post(url: string, body?: any, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'POST', url, body }); + } + + async put(url: string, body?: any, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'PUT', url, body }); + } + + async del(url: string, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'DELETE', url }); + } + + async patch(url: string, body?: any, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'PATCH', url, body }); + } + + /** + * Execute HTTP request using hybrid approach + */ + private async executeRequest(config: RequestConfig): Promise> { + const timeout = config.timeout ?? this.config.timeout ?? 30000; + + // Decide between Bun fetch and Undici based on proxy type + if (this.config.proxy) { + if (ProxyManager.shouldUseBunFetch(this.config.proxy)) { + return this.executeBunRequest(config, timeout); + } else { + return this.executeUndiciRequest(config, timeout); + } + } else { + // No proxy - use fast Bun fetch + return this.executeBunRequest(config, timeout); + } + } + + /** + * Execute request using Bun's native fetch (fastest for HTTP/HTTPS proxies) + */ + private async executeBunRequest(config: RequestConfig, timeout: number): Promise> { + const abortController = new AbortController(); + const timeoutId = setTimeout(() => abortController.abort(), timeout); + + try { + const requestOptions: RequestInit = { + method: config.method || 'GET', + headers: config.headers || {}, + signal: abortController.signal, + }; + + // Add body for non-GET requests + if (config.body && config.method !== 'GET') { + if (typeof config.body === 'object') { + requestOptions.body = JSON.stringify(config.body); + requestOptions.headers = { + 'Content-Type': 'application/json', + ...requestOptions.headers, + }; + } else { + requestOptions.body = config.body; + } + } + + // Add proxy for Bun fetch (HTTP/HTTPS only) + if (this.config.proxy && ProxyManager.shouldUseBunFetch(this.config.proxy)) { + const proxyUrl = ProxyManager.createBunProxyUrl(this.config.proxy); + // Bun supports proxy via environment or direct configuration + (requestOptions as any).proxy = proxyUrl; + } + + const response = await fetch(config.url, requestOptions); + clearTimeout(timeoutId); + + return await this.parseResponse(response); + } catch (error) { + clearTimeout(timeoutId); + if (abortController.signal.aborted) { + throw new HttpError(`Request timeout after ${timeout}ms`); + } + throw new HttpError(`Request failed: ${(error as Error).message}`); + } + } + + /** + * Execute request using Undici (fast for SOCKS proxies) + */ + private async executeUndiciRequest(config: RequestConfig, timeout: number): Promise> { + try { + const requestOptions: any = { + method: config.method || 'GET', + headers: config.headers || {}, + headersTimeout: timeout, + bodyTimeout: timeout, + }; + + // Add body for non-GET requests + if (config.body && config.method !== 'GET') { + if (typeof config.body === 'object') { + requestOptions.body = JSON.stringify(config.body); + requestOptions.headers = { + 'Content-Type': 'application/json', + ...requestOptions.headers, + }; + } else { + requestOptions.body = config.body; + } + } + + // Add SOCKS proxy via Undici agent + if (this.config.proxy && !ProxyManager.shouldUseBunFetch(this.config.proxy)) { + requestOptions.dispatcher = ProxyManager.createUndiciAgent(this.config.proxy); + } + console.log('Executing Undici request', requestOptions) + + const response = await request(config.url, requestOptions); + + // Convert Undici response to our format + const data = await this.parseUndiciResponse(response); + + return { + data, + status: response.statusCode, + headers: response.headers as Record, + ok: response.statusCode >= 200 && response.statusCode < 300, + }; + } catch (error) { + throw new HttpError(`Undici request failed: ${(error as Error).message}`); + } + } + + /** + * Parse standard fetch response + */ + private async parseResponse(response: Response): Promise> { + const responseHeaders: Record = {}; + response.headers.forEach((value, key) => { + responseHeaders[key] = value; + }); + + let data: T; + const contentType = response.headers.get('content-type') || ''; + + if (contentType.includes('application/json')) { + data = await response.json(); + } else if (contentType.includes('text/')) { + data = await response.text() as any; + } else { + data = await response.arrayBuffer() as any; + } + + if (!response.ok) { + throw new HttpError( + `Request failed with status ${response.status}`, + response.status, + { + data, + status: response.status, + headers: responseHeaders, + ok: response.ok, + } + ); + } + + return { + data, + status: response.status, + headers: responseHeaders, + ok: response.ok, + }; + } + + /** + * Parse Undici response + */ + private async parseUndiciResponse(response: any): Promise { + const contentType = response.headers['content-type'] || ''; + + if (contentType.includes('application/json')) { + return await response.body.json(); + } else if (contentType.includes('text/')) { + return await response.body.text(); + } else { + return await response.body.arrayBuffer(); + } + } + + /** + * Merge request config with default config + */ + private mergeConfig(config: RequestConfig): RequestConfig { + return { + ...config, + headers: { + ...this.config.headers, + ...config.headers, + }, + timeout: config.timeout ?? this.config.timeout, + }; + } +} diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 3c0dc22..7e64cae 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -39,11 +39,13 @@ function createTransports(serviceName: string, options?: { if (enableConsole) { targets.push({ target: 'pino-pretty', - level: loggingConfig.LOG_LEVEL, options: { + level: loggingConfig.LOG_LEVEL, + options: { + minimumLevel: 'info', colorize: true, - translateTime: 'yyyy-mm-dd HH:MM:ss.l', - ignore: 'pid,hostname', - messageFormat: '[{service}] {msg}' + translateTime: 'HH:MM:ss.l', + ignore: 'pid,hostname,service,environment,version', + messageFormat: '[{service}] {msg}', } }); }