From d67d07cba61fda049051615bbd637ce2902dc283 Mon Sep 17 00:00:00 2001 From: Bojan Kucera Date: Sun, 8 Jun 2025 08:05:20 -0400 Subject: [PATCH] split up http into adapters --- .../src/services/proxy.service.ts | 13 +- libs/http/src/adapters/axios-adapter.ts | 53 ++++ libs/http/src/adapters/factory.ts | 28 ++ libs/http/src/adapters/fetch-adapter.ts | 66 +++++ libs/http/src/adapters/index.ts | 4 + libs/http/src/adapters/types.ts | 16 ++ libs/http/src/client.ts | 267 ++++-------------- libs/http/src/index.ts | 1 + libs/http/src/types.ts | 3 +- libs/logger/src/logger.ts | 3 +- package.json | 4 +- 11 files changed, 230 insertions(+), 228 deletions(-) create mode 100644 libs/http/src/adapters/axios-adapter.ts create mode 100644 libs/http/src/adapters/factory.ts create mode 100644 libs/http/src/adapters/fetch-adapter.ts create mode 100644 libs/http/src/adapters/index.ts create mode 100644 libs/http/src/adapters/types.ts diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index f44f9e6..c329407 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -7,10 +7,10 @@ export class ProxyService { private logger = new Logger('proxy-service'); private cache: CacheProvider = createCache('hybrid'); private httpClient: HttpClient; - private readonly concurrencyLimit = pLimit(300); + private readonly concurrencyLimit = pLimit(1); private readonly CACHE_KEY = 'proxy'; private readonly CACHE_TTL = 86400; // 24 hours - private readonly CHECK_TIMEOUT = 10000; + private readonly CHECK_TIMEOUT = 3000; private readonly CHECK_IP = '99.246.102.205' private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; private readonly PROXY_SOURCES = [ @@ -40,9 +40,9 @@ export class ProxyService { {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, - {url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' }, + {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https' }, {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', }, - {url: 'https://github.com/officialputuid/KangProxy/blob/KangProxy/https/https.txt',protocol: 'https', }, + {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',protocol: 'https', }, // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', }, // {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', }, // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, @@ -179,10 +179,9 @@ export class ProxyService { ...proxy, isWorking, checkedAt: new Date(), - responseTime: response.responseTime, - }; + responseTime: response.responseTime, }; // console.log('Proxy check result:', proxy); - if (isWorking && !JSON.stringify(response.data).includes(this.CHECK_IP)) { + if (isWorking && JSON.stringify(response.data).includes(this.CHECK_IP)) { await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL); } // else { // TODO diff --git a/libs/http/src/adapters/axios-adapter.ts b/libs/http/src/adapters/axios-adapter.ts new file mode 100644 index 0000000..4b1ff5b --- /dev/null +++ b/libs/http/src/adapters/axios-adapter.ts @@ -0,0 +1,53 @@ +import axios, { type AxiosRequestConfig, type AxiosResponse } from 'axios'; +import type { RequestConfig, HttpResponse } from '../types.js'; +import type { RequestAdapter } from './types.js'; +import { ProxyManager } from '../proxy-manager.js'; +import { HttpError } from '../types.js'; + +/** + * Axios adapter for SOCKS proxies + */ +export class AxiosAdapter implements RequestAdapter { + canHandle(config: RequestConfig): boolean { + // Axios handles SOCKS proxies + return Boolean(config.proxy && (config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5')); + } + + async request(config: RequestConfig, signal: AbortSignal): Promise> { + const { url, method = 'GET', headers, data, proxy } = config; + + if (!proxy) { + throw new Error('Axios adapter requires proxy configuration'); + } + + // Create proxy configuration using ProxyManager + const axiosConfig: AxiosRequestConfig = { + ...ProxyManager.createAxiosConfig(proxy), + url, + method, + headers, + data, + signal, + // Don't throw on non-2xx status codes - let caller handle + validateStatus: () => true, + }; const response: AxiosResponse = await axios(axiosConfig); + + const httpResponse: HttpResponse = { + data: response.data, + status: response.status, + headers: response.headers as Record, + ok: response.status >= 200 && response.status < 300, + }; + + // Throw HttpError for non-2xx status codes + if (!httpResponse.ok) { + throw new HttpError( + `Request failed with status ${response.status}`, + response.status, + httpResponse + ); + } + + return httpResponse; + } +} diff --git a/libs/http/src/adapters/factory.ts b/libs/http/src/adapters/factory.ts new file mode 100644 index 0000000..a7b4b5e --- /dev/null +++ b/libs/http/src/adapters/factory.ts @@ -0,0 +1,28 @@ +import type { RequestConfig } from '../types.js'; +import type { RequestAdapter } from './types.js'; +import { FetchAdapter } from './fetch-adapter.js'; +import { AxiosAdapter } from './axios-adapter.js'; + +/** + * Factory for creating the appropriate request adapter + */ +export class AdapterFactory { + private static adapters: RequestAdapter[] = [ + new AxiosAdapter(), // Check SOCKS first + new FetchAdapter(), // Fallback to fetch for everything else + ]; + + /** + * Get the appropriate adapter for the given configuration + */ + static getAdapter(config: RequestConfig): RequestAdapter { + for (const adapter of this.adapters) { + if (adapter.canHandle(config)) { + return adapter; + } + } + + // Fallback to fetch adapter + return new FetchAdapter(); + } +} diff --git a/libs/http/src/adapters/fetch-adapter.ts b/libs/http/src/adapters/fetch-adapter.ts new file mode 100644 index 0000000..36bb8b4 --- /dev/null +++ b/libs/http/src/adapters/fetch-adapter.ts @@ -0,0 +1,66 @@ +import type { RequestConfig, HttpResponse } from '../types.js'; +import type { RequestAdapter } from './types.js'; +import { ProxyManager } from '../proxy-manager.js'; +import { HttpError } from '../types.js'; + +/** + * Fetch adapter for HTTP/HTTPS proxies and non-proxy requests + */ +export class FetchAdapter implements RequestAdapter { + canHandle(config: RequestConfig): boolean { + // Fetch handles non-proxy requests and HTTP/HTTPS proxies + return !config.proxy || config.proxy.protocol === 'http' || config.proxy.protocol === 'https'; + } + + async request(config: RequestConfig, signal: AbortSignal): Promise> { + const { url, method = 'GET', headers, data, proxy } = config; + + // Prepare fetch options + const fetchOptions: RequestInit = { + method, + headers, + signal, + }; + + // Add body for non-GET requests + if (data && method !== 'GET') { + fetchOptions.body = typeof data === 'string' ? data : JSON.stringify(data); + if (typeof data === 'object') { + fetchOptions.headers = { 'Content-Type': 'application/json', ...fetchOptions.headers }; + } + } + + // Add proxy if needed (using Bun's built-in proxy support) + if (proxy) { + (fetchOptions as any).proxy = ProxyManager.createProxyUrl(proxy); + } const response = await fetch(url, fetchOptions); + + // Parse response based on content type + let responseData: T; + const contentType = response.headers.get('content-type') || ''; + + if (contentType.includes('application/json')) { + responseData = await response.json() as T; + } else { + responseData = await response.text() as T; + } + + const httpResponse: HttpResponse = { + data: responseData, + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + ok: response.ok, + }; + + // Throw HttpError for non-2xx status codes + if (!response.ok) { + throw new HttpError( + `Request failed with status ${response.status}`, + response.status, + httpResponse + ); + } + + return httpResponse; + } +} diff --git a/libs/http/src/adapters/index.ts b/libs/http/src/adapters/index.ts new file mode 100644 index 0000000..ef3dfb8 --- /dev/null +++ b/libs/http/src/adapters/index.ts @@ -0,0 +1,4 @@ +export * from './types.js'; +export * from './fetch-adapter.js'; +export * from './axios-adapter.js'; +export * from './factory.js'; diff --git a/libs/http/src/adapters/types.ts b/libs/http/src/adapters/types.ts new file mode 100644 index 0000000..0c84b71 --- /dev/null +++ b/libs/http/src/adapters/types.ts @@ -0,0 +1,16 @@ +import type { RequestConfig, HttpResponse } from '../types.js'; + +/** + * Request adapter interface for different HTTP implementations + */ +export interface RequestAdapter { + /** + * Execute an HTTP request + */ + request(config: RequestConfig, signal: AbortSignal): Promise>; + + /** + * Check if this adapter can handle the given configuration + */ + canHandle(config: RequestConfig): boolean; +} diff --git a/libs/http/src/client.ts b/libs/http/src/client.ts index 57d72f6..401e0de 100644 --- a/libs/http/src/client.ts +++ b/libs/http/src/client.ts @@ -6,8 +6,7 @@ import type { } from './types.js'; import { HttpError } from './types.js'; import { ProxyManager } from './proxy-manager.js'; -import axios, { type AxiosResponse, AxiosError } from 'axios'; -import { loggingConfig } from '@stock-bot/config'; +import { AdapterFactory } from './adapters/index.js'; export class HttpClient { private readonly config: HttpClientConfig; @@ -15,10 +14,7 @@ export class HttpClient { constructor(config: HttpClientConfig = {}, logger?: Logger) { this.config = config; - this.logger = logger?.child({ //TODO fix pino levels - component: 'http', - // level: loggingConfig?.LOG_LEVEL || 'info', - }); + this.logger = logger?.child({ component: 'http' }); } // Convenience methods @@ -26,262 +22,99 @@ export class HttpClient { return this.request({ ...config, method: 'GET', url }); } - async post(url: string, body?: any, config: Omit = {}): Promise> { - return this.request({ ...config, method: 'POST', url, body }); + async post(url: string, data?: any, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'POST', url, data }); } - async put(url: string, body?: any, config: Omit = {}): Promise> { - return this.request({ ...config, method: 'PUT', url, body }); + async put(url: string, data?: any, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'PUT', url, data }); } async del(url: string, config: Omit = {}): Promise> { return this.request({ ...config, method: 'DELETE', url }); } - async patch(url: string, body?: any, config: Omit = {}): Promise> { - return this.request({ ...config, method: 'PATCH', url, body }); - } /** - * Main request method - unified and simplified + async patch(url: string, data?: any, config: Omit = {}): Promise> { + return this.request({ ...config, method: 'PATCH', url, data }); + } + + /** + * Main request method - clean and simple */ async request(config: RequestConfig): Promise> { const finalConfig = this.mergeConfig(config); const startTime = Date.now(); + this.logger?.debug('Making HTTP request', { method: finalConfig.method, url: finalConfig.url, hasProxy: !!finalConfig.proxy }); - try { // Single decision point for proxy type - only request-level proxy - const proxy = finalConfig.proxy; - const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy); - - const response = await this.makeRequest(finalConfig, useBunFetch); - const responseTime = Date.now() - startTime; - response.responseTime = responseTime; + try { + const response = await this.executeRequest(finalConfig); + response.responseTime = Date.now() - startTime; this.logger?.debug('HTTP request successful', { method: finalConfig.method, url: finalConfig.url, status: response.status, - responseTime: responseTime, + responseTime: response.responseTime, }); return response; } catch (error) { - // this.logger?.warn('HTTP request failed', { - // method: finalConfig.method, - // url: finalConfig.url, - // error: (error as Error).message, - // }); + this.logger?.warn('HTTP request failed', { + method: finalConfig.method, + url: finalConfig.url, + error: (error as Error).message, + }); throw error; } } /** - * Unified request method with consolidated timeout handling + * Execute request with timeout handling - no race conditions */ - private async makeRequest(config: RequestConfig, useBunFetch: boolean): Promise> { + private async executeRequest(config: RequestConfig): Promise> { const timeout = config.timeout ?? this.config.timeout ?? 30000; const controller = new AbortController(); - // Create timeout promise that rejects with proper error - let timeoutId: NodeJS.Timeout | undefined; - const timeoutPromise = new Promise((_, reject) => { - timeoutId = setTimeout(() => { - controller.abort(); - reject(new HttpError(`Request timeout after ${timeout}ms`)); - }, timeout); - }); // Create request promise (don't await here!) - const requestPromise = useBunFetch - ? this.fetchRequest(config, controller.signal) - : this.axiosRequest(config, controller.signal); - + // Set up timeout + const timeoutId = setTimeout(() => { + controller.abort(); + }, timeout); + try { - // Race the promises - const result = await Promise.race([requestPromise, timeoutPromise]); - if (timeoutId) clearTimeout(timeoutId); - return result; + // Get the appropriate adapter + const adapter = AdapterFactory.getAdapter(config); + + // Execute request + const response = await adapter.request(config, controller.signal); + + // Clear timeout on success + clearTimeout(timeoutId); + + return response; } catch (error) { - // If it's our timeout error, handle it - if (error instanceof HttpError && error.message.includes('timeout')) { - this.logger?.warn('Request timed out', { - method: config.method, - url: config.url, - timeout: timeout, - }); - throw error; // Re-throw the timeout error + clearTimeout(timeoutId); + + // Handle timeout + if (controller.signal.aborted) { + throw new HttpError(`Request timeout after ${timeout}ms`); } - // Handle other errors (network, parsing, etc.) - throw error; - } - - } - /** - * Bun fetch implementation (simplified) - */ - private async fetchRequest(config: RequestConfig, signal: AbortSignal): Promise> { - try { - const options = this.buildFetchOptions(config, signal); - - this.logger?.debug('Making request with fetch: ', { url: config.url, options }) - // console.log(options) - const response = await fetch(config.url, options); - // console.log('Fetch response:', response.status); - return this.parseFetchResponse(response); - } catch (error) { - throw signal.aborted - ? new HttpError(`Request timeout`) - : new HttpError(`Request failed: ${(error as Error).message}`); - } - } /** - * Axios implementation (for SOCKS proxies) - */ - private async axiosRequest(config: RequestConfig, signal: AbortSignal): Promise> { - if(config.proxy) { - try { - const axiosProxy = await ProxyManager.createAxiosConfig(config.proxy); - axiosProxy.url = config.url; - axiosProxy.method = config.method || 'GET'; - // console.log(axiosProxy) - // const axiosConfig = { - // ...axiosProxy, - // url: config.url, - // method: config.method || 'GET', - // // headers: config.headers || {}, - // // data: config.body, - // // signal, // Axios supports AbortSignal - // }; - - // console.log('Making request with Axios: ', axiosConfig ); - - const response: AxiosResponse = await axios.request(axiosProxy); - return this.parseAxiosResponse(response); - } catch (error) { - console.error('Axios request error:', error); - - // Handle AbortSignal timeout - if (signal.aborted) { - throw new HttpError(`Request timeout`); - } - - // Handle Axios timeout errors - if (error instanceof AxiosError && error.code === 'ECONNABORTED') { - throw new HttpError(`Request timeout`); - } - - throw new HttpError(`Request failed: ${(error as Error).message}`); + // Re-throw other errors + if (error instanceof HttpError) { + throw error; } - } else { - throw new HttpError(`Request failed: No proxy configured, use fetch instead`); + + throw new HttpError(`Request failed: ${(error as Error).message}`); } } - /** - * Build fetch options (extracted for clarity) - */ - private buildFetchOptions(config: RequestConfig, signal: AbortSignal): RequestInit { - const options: RequestInit = { - method: config.method || 'GET', - headers: config.headers || {}, - signal, - }; - - - // Add body - if (config.body && config.method !== 'GET') { - if (typeof config.body === 'object') { - options.body = JSON.stringify(config.body); - options.headers = { 'Content-Type': 'application/json', ...options.headers }; - } else { - options.body = config.body; - } - } - - // Add proxy (HTTP/HTTPS only) - request level only - if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) { - (options as any).proxy = ProxyManager.createProxyUrl(config.proxy); - } - return options; - } /** - * Build Axios options (for reference, though we're creating instance in ProxyManager) - */ - private buildAxiosOptions(config: RequestConfig, signal: AbortSignal): any { - const options: any = { - method: config.method || 'GET', - headers: config.headers || {}, - signal, // Axios supports AbortSignal - timeout: config.timeout || 30000, - maxRedirects: 5, - validateStatus: () => true // Don't throw on HTTP errors - }; - - // Add body - if (config.body && config.method !== 'GET') { - if (typeof config.body === 'object') { - options.data = config.body; - options.headers = { 'Content-Type': 'application/json', ...options.headers }; - } else { - options.data = config.body; - options.headers = { 'Content-Type': 'text/plain', ...options.headers }; - } - } - - return options; - } /** - * Parse fetch response (simplified) - */ - private async parseFetchResponse(response: Response): Promise> { - const data = await this.parseResponseBody(response); - const headers = Object.fromEntries(response.headers.entries()); - - if (!response.ok) { - throw new HttpError( - `Request failed with status ${response.status}`, - response.status, - { data, status: response.status, headers, ok: response.ok } - ); - } - - return { data, status: response.status, headers, ok: response.ok }; - } - /** - * Parse Axios response - */ - private parseAxiosResponse(response: AxiosResponse): HttpResponse { - const headers = response.headers as Record; - const status = response.status; - const ok = status >= 200 && status < 300; - const data = response.data; - - if (!ok) { - throw new HttpError( - `Request failed with status ${status}`, - status, - { data, status, headers, ok } - ); - } - - return { data, status, headers, ok }; - } - /** - * Unified body parsing (works for fetch response) - */ - private async parseResponseBody(response: Response): Promise { - const contentType = response.headers.get('content-type') || ''; - - if (contentType.includes('application/json')) { - return response.json(); - } else if (contentType.includes('text/')) { - return response.text() as any; - } else { - return response.arrayBuffer() as any; - } - } - /** - * Merge configs - request-level proxy only + * Merge configs with defaults */ private mergeConfig(config: RequestConfig): RequestConfig { return { diff --git a/libs/http/src/index.ts b/libs/http/src/index.ts index 0b65ce4..d29a8a1 100644 --- a/libs/http/src/index.ts +++ b/libs/http/src/index.ts @@ -2,6 +2,7 @@ export * from './types.js'; export * from './client.js'; export * from './proxy-manager.js'; +export * from './adapters/index.js'; // Default export export { HttpClient as default } from './client.js'; diff --git a/libs/http/src/types.ts b/libs/http/src/types.ts index 783207f..f768122 100644 --- a/libs/http/src/types.ts +++ b/libs/http/src/types.ts @@ -7,6 +7,7 @@ export interface ProxyInfo { port: number; username?: string; password?: string; + url?: string; // Full proxy URL for adapters isWorking?: boolean; responseTime?: number; error?: string; @@ -22,7 +23,7 @@ export interface RequestConfig { method?: HttpMethod; url: string; headers?: Record; - body?: any; + data?: any; // Changed from 'body' to 'data' for consistency timeout?: number; proxy?: ProxyInfo; } diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 9b1dc13..39afd55 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -21,12 +21,11 @@ const loggerCache = new Map(); function createTransports(serviceName: string): any { const targets: any[] = []; // const isDev = loggingConfig.LOG_ENVIRONMENT === 'development'; - // Console transport if (loggingConfig.LOG_CONSOLE) { targets.push({ target: 'pino-pretty', - level: loggingConfig.LOG_LEVEL, + level: 'error', // Only show errors on console options: { colorize: true, translateTime: 'yyyy-mm-dd HH:MM:ss.l', diff --git a/package.json b/package.json index 985c662..80afefa 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,9 @@ "infra:down": "docker-compose down", "infra:reset": "docker-compose down -v && docker-compose up -d dragonfly postgres questdb mongodb", "dev:full": "npm run infra:up && npm run docker:admin && turbo run dev", - "dev:clean": "npm run infra:reset && npm run dev:full" + "dev:clean": "npm run infra:reset && npm run dev:full", + "proxy": "bun run ./apps/data-service/src/proxy-demo.ts" + }, "workspaces": [ "libs/*",