split up http into adapters

apps/data-service/src/services/proxy.service.ts

@@ -7,10 +7,10 @@ export class ProxyService {
   private logger = new Logger('proxy-service');
   private cache: CacheProvider = createCache('hybrid');
   private httpClient: HttpClient;
-  private readonly concurrencyLimit = pLimit(300);
+  private readonly concurrencyLimit = pLimit(1);
   private readonly CACHE_KEY = 'proxy';
   private readonly CACHE_TTL = 86400; // 24 hours
-  private readonly CHECK_TIMEOUT = 10000;
+  private readonly CHECK_TIMEOUT = 3000;
   private readonly CHECK_IP = '99.246.102.205'
   private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
   private readonly PROXY_SOURCES = [
@@ -40,9 +40,9 @@ export class ProxyService {
     {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
     {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
     {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
-    {url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' },
+    {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https' },
     {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', },
-    {url: 'https://github.com/officialputuid/KangProxy/blob/KangProxy/https/https.txt',protocol: 'https', },
+    {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',protocol: 'https', },
     // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', },
     // {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', },
     // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
@@ -179,10 +179,9 @@ export class ProxyService {
         ...proxy,
         isWorking,
         checkedAt: new Date(),
-        responseTime: response.responseTime,
-      };
+        responseTime: response.responseTime,      };
       // console.log('Proxy check result:', proxy);
-      if (isWorking && !JSON.stringify(response.data).includes(this.CHECK_IP)) {
+      if (isWorking && JSON.stringify(response.data).includes(this.CHECK_IP)) {
         await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL);
       } 
       // else { // TODO

libs/http/src/adapters/axios-adapter.ts

@@ -0,0 +1,53 @@
+import axios, { type AxiosRequestConfig, type AxiosResponse } from 'axios';
+import type { RequestConfig, HttpResponse } from '../types.js';
+import type { RequestAdapter } from './types.js';
+import { ProxyManager } from '../proxy-manager.js';
+import { HttpError } from '../types.js';
+
+/**
+ * Axios adapter for SOCKS proxies
+ */
+export class AxiosAdapter implements RequestAdapter {
+  canHandle(config: RequestConfig): boolean {
+    // Axios handles SOCKS proxies
+    return Boolean(config.proxy && (config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5'));
+  }
+
+  async request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
+    const { url, method = 'GET', headers, data, proxy } = config;
+
+    if (!proxy) {
+      throw new Error('Axios adapter requires proxy configuration');
+    }
+
+    // Create proxy configuration using ProxyManager
+    const axiosConfig: AxiosRequestConfig = {
+      ...ProxyManager.createAxiosConfig(proxy),
+      url,
+      method,
+      headers,
+      data,
+      signal,
+      // Don't throw on non-2xx status codes - let caller handle
+      validateStatus: () => true,
+    };    const response: AxiosResponse<T> = await axios(axiosConfig);
+
+    const httpResponse: HttpResponse<T> = {
+      data: response.data,
+      status: response.status,
+      headers: response.headers as Record<string, string>,
+      ok: response.status >= 200 && response.status < 300,
+    };
+
+    // Throw HttpError for non-2xx status codes
+    if (!httpResponse.ok) {
+      throw new HttpError(
+        `Request failed with status ${response.status}`,
+        response.status,
+        httpResponse
+      );
+    }
+
+    return httpResponse;
+  }
+}

libs/http/src/adapters/factory.ts

@@ -0,0 +1,28 @@
+import type { RequestConfig } from '../types.js';
+import type { RequestAdapter } from './types.js';
+import { FetchAdapter } from './fetch-adapter.js';
+import { AxiosAdapter } from './axios-adapter.js';
+
+/**
+ * Factory for creating the appropriate request adapter
+ */
+export class AdapterFactory {
+  private static adapters: RequestAdapter[] = [
+    new AxiosAdapter(), // Check SOCKS first
+    new FetchAdapter(), // Fallback to fetch for everything else
+  ];
+
+  /**
+   * Get the appropriate adapter for the given configuration
+   */
+  static getAdapter(config: RequestConfig): RequestAdapter {
+    for (const adapter of this.adapters) {
+      if (adapter.canHandle(config)) {
+        return adapter;
+      }
+    }
+    
+    // Fallback to fetch adapter
+    return new FetchAdapter();
+  }
+}

libs/http/src/adapters/fetch-adapter.ts

@@ -0,0 +1,66 @@
+import type { RequestConfig, HttpResponse } from '../types.js';
+import type { RequestAdapter } from './types.js';
+import { ProxyManager } from '../proxy-manager.js';
+import { HttpError } from '../types.js';
+
+/**
+ * Fetch adapter for HTTP/HTTPS proxies and non-proxy requests
+ */
+export class FetchAdapter implements RequestAdapter {
+  canHandle(config: RequestConfig): boolean {
+    // Fetch handles non-proxy requests and HTTP/HTTPS proxies
+    return !config.proxy || config.proxy.protocol === 'http' || config.proxy.protocol === 'https';
+  }
+
+  async request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
+    const { url, method = 'GET', headers, data, proxy } = config;
+
+    // Prepare fetch options
+    const fetchOptions: RequestInit = {
+      method,
+      headers,
+      signal,
+    };
+
+    // Add body for non-GET requests
+    if (data && method !== 'GET') {
+      fetchOptions.body = typeof data === 'string' ? data : JSON.stringify(data);
+      if (typeof data === 'object') {
+        fetchOptions.headers = { 'Content-Type': 'application/json', ...fetchOptions.headers };
+      }
+    }
+
+    // Add proxy if needed (using Bun's built-in proxy support)
+    if (proxy) {
+      (fetchOptions as any).proxy = ProxyManager.createProxyUrl(proxy);
+    }    const response = await fetch(url, fetchOptions);
+
+    // Parse response based on content type
+    let responseData: T;
+    const contentType = response.headers.get('content-type') || '';
+    
+    if (contentType.includes('application/json')) {
+      responseData = await response.json() as T;
+    } else {
+      responseData = await response.text() as T;
+    }
+
+    const httpResponse: HttpResponse<T> = {
+      data: responseData,
+      status: response.status,
+      headers: Object.fromEntries(response.headers.entries()),
+      ok: response.ok,
+    };
+
+    // Throw HttpError for non-2xx status codes
+    if (!response.ok) {
+      throw new HttpError(
+        `Request failed with status ${response.status}`,
+        response.status,
+        httpResponse
+      );
+    }
+
+    return httpResponse;
+  }
+}

libs/http/src/adapters/index.ts

@@ -0,0 +1,4 @@
+export * from './types.js';
+export * from './fetch-adapter.js';
+export * from './axios-adapter.js';
+export * from './factory.js';

libs/http/src/adapters/types.ts

@@ -0,0 +1,16 @@
+import type { RequestConfig, HttpResponse } from '../types.js';
+
+/**
+ * Request adapter interface for different HTTP implementations
+ */
+export interface RequestAdapter {
+  /**
+   * Execute an HTTP request
+   */
+  request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>>;
+
+  /**
+   * Check if this adapter can handle the given configuration
+   */
+  canHandle(config: RequestConfig): boolean;
+}

libs/http/src/client.ts

@@ -6,8 +6,7 @@ import type {
 } from './types.js';
 import { HttpError } from './types.js';
 import { ProxyManager } from './proxy-manager.js';
-import axios, { type AxiosResponse, AxiosError } from 'axios';
-import { loggingConfig } from '@stock-bot/config';
+import { AdapterFactory } from './adapters/index.js';
 
 export class HttpClient {
   private readonly config: HttpClientConfig;
@@ -15,10 +14,7 @@ export class HttpClient {
 
   constructor(config: HttpClientConfig = {}, logger?: Logger) {
     this.config = config;
-    this.logger = logger?.child({ //TODO fix pino levels
-        component: 'http',
-        // level: loggingConfig?.LOG_LEVEL || 'info',
-      });
+    this.logger = logger?.child({ component: 'http' });
   }
 
   // Convenience methods
@@ -26,262 +22,99 @@ export class HttpClient {
     return this.request<T>({ ...config, method: 'GET', url });
   }
 
-  async post<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
-    return this.request<T>({ ...config, method: 'POST', url, body });
+  async post<T = any>(url: string, data?: any, config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}): Promise<HttpResponse<T>> {
+    return this.request<T>({ ...config, method: 'POST', url, data });
   }
 
-  async put<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
-    return this.request<T>({ ...config, method: 'PUT', url, body });
+  async put<T = any>(url: string, data?: any, config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}): Promise<HttpResponse<T>> {
+    return this.request<T>({ ...config, method: 'PUT', url, data });
   }
 
   async del<T = any>(url: string, config: Omit<RequestConfig, 'method' | 'url'> = {}): Promise<HttpResponse<T>> {
     return this.request<T>({ ...config, method: 'DELETE', url });
   }
 
-  async patch<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
-    return this.request<T>({ ...config, method: 'PATCH', url, body });
-  }  /**
-   * Main request method - unified and simplified
+  async patch<T = any>(url: string, data?: any, config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}): Promise<HttpResponse<T>> {
+    return this.request<T>({ ...config, method: 'PATCH', url, data });
+  }
+
+  /**
+   * Main request method - clean and simple
    */
   async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
     const finalConfig = this.mergeConfig(config);
     const startTime = Date.now();
+    
     this.logger?.debug('Making HTTP request', { 
       method: finalConfig.method,
       url: finalConfig.url,
       hasProxy: !!finalConfig.proxy
     });
 
-    try {      // Single decision point for proxy type - only request-level proxy
-      const proxy = finalConfig.proxy;
-      const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy);
-      
-      const response = await this.makeRequest<T>(finalConfig, useBunFetch);
-      const responseTime = Date.now() - startTime;
-      response.responseTime = responseTime;
+    try {
+      const response = await this.executeRequest<T>(finalConfig);
+      response.responseTime = Date.now() - startTime;
 
       this.logger?.debug('HTTP request successful', {
         method: finalConfig.method,
         url: finalConfig.url,
         status: response.status,
-        responseTime: responseTime,
+        responseTime: response.responseTime,
       });
 
       return response;
     } catch (error) {
-      // this.logger?.warn('HTTP request failed', {
-      //   method: finalConfig.method,
-      //   url: finalConfig.url,
-      //   error: (error as Error).message,
-      // });
+      this.logger?.warn('HTTP request failed', {
+        method: finalConfig.method,
+        url: finalConfig.url,
+        error: (error as Error).message,
+      });
       throw error;
     }
   }
 
   /**
-   * Unified request method with consolidated timeout handling
+   * Execute request with timeout handling - no race conditions
    */
-  private async makeRequest<T>(config: RequestConfig, useBunFetch: boolean): Promise<HttpResponse<T>> {
+  private async executeRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
     const timeout = config.timeout ?? this.config.timeout ?? 30000;
     const controller = new AbortController();
     
-    // Create timeout promise that rejects with proper error
-    let timeoutId: NodeJS.Timeout | undefined;
-    const timeoutPromise = new Promise<never>((_, reject) => {
-      timeoutId = setTimeout(() => {
+    // Set up timeout
+    const timeoutId = setTimeout(() => {
       controller.abort();
-        reject(new HttpError(`Request timeout after ${timeout}ms`));
     }, timeout);
-    });    // Create request promise (don't await here!)
-    const requestPromise = useBunFetch 
-      ? this.fetchRequest<T>(config, controller.signal)
-      : this.axiosRequest<T>(config, controller.signal);
 
     try {
-      // Race the promises
-      const result = await Promise.race([requestPromise, timeoutPromise]);
-      if (timeoutId) clearTimeout(timeoutId);
-      return result;
+      // Get the appropriate adapter
+      const adapter = AdapterFactory.getAdapter(config);
+      
+      // Execute request
+      const response = await adapter.request<T>(config, controller.signal);
+      
+      // Clear timeout on success
+      clearTimeout(timeoutId);
+      
+      return response;
     } catch (error) {
-      // If it's our timeout error, handle it
-      if (error instanceof HttpError && error.message.includes('timeout')) {
-        this.logger?.warn('Request timed out', {
-          method: config.method,
-          url: config.url,
-          timeout: timeout,
-        });
-        throw error; // Re-throw the timeout error
+      clearTimeout(timeoutId);
+      
+      // Handle timeout
+      if (controller.signal.aborted) {
+        throw new HttpError(`Request timeout after ${timeout}ms`);
       }
       
-      // Handle other errors (network, parsing, etc.)
+      // Re-throw other errors
+      if (error instanceof HttpError) {
         throw error;
       }
       
-  }
-  /**
-   * Bun fetch implementation (simplified)
-   */
-  private async fetchRequest<T>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
-    try {
-          const options = this.buildFetchOptions(config, signal);
-
-      this.logger?.debug('Making request with fetch: ', { url: config.url, options })
-      // console.log(options)
-      const response = await fetch(config.url, options);
-      // console.log('Fetch response:', response.status);
-      return this.parseFetchResponse<T>(response);
-    } catch (error) {
-      throw signal.aborted 
-        ? new HttpError(`Request timeout`)
-        : new HttpError(`Request failed: ${(error as Error).message}`);
-    }
-  }  /**
-   * Axios implementation (for SOCKS proxies)
-   */
-  private async axiosRequest<T>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
-    if(config.proxy) {
-      try {
-        const axiosProxy = await ProxyManager.createAxiosConfig(config.proxy);
-        axiosProxy.url = config.url;
-        axiosProxy.method = config.method || 'GET';
-        // console.log(axiosProxy)
-        // const axiosConfig = {
-        //   ...axiosProxy,
-        //   url: config.url,
-        //   method: config.method || 'GET',
-        //   // headers: config.headers || {},
-        //   // data: config.body,
-        //   // signal, // Axios supports AbortSignal
-        // };
-
-        // console.log('Making request with Axios: ', axiosConfig );
-
-        const response: AxiosResponse<T> = await axios.request(axiosProxy);
-        return this.parseAxiosResponse<T>(response);
-      } catch (error) {
-        console.error('Axios request error:', error);
-        
-        // Handle AbortSignal timeout
-        if (signal.aborted) {
-          throw new HttpError(`Request timeout`);
-        }
-        
-        // Handle Axios timeout errors
-        if (error instanceof AxiosError && error.code === 'ECONNABORTED') {
-          throw new HttpError(`Request timeout`);
-        }
-        
       throw new HttpError(`Request failed: ${(error as Error).message}`);
     }
-    } else {
-      throw new HttpError(`Request failed: No proxy configured, use fetch instead`);
-    }
-  }
-  /**
-   * Build fetch options (extracted for clarity)
-   */
-  private buildFetchOptions(config: RequestConfig, signal: AbortSignal): RequestInit {
-    const options: RequestInit = {
-      method: config.method || 'GET',
-      headers: config.headers || {},
-      signal,
-    };
-
-
-    // Add body
-    if (config.body && config.method !== 'GET') {
-      if (typeof config.body === 'object') {
-        options.body = JSON.stringify(config.body);
-        options.headers = { 'Content-Type': 'application/json', ...options.headers };
-      } else {
-        options.body = config.body;
-      }
-    }
-
-    // Add proxy (HTTP/HTTPS only) - request level only
-    if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) {
-      (options as any).proxy = ProxyManager.createProxyUrl(config.proxy);
-    }
-    return options;
-  }  /**
-   * Build Axios options (for reference, though we're creating instance in ProxyManager)
-   */
-  private buildAxiosOptions(config: RequestConfig, signal: AbortSignal): any {
-    const options: any = {
-      method: config.method || 'GET',
-      headers: config.headers || {},
-      signal, // Axios supports AbortSignal
-      timeout: config.timeout || 30000,
-      maxRedirects: 5,
-      validateStatus: () => true // Don't throw on HTTP errors
-    };
-
-    // Add body
-    if (config.body && config.method !== 'GET') {
-      if (typeof config.body === 'object') {
-        options.data = config.body;
-        options.headers = { 'Content-Type': 'application/json', ...options.headers };
-      } else {
-        options.data = config.body;
-        options.headers = { 'Content-Type': 'text/plain', ...options.headers };
-      }
-    }
-
-    return options;
   }
 
   /**
-   * Parse fetch response (simplified)
-   */
-  private async parseFetchResponse<T>(response: Response): Promise<HttpResponse<T>> {
-    const data = await this.parseResponseBody<T>(response);
-    const headers = Object.fromEntries(response.headers.entries());
-
-    if (!response.ok) {
-      throw new HttpError(
-        `Request failed with status ${response.status}`,
-        response.status,
-        { data, status: response.status, headers, ok: response.ok }
-      );
-    }
-
-    return { data, status: response.status, headers, ok: response.ok };
-  }
-  /**
-   * Parse Axios response
-   */
-  private parseAxiosResponse<T>(response: AxiosResponse<T>): HttpResponse<T> {
-    const headers = response.headers as Record<string, string>;
-    const status = response.status;
-    const ok = status >= 200 && status < 300;
-    const data = response.data;
-
-    if (!ok) {
-      throw new HttpError(
-        `Request failed with status ${status}`,
-        status,
-        { data, status, headers, ok }
-      );
-    }
-
-    return { data, status, headers, ok };
-  }
-  /**
-   * Unified body parsing (works for fetch response)
-   */
-  private async parseResponseBody<T>(response: Response): Promise<T> {
-    const contentType = response.headers.get('content-type') || '';
-    
-    if (contentType.includes('application/json')) {
-      return response.json();
-    } else if (contentType.includes('text/')) {
-      return response.text() as any;
-    } else {
-      return response.arrayBuffer() as any;
-    }
-  }
-  /**
-   * Merge configs - request-level proxy only
+   * Merge configs with defaults
    */
   private mergeConfig(config: RequestConfig): RequestConfig {
     return {

libs/http/src/index.ts

@@ -2,6 +2,7 @@
 export * from './types.js';
 export * from './client.js';
 export * from './proxy-manager.js';
+export * from './adapters/index.js';
 
 // Default export
 export { HttpClient as default } from './client.js';

libs/http/src/types.ts

@@ -7,6 +7,7 @@ export interface ProxyInfo {
   port: number;
   username?: string;
   password?: string;
+  url?: string; // Full proxy URL for adapters
   isWorking?: boolean;
   responseTime?: number;
   error?: string;
@@ -22,7 +23,7 @@ export interface RequestConfig {
   method?: HttpMethod;
   url: string;
   headers?: Record<string, string>;
-  body?: any;
+  data?: any; // Changed from 'body' to 'data' for consistency
   timeout?: number;
   proxy?: ProxyInfo;
 }

libs/logger/src/logger.ts

@@ -21,12 +21,11 @@ const loggerCache = new Map<string, pino.Logger>();
 function createTransports(serviceName: string): any {
   const targets: any[] = [];
   // const isDev = loggingConfig.LOG_ENVIRONMENT === 'development';
-
   // Console transport
   if (loggingConfig.LOG_CONSOLE) {
     targets.push({
       target: 'pino-pretty',
-      level: loggingConfig.LOG_LEVEL,
+      level: 'error', // Only show errors on console
       options: {
         colorize: true,
         translateTime: 'yyyy-mm-dd HH:MM:ss.l',

package.json

@@ -39,7 +39,9 @@
     "infra:down": "docker-compose down",
     "infra:reset": "docker-compose down -v && docker-compose up -d dragonfly postgres questdb mongodb",
     "dev:full": "npm run infra:up && npm run docker:admin && turbo run dev",
-    "dev:clean": "npm run infra:reset && npm run dev:full"
+    "dev:clean": "npm run infra:reset && npm run dev:full",
+    "proxy": "bun run ./apps/data-service/src/proxy-demo.ts"
+
   },
   "workspaces": [
     "libs/*",