From a07a71d92a80686a78286f87a6c2d57d88f30702 Mon Sep 17 00:00:00 2001 From: Boki Date: Sun, 22 Jun 2025 09:03:34 -0400 Subject: [PATCH] removed http client for a simple fetch wrapper with logging in utils --- .../proxy/operations/check.operations.ts | 36 +-- .../proxy/operations/fetch.operations.ts | 23 +- .../proxy/operations/query.operations.ts | 2 +- .../proxy/operations/queue.operations.ts | 2 +- .../src/handlers/proxy/proxy.handler.ts | 2 +- .../src/handlers/qm/shared/session-manager.ts | 2 +- .../webshare/operations/fetch.operations.ts | 2 +- libs/core/di/src/adapters/service-adapter.ts | 4 - libs/core/di/src/awilix-container.ts | 7 - libs/core/handlers/src/base/BaseHandler.ts | 2 - .../handlers/src/types/service-container.ts | 1 - libs/services/browser/tsconfig.json | 3 +- libs/services/http/README.md | 283 ------------------ libs/services/http/bunfig.toml | 0 libs/services/http/package.json | 46 --- .../http/src/adapters/axios-adapter.ts | 58 ---- libs/services/http/src/adapters/factory.ts | 28 -- .../http/src/adapters/fetch-adapter.ts | 74 ----- libs/services/http/src/adapters/index.ts | 4 - libs/services/http/src/adapters/types.ts | 16 - libs/services/http/src/client.ts | 183 ----------- libs/services/http/src/index.ts | 9 - libs/services/http/src/proxy-manager.ts | 65 ---- libs/services/http/src/types.ts | 55 ---- libs/services/http/src/user-agent.ts | 6 - .../http/test/http-integration.test.ts | 161 ---------- libs/services/http/test/http.test.ts | 155 ---------- libs/services/http/test/mock-server.test.ts | 132 -------- libs/services/http/test/mock-server.ts | 116 ------- libs/services/http/tsconfig.json | 13 - libs/services/proxy/src/types.ts | 12 +- libs/utils/src/fetch.ts | 27 ++ libs/utils/src/index.ts | 2 + libs/utils/src/user-agent.ts | 30 ++ libs/utils/tsconfig.json | 3 +- scripts/build-libs.sh | 1 - 36 files changed, 100 insertions(+), 1465 deletions(-) delete mode 100644 libs/services/http/README.md delete mode 100644 libs/services/http/bunfig.toml delete mode 100644 libs/services/http/package.json delete mode 100644 libs/services/http/src/adapters/axios-adapter.ts delete mode 100644 libs/services/http/src/adapters/factory.ts delete mode 100644 libs/services/http/src/adapters/fetch-adapter.ts delete mode 100644 libs/services/http/src/adapters/index.ts delete mode 100644 libs/services/http/src/adapters/types.ts delete mode 100644 libs/services/http/src/client.ts delete mode 100644 libs/services/http/src/index.ts delete mode 100644 libs/services/http/src/proxy-manager.ts delete mode 100644 libs/services/http/src/types.ts delete mode 100644 libs/services/http/src/user-agent.ts delete mode 100644 libs/services/http/test/http-integration.test.ts delete mode 100644 libs/services/http/test/http.test.ts delete mode 100644 libs/services/http/test/mock-server.test.ts delete mode 100644 libs/services/http/test/mock-server.ts delete mode 100644 libs/services/http/tsconfig.json create mode 100644 libs/utils/src/fetch.ts create mode 100644 libs/utils/src/user-agent.ts diff --git a/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts b/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts index c73200f..390d823 100644 --- a/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts +++ b/apps/data-ingestion/src/handlers/proxy/operations/check.operations.ts @@ -1,22 +1,13 @@ /** * Proxy Check Operations - Checking proxy functionality */ -import { HttpClient, ProxyInfo } from '@stock-bot/http'; +import type { ProxyInfo } from '@stock-bot/proxy'; import { OperationContext } from '@stock-bot/di'; import { getLogger } from '@stock-bot/logger'; +import { fetch } from '@stock-bot/utils'; import { PROXY_CONFIG } from '../shared/config'; -// Shared HTTP client -let httpClient: HttpClient; - -function getHttpClient(ctx: OperationContext): HttpClient { - if (!httpClient) { - httpClient = new HttpClient({ timeout: 10000 }, ctx.logger); - } - return httpClient; -} - /** * Check if a proxy is working */ @@ -36,22 +27,27 @@ export async function checkProxy(proxy: ProxyInfo): Promise { }); try { - // Test the proxy - const client = getHttpClient(ctx); - const response = await client.get(PROXY_CONFIG.CHECK_URL, { - proxy, - timeout: PROXY_CONFIG.CHECK_TIMEOUT, - }); + // Test the proxy using fetch with proxy support + const proxyUrl = proxy.username && proxy.password + ? `${proxy.protocol}://${encodeURIComponent(proxy.username)}:${encodeURIComponent(proxy.password)}@${proxy.host}:${proxy.port}` + : `${proxy.protocol}://${proxy.host}:${proxy.port}`; + + const response = await fetch(PROXY_CONFIG.CHECK_URL, { + proxy: proxyUrl, + signal: AbortSignal.timeout(PROXY_CONFIG.CHECK_TIMEOUT), + logger: ctx.logger + } as any); + + const data = await response.text(); - const isWorking = response.status >= 200 && response.status < 300; + const isWorking = response.ok; const result: ProxyInfo = { ...proxy, isWorking, lastChecked: new Date(), - responseTime: response.responseTime, }; - if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) { + if (isWorking && !data.includes(PROXY_CONFIG.CHECK_IP)) { success = true; await updateProxyInCache(result, true, ctx); } else { diff --git a/apps/data-ingestion/src/handlers/proxy/operations/fetch.operations.ts b/apps/data-ingestion/src/handlers/proxy/operations/fetch.operations.ts index 2305bee..f92bf5d 100644 --- a/apps/data-ingestion/src/handlers/proxy/operations/fetch.operations.ts +++ b/apps/data-ingestion/src/handlers/proxy/operations/fetch.operations.ts @@ -1,18 +1,13 @@ /** * Proxy Fetch Operations - Fetching proxies from sources */ -import { HttpClient, ProxyInfo } from '@stock-bot/http'; +import type { ProxyInfo } from '@stock-bot/proxy'; import { OperationContext } from '@stock-bot/di'; import { getLogger } from '@stock-bot/logger'; +import { fetch } from '@stock-bot/utils'; import { PROXY_CONFIG } from '../shared/config'; import type { ProxySource } from '../shared/types'; - -// Shared HTTP client -let httpClient: HttpClient; - -function getHttpClient(ctx: OperationContext): HttpClient { - if (!httpClient) { httpClient = new HttpClient({ timeout: 10000 }, ctx.logger); } return httpClient; @@ -44,17 +39,17 @@ export async function fetchProxiesFromSource(source: ProxySource, ctx?: Operatio try { ctx.logger.info(`Fetching proxies from ${source.url}`); - const client = getHttpClient(ctx); - const response = await client.get(source.url, { - timeout: 10000, - }); + const response = await fetch(source.url, { + signal: AbortSignal.timeout(10000), + logger: ctx.logger + } as any); - if (response.status !== 200) { + if (!response.ok) { ctx.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`); return []; } - const text = response.data; + const text = await response.text(); const lines = text.split('\n').filter((line: string) => line.trim()); for (const line of lines) { @@ -69,7 +64,7 @@ export async function fetchProxiesFromSource(source: ProxySource, ctx?: Operatio if (parts.length >= 2) { const proxy: ProxyInfo = { source: source.id, - protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5', + protocol: source.protocol as 'http' | 'https', host: parts[0], port: parseInt(parts[1]), }; diff --git a/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts b/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts index 5a15dcb..221e1ba 100644 --- a/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts +++ b/apps/data-ingestion/src/handlers/proxy/operations/query.operations.ts @@ -1,7 +1,7 @@ /** * Proxy Query Operations - Getting active proxies from cache */ -import { ProxyInfo } from '@stock-bot/http'; +import type { ProxyInfo } from '@stock-bot/proxy'; import { OperationContext } from '@stock-bot/di'; import { PROXY_CONFIG } from '../shared/config'; diff --git a/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts b/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts index 1db582f..b46030f 100644 --- a/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts +++ b/apps/data-ingestion/src/handlers/proxy/operations/queue.operations.ts @@ -1,7 +1,7 @@ /** * Proxy Queue Operations - Queueing proxy operations */ -import { ProxyInfo } from '@stock-bot/http'; +import type { ProxyInfo } from '@stock-bot/proxy'; import { QueueManager } from '@stock-bot/queue'; import { OperationContext } from '@stock-bot/di'; diff --git a/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts b/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts index 5ab4902..e2e57fa 100644 --- a/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts +++ b/apps/data-ingestion/src/handlers/proxy/proxy.handler.ts @@ -1,7 +1,7 @@ /** * Proxy Provider for new queue system */ -import { ProxyInfo } from '@stock-bot/http'; +import type { ProxyInfo } from '@stock-bot/proxy'; import { getLogger } from '@stock-bot/logger'; import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue'; import type { ServiceContainer } from '@stock-bot/di'; diff --git a/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts b/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts index 8bbfbce..723dcec 100644 --- a/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts +++ b/apps/data-ingestion/src/handlers/qm/shared/session-manager.ts @@ -2,7 +2,7 @@ * QM Session Manager - Centralized session state management */ -import { getRandomUserAgent } from '@stock-bot/services/http'; +import { getRandomUserAgent } from '@stock-bot/utils'; import { QM_SESSION_IDS, SESSION_CONFIG } from './config'; import type { QMSession } from './types'; diff --git a/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts b/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts index 662e5bf..ee40fec 100644 --- a/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts +++ b/apps/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts @@ -1,7 +1,7 @@ /** * WebShare Fetch Operations - API integration */ -import { type ProxyInfo } from '@stock-bot/http'; +import type { ProxyInfo } from '@stock-bot/proxy'; import { OperationContext } from '@stock-bot/di'; import { WEBSHARE_CONFIG } from '../shared/config'; diff --git a/libs/core/di/src/adapters/service-adapter.ts b/libs/core/di/src/adapters/service-adapter.ts index 69695af..74df444 100644 --- a/libs/core/di/src/adapters/service-adapter.ts +++ b/libs/core/di/src/adapters/service-adapter.ts @@ -18,10 +18,6 @@ export class DataIngestionServiceAdapter implements IServiceContainer { get logger() { return this.dataServices.logger; } get cache() { return this.dataServices.cache; } get queue() { return this.dataServices.queue; } - get http() { - // HTTP client not in current data services - will be added when needed - return null; - } get proxy(): any { // Proxy manager should be injected via Awilix container // This adapter is for legacy compatibility diff --git a/libs/core/di/src/awilix-container.ts b/libs/core/di/src/awilix-container.ts index 6ced388..1d59b16 100644 --- a/libs/core/di/src/awilix-container.ts +++ b/libs/core/di/src/awilix-container.ts @@ -89,11 +89,6 @@ export function createServiceContainer(config: AppConfig): AwilixContainer { return manager; }).singleton(), - // HTTP client can be added here when decoupled - httpClient: asFunction(() => { - // TODO: Import and create HTTP client when decoupled - return null; - }).singleton(), // MongoDB client with injected logger mongoClient: asFunction(({ mongoConfig, logger }) => { @@ -152,7 +147,6 @@ export function createServiceContainer(config: AppConfig): AwilixContainer { logger: cradle.logger, cache: cradle.cache, proxy: cradle.proxyManager, - http: cradle.httpClient, browser: cradle.browser, mongodb: cradle.mongoClient, postgres: cradle.postgresClient, @@ -224,7 +218,6 @@ export interface ServiceCradle { logger: any; cache: CacheProvider; proxyManager: ProxyManager; - httpClient: any; browser: any; mongoClient: any; postgresClient: any; diff --git a/libs/core/handlers/src/base/BaseHandler.ts b/libs/core/handlers/src/base/BaseHandler.ts index 281cc04..d8a8494 100644 --- a/libs/core/handlers/src/base/BaseHandler.ts +++ b/libs/core/handlers/src/base/BaseHandler.ts @@ -12,7 +12,6 @@ export abstract class BaseHandler implements IHandler { readonly logger; readonly cache; readonly queue; - readonly http; readonly proxy; readonly browser; readonly mongodb; @@ -26,7 +25,6 @@ export abstract class BaseHandler implements IHandler { this.logger = getLogger(this.constructor.name); this.cache = services.cache; this.queue = services.queue; - this.http = services.http; this.proxy = services.proxy; this.browser = services.browser; this.mongodb = services.mongodb; diff --git a/libs/core/handlers/src/types/service-container.ts b/libs/core/handlers/src/types/service-container.ts index e74dab6..333e781 100644 --- a/libs/core/handlers/src/types/service-container.ts +++ b/libs/core/handlers/src/types/service-container.ts @@ -14,7 +14,6 @@ export interface IServiceContainer { readonly logger: any; // Logger instance readonly cache: any; // Cache provider (Redis/Dragonfly) readonly queue: any; // Queue manager (BullMQ) - readonly http: any; // HTTP client with proxy support readonly proxy: ProxyManager; // Proxy manager service readonly browser?: any; // Browser automation (Playwright) diff --git a/libs/services/browser/tsconfig.json b/libs/services/browser/tsconfig.json index 06ee7f5..88fe818 100644 --- a/libs/services/browser/tsconfig.json +++ b/libs/services/browser/tsconfig.json @@ -7,7 +7,6 @@ }, "include": ["src/**/*"], "references": [ - { "path": "../../core/logger" }, - { "path": "../http" } + { "path": "../../core/logger" } ] } diff --git a/libs/services/http/README.md b/libs/services/http/README.md deleted file mode 100644 index 678bf15..0000000 --- a/libs/services/http/README.md +++ /dev/null @@ -1,283 +0,0 @@ -# HTTP Client Library - -A comprehensive HTTP client library for the Stock Bot platform with built-in support for: - -- ✅ **Fetch API** - Modern, promise-based HTTP requests -- ✅ **Proxy Support** - HTTP, HTTPS, SOCKS4, and SOCKS5 proxies -- ✅ **Rate Limiting** - Configurable request rate limiting -- ✅ **Timeout Handling** - Request timeouts with abort controllers -- ✅ **Retry Logic** - Automatic retries with exponential backoff -- ✅ **TypeScript** - Full TypeScript support with type safety -- ✅ **Logging Integration** - Optional logger integration - -## Installation - -```bash -bun add @stock-bot/http -``` - -## Basic Usage - -```typescript -import { HttpClient } from '@stock-bot/http'; - -// Create a client with default configuration -const client = new HttpClient(); - -// Make a GET request -const response = await client.get('https://api.example.com/data'); -console.log(response.data); - -// Make a POST request -const postResponse = await client.post('https://api.example.com/users', { - name: 'John Doe', - email: 'john@example.com' -}); -``` - -## Advanced Configuration - -```typescript -import { HttpClient } from '@stock-bot/http'; -import { logger } from '@stock-bot/logger'; - -const client = new HttpClient({ - baseURL: 'https://api.example.com', - timeout: 10000, // 10 seconds - retries: 3, - retryDelay: 1000, // 1 second base delay - defaultHeaders: { - 'Authorization': 'Bearer token', - 'User-Agent': 'Stock-Bot/1.0' - }, - validateStatus: (status) => status < 400 -}, logger); -``` - -## Proxy Support - -### HTTP/HTTPS Proxy - -```typescript -const client = new HttpClient({ - proxy: { - type: 'http', - host: 'proxy.example.com', - port: 8080, - username: 'user', // optional - password: 'pass' // optional - } -}); -``` - -### SOCKS Proxy - -```typescript -const client = new HttpClient({ - proxy: { - type: 'socks5', - host: 'socks-proxy.example.com', - port: 1080, - username: 'user', // optional - password: 'pass' // optional - } -}); -``` - -## Rate Limiting - -```typescript -const client = new HttpClient({ - rateLimit: { - maxRequests: 100, // Max 100 requests - windowMs: 60 * 1000, // Per 1 minute - skipSuccessfulRequests: false, - skipFailedRequests: true // Don't count failed requests - } -}); - -// Check rate limit status -const status = client.getRateLimitStatus(); -console.log(`${status.currentCount}/${status.maxRequests} requests used`); -``` - -## Request Methods - -```typescript -// GET request -const getData = await client.get('/api/data'); - -// POST request with body -const postData = await client.post('/api/users', { - name: 'John', - email: 'john@example.com' -}); - -// PUT request -const putData = await client.put('/api/users/1', updatedUser); - -// DELETE request -const deleteData = await client.delete('/api/users/1'); - -// PATCH request -const patchData = await client.patch('/api/users/1', { name: 'Jane' }); - -// Custom request -const customResponse = await client.request({ - method: 'POST', - url: '/api/custom', - headers: { 'X-Custom': 'value' }, - body: { data: 'custom' }, - timeout: 5000 -}); -``` - -## Error Handling - -```typescript -import { HttpError, TimeoutError, RateLimitError } from '@stock-bot/http'; - -try { - const response = await client.get('/api/data'); -} catch (error) { - if (error instanceof TimeoutError) { - console.log('Request timed out'); - } else if (error instanceof RateLimitError) { - console.log(`Rate limited: retry after ${error.retryAfter}ms`); - } else if (error instanceof HttpError) { - console.log(`HTTP error ${error.status}: ${error.message}`); - } -} -``` - -## Retry Configuration - -```typescript -const client = new HttpClient({ - retries: 3, // Retry up to 3 times - retryDelay: 1000, // Base delay of 1 second - // Exponential backoff: 1s, 2s, 4s -}); - -// Or per-request retry configuration -const response = await client.get('/api/data', { - retries: 5, - retryDelay: 500 -}); -``` - -## Timeout Handling - -```typescript -// Global timeout -const client = new HttpClient({ - timeout: 30000 // 30 seconds -}); - -// Per-request timeout -const response = await client.get('/api/data', { - timeout: 5000 // 5 seconds for this request -}); -``` - -## Custom Status Validation - -```typescript -const client = new HttpClient({ - validateStatus: (status) => { - // Accept 2xx and 3xx status codes - return status >= 200 && status < 400; - } -}); - -// Or per-request validation -const response = await client.get('/api/data', { - validateStatus: (status) => status === 200 || status === 404 -}); -``` - -## TypeScript Support - -The library is fully typed with TypeScript: - -```typescript -interface User { - id: number; - name: string; - email: string; -} - -// Response data is properly typed -const response = await client.get('/api/users'); -const users: User[] = response.data; - -// Request configuration is validated -const config: RequestConfig = { - method: 'POST', - url: '/api/users', - body: { name: 'John' }, - timeout: 5000 -}; -``` - -## Integration with Logger - -```typescript -import { logger } from '@stock-bot/logger'; -import { HttpClient } from '@stock-bot/http'; - -const client = new HttpClient({ - baseURL: 'https://api.example.com' -}, logger); - -// All requests will be logged with debug/warn/error levels -``` - -## Testing - -```bash -# Run tests -bun test - -# Run with coverage -bun test --coverage - -# Watch mode -bun test --watch -``` - -## Features - -### Proxy Support -- HTTP and HTTPS proxies -- SOCKS4 and SOCKS5 proxies -- Authentication support -- Automatic agent creation - -### Rate Limiting -- Token bucket algorithm -- Configurable window and request limits -- Skip successful/failed requests options -- Real-time status monitoring - -### Retry Logic -- Exponential backoff -- Configurable retry attempts -- Smart retry conditions (5xx errors only) -- Per-request retry override - -### Error Handling -- Typed error classes -- Detailed error information -- Request/response context -- Timeout detection - -### Performance -- Built on modern Fetch API -- Minimal dependencies -- Tree-shakeable exports -- TypeScript optimization - -## License - -MIT License - see LICENSE file for details. diff --git a/libs/services/http/bunfig.toml b/libs/services/http/bunfig.toml deleted file mode 100644 index e69de29..0000000 diff --git a/libs/services/http/package.json b/libs/services/http/package.json deleted file mode 100644 index 08dfbd3..0000000 --- a/libs/services/http/package.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "name": "@stock-bot/http", - "version": "1.0.0", - "description": "HTTP client library with proxy support, rate limiting, and timeout for Stock Bot platform", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "type": "module", - "scripts": { - "build": "tsc", - "test": "bun test", - "test:watch": "bun test --watch", - "test:coverage": "bun test --coverage", - "lint": "eslint src/**/*.ts", - "type-check": "tsc --noEmit", - "clean": "rimraf dist" - }, - "dependencies": { - "@stock-bot/logger": "*", - "@stock-bot/types": "*", - "axios": "^1.9.0", - "http-proxy-agent": "^7.0.2", - "https-proxy-agent": "^7.0.6", - "socks-proxy-agent": "^8.0.5", - "user-agents": "^1.1.567" - }, - "devDependencies": { - "@types/node": "^20.11.0", - "@types/user-agents": "^1.0.4", - "@typescript-eslint/eslint-plugin": "^6.19.0", - "@typescript-eslint/parser": "^6.19.0", - "bun-types": "^1.2.15", - "eslint": "^8.56.0", - "typescript": "^5.3.0" - }, - "exports": { - ".": { - "import": "./dist/index.js", - "require": "./dist/index.js", - "types": "./dist/index.d.ts" - } - }, - "files": [ - "dist", - "README.md" - ] -} diff --git a/libs/services/http/src/adapters/axios-adapter.ts b/libs/services/http/src/adapters/axios-adapter.ts deleted file mode 100644 index 42eb73a..0000000 --- a/libs/services/http/src/adapters/axios-adapter.ts +++ /dev/null @@ -1,58 +0,0 @@ -import axios, { type AxiosRequestConfig, type AxiosResponse } from 'axios'; -import { ProxyManager } from '../proxy-manager'; -import type { HttpResponse, RequestConfig } from '../types'; -import { HttpError } from '../types'; -import type { RequestAdapter } from './types'; - -/** - * Axios adapter for SOCKS proxies - */ -export class AxiosAdapter implements RequestAdapter { - canHandle(config: RequestConfig): boolean { - // Axios handles SOCKS proxies - return Boolean( - config.proxy && - typeof config.proxy !== 'string' && - (config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5') - ); - } - - async request(config: RequestConfig, signal: AbortSignal): Promise> { - const { url, method = 'GET', headers, data, proxy } = config; - - if (!proxy || typeof proxy === 'string') { - throw new Error('Axios adapter requires ProxyInfo configuration'); - } - - // Create proxy configuration using ProxyManager - const axiosConfig: AxiosRequestConfig = { - ...ProxyManager.createAxiosConfig(proxy), - url, - method, - headers, - data, - signal, - // Don't throw on non-2xx status codes - let caller handle - validateStatus: () => true, - }; - const response: AxiosResponse = await axios(axiosConfig); - - const httpResponse: HttpResponse = { - data: response.data, - status: response.status, - headers: response.headers as Record, - ok: response.status >= 200 && response.status < 300, - }; - - // Throw HttpError for non-2xx status codes - if (!httpResponse.ok) { - throw new HttpError( - `Request failed with status ${response.status}`, - response.status, - httpResponse - ); - } - - return httpResponse; - } -} diff --git a/libs/services/http/src/adapters/factory.ts b/libs/services/http/src/adapters/factory.ts deleted file mode 100644 index c185e5c..0000000 --- a/libs/services/http/src/adapters/factory.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { RequestConfig } from '../types'; -import { AxiosAdapter } from './axios-adapter'; -import { FetchAdapter } from './fetch-adapter'; -import type { RequestAdapter } from './types'; - -/** - * Factory for creating the appropriate request adapter - */ -export class AdapterFactory { - private static adapters: RequestAdapter[] = [ - new AxiosAdapter(), // Check SOCKS first - new FetchAdapter(), // Fallback to fetch for everything else - ]; - - /** - * Get the appropriate adapter for the given configuration - */ - static getAdapter(config: RequestConfig): RequestAdapter { - for (const adapter of this.adapters) { - if (adapter.canHandle(config)) { - return adapter; - } - } - - // Fallback to fetch adapter - return new FetchAdapter(); - } -} diff --git a/libs/services/http/src/adapters/fetch-adapter.ts b/libs/services/http/src/adapters/fetch-adapter.ts deleted file mode 100644 index 2a172c9..0000000 --- a/libs/services/http/src/adapters/fetch-adapter.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { ProxyManager } from '../proxy-manager'; -import type { HttpResponse, RequestConfig } from '../types'; -import { HttpError } from '../types'; -import type { RequestAdapter } from './types'; - -/** - * Fetch adapter for HTTP/HTTPS proxies and non-proxy requests - */ -export class FetchAdapter implements RequestAdapter { - canHandle(config: RequestConfig): boolean { - // Fetch handles non-proxy requests and HTTP/HTTPS proxies - if (typeof config.proxy === 'string') { - return config.proxy.startsWith('http'); - } - return !config.proxy || config.proxy.protocol === 'http' || config.proxy.protocol === 'https'; - } - - async request(config: RequestConfig, signal: AbortSignal): Promise> { - const { url, method = 'GET', headers, data, proxy } = config; - - // Prepare fetch options - const fetchOptions: RequestInit = { - method, - headers, - signal, - }; - - // Add body for non-GET requests - if (data && method !== 'GET') { - fetchOptions.body = typeof data === 'string' ? data : JSON.stringify(data); - if (typeof data === 'object') { - fetchOptions.headers = { 'Content-Type': 'application/json', ...fetchOptions.headers }; - } - } - - // Add proxy if needed (using Bun's built-in proxy support) - if (typeof proxy === 'string') { - // If proxy is a URL string, use it directly - (fetchOptions as any).proxy = proxy; - } else if (proxy) { - // If proxy is a ProxyInfo object, create a proxy URL - (fetchOptions as any).proxy = ProxyManager.createProxyUrl(proxy); - } - const response = await fetch(url, fetchOptions); - - // Parse response based on content type - let responseData: T; - const contentType = response.headers.get('content-type') || ''; - - if (contentType.includes('application/json')) { - responseData = (await response.json()) as T; - } else { - responseData = (await response.text()) as T; - } - - const httpResponse: HttpResponse = { - data: responseData, - status: response.status, - headers: Object.fromEntries(response.headers.entries()), - ok: response.ok, - }; - - // Throw HttpError for non-2xx status codes - if (!response.ok) { - throw new HttpError( - `Request failed with status ${response.status}`, - response.status, - httpResponse - ); - } - - return httpResponse; - } -} diff --git a/libs/services/http/src/adapters/index.ts b/libs/services/http/src/adapters/index.ts deleted file mode 100644 index b28aa12..0000000 --- a/libs/services/http/src/adapters/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './types'; -export * from './fetch-adapter'; -export * from './axios-adapter'; -export * from './factory'; diff --git a/libs/services/http/src/adapters/types.ts b/libs/services/http/src/adapters/types.ts deleted file mode 100644 index f363f7f..0000000 --- a/libs/services/http/src/adapters/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { HttpResponse, RequestConfig } from '../types'; - -/** - * Request adapter interface for different HTTP implementations - */ -export interface RequestAdapter { - /** - * Execute an HTTP request - */ - request(config: RequestConfig, signal: AbortSignal): Promise>; - - /** - * Check if this adapter can handle the given configuration - */ - canHandle(config: RequestConfig): boolean; -} diff --git a/libs/services/http/src/client.ts b/libs/services/http/src/client.ts deleted file mode 100644 index c02382a..0000000 --- a/libs/services/http/src/client.ts +++ /dev/null @@ -1,183 +0,0 @@ -import type { Logger } from '@stock-bot/logger'; -import { AdapterFactory } from './adapters/index'; -import type { HttpClientConfig, HttpResponse, RequestConfig } from './types'; -import { HttpError } from './types'; -import { getRandomUserAgent } from './user-agent'; - -export class HttpClient { - private readonly config: HttpClientConfig; - private readonly logger?: Logger; - - constructor(config: HttpClientConfig = {}, logger?: Logger) { - this.config = config; - this.logger = logger?.child('http-client'); - } - - // Convenience methods - async get( - url: string, - config: Omit = {} - ): Promise> { - return this.request({ ...config, method: 'GET', url }); - } - - async post( - url: string, - data?: unknown, - config: Omit = {} - ): Promise> { - return this.request({ ...config, method: 'POST', url, data }); - } - - async put( - url: string, - data?: unknown, - config: Omit = {} - ): Promise> { - return this.request({ ...config, method: 'PUT', url, data }); - } - - async del( - url: string, - config: Omit = {} - ): Promise> { - return this.request({ ...config, method: 'DELETE', url }); - } - - async patch( - url: string, - data?: unknown, - config: Omit = {} - ): Promise> { - return this.request({ ...config, method: 'PATCH', url, data }); - } - - /** - * Main request method - clean and simple - */ - async request(config: RequestConfig): Promise> { - const finalConfig = this.mergeConfig(config); - const startTime = Date.now(); - - this.logger?.debug('Making HTTP request', { - method: finalConfig.method, - url: finalConfig.url, - hasProxy: !!finalConfig.proxy, - }); - - try { - const response = await this.executeRequest(finalConfig); - response.responseTime = Date.now() - startTime; - - this.logger?.debug('HTTP request successful', { - method: finalConfig.method, - url: finalConfig.url, - status: response.status, - responseTime: response.responseTime, - }); - - return response; - } catch (error) { - if (this.logger?.getServiceName() === 'proxy-tasks') { - this.logger?.debug('HTTP request failed', { - method: finalConfig.method, - url: finalConfig.url, - error: (error as Error).message, - }); - } else { - this.logger?.warn('HTTP request failed', { - method: finalConfig.method, - url: finalConfig.url, - error: (error as Error).message, - }); - } - throw error; - } - } - - /** - * Execute request with timeout handling - no race conditions - */ private async executeRequest(config: RequestConfig): Promise> { - const timeout = config.timeout ?? this.config.timeout ?? 30000; - const controller = new AbortController(); - const startTime = Date.now(); - let timeoutId: NodeJS.Timeout | undefined; - - // Set up timeout - // Create a timeout promise that will reject - const timeoutPromise = new Promise((_, reject) => { - timeoutId = setTimeout(() => { - const elapsed = Date.now() - startTime; - this.logger?.warn('Request timeout triggered', { - url: config.url, - method: config.method, - timeout, - elapsed, - }); - - // Attempt to abort (may or may not work with Bun) - controller.abort(); - - // Force rejection regardless of signal behavior - reject(new HttpError(`Request timeout after ${timeout}ms (elapsed: ${elapsed}ms)`)); - }, timeout); - }); - - try { - // Get the appropriate adapter - const adapter = AdapterFactory.getAdapter(config); - - const response = await Promise.race([ - adapter.request(config, controller.signal), - timeoutPromise, - ]); - - this.logger?.debug('Adapter request successful', { - url: config.url, - elapsedMs: Date.now() - startTime, - }); - // Clear timeout on success - clearTimeout(timeoutId); - - return response; - } catch (error) { - const elapsed = Date.now() - startTime; - this.logger?.debug('Adapter request failed', { - url: config.url, - elapsedMs: elapsed, - }); - clearTimeout(timeoutId); - - // Handle timeout - if (controller.signal.aborted) { - throw new HttpError(`Request timeout after ${timeout}ms`); - } - - // Re-throw other errors - if (error instanceof HttpError) { - throw error; - } - - throw new HttpError(`Request failed: ${(error as Error).message}`); - } - } - - /** - * Merge configs with defaults - */ - private mergeConfig(config: RequestConfig): RequestConfig { - // Merge headers with automatic User-Agent assignment - const mergedHeaders = { ...this.config.headers, ...config.headers }; - - // Add random User-Agent if not specified - if (!mergedHeaders['User-Agent'] && !mergedHeaders['user-agent']) { - mergedHeaders['User-Agent'] = getRandomUserAgent(); - } - - return { - ...config, - headers: mergedHeaders, - timeout: config.timeout ?? this.config.timeout, - }; - } -} diff --git a/libs/services/http/src/index.ts b/libs/services/http/src/index.ts deleted file mode 100644 index ad1daa1..0000000 --- a/libs/services/http/src/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -// Re-export all types and classes -export * from './adapters/index'; -export * from './client'; -export * from './proxy-manager'; -export * from './types'; -export * from './user-agent'; - -// Default export -export { HttpClient as default } from './client'; diff --git a/libs/services/http/src/proxy-manager.ts b/libs/services/http/src/proxy-manager.ts deleted file mode 100644 index 451c52b..0000000 --- a/libs/services/http/src/proxy-manager.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { AxiosRequestConfig } from 'axios'; -import { HttpProxyAgent } from 'http-proxy-agent'; -import { HttpsProxyAgent } from 'https-proxy-agent'; -import { SocksProxyAgent } from 'socks-proxy-agent'; -import type { ProxyInfo } from './types'; - -export class ProxyManager { - /** - * Determine if we should use Bun fetch (HTTP/HTTPS) or Axios (SOCKS) - */ - static shouldUseBunFetch(proxy: ProxyInfo): boolean { - return proxy.protocol === 'http' || proxy.protocol === 'https'; - } - /** - * Create proxy URL for both Bun fetch and Axios proxy agents - */ - static createProxyUrl(proxy: ProxyInfo): string { - const { protocol, host, port, username, password } = proxy; - if (username && password) { - return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`; - } - return `${protocol}://${host}:${port}`; - } - - /** - * Create appropriate agent for Axios based on proxy type - */ - static createProxyAgent(proxy: ProxyInfo) { - this.validateConfig(proxy); - - const proxyUrl = this.createProxyUrl(proxy); - switch (proxy.protocol) { - case 'socks4': - case 'socks5': - return new SocksProxyAgent(proxyUrl); - case 'http': - return new HttpProxyAgent(proxyUrl); - case 'https': - return new HttpsProxyAgent(proxyUrl); - default: - throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`); - } - } - /** - * Create Axios instance with proxy configuration - */ - static createAxiosConfig(proxy: ProxyInfo): AxiosRequestConfig { - const agent = this.createProxyAgent(proxy); - return { - httpAgent: agent, - httpsAgent: agent, - }; - } - /** - * Simple proxy config validation - */ - static validateConfig(proxy: ProxyInfo): void { - if (!proxy.host || !proxy.port) { - throw new Error('Proxy host and port are required'); - } - if (!['http', 'https', 'socks4', 'socks5'].includes(proxy.protocol)) { - throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`); - } - } -} diff --git a/libs/services/http/src/types.ts b/libs/services/http/src/types.ts deleted file mode 100644 index 30f2e09..0000000 --- a/libs/services/http/src/types.ts +++ /dev/null @@ -1,55 +0,0 @@ -// Minimal types for fast HTTP client -export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH'; - -export interface ProxyInfo { - source?: string; - protocol: 'http' | 'https' | 'socks4' | 'socks5'; - host: string; - port: number; - username?: string; - password?: string; - url?: string; // Full proxy URL for adapters - isWorking?: boolean; - responseTime?: number; - error?: string; - // Enhanced tracking properties - working?: number; // Number of successful checks - total?: number; // Total number of checks - successRate?: number; // Success rate percentage - averageResponseTime?: number; // Average response time in milliseconds - firstSeen?: Date; // When the proxy was first added to cache - lastChecked?: Date; // When the proxy was last checked -} - -export interface HttpClientConfig { - timeout?: number; - headers?: Record; -} - -export interface RequestConfig { - method?: HttpMethod; - url: string; - headers?: Record; - data?: unknown; // Changed from 'body' to 'data' for consistency - timeout?: number; - proxy?: ProxyInfo | string; // Proxy can be a ProxyInfo object or a URL string -} - -export interface HttpResponse { - data: T; - status: number; - headers: Record; - ok: boolean; - responseTime?: number; -} - -export class HttpError extends Error { - constructor( - message: string, - public status?: number, - public response?: HttpResponse - ) { - super(message); - this.name = 'HttpError'; - } -} diff --git a/libs/services/http/src/user-agent.ts b/libs/services/http/src/user-agent.ts deleted file mode 100644 index 1b25dd1..0000000 --- a/libs/services/http/src/user-agent.ts +++ /dev/null @@ -1,6 +0,0 @@ -import UserAgent from 'user-agents'; - -export function getRandomUserAgent(): string { - const userAgent = new UserAgent(); - return userAgent.toString(); -} diff --git a/libs/services/http/test/http-integration.test.ts b/libs/services/http/test/http-integration.test.ts deleted file mode 100644 index aad154e..0000000 --- a/libs/services/http/test/http-integration.test.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; -import { HttpClient, HttpError } from '../src/index'; -import { MockServer } from './mock-server'; - -/** - * Integration tests for HTTP client with real network scenarios - * These tests use external services and may be affected by network conditions - */ - -let mockServer: MockServer; -let mockServerBaseUrl: string; - -beforeAll(async () => { - mockServer = new MockServer(); - await mockServer.start(); - mockServerBaseUrl = mockServer.getBaseUrl(); -}); - -afterAll(async () => { - await mockServer.stop(); -}); - -describe('HTTP Integration Tests', () => { - let client: HttpClient; - - beforeAll(() => { - client = new HttpClient({ - timeout: 10000, - }); - }); - - describe('Real-world scenarios', () => { - test('should handle JSON API responses', async () => { - try { - const response = await client.get('https://jsonplaceholder.typicode.com/posts/1'); - - expect(response.status).toBe(200); - expect(response.data).toHaveProperty('id'); - expect(response.data).toHaveProperty('title'); - expect(response.data).toHaveProperty('body'); - } catch (error) { - console.warn('External API test skipped due to network issues:', (error as Error).message); - } - }); - - test('should handle large responses', async () => { - try { - const response = await client.get('https://jsonplaceholder.typicode.com/posts'); - - expect(response.status).toBe(200); - expect(Array.isArray(response.data)).toBe(true); - expect(response.data.length).toBeGreaterThan(0); - } catch (error) { - console.warn( - 'Large response test skipped due to network issues:', - (error as Error).message - ); - } - }); - - test('should handle POST with JSON data', async () => { - try { - const postData = { - title: 'Integration Test Post', - body: 'This is a test post from integration tests', - userId: 1, - }; - - const response = await client.post('https://jsonplaceholder.typicode.com/posts', postData); - - expect(response.status).toBe(201); - expect(response.data).toHaveProperty('id'); - expect(response.data.title).toBe(postData.title); - } catch (error) { - console.warn( - 'POST integration test skipped due to network issues:', - (error as Error).message - ); - } - }); - }); - - describe('Error scenarios with mock server', () => { - test('should handle various HTTP status codes', async () => { - const successCodes = [200, 201]; - const errorCodes = [400, 401, 403, 404, 500, 503]; - - // Test success codes - for (const statusCode of successCodes) { - const response = await client.get(`${mockServerBaseUrl}/status/${statusCode}`); - expect(response.status).toBe(statusCode); - } - - // Test error codes (should throw HttpError) - for (const statusCode of errorCodes) { - await expect(client.get(`${mockServerBaseUrl}/status/${statusCode}`)).rejects.toThrow( - HttpError - ); - } - }); - - test('should handle malformed responses gracefully', async () => { - // Mock server returns valid JSON, so this test verifies our client handles it properly - const response = await client.get(`${mockServerBaseUrl}/`); - expect(response.status).toBe(200); - expect(typeof response.data).toBe('object'); - }); - - test('should handle concurrent requests', async () => { - const requests = Array.from({ length: 5 }, (_, i) => - client.get(`${mockServerBaseUrl}/`, { - headers: { 'X-Request-ID': `req-${i}` }, - }) - ); - - const responses = await Promise.all(requests); - - responses.forEach((response, index) => { - expect(response.status).toBe(200); - expect(response.data.headers).toHaveProperty('x-request-id', `req-${index}`); - }); - }); - }); - - describe('Performance and reliability', () => { - test('should handle rapid sequential requests', async () => { - const startTime = Date.now(); - const requests = []; - - for (let i = 0; i < 10; i++) { - requests.push(client.get(`${mockServerBaseUrl}/`)); - } - - const responses = await Promise.all(requests); - const endTime = Date.now(); - - expect(responses).toHaveLength(10); - responses.forEach(response => { - expect(response.status).toBe(200); - }); - - console.log(`Completed 10 requests in ${endTime - startTime}ms`); - }); - - test('should maintain connection efficiency', async () => { - const clientWithKeepAlive = new HttpClient({ - timeout: 5000, - }); - - const requests = Array.from({ length: 3 }, () => - clientWithKeepAlive.get(`${mockServerBaseUrl}/`) - ); - - const responses = await Promise.all(requests); - - responses.forEach(response => { - expect(response.status).toBe(200); - }); - }); - }); -}); diff --git a/libs/services/http/test/http.test.ts b/libs/services/http/test/http.test.ts deleted file mode 100644 index 34543f7..0000000 --- a/libs/services/http/test/http.test.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; -import { HttpClient, HttpError, ProxyManager } from '../src/index'; -import type { ProxyInfo } from '../src/types'; -import { MockServer } from './mock-server'; - -// Global mock server instance -let mockServer: MockServer; -let mockServerBaseUrl: string; - -beforeAll(async () => { - // Start mock server for all tests - mockServer = new MockServer(); - await mockServer.start(); - mockServerBaseUrl = mockServer.getBaseUrl(); -}); - -afterAll(async () => { - // Stop mock server - await mockServer.stop(); -}); - -describe('HttpClient', () => { - let client: HttpClient; - - beforeEach(() => { - client = new HttpClient(); - }); - - describe('Basic functionality', () => { - test('should create client with default config', () => { - expect(client).toBeInstanceOf(HttpClient); - }); - - test('should make GET request', async () => { - const response = await client.get(`${mockServerBaseUrl}/`); - - expect(response.status).toBe(200); - expect(response.data).toHaveProperty('url'); - expect(response.data).toHaveProperty('method', 'GET'); - }); - - test('should make POST request with body', async () => { - const testData = { - title: 'Test Post', - body: 'Test body', - userId: 1, - }; - - const response = await client.post(`${mockServerBaseUrl}/post`, testData); - - expect(response.status).toBe(200); - expect(response.data).toHaveProperty('data'); - expect(response.data.data).toEqual(testData); - }); - - test('should handle custom headers', async () => { - const customHeaders = { - 'X-Custom-Header': 'test-value', - 'User-Agent': 'StockBot-HTTP-Client/1.0', - }; - - const response = await client.get(`${mockServerBaseUrl}/headers`, { - headers: customHeaders, - }); - - expect(response.status).toBe(200); - expect(response.data.headers).toHaveProperty('x-custom-header', 'test-value'); - expect(response.data.headers).toHaveProperty('user-agent', 'StockBot-HTTP-Client/1.0'); - }); - - test('should handle timeout', async () => { - const clientWithTimeout = new HttpClient({ timeout: 1 }); // 1ms timeout - - await expect(clientWithTimeout.get('https://httpbin.org/delay/1')).rejects.toThrow(); - }); - }); - describe('Error handling', () => { - test('should handle HTTP errors', async () => { - await expect(client.get(`${mockServerBaseUrl}/status/404`)).rejects.toThrow(HttpError); - }); - - test('should handle network errors gracefully', async () => { - await expect( - client.get('https://nonexistent-domain-that-will-fail-12345.test') - ).rejects.toThrow(); - }); - - test('should handle invalid URLs', async () => { - await expect(client.get('not:/a:valid/url')).rejects.toThrow(); - }); - }); - - describe('HTTP methods', () => { - test('should make PUT request', async () => { - const testData = { id: 1, name: 'Updated' }; - const response = await client.put(`${mockServerBaseUrl}/post`, testData); - expect(response.status).toBe(200); - }); - - test('should make DELETE request', async () => { - const response = await client.del(`${mockServerBaseUrl}/`); - expect(response.status).toBe(200); - expect(response.data.method).toBe('DELETE'); - }); - - test('should make PATCH request', async () => { - const testData = { name: 'Patched' }; - const response = await client.patch(`${mockServerBaseUrl}/post`, testData); - expect(response.status).toBe(200); - }); - }); -}); - -describe('ProxyManager', () => { - test('should determine when to use Bun fetch', () => { - const httpProxy: ProxyInfo = { - protocol: 'http', - host: 'proxy.example.com', - port: 8080, - }; - - const socksProxy: ProxyInfo = { - protocol: 'socks5', - host: 'proxy.example.com', - port: 1080, - }; - - expect(ProxyManager.shouldUseBunFetch(httpProxy)).toBe(true); - expect(ProxyManager.shouldUseBunFetch(socksProxy)).toBe(false); - }); - - test('should create proxy URL for Bun fetch', () => { - const proxy: ProxyInfo = { - protocol: 'http', - host: 'proxy.example.com', - port: 8080, - username: 'user', - password: 'pass', - }; - - const proxyUrl = ProxyManager.createProxyUrl(proxy); - expect(proxyUrl).toBe('http://user:pass@proxy.example.com:8080'); - }); - - test('should create proxy URL without credentials', () => { - const proxy: ProxyInfo = { - protocol: 'https', - host: 'proxy.example.com', - port: 8080, - }; - - const proxyUrl = ProxyManager.createProxyUrl(proxy); - expect(proxyUrl).toBe('https://proxy.example.com:8080'); - }); -}); diff --git a/libs/services/http/test/mock-server.test.ts b/libs/services/http/test/mock-server.test.ts deleted file mode 100644 index c46e7e0..0000000 --- a/libs/services/http/test/mock-server.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; -import { MockServer } from './mock-server'; - -/** - * Tests for the MockServer utility - * Ensures our test infrastructure works correctly - */ - -describe('MockServer', () => { - let mockServer: MockServer; - let baseUrl: string; - - beforeAll(async () => { - mockServer = new MockServer(); - await mockServer.start(); - baseUrl = mockServer.getBaseUrl(); - }); - - afterAll(async () => { - await mockServer.stop(); - }); - - describe('Server lifecycle', () => { - test('should start and provide base URL', () => { - expect(baseUrl).toMatch(/^http:\/\/localhost:\d+$/); - expect(mockServer.getBaseUrl()).toBe(baseUrl); - }); - - test('should be reachable', async () => { - const response = await fetch(`${baseUrl}/`); - expect(response.ok).toBe(true); - }); - }); - - describe('Status endpoints', () => { - test('should return correct status codes', async () => { - const statusCodes = [200, 201, 400, 401, 403, 404, 500, 503]; - - for (const status of statusCodes) { - const response = await fetch(`${baseUrl}/status/${status}`); - expect(response.status).toBe(status); - } - }); - }); - - describe('Headers endpoint', () => { - test('should echo request headers', async () => { - const response = await fetch(`${baseUrl}/headers`, { - headers: { - 'X-Test-Header': 'test-value', - 'User-Agent': 'MockServer-Test', - }, - }); - - expect(response.ok).toBe(true); - const data = await response.json(); - expect(data.headers).toHaveProperty('x-test-header', 'test-value'); - expect(data.headers).toHaveProperty('user-agent', 'MockServer-Test'); - }); - }); - - describe('Basic auth endpoint', () => { - test('should authenticate valid credentials', async () => { - const username = 'testuser'; - const password = 'testpass'; - const credentials = btoa(`${username}:${password}`); - - const response = await fetch(`${baseUrl}/basic-auth/${username}/${password}`, { - headers: { - Authorization: `Basic ${credentials}`, - }, - }); - - expect(response.ok).toBe(true); - const data = await response.json(); - expect(data.authenticated).toBe(true); - expect(data.user).toBe(username); - }); - - test('should reject invalid credentials', async () => { - const credentials = btoa('wrong:credentials'); - - const response = await fetch(`${baseUrl}/basic-auth/user/pass`, { - headers: { - Authorization: `Basic ${credentials}`, - }, - }); - - expect(response.status).toBe(401); - }); - - test('should reject missing auth header', async () => { - const response = await fetch(`${baseUrl}/basic-auth/user/pass`); - expect(response.status).toBe(401); - }); - }); - - describe('POST endpoint', () => { - test('should echo POST data', async () => { - const testData = { - message: 'Hello, MockServer!', - timestamp: Date.now(), - }; - - const response = await fetch(`${baseUrl}/post`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(testData), - }); - - expect(response.ok).toBe(true); - const data = await response.json(); - expect(data.data).toEqual(testData); - expect(data.method).toBe('POST'); - expect(data.headers).toHaveProperty('content-type', 'application/json'); - }); - }); - - describe('Default endpoint', () => { - test('should return request information', async () => { - const response = await fetch(`${baseUrl}/unknown-endpoint`); - - expect(response.ok).toBe(true); - const data = await response.json(); - expect(data.url).toBe(`${baseUrl}/unknown-endpoint`); - expect(data.method).toBe('GET'); - expect(data.headers).toBeDefined(); - }); - }); -}); diff --git a/libs/services/http/test/mock-server.ts b/libs/services/http/test/mock-server.ts deleted file mode 100644 index ea8c443..0000000 --- a/libs/services/http/test/mock-server.ts +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Mock HTTP server for testing the HTTP client - * Replaces external dependency on httpbin.org with a local server - */ -export class MockServer { - private server: ReturnType | null = null; - private port: number = 0; - - /** - * Start the mock server on a random port - */ - async start(): Promise { - this.server = Bun.serve({ - port: 1, // Use any available port - fetch: this.handleRequest.bind(this), - error: this.handleError.bind(this), - }); - - this.port = this.server.port || 1; - console.log(`Mock server started on port ${this.port}`); - } - - /** - * Stop the mock server - */ - async stop(): Promise { - if (this.server) { - this.server.stop(true); - this.server = null; - this.port = 0; - console.log('Mock server stopped'); - } - } - - /** - * Get the base URL of the mock server - */ - getBaseUrl(): string { - if (!this.server) { - throw new Error('Server not started'); - } - return `http://localhost:${this.port}`; - } - - /** - * Handle incoming requests - */ private async handleRequest(req: Request): Promise { - const url = new URL(req.url); - const path = url.pathname; - - console.log(`Mock server handling request: ${req.method} ${path}`); - - // Status endpoints - if (path.startsWith('/status/')) { - const status = parseInt(path.replace('/status/', ''), 10); - console.log(`Returning status: ${status}`); - return new Response(null, { status }); - } // Headers endpoint - if (path === '/headers') { - const headers = Object.fromEntries([...req.headers.entries()]); - console.log('Headers endpoint called, received headers:', headers); - return Response.json({ headers }); - } // Basic auth endpoint - if (path.startsWith('/basic-auth/')) { - const parts = path.split('/').filter(Boolean); - const expectedUsername = parts[1]; - const expectedPassword = parts[2]; - console.log( - `Basic auth endpoint called: expected user=${expectedUsername}, pass=${expectedPassword}` - ); - - const authHeader = req.headers.get('authorization'); - if (!authHeader || !authHeader.startsWith('Basic ')) { - console.log('Missing or invalid Authorization header'); - return new Response('Unauthorized', { status: 401 }); - } - - const base64Credentials = authHeader.split(' ')[1]; - const credentials = atob(base64Credentials); - const [username, password] = credentials.split(':'); - - if (username === expectedUsername && password === expectedPassword) { - return Response.json({ - authenticated: true, - user: username, - }); - } - - return new Response('Unauthorized', { status: 401 }); - } - - // Echo request body - if (path === '/post' && req.method === 'POST') { - const data = await req.json(); - return Response.json({ - data, - headers: Object.fromEntries([...req.headers.entries()]), - method: req.method, - }); - } - - // Default response - return Response.json({ - url: req.url, - method: req.method, - headers: Object.fromEntries([...req.headers.entries()]), - }); - } - - /** - * Handle errors - */ - private handleError(_error: Error): Response { - return new Response('Server error', { status: 500 }); - } -} diff --git a/libs/services/http/tsconfig.json b/libs/services/http/tsconfig.json deleted file mode 100644 index 79532f7..0000000 --- a/libs/services/http/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "composite": true - }, - "include": ["src/**/*"], - "references": [ - { "path": "../../core/logger" }, - { "path": "../../core/types" } - ] -} diff --git a/libs/services/proxy/src/types.ts b/libs/services/proxy/src/types.ts index 54f4d8f..c36b96e 100644 --- a/libs/services/proxy/src/types.ts +++ b/libs/services/proxy/src/types.ts @@ -5,16 +5,22 @@ export interface ProxyInfo { host: string; port: number; - protocol: 'http' | 'https' | 'socks4' | 'socks5'; + protocol: 'http' | 'https'; // Simplified to only support HTTP/HTTPS username?: string; password?: string; isWorking?: boolean; successRate?: number; - lastChecked?: string; - lastUsed?: string; + lastChecked?: Date; + lastUsed?: Date; responseTime?: number; source?: string; country?: string; + error?: string; + // Tracking properties + working?: number; // Number of successful checks + total?: number; // Total number of checks + averageResponseTime?: number; // Average response time in milliseconds + firstSeen?: Date; // When the proxy was first added } export interface ProxyManagerConfig { diff --git a/libs/utils/src/fetch.ts b/libs/utils/src/fetch.ts new file mode 100644 index 0000000..15b1b49 --- /dev/null +++ b/libs/utils/src/fetch.ts @@ -0,0 +1,27 @@ +/** + * Minimal fetch wrapper with automatic debug logging + * Drop-in replacement for native fetch with logging support + */ + +export function fetch( + input: RequestInfo | URL, + init?: RequestInit & { logger?: any } +): Promise { + const logger = init?.logger || console; + const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url; + const method = init?.method || 'GET'; + + logger.debug('HTTP request', { method, url }); + + return globalThis.fetch(input, init).then(response => { + logger.debug('HTTP response', { + url, + status: response.status, + ok: response.ok + }); + return response; + }).catch(error => { + logger.debug('HTTP error', { url, error: error.message }); + throw error; + }); +} \ No newline at end of file diff --git a/libs/utils/src/index.ts b/libs/utils/src/index.ts index d6b297e..5329f6d 100644 --- a/libs/utils/src/index.ts +++ b/libs/utils/src/index.ts @@ -2,3 +2,5 @@ export * from './calculations/index'; export * from './common'; export * from './dateUtils'; export * from './generic-functions'; +export * from './fetch'; +export * from './user-agent'; diff --git a/libs/utils/src/user-agent.ts b/libs/utils/src/user-agent.ts new file mode 100644 index 0000000..49eea93 --- /dev/null +++ b/libs/utils/src/user-agent.ts @@ -0,0 +1,30 @@ +/** + * User Agent utility for generating random user agents + */ + +// Simple list of common user agents to avoid external dependency +const USER_AGENTS = [ + // Chrome on Windows + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', + // Chrome on Mac + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36', + // Firefox on Windows + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:119.0) Gecko/20100101 Firefox/119.0', + // Firefox on Mac + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/120.0', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/119.0', + // Safari on Mac + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', + // Edge on Windows + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0', +]; + +export function getRandomUserAgent(): string { + const index = Math.floor(Math.random() * USER_AGENTS.length); + return USER_AGENTS[index]!; +} \ No newline at end of file diff --git a/libs/utils/tsconfig.json b/libs/utils/tsconfig.json index d88dafd..bb002c9 100644 --- a/libs/utils/tsconfig.json +++ b/libs/utils/tsconfig.json @@ -13,7 +13,6 @@ { "path": "../core/types" }, { "path": "../data/cache" }, { "path": "../core/config" }, - { "path": "../core/logger" }, - { "path": "../services/http" } + { "path": "../core/logger" } ] } diff --git a/scripts/build-libs.sh b/scripts/build-libs.sh index 66ea0a8..16a61a6 100755 --- a/scripts/build-libs.sh +++ b/scripts/build-libs.sh @@ -43,7 +43,6 @@ libs=( "data/questdb" # QuestDB client - depends on core libs # Service libraries - "services/http" # HTTP client - depends on core libs "services/event-bus" # Event bus - depends on core libs "services/shutdown" # Shutdown - depends on core libs "services/browser" # Browser - depends on core libs