removed http client for a simple fetch wrapper with logging in utils
This commit is contained in:
parent
89cbfb7848
commit
a07a71d92a
36 changed files with 100 additions and 1465 deletions
|
|
@ -1,22 +1,13 @@
|
|||
/**
|
||||
* Proxy Check Operations - Checking proxy functionality
|
||||
*/
|
||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||
import { OperationContext } from '@stock-bot/di';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { fetch } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
|
||||
// Shared HTTP client
|
||||
let httpClient: HttpClient;
|
||||
|
||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
||||
if (!httpClient) {
|
||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
||||
}
|
||||
return httpClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a proxy is working
|
||||
*/
|
||||
|
|
@ -36,22 +27,27 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
|||
});
|
||||
|
||||
try {
|
||||
// Test the proxy
|
||||
const client = getHttpClient(ctx);
|
||||
const response = await client.get(PROXY_CONFIG.CHECK_URL, {
|
||||
proxy,
|
||||
timeout: PROXY_CONFIG.CHECK_TIMEOUT,
|
||||
});
|
||||
// Test the proxy using fetch with proxy support
|
||||
const proxyUrl = proxy.username && proxy.password
|
||||
? `${proxy.protocol}://${encodeURIComponent(proxy.username)}:${encodeURIComponent(proxy.password)}@${proxy.host}:${proxy.port}`
|
||||
: `${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||
|
||||
const isWorking = response.status >= 200 && response.status < 300;
|
||||
const response = await fetch(PROXY_CONFIG.CHECK_URL, {
|
||||
proxy: proxyUrl,
|
||||
signal: AbortSignal.timeout(PROXY_CONFIG.CHECK_TIMEOUT),
|
||||
logger: ctx.logger
|
||||
} as any);
|
||||
|
||||
const data = await response.text();
|
||||
|
||||
const isWorking = response.ok;
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking,
|
||||
lastChecked: new Date(),
|
||||
responseTime: response.responseTime,
|
||||
};
|
||||
|
||||
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
|
||||
if (isWorking && !data.includes(PROXY_CONFIG.CHECK_IP)) {
|
||||
success = true;
|
||||
await updateProxyInCache(result, true, ctx);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1,18 +1,13 @@
|
|||
/**
|
||||
* Proxy Fetch Operations - Fetching proxies from sources
|
||||
*/
|
||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||
import { OperationContext } from '@stock-bot/di';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { fetch } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
import type { ProxySource } from '../shared/types';
|
||||
|
||||
// Shared HTTP client
|
||||
let httpClient: HttpClient;
|
||||
|
||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
||||
if (!httpClient) {
|
||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
||||
}
|
||||
return httpClient;
|
||||
|
|
@ -44,17 +39,17 @@ export async function fetchProxiesFromSource(source: ProxySource, ctx?: Operatio
|
|||
try {
|
||||
ctx.logger.info(`Fetching proxies from ${source.url}`);
|
||||
|
||||
const client = getHttpClient(ctx);
|
||||
const response = await client.get(source.url, {
|
||||
timeout: 10000,
|
||||
});
|
||||
const response = await fetch(source.url, {
|
||||
signal: AbortSignal.timeout(10000),
|
||||
logger: ctx.logger
|
||||
} as any);
|
||||
|
||||
if (response.status !== 200) {
|
||||
if (!response.ok) {
|
||||
ctx.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const text = response.data;
|
||||
const text = await response.text();
|
||||
const lines = text.split('\n').filter((line: string) => line.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
|
|
@ -69,7 +64,7 @@ export async function fetchProxiesFromSource(source: ProxySource, ctx?: Operatio
|
|||
if (parts.length >= 2) {
|
||||
const proxy: ProxyInfo = {
|
||||
source: source.id,
|
||||
protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5',
|
||||
protocol: source.protocol as 'http' | 'https',
|
||||
host: parts[0],
|
||||
port: parseInt(parts[1]),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Proxy Query Operations - Getting active proxies from cache
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||
import { OperationContext } from '@stock-bot/di';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Proxy Queue Operations - Queueing proxy operations
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
import { OperationContext } from '@stock-bot/di';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Proxy Provider for new queue system
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue';
|
||||
import type { ServiceContainer } from '@stock-bot/di';
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* QM Session Manager - Centralized session state management
|
||||
*/
|
||||
|
||||
import { getRandomUserAgent } from '@stock-bot/services/http';
|
||||
import { getRandomUserAgent } from '@stock-bot/utils';
|
||||
import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
|
||||
import type { QMSession } from './types';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* WebShare Fetch Operations - API integration
|
||||
*/
|
||||
import { type ProxyInfo } from '@stock-bot/http';
|
||||
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||
import { OperationContext } from '@stock-bot/di';
|
||||
|
||||
import { WEBSHARE_CONFIG } from '../shared/config';
|
||||
|
|
|
|||
|
|
@ -18,10 +18,6 @@ export class DataIngestionServiceAdapter implements IServiceContainer {
|
|||
get logger() { return this.dataServices.logger; }
|
||||
get cache() { return this.dataServices.cache; }
|
||||
get queue() { return this.dataServices.queue; }
|
||||
get http() {
|
||||
// HTTP client not in current data services - will be added when needed
|
||||
return null;
|
||||
}
|
||||
get proxy(): any {
|
||||
// Proxy manager should be injected via Awilix container
|
||||
// This adapter is for legacy compatibility
|
||||
|
|
|
|||
|
|
@ -89,11 +89,6 @@ export function createServiceContainer(config: AppConfig): AwilixContainer {
|
|||
return manager;
|
||||
}).singleton(),
|
||||
|
||||
// HTTP client can be added here when decoupled
|
||||
httpClient: asFunction(() => {
|
||||
// TODO: Import and create HTTP client when decoupled
|
||||
return null;
|
||||
}).singleton(),
|
||||
|
||||
// MongoDB client with injected logger
|
||||
mongoClient: asFunction(({ mongoConfig, logger }) => {
|
||||
|
|
@ -152,7 +147,6 @@ export function createServiceContainer(config: AppConfig): AwilixContainer {
|
|||
logger: cradle.logger,
|
||||
cache: cradle.cache,
|
||||
proxy: cradle.proxyManager,
|
||||
http: cradle.httpClient,
|
||||
browser: cradle.browser,
|
||||
mongodb: cradle.mongoClient,
|
||||
postgres: cradle.postgresClient,
|
||||
|
|
@ -224,7 +218,6 @@ export interface ServiceCradle {
|
|||
logger: any;
|
||||
cache: CacheProvider;
|
||||
proxyManager: ProxyManager;
|
||||
httpClient: any;
|
||||
browser: any;
|
||||
mongoClient: any;
|
||||
postgresClient: any;
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ export abstract class BaseHandler implements IHandler {
|
|||
readonly logger;
|
||||
readonly cache;
|
||||
readonly queue;
|
||||
readonly http;
|
||||
readonly proxy;
|
||||
readonly browser;
|
||||
readonly mongodb;
|
||||
|
|
@ -26,7 +25,6 @@ export abstract class BaseHandler implements IHandler {
|
|||
this.logger = getLogger(this.constructor.name);
|
||||
this.cache = services.cache;
|
||||
this.queue = services.queue;
|
||||
this.http = services.http;
|
||||
this.proxy = services.proxy;
|
||||
this.browser = services.browser;
|
||||
this.mongodb = services.mongodb;
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ export interface IServiceContainer {
|
|||
readonly logger: any; // Logger instance
|
||||
readonly cache: any; // Cache provider (Redis/Dragonfly)
|
||||
readonly queue: any; // Queue manager (BullMQ)
|
||||
readonly http: any; // HTTP client with proxy support
|
||||
readonly proxy: ProxyManager; // Proxy manager service
|
||||
readonly browser?: any; // Browser automation (Playwright)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
},
|
||||
"include": ["src/**/*"],
|
||||
"references": [
|
||||
{ "path": "../../core/logger" },
|
||||
{ "path": "../http" }
|
||||
{ "path": "../../core/logger" }
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,283 +0,0 @@
|
|||
# HTTP Client Library
|
||||
|
||||
A comprehensive HTTP client library for the Stock Bot platform with built-in support for:
|
||||
|
||||
- ✅ **Fetch API** - Modern, promise-based HTTP requests
|
||||
- ✅ **Proxy Support** - HTTP, HTTPS, SOCKS4, and SOCKS5 proxies
|
||||
- ✅ **Rate Limiting** - Configurable request rate limiting
|
||||
- ✅ **Timeout Handling** - Request timeouts with abort controllers
|
||||
- ✅ **Retry Logic** - Automatic retries with exponential backoff
|
||||
- ✅ **TypeScript** - Full TypeScript support with type safety
|
||||
- ✅ **Logging Integration** - Optional logger integration
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
bun add @stock-bot/http
|
||||
```
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```typescript
|
||||
import { HttpClient } from '@stock-bot/http';
|
||||
|
||||
// Create a client with default configuration
|
||||
const client = new HttpClient();
|
||||
|
||||
// Make a GET request
|
||||
const response = await client.get('https://api.example.com/data');
|
||||
console.log(response.data);
|
||||
|
||||
// Make a POST request
|
||||
const postResponse = await client.post('https://api.example.com/users', {
|
||||
name: 'John Doe',
|
||||
email: 'john@example.com'
|
||||
});
|
||||
```
|
||||
|
||||
## Advanced Configuration
|
||||
|
||||
```typescript
|
||||
import { HttpClient } from '@stock-bot/http';
|
||||
import { logger } from '@stock-bot/logger';
|
||||
|
||||
const client = new HttpClient({
|
||||
baseURL: 'https://api.example.com',
|
||||
timeout: 10000, // 10 seconds
|
||||
retries: 3,
|
||||
retryDelay: 1000, // 1 second base delay
|
||||
defaultHeaders: {
|
||||
'Authorization': 'Bearer token',
|
||||
'User-Agent': 'Stock-Bot/1.0'
|
||||
},
|
||||
validateStatus: (status) => status < 400
|
||||
}, logger);
|
||||
```
|
||||
|
||||
## Proxy Support
|
||||
|
||||
### HTTP/HTTPS Proxy
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
proxy: {
|
||||
type: 'http',
|
||||
host: 'proxy.example.com',
|
||||
port: 8080,
|
||||
username: 'user', // optional
|
||||
password: 'pass' // optional
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### SOCKS Proxy
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
proxy: {
|
||||
type: 'socks5',
|
||||
host: 'socks-proxy.example.com',
|
||||
port: 1080,
|
||||
username: 'user', // optional
|
||||
password: 'pass' // optional
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
rateLimit: {
|
||||
maxRequests: 100, // Max 100 requests
|
||||
windowMs: 60 * 1000, // Per 1 minute
|
||||
skipSuccessfulRequests: false,
|
||||
skipFailedRequests: true // Don't count failed requests
|
||||
}
|
||||
});
|
||||
|
||||
// Check rate limit status
|
||||
const status = client.getRateLimitStatus();
|
||||
console.log(`${status.currentCount}/${status.maxRequests} requests used`);
|
||||
```
|
||||
|
||||
## Request Methods
|
||||
|
||||
```typescript
|
||||
// GET request
|
||||
const getData = await client.get('/api/data');
|
||||
|
||||
// POST request with body
|
||||
const postData = await client.post('/api/users', {
|
||||
name: 'John',
|
||||
email: 'john@example.com'
|
||||
});
|
||||
|
||||
// PUT request
|
||||
const putData = await client.put('/api/users/1', updatedUser);
|
||||
|
||||
// DELETE request
|
||||
const deleteData = await client.delete('/api/users/1');
|
||||
|
||||
// PATCH request
|
||||
const patchData = await client.patch('/api/users/1', { name: 'Jane' });
|
||||
|
||||
// Custom request
|
||||
const customResponse = await client.request({
|
||||
method: 'POST',
|
||||
url: '/api/custom',
|
||||
headers: { 'X-Custom': 'value' },
|
||||
body: { data: 'custom' },
|
||||
timeout: 5000
|
||||
});
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```typescript
|
||||
import { HttpError, TimeoutError, RateLimitError } from '@stock-bot/http';
|
||||
|
||||
try {
|
||||
const response = await client.get('/api/data');
|
||||
} catch (error) {
|
||||
if (error instanceof TimeoutError) {
|
||||
console.log('Request timed out');
|
||||
} else if (error instanceof RateLimitError) {
|
||||
console.log(`Rate limited: retry after ${error.retryAfter}ms`);
|
||||
} else if (error instanceof HttpError) {
|
||||
console.log(`HTTP error ${error.status}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Retry Configuration
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
retries: 3, // Retry up to 3 times
|
||||
retryDelay: 1000, // Base delay of 1 second
|
||||
// Exponential backoff: 1s, 2s, 4s
|
||||
});
|
||||
|
||||
// Or per-request retry configuration
|
||||
const response = await client.get('/api/data', {
|
||||
retries: 5,
|
||||
retryDelay: 500
|
||||
});
|
||||
```
|
||||
|
||||
## Timeout Handling
|
||||
|
||||
```typescript
|
||||
// Global timeout
|
||||
const client = new HttpClient({
|
||||
timeout: 30000 // 30 seconds
|
||||
});
|
||||
|
||||
// Per-request timeout
|
||||
const response = await client.get('/api/data', {
|
||||
timeout: 5000 // 5 seconds for this request
|
||||
});
|
||||
```
|
||||
|
||||
## Custom Status Validation
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
validateStatus: (status) => {
|
||||
// Accept 2xx and 3xx status codes
|
||||
return status >= 200 && status < 400;
|
||||
}
|
||||
});
|
||||
|
||||
// Or per-request validation
|
||||
const response = await client.get('/api/data', {
|
||||
validateStatus: (status) => status === 200 || status === 404
|
||||
});
|
||||
```
|
||||
|
||||
## TypeScript Support
|
||||
|
||||
The library is fully typed with TypeScript:
|
||||
|
||||
```typescript
|
||||
interface User {
|
||||
id: number;
|
||||
name: string;
|
||||
email: string;
|
||||
}
|
||||
|
||||
// Response data is properly typed
|
||||
const response = await client.get<User[]>('/api/users');
|
||||
const users: User[] = response.data;
|
||||
|
||||
// Request configuration is validated
|
||||
const config: RequestConfig = {
|
||||
method: 'POST',
|
||||
url: '/api/users',
|
||||
body: { name: 'John' },
|
||||
timeout: 5000
|
||||
};
|
||||
```
|
||||
|
||||
## Integration with Logger
|
||||
|
||||
```typescript
|
||||
import { logger } from '@stock-bot/logger';
|
||||
import { HttpClient } from '@stock-bot/http';
|
||||
|
||||
const client = new HttpClient({
|
||||
baseURL: 'https://api.example.com'
|
||||
}, logger);
|
||||
|
||||
// All requests will be logged with debug/warn/error levels
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
# Run tests
|
||||
bun test
|
||||
|
||||
# Run with coverage
|
||||
bun test --coverage
|
||||
|
||||
# Watch mode
|
||||
bun test --watch
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### Proxy Support
|
||||
- HTTP and HTTPS proxies
|
||||
- SOCKS4 and SOCKS5 proxies
|
||||
- Authentication support
|
||||
- Automatic agent creation
|
||||
|
||||
### Rate Limiting
|
||||
- Token bucket algorithm
|
||||
- Configurable window and request limits
|
||||
- Skip successful/failed requests options
|
||||
- Real-time status monitoring
|
||||
|
||||
### Retry Logic
|
||||
- Exponential backoff
|
||||
- Configurable retry attempts
|
||||
- Smart retry conditions (5xx errors only)
|
||||
- Per-request retry override
|
||||
|
||||
### Error Handling
|
||||
- Typed error classes
|
||||
- Detailed error information
|
||||
- Request/response context
|
||||
- Timeout detection
|
||||
|
||||
### Performance
|
||||
- Built on modern Fetch API
|
||||
- Minimal dependencies
|
||||
- Tree-shakeable exports
|
||||
- TypeScript optimization
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see LICENSE file for details.
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
{
|
||||
"name": "@stock-bot/http",
|
||||
"version": "1.0.0",
|
||||
"description": "HTTP client library with proxy support, rate limiting, and timeout for Stock Bot platform",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "bun test",
|
||||
"test:watch": "bun test --watch",
|
||||
"test:coverage": "bun test --coverage",
|
||||
"lint": "eslint src/**/*.ts",
|
||||
"type-check": "tsc --noEmit",
|
||||
"clean": "rimraf dist"
|
||||
},
|
||||
"dependencies": {
|
||||
"@stock-bot/logger": "*",
|
||||
"@stock-bot/types": "*",
|
||||
"axios": "^1.9.0",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"user-agents": "^1.1.567"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.0",
|
||||
"@types/user-agents": "^1.0.4",
|
||||
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
||||
"@typescript-eslint/parser": "^6.19.0",
|
||||
"bun-types": "^1.2.15",
|
||||
"eslint": "^8.56.0",
|
||||
"typescript": "^5.3.0"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"require": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
import axios, { type AxiosRequestConfig, type AxiosResponse } from 'axios';
|
||||
import { ProxyManager } from '../proxy-manager';
|
||||
import type { HttpResponse, RequestConfig } from '../types';
|
||||
import { HttpError } from '../types';
|
||||
import type { RequestAdapter } from './types';
|
||||
|
||||
/**
|
||||
* Axios adapter for SOCKS proxies
|
||||
*/
|
||||
export class AxiosAdapter implements RequestAdapter {
|
||||
canHandle(config: RequestConfig): boolean {
|
||||
// Axios handles SOCKS proxies
|
||||
return Boolean(
|
||||
config.proxy &&
|
||||
typeof config.proxy !== 'string' &&
|
||||
(config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5')
|
||||
);
|
||||
}
|
||||
|
||||
async request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
|
||||
const { url, method = 'GET', headers, data, proxy } = config;
|
||||
|
||||
if (!proxy || typeof proxy === 'string') {
|
||||
throw new Error('Axios adapter requires ProxyInfo configuration');
|
||||
}
|
||||
|
||||
// Create proxy configuration using ProxyManager
|
||||
const axiosConfig: AxiosRequestConfig = {
|
||||
...ProxyManager.createAxiosConfig(proxy),
|
||||
url,
|
||||
method,
|
||||
headers,
|
||||
data,
|
||||
signal,
|
||||
// Don't throw on non-2xx status codes - let caller handle
|
||||
validateStatus: () => true,
|
||||
};
|
||||
const response: AxiosResponse<T> = await axios(axiosConfig);
|
||||
|
||||
const httpResponse: HttpResponse<T> = {
|
||||
data: response.data,
|
||||
status: response.status,
|
||||
headers: response.headers as Record<string, string>,
|
||||
ok: response.status >= 200 && response.status < 300,
|
||||
};
|
||||
|
||||
// Throw HttpError for non-2xx status codes
|
||||
if (!httpResponse.ok) {
|
||||
throw new HttpError(
|
||||
`Request failed with status ${response.status}`,
|
||||
response.status,
|
||||
httpResponse
|
||||
);
|
||||
}
|
||||
|
||||
return httpResponse;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
import type { RequestConfig } from '../types';
|
||||
import { AxiosAdapter } from './axios-adapter';
|
||||
import { FetchAdapter } from './fetch-adapter';
|
||||
import type { RequestAdapter } from './types';
|
||||
|
||||
/**
|
||||
* Factory for creating the appropriate request adapter
|
||||
*/
|
||||
export class AdapterFactory {
|
||||
private static adapters: RequestAdapter[] = [
|
||||
new AxiosAdapter(), // Check SOCKS first
|
||||
new FetchAdapter(), // Fallback to fetch for everything else
|
||||
];
|
||||
|
||||
/**
|
||||
* Get the appropriate adapter for the given configuration
|
||||
*/
|
||||
static getAdapter(config: RequestConfig): RequestAdapter {
|
||||
for (const adapter of this.adapters) {
|
||||
if (adapter.canHandle(config)) {
|
||||
return adapter;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to fetch adapter
|
||||
return new FetchAdapter();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
import { ProxyManager } from '../proxy-manager';
|
||||
import type { HttpResponse, RequestConfig } from '../types';
|
||||
import { HttpError } from '../types';
|
||||
import type { RequestAdapter } from './types';
|
||||
|
||||
/**
|
||||
* Fetch adapter for HTTP/HTTPS proxies and non-proxy requests
|
||||
*/
|
||||
export class FetchAdapter implements RequestAdapter {
|
||||
canHandle(config: RequestConfig): boolean {
|
||||
// Fetch handles non-proxy requests and HTTP/HTTPS proxies
|
||||
if (typeof config.proxy === 'string') {
|
||||
return config.proxy.startsWith('http');
|
||||
}
|
||||
return !config.proxy || config.proxy.protocol === 'http' || config.proxy.protocol === 'https';
|
||||
}
|
||||
|
||||
async request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
|
||||
const { url, method = 'GET', headers, data, proxy } = config;
|
||||
|
||||
// Prepare fetch options
|
||||
const fetchOptions: RequestInit = {
|
||||
method,
|
||||
headers,
|
||||
signal,
|
||||
};
|
||||
|
||||
// Add body for non-GET requests
|
||||
if (data && method !== 'GET') {
|
||||
fetchOptions.body = typeof data === 'string' ? data : JSON.stringify(data);
|
||||
if (typeof data === 'object') {
|
||||
fetchOptions.headers = { 'Content-Type': 'application/json', ...fetchOptions.headers };
|
||||
}
|
||||
}
|
||||
|
||||
// Add proxy if needed (using Bun's built-in proxy support)
|
||||
if (typeof proxy === 'string') {
|
||||
// If proxy is a URL string, use it directly
|
||||
(fetchOptions as any).proxy = proxy;
|
||||
} else if (proxy) {
|
||||
// If proxy is a ProxyInfo object, create a proxy URL
|
||||
(fetchOptions as any).proxy = ProxyManager.createProxyUrl(proxy);
|
||||
}
|
||||
const response = await fetch(url, fetchOptions);
|
||||
|
||||
// Parse response based on content type
|
||||
let responseData: T;
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
|
||||
if (contentType.includes('application/json')) {
|
||||
responseData = (await response.json()) as T;
|
||||
} else {
|
||||
responseData = (await response.text()) as T;
|
||||
}
|
||||
|
||||
const httpResponse: HttpResponse<T> = {
|
||||
data: responseData,
|
||||
status: response.status,
|
||||
headers: Object.fromEntries(response.headers.entries()),
|
||||
ok: response.ok,
|
||||
};
|
||||
|
||||
// Throw HttpError for non-2xx status codes
|
||||
if (!response.ok) {
|
||||
throw new HttpError(
|
||||
`Request failed with status ${response.status}`,
|
||||
response.status,
|
||||
httpResponse
|
||||
);
|
||||
}
|
||||
|
||||
return httpResponse;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
export * from './types';
|
||||
export * from './fetch-adapter';
|
||||
export * from './axios-adapter';
|
||||
export * from './factory';
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
import type { HttpResponse, RequestConfig } from '../types';
|
||||
|
||||
/**
|
||||
* Request adapter interface for different HTTP implementations
|
||||
*/
|
||||
export interface RequestAdapter {
|
||||
/**
|
||||
* Execute an HTTP request
|
||||
*/
|
||||
request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>>;
|
||||
|
||||
/**
|
||||
* Check if this adapter can handle the given configuration
|
||||
*/
|
||||
canHandle(config: RequestConfig): boolean;
|
||||
}
|
||||
|
|
@ -1,183 +0,0 @@
|
|||
import type { Logger } from '@stock-bot/logger';
|
||||
import { AdapterFactory } from './adapters/index';
|
||||
import type { HttpClientConfig, HttpResponse, RequestConfig } from './types';
|
||||
import { HttpError } from './types';
|
||||
import { getRandomUserAgent } from './user-agent';
|
||||
|
||||
export class HttpClient {
|
||||
private readonly config: HttpClientConfig;
|
||||
private readonly logger?: Logger;
|
||||
|
||||
constructor(config: HttpClientConfig = {}, logger?: Logger) {
|
||||
this.config = config;
|
||||
this.logger = logger?.child('http-client');
|
||||
}
|
||||
|
||||
// Convenience methods
|
||||
async get<T = any>(
|
||||
url: string,
|
||||
config: Omit<RequestConfig, 'method' | 'url'> = {}
|
||||
): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'GET', url });
|
||||
}
|
||||
|
||||
async post<T = any>(
|
||||
url: string,
|
||||
data?: unknown,
|
||||
config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}
|
||||
): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'POST', url, data });
|
||||
}
|
||||
|
||||
async put<T = any>(
|
||||
url: string,
|
||||
data?: unknown,
|
||||
config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}
|
||||
): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'PUT', url, data });
|
||||
}
|
||||
|
||||
async del<T = any>(
|
||||
url: string,
|
||||
config: Omit<RequestConfig, 'method' | 'url'> = {}
|
||||
): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'DELETE', url });
|
||||
}
|
||||
|
||||
async patch<T = any>(
|
||||
url: string,
|
||||
data?: unknown,
|
||||
config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}
|
||||
): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'PATCH', url, data });
|
||||
}
|
||||
|
||||
/**
|
||||
* Main request method - clean and simple
|
||||
*/
|
||||
async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const finalConfig = this.mergeConfig(config);
|
||||
const startTime = Date.now();
|
||||
|
||||
this.logger?.debug('Making HTTP request', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
hasProxy: !!finalConfig.proxy,
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.executeRequest<T>(finalConfig);
|
||||
response.responseTime = Date.now() - startTime;
|
||||
|
||||
this.logger?.debug('HTTP request successful', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
status: response.status,
|
||||
responseTime: response.responseTime,
|
||||
});
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
if (this.logger?.getServiceName() === 'proxy-tasks') {
|
||||
this.logger?.debug('HTTP request failed', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
} else {
|
||||
this.logger?.warn('HTTP request failed', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute request with timeout handling - no race conditions
|
||||
*/ private async executeRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const timeout = config.timeout ?? this.config.timeout ?? 30000;
|
||||
const controller = new AbortController();
|
||||
const startTime = Date.now();
|
||||
let timeoutId: NodeJS.Timeout | undefined;
|
||||
|
||||
// Set up timeout
|
||||
// Create a timeout promise that will reject
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
const elapsed = Date.now() - startTime;
|
||||
this.logger?.warn('Request timeout triggered', {
|
||||
url: config.url,
|
||||
method: config.method,
|
||||
timeout,
|
||||
elapsed,
|
||||
});
|
||||
|
||||
// Attempt to abort (may or may not work with Bun)
|
||||
controller.abort();
|
||||
|
||||
// Force rejection regardless of signal behavior
|
||||
reject(new HttpError(`Request timeout after ${timeout}ms (elapsed: ${elapsed}ms)`));
|
||||
}, timeout);
|
||||
});
|
||||
|
||||
try {
|
||||
// Get the appropriate adapter
|
||||
const adapter = AdapterFactory.getAdapter(config);
|
||||
|
||||
const response = await Promise.race([
|
||||
adapter.request<T>(config, controller.signal),
|
||||
timeoutPromise,
|
||||
]);
|
||||
|
||||
this.logger?.debug('Adapter request successful', {
|
||||
url: config.url,
|
||||
elapsedMs: Date.now() - startTime,
|
||||
});
|
||||
// Clear timeout on success
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
const elapsed = Date.now() - startTime;
|
||||
this.logger?.debug('Adapter request failed', {
|
||||
url: config.url,
|
||||
elapsedMs: elapsed,
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// Handle timeout
|
||||
if (controller.signal.aborted) {
|
||||
throw new HttpError(`Request timeout after ${timeout}ms`);
|
||||
}
|
||||
|
||||
// Re-throw other errors
|
||||
if (error instanceof HttpError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
throw new HttpError(`Request failed: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge configs with defaults
|
||||
*/
|
||||
private mergeConfig(config: RequestConfig): RequestConfig {
|
||||
// Merge headers with automatic User-Agent assignment
|
||||
const mergedHeaders = { ...this.config.headers, ...config.headers };
|
||||
|
||||
// Add random User-Agent if not specified
|
||||
if (!mergedHeaders['User-Agent'] && !mergedHeaders['user-agent']) {
|
||||
mergedHeaders['User-Agent'] = getRandomUserAgent();
|
||||
}
|
||||
|
||||
return {
|
||||
...config,
|
||||
headers: mergedHeaders,
|
||||
timeout: config.timeout ?? this.config.timeout,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
// Re-export all types and classes
|
||||
export * from './adapters/index';
|
||||
export * from './client';
|
||||
export * from './proxy-manager';
|
||||
export * from './types';
|
||||
export * from './user-agent';
|
||||
|
||||
// Default export
|
||||
export { HttpClient as default } from './client';
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
import { AxiosRequestConfig } from 'axios';
|
||||
import { HttpProxyAgent } from 'http-proxy-agent';
|
||||
import { HttpsProxyAgent } from 'https-proxy-agent';
|
||||
import { SocksProxyAgent } from 'socks-proxy-agent';
|
||||
import type { ProxyInfo } from './types';
|
||||
|
||||
export class ProxyManager {
|
||||
/**
|
||||
* Determine if we should use Bun fetch (HTTP/HTTPS) or Axios (SOCKS)
|
||||
*/
|
||||
static shouldUseBunFetch(proxy: ProxyInfo): boolean {
|
||||
return proxy.protocol === 'http' || proxy.protocol === 'https';
|
||||
}
|
||||
/**
|
||||
* Create proxy URL for both Bun fetch and Axios proxy agents
|
||||
*/
|
||||
static createProxyUrl(proxy: ProxyInfo): string {
|
||||
const { protocol, host, port, username, password } = proxy;
|
||||
if (username && password) {
|
||||
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||
}
|
||||
return `${protocol}://${host}:${port}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create appropriate agent for Axios based on proxy type
|
||||
*/
|
||||
static createProxyAgent(proxy: ProxyInfo) {
|
||||
this.validateConfig(proxy);
|
||||
|
||||
const proxyUrl = this.createProxyUrl(proxy);
|
||||
switch (proxy.protocol) {
|
||||
case 'socks4':
|
||||
case 'socks5':
|
||||
return new SocksProxyAgent(proxyUrl);
|
||||
case 'http':
|
||||
return new HttpProxyAgent(proxyUrl);
|
||||
case 'https':
|
||||
return new HttpsProxyAgent(proxyUrl);
|
||||
default:
|
||||
throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Create Axios instance with proxy configuration
|
||||
*/
|
||||
static createAxiosConfig(proxy: ProxyInfo): AxiosRequestConfig {
|
||||
const agent = this.createProxyAgent(proxy);
|
||||
return {
|
||||
httpAgent: agent,
|
||||
httpsAgent: agent,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Simple proxy config validation
|
||||
*/
|
||||
static validateConfig(proxy: ProxyInfo): void {
|
||||
if (!proxy.host || !proxy.port) {
|
||||
throw new Error('Proxy host and port are required');
|
||||
}
|
||||
if (!['http', 'https', 'socks4', 'socks5'].includes(proxy.protocol)) {
|
||||
throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
// Minimal types for fast HTTP client
|
||||
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH';
|
||||
|
||||
export interface ProxyInfo {
|
||||
source?: string;
|
||||
protocol: 'http' | 'https' | 'socks4' | 'socks5';
|
||||
host: string;
|
||||
port: number;
|
||||
username?: string;
|
||||
password?: string;
|
||||
url?: string; // Full proxy URL for adapters
|
||||
isWorking?: boolean;
|
||||
responseTime?: number;
|
||||
error?: string;
|
||||
// Enhanced tracking properties
|
||||
working?: number; // Number of successful checks
|
||||
total?: number; // Total number of checks
|
||||
successRate?: number; // Success rate percentage
|
||||
averageResponseTime?: number; // Average response time in milliseconds
|
||||
firstSeen?: Date; // When the proxy was first added to cache
|
||||
lastChecked?: Date; // When the proxy was last checked
|
||||
}
|
||||
|
||||
export interface HttpClientConfig {
|
||||
timeout?: number;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface RequestConfig {
|
||||
method?: HttpMethod;
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
data?: unknown; // Changed from 'body' to 'data' for consistency
|
||||
timeout?: number;
|
||||
proxy?: ProxyInfo | string; // Proxy can be a ProxyInfo object or a URL string
|
||||
}
|
||||
|
||||
export interface HttpResponse<T = any> {
|
||||
data: T;
|
||||
status: number;
|
||||
headers: Record<string, string>;
|
||||
ok: boolean;
|
||||
responseTime?: number;
|
||||
}
|
||||
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public status?: number,
|
||||
public response?: HttpResponse
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'HttpError';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
import UserAgent from 'user-agents';
|
||||
|
||||
export function getRandomUserAgent(): string {
|
||||
const userAgent = new UserAgent();
|
||||
return userAgent.toString();
|
||||
}
|
||||
|
|
@ -1,161 +0,0 @@
|
|||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import { HttpClient, HttpError } from '../src/index';
|
||||
import { MockServer } from './mock-server';
|
||||
|
||||
/**
|
||||
* Integration tests for HTTP client with real network scenarios
|
||||
* These tests use external services and may be affected by network conditions
|
||||
*/
|
||||
|
||||
let mockServer: MockServer;
|
||||
let mockServerBaseUrl: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
mockServer = new MockServer();
|
||||
await mockServer.start();
|
||||
mockServerBaseUrl = mockServer.getBaseUrl();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await mockServer.stop();
|
||||
});
|
||||
|
||||
describe('HTTP Integration Tests', () => {
|
||||
let client: HttpClient;
|
||||
|
||||
beforeAll(() => {
|
||||
client = new HttpClient({
|
||||
timeout: 10000,
|
||||
});
|
||||
});
|
||||
|
||||
describe('Real-world scenarios', () => {
|
||||
test('should handle JSON API responses', async () => {
|
||||
try {
|
||||
const response = await client.get('https://jsonplaceholder.typicode.com/posts/1');
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.data).toHaveProperty('id');
|
||||
expect(response.data).toHaveProperty('title');
|
||||
expect(response.data).toHaveProperty('body');
|
||||
} catch (error) {
|
||||
console.warn('External API test skipped due to network issues:', (error as Error).message);
|
||||
}
|
||||
});
|
||||
|
||||
test('should handle large responses', async () => {
|
||||
try {
|
||||
const response = await client.get('https://jsonplaceholder.typicode.com/posts');
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(Array.isArray(response.data)).toBe(true);
|
||||
expect(response.data.length).toBeGreaterThan(0);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'Large response test skipped due to network issues:',
|
||||
(error as Error).message
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('should handle POST with JSON data', async () => {
|
||||
try {
|
||||
const postData = {
|
||||
title: 'Integration Test Post',
|
||||
body: 'This is a test post from integration tests',
|
||||
userId: 1,
|
||||
};
|
||||
|
||||
const response = await client.post('https://jsonplaceholder.typicode.com/posts', postData);
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(response.data).toHaveProperty('id');
|
||||
expect(response.data.title).toBe(postData.title);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'POST integration test skipped due to network issues:',
|
||||
(error as Error).message
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error scenarios with mock server', () => {
|
||||
test('should handle various HTTP status codes', async () => {
|
||||
const successCodes = [200, 201];
|
||||
const errorCodes = [400, 401, 403, 404, 500, 503];
|
||||
|
||||
// Test success codes
|
||||
for (const statusCode of successCodes) {
|
||||
const response = await client.get(`${mockServerBaseUrl}/status/${statusCode}`);
|
||||
expect(response.status).toBe(statusCode);
|
||||
}
|
||||
|
||||
// Test error codes (should throw HttpError)
|
||||
for (const statusCode of errorCodes) {
|
||||
await expect(client.get(`${mockServerBaseUrl}/status/${statusCode}`)).rejects.toThrow(
|
||||
HttpError
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('should handle malformed responses gracefully', async () => {
|
||||
// Mock server returns valid JSON, so this test verifies our client handles it properly
|
||||
const response = await client.get(`${mockServerBaseUrl}/`);
|
||||
expect(response.status).toBe(200);
|
||||
expect(typeof response.data).toBe('object');
|
||||
});
|
||||
|
||||
test('should handle concurrent requests', async () => {
|
||||
const requests = Array.from({ length: 5 }, (_, i) =>
|
||||
client.get(`${mockServerBaseUrl}/`, {
|
||||
headers: { 'X-Request-ID': `req-${i}` },
|
||||
})
|
||||
);
|
||||
|
||||
const responses = await Promise.all(requests);
|
||||
|
||||
responses.forEach((response, index) => {
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.data.headers).toHaveProperty('x-request-id', `req-${index}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Performance and reliability', () => {
|
||||
test('should handle rapid sequential requests', async () => {
|
||||
const startTime = Date.now();
|
||||
const requests = [];
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
requests.push(client.get(`${mockServerBaseUrl}/`));
|
||||
}
|
||||
|
||||
const responses = await Promise.all(requests);
|
||||
const endTime = Date.now();
|
||||
|
||||
expect(responses).toHaveLength(10);
|
||||
responses.forEach(response => {
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
console.log(`Completed 10 requests in ${endTime - startTime}ms`);
|
||||
});
|
||||
|
||||
test('should maintain connection efficiency', async () => {
|
||||
const clientWithKeepAlive = new HttpClient({
|
||||
timeout: 5000,
|
||||
});
|
||||
|
||||
const requests = Array.from({ length: 3 }, () =>
|
||||
clientWithKeepAlive.get(`${mockServerBaseUrl}/`)
|
||||
);
|
||||
|
||||
const responses = await Promise.all(requests);
|
||||
|
||||
responses.forEach(response => {
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { HttpClient, HttpError, ProxyManager } from '../src/index';
|
||||
import type { ProxyInfo } from '../src/types';
|
||||
import { MockServer } from './mock-server';
|
||||
|
||||
// Global mock server instance
|
||||
let mockServer: MockServer;
|
||||
let mockServerBaseUrl: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Start mock server for all tests
|
||||
mockServer = new MockServer();
|
||||
await mockServer.start();
|
||||
mockServerBaseUrl = mockServer.getBaseUrl();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Stop mock server
|
||||
await mockServer.stop();
|
||||
});
|
||||
|
||||
describe('HttpClient', () => {
|
||||
let client: HttpClient;
|
||||
|
||||
beforeEach(() => {
|
||||
client = new HttpClient();
|
||||
});
|
||||
|
||||
describe('Basic functionality', () => {
|
||||
test('should create client with default config', () => {
|
||||
expect(client).toBeInstanceOf(HttpClient);
|
||||
});
|
||||
|
||||
test('should make GET request', async () => {
|
||||
const response = await client.get(`${mockServerBaseUrl}/`);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.data).toHaveProperty('url');
|
||||
expect(response.data).toHaveProperty('method', 'GET');
|
||||
});
|
||||
|
||||
test('should make POST request with body', async () => {
|
||||
const testData = {
|
||||
title: 'Test Post',
|
||||
body: 'Test body',
|
||||
userId: 1,
|
||||
};
|
||||
|
||||
const response = await client.post(`${mockServerBaseUrl}/post`, testData);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.data).toHaveProperty('data');
|
||||
expect(response.data.data).toEqual(testData);
|
||||
});
|
||||
|
||||
test('should handle custom headers', async () => {
|
||||
const customHeaders = {
|
||||
'X-Custom-Header': 'test-value',
|
||||
'User-Agent': 'StockBot-HTTP-Client/1.0',
|
||||
};
|
||||
|
||||
const response = await client.get(`${mockServerBaseUrl}/headers`, {
|
||||
headers: customHeaders,
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.data.headers).toHaveProperty('x-custom-header', 'test-value');
|
||||
expect(response.data.headers).toHaveProperty('user-agent', 'StockBot-HTTP-Client/1.0');
|
||||
});
|
||||
|
||||
test('should handle timeout', async () => {
|
||||
const clientWithTimeout = new HttpClient({ timeout: 1 }); // 1ms timeout
|
||||
|
||||
await expect(clientWithTimeout.get('https://httpbin.org/delay/1')).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
describe('Error handling', () => {
|
||||
test('should handle HTTP errors', async () => {
|
||||
await expect(client.get(`${mockServerBaseUrl}/status/404`)).rejects.toThrow(HttpError);
|
||||
});
|
||||
|
||||
test('should handle network errors gracefully', async () => {
|
||||
await expect(
|
||||
client.get('https://nonexistent-domain-that-will-fail-12345.test')
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
test('should handle invalid URLs', async () => {
|
||||
await expect(client.get('not:/a:valid/url')).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('HTTP methods', () => {
|
||||
test('should make PUT request', async () => {
|
||||
const testData = { id: 1, name: 'Updated' };
|
||||
const response = await client.put(`${mockServerBaseUrl}/post`, testData);
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
test('should make DELETE request', async () => {
|
||||
const response = await client.del(`${mockServerBaseUrl}/`);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.data.method).toBe('DELETE');
|
||||
});
|
||||
|
||||
test('should make PATCH request', async () => {
|
||||
const testData = { name: 'Patched' };
|
||||
const response = await client.patch(`${mockServerBaseUrl}/post`, testData);
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('ProxyManager', () => {
|
||||
test('should determine when to use Bun fetch', () => {
|
||||
const httpProxy: ProxyInfo = {
|
||||
protocol: 'http',
|
||||
host: 'proxy.example.com',
|
||||
port: 8080,
|
||||
};
|
||||
|
||||
const socksProxy: ProxyInfo = {
|
||||
protocol: 'socks5',
|
||||
host: 'proxy.example.com',
|
||||
port: 1080,
|
||||
};
|
||||
|
||||
expect(ProxyManager.shouldUseBunFetch(httpProxy)).toBe(true);
|
||||
expect(ProxyManager.shouldUseBunFetch(socksProxy)).toBe(false);
|
||||
});
|
||||
|
||||
test('should create proxy URL for Bun fetch', () => {
|
||||
const proxy: ProxyInfo = {
|
||||
protocol: 'http',
|
||||
host: 'proxy.example.com',
|
||||
port: 8080,
|
||||
username: 'user',
|
||||
password: 'pass',
|
||||
};
|
||||
|
||||
const proxyUrl = ProxyManager.createProxyUrl(proxy);
|
||||
expect(proxyUrl).toBe('http://user:pass@proxy.example.com:8080');
|
||||
});
|
||||
|
||||
test('should create proxy URL without credentials', () => {
|
||||
const proxy: ProxyInfo = {
|
||||
protocol: 'https',
|
||||
host: 'proxy.example.com',
|
||||
port: 8080,
|
||||
};
|
||||
|
||||
const proxyUrl = ProxyManager.createProxyUrl(proxy);
|
||||
expect(proxyUrl).toBe('https://proxy.example.com:8080');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import { MockServer } from './mock-server';
|
||||
|
||||
/**
|
||||
* Tests for the MockServer utility
|
||||
* Ensures our test infrastructure works correctly
|
||||
*/
|
||||
|
||||
describe('MockServer', () => {
|
||||
let mockServer: MockServer;
|
||||
let baseUrl: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
mockServer = new MockServer();
|
||||
await mockServer.start();
|
||||
baseUrl = mockServer.getBaseUrl();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await mockServer.stop();
|
||||
});
|
||||
|
||||
describe('Server lifecycle', () => {
|
||||
test('should start and provide base URL', () => {
|
||||
expect(baseUrl).toMatch(/^http:\/\/localhost:\d+$/);
|
||||
expect(mockServer.getBaseUrl()).toBe(baseUrl);
|
||||
});
|
||||
|
||||
test('should be reachable', async () => {
|
||||
const response = await fetch(`${baseUrl}/`);
|
||||
expect(response.ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Status endpoints', () => {
|
||||
test('should return correct status codes', async () => {
|
||||
const statusCodes = [200, 201, 400, 401, 403, 404, 500, 503];
|
||||
|
||||
for (const status of statusCodes) {
|
||||
const response = await fetch(`${baseUrl}/status/${status}`);
|
||||
expect(response.status).toBe(status);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Headers endpoint', () => {
|
||||
test('should echo request headers', async () => {
|
||||
const response = await fetch(`${baseUrl}/headers`, {
|
||||
headers: {
|
||||
'X-Test-Header': 'test-value',
|
||||
'User-Agent': 'MockServer-Test',
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const data = await response.json();
|
||||
expect(data.headers).toHaveProperty('x-test-header', 'test-value');
|
||||
expect(data.headers).toHaveProperty('user-agent', 'MockServer-Test');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Basic auth endpoint', () => {
|
||||
test('should authenticate valid credentials', async () => {
|
||||
const username = 'testuser';
|
||||
const password = 'testpass';
|
||||
const credentials = btoa(`${username}:${password}`);
|
||||
|
||||
const response = await fetch(`${baseUrl}/basic-auth/${username}/${password}`, {
|
||||
headers: {
|
||||
Authorization: `Basic ${credentials}`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const data = await response.json();
|
||||
expect(data.authenticated).toBe(true);
|
||||
expect(data.user).toBe(username);
|
||||
});
|
||||
|
||||
test('should reject invalid credentials', async () => {
|
||||
const credentials = btoa('wrong:credentials');
|
||||
|
||||
const response = await fetch(`${baseUrl}/basic-auth/user/pass`, {
|
||||
headers: {
|
||||
Authorization: `Basic ${credentials}`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
|
||||
test('should reject missing auth header', async () => {
|
||||
const response = await fetch(`${baseUrl}/basic-auth/user/pass`);
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST endpoint', () => {
|
||||
test('should echo POST data', async () => {
|
||||
const testData = {
|
||||
message: 'Hello, MockServer!',
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
const response = await fetch(`${baseUrl}/post`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(testData),
|
||||
});
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const data = await response.json();
|
||||
expect(data.data).toEqual(testData);
|
||||
expect(data.method).toBe('POST');
|
||||
expect(data.headers).toHaveProperty('content-type', 'application/json');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Default endpoint', () => {
|
||||
test('should return request information', async () => {
|
||||
const response = await fetch(`${baseUrl}/unknown-endpoint`);
|
||||
|
||||
expect(response.ok).toBe(true);
|
||||
const data = await response.json();
|
||||
expect(data.url).toBe(`${baseUrl}/unknown-endpoint`);
|
||||
expect(data.method).toBe('GET');
|
||||
expect(data.headers).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,116 +0,0 @@
|
|||
/**
|
||||
* Mock HTTP server for testing the HTTP client
|
||||
* Replaces external dependency on httpbin.org with a local server
|
||||
*/
|
||||
export class MockServer {
|
||||
private server: ReturnType<typeof Bun.serve> | null = null;
|
||||
private port: number = 0;
|
||||
|
||||
/**
|
||||
* Start the mock server on a random port
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
this.server = Bun.serve({
|
||||
port: 1, // Use any available port
|
||||
fetch: this.handleRequest.bind(this),
|
||||
error: this.handleError.bind(this),
|
||||
});
|
||||
|
||||
this.port = this.server.port || 1;
|
||||
console.log(`Mock server started on port ${this.port}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the mock server
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (this.server) {
|
||||
this.server.stop(true);
|
||||
this.server = null;
|
||||
this.port = 0;
|
||||
console.log('Mock server stopped');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the base URL of the mock server
|
||||
*/
|
||||
getBaseUrl(): string {
|
||||
if (!this.server) {
|
||||
throw new Error('Server not started');
|
||||
}
|
||||
return `http://localhost:${this.port}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle incoming requests
|
||||
*/ private async handleRequest(req: Request): Promise<Response> {
|
||||
const url = new URL(req.url);
|
||||
const path = url.pathname;
|
||||
|
||||
console.log(`Mock server handling request: ${req.method} ${path}`);
|
||||
|
||||
// Status endpoints
|
||||
if (path.startsWith('/status/')) {
|
||||
const status = parseInt(path.replace('/status/', ''), 10);
|
||||
console.log(`Returning status: ${status}`);
|
||||
return new Response(null, { status });
|
||||
} // Headers endpoint
|
||||
if (path === '/headers') {
|
||||
const headers = Object.fromEntries([...req.headers.entries()]);
|
||||
console.log('Headers endpoint called, received headers:', headers);
|
||||
return Response.json({ headers });
|
||||
} // Basic auth endpoint
|
||||
if (path.startsWith('/basic-auth/')) {
|
||||
const parts = path.split('/').filter(Boolean);
|
||||
const expectedUsername = parts[1];
|
||||
const expectedPassword = parts[2];
|
||||
console.log(
|
||||
`Basic auth endpoint called: expected user=${expectedUsername}, pass=${expectedPassword}`
|
||||
);
|
||||
|
||||
const authHeader = req.headers.get('authorization');
|
||||
if (!authHeader || !authHeader.startsWith('Basic ')) {
|
||||
console.log('Missing or invalid Authorization header');
|
||||
return new Response('Unauthorized', { status: 401 });
|
||||
}
|
||||
|
||||
const base64Credentials = authHeader.split(' ')[1];
|
||||
const credentials = atob(base64Credentials);
|
||||
const [username, password] = credentials.split(':');
|
||||
|
||||
if (username === expectedUsername && password === expectedPassword) {
|
||||
return Response.json({
|
||||
authenticated: true,
|
||||
user: username,
|
||||
});
|
||||
}
|
||||
|
||||
return new Response('Unauthorized', { status: 401 });
|
||||
}
|
||||
|
||||
// Echo request body
|
||||
if (path === '/post' && req.method === 'POST') {
|
||||
const data = await req.json();
|
||||
return Response.json({
|
||||
data,
|
||||
headers: Object.fromEntries([...req.headers.entries()]),
|
||||
method: req.method,
|
||||
});
|
||||
}
|
||||
|
||||
// Default response
|
||||
return Response.json({
|
||||
url: req.url,
|
||||
method: req.method,
|
||||
headers: Object.fromEntries([...req.headers.entries()]),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle errors
|
||||
*/
|
||||
private handleError(_error: Error): Response {
|
||||
return new Response('Server error', { status: 500 });
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
{
|
||||
"extends": "../../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"composite": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"references": [
|
||||
{ "path": "../../core/logger" },
|
||||
{ "path": "../../core/types" }
|
||||
]
|
||||
}
|
||||
|
|
@ -5,16 +5,22 @@
|
|||
export interface ProxyInfo {
|
||||
host: string;
|
||||
port: number;
|
||||
protocol: 'http' | 'https' | 'socks4' | 'socks5';
|
||||
protocol: 'http' | 'https'; // Simplified to only support HTTP/HTTPS
|
||||
username?: string;
|
||||
password?: string;
|
||||
isWorking?: boolean;
|
||||
successRate?: number;
|
||||
lastChecked?: string;
|
||||
lastUsed?: string;
|
||||
lastChecked?: Date;
|
||||
lastUsed?: Date;
|
||||
responseTime?: number;
|
||||
source?: string;
|
||||
country?: string;
|
||||
error?: string;
|
||||
// Tracking properties
|
||||
working?: number; // Number of successful checks
|
||||
total?: number; // Total number of checks
|
||||
averageResponseTime?: number; // Average response time in milliseconds
|
||||
firstSeen?: Date; // When the proxy was first added
|
||||
}
|
||||
|
||||
export interface ProxyManagerConfig {
|
||||
|
|
|
|||
27
libs/utils/src/fetch.ts
Normal file
27
libs/utils/src/fetch.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* Minimal fetch wrapper with automatic debug logging
|
||||
* Drop-in replacement for native fetch with logging support
|
||||
*/
|
||||
|
||||
export function fetch(
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit & { logger?: any }
|
||||
): Promise<Response> {
|
||||
const logger = init?.logger || console;
|
||||
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||
const method = init?.method || 'GET';
|
||||
|
||||
logger.debug('HTTP request', { method, url });
|
||||
|
||||
return globalThis.fetch(input, init).then(response => {
|
||||
logger.debug('HTTP response', {
|
||||
url,
|
||||
status: response.status,
|
||||
ok: response.ok
|
||||
});
|
||||
return response;
|
||||
}).catch(error => {
|
||||
logger.debug('HTTP error', { url, error: error.message });
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
|
@ -2,3 +2,5 @@ export * from './calculations/index';
|
|||
export * from './common';
|
||||
export * from './dateUtils';
|
||||
export * from './generic-functions';
|
||||
export * from './fetch';
|
||||
export * from './user-agent';
|
||||
|
|
|
|||
30
libs/utils/src/user-agent.ts
Normal file
30
libs/utils/src/user-agent.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
/**
|
||||
* User Agent utility for generating random user agents
|
||||
*/
|
||||
|
||||
// Simple list of common user agents to avoid external dependency
|
||||
const USER_AGENTS = [
|
||||
// Chrome on Windows
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||||
// Chrome on Mac
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||||
// Firefox on Windows
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:119.0) Gecko/20100101 Firefox/119.0',
|
||||
// Firefox on Mac
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/120.0',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/119.0',
|
||||
// Safari on Mac
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
|
||||
// Edge on Windows
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
|
||||
];
|
||||
|
||||
export function getRandomUserAgent(): string {
|
||||
const index = Math.floor(Math.random() * USER_AGENTS.length);
|
||||
return USER_AGENTS[index]!;
|
||||
}
|
||||
|
|
@ -13,7 +13,6 @@
|
|||
{ "path": "../core/types" },
|
||||
{ "path": "../data/cache" },
|
||||
{ "path": "../core/config" },
|
||||
{ "path": "../core/logger" },
|
||||
{ "path": "../services/http" }
|
||||
{ "path": "../core/logger" }
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ libs=(
|
|||
"data/questdb" # QuestDB client - depends on core libs
|
||||
|
||||
# Service libraries
|
||||
"services/http" # HTTP client - depends on core libs
|
||||
"services/event-bus" # Event bus - depends on core libs
|
||||
"services/shutdown" # Shutdown - depends on core libs
|
||||
"services/browser" # Browser - depends on core libs
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue