fixed all libs to be buildiable and dependency hell from removing some
This commit is contained in:
parent
5c64b1ccf8
commit
a282dac6cd
40 changed files with 4050 additions and 8219 deletions
283
libs/http-client/README.md
Normal file
283
libs/http-client/README.md
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
# HTTP Client Library
|
||||
|
||||
A comprehensive HTTP client library for the Stock Bot platform with built-in support for:
|
||||
|
||||
- ✅ **Fetch API** - Modern, promise-based HTTP requests
|
||||
- ✅ **Proxy Support** - HTTP, HTTPS, SOCKS4, and SOCKS5 proxies
|
||||
- ✅ **Rate Limiting** - Configurable request rate limiting
|
||||
- ✅ **Timeout Handling** - Request timeouts with abort controllers
|
||||
- ✅ **Retry Logic** - Automatic retries with exponential backoff
|
||||
- ✅ **TypeScript** - Full TypeScript support with type safety
|
||||
- ✅ **Logging Integration** - Optional logger integration
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
bun add @stock-bot/http-client
|
||||
```
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```typescript
|
||||
import { HttpClient } from '@stock-bot/http-client';
|
||||
|
||||
// Create a client with default configuration
|
||||
const client = new HttpClient();
|
||||
|
||||
// Make a GET request
|
||||
const response = await client.get('https://api.example.com/data');
|
||||
console.log(response.data);
|
||||
|
||||
// Make a POST request
|
||||
const postResponse = await client.post('https://api.example.com/users', {
|
||||
name: 'John Doe',
|
||||
email: 'john@example.com'
|
||||
});
|
||||
```
|
||||
|
||||
## Advanced Configuration
|
||||
|
||||
```typescript
|
||||
import { HttpClient } from '@stock-bot/http-client';
|
||||
import { logger } from '@stock-bot/logger';
|
||||
|
||||
const client = new HttpClient({
|
||||
baseURL: 'https://api.example.com',
|
||||
timeout: 10000, // 10 seconds
|
||||
retries: 3,
|
||||
retryDelay: 1000, // 1 second base delay
|
||||
defaultHeaders: {
|
||||
'Authorization': 'Bearer token',
|
||||
'User-Agent': 'Stock-Bot/1.0'
|
||||
},
|
||||
validateStatus: (status) => status < 400
|
||||
}, logger);
|
||||
```
|
||||
|
||||
## Proxy Support
|
||||
|
||||
### HTTP/HTTPS Proxy
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
proxy: {
|
||||
type: 'http',
|
||||
host: 'proxy.example.com',
|
||||
port: 8080,
|
||||
username: 'user', // optional
|
||||
password: 'pass' // optional
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### SOCKS Proxy
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
proxy: {
|
||||
type: 'socks5',
|
||||
host: 'socks-proxy.example.com',
|
||||
port: 1080,
|
||||
username: 'user', // optional
|
||||
password: 'pass' // optional
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
rateLimit: {
|
||||
maxRequests: 100, // Max 100 requests
|
||||
windowMs: 60 * 1000, // Per 1 minute
|
||||
skipSuccessfulRequests: false,
|
||||
skipFailedRequests: true // Don't count failed requests
|
||||
}
|
||||
});
|
||||
|
||||
// Check rate limit status
|
||||
const status = client.getRateLimitStatus();
|
||||
console.log(`${status.currentCount}/${status.maxRequests} requests used`);
|
||||
```
|
||||
|
||||
## Request Methods
|
||||
|
||||
```typescript
|
||||
// GET request
|
||||
const getData = await client.get('/api/data');
|
||||
|
||||
// POST request with body
|
||||
const postData = await client.post('/api/users', {
|
||||
name: 'John',
|
||||
email: 'john@example.com'
|
||||
});
|
||||
|
||||
// PUT request
|
||||
const putData = await client.put('/api/users/1', updatedUser);
|
||||
|
||||
// DELETE request
|
||||
const deleteData = await client.delete('/api/users/1');
|
||||
|
||||
// PATCH request
|
||||
const patchData = await client.patch('/api/users/1', { name: 'Jane' });
|
||||
|
||||
// Custom request
|
||||
const customResponse = await client.request({
|
||||
method: 'POST',
|
||||
url: '/api/custom',
|
||||
headers: { 'X-Custom': 'value' },
|
||||
body: { data: 'custom' },
|
||||
timeout: 5000
|
||||
});
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
```typescript
|
||||
import { HttpError, TimeoutError, RateLimitError } from '@stock-bot/http-client';
|
||||
|
||||
try {
|
||||
const response = await client.get('/api/data');
|
||||
} catch (error) {
|
||||
if (error instanceof TimeoutError) {
|
||||
console.log('Request timed out');
|
||||
} else if (error instanceof RateLimitError) {
|
||||
console.log(`Rate limited: retry after ${error.retryAfter}ms`);
|
||||
} else if (error instanceof HttpError) {
|
||||
console.log(`HTTP error ${error.status}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Retry Configuration
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
retries: 3, // Retry up to 3 times
|
||||
retryDelay: 1000, // Base delay of 1 second
|
||||
// Exponential backoff: 1s, 2s, 4s
|
||||
});
|
||||
|
||||
// Or per-request retry configuration
|
||||
const response = await client.get('/api/data', {
|
||||
retries: 5,
|
||||
retryDelay: 500
|
||||
});
|
||||
```
|
||||
|
||||
## Timeout Handling
|
||||
|
||||
```typescript
|
||||
// Global timeout
|
||||
const client = new HttpClient({
|
||||
timeout: 30000 // 30 seconds
|
||||
});
|
||||
|
||||
// Per-request timeout
|
||||
const response = await client.get('/api/data', {
|
||||
timeout: 5000 // 5 seconds for this request
|
||||
});
|
||||
```
|
||||
|
||||
## Custom Status Validation
|
||||
|
||||
```typescript
|
||||
const client = new HttpClient({
|
||||
validateStatus: (status) => {
|
||||
// Accept 2xx and 3xx status codes
|
||||
return status >= 200 && status < 400;
|
||||
}
|
||||
});
|
||||
|
||||
// Or per-request validation
|
||||
const response = await client.get('/api/data', {
|
||||
validateStatus: (status) => status === 200 || status === 404
|
||||
});
|
||||
```
|
||||
|
||||
## TypeScript Support
|
||||
|
||||
The library is fully typed with TypeScript:
|
||||
|
||||
```typescript
|
||||
interface User {
|
||||
id: number;
|
||||
name: string;
|
||||
email: string;
|
||||
}
|
||||
|
||||
// Response data is properly typed
|
||||
const response = await client.get<User[]>('/api/users');
|
||||
const users: User[] = response.data;
|
||||
|
||||
// Request configuration is validated
|
||||
const config: RequestConfig = {
|
||||
method: 'POST',
|
||||
url: '/api/users',
|
||||
body: { name: 'John' },
|
||||
timeout: 5000
|
||||
};
|
||||
```
|
||||
|
||||
## Integration with Logger
|
||||
|
||||
```typescript
|
||||
import { logger } from '@stock-bot/logger';
|
||||
import { HttpClient } from '@stock-bot/http-client';
|
||||
|
||||
const client = new HttpClient({
|
||||
baseURL: 'https://api.example.com'
|
||||
}, logger);
|
||||
|
||||
// All requests will be logged with debug/warn/error levels
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
# Run tests
|
||||
bun test
|
||||
|
||||
# Run with coverage
|
||||
bun test --coverage
|
||||
|
||||
# Watch mode
|
||||
bun test --watch
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### Proxy Support
|
||||
- HTTP and HTTPS proxies
|
||||
- SOCKS4 and SOCKS5 proxies
|
||||
- Authentication support
|
||||
- Automatic agent creation
|
||||
|
||||
### Rate Limiting
|
||||
- Token bucket algorithm
|
||||
- Configurable window and request limits
|
||||
- Skip successful/failed requests options
|
||||
- Real-time status monitoring
|
||||
|
||||
### Retry Logic
|
||||
- Exponential backoff
|
||||
- Configurable retry attempts
|
||||
- Smart retry conditions (5xx errors only)
|
||||
- Per-request retry override
|
||||
|
||||
### Error Handling
|
||||
- Typed error classes
|
||||
- Detailed error information
|
||||
- Request/response context
|
||||
- Timeout detection
|
||||
|
||||
### Performance
|
||||
- Built on modern Fetch API
|
||||
- Minimal dependencies
|
||||
- Tree-shakeable exports
|
||||
- TypeScript optimization
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see LICENSE file for details.
|
||||
35
libs/http-client/bun.lock
Normal file
35
libs/http-client/bun.lock
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"lockfileVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"dependencies": {
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"zod": "^3.25.51",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"agent-base": ["agent-base@7.1.3", "", {}, "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw=="],
|
||||
|
||||
"debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="],
|
||||
|
||||
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
|
||||
|
||||
"ip-address": ["ip-address@9.0.5", "", { "dependencies": { "jsbn": "1.1.0", "sprintf-js": "^1.1.3" } }, "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g=="],
|
||||
|
||||
"jsbn": ["jsbn@1.1.0", "", {}, "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"smart-buffer": ["smart-buffer@4.2.0", "", {}, "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="],
|
||||
|
||||
"socks": ["socks@2.8.4", "", { "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" } }, "sha512-D3YaD0aRxR3mEcqnidIs7ReYJFVzWdd6fXJYUM8ixcQcJRGTka/b3saV0KflYhyVJXKhb947GndU35SxYNResQ=="],
|
||||
|
||||
"socks-proxy-agent": ["socks-proxy-agent@8.0.5", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "^4.3.4", "socks": "^2.8.3" } }, "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw=="],
|
||||
|
||||
"sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="],
|
||||
|
||||
"zod": ["zod@3.25.51", "", {}, "sha512-TQSnBldh+XSGL+opiSIq0575wvDPqu09AqWe1F7JhUMKY+M91/aGlK4MhpVNO7MgYfHcVCB1ffwAUTJzllKJqg=="],
|
||||
}
|
||||
}
|
||||
0
libs/http-client/bunfig.toml
Normal file
0
libs/http-client/bunfig.toml
Normal file
46
libs/http-client/package.json
Normal file
46
libs/http-client/package.json
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"name": "@stock-bot/http-client",
|
||||
"version": "1.0.0",
|
||||
"description": "HTTP client library with proxy support, rate limiting, and timeout for Stock Bot platform",
|
||||
"main": "src/index.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "bun test",
|
||||
"test:watch": "bun test --watch",
|
||||
"test:coverage": "bun test --coverage",
|
||||
"lint": "eslint src/**/*.ts",
|
||||
"type-check": "tsc --noEmit",
|
||||
"dev": "tsc --watch"
|
||||
},
|
||||
"dependencies": {
|
||||
"@stock-bot/logger": "*",
|
||||
"@stock-bot/types": "*",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"zod": "^3.25.51"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.0",
|
||||
"typescript": "^5.3.0",
|
||||
"eslint": "^8.56.0",
|
||||
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
||||
"@typescript-eslint/parser": "^6.19.0",
|
||||
"bun-types": "^1.2.15"
|
||||
},
|
||||
"keywords": [
|
||||
"http",
|
||||
"client",
|
||||
"fetch",
|
||||
"proxy",
|
||||
"rate-limiting",
|
||||
"timeout",
|
||||
"stock-bot"
|
||||
],
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./src/index.ts",
|
||||
"require": "./dist/index.js"
|
||||
}
|
||||
}
|
||||
}
|
||||
291
libs/http-client/src/client.ts
Normal file
291
libs/http-client/src/client.ts
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
import type { Logger } from '@stock-bot/logger';
|
||||
import type {
|
||||
HttpClientConfig,
|
||||
RequestConfig,
|
||||
HttpResponse,
|
||||
HttpMethod,
|
||||
ProxyConfig,
|
||||
} from './types.js';
|
||||
import {
|
||||
HttpError,
|
||||
TimeoutError,
|
||||
HttpClientConfigSchema,
|
||||
RequestConfigSchema,
|
||||
} from './types.js';
|
||||
import { RateLimiter } from './rate-limiter.js';
|
||||
import { ProxyManager } from './proxy-manager.js';
|
||||
|
||||
export class HttpClient {
|
||||
private readonly config: HttpClientConfig;
|
||||
private readonly rateLimiter?: RateLimiter;
|
||||
private readonly logger?: Logger;
|
||||
|
||||
constructor(config: Partial<HttpClientConfig> = {}, logger?: Logger) {
|
||||
// Validate and set default configuration
|
||||
this.config = HttpClientConfigSchema.parse(config);
|
||||
this.logger = logger;
|
||||
|
||||
// Initialize rate limiter if configured
|
||||
if (this.config.rateLimit) {
|
||||
this.rateLimiter = new RateLimiter(this.config.rateLimit);
|
||||
}
|
||||
|
||||
// Validate proxy configuration if provided
|
||||
if (this.config.proxy) {
|
||||
ProxyManager.validateConfig(this.config.proxy);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an HTTP request
|
||||
*/
|
||||
async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
// Validate request configuration
|
||||
const validatedConfig = RequestConfigSchema.parse(config);
|
||||
|
||||
// Merge with default configuration
|
||||
const finalConfig = this.mergeConfig(validatedConfig);
|
||||
|
||||
// Check rate limiting
|
||||
if (this.rateLimiter) {
|
||||
await this.rateLimiter.checkRateLimit();
|
||||
}
|
||||
|
||||
this.logger?.debug('Making HTTP request', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url
|
||||
});
|
||||
|
||||
let lastError: Error | undefined;
|
||||
const maxRetries = finalConfig.retries ?? this.config.retries;
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
const response = await this.executeRequest<T>(finalConfig);
|
||||
|
||||
// Record successful request for rate limiting
|
||||
if (this.rateLimiter) {
|
||||
this.rateLimiter.recordRequest(true);
|
||||
}
|
||||
|
||||
this.logger?.debug('HTTP request successful', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
status: response.status,
|
||||
attempt: attempt + 1,
|
||||
});
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
|
||||
// Record failed request for rate limiting
|
||||
if (this.rateLimiter) {
|
||||
this.rateLimiter.recordRequest(false);
|
||||
}
|
||||
|
||||
this.logger?.warn('HTTP request failed', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
attempt: attempt + 1,
|
||||
error: lastError.message,
|
||||
});
|
||||
|
||||
// Don't retry on certain errors
|
||||
if (error instanceof TimeoutError ||
|
||||
(error instanceof HttpError && error.status && error.status < 500)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Wait before retrying (except on last attempt)
|
||||
if (attempt < maxRetries) {
|
||||
const delay = finalConfig.retryDelay ?? this.config.retryDelay;
|
||||
await this.sleep(delay * Math.pow(2, attempt)); // Exponential backoff
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience methods for common HTTP methods
|
||||
*/
|
||||
async get<T = any>(url: string, config: Omit<RequestConfig, 'method' | 'url'> = {}): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'GET', url });
|
||||
}
|
||||
|
||||
async post<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'POST', url, body });
|
||||
}
|
||||
|
||||
async put<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'PUT', url, body });
|
||||
}
|
||||
|
||||
async delete<T = any>(url: string, config: Omit<RequestConfig, 'method' | 'url'> = {}): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'DELETE', url });
|
||||
}
|
||||
|
||||
async patch<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'PATCH', url, body });
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the actual HTTP request
|
||||
*/
|
||||
private async executeRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const url = this.buildUrl(config.url);
|
||||
const timeout = config.timeout ?? this.config.timeout;
|
||||
|
||||
// Create abort controller for timeout
|
||||
const abortController = new AbortController();
|
||||
const timeoutId = setTimeout(() => {
|
||||
abortController.abort();
|
||||
}, timeout);
|
||||
|
||||
try {
|
||||
// Prepare request options
|
||||
const requestOptions: RequestInit = {
|
||||
method: config.method,
|
||||
headers: config.headers,
|
||||
signal: abortController.signal,
|
||||
};
|
||||
|
||||
// Add body for non-GET requests
|
||||
if (config.body && config.method !== 'GET' && config.method !== 'HEAD') {
|
||||
if (typeof config.body === 'object') {
|
||||
requestOptions.body = JSON.stringify(config.body);
|
||||
requestOptions.headers = {
|
||||
'Content-Type': 'application/json',
|
||||
...requestOptions.headers,
|
||||
};
|
||||
} else {
|
||||
requestOptions.body = config.body;
|
||||
}
|
||||
}
|
||||
|
||||
// Add proxy agent if configured
|
||||
if (this.config.proxy) {
|
||||
const agent = ProxyManager.createAgent(this.config.proxy);
|
||||
(requestOptions as any).agent = agent;
|
||||
}
|
||||
|
||||
// Make the request
|
||||
const response = await fetch(url, requestOptions);
|
||||
|
||||
// Clear timeout
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// Check if response status is valid
|
||||
const validateStatus = config.validateStatus ?? this.config.validateStatus ?? this.defaultValidateStatus;
|
||||
if (!validateStatus(response.status)) {
|
||||
throw new HttpError(
|
||||
`Request failed with status ${response.status}`,
|
||||
response.status,
|
||||
undefined,
|
||||
config
|
||||
);
|
||||
}
|
||||
|
||||
// Parse response
|
||||
const responseHeaders: Record<string, string> = {};
|
||||
response.headers.forEach((value, key) => {
|
||||
responseHeaders[key] = value;
|
||||
});
|
||||
|
||||
let data: T;
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
|
||||
if (contentType.includes('application/json')) {
|
||||
data = await response.json();
|
||||
} else if (contentType.includes('text/')) {
|
||||
data = await response.text() as any;
|
||||
} else {
|
||||
data = await response.arrayBuffer() as any;
|
||||
}
|
||||
|
||||
return {
|
||||
data,
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: responseHeaders,
|
||||
config,
|
||||
};
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (abortController.signal.aborted) {
|
||||
throw new TimeoutError(config, timeout);
|
||||
}
|
||||
|
||||
if (error instanceof HttpError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
throw new HttpError(`Request failed: ${(error as Error).message}`, undefined, undefined, config);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge request config with default config
|
||||
*/
|
||||
private mergeConfig(config: RequestConfig): RequestConfig {
|
||||
return {
|
||||
...config,
|
||||
headers: {
|
||||
...this.config.defaultHeaders,
|
||||
...config.headers,
|
||||
},
|
||||
timeout: config.timeout ?? this.config.timeout,
|
||||
retries: config.retries ?? this.config.retries,
|
||||
retryDelay: config.retryDelay ?? this.config.retryDelay,
|
||||
validateStatus: config.validateStatus ?? this.config.validateStatus,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build full URL from base URL and request URL
|
||||
*/
|
||||
private buildUrl(url: string): string {
|
||||
if (url.startsWith('http://') || url.startsWith('https://')) {
|
||||
return url;
|
||||
}
|
||||
|
||||
if (this.config.baseURL) {
|
||||
return new URL(url, this.config.baseURL).toString();
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default status validator
|
||||
*/
|
||||
private defaultValidateStatus(status: number): boolean {
|
||||
return status >= 200 && status < 300;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep utility for retries
|
||||
*/
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current rate limiting status
|
||||
*/
|
||||
getRateLimitStatus() {
|
||||
if (!this.rateLimiter) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
currentCount: this.rateLimiter.getCurrentCount(),
|
||||
maxRequests: this.config.rateLimit!.maxRequests,
|
||||
windowMs: this.config.rateLimit!.windowMs,
|
||||
timeUntilReset: this.rateLimiter.getTimeUntilReset(),
|
||||
};
|
||||
}
|
||||
}
|
||||
8
libs/http-client/src/index.ts
Normal file
8
libs/http-client/src/index.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
// Re-export all types and classes
|
||||
export * from './types.js';
|
||||
export * from './client.js';
|
||||
export * from './rate-limiter.js';
|
||||
export * from './proxy-manager.js';
|
||||
|
||||
// Default export
|
||||
export { HttpClient as default } from './client.js';
|
||||
48
libs/http-client/src/proxy-manager.ts
Normal file
48
libs/http-client/src/proxy-manager.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import { HttpsProxyAgent } from 'https-proxy-agent';
|
||||
import { SocksProxyAgent } from 'socks-proxy-agent';
|
||||
import type { ProxyConfig } from './types.js';
|
||||
|
||||
export class ProxyManager {
|
||||
/**
|
||||
* Create appropriate proxy agent based on configuration
|
||||
*/
|
||||
static createAgent(proxy: ProxyConfig): HttpsProxyAgent<string> | SocksProxyAgent {
|
||||
const { type, host, port, username, password } = proxy;
|
||||
|
||||
let proxyUrl: string;
|
||||
|
||||
if (username && password) {
|
||||
proxyUrl = `${type}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||
} else {
|
||||
proxyUrl = `${type}://${host}:${port}`;
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case 'http':
|
||||
case 'https':
|
||||
return new HttpsProxyAgent(proxyUrl);
|
||||
case 'socks4':
|
||||
case 'socks5':
|
||||
return new SocksProxyAgent(proxyUrl);
|
||||
default:
|
||||
throw new Error(`Unsupported proxy type: ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate proxy configuration
|
||||
*/
|
||||
static validateConfig(proxy: ProxyConfig): void {
|
||||
if (!proxy.host) {
|
||||
throw new Error('Proxy host is required');
|
||||
}
|
||||
|
||||
if (!proxy.port || proxy.port < 1 || proxy.port > 65535) {
|
||||
throw new Error('Invalid proxy port');
|
||||
}
|
||||
|
||||
if (!['http', 'https', 'socks4', 'socks5'].includes(proxy.type)) {
|
||||
throw new Error(`Invalid proxy type: ${proxy.type}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
79
libs/http-client/src/rate-limiter.ts
Normal file
79
libs/http-client/src/rate-limiter.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import type { RateLimitConfig } from './types.js';
|
||||
import { RateLimitError } from './types.js';
|
||||
|
||||
interface RequestRecord {
|
||||
timestamp: number;
|
||||
success: boolean;
|
||||
}
|
||||
|
||||
export class RateLimiter {
|
||||
private requests: RequestRecord[] = [];
|
||||
private readonly config: RateLimitConfig;
|
||||
|
||||
constructor(config: RateLimitConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if request is allowed based on rate limiting configuration
|
||||
* @returns Promise that resolves when request can proceed
|
||||
*/
|
||||
async checkRateLimit(): Promise<void> {
|
||||
const now = Date.now();
|
||||
const windowStart = now - this.config.windowMs;
|
||||
|
||||
// Remove old requests outside the window
|
||||
this.requests = this.requests.filter(req => req.timestamp > windowStart);
|
||||
|
||||
// Filter requests based on configuration
|
||||
const relevantRequests = this.requests.filter(req => {
|
||||
if (this.config.skipSuccessfulRequests && req.success) {
|
||||
return false;
|
||||
}
|
||||
if (this.config.skipFailedRequests && !req.success) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (relevantRequests.length >= this.config.maxRequests) {
|
||||
const oldestRequest = relevantRequests[0];
|
||||
const retryAfter = oldestRequest.timestamp + this.config.windowMs - now;
|
||||
throw new RateLimitError(this.config.maxRequests, this.config.windowMs, retryAfter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a request for rate limiting purposes
|
||||
*/
|
||||
recordRequest(success: boolean): void {
|
||||
this.requests.push({
|
||||
timestamp: Date.now(),
|
||||
success,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current request count in the window
|
||||
*/
|
||||
getCurrentCount(): number {
|
||||
const now = Date.now();
|
||||
const windowStart = now - this.config.windowMs;
|
||||
return this.requests.filter(req => req.timestamp > windowStart).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get time until next request is allowed
|
||||
*/
|
||||
getTimeUntilReset(): number {
|
||||
if (this.requests.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const oldestRequest = this.requests[0];
|
||||
const resetTime = oldestRequest.timestamp + this.config.windowMs;
|
||||
|
||||
return Math.max(0, resetTime - now);
|
||||
}
|
||||
}
|
||||
93
libs/http-client/src/types.ts
Normal file
93
libs/http-client/src/types.ts
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import { z } from 'zod';
|
||||
|
||||
// HTTP Methods
|
||||
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
|
||||
|
||||
// Proxy configuration
|
||||
export const ProxyConfigSchema = z.object({
|
||||
type: z.enum(['http', 'https', 'socks4', 'socks5']),
|
||||
host: z.string(),
|
||||
port: z.number().min(1).max(65535),
|
||||
username: z.string().optional(),
|
||||
password: z.string().optional(),
|
||||
});
|
||||
|
||||
export type ProxyConfig = z.infer<typeof ProxyConfigSchema>;
|
||||
|
||||
// Rate limiting configuration
|
||||
export const RateLimitConfigSchema = z.object({
|
||||
maxRequests: z.number().min(1),
|
||||
windowMs: z.number().min(1),
|
||||
skipSuccessfulRequests: z.boolean().default(false),
|
||||
skipFailedRequests: z.boolean().default(false),
|
||||
});
|
||||
|
||||
export type RateLimitConfig = z.infer<typeof RateLimitConfigSchema>;
|
||||
|
||||
// HTTP client configuration
|
||||
export const HttpClientConfigSchema = z.object({
|
||||
baseURL: z.string().url().optional(),
|
||||
timeout: z.number().min(1).default(30000), // 30 seconds default
|
||||
retries: z.number().min(0).default(3),
|
||||
retryDelay: z.number().min(0).default(1000), // 1 second default
|
||||
proxy: ProxyConfigSchema.optional(),
|
||||
rateLimit: RateLimitConfigSchema.optional(),
|
||||
defaultHeaders: z.record(z.string()).default({}),
|
||||
validateStatus: z.function().args(z.number()).returns(z.boolean()).optional(),
|
||||
});
|
||||
|
||||
export type HttpClientConfig = z.infer<typeof HttpClientConfigSchema>;
|
||||
|
||||
// Request configuration
|
||||
export const RequestConfigSchema = z.object({
|
||||
method: z.enum(['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS']).default('GET'),
|
||||
url: z.string(),
|
||||
headers: z.record(z.string()).default({}).optional(),
|
||||
body: z.any().optional(),
|
||||
timeout: z.number().min(1).optional(),
|
||||
retries: z.number().min(0).optional(),
|
||||
retryDelay: z.number().min(0).optional(),
|
||||
validateStatus: z.function().args(z.number()).returns(z.boolean()).optional(),
|
||||
});
|
||||
|
||||
export type RequestConfig = z.infer<typeof RequestConfigSchema>;
|
||||
|
||||
// Response type
|
||||
export interface HttpResponse<T = any> {
|
||||
data: T;
|
||||
status: number;
|
||||
statusText: string;
|
||||
headers: Record<string, string>;
|
||||
config: RequestConfig;
|
||||
}
|
||||
|
||||
// Error types
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public status?: number,
|
||||
public response?: HttpResponse,
|
||||
public config?: RequestConfig
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'HttpError';
|
||||
}
|
||||
}
|
||||
|
||||
export class TimeoutError extends HttpError {
|
||||
constructor(config: RequestConfig, timeout: number) {
|
||||
super(`Request timeout after ${timeout}ms`, undefined, undefined, config);
|
||||
this.name = 'TimeoutError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RateLimitError extends HttpError {
|
||||
constructor(
|
||||
public maxRequests: number,
|
||||
public windowMs: number,
|
||||
public retryAfter?: number
|
||||
) {
|
||||
super(`Rate limit exceeded: ${maxRequests} requests per ${windowMs}ms`);
|
||||
this.name = 'RateLimitError';
|
||||
}
|
||||
}
|
||||
190
libs/http-client/test/http-client.test.ts
Normal file
190
libs/http-client/test/http-client.test.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import { describe, test, expect, beforeEach } from 'bun:test';
|
||||
import { HttpClient, RateLimiter, ProxyManager } from '../src/index.js';
|
||||
import type { RateLimitConfig, ProxyConfig } from '../src/types.js';
|
||||
import { RateLimitError } from '../src/types.js';
|
||||
|
||||
describe('HttpClient', () => {
|
||||
let client: HttpClient;
|
||||
|
||||
beforeEach(() => {
|
||||
client = new HttpClient();
|
||||
});
|
||||
|
||||
test('should create client with default config', () => {
|
||||
expect(client).toBeInstanceOf(HttpClient);
|
||||
});
|
||||
|
||||
test('should make GET request', async () => {
|
||||
// Using a mock endpoint that returns JSON
|
||||
const response = await client.get('https://jsonplaceholder.typicode.com/posts/1');
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.data).toHaveProperty('id');
|
||||
expect(response.data).toHaveProperty('title');
|
||||
});
|
||||
|
||||
test('should handle timeout', async () => {
|
||||
const clientWithTimeout = new HttpClient({ timeout: 1 }); // 1ms timeout
|
||||
|
||||
await expect(
|
||||
clientWithTimeout.get('https://jsonplaceholder.typicode.com/posts/1')
|
||||
).rejects.toThrow('timeout');
|
||||
});
|
||||
|
||||
test('should handle 404 error', async () => {
|
||||
await expect(
|
||||
client.get('https://jsonplaceholder.typicode.com/posts/999999')
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
test('should make POST request with body', async () => {
|
||||
const response = await client.post('https://jsonplaceholder.typicode.com/posts', {
|
||||
title: 'Test Post',
|
||||
body: 'Test body',
|
||||
userId: 1,
|
||||
});
|
||||
|
||||
expect(response.status).toBe(201);
|
||||
expect(response.data).toHaveProperty('id');
|
||||
});
|
||||
|
||||
test('should use base URL', async () => {
|
||||
const clientWithBase = new HttpClient({
|
||||
baseURL: 'https://jsonplaceholder.typicode.com'
|
||||
});
|
||||
|
||||
const response = await clientWithBase.get('/posts/1');
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
test('should merge headers', async () => {
|
||||
const clientWithHeaders = new HttpClient({
|
||||
defaultHeaders: {
|
||||
'X-API-Key': 'test-key',
|
||||
}
|
||||
});
|
||||
|
||||
const response = await clientWithHeaders.get('https://jsonplaceholder.typicode.com/posts/1', {
|
||||
headers: {
|
||||
'X-Custom': 'custom-value',
|
||||
}
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('RateLimiter', () => {
|
||||
let rateLimiter: RateLimiter;
|
||||
const config: RateLimitConfig = {
|
||||
maxRequests: 2,
|
||||
windowMs: 1000, // 1 second
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
rateLimiter = new RateLimiter(config);
|
||||
});
|
||||
|
||||
test('should allow requests within limit', async () => {
|
||||
await expect(rateLimiter.checkRateLimit()).resolves.toBeUndefined();
|
||||
rateLimiter.recordRequest(true);
|
||||
|
||||
await expect(rateLimiter.checkRateLimit()).resolves.toBeUndefined();
|
||||
rateLimiter.recordRequest(true);
|
||||
});
|
||||
|
||||
test('should reject requests exceeding limit', async () => {
|
||||
// Fill up the rate limit
|
||||
await rateLimiter.checkRateLimit();
|
||||
rateLimiter.recordRequest(true);
|
||||
await rateLimiter.checkRateLimit();
|
||||
rateLimiter.recordRequest(true);
|
||||
|
||||
// This should throw
|
||||
await expect(rateLimiter.checkRateLimit()).rejects.toThrow(RateLimitError);
|
||||
});
|
||||
|
||||
test('should reset after window', async () => {
|
||||
const shortConfig: RateLimitConfig = {
|
||||
maxRequests: 1,
|
||||
windowMs: 100, // 100ms
|
||||
};
|
||||
const shortRateLimiter = new RateLimiter(shortConfig);
|
||||
|
||||
await shortRateLimiter.checkRateLimit();
|
||||
shortRateLimiter.recordRequest(true);
|
||||
|
||||
// Should be at limit
|
||||
await expect(shortRateLimiter.checkRateLimit()).rejects.toThrow(RateLimitError);
|
||||
|
||||
// Wait for window to reset
|
||||
await new Promise(resolve => setTimeout(resolve, 150));
|
||||
|
||||
// Should be allowed again
|
||||
await expect(shortRateLimiter.checkRateLimit()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test('should get current count', () => {
|
||||
expect(rateLimiter.getCurrentCount()).toBe(0);
|
||||
|
||||
rateLimiter.recordRequest(true);
|
||||
expect(rateLimiter.getCurrentCount()).toBe(1);
|
||||
|
||||
rateLimiter.recordRequest(false);
|
||||
expect(rateLimiter.getCurrentCount()).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ProxyManager', () => {
|
||||
test('should validate proxy config', () => {
|
||||
const validConfig: ProxyConfig = {
|
||||
type: 'http',
|
||||
host: 'proxy.example.com',
|
||||
port: 8080,
|
||||
};
|
||||
|
||||
expect(() => ProxyManager.validateConfig(validConfig)).not.toThrow();
|
||||
});
|
||||
|
||||
test('should reject invalid proxy config', () => {
|
||||
const invalidConfig = {
|
||||
type: 'http',
|
||||
host: '',
|
||||
port: 8080,
|
||||
} as ProxyConfig;
|
||||
|
||||
expect(() => ProxyManager.validateConfig(invalidConfig)).toThrow('Proxy host is required');
|
||||
});
|
||||
|
||||
test('should reject invalid port', () => {
|
||||
const invalidConfig = {
|
||||
type: 'http',
|
||||
host: 'proxy.example.com',
|
||||
port: 70000,
|
||||
} as ProxyConfig;
|
||||
|
||||
expect(() => ProxyManager.validateConfig(invalidConfig)).toThrow('Invalid proxy port');
|
||||
});
|
||||
|
||||
test('should create HTTP proxy agent', () => {
|
||||
const config: ProxyConfig = {
|
||||
type: 'http',
|
||||
host: 'proxy.example.com',
|
||||
port: 8080,
|
||||
};
|
||||
|
||||
const agent = ProxyManager.createAgent(config);
|
||||
expect(agent).toBeDefined();
|
||||
});
|
||||
|
||||
test('should create SOCKS proxy agent', () => {
|
||||
const config: ProxyConfig = {
|
||||
type: 'socks5',
|
||||
host: 'proxy.example.com',
|
||||
port: 1080,
|
||||
};
|
||||
|
||||
const agent = ProxyManager.createAgent(config);
|
||||
expect(agent).toBeDefined();
|
||||
});
|
||||
});
|
||||
21
libs/http-client/tsconfig.json
Normal file
21
libs/http-client/tsconfig.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"declaration": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*"
|
||||
],
|
||||
"exclude": [
|
||||
"dist",
|
||||
"node_modules",
|
||||
"**/*.test.ts",
|
||||
"**/*.spec.ts"
|
||||
],
|
||||
"references": [
|
||||
{ "path": "../logger" },
|
||||
{ "path": "../types" }
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue