removed http client for a simple fetch wrapper with logging in utils
This commit is contained in:
parent
89cbfb7848
commit
a07a71d92a
36 changed files with 100 additions and 1465 deletions
|
|
@ -1,22 +1,13 @@
|
||||||
/**
|
/**
|
||||||
* Proxy Check Operations - Checking proxy functionality
|
* Proxy Check Operations - Checking proxy functionality
|
||||||
*/
|
*/
|
||||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||||
import { OperationContext } from '@stock-bot/di';
|
import { OperationContext } from '@stock-bot/di';
|
||||||
import { getLogger } from '@stock-bot/logger';
|
import { getLogger } from '@stock-bot/logger';
|
||||||
|
import { fetch } from '@stock-bot/utils';
|
||||||
|
|
||||||
import { PROXY_CONFIG } from '../shared/config';
|
import { PROXY_CONFIG } from '../shared/config';
|
||||||
|
|
||||||
// Shared HTTP client
|
|
||||||
let httpClient: HttpClient;
|
|
||||||
|
|
||||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
|
||||||
if (!httpClient) {
|
|
||||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
|
||||||
}
|
|
||||||
return httpClient;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a proxy is working
|
* Check if a proxy is working
|
||||||
*/
|
*/
|
||||||
|
|
@ -36,22 +27,27 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Test the proxy
|
// Test the proxy using fetch with proxy support
|
||||||
const client = getHttpClient(ctx);
|
const proxyUrl = proxy.username && proxy.password
|
||||||
const response = await client.get(PROXY_CONFIG.CHECK_URL, {
|
? `${proxy.protocol}://${encodeURIComponent(proxy.username)}:${encodeURIComponent(proxy.password)}@${proxy.host}:${proxy.port}`
|
||||||
proxy,
|
: `${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||||
timeout: PROXY_CONFIG.CHECK_TIMEOUT,
|
|
||||||
});
|
const response = await fetch(PROXY_CONFIG.CHECK_URL, {
|
||||||
|
proxy: proxyUrl,
|
||||||
|
signal: AbortSignal.timeout(PROXY_CONFIG.CHECK_TIMEOUT),
|
||||||
|
logger: ctx.logger
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const data = await response.text();
|
||||||
|
|
||||||
const isWorking = response.status >= 200 && response.status < 300;
|
const isWorking = response.ok;
|
||||||
const result: ProxyInfo = {
|
const result: ProxyInfo = {
|
||||||
...proxy,
|
...proxy,
|
||||||
isWorking,
|
isWorking,
|
||||||
lastChecked: new Date(),
|
lastChecked: new Date(),
|
||||||
responseTime: response.responseTime,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
|
if (isWorking && !data.includes(PROXY_CONFIG.CHECK_IP)) {
|
||||||
success = true;
|
success = true;
|
||||||
await updateProxyInCache(result, true, ctx);
|
await updateProxyInCache(result, true, ctx);
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,13 @@
|
||||||
/**
|
/**
|
||||||
* Proxy Fetch Operations - Fetching proxies from sources
|
* Proxy Fetch Operations - Fetching proxies from sources
|
||||||
*/
|
*/
|
||||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||||
import { OperationContext } from '@stock-bot/di';
|
import { OperationContext } from '@stock-bot/di';
|
||||||
import { getLogger } from '@stock-bot/logger';
|
import { getLogger } from '@stock-bot/logger';
|
||||||
|
import { fetch } from '@stock-bot/utils';
|
||||||
|
|
||||||
import { PROXY_CONFIG } from '../shared/config';
|
import { PROXY_CONFIG } from '../shared/config';
|
||||||
import type { ProxySource } from '../shared/types';
|
import type { ProxySource } from '../shared/types';
|
||||||
|
|
||||||
// Shared HTTP client
|
|
||||||
let httpClient: HttpClient;
|
|
||||||
|
|
||||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
|
||||||
if (!httpClient) {
|
|
||||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
||||||
}
|
}
|
||||||
return httpClient;
|
return httpClient;
|
||||||
|
|
@ -44,17 +39,17 @@ export async function fetchProxiesFromSource(source: ProxySource, ctx?: Operatio
|
||||||
try {
|
try {
|
||||||
ctx.logger.info(`Fetching proxies from ${source.url}`);
|
ctx.logger.info(`Fetching proxies from ${source.url}`);
|
||||||
|
|
||||||
const client = getHttpClient(ctx);
|
const response = await fetch(source.url, {
|
||||||
const response = await client.get(source.url, {
|
signal: AbortSignal.timeout(10000),
|
||||||
timeout: 10000,
|
logger: ctx.logger
|
||||||
});
|
} as any);
|
||||||
|
|
||||||
if (response.status !== 200) {
|
if (!response.ok) {
|
||||||
ctx.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
|
ctx.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
const text = response.data;
|
const text = await response.text();
|
||||||
const lines = text.split('\n').filter((line: string) => line.trim());
|
const lines = text.split('\n').filter((line: string) => line.trim());
|
||||||
|
|
||||||
for (const line of lines) {
|
for (const line of lines) {
|
||||||
|
|
@ -69,7 +64,7 @@ export async function fetchProxiesFromSource(source: ProxySource, ctx?: Operatio
|
||||||
if (parts.length >= 2) {
|
if (parts.length >= 2) {
|
||||||
const proxy: ProxyInfo = {
|
const proxy: ProxyInfo = {
|
||||||
source: source.id,
|
source: source.id,
|
||||||
protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5',
|
protocol: source.protocol as 'http' | 'https',
|
||||||
host: parts[0],
|
host: parts[0],
|
||||||
port: parseInt(parts[1]),
|
port: parseInt(parts[1]),
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* Proxy Query Operations - Getting active proxies from cache
|
* Proxy Query Operations - Getting active proxies from cache
|
||||||
*/
|
*/
|
||||||
import { ProxyInfo } from '@stock-bot/http';
|
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||||
import { OperationContext } from '@stock-bot/di';
|
import { OperationContext } from '@stock-bot/di';
|
||||||
|
|
||||||
import { PROXY_CONFIG } from '../shared/config';
|
import { PROXY_CONFIG } from '../shared/config';
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* Proxy Queue Operations - Queueing proxy operations
|
* Proxy Queue Operations - Queueing proxy operations
|
||||||
*/
|
*/
|
||||||
import { ProxyInfo } from '@stock-bot/http';
|
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||||
import { QueueManager } from '@stock-bot/queue';
|
import { QueueManager } from '@stock-bot/queue';
|
||||||
import { OperationContext } from '@stock-bot/di';
|
import { OperationContext } from '@stock-bot/di';
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* Proxy Provider for new queue system
|
* Proxy Provider for new queue system
|
||||||
*/
|
*/
|
||||||
import { ProxyInfo } from '@stock-bot/http';
|
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||||
import { getLogger } from '@stock-bot/logger';
|
import { getLogger } from '@stock-bot/logger';
|
||||||
import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue';
|
import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue';
|
||||||
import type { ServiceContainer } from '@stock-bot/di';
|
import type { ServiceContainer } from '@stock-bot/di';
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
* QM Session Manager - Centralized session state management
|
* QM Session Manager - Centralized session state management
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { getRandomUserAgent } from '@stock-bot/services/http';
|
import { getRandomUserAgent } from '@stock-bot/utils';
|
||||||
import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
|
import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
|
||||||
import type { QMSession } from './types';
|
import type { QMSession } from './types';
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
/**
|
/**
|
||||||
* WebShare Fetch Operations - API integration
|
* WebShare Fetch Operations - API integration
|
||||||
*/
|
*/
|
||||||
import { type ProxyInfo } from '@stock-bot/http';
|
import type { ProxyInfo } from '@stock-bot/proxy';
|
||||||
import { OperationContext } from '@stock-bot/di';
|
import { OperationContext } from '@stock-bot/di';
|
||||||
|
|
||||||
import { WEBSHARE_CONFIG } from '../shared/config';
|
import { WEBSHARE_CONFIG } from '../shared/config';
|
||||||
|
|
|
||||||
|
|
@ -18,10 +18,6 @@ export class DataIngestionServiceAdapter implements IServiceContainer {
|
||||||
get logger() { return this.dataServices.logger; }
|
get logger() { return this.dataServices.logger; }
|
||||||
get cache() { return this.dataServices.cache; }
|
get cache() { return this.dataServices.cache; }
|
||||||
get queue() { return this.dataServices.queue; }
|
get queue() { return this.dataServices.queue; }
|
||||||
get http() {
|
|
||||||
// HTTP client not in current data services - will be added when needed
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
get proxy(): any {
|
get proxy(): any {
|
||||||
// Proxy manager should be injected via Awilix container
|
// Proxy manager should be injected via Awilix container
|
||||||
// This adapter is for legacy compatibility
|
// This adapter is for legacy compatibility
|
||||||
|
|
|
||||||
|
|
@ -89,11 +89,6 @@ export function createServiceContainer(config: AppConfig): AwilixContainer {
|
||||||
return manager;
|
return manager;
|
||||||
}).singleton(),
|
}).singleton(),
|
||||||
|
|
||||||
// HTTP client can be added here when decoupled
|
|
||||||
httpClient: asFunction(() => {
|
|
||||||
// TODO: Import and create HTTP client when decoupled
|
|
||||||
return null;
|
|
||||||
}).singleton(),
|
|
||||||
|
|
||||||
// MongoDB client with injected logger
|
// MongoDB client with injected logger
|
||||||
mongoClient: asFunction(({ mongoConfig, logger }) => {
|
mongoClient: asFunction(({ mongoConfig, logger }) => {
|
||||||
|
|
@ -152,7 +147,6 @@ export function createServiceContainer(config: AppConfig): AwilixContainer {
|
||||||
logger: cradle.logger,
|
logger: cradle.logger,
|
||||||
cache: cradle.cache,
|
cache: cradle.cache,
|
||||||
proxy: cradle.proxyManager,
|
proxy: cradle.proxyManager,
|
||||||
http: cradle.httpClient,
|
|
||||||
browser: cradle.browser,
|
browser: cradle.browser,
|
||||||
mongodb: cradle.mongoClient,
|
mongodb: cradle.mongoClient,
|
||||||
postgres: cradle.postgresClient,
|
postgres: cradle.postgresClient,
|
||||||
|
|
@ -224,7 +218,6 @@ export interface ServiceCradle {
|
||||||
logger: any;
|
logger: any;
|
||||||
cache: CacheProvider;
|
cache: CacheProvider;
|
||||||
proxyManager: ProxyManager;
|
proxyManager: ProxyManager;
|
||||||
httpClient: any;
|
|
||||||
browser: any;
|
browser: any;
|
||||||
mongoClient: any;
|
mongoClient: any;
|
||||||
postgresClient: any;
|
postgresClient: any;
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ export abstract class BaseHandler implements IHandler {
|
||||||
readonly logger;
|
readonly logger;
|
||||||
readonly cache;
|
readonly cache;
|
||||||
readonly queue;
|
readonly queue;
|
||||||
readonly http;
|
|
||||||
readonly proxy;
|
readonly proxy;
|
||||||
readonly browser;
|
readonly browser;
|
||||||
readonly mongodb;
|
readonly mongodb;
|
||||||
|
|
@ -26,7 +25,6 @@ export abstract class BaseHandler implements IHandler {
|
||||||
this.logger = getLogger(this.constructor.name);
|
this.logger = getLogger(this.constructor.name);
|
||||||
this.cache = services.cache;
|
this.cache = services.cache;
|
||||||
this.queue = services.queue;
|
this.queue = services.queue;
|
||||||
this.http = services.http;
|
|
||||||
this.proxy = services.proxy;
|
this.proxy = services.proxy;
|
||||||
this.browser = services.browser;
|
this.browser = services.browser;
|
||||||
this.mongodb = services.mongodb;
|
this.mongodb = services.mongodb;
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ export interface IServiceContainer {
|
||||||
readonly logger: any; // Logger instance
|
readonly logger: any; // Logger instance
|
||||||
readonly cache: any; // Cache provider (Redis/Dragonfly)
|
readonly cache: any; // Cache provider (Redis/Dragonfly)
|
||||||
readonly queue: any; // Queue manager (BullMQ)
|
readonly queue: any; // Queue manager (BullMQ)
|
||||||
readonly http: any; // HTTP client with proxy support
|
|
||||||
readonly proxy: ProxyManager; // Proxy manager service
|
readonly proxy: ProxyManager; // Proxy manager service
|
||||||
readonly browser?: any; // Browser automation (Playwright)
|
readonly browser?: any; // Browser automation (Playwright)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@
|
||||||
},
|
},
|
||||||
"include": ["src/**/*"],
|
"include": ["src/**/*"],
|
||||||
"references": [
|
"references": [
|
||||||
{ "path": "../../core/logger" },
|
{ "path": "../../core/logger" }
|
||||||
{ "path": "../http" }
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,283 +0,0 @@
|
||||||
# HTTP Client Library
|
|
||||||
|
|
||||||
A comprehensive HTTP client library for the Stock Bot platform with built-in support for:
|
|
||||||
|
|
||||||
- ✅ **Fetch API** - Modern, promise-based HTTP requests
|
|
||||||
- ✅ **Proxy Support** - HTTP, HTTPS, SOCKS4, and SOCKS5 proxies
|
|
||||||
- ✅ **Rate Limiting** - Configurable request rate limiting
|
|
||||||
- ✅ **Timeout Handling** - Request timeouts with abort controllers
|
|
||||||
- ✅ **Retry Logic** - Automatic retries with exponential backoff
|
|
||||||
- ✅ **TypeScript** - Full TypeScript support with type safety
|
|
||||||
- ✅ **Logging Integration** - Optional logger integration
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bun add @stock-bot/http
|
|
||||||
```
|
|
||||||
|
|
||||||
## Basic Usage
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { HttpClient } from '@stock-bot/http';
|
|
||||||
|
|
||||||
// Create a client with default configuration
|
|
||||||
const client = new HttpClient();
|
|
||||||
|
|
||||||
// Make a GET request
|
|
||||||
const response = await client.get('https://api.example.com/data');
|
|
||||||
console.log(response.data);
|
|
||||||
|
|
||||||
// Make a POST request
|
|
||||||
const postResponse = await client.post('https://api.example.com/users', {
|
|
||||||
name: 'John Doe',
|
|
||||||
email: 'john@example.com'
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
## Advanced Configuration
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { HttpClient } from '@stock-bot/http';
|
|
||||||
import { logger } from '@stock-bot/logger';
|
|
||||||
|
|
||||||
const client = new HttpClient({
|
|
||||||
baseURL: 'https://api.example.com',
|
|
||||||
timeout: 10000, // 10 seconds
|
|
||||||
retries: 3,
|
|
||||||
retryDelay: 1000, // 1 second base delay
|
|
||||||
defaultHeaders: {
|
|
||||||
'Authorization': 'Bearer token',
|
|
||||||
'User-Agent': 'Stock-Bot/1.0'
|
|
||||||
},
|
|
||||||
validateStatus: (status) => status < 400
|
|
||||||
}, logger);
|
|
||||||
```
|
|
||||||
|
|
||||||
## Proxy Support
|
|
||||||
|
|
||||||
### HTTP/HTTPS Proxy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const client = new HttpClient({
|
|
||||||
proxy: {
|
|
||||||
type: 'http',
|
|
||||||
host: 'proxy.example.com',
|
|
||||||
port: 8080,
|
|
||||||
username: 'user', // optional
|
|
||||||
password: 'pass' // optional
|
|
||||||
}
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
### SOCKS Proxy
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const client = new HttpClient({
|
|
||||||
proxy: {
|
|
||||||
type: 'socks5',
|
|
||||||
host: 'socks-proxy.example.com',
|
|
||||||
port: 1080,
|
|
||||||
username: 'user', // optional
|
|
||||||
password: 'pass' // optional
|
|
||||||
}
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
## Rate Limiting
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const client = new HttpClient({
|
|
||||||
rateLimit: {
|
|
||||||
maxRequests: 100, // Max 100 requests
|
|
||||||
windowMs: 60 * 1000, // Per 1 minute
|
|
||||||
skipSuccessfulRequests: false,
|
|
||||||
skipFailedRequests: true // Don't count failed requests
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Check rate limit status
|
|
||||||
const status = client.getRateLimitStatus();
|
|
||||||
console.log(`${status.currentCount}/${status.maxRequests} requests used`);
|
|
||||||
```
|
|
||||||
|
|
||||||
## Request Methods
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// GET request
|
|
||||||
const getData = await client.get('/api/data');
|
|
||||||
|
|
||||||
// POST request with body
|
|
||||||
const postData = await client.post('/api/users', {
|
|
||||||
name: 'John',
|
|
||||||
email: 'john@example.com'
|
|
||||||
});
|
|
||||||
|
|
||||||
// PUT request
|
|
||||||
const putData = await client.put('/api/users/1', updatedUser);
|
|
||||||
|
|
||||||
// DELETE request
|
|
||||||
const deleteData = await client.delete('/api/users/1');
|
|
||||||
|
|
||||||
// PATCH request
|
|
||||||
const patchData = await client.patch('/api/users/1', { name: 'Jane' });
|
|
||||||
|
|
||||||
// Custom request
|
|
||||||
const customResponse = await client.request({
|
|
||||||
method: 'POST',
|
|
||||||
url: '/api/custom',
|
|
||||||
headers: { 'X-Custom': 'value' },
|
|
||||||
body: { data: 'custom' },
|
|
||||||
timeout: 5000
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
## Error Handling
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { HttpError, TimeoutError, RateLimitError } from '@stock-bot/http';
|
|
||||||
|
|
||||||
try {
|
|
||||||
const response = await client.get('/api/data');
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof TimeoutError) {
|
|
||||||
console.log('Request timed out');
|
|
||||||
} else if (error instanceof RateLimitError) {
|
|
||||||
console.log(`Rate limited: retry after ${error.retryAfter}ms`);
|
|
||||||
} else if (error instanceof HttpError) {
|
|
||||||
console.log(`HTTP error ${error.status}: ${error.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Retry Configuration
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const client = new HttpClient({
|
|
||||||
retries: 3, // Retry up to 3 times
|
|
||||||
retryDelay: 1000, // Base delay of 1 second
|
|
||||||
// Exponential backoff: 1s, 2s, 4s
|
|
||||||
});
|
|
||||||
|
|
||||||
// Or per-request retry configuration
|
|
||||||
const response = await client.get('/api/data', {
|
|
||||||
retries: 5,
|
|
||||||
retryDelay: 500
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
## Timeout Handling
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Global timeout
|
|
||||||
const client = new HttpClient({
|
|
||||||
timeout: 30000 // 30 seconds
|
|
||||||
});
|
|
||||||
|
|
||||||
// Per-request timeout
|
|
||||||
const response = await client.get('/api/data', {
|
|
||||||
timeout: 5000 // 5 seconds for this request
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
## Custom Status Validation
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const client = new HttpClient({
|
|
||||||
validateStatus: (status) => {
|
|
||||||
// Accept 2xx and 3xx status codes
|
|
||||||
return status >= 200 && status < 400;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Or per-request validation
|
|
||||||
const response = await client.get('/api/data', {
|
|
||||||
validateStatus: (status) => status === 200 || status === 404
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
## TypeScript Support
|
|
||||||
|
|
||||||
The library is fully typed with TypeScript:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface User {
|
|
||||||
id: number;
|
|
||||||
name: string;
|
|
||||||
email: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Response data is properly typed
|
|
||||||
const response = await client.get<User[]>('/api/users');
|
|
||||||
const users: User[] = response.data;
|
|
||||||
|
|
||||||
// Request configuration is validated
|
|
||||||
const config: RequestConfig = {
|
|
||||||
method: 'POST',
|
|
||||||
url: '/api/users',
|
|
||||||
body: { name: 'John' },
|
|
||||||
timeout: 5000
|
|
||||||
};
|
|
||||||
```
|
|
||||||
|
|
||||||
## Integration with Logger
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { logger } from '@stock-bot/logger';
|
|
||||||
import { HttpClient } from '@stock-bot/http';
|
|
||||||
|
|
||||||
const client = new HttpClient({
|
|
||||||
baseURL: 'https://api.example.com'
|
|
||||||
}, logger);
|
|
||||||
|
|
||||||
// All requests will be logged with debug/warn/error levels
|
|
||||||
```
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Run tests
|
|
||||||
bun test
|
|
||||||
|
|
||||||
# Run with coverage
|
|
||||||
bun test --coverage
|
|
||||||
|
|
||||||
# Watch mode
|
|
||||||
bun test --watch
|
|
||||||
```
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
### Proxy Support
|
|
||||||
- HTTP and HTTPS proxies
|
|
||||||
- SOCKS4 and SOCKS5 proxies
|
|
||||||
- Authentication support
|
|
||||||
- Automatic agent creation
|
|
||||||
|
|
||||||
### Rate Limiting
|
|
||||||
- Token bucket algorithm
|
|
||||||
- Configurable window and request limits
|
|
||||||
- Skip successful/failed requests options
|
|
||||||
- Real-time status monitoring
|
|
||||||
|
|
||||||
### Retry Logic
|
|
||||||
- Exponential backoff
|
|
||||||
- Configurable retry attempts
|
|
||||||
- Smart retry conditions (5xx errors only)
|
|
||||||
- Per-request retry override
|
|
||||||
|
|
||||||
### Error Handling
|
|
||||||
- Typed error classes
|
|
||||||
- Detailed error information
|
|
||||||
- Request/response context
|
|
||||||
- Timeout detection
|
|
||||||
|
|
||||||
### Performance
|
|
||||||
- Built on modern Fetch API
|
|
||||||
- Minimal dependencies
|
|
||||||
- Tree-shakeable exports
|
|
||||||
- TypeScript optimization
|
|
||||||
|
|
||||||
## License
|
|
||||||
|
|
||||||
MIT License - see LICENSE file for details.
|
|
||||||
|
|
@ -1,46 +0,0 @@
|
||||||
{
|
|
||||||
"name": "@stock-bot/http",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"description": "HTTP client library with proxy support, rate limiting, and timeout for Stock Bot platform",
|
|
||||||
"main": "dist/index.js",
|
|
||||||
"types": "dist/index.d.ts",
|
|
||||||
"type": "module",
|
|
||||||
"scripts": {
|
|
||||||
"build": "tsc",
|
|
||||||
"test": "bun test",
|
|
||||||
"test:watch": "bun test --watch",
|
|
||||||
"test:coverage": "bun test --coverage",
|
|
||||||
"lint": "eslint src/**/*.ts",
|
|
||||||
"type-check": "tsc --noEmit",
|
|
||||||
"clean": "rimraf dist"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"@stock-bot/logger": "*",
|
|
||||||
"@stock-bot/types": "*",
|
|
||||||
"axios": "^1.9.0",
|
|
||||||
"http-proxy-agent": "^7.0.2",
|
|
||||||
"https-proxy-agent": "^7.0.6",
|
|
||||||
"socks-proxy-agent": "^8.0.5",
|
|
||||||
"user-agents": "^1.1.567"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"@types/node": "^20.11.0",
|
|
||||||
"@types/user-agents": "^1.0.4",
|
|
||||||
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
|
||||||
"@typescript-eslint/parser": "^6.19.0",
|
|
||||||
"bun-types": "^1.2.15",
|
|
||||||
"eslint": "^8.56.0",
|
|
||||||
"typescript": "^5.3.0"
|
|
||||||
},
|
|
||||||
"exports": {
|
|
||||||
".": {
|
|
||||||
"import": "./dist/index.js",
|
|
||||||
"require": "./dist/index.js",
|
|
||||||
"types": "./dist/index.d.ts"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"files": [
|
|
||||||
"dist",
|
|
||||||
"README.md"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -1,58 +0,0 @@
|
||||||
import axios, { type AxiosRequestConfig, type AxiosResponse } from 'axios';
|
|
||||||
import { ProxyManager } from '../proxy-manager';
|
|
||||||
import type { HttpResponse, RequestConfig } from '../types';
|
|
||||||
import { HttpError } from '../types';
|
|
||||||
import type { RequestAdapter } from './types';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Axios adapter for SOCKS proxies
|
|
||||||
*/
|
|
||||||
export class AxiosAdapter implements RequestAdapter {
|
|
||||||
canHandle(config: RequestConfig): boolean {
|
|
||||||
// Axios handles SOCKS proxies
|
|
||||||
return Boolean(
|
|
||||||
config.proxy &&
|
|
||||||
typeof config.proxy !== 'string' &&
|
|
||||||
(config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5')
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
|
|
||||||
const { url, method = 'GET', headers, data, proxy } = config;
|
|
||||||
|
|
||||||
if (!proxy || typeof proxy === 'string') {
|
|
||||||
throw new Error('Axios adapter requires ProxyInfo configuration');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create proxy configuration using ProxyManager
|
|
||||||
const axiosConfig: AxiosRequestConfig = {
|
|
||||||
...ProxyManager.createAxiosConfig(proxy),
|
|
||||||
url,
|
|
||||||
method,
|
|
||||||
headers,
|
|
||||||
data,
|
|
||||||
signal,
|
|
||||||
// Don't throw on non-2xx status codes - let caller handle
|
|
||||||
validateStatus: () => true,
|
|
||||||
};
|
|
||||||
const response: AxiosResponse<T> = await axios(axiosConfig);
|
|
||||||
|
|
||||||
const httpResponse: HttpResponse<T> = {
|
|
||||||
data: response.data,
|
|
||||||
status: response.status,
|
|
||||||
headers: response.headers as Record<string, string>,
|
|
||||||
ok: response.status >= 200 && response.status < 300,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Throw HttpError for non-2xx status codes
|
|
||||||
if (!httpResponse.ok) {
|
|
||||||
throw new HttpError(
|
|
||||||
`Request failed with status ${response.status}`,
|
|
||||||
response.status,
|
|
||||||
httpResponse
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return httpResponse;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
import type { RequestConfig } from '../types';
|
|
||||||
import { AxiosAdapter } from './axios-adapter';
|
|
||||||
import { FetchAdapter } from './fetch-adapter';
|
|
||||||
import type { RequestAdapter } from './types';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory for creating the appropriate request adapter
|
|
||||||
*/
|
|
||||||
export class AdapterFactory {
|
|
||||||
private static adapters: RequestAdapter[] = [
|
|
||||||
new AxiosAdapter(), // Check SOCKS first
|
|
||||||
new FetchAdapter(), // Fallback to fetch for everything else
|
|
||||||
];
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the appropriate adapter for the given configuration
|
|
||||||
*/
|
|
||||||
static getAdapter(config: RequestConfig): RequestAdapter {
|
|
||||||
for (const adapter of this.adapters) {
|
|
||||||
if (adapter.canHandle(config)) {
|
|
||||||
return adapter;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback to fetch adapter
|
|
||||||
return new FetchAdapter();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,74 +0,0 @@
|
||||||
import { ProxyManager } from '../proxy-manager';
|
|
||||||
import type { HttpResponse, RequestConfig } from '../types';
|
|
||||||
import { HttpError } from '../types';
|
|
||||||
import type { RequestAdapter } from './types';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetch adapter for HTTP/HTTPS proxies and non-proxy requests
|
|
||||||
*/
|
|
||||||
export class FetchAdapter implements RequestAdapter {
|
|
||||||
canHandle(config: RequestConfig): boolean {
|
|
||||||
// Fetch handles non-proxy requests and HTTP/HTTPS proxies
|
|
||||||
if (typeof config.proxy === 'string') {
|
|
||||||
return config.proxy.startsWith('http');
|
|
||||||
}
|
|
||||||
return !config.proxy || config.proxy.protocol === 'http' || config.proxy.protocol === 'https';
|
|
||||||
}
|
|
||||||
|
|
||||||
async request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
|
|
||||||
const { url, method = 'GET', headers, data, proxy } = config;
|
|
||||||
|
|
||||||
// Prepare fetch options
|
|
||||||
const fetchOptions: RequestInit = {
|
|
||||||
method,
|
|
||||||
headers,
|
|
||||||
signal,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Add body for non-GET requests
|
|
||||||
if (data && method !== 'GET') {
|
|
||||||
fetchOptions.body = typeof data === 'string' ? data : JSON.stringify(data);
|
|
||||||
if (typeof data === 'object') {
|
|
||||||
fetchOptions.headers = { 'Content-Type': 'application/json', ...fetchOptions.headers };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add proxy if needed (using Bun's built-in proxy support)
|
|
||||||
if (typeof proxy === 'string') {
|
|
||||||
// If proxy is a URL string, use it directly
|
|
||||||
(fetchOptions as any).proxy = proxy;
|
|
||||||
} else if (proxy) {
|
|
||||||
// If proxy is a ProxyInfo object, create a proxy URL
|
|
||||||
(fetchOptions as any).proxy = ProxyManager.createProxyUrl(proxy);
|
|
||||||
}
|
|
||||||
const response = await fetch(url, fetchOptions);
|
|
||||||
|
|
||||||
// Parse response based on content type
|
|
||||||
let responseData: T;
|
|
||||||
const contentType = response.headers.get('content-type') || '';
|
|
||||||
|
|
||||||
if (contentType.includes('application/json')) {
|
|
||||||
responseData = (await response.json()) as T;
|
|
||||||
} else {
|
|
||||||
responseData = (await response.text()) as T;
|
|
||||||
}
|
|
||||||
|
|
||||||
const httpResponse: HttpResponse<T> = {
|
|
||||||
data: responseData,
|
|
||||||
status: response.status,
|
|
||||||
headers: Object.fromEntries(response.headers.entries()),
|
|
||||||
ok: response.ok,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Throw HttpError for non-2xx status codes
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new HttpError(
|
|
||||||
`Request failed with status ${response.status}`,
|
|
||||||
response.status,
|
|
||||||
httpResponse
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return httpResponse;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
export * from './types';
|
|
||||||
export * from './fetch-adapter';
|
|
||||||
export * from './axios-adapter';
|
|
||||||
export * from './factory';
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
import type { HttpResponse, RequestConfig } from '../types';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Request adapter interface for different HTTP implementations
|
|
||||||
*/
|
|
||||||
export interface RequestAdapter {
|
|
||||||
/**
|
|
||||||
* Execute an HTTP request
|
|
||||||
*/
|
|
||||||
request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>>;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if this adapter can handle the given configuration
|
|
||||||
*/
|
|
||||||
canHandle(config: RequestConfig): boolean;
|
|
||||||
}
|
|
||||||
|
|
@ -1,183 +0,0 @@
|
||||||
import type { Logger } from '@stock-bot/logger';
|
|
||||||
import { AdapterFactory } from './adapters/index';
|
|
||||||
import type { HttpClientConfig, HttpResponse, RequestConfig } from './types';
|
|
||||||
import { HttpError } from './types';
|
|
||||||
import { getRandomUserAgent } from './user-agent';
|
|
||||||
|
|
||||||
export class HttpClient {
|
|
||||||
private readonly config: HttpClientConfig;
|
|
||||||
private readonly logger?: Logger;
|
|
||||||
|
|
||||||
constructor(config: HttpClientConfig = {}, logger?: Logger) {
|
|
||||||
this.config = config;
|
|
||||||
this.logger = logger?.child('http-client');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convenience methods
|
|
||||||
async get<T = any>(
|
|
||||||
url: string,
|
|
||||||
config: Omit<RequestConfig, 'method' | 'url'> = {}
|
|
||||||
): Promise<HttpResponse<T>> {
|
|
||||||
return this.request<T>({ ...config, method: 'GET', url });
|
|
||||||
}
|
|
||||||
|
|
||||||
async post<T = any>(
|
|
||||||
url: string,
|
|
||||||
data?: unknown,
|
|
||||||
config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}
|
|
||||||
): Promise<HttpResponse<T>> {
|
|
||||||
return this.request<T>({ ...config, method: 'POST', url, data });
|
|
||||||
}
|
|
||||||
|
|
||||||
async put<T = any>(
|
|
||||||
url: string,
|
|
||||||
data?: unknown,
|
|
||||||
config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}
|
|
||||||
): Promise<HttpResponse<T>> {
|
|
||||||
return this.request<T>({ ...config, method: 'PUT', url, data });
|
|
||||||
}
|
|
||||||
|
|
||||||
async del<T = any>(
|
|
||||||
url: string,
|
|
||||||
config: Omit<RequestConfig, 'method' | 'url'> = {}
|
|
||||||
): Promise<HttpResponse<T>> {
|
|
||||||
return this.request<T>({ ...config, method: 'DELETE', url });
|
|
||||||
}
|
|
||||||
|
|
||||||
async patch<T = any>(
|
|
||||||
url: string,
|
|
||||||
data?: unknown,
|
|
||||||
config: Omit<RequestConfig, 'method' | 'url' | 'data'> = {}
|
|
||||||
): Promise<HttpResponse<T>> {
|
|
||||||
return this.request<T>({ ...config, method: 'PATCH', url, data });
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Main request method - clean and simple
|
|
||||||
*/
|
|
||||||
async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
|
|
||||||
const finalConfig = this.mergeConfig(config);
|
|
||||||
const startTime = Date.now();
|
|
||||||
|
|
||||||
this.logger?.debug('Making HTTP request', {
|
|
||||||
method: finalConfig.method,
|
|
||||||
url: finalConfig.url,
|
|
||||||
hasProxy: !!finalConfig.proxy,
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
const response = await this.executeRequest<T>(finalConfig);
|
|
||||||
response.responseTime = Date.now() - startTime;
|
|
||||||
|
|
||||||
this.logger?.debug('HTTP request successful', {
|
|
||||||
method: finalConfig.method,
|
|
||||||
url: finalConfig.url,
|
|
||||||
status: response.status,
|
|
||||||
responseTime: response.responseTime,
|
|
||||||
});
|
|
||||||
|
|
||||||
return response;
|
|
||||||
} catch (error) {
|
|
||||||
if (this.logger?.getServiceName() === 'proxy-tasks') {
|
|
||||||
this.logger?.debug('HTTP request failed', {
|
|
||||||
method: finalConfig.method,
|
|
||||||
url: finalConfig.url,
|
|
||||||
error: (error as Error).message,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
this.logger?.warn('HTTP request failed', {
|
|
||||||
method: finalConfig.method,
|
|
||||||
url: finalConfig.url,
|
|
||||||
error: (error as Error).message,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute request with timeout handling - no race conditions
|
|
||||||
*/ private async executeRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
|
|
||||||
const timeout = config.timeout ?? this.config.timeout ?? 30000;
|
|
||||||
const controller = new AbortController();
|
|
||||||
const startTime = Date.now();
|
|
||||||
let timeoutId: NodeJS.Timeout | undefined;
|
|
||||||
|
|
||||||
// Set up timeout
|
|
||||||
// Create a timeout promise that will reject
|
|
||||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
||||||
timeoutId = setTimeout(() => {
|
|
||||||
const elapsed = Date.now() - startTime;
|
|
||||||
this.logger?.warn('Request timeout triggered', {
|
|
||||||
url: config.url,
|
|
||||||
method: config.method,
|
|
||||||
timeout,
|
|
||||||
elapsed,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Attempt to abort (may or may not work with Bun)
|
|
||||||
controller.abort();
|
|
||||||
|
|
||||||
// Force rejection regardless of signal behavior
|
|
||||||
reject(new HttpError(`Request timeout after ${timeout}ms (elapsed: ${elapsed}ms)`));
|
|
||||||
}, timeout);
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Get the appropriate adapter
|
|
||||||
const adapter = AdapterFactory.getAdapter(config);
|
|
||||||
|
|
||||||
const response = await Promise.race([
|
|
||||||
adapter.request<T>(config, controller.signal),
|
|
||||||
timeoutPromise,
|
|
||||||
]);
|
|
||||||
|
|
||||||
this.logger?.debug('Adapter request successful', {
|
|
||||||
url: config.url,
|
|
||||||
elapsedMs: Date.now() - startTime,
|
|
||||||
});
|
|
||||||
// Clear timeout on success
|
|
||||||
clearTimeout(timeoutId);
|
|
||||||
|
|
||||||
return response;
|
|
||||||
} catch (error) {
|
|
||||||
const elapsed = Date.now() - startTime;
|
|
||||||
this.logger?.debug('Adapter request failed', {
|
|
||||||
url: config.url,
|
|
||||||
elapsedMs: elapsed,
|
|
||||||
});
|
|
||||||
clearTimeout(timeoutId);
|
|
||||||
|
|
||||||
// Handle timeout
|
|
||||||
if (controller.signal.aborted) {
|
|
||||||
throw new HttpError(`Request timeout after ${timeout}ms`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Re-throw other errors
|
|
||||||
if (error instanceof HttpError) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new HttpError(`Request failed: ${(error as Error).message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Merge configs with defaults
|
|
||||||
*/
|
|
||||||
private mergeConfig(config: RequestConfig): RequestConfig {
|
|
||||||
// Merge headers with automatic User-Agent assignment
|
|
||||||
const mergedHeaders = { ...this.config.headers, ...config.headers };
|
|
||||||
|
|
||||||
// Add random User-Agent if not specified
|
|
||||||
if (!mergedHeaders['User-Agent'] && !mergedHeaders['user-agent']) {
|
|
||||||
mergedHeaders['User-Agent'] = getRandomUserAgent();
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
...config,
|
|
||||||
headers: mergedHeaders,
|
|
||||||
timeout: config.timeout ?? this.config.timeout,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
// Re-export all types and classes
|
|
||||||
export * from './adapters/index';
|
|
||||||
export * from './client';
|
|
||||||
export * from './proxy-manager';
|
|
||||||
export * from './types';
|
|
||||||
export * from './user-agent';
|
|
||||||
|
|
||||||
// Default export
|
|
||||||
export { HttpClient as default } from './client';
|
|
||||||
|
|
@ -1,65 +0,0 @@
|
||||||
import { AxiosRequestConfig } from 'axios';
|
|
||||||
import { HttpProxyAgent } from 'http-proxy-agent';
|
|
||||||
import { HttpsProxyAgent } from 'https-proxy-agent';
|
|
||||||
import { SocksProxyAgent } from 'socks-proxy-agent';
|
|
||||||
import type { ProxyInfo } from './types';
|
|
||||||
|
|
||||||
export class ProxyManager {
|
|
||||||
/**
|
|
||||||
* Determine if we should use Bun fetch (HTTP/HTTPS) or Axios (SOCKS)
|
|
||||||
*/
|
|
||||||
static shouldUseBunFetch(proxy: ProxyInfo): boolean {
|
|
||||||
return proxy.protocol === 'http' || proxy.protocol === 'https';
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Create proxy URL for both Bun fetch and Axios proxy agents
|
|
||||||
*/
|
|
||||||
static createProxyUrl(proxy: ProxyInfo): string {
|
|
||||||
const { protocol, host, port, username, password } = proxy;
|
|
||||||
if (username && password) {
|
|
||||||
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
|
||||||
}
|
|
||||||
return `${protocol}://${host}:${port}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create appropriate agent for Axios based on proxy type
|
|
||||||
*/
|
|
||||||
static createProxyAgent(proxy: ProxyInfo) {
|
|
||||||
this.validateConfig(proxy);
|
|
||||||
|
|
||||||
const proxyUrl = this.createProxyUrl(proxy);
|
|
||||||
switch (proxy.protocol) {
|
|
||||||
case 'socks4':
|
|
||||||
case 'socks5':
|
|
||||||
return new SocksProxyAgent(proxyUrl);
|
|
||||||
case 'http':
|
|
||||||
return new HttpProxyAgent(proxyUrl);
|
|
||||||
case 'https':
|
|
||||||
return new HttpsProxyAgent(proxyUrl);
|
|
||||||
default:
|
|
||||||
throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Create Axios instance with proxy configuration
|
|
||||||
*/
|
|
||||||
static createAxiosConfig(proxy: ProxyInfo): AxiosRequestConfig {
|
|
||||||
const agent = this.createProxyAgent(proxy);
|
|
||||||
return {
|
|
||||||
httpAgent: agent,
|
|
||||||
httpsAgent: agent,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Simple proxy config validation
|
|
||||||
*/
|
|
||||||
static validateConfig(proxy: ProxyInfo): void {
|
|
||||||
if (!proxy.host || !proxy.port) {
|
|
||||||
throw new Error('Proxy host and port are required');
|
|
||||||
}
|
|
||||||
if (!['http', 'https', 'socks4', 'socks5'].includes(proxy.protocol)) {
|
|
||||||
throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,55 +0,0 @@
|
||||||
// Minimal types for fast HTTP client
|
|
||||||
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH';
|
|
||||||
|
|
||||||
export interface ProxyInfo {
|
|
||||||
source?: string;
|
|
||||||
protocol: 'http' | 'https' | 'socks4' | 'socks5';
|
|
||||||
host: string;
|
|
||||||
port: number;
|
|
||||||
username?: string;
|
|
||||||
password?: string;
|
|
||||||
url?: string; // Full proxy URL for adapters
|
|
||||||
isWorking?: boolean;
|
|
||||||
responseTime?: number;
|
|
||||||
error?: string;
|
|
||||||
// Enhanced tracking properties
|
|
||||||
working?: number; // Number of successful checks
|
|
||||||
total?: number; // Total number of checks
|
|
||||||
successRate?: number; // Success rate percentage
|
|
||||||
averageResponseTime?: number; // Average response time in milliseconds
|
|
||||||
firstSeen?: Date; // When the proxy was first added to cache
|
|
||||||
lastChecked?: Date; // When the proxy was last checked
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface HttpClientConfig {
|
|
||||||
timeout?: number;
|
|
||||||
headers?: Record<string, string>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface RequestConfig {
|
|
||||||
method?: HttpMethod;
|
|
||||||
url: string;
|
|
||||||
headers?: Record<string, string>;
|
|
||||||
data?: unknown; // Changed from 'body' to 'data' for consistency
|
|
||||||
timeout?: number;
|
|
||||||
proxy?: ProxyInfo | string; // Proxy can be a ProxyInfo object or a URL string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface HttpResponse<T = any> {
|
|
||||||
data: T;
|
|
||||||
status: number;
|
|
||||||
headers: Record<string, string>;
|
|
||||||
ok: boolean;
|
|
||||||
responseTime?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class HttpError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public status?: number,
|
|
||||||
public response?: HttpResponse
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = 'HttpError';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
import UserAgent from 'user-agents';
|
|
||||||
|
|
||||||
export function getRandomUserAgent(): string {
|
|
||||||
const userAgent = new UserAgent();
|
|
||||||
return userAgent.toString();
|
|
||||||
}
|
|
||||||
|
|
@ -1,161 +0,0 @@
|
||||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
|
||||||
import { HttpClient, HttpError } from '../src/index';
|
|
||||||
import { MockServer } from './mock-server';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Integration tests for HTTP client with real network scenarios
|
|
||||||
* These tests use external services and may be affected by network conditions
|
|
||||||
*/
|
|
||||||
|
|
||||||
let mockServer: MockServer;
|
|
||||||
let mockServerBaseUrl: string;
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
mockServer = new MockServer();
|
|
||||||
await mockServer.start();
|
|
||||||
mockServerBaseUrl = mockServer.getBaseUrl();
|
|
||||||
});
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
await mockServer.stop();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('HTTP Integration Tests', () => {
|
|
||||||
let client: HttpClient;
|
|
||||||
|
|
||||||
beforeAll(() => {
|
|
||||||
client = new HttpClient({
|
|
||||||
timeout: 10000,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Real-world scenarios', () => {
|
|
||||||
test('should handle JSON API responses', async () => {
|
|
||||||
try {
|
|
||||||
const response = await client.get('https://jsonplaceholder.typicode.com/posts/1');
|
|
||||||
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(response.data).toHaveProperty('id');
|
|
||||||
expect(response.data).toHaveProperty('title');
|
|
||||||
expect(response.data).toHaveProperty('body');
|
|
||||||
} catch (error) {
|
|
||||||
console.warn('External API test skipped due to network issues:', (error as Error).message);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle large responses', async () => {
|
|
||||||
try {
|
|
||||||
const response = await client.get('https://jsonplaceholder.typicode.com/posts');
|
|
||||||
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(Array.isArray(response.data)).toBe(true);
|
|
||||||
expect(response.data.length).toBeGreaterThan(0);
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(
|
|
||||||
'Large response test skipped due to network issues:',
|
|
||||||
(error as Error).message
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle POST with JSON data', async () => {
|
|
||||||
try {
|
|
||||||
const postData = {
|
|
||||||
title: 'Integration Test Post',
|
|
||||||
body: 'This is a test post from integration tests',
|
|
||||||
userId: 1,
|
|
||||||
};
|
|
||||||
|
|
||||||
const response = await client.post('https://jsonplaceholder.typicode.com/posts', postData);
|
|
||||||
|
|
||||||
expect(response.status).toBe(201);
|
|
||||||
expect(response.data).toHaveProperty('id');
|
|
||||||
expect(response.data.title).toBe(postData.title);
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(
|
|
||||||
'POST integration test skipped due to network issues:',
|
|
||||||
(error as Error).message
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Error scenarios with mock server', () => {
|
|
||||||
test('should handle various HTTP status codes', async () => {
|
|
||||||
const successCodes = [200, 201];
|
|
||||||
const errorCodes = [400, 401, 403, 404, 500, 503];
|
|
||||||
|
|
||||||
// Test success codes
|
|
||||||
for (const statusCode of successCodes) {
|
|
||||||
const response = await client.get(`${mockServerBaseUrl}/status/${statusCode}`);
|
|
||||||
expect(response.status).toBe(statusCode);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test error codes (should throw HttpError)
|
|
||||||
for (const statusCode of errorCodes) {
|
|
||||||
await expect(client.get(`${mockServerBaseUrl}/status/${statusCode}`)).rejects.toThrow(
|
|
||||||
HttpError
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle malformed responses gracefully', async () => {
|
|
||||||
// Mock server returns valid JSON, so this test verifies our client handles it properly
|
|
||||||
const response = await client.get(`${mockServerBaseUrl}/`);
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(typeof response.data).toBe('object');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle concurrent requests', async () => {
|
|
||||||
const requests = Array.from({ length: 5 }, (_, i) =>
|
|
||||||
client.get(`${mockServerBaseUrl}/`, {
|
|
||||||
headers: { 'X-Request-ID': `req-${i}` },
|
|
||||||
})
|
|
||||||
);
|
|
||||||
|
|
||||||
const responses = await Promise.all(requests);
|
|
||||||
|
|
||||||
responses.forEach((response, index) => {
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(response.data.headers).toHaveProperty('x-request-id', `req-${index}`);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Performance and reliability', () => {
|
|
||||||
test('should handle rapid sequential requests', async () => {
|
|
||||||
const startTime = Date.now();
|
|
||||||
const requests = [];
|
|
||||||
|
|
||||||
for (let i = 0; i < 10; i++) {
|
|
||||||
requests.push(client.get(`${mockServerBaseUrl}/`));
|
|
||||||
}
|
|
||||||
|
|
||||||
const responses = await Promise.all(requests);
|
|
||||||
const endTime = Date.now();
|
|
||||||
|
|
||||||
expect(responses).toHaveLength(10);
|
|
||||||
responses.forEach(response => {
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`Completed 10 requests in ${endTime - startTime}ms`);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should maintain connection efficiency', async () => {
|
|
||||||
const clientWithKeepAlive = new HttpClient({
|
|
||||||
timeout: 5000,
|
|
||||||
});
|
|
||||||
|
|
||||||
const requests = Array.from({ length: 3 }, () =>
|
|
||||||
clientWithKeepAlive.get(`${mockServerBaseUrl}/`)
|
|
||||||
);
|
|
||||||
|
|
||||||
const responses = await Promise.all(requests);
|
|
||||||
|
|
||||||
responses.forEach(response => {
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,155 +0,0 @@
|
||||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
|
||||||
import { HttpClient, HttpError, ProxyManager } from '../src/index';
|
|
||||||
import type { ProxyInfo } from '../src/types';
|
|
||||||
import { MockServer } from './mock-server';
|
|
||||||
|
|
||||||
// Global mock server instance
|
|
||||||
let mockServer: MockServer;
|
|
||||||
let mockServerBaseUrl: string;
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
// Start mock server for all tests
|
|
||||||
mockServer = new MockServer();
|
|
||||||
await mockServer.start();
|
|
||||||
mockServerBaseUrl = mockServer.getBaseUrl();
|
|
||||||
});
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
// Stop mock server
|
|
||||||
await mockServer.stop();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('HttpClient', () => {
|
|
||||||
let client: HttpClient;
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
client = new HttpClient();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Basic functionality', () => {
|
|
||||||
test('should create client with default config', () => {
|
|
||||||
expect(client).toBeInstanceOf(HttpClient);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should make GET request', async () => {
|
|
||||||
const response = await client.get(`${mockServerBaseUrl}/`);
|
|
||||||
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(response.data).toHaveProperty('url');
|
|
||||||
expect(response.data).toHaveProperty('method', 'GET');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should make POST request with body', async () => {
|
|
||||||
const testData = {
|
|
||||||
title: 'Test Post',
|
|
||||||
body: 'Test body',
|
|
||||||
userId: 1,
|
|
||||||
};
|
|
||||||
|
|
||||||
const response = await client.post(`${mockServerBaseUrl}/post`, testData);
|
|
||||||
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(response.data).toHaveProperty('data');
|
|
||||||
expect(response.data.data).toEqual(testData);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle custom headers', async () => {
|
|
||||||
const customHeaders = {
|
|
||||||
'X-Custom-Header': 'test-value',
|
|
||||||
'User-Agent': 'StockBot-HTTP-Client/1.0',
|
|
||||||
};
|
|
||||||
|
|
||||||
const response = await client.get(`${mockServerBaseUrl}/headers`, {
|
|
||||||
headers: customHeaders,
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(response.data.headers).toHaveProperty('x-custom-header', 'test-value');
|
|
||||||
expect(response.data.headers).toHaveProperty('user-agent', 'StockBot-HTTP-Client/1.0');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle timeout', async () => {
|
|
||||||
const clientWithTimeout = new HttpClient({ timeout: 1 }); // 1ms timeout
|
|
||||||
|
|
||||||
await expect(clientWithTimeout.get('https://httpbin.org/delay/1')).rejects.toThrow();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
describe('Error handling', () => {
|
|
||||||
test('should handle HTTP errors', async () => {
|
|
||||||
await expect(client.get(`${mockServerBaseUrl}/status/404`)).rejects.toThrow(HttpError);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle network errors gracefully', async () => {
|
|
||||||
await expect(
|
|
||||||
client.get('https://nonexistent-domain-that-will-fail-12345.test')
|
|
||||||
).rejects.toThrow();
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should handle invalid URLs', async () => {
|
|
||||||
await expect(client.get('not:/a:valid/url')).rejects.toThrow();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('HTTP methods', () => {
|
|
||||||
test('should make PUT request', async () => {
|
|
||||||
const testData = { id: 1, name: 'Updated' };
|
|
||||||
const response = await client.put(`${mockServerBaseUrl}/post`, testData);
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should make DELETE request', async () => {
|
|
||||||
const response = await client.del(`${mockServerBaseUrl}/`);
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
expect(response.data.method).toBe('DELETE');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should make PATCH request', async () => {
|
|
||||||
const testData = { name: 'Patched' };
|
|
||||||
const response = await client.patch(`${mockServerBaseUrl}/post`, testData);
|
|
||||||
expect(response.status).toBe(200);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('ProxyManager', () => {
|
|
||||||
test('should determine when to use Bun fetch', () => {
|
|
||||||
const httpProxy: ProxyInfo = {
|
|
||||||
protocol: 'http',
|
|
||||||
host: 'proxy.example.com',
|
|
||||||
port: 8080,
|
|
||||||
};
|
|
||||||
|
|
||||||
const socksProxy: ProxyInfo = {
|
|
||||||
protocol: 'socks5',
|
|
||||||
host: 'proxy.example.com',
|
|
||||||
port: 1080,
|
|
||||||
};
|
|
||||||
|
|
||||||
expect(ProxyManager.shouldUseBunFetch(httpProxy)).toBe(true);
|
|
||||||
expect(ProxyManager.shouldUseBunFetch(socksProxy)).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should create proxy URL for Bun fetch', () => {
|
|
||||||
const proxy: ProxyInfo = {
|
|
||||||
protocol: 'http',
|
|
||||||
host: 'proxy.example.com',
|
|
||||||
port: 8080,
|
|
||||||
username: 'user',
|
|
||||||
password: 'pass',
|
|
||||||
};
|
|
||||||
|
|
||||||
const proxyUrl = ProxyManager.createProxyUrl(proxy);
|
|
||||||
expect(proxyUrl).toBe('http://user:pass@proxy.example.com:8080');
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should create proxy URL without credentials', () => {
|
|
||||||
const proxy: ProxyInfo = {
|
|
||||||
protocol: 'https',
|
|
||||||
host: 'proxy.example.com',
|
|
||||||
port: 8080,
|
|
||||||
};
|
|
||||||
|
|
||||||
const proxyUrl = ProxyManager.createProxyUrl(proxy);
|
|
||||||
expect(proxyUrl).toBe('https://proxy.example.com:8080');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,132 +0,0 @@
|
||||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
|
||||||
import { MockServer } from './mock-server';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Tests for the MockServer utility
|
|
||||||
* Ensures our test infrastructure works correctly
|
|
||||||
*/
|
|
||||||
|
|
||||||
describe('MockServer', () => {
|
|
||||||
let mockServer: MockServer;
|
|
||||||
let baseUrl: string;
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
mockServer = new MockServer();
|
|
||||||
await mockServer.start();
|
|
||||||
baseUrl = mockServer.getBaseUrl();
|
|
||||||
});
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
await mockServer.stop();
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Server lifecycle', () => {
|
|
||||||
test('should start and provide base URL', () => {
|
|
||||||
expect(baseUrl).toMatch(/^http:\/\/localhost:\d+$/);
|
|
||||||
expect(mockServer.getBaseUrl()).toBe(baseUrl);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should be reachable', async () => {
|
|
||||||
const response = await fetch(`${baseUrl}/`);
|
|
||||||
expect(response.ok).toBe(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Status endpoints', () => {
|
|
||||||
test('should return correct status codes', async () => {
|
|
||||||
const statusCodes = [200, 201, 400, 401, 403, 404, 500, 503];
|
|
||||||
|
|
||||||
for (const status of statusCodes) {
|
|
||||||
const response = await fetch(`${baseUrl}/status/${status}`);
|
|
||||||
expect(response.status).toBe(status);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Headers endpoint', () => {
|
|
||||||
test('should echo request headers', async () => {
|
|
||||||
const response = await fetch(`${baseUrl}/headers`, {
|
|
||||||
headers: {
|
|
||||||
'X-Test-Header': 'test-value',
|
|
||||||
'User-Agent': 'MockServer-Test',
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(response.ok).toBe(true);
|
|
||||||
const data = await response.json();
|
|
||||||
expect(data.headers).toHaveProperty('x-test-header', 'test-value');
|
|
||||||
expect(data.headers).toHaveProperty('user-agent', 'MockServer-Test');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Basic auth endpoint', () => {
|
|
||||||
test('should authenticate valid credentials', async () => {
|
|
||||||
const username = 'testuser';
|
|
||||||
const password = 'testpass';
|
|
||||||
const credentials = btoa(`${username}:${password}`);
|
|
||||||
|
|
||||||
const response = await fetch(`${baseUrl}/basic-auth/${username}/${password}`, {
|
|
||||||
headers: {
|
|
||||||
Authorization: `Basic ${credentials}`,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(response.ok).toBe(true);
|
|
||||||
const data = await response.json();
|
|
||||||
expect(data.authenticated).toBe(true);
|
|
||||||
expect(data.user).toBe(username);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should reject invalid credentials', async () => {
|
|
||||||
const credentials = btoa('wrong:credentials');
|
|
||||||
|
|
||||||
const response = await fetch(`${baseUrl}/basic-auth/user/pass`, {
|
|
||||||
headers: {
|
|
||||||
Authorization: `Basic ${credentials}`,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(response.status).toBe(401);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('should reject missing auth header', async () => {
|
|
||||||
const response = await fetch(`${baseUrl}/basic-auth/user/pass`);
|
|
||||||
expect(response.status).toBe(401);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('POST endpoint', () => {
|
|
||||||
test('should echo POST data', async () => {
|
|
||||||
const testData = {
|
|
||||||
message: 'Hello, MockServer!',
|
|
||||||
timestamp: Date.now(),
|
|
||||||
};
|
|
||||||
|
|
||||||
const response = await fetch(`${baseUrl}/post`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify(testData),
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(response.ok).toBe(true);
|
|
||||||
const data = await response.json();
|
|
||||||
expect(data.data).toEqual(testData);
|
|
||||||
expect(data.method).toBe('POST');
|
|
||||||
expect(data.headers).toHaveProperty('content-type', 'application/json');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('Default endpoint', () => {
|
|
||||||
test('should return request information', async () => {
|
|
||||||
const response = await fetch(`${baseUrl}/unknown-endpoint`);
|
|
||||||
|
|
||||||
expect(response.ok).toBe(true);
|
|
||||||
const data = await response.json();
|
|
||||||
expect(data.url).toBe(`${baseUrl}/unknown-endpoint`);
|
|
||||||
expect(data.method).toBe('GET');
|
|
||||||
expect(data.headers).toBeDefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,116 +0,0 @@
|
||||||
/**
|
|
||||||
* Mock HTTP server for testing the HTTP client
|
|
||||||
* Replaces external dependency on httpbin.org with a local server
|
|
||||||
*/
|
|
||||||
export class MockServer {
|
|
||||||
private server: ReturnType<typeof Bun.serve> | null = null;
|
|
||||||
private port: number = 0;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start the mock server on a random port
|
|
||||||
*/
|
|
||||||
async start(): Promise<void> {
|
|
||||||
this.server = Bun.serve({
|
|
||||||
port: 1, // Use any available port
|
|
||||||
fetch: this.handleRequest.bind(this),
|
|
||||||
error: this.handleError.bind(this),
|
|
||||||
});
|
|
||||||
|
|
||||||
this.port = this.server.port || 1;
|
|
||||||
console.log(`Mock server started on port ${this.port}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stop the mock server
|
|
||||||
*/
|
|
||||||
async stop(): Promise<void> {
|
|
||||||
if (this.server) {
|
|
||||||
this.server.stop(true);
|
|
||||||
this.server = null;
|
|
||||||
this.port = 0;
|
|
||||||
console.log('Mock server stopped');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the base URL of the mock server
|
|
||||||
*/
|
|
||||||
getBaseUrl(): string {
|
|
||||||
if (!this.server) {
|
|
||||||
throw new Error('Server not started');
|
|
||||||
}
|
|
||||||
return `http://localhost:${this.port}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle incoming requests
|
|
||||||
*/ private async handleRequest(req: Request): Promise<Response> {
|
|
||||||
const url = new URL(req.url);
|
|
||||||
const path = url.pathname;
|
|
||||||
|
|
||||||
console.log(`Mock server handling request: ${req.method} ${path}`);
|
|
||||||
|
|
||||||
// Status endpoints
|
|
||||||
if (path.startsWith('/status/')) {
|
|
||||||
const status = parseInt(path.replace('/status/', ''), 10);
|
|
||||||
console.log(`Returning status: ${status}`);
|
|
||||||
return new Response(null, { status });
|
|
||||||
} // Headers endpoint
|
|
||||||
if (path === '/headers') {
|
|
||||||
const headers = Object.fromEntries([...req.headers.entries()]);
|
|
||||||
console.log('Headers endpoint called, received headers:', headers);
|
|
||||||
return Response.json({ headers });
|
|
||||||
} // Basic auth endpoint
|
|
||||||
if (path.startsWith('/basic-auth/')) {
|
|
||||||
const parts = path.split('/').filter(Boolean);
|
|
||||||
const expectedUsername = parts[1];
|
|
||||||
const expectedPassword = parts[2];
|
|
||||||
console.log(
|
|
||||||
`Basic auth endpoint called: expected user=${expectedUsername}, pass=${expectedPassword}`
|
|
||||||
);
|
|
||||||
|
|
||||||
const authHeader = req.headers.get('authorization');
|
|
||||||
if (!authHeader || !authHeader.startsWith('Basic ')) {
|
|
||||||
console.log('Missing or invalid Authorization header');
|
|
||||||
return new Response('Unauthorized', { status: 401 });
|
|
||||||
}
|
|
||||||
|
|
||||||
const base64Credentials = authHeader.split(' ')[1];
|
|
||||||
const credentials = atob(base64Credentials);
|
|
||||||
const [username, password] = credentials.split(':');
|
|
||||||
|
|
||||||
if (username === expectedUsername && password === expectedPassword) {
|
|
||||||
return Response.json({
|
|
||||||
authenticated: true,
|
|
||||||
user: username,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Response('Unauthorized', { status: 401 });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Echo request body
|
|
||||||
if (path === '/post' && req.method === 'POST') {
|
|
||||||
const data = await req.json();
|
|
||||||
return Response.json({
|
|
||||||
data,
|
|
||||||
headers: Object.fromEntries([...req.headers.entries()]),
|
|
||||||
method: req.method,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default response
|
|
||||||
return Response.json({
|
|
||||||
url: req.url,
|
|
||||||
method: req.method,
|
|
||||||
headers: Object.fromEntries([...req.headers.entries()]),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handle errors
|
|
||||||
*/
|
|
||||||
private handleError(_error: Error): Response {
|
|
||||||
return new Response('Server error', { status: 500 });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,13 +0,0 @@
|
||||||
{
|
|
||||||
"extends": "../../../tsconfig.json",
|
|
||||||
"compilerOptions": {
|
|
||||||
"outDir": "./dist",
|
|
||||||
"rootDir": "./src",
|
|
||||||
"composite": true
|
|
||||||
},
|
|
||||||
"include": ["src/**/*"],
|
|
||||||
"references": [
|
|
||||||
{ "path": "../../core/logger" },
|
|
||||||
{ "path": "../../core/types" }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -5,16 +5,22 @@
|
||||||
export interface ProxyInfo {
|
export interface ProxyInfo {
|
||||||
host: string;
|
host: string;
|
||||||
port: number;
|
port: number;
|
||||||
protocol: 'http' | 'https' | 'socks4' | 'socks5';
|
protocol: 'http' | 'https'; // Simplified to only support HTTP/HTTPS
|
||||||
username?: string;
|
username?: string;
|
||||||
password?: string;
|
password?: string;
|
||||||
isWorking?: boolean;
|
isWorking?: boolean;
|
||||||
successRate?: number;
|
successRate?: number;
|
||||||
lastChecked?: string;
|
lastChecked?: Date;
|
||||||
lastUsed?: string;
|
lastUsed?: Date;
|
||||||
responseTime?: number;
|
responseTime?: number;
|
||||||
source?: string;
|
source?: string;
|
||||||
country?: string;
|
country?: string;
|
||||||
|
error?: string;
|
||||||
|
// Tracking properties
|
||||||
|
working?: number; // Number of successful checks
|
||||||
|
total?: number; // Total number of checks
|
||||||
|
averageResponseTime?: number; // Average response time in milliseconds
|
||||||
|
firstSeen?: Date; // When the proxy was first added
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ProxyManagerConfig {
|
export interface ProxyManagerConfig {
|
||||||
|
|
|
||||||
27
libs/utils/src/fetch.ts
Normal file
27
libs/utils/src/fetch.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
/**
|
||||||
|
* Minimal fetch wrapper with automatic debug logging
|
||||||
|
* Drop-in replacement for native fetch with logging support
|
||||||
|
*/
|
||||||
|
|
||||||
|
export function fetch(
|
||||||
|
input: RequestInfo | URL,
|
||||||
|
init?: RequestInit & { logger?: any }
|
||||||
|
): Promise<Response> {
|
||||||
|
const logger = init?.logger || console;
|
||||||
|
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
|
||||||
|
const method = init?.method || 'GET';
|
||||||
|
|
||||||
|
logger.debug('HTTP request', { method, url });
|
||||||
|
|
||||||
|
return globalThis.fetch(input, init).then(response => {
|
||||||
|
logger.debug('HTTP response', {
|
||||||
|
url,
|
||||||
|
status: response.status,
|
||||||
|
ok: response.ok
|
||||||
|
});
|
||||||
|
return response;
|
||||||
|
}).catch(error => {
|
||||||
|
logger.debug('HTTP error', { url, error: error.message });
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
@ -2,3 +2,5 @@ export * from './calculations/index';
|
||||||
export * from './common';
|
export * from './common';
|
||||||
export * from './dateUtils';
|
export * from './dateUtils';
|
||||||
export * from './generic-functions';
|
export * from './generic-functions';
|
||||||
|
export * from './fetch';
|
||||||
|
export * from './user-agent';
|
||||||
|
|
|
||||||
30
libs/utils/src/user-agent.ts
Normal file
30
libs/utils/src/user-agent.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
/**
|
||||||
|
* User Agent utility for generating random user agents
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Simple list of common user agents to avoid external dependency
|
||||||
|
const USER_AGENTS = [
|
||||||
|
// Chrome on Windows
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||||||
|
// Chrome on Mac
|
||||||
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||||
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
||||||
|
// Firefox on Windows
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:119.0) Gecko/20100101 Firefox/119.0',
|
||||||
|
// Firefox on Mac
|
||||||
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/120.0',
|
||||||
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15) Gecko/20100101 Firefox/119.0',
|
||||||
|
// Safari on Mac
|
||||||
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
|
||||||
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
|
||||||
|
// Edge on Windows
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
|
||||||
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
|
||||||
|
];
|
||||||
|
|
||||||
|
export function getRandomUserAgent(): string {
|
||||||
|
const index = Math.floor(Math.random() * USER_AGENTS.length);
|
||||||
|
return USER_AGENTS[index]!;
|
||||||
|
}
|
||||||
|
|
@ -13,7 +13,6 @@
|
||||||
{ "path": "../core/types" },
|
{ "path": "../core/types" },
|
||||||
{ "path": "../data/cache" },
|
{ "path": "../data/cache" },
|
||||||
{ "path": "../core/config" },
|
{ "path": "../core/config" },
|
||||||
{ "path": "../core/logger" },
|
{ "path": "../core/logger" }
|
||||||
{ "path": "../services/http" }
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -43,7 +43,6 @@ libs=(
|
||||||
"data/questdb" # QuestDB client - depends on core libs
|
"data/questdb" # QuestDB client - depends on core libs
|
||||||
|
|
||||||
# Service libraries
|
# Service libraries
|
||||||
"services/http" # HTTP client - depends on core libs
|
|
||||||
"services/event-bus" # Event bus - depends on core libs
|
"services/event-bus" # Event bus - depends on core libs
|
||||||
"services/shutdown" # Shutdown - depends on core libs
|
"services/shutdown" # Shutdown - depends on core libs
|
||||||
"services/browser" # Browser - depends on core libs
|
"services/browser" # Browser - depends on core libs
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue