simplify http-client and switch to bun fetch + undeci

This commit is contained in:
Bojan Kucera 2025-06-06 23:24:22 -04:00
parent 08bb21cee7
commit 7eeccfe8f2
7 changed files with 43 additions and 740 deletions

View file

@ -1,306 +0,0 @@
import type { Logger } from '@stock-bot/logger';
import type {
HttpClientConfig,
RequestConfig,
HttpResponse,
} from './types.js';
import {
HttpError,
TimeoutError,
validateHttpClientConfig,
validateRequestConfig,
} from './types.js';
import { RateLimiter } from './rate-limiter.js';
import { ProxyManager } from './proxy-manager.js';
export class HttpClient {
private readonly config: HttpClientConfig;
private readonly rateLimiter?: RateLimiter;
private readonly logger?: Logger;
constructor(config: Partial<HttpClientConfig> = {}, logger?: Logger) {
// Validate and set default configuration
this.config = validateHttpClientConfig(config);
this.logger = logger;
// Initialize rate limiter if configured
if (this.config.rateLimit) {
this.rateLimiter = new RateLimiter(this.config.rateLimit);
}
// Validate proxy configuration if provided
if (this.config.proxy) {
ProxyManager.validateConfig(this.config.proxy);
}
}
/**
* Make an HTTP request
*/ async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
// Validate request configuration
const validatedConfig = validateRequestConfig(config);
// Merge with default configuration
const finalConfig = this.mergeConfig(validatedConfig);
// Check rate limiting
if (this.rateLimiter) {
await this.rateLimiter.checkRateLimit();
}
this.logger?.debug('Making HTTP request', {
method: finalConfig.method,
url: finalConfig.url
}); let lastError: Error | undefined;
const maxRetries = finalConfig.retries ?? 3;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
const response = await this.executeRequest<T>(finalConfig);
// Record successful request for rate limiting
if (this.rateLimiter) {
this.rateLimiter.recordRequest(true);
}
this.logger?.debug('HTTP request successful', {
method: finalConfig.method,
url: finalConfig.url,
status: response.status,
attempt: attempt + 1,
});
return response;
} catch (error) {
lastError = error as Error;
// Record failed request for rate limiting
if (this.rateLimiter) {
this.rateLimiter.recordRequest(false);
}
this.logger?.warn('HTTP request failed', {
method: finalConfig.method,
url: finalConfig.url,
attempt: attempt + 1,
error: lastError.message,
});
// Don't retry on certain errors
if (error instanceof TimeoutError ||
(error instanceof HttpError && error.status && error.status < 500)) {
break;
}
// Wait before retrying (except on last attempt)
if (attempt < maxRetries) {
const delay = finalConfig.retryDelay ?? 1000;
await this.sleep(delay * Math.pow(2, attempt)); // Exponential backoff
}
}
}
throw lastError;
}
/**
* Convenience methods for common HTTP methods
*/
async get<T = any>(url: string, config: Omit<RequestConfig, 'method' | 'url'> = {}): Promise<HttpResponse<T>> {
return this.request<T>({ ...config, method: 'GET', url });
}
async post<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
return this.request<T>({ ...config, method: 'POST', url, body });
}
async put<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
return this.request<T>({ ...config, method: 'PUT', url, body });
}
async delete<T = any>(url: string, config: Omit<RequestConfig, 'method' | 'url'> = {}): Promise<HttpResponse<T>> {
return this.request<T>({ ...config, method: 'DELETE', url });
}
async patch<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
return this.request<T>({ ...config, method: 'PATCH', url, body });
}
/**
* Execute the actual HTTP request
*/ private async executeRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
const url = this.buildUrl(config.url);
const timeout = config.timeout ?? 30000;
// Create abort controller for timeout
const abortController = new AbortController();
const timeoutId = setTimeout(() => {
abortController.abort();
}, timeout);
try { // Prepare request options
const requestOptions: RequestInit = {
method: config.method,
headers: config.headers ? {...config.headers} : {},
signal: abortController.signal,
};
// Add basic auth if provided
if (config.auth) {
const authValue = `Basic ${Buffer.from(`${config.auth.username}:${config.auth.password}`).toString('base64')}`;
requestOptions.headers = {
...requestOptions.headers,
'Authorization': authValue
};
}
// Add body for non-GET requests
if (config.body && config.method !== 'GET' && config.method !== 'HEAD') {
if (typeof config.body === 'object') {
requestOptions.body = JSON.stringify(config.body);
requestOptions.headers = {
'Content-Type': 'application/json',
...requestOptions.headers,
};
} else {
requestOptions.body = config.body;
}
}
// Add proxy agent if configured
if (this.config.proxy) {
const agent = ProxyManager.createAgent(this.config.proxy);
(requestOptions as any).agent = agent;
console.log('Using proxy agent:', this.config.proxy);
}
// Make the request
const response = await fetch(url, requestOptions);
// Clear timeout
clearTimeout(timeoutId);
// Check if response status is valid
const validateStatus = config.validateStatus ?? this.config.validateStatus ?? this.defaultValidateStatus;
if (!validateStatus(response.status)) {
throw new HttpError(
`Request failed with status ${response.status}`,
response.status,
undefined,
config
);
}
// Parse response
const responseHeaders: Record<string, string> = {};
response.headers.forEach((value, key) => {
responseHeaders[key] = value;
});
let data: T;
const contentType = response.headers.get('content-type') || '';
if (contentType.includes('application/json')) {
data = await response.json();
} else if (contentType.includes('text/')) {
data = await response.text() as any;
} else {
data = await response.arrayBuffer() as any;
}
return {
data,
status: response.status,
statusText: response.statusText,
headers: responseHeaders,
config,
};
} catch (error) {
clearTimeout(timeoutId);
if (abortController.signal.aborted) {
throw new TimeoutError(config, timeout);
}
if (error instanceof HttpError) {
throw error;
}
throw new HttpError(`Request failed: ${(error as Error).message}`, undefined, undefined, config);
}
}
/**
* Merge request config with default config
*/ private mergeConfig(config: RequestConfig): RequestConfig {
return {
...config,
headers: {
...this.config.defaultHeaders,
...config.headers,
},
timeout: config.timeout ?? this.config.timeout ?? 30000,
retries: config.retries ?? this.config.retries ?? 3,
retryDelay: config.retryDelay ?? this.config.retryDelay ?? 1000,
validateStatus: config.validateStatus ?? this.config.validateStatus ?? this.defaultValidateStatus,
};
}
/**
* Build full URL from base URL and request URL
*/
private buildUrl(url: string): string {
// If it's already a full URL, return it as is
if (url.startsWith('http://') || url.startsWith('https://')) {
return url;
}
// If we have a baseURL, combine them
if (this.config.baseURL) {
try {
// Try to use URL constructor
return new URL(url, this.config.baseURL).toString();
} catch (e) {
// Fall back to string concatenation if URL constructor fails
const base = this.config.baseURL.endsWith('/') ? this.config.baseURL : `${this.config.baseURL}/`;
const path = url.startsWith('/') ? url.substring(1) : url;
return `${base}${path}`;
}
}
// No baseURL, so prepend https:// if it's a domain-like string
if (!url.includes('://') && url.includes('.')) {
return `https://${url}`;
}
return url;
}
/**
* Default status validator
*/
private defaultValidateStatus(status: number): boolean {
return status >= 200 && status < 300;
}
/**
* Sleep utility for retries
*/
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Get current rate limiting status
*/
getRateLimitStatus() {
if (!this.rateLimiter) {
return null;
}
return {
currentCount: this.rateLimiter.getCurrentCount(),
maxRequests: this.config.rateLimit!.maxRequests,
windowMs: this.config.rateLimit!.windowMs,
timeUntilReset: this.rateLimiter.getTimeUntilReset(),
};
}
}

View file

@ -1,7 +1,6 @@
// Re-export all types and classes
export * from './types.js';
export * from './client.js';
export * from './rate-limiter.js';
export * from './proxy-manager.js';
// Default export

View file

@ -1,47 +1,51 @@
import { HttpsProxyAgent } from 'https-proxy-agent';
import { SocksProxyAgent } from 'socks-proxy-agent';
import { ProxyAgent } from 'undici';
import type { ProxyConfig } from './types.js';
import { validateProxyConfig } from './types.js';
import { HttpProxyAgent } from 'http-proxy-agent';
export class ProxyManager {
/**
* Create appropriate proxy agent based on configuration
* Determine if we should use Bun fetch (HTTP/HTTPS) or Undici (SOCKS)
*/
static createAgent(proxy: ProxyConfig): HttpsProxyAgent<string> | SocksProxyAgent {
const { protocol, host, port, username, password } = proxy;
static shouldUseBunFetch(proxy: ProxyConfig): boolean {
return proxy.protocol === 'http' || proxy.protocol === 'https';
}
/**
* Create Bun fetch proxy URL for HTTP/HTTPS proxies
*/
static createBunProxyUrl(proxy: ProxyConfig): string {
const { protocol, host, port, username, password } = proxy;
if (username && password) {
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
}
return `${protocol}://${host}:${port}`;
}
/**
* Create Undici proxy agent for SOCKS proxies
*/
static createUndiciAgent(proxy: ProxyConfig): ProxyAgent {
const { protocol, host, port, username, password } = proxy;
let proxyUrl: string;
console.log('Creating proxy agent with config:', {
protocol,
host,
port,
username,
password
});
if (username && password) {
proxyUrl = `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
} else {
proxyUrl = `${protocol}://${host}:${port}`;
}
switch (protocol) {
case 'http':
return new HttpProxyAgent(proxyUrl);
case 'https':
return new HttpsProxyAgent(proxyUrl);
case 'socks4':
case 'socks5':
return new SocksProxyAgent(proxyUrl);
default:
throw new Error(`Unsupported proxy protocol: ${protocol}`);
}
return new ProxyAgent(proxyUrl);
}
/**
* Validate proxy configuration
* Simple proxy config validation
*/
static validateConfig(proxy: ProxyConfig): void {
// Use the centralized validation function
validateProxyConfig(proxy);
if (!proxy.host || !proxy.port) {
throw new Error('Proxy host and port are required');
}
if (!['http', 'https', 'socks4', 'socks5'].includes(proxy.protocol)) {
throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
}
}
}

View file

@ -1,79 +0,0 @@
import type { RateLimitConfig } from './types.js';
import { RateLimitError } from './types.js';
interface RequestRecord {
timestamp: number;
success: boolean;
}
export class RateLimiter {
private requests: RequestRecord[] = [];
private readonly config: RateLimitConfig;
constructor(config: RateLimitConfig) {
this.config = config;
}
/**
* Check if request is allowed based on rate limiting configuration
* @returns Promise that resolves when request can proceed
*/
async checkRateLimit(): Promise<void> {
const now = Date.now();
const windowStart = now - this.config.windowMs;
// Remove old requests outside the window
this.requests = this.requests.filter(req => req.timestamp > windowStart);
// Filter requests based on configuration
const relevantRequests = this.requests.filter(req => {
if (this.config.skipSuccessfulRequests && req.success) {
return false;
}
if (this.config.skipFailedRequests && !req.success) {
return false;
}
return true;
});
if (relevantRequests.length >= this.config.maxRequests) {
const oldestRequest = relevantRequests[0];
const retryAfter = oldestRequest.timestamp + this.config.windowMs - now;
throw new RateLimitError(this.config.maxRequests, this.config.windowMs, retryAfter);
}
}
/**
* Record a request for rate limiting purposes
*/
recordRequest(success: boolean): void {
this.requests.push({
timestamp: Date.now(),
success,
});
}
/**
* Get current request count in the window
*/
getCurrentCount(): number {
const now = Date.now();
const windowStart = now - this.config.windowMs;
return this.requests.filter(req => req.timestamp > windowStart).length;
}
/**
* Get time until next request is allowed
*/
getTimeUntilReset(): number {
if (this.requests.length === 0) {
return 0;
}
const now = Date.now();
const oldestRequest = this.requests[0];
const resetTime = oldestRequest.timestamp + this.config.windowMs;
return Math.max(0, resetTime - now);
}
}

View file

@ -1,7 +1,6 @@
// HTTP Methods
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
// Minimal types for fast HTTP client
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH';
// Proxy configuration
export interface ProxyConfig {
protocol: 'http' | 'https' | 'socks4' | 'socks5';
host: string;
@ -10,352 +9,34 @@ export interface ProxyConfig {
password?: string;
}
// Rate limiting configuration
export interface RateLimitConfig {
maxRequests: number;
windowMs: number;
skipSuccessfulRequests?: boolean;
skipFailedRequests?: boolean;
}
// HTTP client configuration
export interface HttpClientConfig {
baseURL?: string;
timeout?: number;
retries?: number;
retryDelay?: number;
proxy?: ProxyConfig;
rateLimit?: RateLimitConfig;
defaultHeaders?: Record<string, string>;
validateStatus?: (status: number) => boolean;
headers?: Record<string, string>;
}
// Request configuration
export interface RequestConfig {
method?: HttpMethod;
url: string;
headers?: Record<string, string>;
proxy?: ProxyConfig;
body?: any;
timeout?: number;
retries?: number;
retryDelay?: number;
validateStatus?: (status: number) => boolean;
params?: Record<string, any>;
auth?: {
username: string;
password: string;
};
}
// Response type
export interface HttpResponse<T = any> {
data: T;
status: number;
statusText: string;
headers: Record<string, string>;
config: RequestConfig;
ok: boolean;
}
// Error types
export class HttpError extends Error {
constructor(
message: string,
public status?: number,
public response?: HttpResponse,
public config?: RequestConfig
public response?: HttpResponse
) {
super(message);
this.name = 'HttpError';
}
}
export class TimeoutError extends HttpError {
constructor(config: RequestConfig, timeout: number) {
super(`Request timeout after ${timeout}ms`, undefined, undefined, config);
this.name = 'TimeoutError';
}
}
export class RateLimitError extends HttpError {
constructor(
public maxRequests: number,
public windowMs: number,
public retryAfter?: number
) {
super(`Rate limit exceeded: ${maxRequests} requests per ${windowMs}ms`);
this.name = 'RateLimitError';
}
}
// Validation functions
export function validateProxyConfig(config: unknown): ProxyConfig {
if (!config || typeof config !== 'object') {
throw new Error('Proxy configuration must be an object');
}
const proxy = config as Record<string, unknown>;
// Validate protocol
if (!proxy.protocol || !['http', 'https', 'socks4', 'socks5'].includes(proxy.protocol as string)) {
throw new Error('Proxy protocol must be one of: http, https, socks4, socks5');
}
// Validate host
if (!proxy.host || typeof proxy.host !== 'string' || proxy.host.trim().length === 0) {
throw new Error('Proxy host is required');
}
// Validate port
if (typeof proxy.port !== 'number' || proxy.port < 1 || proxy.port > 65535 || !Number.isInteger(proxy.port)) {
throw new Error('Invalid proxy port');
}
// Validate optional auth
if (proxy.username !== undefined && typeof proxy.username !== 'string') {
throw new Error('Proxy username must be a string');
}
if (proxy.password !== undefined && typeof proxy.password !== 'string') {
throw new Error('Proxy password must be a string');
}
return {
protocol: proxy.protocol as ProxyConfig['protocol'],
host: proxy.host.trim(),
port: proxy.port,
username: proxy.username as string | undefined,
password: proxy.password as string | undefined,
};
}
export function validateRateLimitConfig(config: unknown): RateLimitConfig {
if (!config || typeof config !== 'object') {
throw new Error('Rate limit configuration must be an object');
}
const rateLimit = config as Record<string, unknown>;
// Validate maxRequests
if (typeof rateLimit.maxRequests !== 'number' || rateLimit.maxRequests < 1 || !Number.isInteger(rateLimit.maxRequests)) {
throw new Error('maxRequests must be a positive integer');
}
// Validate windowMs
if (typeof rateLimit.windowMs !== 'number' || rateLimit.windowMs < 1 || !Number.isInteger(rateLimit.windowMs)) {
throw new Error('windowMs must be a positive integer');
}
// Validate optional booleans
if (rateLimit.skipSuccessfulRequests !== undefined && typeof rateLimit.skipSuccessfulRequests !== 'boolean') {
throw new Error('skipSuccessfulRequests must be a boolean');
}
if (rateLimit.skipFailedRequests !== undefined && typeof rateLimit.skipFailedRequests !== 'boolean') {
throw new Error('skipFailedRequests must be a boolean');
}
return {
maxRequests: rateLimit.maxRequests,
windowMs: rateLimit.windowMs,
skipSuccessfulRequests: rateLimit.skipSuccessfulRequests ?? false,
skipFailedRequests: rateLimit.skipFailedRequests ?? false,
};
}
export function validateHttpClientConfig(config: unknown = {}): HttpClientConfig {
if (!config || typeof config !== 'object') {
config = {};
}
const cfg = config as Record<string, unknown>;
// Validate baseURL
let baseURL: string | undefined;
if (cfg.baseURL !== undefined) {
if (typeof cfg.baseURL !== 'string') {
throw new Error('baseURL must be a string');
}
// Simple URL validation - just ensure it starts with http:// or https://
if (cfg.baseURL && !cfg.baseURL.startsWith('http://') && !cfg.baseURL.startsWith('https://')) {
baseURL = `https://${cfg.baseURL}`;
} else {
baseURL = cfg.baseURL;
}
}
// Validate timeout
const timeout = cfg.timeout !== undefined
? validatePositiveNumber(cfg.timeout, 'timeout')
: 30000;
// Validate retries
const retries = cfg.retries !== undefined
? validateNonNegativeInteger(cfg.retries, 'retries')
: 3;
// Validate retryDelay
const retryDelay = cfg.retryDelay !== undefined
? validateNonNegativeNumber(cfg.retryDelay, 'retryDelay')
: 1000;
// Validate headers
let defaultHeaders: Record<string, string> = {};
if (cfg.defaultHeaders !== undefined) {
if (!cfg.defaultHeaders || typeof cfg.defaultHeaders !== 'object') {
throw new Error('defaultHeaders must be an object');
}
defaultHeaders = cfg.defaultHeaders as Record<string, string>;
// Validate header values are strings
for (const [key, value] of Object.entries(defaultHeaders)) {
if (typeof value !== 'string') {
throw new Error(`Header value for '${key}' must be a string`);
}
}
}
// Validate validateStatus function
let validateStatus: ((status: number) => boolean) | undefined;
if (cfg.validateStatus !== undefined) {
if (typeof cfg.validateStatus !== 'function') {
throw new Error('validateStatus must be a function');
}
validateStatus = cfg.validateStatus as (status: number) => boolean;
}
// Validate proxy
let proxy: ProxyConfig | undefined;
if (cfg.proxy !== undefined) {
proxy = validateProxyConfig(cfg.proxy);
}
// Validate rateLimit
let rateLimit: RateLimitConfig | undefined;
if (cfg.rateLimit !== undefined) {
rateLimit = validateRateLimitConfig(cfg.rateLimit);
}
return {
baseURL,
timeout,
retries,
retryDelay,
defaultHeaders,
validateStatus,
proxy,
rateLimit,
};
}
export function validateRequestConfig(config: unknown): RequestConfig {
if (!config || typeof config !== 'object') {
throw new Error('Request configuration must be an object');
}
const req = config as Record<string, unknown>;
// Validate URL (required)
if (!req.url || typeof req.url !== 'string' || req.url.trim().length === 0) {
throw new Error('url is required and must be a non-empty string');
}
// Validate method
const validMethods: HttpMethod[] = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'];
const method: HttpMethod = req.method !== undefined
? validateEnum(req.method, validMethods, 'method') as HttpMethod
: 'GET';
// Validate headers
let headers: Record<string, string> = {};
if (req.headers !== undefined) {
if (typeof req.headers !== 'object' || req.headers === null) {
throw new Error('headers must be an object');
}
headers = req.headers as Record<string, string>;
}
// Validate optional numbers
const timeout = req.timeout !== undefined
? validatePositiveNumber(req.timeout, 'timeout')
: undefined;
const retries = req.retries !== undefined
? validateNonNegativeInteger(req.retries, 'retries')
: undefined;
const retryDelay = req.retryDelay !== undefined
? validateNonNegativeNumber(req.retryDelay, 'retryDelay')
: undefined;
// Validate validateStatus function
let validateStatus: ((status: number) => boolean) | undefined;
if (req.validateStatus !== undefined) {
if (typeof req.validateStatus !== 'function') {
throw new Error('validateStatus must be a function');
}
validateStatus = req.validateStatus as (status: number) => boolean;
}
// Validate params
let params: Record<string, any> | undefined;
if (req.params !== undefined) {
if (typeof req.params !== 'object' || req.params === null) {
throw new Error('params must be an object');
}
params = req.params as Record<string, any>;
}
// Validate auth
let auth: { username: string; password: string } | undefined;
if (req.auth !== undefined) {
if (!req.auth || typeof req.auth !== 'object') {
throw new Error('auth must be an object');
}
const authObj = req.auth as Record<string, unknown>;
if (typeof authObj.username !== 'string' || typeof authObj.password !== 'string') {
throw new Error('auth must have username and password strings');
}
auth = { username: authObj.username, password: authObj.password };
}
return {
method,
url: req.url.trim(),
headers,
body: req.body,
timeout,
retries,
retryDelay,
validateStatus,
params,
auth,
};
}
// Helper validation functions
function validatePositiveNumber(value: unknown, fieldName: string): number {
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
throw new Error(`${fieldName} must be a positive number`);
}
return value;
}
function validateNonNegativeNumber(value: unknown, fieldName: string): number {
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
throw new Error(`${fieldName} must be a non-negative number`);
}
return value;
}
function validateNonNegativeInteger(value: unknown, fieldName: string): number {
if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
throw new Error(`${fieldName} must be a non-negative integer`);
}
return value;
}
function validateEnum<T>(value: unknown, validValues: T[], fieldName: string): T {
if (!validValues.includes(value as T)) {
throw new Error(`${fieldName} must be one of: ${validValues.join(', ')}`);
}
return value as T;
}