simplified httpclient lib
This commit is contained in:
parent
38b523d2c0
commit
f70a8be1cb
6 changed files with 256 additions and 203 deletions
|
|
@ -16,10 +16,10 @@
|
|||
"dependencies": {
|
||||
"@stock-bot/logger": "*",
|
||||
"@stock-bot/types": "*",
|
||||
"got": "^14.4.7",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"undici": "^7.10.0"
|
||||
"socks-proxy-agent": "^8.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.0",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import type {
|
|||
} from './types.js';
|
||||
import { HttpError } from './types.js';
|
||||
import { ProxyManager } from './proxy-manager.js';
|
||||
import { request } from 'undici';
|
||||
import got from 'got';
|
||||
|
||||
export class HttpClient {
|
||||
private readonly config: HttpClientConfig;
|
||||
|
|
@ -15,49 +15,9 @@ export class HttpClient {
|
|||
constructor(config: HttpClientConfig = {}, logger?: Logger) {
|
||||
this.config = config;
|
||||
this.logger = logger;
|
||||
|
||||
// Validate proxy configuration if provided
|
||||
if (this.config.proxy) {
|
||||
ProxyManager.validateConfig(this.config.proxy);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an HTTP request using hybrid approach:
|
||||
* - Bun fetch for HTTP/HTTPS proxies (fastest ~150-300ms)
|
||||
* - Undici for SOCKS proxies (fast ~300-600ms)
|
||||
*/
|
||||
async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const finalConfig = this.mergeConfig(config);
|
||||
|
||||
this.logger?.debug('Making HTTP request', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.executeRequest<T>(finalConfig);
|
||||
|
||||
this.logger?.debug('HTTP request successful', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
status: response.status,
|
||||
});
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
this.logger?.warn('HTTP request failed', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience methods for common HTTP methods
|
||||
*/
|
||||
// Convenience methods
|
||||
async get<T = any>(url: string, config: Omit<RequestConfig, 'method' | 'url'> = {}): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'GET', url });
|
||||
}
|
||||
|
|
@ -77,186 +37,209 @@ export class HttpClient {
|
|||
async patch<T = any>(url: string, body?: any, config: Omit<RequestConfig, 'method' | 'url' | 'body'> = {}): Promise<HttpResponse<T>> {
|
||||
return this.request<T>({ ...config, method: 'PATCH', url, body });
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute HTTP request using hybrid approach
|
||||
* Main request method - unified and simplified
|
||||
*/
|
||||
private async executeRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const timeout = config.timeout ?? this.config.timeout ?? 30000;
|
||||
async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const finalConfig = this.mergeConfig(config);
|
||||
|
||||
// Decide between Bun fetch and Undici based on proxy type
|
||||
if (this.config.proxy) {
|
||||
if (ProxyManager.shouldUseBunFetch(this.config.proxy)) {
|
||||
return this.executeBunRequest<T>(config, timeout);
|
||||
} else {
|
||||
return this.executeUndiciRequest<T>(config, timeout);
|
||||
}
|
||||
} else {
|
||||
// No proxy - use fast Bun fetch
|
||||
return this.executeBunRequest<T>(config, timeout);
|
||||
this.logger?.debug('Making HTTP request', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
hasProxy: !!finalConfig.proxy
|
||||
});
|
||||
|
||||
try {
|
||||
// Single decision point for proxy type - only request-level proxy
|
||||
const proxy = finalConfig.proxy;
|
||||
const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy);
|
||||
|
||||
const response = useBunFetch
|
||||
? await this.fetchRequest<T>(finalConfig)
|
||||
: await this.gotRequest<T>(finalConfig);
|
||||
|
||||
this.logger?.debug('HTTP request successful', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
status: response.status,
|
||||
});
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
this.logger?.warn('HTTP request failed', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute request using Bun's native fetch (fastest for HTTP/HTTPS proxies)
|
||||
* Bun fetch implementation (simplified)
|
||||
*/
|
||||
private async executeBunRequest<T>(config: RequestConfig, timeout: number): Promise<HttpResponse<T>> {
|
||||
const abortController = new AbortController();
|
||||
const timeoutId = setTimeout(() => abortController.abort(), timeout);
|
||||
private async fetchRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const timeout = config.timeout ?? this.config.timeout ?? 30000;
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
||||
|
||||
try {
|
||||
const requestOptions: RequestInit = {
|
||||
method: config.method || 'GET',
|
||||
headers: config.headers || {},
|
||||
signal: abortController.signal,
|
||||
};
|
||||
|
||||
// Add body for non-GET requests
|
||||
if (config.body && config.method !== 'GET') {
|
||||
if (typeof config.body === 'object') {
|
||||
requestOptions.body = JSON.stringify(config.body);
|
||||
requestOptions.headers = {
|
||||
'Content-Type': 'application/json',
|
||||
...requestOptions.headers,
|
||||
};
|
||||
} else {
|
||||
requestOptions.body = config.body;
|
||||
}
|
||||
}
|
||||
|
||||
// Add proxy for Bun fetch (HTTP/HTTPS only)
|
||||
if (this.config.proxy && ProxyManager.shouldUseBunFetch(this.config.proxy)) {
|
||||
const proxyUrl = ProxyManager.createBunProxyUrl(this.config.proxy);
|
||||
// Bun supports proxy via environment or direct configuration
|
||||
(requestOptions as any).proxy = proxyUrl;
|
||||
}
|
||||
|
||||
const response = await fetch(config.url, requestOptions);
|
||||
const options = this.buildFetchOptions(config, controller.signal);
|
||||
const response = await fetch(config.url, options);
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
return await this.parseResponse<T>(response);
|
||||
return this.parseFetchResponse<T>(response);
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
if (abortController.signal.aborted) {
|
||||
throw new HttpError(`Request timeout after ${timeout}ms`);
|
||||
}
|
||||
throw controller.signal.aborted
|
||||
? new HttpError(`Request timeout after ${timeout}ms`)
|
||||
: new HttpError(`Request failed: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Got implementation (simplified for SOCKS proxies)
|
||||
*/
|
||||
private async gotRequest<T>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const timeout = config.timeout ?? this.config.timeout ?? 30000;
|
||||
|
||||
try {
|
||||
const options = this.buildGotOptions(config, timeout);
|
||||
const response = await got(config.url, options);
|
||||
|
||||
return this.parseGotResponse<T>(response);
|
||||
} catch (error) {
|
||||
throw new HttpError(`Request failed: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute request using Undici (fast for SOCKS proxies)
|
||||
* Build fetch options (extracted for clarity)
|
||||
*/
|
||||
private async executeUndiciRequest<T>(config: RequestConfig, timeout: number): Promise<HttpResponse<T>> {
|
||||
try {
|
||||
const requestOptions: any = {
|
||||
method: config.method || 'GET',
|
||||
headers: config.headers || {},
|
||||
headersTimeout: timeout,
|
||||
bodyTimeout: timeout,
|
||||
};
|
||||
private buildFetchOptions(config: RequestConfig, signal: AbortSignal): RequestInit {
|
||||
const options: RequestInit = {
|
||||
method: config.method || 'GET',
|
||||
headers: config.headers || {},
|
||||
signal,
|
||||
};
|
||||
|
||||
// Add body for non-GET requests
|
||||
if (config.body && config.method !== 'GET') {
|
||||
if (typeof config.body === 'object') {
|
||||
requestOptions.body = JSON.stringify(config.body);
|
||||
requestOptions.headers = {
|
||||
'Content-Type': 'application/json',
|
||||
...requestOptions.headers,
|
||||
};
|
||||
} else {
|
||||
requestOptions.body = config.body;
|
||||
}
|
||||
// Add body
|
||||
if (config.body && config.method !== 'GET') {
|
||||
if (typeof config.body === 'object') {
|
||||
options.body = JSON.stringify(config.body);
|
||||
options.headers = { 'Content-Type': 'application/json', ...options.headers };
|
||||
} else {
|
||||
options.body = config.body;
|
||||
}
|
||||
|
||||
// Add SOCKS proxy via Undici agent
|
||||
if (this.config.proxy && !ProxyManager.shouldUseBunFetch(this.config.proxy)) {
|
||||
requestOptions.dispatcher = ProxyManager.createUndiciAgent(this.config.proxy);
|
||||
}
|
||||
console.log('Executing Undici request', requestOptions)
|
||||
|
||||
const response = await request(config.url, requestOptions);
|
||||
|
||||
// Convert Undici response to our format
|
||||
const data = await this.parseUndiciResponse<T>(response);
|
||||
|
||||
return {
|
||||
data,
|
||||
status: response.statusCode,
|
||||
headers: response.headers as Record<string, string>,
|
||||
ok: response.statusCode >= 200 && response.statusCode < 300,
|
||||
};
|
||||
} catch (error) {
|
||||
throw new HttpError(`Undici request failed: ${(error as Error).message}`);
|
||||
}
|
||||
|
||||
// Add proxy (HTTP/HTTPS only) - request level only
|
||||
if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) {
|
||||
(options as any).proxy = ProxyManager.createBunProxyUrl(config.proxy);
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
/**
|
||||
* Build Got options (extracted for clarity)
|
||||
*/
|
||||
private buildGotOptions(config: RequestConfig, timeout: number): any {
|
||||
const options: any = {
|
||||
method: config.method || 'GET',
|
||||
headers: config.headers || {},
|
||||
timeout: {
|
||||
request: timeout,
|
||||
connect: 10000
|
||||
},
|
||||
retry: {
|
||||
limit: 3,
|
||||
methods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH']
|
||||
},
|
||||
throwHttpErrors: false,
|
||||
responseType: 'json'
|
||||
};
|
||||
|
||||
// Add body
|
||||
if (config.body && config.method !== 'GET') {
|
||||
if (typeof config.body === 'object') {
|
||||
options.json = config.body;
|
||||
} else {
|
||||
options.body = config.body;
|
||||
options.headers = { 'Content-Type': 'text/plain', ...options.headers };
|
||||
}
|
||||
}
|
||||
|
||||
// Add SOCKS proxy via agent - request level only
|
||||
if (config.proxy && !ProxyManager.shouldUseBunFetch(config.proxy)) {
|
||||
ProxyManager.validateConfig(config.proxy);
|
||||
const agent = ProxyManager.createGotAgent(config.proxy);
|
||||
options.agent = {
|
||||
http: agent,
|
||||
https: agent
|
||||
};
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse standard fetch response
|
||||
* Parse fetch response (simplified)
|
||||
*/
|
||||
private async parseResponse<T>(response: Response): Promise<HttpResponse<T>> {
|
||||
const responseHeaders: Record<string, string> = {};
|
||||
response.headers.forEach((value, key) => {
|
||||
responseHeaders[key] = value;
|
||||
});
|
||||
|
||||
let data: T;
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
|
||||
if (contentType.includes('application/json')) {
|
||||
data = await response.json();
|
||||
} else if (contentType.includes('text/')) {
|
||||
data = await response.text() as any;
|
||||
} else {
|
||||
data = await response.arrayBuffer() as any;
|
||||
}
|
||||
private async parseFetchResponse<T>(response: Response): Promise<HttpResponse<T>> {
|
||||
const data = await this.parseResponseBody<T>(response);
|
||||
const headers = Object.fromEntries(response.headers.entries());
|
||||
|
||||
if (!response.ok) {
|
||||
throw new HttpError(
|
||||
`Request failed with status ${response.status}`,
|
||||
response.status,
|
||||
{
|
||||
data,
|
||||
status: response.status,
|
||||
headers: responseHeaders,
|
||||
ok: response.ok,
|
||||
}
|
||||
{ data, status: response.status, headers, ok: response.ok }
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
data,
|
||||
status: response.status,
|
||||
headers: responseHeaders,
|
||||
ok: response.ok,
|
||||
};
|
||||
return { data, status: response.status, headers, ok: response.ok };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Undici response
|
||||
* Parse Got response (simplified)
|
||||
*/
|
||||
private async parseUndiciResponse<T>(response: any): Promise<T> {
|
||||
const contentType = response.headers['content-type'] || '';
|
||||
private parseGotResponse<T>(response: any): HttpResponse<T> {
|
||||
const headers = response.headers as Record<string, string>;
|
||||
const status = response.statusCode;
|
||||
const ok = status >= 200 && status < 300;
|
||||
const data = response.body as T;
|
||||
|
||||
if (!ok) {
|
||||
throw new HttpError(
|
||||
`Request failed with status ${status}`,
|
||||
status,
|
||||
{ data, status, headers, ok }
|
||||
);
|
||||
}
|
||||
|
||||
return { data, status, headers, ok };
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified body parsing (works for fetch response)
|
||||
*/
|
||||
private async parseResponseBody<T>(response: Response): Promise<T> {
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
|
||||
if (contentType.includes('application/json')) {
|
||||
return await response.body.json();
|
||||
return response.json();
|
||||
} else if (contentType.includes('text/')) {
|
||||
return await response.body.text();
|
||||
return response.text() as any;
|
||||
} else {
|
||||
return await response.body.arrayBuffer();
|
||||
return response.arrayBuffer() as any;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge request config with default config
|
||||
* Merge configs - request-level proxy only
|
||||
*/
|
||||
private mergeConfig(config: RequestConfig): RequestConfig {
|
||||
return {
|
||||
...config,
|
||||
headers: {
|
||||
...this.config.headers,
|
||||
...config.headers,
|
||||
},
|
||||
headers: { ...this.config.headers, ...config.headers },
|
||||
timeout: config.timeout ?? this.config.timeout,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import { ProxyAgent } from 'undici';
|
||||
import got from 'got';
|
||||
import { SocksProxyAgent } from 'socks-proxy-agent';
|
||||
import { HttpsProxyAgent } from 'https-proxy-agent';
|
||||
import { HttpProxyAgent } from 'http-proxy-agent';
|
||||
import type { ProxyConfig } from './types.js';
|
||||
|
||||
export class ProxyManager {
|
||||
/**
|
||||
* Determine if we should use Bun fetch (HTTP/HTTPS) or Undici (SOCKS)
|
||||
* Determine if we should use Bun fetch (HTTP/HTTPS) or Got (SOCKS)
|
||||
*/
|
||||
static shouldUseBunFetch(proxy: ProxyConfig): boolean {
|
||||
return proxy.protocol === 'http' || proxy.protocol === 'https';
|
||||
|
|
@ -23,35 +25,56 @@ export class ProxyManager {
|
|||
}
|
||||
|
||||
/**
|
||||
* Create agent for SOCKS proxies (used with undici)
|
||||
* Create appropriate agent for Got based on proxy type
|
||||
*/
|
||||
static createSocksAgent(proxy: ProxyConfig): SocksProxyAgent {
|
||||
const { protocol, host, port, username, password } = proxy;
|
||||
static createGotAgent(proxy: ProxyConfig) {
|
||||
this.validateConfig(proxy);
|
||||
|
||||
let proxyUrl: string;
|
||||
if (username && password) {
|
||||
proxyUrl = `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||
} else {
|
||||
proxyUrl = `${protocol}://${host}:${port}`;
|
||||
}
|
||||
const proxyUrl = this.buildProxyUrl(proxy);
|
||||
|
||||
return new SocksProxyAgent(proxyUrl);
|
||||
switch (proxy.protocol) {
|
||||
case 'socks4':
|
||||
case 'socks5':
|
||||
return new SocksProxyAgent(proxyUrl);
|
||||
case 'http':
|
||||
return new HttpProxyAgent(proxyUrl);
|
||||
case 'https':
|
||||
return new HttpsProxyAgent(proxyUrl);
|
||||
default:
|
||||
throw new Error(`Unsupported proxy protocol: ${proxy.protocol}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Undici proxy agent for HTTP/HTTPS proxies (fallback)
|
||||
* Create Got instance with proxy configuration
|
||||
*/
|
||||
static createUndiciAgent(proxy: ProxyConfig): ProxyAgent {
|
||||
static createGotInstance(proxy: ProxyConfig) {
|
||||
const agent = this.createGotAgent(proxy);
|
||||
|
||||
return got.extend({
|
||||
agent: {
|
||||
http: agent,
|
||||
https: agent
|
||||
},
|
||||
timeout: {
|
||||
request: 30000,
|
||||
connect: 10000
|
||||
},
|
||||
retry: {
|
||||
limit: 3,
|
||||
methods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH']
|
||||
},
|
||||
throwHttpErrors: false // We'll handle errors ourselves
|
||||
});
|
||||
}
|
||||
|
||||
private static buildProxyUrl(proxy: ProxyConfig): string {
|
||||
const { protocol, host, port, username, password } = proxy;
|
||||
|
||||
let proxyUrl: string;
|
||||
if (username && password) {
|
||||
proxyUrl = `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||
} else {
|
||||
proxyUrl = `${protocol}://${host}:${port}`;
|
||||
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||
}
|
||||
|
||||
return new ProxyAgent(proxyUrl);
|
||||
return `${protocol}://${host}:${port}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ export interface ProxyConfig {
|
|||
|
||||
export interface HttpClientConfig {
|
||||
timeout?: number;
|
||||
proxy?: ProxyConfig;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@stock-bot/config": "*",
|
||||
"got": "^14.4.7",
|
||||
"pino": "^9.7.0",
|
||||
"pino-loki": "^2.6.0",
|
||||
"pino-pretty": "^13.0.0"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue