fixed up some stuff
This commit is contained in:
parent
81d88357ca
commit
6cb6eb192c
3 changed files with 93 additions and 92 deletions
|
|
@ -3,56 +3,42 @@ import createCache, { type CacheProvider } from '@stock-bot/cache';
|
|||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import pLimit from 'p-limit';
|
||||
|
||||
export interface ProxyCheckResult {
|
||||
proxy: ProxyInfo;
|
||||
isWorking: boolean;
|
||||
responseTime: number;
|
||||
error?: string;
|
||||
checkedAt: Date;
|
||||
}
|
||||
|
||||
export interface ProxyStats {
|
||||
total: number;
|
||||
working: number;
|
||||
avgResponseTime: number;
|
||||
}
|
||||
|
||||
export class ProxyService {
|
||||
private logger = new Logger('proxy-service');
|
||||
private cache: CacheProvider = createCache('hybrid');
|
||||
private httpClient: HttpClient;
|
||||
private readonly concurrencyLimit = pLimit(1000);
|
||||
private readonly concurrencyLimit = pLimit(500);
|
||||
private readonly CACHE_KEY = 'proxy';
|
||||
private readonly CACHE_TTL = 86400; // 24 hours
|
||||
private readonly CHECK_TIMEOUT = 5000;
|
||||
private readonly CHECK_URL = 'https://httpbin.org/ip';
|
||||
private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
|
||||
private readonly PROXY_SOURCES = [
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
||||
]
|
||||
|
||||
constructor() {
|
||||
|
|
@ -69,10 +55,26 @@ export class ProxyService {
|
|||
this.concurrencyLimit(() => this.fetchProxiesFromSource(source))
|
||||
)
|
||||
const result = await Promise.all(sources);
|
||||
this.checkProxies(result.flat())
|
||||
const allProxies: ProxyInfo[] = result.flat();
|
||||
await this.checkProxies(this.removeDuplicateProxies(allProxies))
|
||||
return true
|
||||
}
|
||||
|
||||
private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
|
||||
const seen = new Set<string>();
|
||||
const unique: ProxyInfo[] = [];
|
||||
|
||||
for (const proxy of proxies) {
|
||||
const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
unique.push(proxy);
|
||||
}
|
||||
}
|
||||
|
||||
return unique;
|
||||
}
|
||||
|
||||
|
||||
async fetchProxiesFromSource(source: { url: string; protocol: string }): Promise<ProxyInfo[]> {
|
||||
const allProxies: ProxyInfo[] = [];
|
||||
|
|
@ -125,9 +127,8 @@ export class ProxyService {
|
|||
/**
|
||||
* Check if a proxy is working
|
||||
*/
|
||||
async checkProxy(proxy: ProxyInfo): Promise<ProxyCheckResult> {
|
||||
const startTime = Date.now();
|
||||
console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
async checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||
console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount);
|
||||
try {
|
||||
|
||||
// Test the proxy
|
||||
|
|
@ -135,19 +136,15 @@ export class ProxyService {
|
|||
proxy,
|
||||
timeout: this.CHECK_TIMEOUT
|
||||
});
|
||||
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
const isWorking = response.status >= 200 && response.status < 300;
|
||||
|
||||
const result: ProxyCheckResult = {
|
||||
proxy,
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking,
|
||||
responseTime,
|
||||
checkedAt: new Date()
|
||||
checkedAt: new Date(),
|
||||
responseTime: response.responseTime,
|
||||
};
|
||||
|
||||
// Cache the result
|
||||
// await this.cache.set(cacheKey, result, this.CACHE_TTL);
|
||||
|
||||
if (isWorking) {
|
||||
await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL);
|
||||
|
|
@ -159,26 +156,23 @@ export class ProxyService {
|
|||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
isWorking,
|
||||
responseTime
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
const responseTime = Date.now() - startTime;
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
const result: ProxyCheckResult = {
|
||||
proxy,
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking: false,
|
||||
responseTime,
|
||||
error: errorMessage,
|
||||
checkedAt: new Date()
|
||||
};
|
||||
|
||||
// Cache failed result for shorter time
|
||||
// await this.cache.set(cacheKey, result, 300); // 5 minutes
|
||||
await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
// await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
|
||||
this.logger.debug('Proxy check failed', {
|
||||
host: proxy.host,
|
||||
|
|
@ -193,7 +187,7 @@ export class ProxyService {
|
|||
/**
|
||||
* Check multiple proxies concurrently
|
||||
*/
|
||||
async checkProxies(proxies: ProxyInfo[]): Promise<ProxyCheckResult[]> {
|
||||
async checkProxies(proxies: ProxyInfo[]): Promise<ProxyInfo[]> {
|
||||
this.logger.info('Checking proxies', { count: proxies.length });
|
||||
|
||||
const checkPromises = proxies.map(proxy =>
|
||||
|
|
@ -239,18 +233,6 @@ export class ProxyService {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get proxy statistics
|
||||
*/
|
||||
async getStats(): Promise<ProxyStats> {
|
||||
// Simplified stats - in production you'd track these properly
|
||||
return {
|
||||
total: 0,
|
||||
working: 0,
|
||||
avgResponseTime: 0
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear proxy cache
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import type {
|
|||
import { HttpError } from './types.js';
|
||||
import { ProxyManager } from './proxy-manager.js';
|
||||
import got from 'got';
|
||||
import { clear } from 'node:console';
|
||||
|
||||
export class HttpClient {
|
||||
private readonly config: HttpClientConfig;
|
||||
|
|
@ -43,6 +44,7 @@ export class HttpClient {
|
|||
*/
|
||||
async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
|
||||
const finalConfig = this.mergeConfig(config);
|
||||
const startTime = Date.now();
|
||||
this.logger?.debug('Making HTTP request', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
|
|
@ -55,11 +57,14 @@ export class HttpClient {
|
|||
const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy);
|
||||
|
||||
const response = await this.makeRequest<T>(finalConfig, useBunFetch);
|
||||
|
||||
const responseTime = Date.now() - startTime;
|
||||
response.responseTime = responseTime;
|
||||
|
||||
this.logger?.debug('HTTP request successful', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
status: response.status,
|
||||
responseTime: responseTime,
|
||||
});
|
||||
|
||||
return response;
|
||||
|
|
@ -79,28 +84,41 @@ export class HttpClient {
|
|||
private async makeRequest<T>(config: RequestConfig, useBunFetch: boolean): Promise<HttpResponse<T>> {
|
||||
const timeout = config.timeout ?? this.config.timeout ?? 30000;
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
||||
|
||||
// Create timeout promise that rejects with proper error
|
||||
let timeoutId: NodeJS.Timeout | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
controller.abort();
|
||||
reject(new HttpError(`Request timeout after ${timeout}ms`));
|
||||
}, timeout);
|
||||
});
|
||||
|
||||
// Create request promise (don't await here!)
|
||||
const requestPromise = useBunFetch
|
||||
? this.fetchRequest<T>(config, controller.signal)
|
||||
: this.gotRequest<T>(config, controller.signal);
|
||||
|
||||
try {
|
||||
const response = useBunFetch
|
||||
? await this.fetchRequest<T>(config, controller.signal)
|
||||
: await this.gotRequest<T>(config, controller.signal);
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
return response;
|
||||
// Race the promises
|
||||
const result = await Promise.race([requestPromise, timeoutPromise]);
|
||||
if (timeoutId) clearTimeout(timeoutId);
|
||||
return result;
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// Unified timeout error handling
|
||||
if (controller.signal.aborted) {
|
||||
throw new HttpError(`Request timeout after ${timeout}ms`);
|
||||
}
|
||||
if ((error as any).name === 'TimeoutError') {
|
||||
throw new HttpError(`Request timeout after ${timeout}ms`);
|
||||
// If it's our timeout error, handle it
|
||||
if (error instanceof HttpError && error.message.includes('timeout')) {
|
||||
this.logger?.warn('Request timed out', {
|
||||
method: config.method,
|
||||
url: config.url,
|
||||
timeout: timeout,
|
||||
});
|
||||
throw error; // Re-throw the timeout error
|
||||
}
|
||||
|
||||
throw error; // Re-throw other errors as-is
|
||||
// Handle other errors (network, parsing, etc.)
|
||||
throw error;
|
||||
}
|
||||
|
||||
}
|
||||
/**
|
||||
* Bun fetch implementation (simplified)
|
||||
|
|
@ -108,9 +126,9 @@ export class HttpClient {
|
|||
private async fetchRequest<T>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
|
||||
try {
|
||||
|
||||
// const options = this.buildFetchOptions(config, signal);
|
||||
const options = this.buildFetchOptions(config, signal);
|
||||
// console.log('Using fetch with proxy:', config.proxy);
|
||||
const options = config.proxy? { proxy: config.proxy.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal);
|
||||
// const options = config.proxy? { proxy: config.proxy.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal);
|
||||
|
||||
this.logger?.debug('Making request with fetch: ', { url: config.url, options })
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ export interface HttpResponse<T = any> {
|
|||
status: number;
|
||||
headers: Record<string, string>;
|
||||
ok: boolean;
|
||||
responseTime?: number;
|
||||
}
|
||||
|
||||
export class HttpError extends Error {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue