boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fixed up some stuff

Browse source
This commit is contained in:
Bojan Kucera 2025-06-07 20:34:46 -04:00
parent 81d88357ca
commit 6cb6eb192c
3 changed files with 93 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

128
apps/data-service/src/services/proxy.service.ts
View file

@ -3,56 +3,42 @@ import createCache, { type CacheProvider } from '@stock-bot/cache';
import { HttpClient, ProxyInfo } from '@stock-bot/http'; import { HttpClient, ProxyInfo } from '@stock-bot/http';
import pLimit from 'p-limit'; import pLimit from 'p-limit';
export interface ProxyCheckResult {
proxy: ProxyInfo;
isWorking: boolean;
responseTime: number;
error?: string;
checkedAt: Date;
}
export interface ProxyStats {
total: number;
working: number;
avgResponseTime: number;
}
export class ProxyService { export class ProxyService {
private logger = new Logger('proxy-service'); private logger = new Logger('proxy-service');
private cache: CacheProvider = createCache('hybrid'); private cache: CacheProvider = createCache('hybrid');
private httpClient: HttpClient; private httpClient: HttpClient;
private readonly concurrencyLimit = pLimit(1000); private readonly concurrencyLimit = pLimit(500);
private readonly CACHE_KEY = 'proxy'; private readonly CACHE_KEY = 'proxy';
private readonly CACHE_TTL = 86400; // 24 hours private readonly CACHE_TTL = 86400; // 24 hours
private readonly CHECK_TIMEOUT = 5000; private readonly CHECK_TIMEOUT = 5000;
private readonly CHECK_URL = 'https://httpbin.org/ip'; private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
private readonly PROXY_SOURCES = [ private readonly PROXY_SOURCES = [
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
// {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', }, // {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', }, // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
] ]
constructor() { constructor() {
@ -69,10 +55,26 @@ export class ProxyService {
this.concurrencyLimit(() => this.fetchProxiesFromSource(source)) this.concurrencyLimit(() => this.fetchProxiesFromSource(source))
) )
const result = await Promise.all(sources); const result = await Promise.all(sources);
this.checkProxies(result.flat()) const allProxies: ProxyInfo[] = result.flat();
await this.checkProxies(this.removeDuplicateProxies(allProxies))
return true return true
} }
private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
const seen = new Set<string>();
const unique: ProxyInfo[] = [];
for (const proxy of proxies) {
const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
if (!seen.has(key)) {
seen.add(key);
unique.push(proxy);
}
}
return unique;
}
async fetchProxiesFromSource(source: { url: string; protocol: string }): Promise<ProxyInfo[]> { async fetchProxiesFromSource(source: { url: string; protocol: string }): Promise<ProxyInfo[]> {
const allProxies: ProxyInfo[] = []; const allProxies: ProxyInfo[] = [];
@ -125,9 +127,8 @@ export class ProxyService {
/** /**
* Check if a proxy is working * Check if a proxy is working
*/ */
async checkProxy(proxy: ProxyInfo): Promise<ProxyCheckResult> { async checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
const startTime = Date.now(); console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount);
console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`);
try { try {
// Test the proxy // Test the proxy
@ -135,19 +136,15 @@ export class ProxyService {
proxy, proxy,
timeout: this.CHECK_TIMEOUT timeout: this.CHECK_TIMEOUT
}); });
const responseTime = Date.now() - startTime;
const isWorking = response.status >= 200 && response.status < 300; const isWorking = response.status >= 200 && response.status < 300;
const result: ProxyCheckResult = { const result: ProxyInfo = {
proxy, ...proxy,
isWorking, isWorking,
responseTime, checkedAt: new Date(),
checkedAt: new Date() responseTime: response.responseTime,
}; };
// Cache the result
// await this.cache.set(cacheKey, result, this.CACHE_TTL);
if (isWorking) { if (isWorking) {
await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL); await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL);
@ -159,26 +156,23 @@ export class ProxyService {
host: proxy.host, host: proxy.host,
port: proxy.port, port: proxy.port,
isWorking, isWorking,
responseTime
}); });
return result; return result;
} catch (error) { } catch (error) {
const responseTime = Date.now() - startTime;
const errorMessage = error instanceof Error ? error.message : String(error); const errorMessage = error instanceof Error ? error.message : String(error);
const result: ProxyCheckResult = { const result: ProxyInfo = {
proxy, ...proxy,
isWorking: false, isWorking: false,
responseTime,
error: errorMessage, error: errorMessage,
checkedAt: new Date() checkedAt: new Date()
}; };
// Cache failed result for shorter time // Cache failed result for shorter time
// await this.cache.set(cacheKey, result, 300); // 5 minutes // await this.cache.set(cacheKey, result, 300); // 5 minutes
await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); // await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
this.logger.debug('Proxy check failed', { this.logger.debug('Proxy check failed', {
host: proxy.host, host: proxy.host,
@ -193,7 +187,7 @@ export class ProxyService {
/** /**
* Check multiple proxies concurrently * Check multiple proxies concurrently
*/ */
async checkProxies(proxies: ProxyInfo[]): Promise<ProxyCheckResult[]> { async checkProxies(proxies: ProxyInfo[]): Promise<ProxyInfo[]> {
this.logger.info('Checking proxies', { count: proxies.length }); this.logger.info('Checking proxies', { count: proxies.length });
const checkPromises = proxies.map(proxy => const checkPromises = proxies.map(proxy =>
@ -239,18 +233,6 @@ export class ProxyService {
}); });
} }
/**
* Get proxy statistics
*/
async getStats(): Promise<ProxyStats> {
// Simplified stats - in production you'd track these properly
return {
total: 0,
working: 0,
avgResponseTime: 0
};
}
/** /**
* Clear proxy cache * Clear proxy cache
*/ */

56
libs/http/src/client.ts
View file

@ -7,6 +7,7 @@ import type {
import { HttpError } from './types.js'; import { HttpError } from './types.js';
import { ProxyManager } from './proxy-manager.js'; import { ProxyManager } from './proxy-manager.js';
import got from 'got'; import got from 'got';
import { clear } from 'node:console';
export class HttpClient { export class HttpClient {
private readonly config: HttpClientConfig; private readonly config: HttpClientConfig;
@ -43,6 +44,7 @@ export class HttpClient {
*/ */
async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> { async request<T = any>(config: RequestConfig): Promise<HttpResponse<T>> {
const finalConfig = this.mergeConfig(config); const finalConfig = this.mergeConfig(config);
const startTime = Date.now();
this.logger?.debug('Making HTTP request', { this.logger?.debug('Making HTTP request', {
method: finalConfig.method, method: finalConfig.method,
url: finalConfig.url, url: finalConfig.url,
@ -55,11 +57,14 @@ export class HttpClient {
const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy); const useBunFetch = !proxy || ProxyManager.shouldUseBunFetch(proxy);
const response = await this.makeRequest<T>(finalConfig, useBunFetch); const response = await this.makeRequest<T>(finalConfig, useBunFetch);
const responseTime = Date.now() - startTime;
response.responseTime = responseTime;
this.logger?.debug('HTTP request successful', { this.logger?.debug('HTTP request successful', {
method: finalConfig.method, method: finalConfig.method,
url: finalConfig.url, url: finalConfig.url,
status: response.status, status: response.status,
responseTime: responseTime,
}); });
return response; return response;
@ -79,28 +84,41 @@ export class HttpClient {
private async makeRequest<T>(config: RequestConfig, useBunFetch: boolean): Promise<HttpResponse<T>> { private async makeRequest<T>(config: RequestConfig, useBunFetch: boolean): Promise<HttpResponse<T>> {
const timeout = config.timeout ?? this.config.timeout ?? 30000; const timeout = config.timeout ?? this.config.timeout ?? 30000;
const controller = new AbortController(); const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
// Create timeout promise that rejects with proper error
let timeoutId: NodeJS.Timeout | undefined;
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => {
controller.abort();
reject(new HttpError(`Request timeout after ${timeout}ms`));
}, timeout);
});
// Create request promise (don't await here!)
const requestPromise = useBunFetch
? this.fetchRequest<T>(config, controller.signal)
: this.gotRequest<T>(config, controller.signal);
try { try {
const response = useBunFetch // Race the promises
? await this.fetchRequest<T>(config, controller.signal) const result = await Promise.race([requestPromise, timeoutPromise]);
: await this.gotRequest<T>(config, controller.signal); if (timeoutId) clearTimeout(timeoutId);
return result;
clearTimeout(timeoutId);
return response;
} catch (error) { } catch (error) {
clearTimeout(timeoutId); // If it's our timeout error, handle it
if (error instanceof HttpError && error.message.includes('timeout')) {
// Unified timeout error handling this.logger?.warn('Request timed out', {
if (controller.signal.aborted) { method: config.method,
throw new HttpError(`Request timeout after ${timeout}ms`); url: config.url,
} timeout: timeout,
if ((error as any).name === 'TimeoutError') { });
throw new HttpError(`Request timeout after ${timeout}ms`); throw error; // Re-throw the timeout error
} }
throw error; // Re-throw other errors as-is // Handle other errors (network, parsing, etc.)
throw error;
} }
} }
/** /**
* Bun fetch implementation (simplified) * Bun fetch implementation (simplified)
@ -108,9 +126,9 @@ export class HttpClient {
private async fetchRequest<T>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> { private async fetchRequest<T>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
try { try {
// const options = this.buildFetchOptions(config, signal); const options = this.buildFetchOptions(config, signal);
// console.log('Using fetch with proxy:', config.proxy); // console.log('Using fetch with proxy:', config.proxy);
const options = config.proxy? { proxy: config.proxy.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal); // const options = config.proxy? { proxy: config.proxy.protocol + '://' + config.proxy?.host + ':' + config.proxy?.port } : {}//this.buildGotOptions(config, signal);
this.logger?.debug('Making request with fetch: ', { url: config.url, options }) this.logger?.debug('Making request with fetch: ', { url: config.url, options })

1
libs/http/src/types.ts
View file

@ -32,6 +32,7 @@ export interface HttpResponse<T = any> {
status: number; status: number;
headers: Record<string, string>; headers: Record<string, string>;
ok: boolean; ok: boolean;
responseTime?: number;
} }
export class HttpError extends Error { export class HttpError extends Error {