fixed up some stuff
This commit is contained in:
parent
81d88357ca
commit
6cb6eb192c
3 changed files with 93 additions and 92 deletions
|
|
@ -3,56 +3,42 @@ import createCache, { type CacheProvider } from '@stock-bot/cache';
|
|||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import pLimit from 'p-limit';
|
||||
|
||||
export interface ProxyCheckResult {
|
||||
proxy: ProxyInfo;
|
||||
isWorking: boolean;
|
||||
responseTime: number;
|
||||
error?: string;
|
||||
checkedAt: Date;
|
||||
}
|
||||
|
||||
export interface ProxyStats {
|
||||
total: number;
|
||||
working: number;
|
||||
avgResponseTime: number;
|
||||
}
|
||||
|
||||
export class ProxyService {
|
||||
private logger = new Logger('proxy-service');
|
||||
private cache: CacheProvider = createCache('hybrid');
|
||||
private httpClient: HttpClient;
|
||||
private readonly concurrencyLimit = pLimit(1000);
|
||||
private readonly concurrencyLimit = pLimit(500);
|
||||
private readonly CACHE_KEY = 'proxy';
|
||||
private readonly CACHE_TTL = 86400; // 24 hours
|
||||
private readonly CHECK_TIMEOUT = 5000;
|
||||
private readonly CHECK_URL = 'https://httpbin.org/ip';
|
||||
private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
|
||||
private readonly PROXY_SOURCES = [
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
||||
]
|
||||
|
||||
constructor() {
|
||||
|
|
@ -69,10 +55,26 @@ export class ProxyService {
|
|||
this.concurrencyLimit(() => this.fetchProxiesFromSource(source))
|
||||
)
|
||||
const result = await Promise.all(sources);
|
||||
this.checkProxies(result.flat())
|
||||
const allProxies: ProxyInfo[] = result.flat();
|
||||
await this.checkProxies(this.removeDuplicateProxies(allProxies))
|
||||
return true
|
||||
}
|
||||
|
||||
private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
|
||||
const seen = new Set<string>();
|
||||
const unique: ProxyInfo[] = [];
|
||||
|
||||
for (const proxy of proxies) {
|
||||
const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
unique.push(proxy);
|
||||
}
|
||||
}
|
||||
|
||||
return unique;
|
||||
}
|
||||
|
||||
|
||||
async fetchProxiesFromSource(source: { url: string; protocol: string }): Promise<ProxyInfo[]> {
|
||||
const allProxies: ProxyInfo[] = [];
|
||||
|
|
@ -125,9 +127,8 @@ export class ProxyService {
|
|||
/**
|
||||
* Check if a proxy is working
|
||||
*/
|
||||
async checkProxy(proxy: ProxyInfo): Promise<ProxyCheckResult> {
|
||||
const startTime = Date.now();
|
||||
console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
async checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||
console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount);
|
||||
try {
|
||||
|
||||
// Test the proxy
|
||||
|
|
@ -135,19 +136,15 @@ export class ProxyService {
|
|||
proxy,
|
||||
timeout: this.CHECK_TIMEOUT
|
||||
});
|
||||
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
const isWorking = response.status >= 200 && response.status < 300;
|
||||
|
||||
const result: ProxyCheckResult = {
|
||||
proxy,
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking,
|
||||
responseTime,
|
||||
checkedAt: new Date()
|
||||
checkedAt: new Date(),
|
||||
responseTime: response.responseTime,
|
||||
};
|
||||
|
||||
// Cache the result
|
||||
// await this.cache.set(cacheKey, result, this.CACHE_TTL);
|
||||
|
||||
if (isWorking) {
|
||||
await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL);
|
||||
|
|
@ -159,26 +156,23 @@ export class ProxyService {
|
|||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
isWorking,
|
||||
responseTime
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
const responseTime = Date.now() - startTime;
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
const result: ProxyCheckResult = {
|
||||
proxy,
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking: false,
|
||||
responseTime,
|
||||
error: errorMessage,
|
||||
checkedAt: new Date()
|
||||
};
|
||||
|
||||
// Cache failed result for shorter time
|
||||
// await this.cache.set(cacheKey, result, 300); // 5 minutes
|
||||
await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
// await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
|
||||
this.logger.debug('Proxy check failed', {
|
||||
host: proxy.host,
|
||||
|
|
@ -193,7 +187,7 @@ export class ProxyService {
|
|||
/**
|
||||
* Check multiple proxies concurrently
|
||||
*/
|
||||
async checkProxies(proxies: ProxyInfo[]): Promise<ProxyCheckResult[]> {
|
||||
async checkProxies(proxies: ProxyInfo[]): Promise<ProxyInfo[]> {
|
||||
this.logger.info('Checking proxies', { count: proxies.length });
|
||||
|
||||
const checkPromises = proxies.map(proxy =>
|
||||
|
|
@ -239,18 +233,6 @@ export class ProxyService {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get proxy statistics
|
||||
*/
|
||||
async getStats(): Promise<ProxyStats> {
|
||||
// Simplified stats - in production you'd track these properly
|
||||
return {
|
||||
total: 0,
|
||||
working: 0,
|
||||
avgResponseTime: 0
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear proxy cache
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue