boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

bun fetch dont support sock proxy :(

Browse source
This commit is contained in:
Bojan Kucera 2025-06-06 23:08:54 -04:00
parent 5904af213f
commit 08bb21cee7
5 changed files with 89 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

151
apps/data-service/src/proxy-demo.ts
View file

@ -3,86 +3,86 @@ import { proxyService, ProxySource } from './services/proxy.service.js';
/** /**
* Example usage of the ProxyService * Example usage of the ProxyService
*/ */
async function demonstrateProxyService() { // async function demonstrateProxyService() {
console.log('🚀 Starting Proxy Service Demo...'); // console.log('🚀 Starting Proxy Service Demo...');
try { // try {
// 1. Start the proxy refresh job (scrapes proxies every 30 minutes) // // 1. Start the proxy refresh job (scrapes proxies every 30 minutes)
console.log('📥 Starting proxy refresh job...'); // console.log('📥 Starting proxy refresh job...');
await proxyService.queueRefreshProxies(30 * 60 * 1000); // 30 minutes // await proxyService.queueRefreshProxies(30 * 60 * 1000); // 30 minutes
// 2. Start health checks (checks working proxies every 15 minutes) // // 2. Start health checks (checks working proxies every 15 minutes)
console.log('🔍 Starting proxy health checks...'); // console.log('🔍 Starting proxy health checks...');
await proxyService.startHealthChecks(15 * 60 * 1000); // 15 minutes // await proxyService.startHealthChecks(15 * 60 * 1000); // 15 minutes
// 3. Manually scrape proxies // // 3. Manually scrape proxies
console.log('🌐 Manually scraping proxies...'); // console.log('🌐 Manually scraping proxies...');
const scrapedCount = await proxyService.scrapeProxies(); // const scrapedCount = await proxyService.scrapeProxies();
console.log(`✅ Scraped ${scrapedCount} unique proxies`); // console.log(`✅ Scraped ${scrapedCount} unique proxies`);
// 4. Wait a bit for some validation to complete // // 4. Wait a bit for some validation to complete
await new Promise(resolve => setTimeout(resolve, 5000)); // await new Promise(resolve => setTimeout(resolve, 5000));
// 5. Get proxy statistics // // 5. Get proxy statistics
console.log('📊 Getting proxy statistics...'); // console.log('📊 Getting proxy statistics...');
const stats = await proxyService.getProxyStats(); // const stats = await proxyService.getProxyStats();
console.log('Stats:', { // console.log('Stats:', {
total: stats.total, // total: stats.total,
working: stats.working, // working: stats.working,
failed: stats.failed, // failed: stats.failed,
avgResponseTime: stats.avgResponseTime + 'ms' // avgResponseTime: stats.avgResponseTime + 'ms'
}); // });
// 6. Get a working proxy // // 6. Get a working proxy
console.log('🎯 Getting a working proxy...'); // console.log('🎯 Getting a working proxy...');
const workingProxy = await proxyService.getWorkingProxy(); // const workingProxy = await proxyService.getWorkingProxy();
if (workingProxy) { // if (workingProxy) {
console.log('Working proxy found:', { // console.log('Working proxy found:', {
host: workingProxy.host, // host: workingProxy.host,
port: workingProxy.port, // port: workingProxy.port,
protocol: workingProxy.protocol // protocol: workingProxy.protocol
}); // });
// 7. Test the proxy // // 7. Test the proxy
console.log('🧪 Testing proxy...'); // console.log('🧪 Testing proxy...');
const testResult = await proxyService.checkProxy(workingProxy); // const testResult = await proxyService.checkProxy(workingProxy);
console.log('Test result:', { // console.log('Test result:', {
isWorking: testResult.isWorking, // isWorking: testResult.isWorking,
responseTime: testResult.responseTime + 'ms', // responseTime: testResult.responseTime + 'ms',
error: testResult.error || 'None' // error: testResult.error || 'None'
}); // });
} else { // } else {
console.log('❌ No working proxies available yet'); // console.log('❌ No working proxies available yet');
} // }
// 8. Get multiple working proxies // // 8. Get multiple working proxies
console.log('📋 Getting multiple working proxies...'); // console.log('📋 Getting multiple working proxies...');
const workingProxies = await proxyService.getWorkingProxies(5); // const workingProxies = await proxyService.getWorkingProxies(5);
console.log(`Found ${workingProxies.length} working proxies`); // console.log(`Found ${workingProxies.length} working proxies`);
// 9. Example: Using a proxy with HttpClient // // 9. Example: Using a proxy with HttpClient
if (workingProxies.length > 0) { // if (workingProxies.length > 0) {
console.log('🔄 Example: Using proxy with HttpClient...'); // console.log('🔄 Example: Using proxy with HttpClient...');
try { // try {
const { HttpClient } = await import('@stock-bot/http-client'); // const { HttpClient } = await import('@stock-bot/http-client');
const proxyClient = new HttpClient({ // const proxyClient = new HttpClient({
proxy: workingProxies[0], // proxy: workingProxies[0],
timeout: 10000 // timeout: 10000
}); // });
const response = await proxyClient.get('http://httpbin.org/ip'); // const response = await proxyClient.get('http://httpbin.org/ip');
console.log('✅ Request through proxy successful:', response.data); // console.log('✅ Request through proxy successful:', response.data);
} catch (error) { // } catch (error) {
console.log('❌ Request through proxy failed:', error); // console.log('❌ Request through proxy failed:', error);
} // }
} // }
console.log('🎉 Proxy Service Demo completed!'); // console.log('🎉 Proxy Service Demo completed!');
} catch (error) { // } catch (error) {
console.error('❌ Demo failed:', error); // console.error('❌ Demo failed:', error);
} // }
} // }
/** /**
* Example: Custom proxy source * Example: Custom proxy source
@ -92,8 +92,8 @@ async function demonstrateCustomProxySource() {
const customSources : ProxySource[] = [ const customSources : ProxySource[] = [
{ {
url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt', url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',
protocol: 'socks5' protocol: 'http'
} }
]; ];
@ -107,13 +107,14 @@ async function demonstrateCustomProxySource() {
// Export functions for use in other files // Export functions for use in other files
export { export {
demonstrateProxyService, // demonstrateProxyService,
demonstrateCustomProxySource demonstrateCustomProxySource
}; };
demonstrateCustomProxySource()
// If this file is run directly, execute the demo // If this file is run directly, execute the demo
if (import.meta.main) { // if (import.meta.main) {
demonstrateProxyService() // demonstrateProxyService()
// demonstrateCustomProxySource() // demonstrateCustomProxySource()
.catch(console.error); // .catch(console.error);
} // }

3
apps/data-service/src/services/proxy.service.ts
View file

@ -1,6 +1,6 @@
import { createLogger } from '@stock-bot/logger'; import { createLogger } from '@stock-bot/logger';
import createCache, { type CacheProvider } from '@stock-bot/cache'; import createCache, { type CacheProvider } from '@stock-bot/cache';
import { HttpClient, ProxyConfig } from '@stock-bot/http-client'; import { HttpClient, ProxyConfig , RequestConfig } from '@stock-bot/http-client';
import type { Logger as PinoLogger } from 'pino'; import type { Logger as PinoLogger } from 'pino';
export interface ProxySource { export interface ProxySource {
@ -267,6 +267,7 @@ export class ProxyService {
const startTime = Date.now(); const startTime = Date.now();
try { try {
// Create a new HttpClient instance with the proxy // Create a new HttpClient instance with the proxy
const proxyClient = new HttpClient({ const proxyClient = new HttpClient({
timeout: this.CHECK_TIMEOUT, timeout: this.CHECK_TIMEOUT,

1
libs/http-client/src/client.ts
View file

@ -170,6 +170,7 @@ export class HttpClient {
if (this.config.proxy) { if (this.config.proxy) {
const agent = ProxyManager.createAgent(this.config.proxy); const agent = ProxyManager.createAgent(this.config.proxy);
(requestOptions as any).agent = agent; (requestOptions as any).agent = agent;
console.log('Using proxy agent:', this.config.proxy);
} }
// Make the request // Make the request

10
libs/http-client/src/proxy-manager.ts
View file

@ -2,6 +2,7 @@ import { HttpsProxyAgent } from 'https-proxy-agent';
import { SocksProxyAgent } from 'socks-proxy-agent'; import { SocksProxyAgent } from 'socks-proxy-agent';
import type { ProxyConfig } from './types.js'; import type { ProxyConfig } from './types.js';
import { validateProxyConfig } from './types.js'; import { validateProxyConfig } from './types.js';
import { HttpProxyAgent } from 'http-proxy-agent';
export class ProxyManager { export class ProxyManager {
/** /**
@ -11,7 +12,13 @@ export class ProxyManager {
const { protocol, host, port, username, password } = proxy; const { protocol, host, port, username, password } = proxy;
let proxyUrl: string; let proxyUrl: string;
console.log('Creating proxy agent with config:', {
protocol,
host,
port,
username,
password
});
if (username && password) { if (username && password) {
proxyUrl = `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`; proxyUrl = `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
} else { } else {
@ -20,6 +27,7 @@ export class ProxyManager {
switch (protocol) { switch (protocol) {
case 'http': case 'http':
return new HttpProxyAgent(proxyUrl);
case 'https': case 'https':
return new HttpsProxyAgent(proxyUrl); return new HttpsProxyAgent(proxyUrl);
case 'socks4': case 'socks4':

1
libs/http-client/src/types.ts
View file

@ -35,6 +35,7 @@ export interface RequestConfig {
method?: HttpMethod; method?: HttpMethod;
url: string; url: string;
headers?: Record<string, string>; headers?: Record<string, string>;
proxy?: ProxyConfig;
body?: any; body?: any;
timeout?: number; timeout?: number;
retries?: number; retries?: number;