boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

clean up proxyManager

Browse source
This commit is contained in:
Boki 2025-06-20 12:31:49 -04:00
parent da916222c1
commit 76d55fe35f
10 changed files with 38 additions and 438 deletions
Download patch file Download diff file Expand all files Collapse all files

6
apps/data-service/src/providers/qm.tasks.ts
View file

@ -2,7 +2,7 @@ import { getRandomUserAgent } from '@stock-bot/http';
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient } from '@stock-bot/mongodb-client';
import { QueueManager } from '@stock-bot/queue';
import { ProxyManager } from '@stock-bot/utils';
import { getRandomProxy } from '@stock-bot/utils';
// Shared instances (module-scoped, not global)
let isInitialized = false; // Track if resources are initialized
@ -90,7 +90,7 @@ export async function createSessions(): Promise<void> {
while (sessionCache[sessionId].length < 50) {
logger.info(`Creating new session for ${sessionId}`);
const proxyInfo = await ProxyManager.getInstance().getRandomProxy();
const proxyInfo = await getRandomProxy();
if (!proxyInfo) {
logger.error('No proxy available for QM session creation');
break; // Skip session creation if no proxy is available
@ -294,7 +294,7 @@ async function searchAndSpawnJobs(
// API call function to search symbols via QM
async function searchQMSymbolsAPI(query: string): Promise<string[]> {
const proxyInfo = await ProxyManager.getInstance().getRandomProxy();
const proxyInfo = await getRandomProxy();
if (!proxyInfo) {
throw new Error('No proxy available for QM API call');

67
apps/data-service/src/providers/webshare.provider.ts
View file

@ -7,7 +7,7 @@ import {
handlerRegistry,
type HandlerConfigWithSchedule,
} from '@stock-bot/queue';
import { ProxyManager } from '@stock-bot/utils';
import { updateProxies } from '@stock-bot/utils';
const logger = getLogger('webshare-provider');
@ -28,7 +28,7 @@ export function initializeWebShareProvider() {
if (proxies.length > 0) {
// Update the centralized proxy manager
await ProxyManager.getInstance().updateProxies(proxies);
await updateProxies(proxies);
logger.info('Updated proxy manager with WebShare proxies', {
count: proxies.length,
@ -57,40 +57,6 @@ export function initializeWebShareProvider() {
};
}
}),
'validate-proxies': createJobHandler(async () => {
logger.info('Validating existing proxies');
const { validateStoredProxies } = await import('./webshare.tasks');
try {
const validationResults = await validateStoredProxies();
// Update proxy manager with validated proxies
await ProxyManager.getInstance().updateProxies(validationResults.workingProxies);
logger.info('Proxy validation completed', {
totalChecked: validationResults.totalChecked,
workingCount: validationResults.workingCount,
successRate: ((validationResults.workingCount / validationResults.totalChecked) * 100).toFixed(1) + '%',
});
return validationResults;
} catch (error) {
logger.error('Failed to validate proxies', { error });
return {
workingProxies: [],
totalChecked: 0,
workingCount: 0,
error: error instanceof Error ? error.message : 'Unknown error',
};
}
}),
'get-stats': createJobHandler(async () => {
const stats = ProxyManager.getInstance().getStats();
logger.info('Proxy manager statistics', stats);
return stats;
}),
},
scheduledJobs: [
@ -103,22 +69,6 @@ export function initializeWebShareProvider() {
description: 'Fetch fresh proxies from WebShare API',
immediately: true, // Run on startup
},
{
type: 'webshare-validate',
operation: 'validate-proxies',
payload: {},
cronPattern: '0 */2 * * *', // Every 2 hours
priority: 4,
description: 'Validate and clean existing proxies',
},
{
type: 'webshare-stats',
operation: 'get-stats',
payload: {},
cronPattern: '0 * * * *', // Every hour
priority: 5,
description: 'Log proxy manager statistics',
},
],
};
@ -126,20 +76,7 @@ export function initializeWebShareProvider() {
logger.debug('WebShare provider registered successfully');
}
// Legacy function for backward compatibility - now uses centralized proxy manager
export async function getProxy(): Promise<string | null> {
const proxy = ProxyManager.getInstance().getRandomProxy();
if (!proxy) {
return null;
}
// Convert ProxyInfo back to string format for backward compatibility
const auth = proxy.username && proxy.password ? `${proxy.username}:${proxy.password}@` : '';
return `${proxy.protocol}://${auth}${proxy.host}:${proxy.port}`;
}
export const webShareProvider = {
initialize: initializeWebShareProvider,
getProxy,
};

177
apps/data-service/src/providers/webshare.tasks.ts
View file

@ -1,9 +1,8 @@
/**
* WebShare Tasks - API integration and proxy validation
* WebShare Tasks - API integration
*/
import { getLogger } from '@stock-bot/logger';
import { HttpClient, type ProxyInfo } from '@stock-bot/http';
import { ProxyManager } from '@stock-bot/utils';
import { type ProxyInfo } from '@stock-bot/http';
const logger = getLogger('webshare-tasks');
@ -65,7 +64,7 @@ export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
port: proxy.port,
username: proxy.username,
password: proxy.password,
isWorking: true, // Assume working until proven otherwise
isWorking: true, // WebShare provides working proxies
firstSeen: new Date(),
lastChecked: new Date(),
}));
@ -80,174 +79,4 @@ export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
logger.error('Failed to fetch proxies from WebShare', { error });
return [];
}
}
/**
* Validate stored proxies by testing connectivity
*/
export async function validateStoredProxies(): Promise<{
workingProxies: ProxyInfo[];
totalChecked: number;
workingCount: number;
}> {
const httpClient = new HttpClient({ timeout: 10000 }, logger);
const testUrl = 'https://httpbin.org/ip'; // Simple IP echo service
// Get all proxies from proxy manager
const allProxies = ProxyManager.getInstance().getAllProxies();
if (allProxies.length === 0) {
logger.warn('No proxies available for validation');
return {
workingProxies: [],
totalChecked: 0,
workingCount: 0,
};
}
logger.info('Starting proxy validation', { totalProxies: allProxies.length });
const workingProxies: ProxyInfo[] = [];
const validationPromises = allProxies.map(async (proxy) => {
try {
const startTime = Date.now();
const response = await httpClient.get(testUrl, {
proxy,
timeout: 10000,
});
const responseTime = Date.now() - startTime;
const isWorking = response.status === 200;
if (isWorking) {
// Update proxy with success metrics
const updatedProxy: ProxyInfo = {
...proxy,
isWorking: true,
lastChecked: new Date(),
responseTime,
total: (proxy.total || 0) + 1,
working: (proxy.working || 0) + 1,
averageResponseTime: proxy.averageResponseTime
? (proxy.averageResponseTime + responseTime) / 2
: responseTime,
};
// Calculate success rate
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: 100;
workingProxies.push(updatedProxy);
logger.debug('Proxy validation successful', {
host: proxy.host,
port: proxy.port,
responseTime,
successRate: updatedProxy.successRate?.toFixed(1) + '%',
});
} else {
logger.debug('Proxy validation failed', {
host: proxy.host,
port: proxy.port,
status: response.status,
});
}
} catch (error) {
// Update proxy with failure metrics
const updatedProxy: ProxyInfo = {
...proxy,
isWorking: false,
lastChecked: new Date(),
error: error instanceof Error ? error.message : 'Unknown error',
total: (proxy.total || 0) + 1,
working: proxy.working || 0,
};
// Calculate success rate
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: 0;
logger.debug('Proxy validation error', {
host: proxy.host,
port: proxy.port,
error: error instanceof Error ? error.message : 'Unknown error',
});
}
});
// Wait for all validations to complete
await Promise.all(validationPromises);
const results = {
workingProxies,
totalChecked: allProxies.length,
workingCount: workingProxies.length,
};
logger.info('Proxy validation completed', {
totalChecked: results.totalChecked,
workingCount: results.workingCount,
successRate: ((results.workingCount / results.totalChecked) * 100).toFixed(1) + '%',
});
return results;
}
/**
* Test a single proxy for connectivity
*/
export async function testProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
const httpClient = new HttpClient({ timeout: 10000 }, logger);
const testUrl = 'https://httpbin.org/ip';
try {
const startTime = Date.now();
const response = await httpClient.get(testUrl, {
proxy,
timeout: 10000,
});
const responseTime = Date.now() - startTime;
const isWorking = response.status === 200;
const updatedProxy: ProxyInfo = {
...proxy,
isWorking,
lastChecked: new Date(),
responseTime,
total: (proxy.total || 0) + 1,
working: isWorking ? (proxy.working || 0) + 1 : (proxy.working || 0),
};
// Calculate success rate
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: (isWorking ? 100 : 0);
// Update average response time
if (isWorking && responseTime) {
updatedProxy.averageResponseTime = proxy.averageResponseTime
? (proxy.averageResponseTime + responseTime) / 2
: responseTime;
}
return updatedProxy;
} catch (error) {
const updatedProxy: ProxyInfo = {
...proxy,
isWorking: false,
lastChecked: new Date(),
error: error instanceof Error ? error.message : 'Unknown error',
total: (proxy.total || 0) + 1,
working: proxy.working || 0,
};
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: 0;
return updatedProxy;
}
}

18
libs/proxy/package.json
View file

@ -1,18 +0,0 @@
{
"name": "@stock-bot/proxy",
"version": "1.0.0",
"description": "Simple proxy management library",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"type": "module",
"scripts": {
"build": "tsc",
"test": "bun test",
"dev": "tsc --watch"
},
"devDependencies": {
"@types/node": "^20.0.0",
"typescript": "^5.0.0"
},
"peerDependencies": {}
}

104
libs/proxy/src/index.ts
View file

@ -1,104 +0,0 @@
// Simple proxy list manager
let proxies: string[] = [];
let currentIndex = 0;
const DEFAULT_PROXY_URL =
'https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all';
/**
* Fetch proxy list from URL and store in module
*/
export async function refreshProxies(fetchUrl: string = DEFAULT_PROXY_URL): Promise<string[]> {
try {
const response = await fetch(fetchUrl);
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const data = await response.text();
const newProxies = data
.trim()
.split('\n')
.map(line => line.trim())
.filter(line => line && line.includes(':'))
.map(line => {
// Convert host:port to http://host:port format
return line.startsWith('http') ? line : `http://${line}`;
});
proxies = newProxies;
currentIndex = 0;
return proxies;
} catch (error) {
throw new Error(`Failed to fetch proxies: ${error}`);
}
}
/**
* Get next proxy URL in round-robin fashion
*/
export function getProxyURL(): string | null {
if (proxies.length === 0) {
return null;
}
const proxy = proxies[currentIndex];
currentIndex = (currentIndex + 1) % proxies.length;
if (!proxy) {
return null;
}
// Ensure the proxy URL is in http://host:port format
return proxy;
}
/**
* Get multiple proxy URLs
*/
export function getProxyURLs(count: number): string[] {
const urls: string[] = [];
for (let i = 0; i < count; i++) {
const url = getProxyURL();
if (url) {
urls.push(url);
}
}
return urls;
}
/**
* Get random proxy URL
*/
export function getRandomProxyURL(): string | null {
if (proxies.length === 0) {
return null;
}
const randomIndex = Math.floor(Math.random() * proxies.length);
const proxy = proxies[randomIndex];
if (!proxy) {
return null;
}
return proxy;
}
/**
* Get current proxy count
*/
export function getProxyCount(): number {
return proxies.length;
}
/**
* Get all proxies
*/
export function getAllProxies(): string[] {
return [...proxies];
}
/**
* Initialize proxy manager with initial fetch
*/
export async function initializeProxies(fetchUrl?: string): Promise<void> {
await refreshProxies(fetchUrl);
}

22
libs/proxy/src/types.ts
View file

@ -1,22 +0,0 @@
export interface ProxyInfo {
host: string;
port: number;
protocol: 'http' | 'https' | 'socks4' | 'socks5';
username?: string;
password?: string;
country?: string;
isActive?: boolean;
}
export interface ProxyManagerOptions {
fetchUrl?: string;
refreshIntervalMs?: number;
maxRetries?: number;
timeout?: number;
}
export interface ProxyResponse {
proxies: ProxyInfo[];
totalCount: number;
activeCount: number;
}

10
libs/proxy/tsconfig.json
View file

@ -1,10 +0,0 @@
{
"extends": "../../tsconfig.lib.json",
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src"
},
"include": ["src/**/*"],
"references": [
]
}

19
libs/proxy/turbo.json
View file

@ -1,19 +0,0 @@
{
"extends": ["//"],
"tasks": {
"build": {
"dependsOn": [],
"outputs": ["dist/**"],
"inputs": [
"src/**",
"package.json",
"tsconfig.json",
"!**/*.test.ts",
"!**/*.spec.ts",
"!**/test/**",
"!**/tests/**",
"!**/__tests__/**"
]
}
}
}

9
libs/utils/src/proxy/index.ts
View file

@ -1,5 +1,12 @@
/**
* Proxy management utilities
*/
export { default as ProxyManager } from './proxy-manager';
export {
default as ProxyManager,
getProxy,
getRandomProxy,
getAllProxies,
getWorkingProxies,
updateProxies
} from './proxy-manager';
export type { ProxyInfo } from '@stock-bot/http'; // Re-export for convenience

44
libs/utils/src/proxy/proxy-manager.ts
View file

@ -180,27 +180,6 @@ export class ProxyManager {
}
}
/**
* Get proxy statistics
*/
getStats(): {
totalProxies: number;
workingProxies: number;
lastUpdate: Date | null;
successRate: number;
} {
const workingProxies = this.proxies.filter(p => p.isWorking !== false);
const totalSuccessRate = this.proxies.reduce((sum, p) => sum + (p.successRate || 0), 0);
const avgSuccessRate = this.proxies.length > 0 ? totalSuccessRate / this.proxies.length : 0;
return {
totalProxies: this.proxies.length,
workingProxies: workingProxies.length,
lastUpdate: this.lastUpdate,
successRate: avgSuccessRate,
};
}
/**
* Clear all proxies from memory and cache
*/
@ -274,4 +253,25 @@ export class ProxyManager {
}
// Export the class as default
export default ProxyManager;
export default ProxyManager;
// Convenience functions for easier imports
export function getProxy(): ProxyInfo | null {
return ProxyManager.getInstance().getRandomProxy();
}
export function getRandomProxy(): ProxyInfo | null {
return ProxyManager.getInstance().getRandomProxy();
}
export function getAllProxies(): ProxyInfo[] {
return ProxyManager.getInstance().getAllProxies();
}
export function getWorkingProxies(): ProxyInfo[] {
return ProxyManager.getInstance().getWorkingProxies();
}
export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
return ProxyManager.getInstance().updateProxies(proxies);
}