diff --git a/apps/data-service/src/providers/qm.tasks.ts b/apps/data-service/src/providers/qm.tasks.ts index 2ca6c08..f4117df 100644 --- a/apps/data-service/src/providers/qm.tasks.ts +++ b/apps/data-service/src/providers/qm.tasks.ts @@ -2,7 +2,7 @@ import { getRandomUserAgent } from '@stock-bot/http'; import { getLogger } from '@stock-bot/logger'; import { getMongoDBClient } from '@stock-bot/mongodb-client'; import { QueueManager } from '@stock-bot/queue'; -import { ProxyManager } from '@stock-bot/utils'; +import { getRandomProxy } from '@stock-bot/utils'; // Shared instances (module-scoped, not global) let isInitialized = false; // Track if resources are initialized @@ -90,7 +90,7 @@ export async function createSessions(): Promise { while (sessionCache[sessionId].length < 50) { logger.info(`Creating new session for ${sessionId}`); - const proxyInfo = await ProxyManager.getInstance().getRandomProxy(); + const proxyInfo = await getRandomProxy(); if (!proxyInfo) { logger.error('No proxy available for QM session creation'); break; // Skip session creation if no proxy is available @@ -294,7 +294,7 @@ async function searchAndSpawnJobs( // API call function to search symbols via QM async function searchQMSymbolsAPI(query: string): Promise { - const proxyInfo = await ProxyManager.getInstance().getRandomProxy(); + const proxyInfo = await getRandomProxy(); if (!proxyInfo) { throw new Error('No proxy available for QM API call'); diff --git a/apps/data-service/src/providers/webshare.provider.ts b/apps/data-service/src/providers/webshare.provider.ts index 9299b09..5f8cfb1 100644 --- a/apps/data-service/src/providers/webshare.provider.ts +++ b/apps/data-service/src/providers/webshare.provider.ts @@ -7,7 +7,7 @@ import { handlerRegistry, type HandlerConfigWithSchedule, } from '@stock-bot/queue'; -import { ProxyManager } from '@stock-bot/utils'; +import { updateProxies } from '@stock-bot/utils'; const logger = getLogger('webshare-provider'); @@ -28,7 +28,7 @@ export function initializeWebShareProvider() { if (proxies.length > 0) { // Update the centralized proxy manager - await ProxyManager.getInstance().updateProxies(proxies); + await updateProxies(proxies); logger.info('Updated proxy manager with WebShare proxies', { count: proxies.length, @@ -57,40 +57,6 @@ export function initializeWebShareProvider() { }; } }), - - 'validate-proxies': createJobHandler(async () => { - logger.info('Validating existing proxies'); - const { validateStoredProxies } = await import('./webshare.tasks'); - - try { - const validationResults = await validateStoredProxies(); - - // Update proxy manager with validated proxies - await ProxyManager.getInstance().updateProxies(validationResults.workingProxies); - - logger.info('Proxy validation completed', { - totalChecked: validationResults.totalChecked, - workingCount: validationResults.workingCount, - successRate: ((validationResults.workingCount / validationResults.totalChecked) * 100).toFixed(1) + '%', - }); - - return validationResults; - } catch (error) { - logger.error('Failed to validate proxies', { error }); - return { - workingProxies: [], - totalChecked: 0, - workingCount: 0, - error: error instanceof Error ? error.message : 'Unknown error', - }; - } - }), - - 'get-stats': createJobHandler(async () => { - const stats = ProxyManager.getInstance().getStats(); - logger.info('Proxy manager statistics', stats); - return stats; - }), }, scheduledJobs: [ @@ -103,22 +69,6 @@ export function initializeWebShareProvider() { description: 'Fetch fresh proxies from WebShare API', immediately: true, // Run on startup }, - { - type: 'webshare-validate', - operation: 'validate-proxies', - payload: {}, - cronPattern: '0 */2 * * *', // Every 2 hours - priority: 4, - description: 'Validate and clean existing proxies', - }, - { - type: 'webshare-stats', - operation: 'get-stats', - payload: {}, - cronPattern: '0 * * * *', // Every hour - priority: 5, - description: 'Log proxy manager statistics', - }, ], }; @@ -126,20 +76,7 @@ export function initializeWebShareProvider() { logger.debug('WebShare provider registered successfully'); } -// Legacy function for backward compatibility - now uses centralized proxy manager -export async function getProxy(): Promise { - const proxy = ProxyManager.getInstance().getRandomProxy(); - if (!proxy) { - return null; - } - - // Convert ProxyInfo back to string format for backward compatibility - const auth = proxy.username && proxy.password ? `${proxy.username}:${proxy.password}@` : ''; - return `${proxy.protocol}://${auth}${proxy.host}:${proxy.port}`; -} - export const webShareProvider = { initialize: initializeWebShareProvider, - getProxy, }; diff --git a/apps/data-service/src/providers/webshare.tasks.ts b/apps/data-service/src/providers/webshare.tasks.ts index 0fe2299..3024856 100644 --- a/apps/data-service/src/providers/webshare.tasks.ts +++ b/apps/data-service/src/providers/webshare.tasks.ts @@ -1,9 +1,8 @@ /** - * WebShare Tasks - API integration and proxy validation + * WebShare Tasks - API integration */ import { getLogger } from '@stock-bot/logger'; -import { HttpClient, type ProxyInfo } from '@stock-bot/http'; -import { ProxyManager } from '@stock-bot/utils'; +import { type ProxyInfo } from '@stock-bot/http'; const logger = getLogger('webshare-tasks'); @@ -65,7 +64,7 @@ export async function fetchWebShareProxies(): Promise { port: proxy.port, username: proxy.username, password: proxy.password, - isWorking: true, // Assume working until proven otherwise + isWorking: true, // WebShare provides working proxies firstSeen: new Date(), lastChecked: new Date(), })); @@ -80,174 +79,4 @@ export async function fetchWebShareProxies(): Promise { logger.error('Failed to fetch proxies from WebShare', { error }); return []; } -} - -/** - * Validate stored proxies by testing connectivity - */ -export async function validateStoredProxies(): Promise<{ - workingProxies: ProxyInfo[]; - totalChecked: number; - workingCount: number; -}> { - const httpClient = new HttpClient({ timeout: 10000 }, logger); - const testUrl = 'https://httpbin.org/ip'; // Simple IP echo service - - // Get all proxies from proxy manager - const allProxies = ProxyManager.getInstance().getAllProxies(); - - if (allProxies.length === 0) { - logger.warn('No proxies available for validation'); - return { - workingProxies: [], - totalChecked: 0, - workingCount: 0, - }; - } - - logger.info('Starting proxy validation', { totalProxies: allProxies.length }); - - const workingProxies: ProxyInfo[] = []; - const validationPromises = allProxies.map(async (proxy) => { - try { - const startTime = Date.now(); - const response = await httpClient.get(testUrl, { - proxy, - timeout: 10000, - }); - - const responseTime = Date.now() - startTime; - const isWorking = response.status === 200; - - if (isWorking) { - // Update proxy with success metrics - const updatedProxy: ProxyInfo = { - ...proxy, - isWorking: true, - lastChecked: new Date(), - responseTime, - total: (proxy.total || 0) + 1, - working: (proxy.working || 0) + 1, - averageResponseTime: proxy.averageResponseTime - ? (proxy.averageResponseTime + responseTime) / 2 - : responseTime, - }; - - // Calculate success rate - updatedProxy.successRate = updatedProxy.total > 0 - ? (updatedProxy.working / updatedProxy.total) * 100 - : 100; - - workingProxies.push(updatedProxy); - - logger.debug('Proxy validation successful', { - host: proxy.host, - port: proxy.port, - responseTime, - successRate: updatedProxy.successRate?.toFixed(1) + '%', - }); - } else { - logger.debug('Proxy validation failed', { - host: proxy.host, - port: proxy.port, - status: response.status, - }); - } - } catch (error) { - // Update proxy with failure metrics - const updatedProxy: ProxyInfo = { - ...proxy, - isWorking: false, - lastChecked: new Date(), - error: error instanceof Error ? error.message : 'Unknown error', - total: (proxy.total || 0) + 1, - working: proxy.working || 0, - }; - - // Calculate success rate - updatedProxy.successRate = updatedProxy.total > 0 - ? (updatedProxy.working / updatedProxy.total) * 100 - : 0; - - logger.debug('Proxy validation error', { - host: proxy.host, - port: proxy.port, - error: error instanceof Error ? error.message : 'Unknown error', - }); - } - }); - - // Wait for all validations to complete - await Promise.all(validationPromises); - - const results = { - workingProxies, - totalChecked: allProxies.length, - workingCount: workingProxies.length, - }; - - logger.info('Proxy validation completed', { - totalChecked: results.totalChecked, - workingCount: results.workingCount, - successRate: ((results.workingCount / results.totalChecked) * 100).toFixed(1) + '%', - }); - - return results; -} - -/** - * Test a single proxy for connectivity - */ -export async function testProxy(proxy: ProxyInfo): Promise { - const httpClient = new HttpClient({ timeout: 10000 }, logger); - const testUrl = 'https://httpbin.org/ip'; - - try { - const startTime = Date.now(); - const response = await httpClient.get(testUrl, { - proxy, - timeout: 10000, - }); - - const responseTime = Date.now() - startTime; - const isWorking = response.status === 200; - - const updatedProxy: ProxyInfo = { - ...proxy, - isWorking, - lastChecked: new Date(), - responseTime, - total: (proxy.total || 0) + 1, - working: isWorking ? (proxy.working || 0) + 1 : (proxy.working || 0), - }; - - // Calculate success rate - updatedProxy.successRate = updatedProxy.total > 0 - ? (updatedProxy.working / updatedProxy.total) * 100 - : (isWorking ? 100 : 0); - - // Update average response time - if (isWorking && responseTime) { - updatedProxy.averageResponseTime = proxy.averageResponseTime - ? (proxy.averageResponseTime + responseTime) / 2 - : responseTime; - } - - return updatedProxy; - } catch (error) { - const updatedProxy: ProxyInfo = { - ...proxy, - isWorking: false, - lastChecked: new Date(), - error: error instanceof Error ? error.message : 'Unknown error', - total: (proxy.total || 0) + 1, - working: proxy.working || 0, - }; - - updatedProxy.successRate = updatedProxy.total > 0 - ? (updatedProxy.working / updatedProxy.total) * 100 - : 0; - - return updatedProxy; - } } \ No newline at end of file diff --git a/libs/proxy/package.json b/libs/proxy/package.json deleted file mode 100644 index bbef6d0..0000000 --- a/libs/proxy/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@stock-bot/proxy", - "version": "1.0.0", - "description": "Simple proxy management library", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "type": "module", - "scripts": { - "build": "tsc", - "test": "bun test", - "dev": "tsc --watch" - }, - "devDependencies": { - "@types/node": "^20.0.0", - "typescript": "^5.0.0" - }, - "peerDependencies": {} -} diff --git a/libs/proxy/src/index.ts b/libs/proxy/src/index.ts deleted file mode 100644 index d7ab99f..0000000 --- a/libs/proxy/src/index.ts +++ /dev/null @@ -1,104 +0,0 @@ -// Simple proxy list manager -let proxies: string[] = []; -let currentIndex = 0; - -const DEFAULT_PROXY_URL = - 'https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all'; - -/** - * Fetch proxy list from URL and store in module - */ -export async function refreshProxies(fetchUrl: string = DEFAULT_PROXY_URL): Promise { - try { - const response = await fetch(fetchUrl); - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - const data = await response.text(); - const newProxies = data - .trim() - .split('\n') - .map(line => line.trim()) - .filter(line => line && line.includes(':')) - .map(line => { - // Convert host:port to http://host:port format - return line.startsWith('http') ? line : `http://${line}`; - }); - - proxies = newProxies; - currentIndex = 0; - - return proxies; - } catch (error) { - throw new Error(`Failed to fetch proxies: ${error}`); - } -} - -/** - * Get next proxy URL in round-robin fashion - */ -export function getProxyURL(): string | null { - if (proxies.length === 0) { - return null; - } - - const proxy = proxies[currentIndex]; - currentIndex = (currentIndex + 1) % proxies.length; - if (!proxy) { - return null; - } - // Ensure the proxy URL is in http://host:port format - return proxy; -} - -/** - * Get multiple proxy URLs - */ -export function getProxyURLs(count: number): string[] { - const urls: string[] = []; - for (let i = 0; i < count; i++) { - const url = getProxyURL(); - if (url) { - urls.push(url); - } - } - return urls; -} - -/** - * Get random proxy URL - */ -export function getRandomProxyURL(): string | null { - if (proxies.length === 0) { - return null; - } - - const randomIndex = Math.floor(Math.random() * proxies.length); - const proxy = proxies[randomIndex]; - if (!proxy) { - return null; - } - return proxy; -} - -/** - * Get current proxy count - */ -export function getProxyCount(): number { - return proxies.length; -} - -/** - * Get all proxies - */ -export function getAllProxies(): string[] { - return [...proxies]; -} - -/** - * Initialize proxy manager with initial fetch - */ -export async function initializeProxies(fetchUrl?: string): Promise { - await refreshProxies(fetchUrl); -} diff --git a/libs/proxy/src/types.ts b/libs/proxy/src/types.ts deleted file mode 100644 index a4f94ed..0000000 --- a/libs/proxy/src/types.ts +++ /dev/null @@ -1,22 +0,0 @@ -export interface ProxyInfo { - host: string; - port: number; - protocol: 'http' | 'https' | 'socks4' | 'socks5'; - username?: string; - password?: string; - country?: string; - isActive?: boolean; -} - -export interface ProxyManagerOptions { - fetchUrl?: string; - refreshIntervalMs?: number; - maxRetries?: number; - timeout?: number; -} - -export interface ProxyResponse { - proxies: ProxyInfo[]; - totalCount: number; - activeCount: number; -} diff --git a/libs/proxy/tsconfig.json b/libs/proxy/tsconfig.json deleted file mode 100644 index 969ce3b..0000000 --- a/libs/proxy/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../tsconfig.lib.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"], - "references": [ - ] -} diff --git a/libs/proxy/turbo.json b/libs/proxy/turbo.json deleted file mode 100644 index 6d18e62..0000000 --- a/libs/proxy/turbo.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "extends": ["//"], - "tasks": { - "build": { - "dependsOn": [], - "outputs": ["dist/**"], - "inputs": [ - "src/**", - "package.json", - "tsconfig.json", - "!**/*.test.ts", - "!**/*.spec.ts", - "!**/test/**", - "!**/tests/**", - "!**/__tests__/**" - ] - } - } -} diff --git a/libs/utils/src/proxy/index.ts b/libs/utils/src/proxy/index.ts index 56a1ecc..91e1b41 100644 --- a/libs/utils/src/proxy/index.ts +++ b/libs/utils/src/proxy/index.ts @@ -1,5 +1,12 @@ /** * Proxy management utilities */ -export { default as ProxyManager } from './proxy-manager'; +export { + default as ProxyManager, + getProxy, + getRandomProxy, + getAllProxies, + getWorkingProxies, + updateProxies +} from './proxy-manager'; export type { ProxyInfo } from '@stock-bot/http'; // Re-export for convenience \ No newline at end of file diff --git a/libs/utils/src/proxy/proxy-manager.ts b/libs/utils/src/proxy/proxy-manager.ts index 296ec20..9a306a6 100644 --- a/libs/utils/src/proxy/proxy-manager.ts +++ b/libs/utils/src/proxy/proxy-manager.ts @@ -180,27 +180,6 @@ export class ProxyManager { } } - /** - * Get proxy statistics - */ - getStats(): { - totalProxies: number; - workingProxies: number; - lastUpdate: Date | null; - successRate: number; - } { - const workingProxies = this.proxies.filter(p => p.isWorking !== false); - const totalSuccessRate = this.proxies.reduce((sum, p) => sum + (p.successRate || 0), 0); - const avgSuccessRate = this.proxies.length > 0 ? totalSuccessRate / this.proxies.length : 0; - - return { - totalProxies: this.proxies.length, - workingProxies: workingProxies.length, - lastUpdate: this.lastUpdate, - successRate: avgSuccessRate, - }; - } - /** * Clear all proxies from memory and cache */ @@ -274,4 +253,25 @@ export class ProxyManager { } // Export the class as default -export default ProxyManager; \ No newline at end of file +export default ProxyManager; + +// Convenience functions for easier imports +export function getProxy(): ProxyInfo | null { + return ProxyManager.getInstance().getRandomProxy(); +} + +export function getRandomProxy(): ProxyInfo | null { + return ProxyManager.getInstance().getRandomProxy(); +} + +export function getAllProxies(): ProxyInfo[] { + return ProxyManager.getInstance().getAllProxies(); +} + +export function getWorkingProxies(): ProxyInfo[] { + return ProxyManager.getInstance().getWorkingProxies(); +} + +export async function updateProxies(proxies: ProxyInfo[]): Promise { + return ProxyManager.getInstance().updateProxies(proxies); +} \ No newline at end of file