clean up proxyManager

apps/data-service/src/providers/qm.tasks.ts

@@ -2,7 +2,7 @@ import { getRandomUserAgent } from '@stock-bot/http';
 import { getLogger } from '@stock-bot/logger';
 import { getMongoDBClient } from '@stock-bot/mongodb-client';
 import { QueueManager } from '@stock-bot/queue';
-import { ProxyManager } from '@stock-bot/utils';
+import { getRandomProxy } from '@stock-bot/utils';
 
 // Shared instances (module-scoped, not global)
 let isInitialized = false; // Track if resources are initialized
@@ -90,7 +90,7 @@ export async function createSessions(): Promise<void> {
 
       while (sessionCache[sessionId].length < 50) {
         logger.info(`Creating new session for ${sessionId}`);
-        const proxyInfo = await ProxyManager.getInstance().getRandomProxy();
+        const proxyInfo = await getRandomProxy();
         if (!proxyInfo) {
           logger.error('No proxy available for QM session creation');
           break; // Skip session creation if no proxy is available
@@ -294,7 +294,7 @@ async function searchAndSpawnJobs(
 
 // API call function to search symbols via QM
 async function searchQMSymbolsAPI(query: string): Promise<string[]> {
-  const proxyInfo = await ProxyManager.getInstance().getRandomProxy();
+  const proxyInfo = await getRandomProxy();
 
   if (!proxyInfo) {
     throw new Error('No proxy available for QM API call');

apps/data-service/src/providers/webshare.provider.ts

@@ -7,7 +7,7 @@ import {
   handlerRegistry,
   type HandlerConfigWithSchedule,
 } from '@stock-bot/queue';
-import { ProxyManager } from '@stock-bot/utils';
+import { updateProxies } from '@stock-bot/utils';
 
 const logger = getLogger('webshare-provider');
 
@@ -28,7 +28,7 @@ export function initializeWebShareProvider() {
           
           if (proxies.length > 0) {
             // Update the centralized proxy manager
-            await ProxyManager.getInstance().updateProxies(proxies);
+            await updateProxies(proxies);
             
             logger.info('Updated proxy manager with WebShare proxies', { 
               count: proxies.length,
@@ -57,40 +57,6 @@ export function initializeWebShareProvider() {
           };
         }
       }),
-
-      'validate-proxies': createJobHandler(async () => {
-        logger.info('Validating existing proxies');
-        const { validateStoredProxies } = await import('./webshare.tasks');
-        
-        try {
-          const validationResults = await validateStoredProxies();
-          
-          // Update proxy manager with validated proxies
-          await ProxyManager.getInstance().updateProxies(validationResults.workingProxies);
-          
-          logger.info('Proxy validation completed', {
-            totalChecked: validationResults.totalChecked,
-            workingCount: validationResults.workingCount,
-            successRate: ((validationResults.workingCount / validationResults.totalChecked) * 100).toFixed(1) + '%',
-          });
-          
-          return validationResults;
-        } catch (error) {
-          logger.error('Failed to validate proxies', { error });
-          return {
-            workingProxies: [],
-            totalChecked: 0,
-            workingCount: 0,
-            error: error instanceof Error ? error.message : 'Unknown error',
-          };
-        }
-      }),
-
-      'get-stats': createJobHandler(async () => {
-        const stats = ProxyManager.getInstance().getStats();
-        logger.info('Proxy manager statistics', stats);
-        return stats;
-      }),
     },
 
     scheduledJobs: [
@@ -103,22 +69,6 @@ export function initializeWebShareProvider() {
         description: 'Fetch fresh proxies from WebShare API',
         immediately: true, // Run on startup
       },
-      {
-        type: 'webshare-validate',
-        operation: 'validate-proxies',
-        payload: {},
-        cronPattern: '0 */2 * * *', // Every 2 hours
-        priority: 4,
-        description: 'Validate and clean existing proxies',
-      },
-      {
-        type: 'webshare-stats',
-        operation: 'get-stats',
-        payload: {},
-        cronPattern: '0 * * * *', // Every hour
-        priority: 5,
-        description: 'Log proxy manager statistics',
-      },
     ],
   };
 
@@ -126,20 +76,7 @@ export function initializeWebShareProvider() {
   logger.debug('WebShare provider registered successfully');
 }
 
-// Legacy function for backward compatibility - now uses centralized proxy manager
-export async function getProxy(): Promise<string | null> {
-  const proxy = ProxyManager.getInstance().getRandomProxy();
-  if (!proxy) {
-    return null;
-  }
-  
-  // Convert ProxyInfo back to string format for backward compatibility
-  const auth = proxy.username && proxy.password ? `${proxy.username}:${proxy.password}@` : '';
-  return `${proxy.protocol}://${auth}${proxy.host}:${proxy.port}`;
-}
-
 export const webShareProvider = {
   initialize: initializeWebShareProvider,
-  getProxy,
 };
 

apps/data-service/src/providers/webshare.tasks.ts

@@ -1,9 +1,8 @@
 /**
- * WebShare Tasks - API integration and proxy validation
+ * WebShare Tasks - API integration
  */
 import { getLogger } from '@stock-bot/logger';
-import { HttpClient, type ProxyInfo } from '@stock-bot/http';
+import { type ProxyInfo } from '@stock-bot/http';
-import { ProxyManager } from '@stock-bot/utils';
 
 const logger = getLogger('webshare-tasks');
 
@@ -65,7 +64,7 @@ export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
       port: proxy.port,
       username: proxy.username,
       password: proxy.password,
-      isWorking: true, // Assume working until proven otherwise
+      isWorking: true, // WebShare provides working proxies
       firstSeen: new Date(),
       lastChecked: new Date(),
     }));
@@ -80,174 +79,4 @@ export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
     logger.error('Failed to fetch proxies from WebShare', { error });
     return [];
   }
-}
-
-/**
- * Validate stored proxies by testing connectivity
- */
-export async function validateStoredProxies(): Promise<{
-  workingProxies: ProxyInfo[];
-  totalChecked: number;
-  workingCount: number;
-}> {
-  const httpClient = new HttpClient({ timeout: 10000 }, logger);
-  const testUrl = 'https://httpbin.org/ip'; // Simple IP echo service
-  
-  // Get all proxies from proxy manager
-  const allProxies = ProxyManager.getInstance().getAllProxies();
-  
-  if (allProxies.length === 0) {
-    logger.warn('No proxies available for validation');
-    return {
-      workingProxies: [],
-      totalChecked: 0,
-      workingCount: 0,
-    };
-  }
-
-  logger.info('Starting proxy validation', { totalProxies: allProxies.length });
-  
-  const workingProxies: ProxyInfo[] = [];
-  const validationPromises = allProxies.map(async (proxy) => {
-    try {
-      const startTime = Date.now();
-      const response = await httpClient.get(testUrl, {
-        proxy,
-        timeout: 10000,
-      });
-
-      const responseTime = Date.now() - startTime;
-      const isWorking = response.status === 200;
-
-      if (isWorking) {
-        // Update proxy with success metrics
-        const updatedProxy: ProxyInfo = {
-          ...proxy,
-          isWorking: true,
-          lastChecked: new Date(),
-          responseTime,
-          total: (proxy.total || 0) + 1,
-          working: (proxy.working || 0) + 1,
-          averageResponseTime: proxy.averageResponseTime 
-            ? (proxy.averageResponseTime + responseTime) / 2 
-            : responseTime,
-        };
-        
-        // Calculate success rate
-        updatedProxy.successRate = updatedProxy.total > 0 
-          ? (updatedProxy.working / updatedProxy.total) * 100 
-          : 100;
-
-        workingProxies.push(updatedProxy);
-
-        logger.debug('Proxy validation successful', {
-          host: proxy.host,
-          port: proxy.port,
-          responseTime,
-          successRate: updatedProxy.successRate?.toFixed(1) + '%',
-        });
-      } else {
-        logger.debug('Proxy validation failed', {
-          host: proxy.host,
-          port: proxy.port,
-          status: response.status,
-        });
-      }
-    } catch (error) {
-      // Update proxy with failure metrics
-      const updatedProxy: ProxyInfo = {
-        ...proxy,
-        isWorking: false,
-        lastChecked: new Date(),
-        error: error instanceof Error ? error.message : 'Unknown error',
-        total: (proxy.total || 0) + 1,
-        working: proxy.working || 0,
-      };
-
-      // Calculate success rate
-      updatedProxy.successRate = updatedProxy.total > 0 
-        ? (updatedProxy.working / updatedProxy.total) * 100 
-        : 0;
-
-      logger.debug('Proxy validation error', {
-        host: proxy.host,
-        port: proxy.port,
-        error: error instanceof Error ? error.message : 'Unknown error',
-      });
-    }
-  });
-
-  // Wait for all validations to complete
-  await Promise.all(validationPromises);
-
-  const results = {
-    workingProxies,
-    totalChecked: allProxies.length,
-    workingCount: workingProxies.length,
-  };
-
-  logger.info('Proxy validation completed', {
-    totalChecked: results.totalChecked,
-    workingCount: results.workingCount,
-    successRate: ((results.workingCount / results.totalChecked) * 100).toFixed(1) + '%',
-  });
-
-  return results;
-}
-
-/**
- * Test a single proxy for connectivity
- */
-export async function testProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
-  const httpClient = new HttpClient({ timeout: 10000 }, logger);
-  const testUrl = 'https://httpbin.org/ip';
-
-  try {
-    const startTime = Date.now();
-    const response = await httpClient.get(testUrl, {
-      proxy,
-      timeout: 10000,
-    });
-
-    const responseTime = Date.now() - startTime;
-    const isWorking = response.status === 200;
-
-    const updatedProxy: ProxyInfo = {
-      ...proxy,
-      isWorking,
-      lastChecked: new Date(),
-      responseTime,
-      total: (proxy.total || 0) + 1,
-      working: isWorking ? (proxy.working || 0) + 1 : (proxy.working || 0),
-    };
-
-    // Calculate success rate
-    updatedProxy.successRate = updatedProxy.total > 0 
-      ? (updatedProxy.working / updatedProxy.total) * 100 
-      : (isWorking ? 100 : 0);
-
-    // Update average response time
-    if (isWorking && responseTime) {
-      updatedProxy.averageResponseTime = proxy.averageResponseTime
-        ? (proxy.averageResponseTime + responseTime) / 2
-        : responseTime;
-    }
-
-    return updatedProxy;
-  } catch (error) {
-    const updatedProxy: ProxyInfo = {
-      ...proxy,
-      isWorking: false,
-      lastChecked: new Date(),
-      error: error instanceof Error ? error.message : 'Unknown error',
-      total: (proxy.total || 0) + 1,
-      working: proxy.working || 0,
-    };
-
-    updatedProxy.successRate = updatedProxy.total > 0 
-      ? (updatedProxy.working / updatedProxy.total) * 100 
-      : 0;
-
-    return updatedProxy;
-  }
 }

libs/proxy/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "@stock-bot/proxy",
-  "version": "1.0.0",
-  "description": "Simple proxy management library",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "type": "module",
-  "scripts": {
-    "build": "tsc",
-    "test": "bun test",
-    "dev": "tsc --watch"
-  },
-  "devDependencies": {
-    "@types/node": "^20.0.0",
-    "typescript": "^5.0.0"
-  },
-  "peerDependencies": {}
-}

libs/proxy/src/index.ts

@@ -1,104 +0,0 @@
-// Simple proxy list manager
-let proxies: string[] = [];
-let currentIndex = 0;
-
-const DEFAULT_PROXY_URL =
-  'https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all';
-
-/**
- * Fetch proxy list from URL and store in module
- */
-export async function refreshProxies(fetchUrl: string = DEFAULT_PROXY_URL): Promise<string[]> {
-  try {
-    const response = await fetch(fetchUrl);
-    if (!response.ok) {
-      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-    }
-
-    const data = await response.text();
-    const newProxies = data
-      .trim()
-      .split('\n')
-      .map(line => line.trim())
-      .filter(line => line && line.includes(':'))
-      .map(line => {
-        // Convert host:port to http://host:port format
-        return line.startsWith('http') ? line : `http://${line}`;
-      });
-
-    proxies = newProxies;
-    currentIndex = 0;
-
-    return proxies;
-  } catch (error) {
-    throw new Error(`Failed to fetch proxies: ${error}`);
-  }
-}
-
-/**
- * Get next proxy URL in round-robin fashion
- */
-export function getProxyURL(): string | null {
-  if (proxies.length === 0) {
-    return null;
-  }
-
-  const proxy = proxies[currentIndex];
-  currentIndex = (currentIndex + 1) % proxies.length;
-  if (!proxy) {
-    return null;
-  }
-  // Ensure the proxy URL is in http://host:port format
-  return proxy;
-}
-
-/**
- * Get multiple proxy URLs
- */
-export function getProxyURLs(count: number): string[] {
-  const urls: string[] = [];
-  for (let i = 0; i < count; i++) {
-    const url = getProxyURL();
-    if (url) {
-      urls.push(url);
-    }
-  }
-  return urls;
-}
-
-/**
- * Get random proxy URL
- */
-export function getRandomProxyURL(): string | null {
-  if (proxies.length === 0) {
-    return null;
-  }
-
-  const randomIndex = Math.floor(Math.random() * proxies.length);
-  const proxy = proxies[randomIndex];
-  if (!proxy) {
-    return null;
-  }
-  return proxy;
-}
-
-/**
- * Get current proxy count
- */
-export function getProxyCount(): number {
-  return proxies.length;
-}
-
-/**
- * Get all proxies
- */
-export function getAllProxies(): string[] {
-  return [...proxies];
-}
-
-/**
- * Initialize proxy manager with initial fetch
- */
-export async function initializeProxies(fetchUrl?: string): Promise<void> {
-  await refreshProxies(fetchUrl);
-}

libs/proxy/src/types.ts

@@ -1,22 +0,0 @@
-export interface ProxyInfo {
-  host: string;
-  port: number;
-  protocol: 'http' | 'https' | 'socks4' | 'socks5';
-  username?: string;
-  password?: string;
-  country?: string;
-  isActive?: boolean;
-}
-
-export interface ProxyManagerOptions {
-  fetchUrl?: string;
-  refreshIntervalMs?: number;
-  maxRetries?: number;
-  timeout?: number;
-}
-
-export interface ProxyResponse {
-  proxies: ProxyInfo[];
-  totalCount: number;
-  activeCount: number;
-}

libs/proxy/tsconfig.json

@@ -1,10 +0,0 @@
-{
-  "extends": "../../tsconfig.lib.json",
-  "compilerOptions": {
-    "outDir": "./dist",
-    "rootDir": "./src"
-  },
-  "include": ["src/**/*"],
-  "references": [
-  ]
-}

libs/proxy/turbo.json

@@ -1,19 +0,0 @@
-{
-  "extends": ["//"],
-  "tasks": {
-    "build": {
-      "dependsOn": [],
-      "outputs": ["dist/**"],
-      "inputs": [
-        "src/**",
-        "package.json",
-        "tsconfig.json",
-        "!**/*.test.ts",
-        "!**/*.spec.ts",
-        "!**/test/**",
-        "!**/tests/**",
-        "!**/__tests__/**"
-      ]
-    }
-  }
-}

libs/utils/src/proxy/index.ts

@@ -1,5 +1,12 @@
 /**
  * Proxy management utilities
  */
-export { default as ProxyManager } from './proxy-manager';
+export { 
+  default as ProxyManager,
+  getProxy,
+  getRandomProxy,
+  getAllProxies,
+  getWorkingProxies,
+  updateProxies
+} from './proxy-manager';
 export type { ProxyInfo } from '@stock-bot/http'; // Re-export for convenience

libs/utils/src/proxy/proxy-manager.ts

@@ -180,27 +180,6 @@ export class ProxyManager {
     }
   }
 
-  /**
-   * Get proxy statistics
-   */
-  getStats(): {
-    totalProxies: number;
-    workingProxies: number;
-    lastUpdate: Date | null;
-    successRate: number;
-  } {
-    const workingProxies = this.proxies.filter(p => p.isWorking !== false);
-    const totalSuccessRate = this.proxies.reduce((sum, p) => sum + (p.successRate || 0), 0);
-    const avgSuccessRate = this.proxies.length > 0 ? totalSuccessRate / this.proxies.length : 0;
-
-    return {
-      totalProxies: this.proxies.length,
-      workingProxies: workingProxies.length,
-      lastUpdate: this.lastUpdate,
-      successRate: avgSuccessRate,
-    };
-  }
-
   /**
    * Clear all proxies from memory and cache
    */
@@ -274,4 +253,25 @@ export class ProxyManager {
 }
 
 // Export the class as default
-export default ProxyManager;
+export default ProxyManager;
+
+// Convenience functions for easier imports
+export function getProxy(): ProxyInfo | null {
+  return ProxyManager.getInstance().getRandomProxy();
+}
+
+export function getRandomProxy(): ProxyInfo | null {
+  return ProxyManager.getInstance().getRandomProxy();
+}
+
+export function getAllProxies(): ProxyInfo[] {
+  return ProxyManager.getInstance().getAllProxies();
+}
+
+export function getWorkingProxies(): ProxyInfo[] {
+  return ProxyManager.getInstance().getWorkingProxies();
+}
+
+export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
+  return ProxyManager.getInstance().updateProxies(proxies);
+}