refactored out proxymanager from webshare to make it reusable

apps/data-service/src/providers/webshare.tasks.ts

@@ -0,0 +1,253 @@
+/**
+ * WebShare Tasks - API integration and proxy validation
+ */
+import { getLogger } from '@stock-bot/logger';
+import { HttpClient, type ProxyInfo } from '@stock-bot/http';
+import { proxyManager } from '@stock-bot/utils';
+
+const logger = getLogger('webshare-tasks');
+
+/**
+ * Fetch proxies from WebShare API and convert to ProxyInfo format
+ */
+export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
+  try {
+    // Get configuration from config system
+    const { getConfig } = await import('@stock-bot/config');
+    const config = getConfig();
+
+    const apiKey = config.webshare?.apiKey;
+    const apiUrl = config.webshare?.apiUrl;
+
+    if (!apiKey || !apiUrl) {
+      logger.error('Missing WebShare configuration', {
+        hasApiKey: !!apiKey,
+        hasApiUrl: !!apiUrl,
+      });
+      return [];
+    }
+
+    logger.info('Fetching proxies from WebShare API', { apiUrl });
+
+    const response = await fetch(`${apiUrl}proxy/list/?mode=direct&page=1&page_size=100`, {
+      method: 'GET',
+      headers: {
+        Authorization: `Token ${apiKey}`,
+        'Content-Type': 'application/json',
+      },
+    });
+
+    if (!response.ok) {
+      logger.error('WebShare API request failed', {
+        status: response.status,
+        statusText: response.statusText,
+      });
+      return [];
+    }
+
+    const data = await response.json();
+
+    if (!data.results || !Array.isArray(data.results)) {
+      logger.error('Invalid response format from WebShare API', { data });
+      return [];
+    }
+
+    // Transform proxy data to ProxyInfo format
+    const proxies: ProxyInfo[] = data.results.map((proxy: {
+      username: string;
+      password: string;
+      proxy_address: string;
+      port: number;
+    }) => ({
+      source: 'webshare',
+      protocol: 'http' as const,
+      host: proxy.proxy_address,
+      port: proxy.port,
+      username: proxy.username,
+      password: proxy.password,
+      isWorking: true, // Assume working until proven otherwise
+      firstSeen: new Date(),
+      lastChecked: new Date(),
+    }));
+
+    logger.info('Successfully fetched proxies from WebShare', {
+      count: proxies.length,
+      total: data.count || proxies.length,
+    });
+
+    return proxies;
+  } catch (error) {
+    logger.error('Failed to fetch proxies from WebShare', { error });
+    return [];
+  }
+}
+
+/**
+ * Validate stored proxies by testing connectivity
+ */
+export async function validateStoredProxies(): Promise<{
+  workingProxies: ProxyInfo[];
+  totalChecked: number;
+  workingCount: number;
+}> {
+  const httpClient = new HttpClient({ timeout: 10000 }, logger);
+  const testUrl = 'https://httpbin.org/ip'; // Simple IP echo service
+  
+  // Get all proxies from proxy manager
+  const allProxies = await proxyManager.getAllProxies();
+  
+  if (allProxies.length === 0) {
+    logger.warn('No proxies available for validation');
+    return {
+      workingProxies: [],
+      totalChecked: 0,
+      workingCount: 0,
+    };
+  }
+
+  logger.info('Starting proxy validation', { totalProxies: allProxies.length });
+  
+  const workingProxies: ProxyInfo[] = [];
+  const validationPromises = allProxies.map(async (proxy) => {
+    try {
+      const startTime = Date.now();
+      const response = await httpClient.get(testUrl, {
+        proxy,
+        timeout: 10000,
+      });
+
+      const responseTime = Date.now() - startTime;
+      const isWorking = response.status === 200;
+
+      if (isWorking) {
+        // Update proxy with success metrics
+        const updatedProxy: ProxyInfo = {
+          ...proxy,
+          isWorking: true,
+          lastChecked: new Date(),
+          responseTime,
+          total: (proxy.total || 0) + 1,
+          working: (proxy.working || 0) + 1,
+          averageResponseTime: proxy.averageResponseTime 
+            ? (proxy.averageResponseTime + responseTime) / 2 
+            : responseTime,
+        };
+        
+        // Calculate success rate
+        updatedProxy.successRate = updatedProxy.total > 0 
+          ? (updatedProxy.working / updatedProxy.total) * 100 
+          : 100;
+
+        workingProxies.push(updatedProxy);
+
+        logger.debug('Proxy validation successful', {
+          host: proxy.host,
+          port: proxy.port,
+          responseTime,
+          successRate: updatedProxy.successRate?.toFixed(1) + '%',
+        });
+      } else {
+        logger.debug('Proxy validation failed', {
+          host: proxy.host,
+          port: proxy.port,
+          status: response.status,
+        });
+      }
+    } catch (error) {
+      // Update proxy with failure metrics
+      const updatedProxy: ProxyInfo = {
+        ...proxy,
+        isWorking: false,
+        lastChecked: new Date(),
+        error: error instanceof Error ? error.message : 'Unknown error',
+        total: (proxy.total || 0) + 1,
+        working: proxy.working || 0,
+      };
+
+      // Calculate success rate
+      updatedProxy.successRate = updatedProxy.total > 0 
+        ? (updatedProxy.working / updatedProxy.total) * 100 
+        : 0;
+
+      logger.debug('Proxy validation error', {
+        host: proxy.host,
+        port: proxy.port,
+        error: error instanceof Error ? error.message : 'Unknown error',
+      });
+    }
+  });
+
+  // Wait for all validations to complete
+  await Promise.all(validationPromises);
+
+  const results = {
+    workingProxies,
+    totalChecked: allProxies.length,
+    workingCount: workingProxies.length,
+  };
+
+  logger.info('Proxy validation completed', {
+    totalChecked: results.totalChecked,
+    workingCount: results.workingCount,
+    successRate: ((results.workingCount / results.totalChecked) * 100).toFixed(1) + '%',
+  });
+
+  return results;
+}
+
+/**
+ * Test a single proxy for connectivity
+ */
+export async function testProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
+  const httpClient = new HttpClient({ timeout: 10000 }, logger);
+  const testUrl = 'https://httpbin.org/ip';
+
+  try {
+    const startTime = Date.now();
+    const response = await httpClient.get(testUrl, {
+      proxy,
+      timeout: 10000,
+    });
+
+    const responseTime = Date.now() - startTime;
+    const isWorking = response.status === 200;
+
+    const updatedProxy: ProxyInfo = {
+      ...proxy,
+      isWorking,
+      lastChecked: new Date(),
+      responseTime,
+      total: (proxy.total || 0) + 1,
+      working: isWorking ? (proxy.working || 0) + 1 : (proxy.working || 0),
+    };
+
+    // Calculate success rate
+    updatedProxy.successRate = updatedProxy.total > 0 
+      ? (updatedProxy.working / updatedProxy.total) * 100 
+      : (isWorking ? 100 : 0);
+
+    // Update average response time
+    if (isWorking && responseTime) {
+      updatedProxy.averageResponseTime = proxy.averageResponseTime
+        ? (proxy.averageResponseTime + responseTime) / 2
+        : responseTime;
+    }
+
+    return updatedProxy;
+  } catch (error) {
+    const updatedProxy: ProxyInfo = {
+      ...proxy,
+      isWorking: false,
+      lastChecked: new Date(),
+      error: error instanceof Error ? error.message : 'Unknown error',
+      total: (proxy.total || 0) + 1,
+      working: proxy.working || 0,
+    };
+
+    updatedProxy.successRate = updatedProxy.total > 0 
+      ? (updatedProxy.working / updatedProxy.total) * 100 
+      : 0;
+
+    return updatedProxy;
+  }
+}