test

apps/data-service/src/providers/proxy.tasks.ts

@@ -8,16 +8,21 @@ export interface ProxySource {
   id: string;
   url: string;
   protocol: string;
+  working?: number; // Optional, used for stats
+  total?: number; // Optional, used for stats
+  lastChecked?: Date; // Optional, used for stats
 }
 
 // Shared configuration and utilities
 const PROXY_CONFIG = {
   CACHE_KEY: 'proxy',
+  CACHE_STATS_KEY: 'proxy:stats',
   CACHE_TTL: 86400, // 24 hours
   CHECK_TIMEOUT: 7000,
   CHECK_IP: '99.246.102.205',
   CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
-  CONCURRENCY_LIMIT: 100,  PROXY_SOURCES: [
+  CONCURRENCY_LIMIT: 100,
+  PROXY_SOURCES: [
     {id: 'prxchk', url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt', protocol: 'http'},
     {id: 'casals', url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http', protocol: 'http'},
     {id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'},
@@ -52,6 +57,39 @@ let logger: ReturnType<typeof getLogger>;
 let cache: CacheProvider;
 let httpClient: HttpClient;
 let concurrencyLimit: ReturnType<typeof pLimit>;
+let proxyStats: ProxySource[] = []
+
+
+// make a function that takes in source id and a boolean success and updates the proxyStats array
+function updateProxyStats(sourceId: string, success: boolean) {
+  const source = proxyStats.find(s => s.id === sourceId);
+  if (source !== undefined && source !== null && source.working && source.total) {
+    source.total += 1;
+    if (success) {
+      source.working += 1;
+    }
+    return source;
+  } else {
+    logger.warn(`Unknown proxy source: ${sourceId}`);
+  }
+}
+
+// make a function that resets proxyStats
+async function resetProxyStats(): Promise<void> {
+    proxyStats = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
+      id: source.id,
+      total: 0,
+      working: 0,
+      lastChecked: new Date(),
+      protocol: source.protocol,
+      url: source.url,
+    }));
+    // for (const source of proxyStats) {
+    //   await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
+    // }
+    return Promise.resolve();
+}
+
 
 // Initialize shared resources
 function initializeSharedResources() {
@@ -103,6 +141,7 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
 
 export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
   initializeSharedResources();
+  await resetProxyStats();
   
   const sources = PROXY_CONFIG.PROXY_SOURCES.map(source => 
     concurrencyLimit(() => fetchProxiesFromSource(source))
@@ -194,19 +233,23 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
       responseTime: response.responseTime,
     };
 
-    if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
+    // if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
-      success = true;
+    //   success = true;
-      await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
+    //   await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
-    } else {
+    // } else {
-      await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
+    //   await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
-    }
+    // }
+
+    // if( proxy.source ){
+    //   updateProxyStats(proxy.source, success);
+    // }
 
     logger.debug('Proxy check completed', {
       host: proxy.host,
       port: proxy.port,
       isWorking,
     });
-
+    
     return result;
 
   } catch (error) {
@@ -220,9 +263,12 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
     };
 
     // If the proxy check failed, remove it from cache - success is here cause i think abort signal fails sometimes
-    if (!success) {
+    // if (!success) {
-      await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
+    //   await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result);
-    }
+    // }
+    // if( proxy.source ){
+    //   updateProxyStats(proxy.source, success);
+    // }
 
     logger.debug('Proxy check failed', {
       host: proxy.host,

apps/data-service/src/services/queue.service.ts

@@ -43,7 +43,7 @@ export class QueueService {
       enableReadyCheck: false,
       lazyConnect: true,
       // Disable Redis Cluster mode if you're using standalone Redis/Dragonfly
-      enableOfflineQueue: false
+      enableOfflineQueue: true
     };
 
     // Worker configuration