refactoring

apps/data-ingestion/src/handlers/qm/actions/session.action.ts

@@ -18,14 +18,14 @@ export async function checkSessions(handler: BaseHandler): Promise<{
   const cleanedCount = sessionManager.cleanupFailedSessions();
   // Check which session IDs need more sessions and queue creation jobs
   let queuedCount = 0;
-  for (const sessionId of Object.values(QM_SESSION_IDS)) {
+  for (const [sessionType, sessionId] of Object.entries(QM_SESSION_IDS)) {
     console.log(`Checking session ID: ${sessionId}`);
     if (sessionManager.needsMoreSessions(sessionId)) {
       const currentCount = sessionManager.getSessions(sessionId).length;
       const neededSessions = SESSION_CONFIG.MAX_SESSIONS - currentCount;
       for (let i = 0; i < neededSessions; i++) {
-        await handler.scheduleOperation('create-session', { sessionId });
-        handler.services.logger.log(`Queued job to create session for ${sessionId}`);
+        await handler.scheduleOperation('create-session', { sessionId , sessionType });
+        handler.services.logger.log(`Queued job to create session for ${sessionType}`);
         queuedCount++;
       }
     }
@@ -46,20 +46,24 @@ export async function createSingleSession(
   input: any
 ): Promise<{ sessionId: string; status: string; sessionType: string }> {
   
-  const { sessionId: sessionType = 'default' } = input || {};
+  const { sessionId, sessionType } = input || {};
   const sessionManager = QMSessionManager.getInstance();
   
-  // TODO: Get actual proxy and headers from proxy service
-  const session = {
-    // proxy: handler.services.getRandomProxy(),
-    headers: sessionManager.getQmHeaders(),
-    successfulCalls: 0,
-    failedCalls: 0,
-    lastUsed: new Date()
-  };
+  // Get proxy from proxy service
+  const proxyString = handler.services.proxy.getProxy();
+  
+  // const session = {
+  //   proxy: proxyString || 'http://proxy:8080',
+  //   headers: sessionManager.getQmHeaders(),
+  //   successfulCalls: 0,
+  //   failedCalls: 0,
+  //   lastUsed: new Date()
+  // };
+
+  handler.services.logger.info(`Creating session for ${sessionType}`)
   
   // Add session to manager
-  sessionManager.addSession(sessionType, session);
+  // sessionManager.addSession(sessionType, session);
   
   return {
     sessionId: sessionType,

bun.lock

@@ -314,6 +314,18 @@
         "typescript": "^5.3.0",
       },
     },
+    "libs/services/proxy": {
+      "name": "@stock-bot/proxy",
+      "version": "0.1.0",
+      "dependencies": {
+        "@stock-bot/cache": "workspace:*",
+        "@stock-bot/http": "workspace:*",
+        "@stock-bot/logger": "workspace:*",
+      },
+      "devDependencies": {
+        "typescript": "^5.0.0",
+      },
+    },
     "libs/services/queue": {
       "name": "@stock-bot/queue",
       "version": "1.0.0",
@@ -782,6 +794,8 @@
 
     "@stock-bot/postgres": ["@stock-bot/postgres@workspace:libs/data/postgres"],
 
+    "@stock-bot/proxy": ["@stock-bot/proxy@workspace:libs/services/proxy"],
+
     "@stock-bot/questdb": ["@stock-bot/questdb@workspace:libs/data/questdb"],
 
     "@stock-bot/queue": ["@stock-bot/queue@workspace:libs/services/queue"],

libs/core/di/src/adapters/service-adapter.ts

@@ -5,6 +5,7 @@
 
 import type { IServiceContainer } from '@stock-bot/handlers';
 import type { IDataIngestionServices } from '../service-interfaces';
+import { ProxyManager } from '@stock-bot/proxy';
 
 /**
  * Adapter that converts IDataIngestionServices to IServiceContainer
@@ -22,6 +23,10 @@ export class DataIngestionServiceAdapter implements IServiceContainer {
     // HTTP client not in current data services - will be added when needed
     return null;
   }
+  get proxy() { 
+    // Return singleton proxy manager instance
+    return ProxyManager.getInstance();
+  }
 
   // Database clients
   get mongodb() { return this.dataServices.mongodb; }

libs/core/di/src/service-factory.ts

@@ -5,6 +5,7 @@
 import { getLogger } from '@stock-bot/logger';
 import { ConnectionFactory } from './connection-factory';
 import { PoolSizeCalculator } from './pool-size-calculator';
+import { ProxyManager } from '@stock-bot/proxy';
 import type {
   IDataIngestionServices,
   IServiceFactory,
@@ -44,6 +45,10 @@ export class DataIngestionServiceFactory implements IServiceFactory {
         this.createQueueConnection(connectionFactory, config)
       ]);
 
+      // Initialize proxy manager
+      logger.info('Initializing proxy manager...');
+      await ProxyManager.initialize();
+
       const services: IDataIngestionServices = {
         mongodb: mongoPool.client,
         postgres: postgresPool.client,

libs/core/handlers/src/types/service-container.ts

@@ -3,6 +3,8 @@
  * Simple, comprehensive container with all services available
  */
 
+import type { ProxyManager } from '@stock-bot/proxy';
+
 /**
  * Universal service container with all common services
  * Designed to work across different service contexts (data-ingestion, processing, etc.)
@@ -13,6 +15,7 @@ export interface IServiceContainer {
   readonly cache: any;      // Cache provider (Redis/Dragonfly)
   readonly queue: any;      // Queue manager (BullMQ)
   readonly http: any;       // HTTP client with proxy support
+  readonly proxy: ProxyManager; // Proxy manager service
 
   // Database clients
   readonly mongodb: any;    // MongoDB client

libs/services/proxy/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@stock-bot/proxy",
+  "version": "0.1.0",
+  "description": "Proxy management and synchronization services",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@stock-bot/logger": "workspace:*",
+    "@stock-bot/cache": "workspace:*",
+    "@stock-bot/http": "workspace:*"
+  },
+  "devDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  }
+}

libs/services/proxy/src/index.ts

@@ -0,0 +1,36 @@
+/**
+ * Proxy Service Library
+ * Centralized proxy management and synchronization
+ */
+
+// Main classes
+export { ProxyManager } from './proxy-manager';
+export { ProxySyncService } from './proxy-sync';
+
+// Types
+export type { 
+  ProxyInfo, 
+  ProxyManagerConfig, 
+  ProxySyncConfig, 
+  ProxyStats 
+} from './types';
+
+// Convenience functions
+export {
+  getProxy,
+  getRandomProxy,
+  getAllProxies,
+  getWorkingProxies,
+  updateProxies,
+  getProxyStats
+} from './proxy-manager';
+
+export {
+  getProxySyncService,
+  startProxySync,
+  stopProxySync,
+  syncProxiesOnce
+} from './proxy-sync';
+
+// Default export
+export { ProxyManager as default } from './proxy-manager';

libs/services/proxy/src/proxy-manager.ts

@@ -0,0 +1,345 @@
+/**
+ * Centralized Proxy Manager - Handles proxy storage, retrieval, and caching
+ */
+import { createCache, type CacheProvider } from '@stock-bot/cache';
+import { getDatabaseConfig } from '@stock-bot/config';
+import { getLogger } from '@stock-bot/logger';
+import type { ProxyInfo, ProxyManagerConfig, ProxyStats } from './types';
+
+const logger = getLogger('proxy-manager');
+
+export class ProxyManager {
+  private static instance: ProxyManager | null = null;
+  private cache: CacheProvider;
+  private proxies: ProxyInfo[] = [];
+  private proxyIndex: number = 0;
+  private lastUpdate: Date | null = null;
+  private isInitialized = false;
+  private config: ProxyManagerConfig;
+
+  private constructor(config: ProxyManagerConfig = {}) {
+    this.config = {
+      cachePrefix: 'proxies:',
+      ttl: 86400, // 24 hours
+      enableMetrics: true,
+      ...config
+    };
+
+    const databaseConfig = getDatabaseConfig();
+    this.cache = createCache({
+      redisConfig: databaseConfig.dragonfly,
+      keyPrefix: this.config.cachePrefix,
+      ttl: this.config.ttl,
+      enableMetrics: this.config.enableMetrics,
+    });
+  }
+
+  /**
+   * Internal initialization - loads existing proxies from cache
+   */
+  private async initializeInternal(): Promise<void> {
+    if (this.isInitialized) {
+      return;
+    }
+
+    try {
+      logger.info('Initializing proxy manager...');
+      
+      // Wait for cache to be ready
+      await this.cache.waitForReady(10000); // Wait up to 10 seconds
+      logger.debug('Cache is ready');
+      
+      await this.loadFromCache();
+      this.isInitialized = true;
+      logger.info('Proxy manager initialized', {
+        proxiesLoaded: this.proxies.length,
+        lastUpdate: this.lastUpdate,
+      });
+    } catch (error) {
+      logger.error('Failed to initialize proxy manager', { error });
+      this.isInitialized = true; // Set to true anyway to avoid infinite retries
+    }
+  }
+
+  getProxy(): string | null {
+    if (this.proxies.length === 0) {
+      logger.warn('No proxies available in memory');
+      return null;
+    }
+    
+    // Cycle through proxies
+    if (this.proxyIndex >= this.proxies.length) {
+      this.proxyIndex = 0;
+    }
+    
+    const proxyInfo = this.proxies[this.proxyIndex++];
+    if (!proxyInfo) {
+      return null;
+    }
+    
+    // Build proxy URL with optional auth
+    let proxyUrl = `${proxyInfo.protocol}://`;
+    if (proxyInfo.username && proxyInfo.password) {
+      proxyUrl += `${proxyInfo.username}:${proxyInfo.password}@`;
+    }
+    proxyUrl += `${proxyInfo.host}:${proxyInfo.port}`;
+    
+    return proxyUrl;
+  }
+  /**
+   * Get a random working proxy from the available pool (synchronous)
+   */
+  getRandomProxy(): ProxyInfo | null {
+    // Ensure initialized
+    if (!this.isInitialized) {
+      throw new Error('ProxyManager not initialized');
+    }
+
+    // Return null if no proxies available
+    if (this.proxies.length === 0) {
+      logger.warn('No proxies available in memory');
+      return null;
+    }
+
+    // Filter for working proxies (not explicitly marked as non-working)
+    const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false);
+    
+    if (workingProxies.length === 0) {
+      logger.warn('No working proxies available');
+      return null;
+    }
+
+    // Return random proxy with preference for recently successful ones
+    const sortedProxies = workingProxies.sort((a, b) => {
+      // Prefer proxies with better success rates
+      const aRate = a.successRate || 0;
+      const bRate = b.successRate || 0;
+      return bRate - aRate;
+    });
+
+    // Take from top 50% of best performing proxies
+    const topProxies = sortedProxies.slice(0, Math.max(1, Math.floor(sortedProxies.length * 0.5)));
+    const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)];
+
+    if (!selectedProxy) {
+      logger.warn('No proxy selected from available pool');
+      return null;
+    }
+
+    logger.debug('Selected proxy', {
+      host: selectedProxy.host,
+      port: selectedProxy.port,
+      successRate: selectedProxy.successRate,
+      totalAvailable: workingProxies.length,
+    });
+
+    return selectedProxy;
+  }
+
+  /**
+   * Get all working proxies (synchronous)
+   */
+  getWorkingProxies(): ProxyInfo[] {
+    if (!this.isInitialized) {
+      throw new Error('ProxyManager not initialized');
+    }
+
+    return this.proxies.filter(proxy => proxy.isWorking !== false);
+  }
+
+  /**
+   * Get all proxies (working and non-working)
+   */
+  getAllProxies(): ProxyInfo[] {
+    if (!this.isInitialized) {
+      throw new Error('ProxyManager not initialized');
+    }
+
+    return [...this.proxies];
+  }
+
+  /**
+   * Get proxy statistics
+   */
+  getStats(): ProxyStats {
+    if (!this.isInitialized) {
+      throw new Error('ProxyManager not initialized');
+    }
+
+    return {
+      total: this.proxies.length,
+      working: this.proxies.filter(p => p.isWorking !== false).length,
+      failed: this.proxies.filter(p => p.isWorking === false).length,
+      lastUpdate: this.lastUpdate
+    };
+  }
+
+  /**
+   * Update the proxy pool with new proxies
+   */
+  async updateProxies(proxies: ProxyInfo[]): Promise<void> {
+    try {
+      logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
+      
+      this.proxies = proxies;
+      this.lastUpdate = new Date();
+
+      // Store to cache
+      await this.cache.set('active-proxies', proxies);
+      await this.cache.set('last-update', this.lastUpdate.toISOString());
+
+      const workingCount = proxies.filter(p => p.isWorking !== false).length;
+      logger.info('Proxy pool updated successfully', {
+        totalProxies: proxies.length,
+        workingProxies: workingCount,
+        lastUpdate: this.lastUpdate,
+      });
+    } catch (error) {
+      logger.error('Failed to update proxy pool', { error });
+      throw error;
+    }
+  }
+
+  /**
+   * Add or update a single proxy in the pool
+   */
+  async updateProxy(proxy: ProxyInfo): Promise<void> {
+    const existingIndex = this.proxies.findIndex(
+      p => p.host === proxy.host && p.port === proxy.port && p.protocol === proxy.protocol
+    );
+
+    if (existingIndex >= 0) {
+      this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy };
+      logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
+    } else {
+      this.proxies.push(proxy);
+      logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
+    }
+
+    // Update cache
+    await this.updateProxies(this.proxies);
+  }
+
+  /**
+   * Remove a proxy from the pool
+   */
+  async removeProxy(host: string, port: number, protocol: string): Promise<void> {
+    const initialLength = this.proxies.length;
+    this.proxies = this.proxies.filter(
+      p => !(p.host === host && p.port === port && p.protocol === protocol)
+    );
+
+    if (this.proxies.length < initialLength) {
+      await this.updateProxies(this.proxies);
+      logger.debug('Removed proxy', { host, port, protocol });
+    }
+  }
+
+  /**
+   * Clear all proxies from memory and cache
+   */
+  async clearProxies(): Promise<void> {
+    this.proxies = [];
+    this.lastUpdate = null;
+    
+    await this.cache.del('active-proxies');
+    await this.cache.del('last-update');
+    
+    logger.info('Cleared all proxies');
+  }
+
+  /**
+   * Check if proxy manager is ready
+   */
+  isReady(): boolean {
+    return this.isInitialized;
+  }
+
+  /**
+   * Load proxies from cache storage
+   */
+  private async loadFromCache(): Promise<void> {
+    try {
+      const cachedProxies = await this.cache.get<ProxyInfo[]>('active-proxies');
+      const lastUpdateStr = await this.cache.get<string>('last-update');
+
+      if (cachedProxies && Array.isArray(cachedProxies)) {
+        this.proxies = cachedProxies;
+        this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null;
+        
+        logger.debug('Loaded proxies from cache', {
+          count: this.proxies.length,
+          lastUpdate: this.lastUpdate,
+        });
+      } else {
+        logger.debug('No cached proxies found');
+      }
+    } catch (error) {
+      logger.error('Failed to load proxies from cache', { error });
+    }
+  }
+
+  /**
+   * Initialize the singleton instance
+   */
+  static async initialize(config?: ProxyManagerConfig): Promise<void> {
+    if (!ProxyManager.instance) {
+      ProxyManager.instance = new ProxyManager(config);
+      await ProxyManager.instance.initializeInternal();
+      
+      // Perform initial sync with proxy:active:* storage
+      try {
+        const { syncProxiesOnce } = await import('./proxy-sync');
+        await syncProxiesOnce();
+        logger.info('Initial proxy sync completed');
+      } catch (error) {
+        logger.error('Failed to perform initial proxy sync', { error });
+      }
+    }
+  }
+
+  /**
+   * Get the singleton instance (must be initialized first)
+   */
+  static getInstance(): ProxyManager {
+    if (!ProxyManager.instance) {
+      throw new Error('ProxyManager not initialized. Call ProxyManager.initialize() first.');
+    }
+    return ProxyManager.instance;
+  }
+
+  /**
+   * Reset the singleton instance (for testing)
+   */
+  static reset(): void {
+    ProxyManager.instance = null;
+  }
+}
+
+// Export the class as default
+export default ProxyManager;
+
+// Convenience functions for easier imports
+export function getProxy(): string | null {
+  return ProxyManager.getInstance().getProxy();
+}
+
+export function getRandomProxy(): ProxyInfo | null {
+  return ProxyManager.getInstance().getRandomProxy();
+}
+
+export function getAllProxies(): ProxyInfo[] {
+  return ProxyManager.getInstance().getAllProxies();
+}
+
+export function getWorkingProxies(): ProxyInfo[] {
+  return ProxyManager.getInstance().getWorkingProxies();
+}
+
+export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
+  return ProxyManager.getInstance().updateProxies(proxies);
+}
+
+export function getProxyStats(): ProxyStats {
+  return ProxyManager.getInstance().getStats();
+}

libs/services/proxy/src/proxy-sync.ts

@@ -0,0 +1,170 @@
+/**
+ * Proxy Storage Synchronization Service
+ * 
+ * This service bridges the gap between two proxy storage systems:
+ * 1. proxy:active:* keys (used by proxy tasks for individual proxy storage)
+ * 2. proxies:active-proxies (used by ProxyManager for centralized storage)
+ */
+
+import { createCache, type CacheProvider } from '@stock-bot/cache';
+import { getDatabaseConfig } from '@stock-bot/config';
+import { getLogger } from '@stock-bot/logger';
+import type { ProxyInfo, ProxySyncConfig } from './types';
+import { ProxyManager } from './proxy-manager';
+
+const logger = getLogger('proxy-sync');
+
+export class ProxySyncService {
+  private cache: CacheProvider;
+  private syncInterval: Timer | null = null;
+  private isRunning = false;
+  private config: ProxySyncConfig;
+
+  constructor(config: ProxySyncConfig = {}) {
+    this.config = {
+      intervalMs: 300000, // 5 minutes
+      enableAutoSync: true,
+      ...config
+    };
+
+    const databaseConfig = getDatabaseConfig();
+    this.cache = createCache({
+      redisConfig: databaseConfig.dragonfly,
+      keyPrefix: '', // No prefix to access all keys
+      ttl: 86400,
+    });
+  }
+
+  /**
+   * Start the synchronization service
+   * @param intervalMs - Sync interval in milliseconds (default: 5 minutes)
+   */
+  async start(intervalMs?: number): Promise<void> {
+    const interval = intervalMs || this.config.intervalMs!;
+    
+    if (this.isRunning) {
+      logger.warn('Proxy sync service is already running');
+      return;
+    }
+
+    this.isRunning = true;
+    logger.info('Starting proxy sync service', { intervalMs: interval });
+
+    // Wait for cache to be ready before initial sync
+    await this.cache.waitForReady(10000);
+
+    // Initial sync
+    await this.syncProxies();
+
+    // Set up periodic sync if enabled
+    if (this.config.enableAutoSync) {
+      this.syncInterval = setInterval(async () => {
+        try {
+          await this.syncProxies();
+        } catch (error) {
+          logger.error('Error during periodic sync', { error });
+        }
+      }, interval);
+    }
+  }
+
+  /**
+   * Stop the synchronization service
+   */
+  stop(): void {
+    if (this.syncInterval) {
+      clearInterval(this.syncInterval);
+      this.syncInterval = null;
+    }
+    this.isRunning = false;
+    logger.info('Stopped proxy sync service');
+  }
+
+  /**
+   * Perform a one-time synchronization
+   */
+  async syncProxies(): Promise<void> {
+    try {
+      logger.debug('Starting proxy synchronization');
+
+      // Wait for cache to be ready
+      await this.cache.waitForReady(5000);
+
+      // Collect all proxies from proxy:active:* storage
+      const proxyKeys = await this.cache.keys('proxy:active:*');
+      
+      if (proxyKeys.length === 0) {
+        logger.debug('No proxies found in proxy:active:* storage');
+        return;
+      }
+
+      const allProxies: ProxyInfo[] = [];
+      
+      // Fetch all proxies in parallel for better performance
+      const proxyPromises = proxyKeys.map(key => this.cache.get<ProxyInfo>(key));
+      const proxyResults = await Promise.all(proxyPromises);
+      
+      for (const proxy of proxyResults) {
+        if (proxy) {
+          allProxies.push(proxy);
+        }
+      }
+
+      const workingCount = allProxies.filter(p => p.isWorking).length;
+      
+      logger.info('Collected proxies from storage', {
+        total: allProxies.length,
+        working: workingCount,
+      });
+
+      // Update ProxyManager with all proxies
+      const manager = ProxyManager.getInstance();
+      await manager.updateProxies(allProxies);
+
+      logger.info('Proxy synchronization completed', {
+        synchronized: allProxies.length,
+        working: workingCount,
+      });
+    } catch (error) {
+      logger.error('Failed to sync proxies', { error });
+      throw error;
+    }
+  }
+
+  /**
+   * Get synchronization status
+   */
+  getStatus(): { isRunning: boolean; config: ProxySyncConfig } {
+    return {
+      isRunning: this.isRunning,
+      config: this.config
+    };
+  }
+}
+
+// Export singleton instance
+let syncServiceInstance: ProxySyncService | null = null;
+
+export function getProxySyncService(config?: ProxySyncConfig): ProxySyncService {
+  if (!syncServiceInstance) {
+    syncServiceInstance = new ProxySyncService(config);
+  }
+  return syncServiceInstance;
+}
+
+// Convenience functions
+export async function startProxySync(intervalMs?: number, config?: ProxySyncConfig): Promise<void> {
+  const service = getProxySyncService(config);
+  await service.start(intervalMs);
+}
+
+export function stopProxySync(): void {
+  if (syncServiceInstance) {
+    syncServiceInstance.stop();
+  }
+}
+
+export async function syncProxiesOnce(): Promise<void> {
+  const service = getProxySyncService();
+  await service.syncProxies();
+}

libs/services/proxy/src/types.ts

@@ -0,0 +1,36 @@
+/**
+ * Proxy service types and interfaces
+ */
+
+export interface ProxyInfo {
+  host: string;
+  port: number;
+  protocol: 'http' | 'https' | 'socks4' | 'socks5';
+  username?: string;
+  password?: string;
+  isWorking?: boolean;
+  successRate?: number;
+  lastChecked?: string;
+  lastUsed?: string;
+  responseTime?: number;
+  source?: string;
+  country?: string;
+}
+
+export interface ProxyManagerConfig {
+  cachePrefix?: string;
+  ttl?: number;
+  enableMetrics?: boolean;
+}
+
+export interface ProxySyncConfig {
+  intervalMs?: number;
+  enableAutoSync?: boolean;
+}
+
+export interface ProxyStats {
+  total: number;
+  working: number;
+  failed: number;
+  lastUpdate: Date | null;
+}

libs/services/proxy/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["dist", "node_modules"]
+}

scripts/build-libs.sh

@@ -48,6 +48,7 @@ libs=(
     "services/shutdown"    # Shutdown - depends on core libs
     "services/browser"     # Browser - depends on core libs
     "services/queue"       # Queue - depends on core libs and cache
+    "services/proxy"       # Proxy manager - depends on core libs and cache
     
     # Utils
     "utils"                # Utilities - depends on many libs