refactoring
This commit is contained in:
parent
3fb9df425c
commit
62a2f15dab
12 changed files with 670 additions and 13 deletions
345
libs/services/proxy/src/proxy-manager.ts
Normal file
345
libs/services/proxy/src/proxy-manager.ts
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
/**
|
||||
* Centralized Proxy Manager - Handles proxy storage, retrieval, and caching
|
||||
*/
|
||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
||||
import { getDatabaseConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { ProxyInfo, ProxyManagerConfig, ProxyStats } from './types';
|
||||
|
||||
const logger = getLogger('proxy-manager');
|
||||
|
||||
export class ProxyManager {
|
||||
private static instance: ProxyManager | null = null;
|
||||
private cache: CacheProvider;
|
||||
private proxies: ProxyInfo[] = [];
|
||||
private proxyIndex: number = 0;
|
||||
private lastUpdate: Date | null = null;
|
||||
private isInitialized = false;
|
||||
private config: ProxyManagerConfig;
|
||||
|
||||
private constructor(config: ProxyManagerConfig = {}) {
|
||||
this.config = {
|
||||
cachePrefix: 'proxies:',
|
||||
ttl: 86400, // 24 hours
|
||||
enableMetrics: true,
|
||||
...config
|
||||
};
|
||||
|
||||
const databaseConfig = getDatabaseConfig();
|
||||
this.cache = createCache({
|
||||
redisConfig: databaseConfig.dragonfly,
|
||||
keyPrefix: this.config.cachePrefix,
|
||||
ttl: this.config.ttl,
|
||||
enableMetrics: this.config.enableMetrics,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal initialization - loads existing proxies from cache
|
||||
*/
|
||||
private async initializeInternal(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info('Initializing proxy manager...');
|
||||
|
||||
// Wait for cache to be ready
|
||||
await this.cache.waitForReady(10000); // Wait up to 10 seconds
|
||||
logger.debug('Cache is ready');
|
||||
|
||||
await this.loadFromCache();
|
||||
this.isInitialized = true;
|
||||
logger.info('Proxy manager initialized', {
|
||||
proxiesLoaded: this.proxies.length,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize proxy manager', { error });
|
||||
this.isInitialized = true; // Set to true anyway to avoid infinite retries
|
||||
}
|
||||
}
|
||||
|
||||
getProxy(): string | null {
|
||||
if (this.proxies.length === 0) {
|
||||
logger.warn('No proxies available in memory');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Cycle through proxies
|
||||
if (this.proxyIndex >= this.proxies.length) {
|
||||
this.proxyIndex = 0;
|
||||
}
|
||||
|
||||
const proxyInfo = this.proxies[this.proxyIndex++];
|
||||
if (!proxyInfo) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Build proxy URL with optional auth
|
||||
let proxyUrl = `${proxyInfo.protocol}://`;
|
||||
if (proxyInfo.username && proxyInfo.password) {
|
||||
proxyUrl += `${proxyInfo.username}:${proxyInfo.password}@`;
|
||||
}
|
||||
proxyUrl += `${proxyInfo.host}:${proxyInfo.port}`;
|
||||
|
||||
return proxyUrl;
|
||||
}
|
||||
/**
|
||||
* Get a random working proxy from the available pool (synchronous)
|
||||
*/
|
||||
getRandomProxy(): ProxyInfo | null {
|
||||
// Ensure initialized
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('ProxyManager not initialized');
|
||||
}
|
||||
|
||||
// Return null if no proxies available
|
||||
if (this.proxies.length === 0) {
|
||||
logger.warn('No proxies available in memory');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Filter for working proxies (not explicitly marked as non-working)
|
||||
const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false);
|
||||
|
||||
if (workingProxies.length === 0) {
|
||||
logger.warn('No working proxies available');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Return random proxy with preference for recently successful ones
|
||||
const sortedProxies = workingProxies.sort((a, b) => {
|
||||
// Prefer proxies with better success rates
|
||||
const aRate = a.successRate || 0;
|
||||
const bRate = b.successRate || 0;
|
||||
return bRate - aRate;
|
||||
});
|
||||
|
||||
// Take from top 50% of best performing proxies
|
||||
const topProxies = sortedProxies.slice(0, Math.max(1, Math.floor(sortedProxies.length * 0.5)));
|
||||
const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)];
|
||||
|
||||
if (!selectedProxy) {
|
||||
logger.warn('No proxy selected from available pool');
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.debug('Selected proxy', {
|
||||
host: selectedProxy.host,
|
||||
port: selectedProxy.port,
|
||||
successRate: selectedProxy.successRate,
|
||||
totalAvailable: workingProxies.length,
|
||||
});
|
||||
|
||||
return selectedProxy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all working proxies (synchronous)
|
||||
*/
|
||||
getWorkingProxies(): ProxyInfo[] {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('ProxyManager not initialized');
|
||||
}
|
||||
|
||||
return this.proxies.filter(proxy => proxy.isWorking !== false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all proxies (working and non-working)
|
||||
*/
|
||||
getAllProxies(): ProxyInfo[] {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('ProxyManager not initialized');
|
||||
}
|
||||
|
||||
return [...this.proxies];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get proxy statistics
|
||||
*/
|
||||
getStats(): ProxyStats {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('ProxyManager not initialized');
|
||||
}
|
||||
|
||||
return {
|
||||
total: this.proxies.length,
|
||||
working: this.proxies.filter(p => p.isWorking !== false).length,
|
||||
failed: this.proxies.filter(p => p.isWorking === false).length,
|
||||
lastUpdate: this.lastUpdate
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the proxy pool with new proxies
|
||||
*/
|
||||
async updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
||||
try {
|
||||
logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
|
||||
|
||||
this.proxies = proxies;
|
||||
this.lastUpdate = new Date();
|
||||
|
||||
// Store to cache
|
||||
await this.cache.set('active-proxies', proxies);
|
||||
await this.cache.set('last-update', this.lastUpdate.toISOString());
|
||||
|
||||
const workingCount = proxies.filter(p => p.isWorking !== false).length;
|
||||
logger.info('Proxy pool updated successfully', {
|
||||
totalProxies: proxies.length,
|
||||
workingProxies: workingCount,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to update proxy pool', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add or update a single proxy in the pool
|
||||
*/
|
||||
async updateProxy(proxy: ProxyInfo): Promise<void> {
|
||||
const existingIndex = this.proxies.findIndex(
|
||||
p => p.host === proxy.host && p.port === proxy.port && p.protocol === proxy.protocol
|
||||
);
|
||||
|
||||
if (existingIndex >= 0) {
|
||||
this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy };
|
||||
logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
|
||||
} else {
|
||||
this.proxies.push(proxy);
|
||||
logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
|
||||
}
|
||||
|
||||
// Update cache
|
||||
await this.updateProxies(this.proxies);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a proxy from the pool
|
||||
*/
|
||||
async removeProxy(host: string, port: number, protocol: string): Promise<void> {
|
||||
const initialLength = this.proxies.length;
|
||||
this.proxies = this.proxies.filter(
|
||||
p => !(p.host === host && p.port === port && p.protocol === protocol)
|
||||
);
|
||||
|
||||
if (this.proxies.length < initialLength) {
|
||||
await this.updateProxies(this.proxies);
|
||||
logger.debug('Removed proxy', { host, port, protocol });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all proxies from memory and cache
|
||||
*/
|
||||
async clearProxies(): Promise<void> {
|
||||
this.proxies = [];
|
||||
this.lastUpdate = null;
|
||||
|
||||
await this.cache.del('active-proxies');
|
||||
await this.cache.del('last-update');
|
||||
|
||||
logger.info('Cleared all proxies');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if proxy manager is ready
|
||||
*/
|
||||
isReady(): boolean {
|
||||
return this.isInitialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load proxies from cache storage
|
||||
*/
|
||||
private async loadFromCache(): Promise<void> {
|
||||
try {
|
||||
const cachedProxies = await this.cache.get<ProxyInfo[]>('active-proxies');
|
||||
const lastUpdateStr = await this.cache.get<string>('last-update');
|
||||
|
||||
if (cachedProxies && Array.isArray(cachedProxies)) {
|
||||
this.proxies = cachedProxies;
|
||||
this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null;
|
||||
|
||||
logger.debug('Loaded proxies from cache', {
|
||||
count: this.proxies.length,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} else {
|
||||
logger.debug('No cached proxies found');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to load proxies from cache', { error });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the singleton instance
|
||||
*/
|
||||
static async initialize(config?: ProxyManagerConfig): Promise<void> {
|
||||
if (!ProxyManager.instance) {
|
||||
ProxyManager.instance = new ProxyManager(config);
|
||||
await ProxyManager.instance.initializeInternal();
|
||||
|
||||
// Perform initial sync with proxy:active:* storage
|
||||
try {
|
||||
const { syncProxiesOnce } = await import('./proxy-sync');
|
||||
await syncProxiesOnce();
|
||||
logger.info('Initial proxy sync completed');
|
||||
} catch (error) {
|
||||
logger.error('Failed to perform initial proxy sync', { error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the singleton instance (must be initialized first)
|
||||
*/
|
||||
static getInstance(): ProxyManager {
|
||||
if (!ProxyManager.instance) {
|
||||
throw new Error('ProxyManager not initialized. Call ProxyManager.initialize() first.');
|
||||
}
|
||||
return ProxyManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the singleton instance (for testing)
|
||||
*/
|
||||
static reset(): void {
|
||||
ProxyManager.instance = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Export the class as default
|
||||
export default ProxyManager;
|
||||
|
||||
// Convenience functions for easier imports
|
||||
export function getProxy(): string | null {
|
||||
return ProxyManager.getInstance().getProxy();
|
||||
}
|
||||
|
||||
export function getRandomProxy(): ProxyInfo | null {
|
||||
return ProxyManager.getInstance().getRandomProxy();
|
||||
}
|
||||
|
||||
export function getAllProxies(): ProxyInfo[] {
|
||||
return ProxyManager.getInstance().getAllProxies();
|
||||
}
|
||||
|
||||
export function getWorkingProxies(): ProxyInfo[] {
|
||||
return ProxyManager.getInstance().getWorkingProxies();
|
||||
}
|
||||
|
||||
export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
||||
return ProxyManager.getInstance().updateProxies(proxies);
|
||||
}
|
||||
|
||||
export function getProxyStats(): ProxyStats {
|
||||
return ProxyManager.getInstance().getStats();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue