refactoring continuing
This commit is contained in:
parent
742e590382
commit
a0a3b26177
20 changed files with 394 additions and 798 deletions
|
|
@ -5,32 +5,15 @@
|
|||
|
||||
// Main classes
|
||||
export { ProxyManager } from './proxy-manager';
|
||||
export { ProxySyncService } from './proxy-sync';
|
||||
|
||||
// Types
|
||||
export type {
|
||||
ProxyInfo,
|
||||
ProxyManagerConfig,
|
||||
ProxySyncConfig,
|
||||
ProxyStats
|
||||
export type {
|
||||
ProxyInfo,
|
||||
ProxyManagerConfig, ProxyStats, ProxySyncConfig
|
||||
} from './types';
|
||||
|
||||
// Convenience functions
|
||||
export {
|
||||
getProxy,
|
||||
getRandomProxy,
|
||||
getAllProxies,
|
||||
getWorkingProxies,
|
||||
updateProxies,
|
||||
getProxyStats
|
||||
} from './proxy-manager';
|
||||
|
||||
export {
|
||||
getProxySyncService,
|
||||
startProxySync,
|
||||
stopProxySync,
|
||||
syncProxiesOnce
|
||||
} from './proxy-sync';
|
||||
// Note: Convenience functions removed as ProxyManager is no longer a singleton
|
||||
// Create an instance and use its methods directly
|
||||
|
||||
// Default export
|
||||
export { ProxyManager as default } from './proxy-manager';
|
||||
export { ProxyManager as default } from './proxy-manager';
|
||||
|
|
|
|||
|
|
@ -1,37 +1,21 @@
|
|||
/**
|
||||
* Centralized Proxy Manager - Handles proxy storage, retrieval, and caching
|
||||
*/
|
||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
||||
import { getDatabaseConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { CacheProvider } from '@stock-bot/cache';
|
||||
import type { ProxyInfo, ProxyManagerConfig, ProxyStats } from './types';
|
||||
|
||||
const logger = getLogger('proxy-manager');
|
||||
|
||||
export class ProxyManager {
|
||||
private static instance: ProxyManager | null = null;
|
||||
private cache: CacheProvider;
|
||||
private proxies: ProxyInfo[] = [];
|
||||
private proxyIndex: number = 0;
|
||||
private lastUpdate: Date | null = null;
|
||||
private isInitialized = false;
|
||||
private config: ProxyManagerConfig;
|
||||
private logger: any;
|
||||
|
||||
private constructor(config: ProxyManagerConfig = {}) {
|
||||
this.config = {
|
||||
cachePrefix: 'proxies:',
|
||||
ttl: 86400, // 24 hours
|
||||
enableMetrics: true,
|
||||
...config
|
||||
};
|
||||
|
||||
const databaseConfig = getDatabaseConfig();
|
||||
this.cache = createCache({
|
||||
redisConfig: databaseConfig.dragonfly,
|
||||
keyPrefix: this.config.cachePrefix,
|
||||
ttl: this.config.ttl,
|
||||
enableMetrics: this.config.enableMetrics,
|
||||
});
|
||||
constructor(cache: CacheProvider, _config: ProxyManagerConfig = {}, logger?: any) {
|
||||
this.cache = cache;
|
||||
this.logger = logger || console;
|
||||
// Config can be used in the future for customization
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -43,27 +27,27 @@ export class ProxyManager {
|
|||
}
|
||||
|
||||
try {
|
||||
logger.info('Initializing proxy manager...');
|
||||
this.logger.info('Initializing proxy manager...');
|
||||
|
||||
// Wait for cache to be ready
|
||||
await this.cache.waitForReady(10000); // Wait up to 10 seconds
|
||||
logger.debug('Cache is ready');
|
||||
this.logger.debug('Cache is ready');
|
||||
|
||||
await this.loadFromCache();
|
||||
this.isInitialized = true;
|
||||
logger.info('Proxy manager initialized', {
|
||||
this.logger.info('Proxy manager initialized', {
|
||||
proxiesLoaded: this.proxies.length,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize proxy manager', { error });
|
||||
this.logger.error('Failed to initialize proxy manager', { error });
|
||||
this.isInitialized = true; // Set to true anyway to avoid infinite retries
|
||||
}
|
||||
}
|
||||
|
||||
getProxy(): string | null {
|
||||
if (this.proxies.length === 0) {
|
||||
logger.warn('No proxies available in memory');
|
||||
this.logger.warn('No proxies available in memory');
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -97,7 +81,7 @@ export class ProxyManager {
|
|||
|
||||
// Return null if no proxies available
|
||||
if (this.proxies.length === 0) {
|
||||
logger.warn('No proxies available in memory');
|
||||
this.logger.warn('No proxies available in memory');
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -105,7 +89,7 @@ export class ProxyManager {
|
|||
const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false);
|
||||
|
||||
if (workingProxies.length === 0) {
|
||||
logger.warn('No working proxies available');
|
||||
this.logger.warn('No working proxies available');
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -122,11 +106,11 @@ export class ProxyManager {
|
|||
const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)];
|
||||
|
||||
if (!selectedProxy) {
|
||||
logger.warn('No proxy selected from available pool');
|
||||
this.logger.warn('No proxy selected from available pool');
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.debug('Selected proxy', {
|
||||
this.logger.debug('Selected proxy', {
|
||||
host: selectedProxy.host,
|
||||
port: selectedProxy.port,
|
||||
successRate: selectedProxy.successRate,
|
||||
|
|
@ -178,8 +162,13 @@ export class ProxyManager {
|
|||
* Update the proxy pool with new proxies
|
||||
*/
|
||||
async updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
||||
// Ensure manager is initialized before updating
|
||||
if (!this.isInitialized) {
|
||||
await this.initializeInternal();
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
|
||||
this.logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
|
||||
|
||||
this.proxies = proxies;
|
||||
this.lastUpdate = new Date();
|
||||
|
|
@ -189,13 +178,13 @@ export class ProxyManager {
|
|||
await this.cache.set('last-update', this.lastUpdate.toISOString());
|
||||
|
||||
const workingCount = proxies.filter(p => p.isWorking !== false).length;
|
||||
logger.info('Proxy pool updated successfully', {
|
||||
this.logger.info('Proxy pool updated successfully', {
|
||||
totalProxies: proxies.length,
|
||||
workingProxies: workingCount,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to update proxy pool', { error });
|
||||
this.logger.error('Failed to update proxy pool', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
|
@ -210,10 +199,10 @@ export class ProxyManager {
|
|||
|
||||
if (existingIndex >= 0) {
|
||||
this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy };
|
||||
logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
|
||||
this.logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
|
||||
} else {
|
||||
this.proxies.push(proxy);
|
||||
logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
|
||||
this.logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
|
||||
}
|
||||
|
||||
// Update cache
|
||||
|
|
@ -231,7 +220,7 @@ export class ProxyManager {
|
|||
|
||||
if (this.proxies.length < initialLength) {
|
||||
await this.updateProxies(this.proxies);
|
||||
logger.debug('Removed proxy', { host, port, protocol });
|
||||
this.logger.debug('Removed proxy', { host, port, protocol });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -245,7 +234,7 @@ export class ProxyManager {
|
|||
await this.cache.del('active-proxies');
|
||||
await this.cache.del('last-update');
|
||||
|
||||
logger.info('Cleared all proxies');
|
||||
this.logger.info('Cleared all proxies');
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -267,79 +256,29 @@ export class ProxyManager {
|
|||
this.proxies = cachedProxies;
|
||||
this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null;
|
||||
|
||||
logger.debug('Loaded proxies from cache', {
|
||||
this.logger.debug('Loaded proxies from cache', {
|
||||
count: this.proxies.length,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} else {
|
||||
logger.debug('No cached proxies found');
|
||||
this.logger.debug('No cached proxies found');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to load proxies from cache', { error });
|
||||
this.logger.error('Failed to load proxies from cache', { error });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the singleton instance
|
||||
* Initialize the proxy manager
|
||||
*/
|
||||
static async initialize(config?: ProxyManagerConfig): Promise<void> {
|
||||
if (!ProxyManager.instance) {
|
||||
ProxyManager.instance = new ProxyManager(config);
|
||||
await ProxyManager.instance.initializeInternal();
|
||||
|
||||
// Perform initial sync with proxy:active:* storage
|
||||
try {
|
||||
const { syncProxiesOnce } = await import('./proxy-sync');
|
||||
await syncProxiesOnce();
|
||||
logger.info('Initial proxy sync completed');
|
||||
} catch (error) {
|
||||
logger.error('Failed to perform initial proxy sync', { error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the singleton instance (must be initialized first)
|
||||
*/
|
||||
static getInstance(): ProxyManager {
|
||||
if (!ProxyManager.instance) {
|
||||
throw new Error('ProxyManager not initialized. Call ProxyManager.initialize() first.');
|
||||
}
|
||||
return ProxyManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the singleton instance (for testing)
|
||||
*/
|
||||
static reset(): void {
|
||||
ProxyManager.instance = null;
|
||||
async initialize(): Promise<void> {
|
||||
await this.initializeInternal();
|
||||
|
||||
// Note: Initial proxy sync should be handled by the container or application
|
||||
// that creates ProxyManager instance
|
||||
this.logger.info('ProxyManager initialized - proxy sync should be handled externally');
|
||||
}
|
||||
}
|
||||
|
||||
// Export the class as default
|
||||
export default ProxyManager;
|
||||
|
||||
// Convenience functions for easier imports
|
||||
export function getProxy(): string | null {
|
||||
return ProxyManager.getInstance().getProxy();
|
||||
}
|
||||
|
||||
export function getRandomProxy(): ProxyInfo | null {
|
||||
return ProxyManager.getInstance().getRandomProxy();
|
||||
}
|
||||
|
||||
export function getAllProxies(): ProxyInfo[] {
|
||||
return ProxyManager.getInstance().getAllProxies();
|
||||
}
|
||||
|
||||
export function getWorkingProxies(): ProxyInfo[] {
|
||||
return ProxyManager.getInstance().getWorkingProxies();
|
||||
}
|
||||
|
||||
export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
||||
return ProxyManager.getInstance().updateProxies(proxies);
|
||||
}
|
||||
|
||||
export function getProxyStats(): ProxyStats {
|
||||
return ProxyManager.getInstance().getStats();
|
||||
}
|
||||
export default ProxyManager;
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
/**
|
||||
* Proxy Storage Synchronization Service
|
||||
*
|
||||
* This service bridges the gap between two proxy storage systems:
|
||||
* 1. proxy:active:* keys (used by proxy tasks for individual proxy storage)
|
||||
* 2. proxies:active-proxies (used by ProxyManager for centralized storage)
|
||||
*/
|
||||
|
||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
||||
import { getDatabaseConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { ProxyInfo, ProxySyncConfig } from './types';
|
||||
import { ProxyManager } from './proxy-manager';
|
||||
|
||||
const logger = getLogger('proxy-sync');
|
||||
|
||||
export class ProxySyncService {
|
||||
private cache: CacheProvider;
|
||||
private syncInterval: Timer | null = null;
|
||||
private isRunning = false;
|
||||
private config: ProxySyncConfig;
|
||||
|
||||
constructor(config: ProxySyncConfig = {}) {
|
||||
this.config = {
|
||||
intervalMs: 300000, // 5 minutes
|
||||
enableAutoSync: true,
|
||||
...config
|
||||
};
|
||||
|
||||
const databaseConfig = getDatabaseConfig();
|
||||
this.cache = createCache({
|
||||
redisConfig: databaseConfig.dragonfly,
|
||||
keyPrefix: '', // No prefix to access all keys
|
||||
ttl: 86400,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the synchronization service
|
||||
* @param intervalMs - Sync interval in milliseconds (default: 5 minutes)
|
||||
*/
|
||||
async start(intervalMs?: number): Promise<void> {
|
||||
const interval = intervalMs || this.config.intervalMs!;
|
||||
|
||||
if (this.isRunning) {
|
||||
logger.warn('Proxy sync service is already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
logger.info('Starting proxy sync service', { intervalMs: interval });
|
||||
|
||||
// Wait for cache to be ready before initial sync
|
||||
await this.cache.waitForReady(10000);
|
||||
|
||||
// Initial sync
|
||||
await this.syncProxies();
|
||||
|
||||
// Set up periodic sync if enabled
|
||||
if (this.config.enableAutoSync) {
|
||||
this.syncInterval = setInterval(async () => {
|
||||
try {
|
||||
await this.syncProxies();
|
||||
} catch (error) {
|
||||
logger.error('Error during periodic sync', { error });
|
||||
}
|
||||
}, interval);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the synchronization service
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.syncInterval) {
|
||||
clearInterval(this.syncInterval);
|
||||
this.syncInterval = null;
|
||||
}
|
||||
this.isRunning = false;
|
||||
logger.info('Stopped proxy sync service');
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a one-time synchronization
|
||||
*/
|
||||
async syncProxies(): Promise<void> {
|
||||
try {
|
||||
logger.debug('Starting proxy synchronization');
|
||||
|
||||
// Wait for cache to be ready
|
||||
await this.cache.waitForReady(5000);
|
||||
|
||||
// Collect all proxies from proxy:active:* storage
|
||||
const proxyKeys = await this.cache.keys('proxy:active:*');
|
||||
|
||||
if (proxyKeys.length === 0) {
|
||||
logger.debug('No proxies found in proxy:active:* storage');
|
||||
return;
|
||||
}
|
||||
|
||||
const allProxies: ProxyInfo[] = [];
|
||||
|
||||
// Fetch all proxies in parallel for better performance
|
||||
const proxyPromises = proxyKeys.map(key => this.cache.get<ProxyInfo>(key));
|
||||
const proxyResults = await Promise.all(proxyPromises);
|
||||
|
||||
for (const proxy of proxyResults) {
|
||||
if (proxy) {
|
||||
allProxies.push(proxy);
|
||||
}
|
||||
}
|
||||
|
||||
const workingCount = allProxies.filter(p => p.isWorking).length;
|
||||
|
||||
logger.info('Collected proxies from storage', {
|
||||
total: allProxies.length,
|
||||
working: workingCount,
|
||||
});
|
||||
|
||||
// Update ProxyManager with all proxies
|
||||
const manager = ProxyManager.getInstance();
|
||||
await manager.updateProxies(allProxies);
|
||||
|
||||
logger.info('Proxy synchronization completed', {
|
||||
synchronized: allProxies.length,
|
||||
working: workingCount,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to sync proxies', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get synchronization status
|
||||
*/
|
||||
getStatus(): { isRunning: boolean; config: ProxySyncConfig } {
|
||||
return {
|
||||
isRunning: this.isRunning,
|
||||
config: this.config
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
let syncServiceInstance: ProxySyncService | null = null;
|
||||
|
||||
export function getProxySyncService(config?: ProxySyncConfig): ProxySyncService {
|
||||
if (!syncServiceInstance) {
|
||||
syncServiceInstance = new ProxySyncService(config);
|
||||
}
|
||||
return syncServiceInstance;
|
||||
}
|
||||
|
||||
// Convenience functions
|
||||
export async function startProxySync(intervalMs?: number, config?: ProxySyncConfig): Promise<void> {
|
||||
const service = getProxySyncService(config);
|
||||
await service.start(intervalMs);
|
||||
}
|
||||
|
||||
export function stopProxySync(): void {
|
||||
if (syncServiceInstance) {
|
||||
syncServiceInstance.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export async function syncProxiesOnce(): Promise<void> {
|
||||
const service = getProxySyncService();
|
||||
await service.syncProxies();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue