refactoring continuing
This commit is contained in:
parent
742e590382
commit
a0a3b26177
20 changed files with 394 additions and 798 deletions
|
|
@ -2,4 +2,3 @@ export * from './calculations/index';
|
|||
export * from './common';
|
||||
export * from './dateUtils';
|
||||
export * from './generic-functions';
|
||||
export * from './proxy';
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
/**
|
||||
* Proxy management utilities
|
||||
*/
|
||||
export {
|
||||
default as ProxyManager,
|
||||
getProxy,
|
||||
getRandomProxy,
|
||||
getAllProxies,
|
||||
getWorkingProxies,
|
||||
updateProxies
|
||||
} from './proxy-manager';
|
||||
|
||||
export {
|
||||
ProxySyncService,
|
||||
getProxySyncService,
|
||||
startProxySync,
|
||||
stopProxySync,
|
||||
syncProxiesOnce
|
||||
} from './proxy-sync';
|
||||
|
||||
export type { ProxyInfo } from '@stock-bot/http'; // Re-export for convenience
|
||||
|
|
@ -1,291 +0,0 @@
|
|||
/**
|
||||
* Centralized Proxy Manager - Handles proxy storage, retrieval, and caching
|
||||
*/
|
||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
||||
import { getDatabaseConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { ProxyInfo } from '@stock-bot/http';
|
||||
|
||||
const logger = getLogger('proxy-manager');
|
||||
|
||||
export class ProxyManager {
|
||||
private static instance: ProxyManager | null = null;
|
||||
private cache: CacheProvider;
|
||||
private proxies: ProxyInfo[] = [];
|
||||
private lastUpdate: Date | null = null;
|
||||
private isInitialized = false;
|
||||
|
||||
private constructor() {
|
||||
const databaseConfig = getDatabaseConfig();
|
||||
this.cache = createCache({
|
||||
redisConfig: databaseConfig.dragonfly,
|
||||
keyPrefix: 'proxies:',
|
||||
ttl: 86400, // 24 hours
|
||||
enableMetrics: true,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal initialization - loads existing proxies from cache
|
||||
*/
|
||||
private async initializeInternal(): Promise<void> {
|
||||
if (this.isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
logger.info('Initializing proxy manager...');
|
||||
|
||||
// Wait for cache to be ready
|
||||
await this.cache.waitForReady(10000); // Wait up to 10 seconds
|
||||
logger.debug('Cache is ready');
|
||||
|
||||
await this.loadFromCache();
|
||||
this.isInitialized = true;
|
||||
logger.info('Proxy manager initialized', {
|
||||
proxiesLoaded: this.proxies.length,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize proxy manager', { error });
|
||||
this.isInitialized = true; // Set to true anyway to avoid infinite retries
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a random working proxy from the available pool (synchronous)
|
||||
*/
|
||||
getRandomProxy(): ProxyInfo | null {
|
||||
// Ensure initialized
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('ProxyManager not initialized');
|
||||
}
|
||||
|
||||
// Return null if no proxies available
|
||||
if (this.proxies.length === 0) {
|
||||
logger.warn('No proxies available in memory');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Filter for working proxies (not explicitly marked as non-working)
|
||||
const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false);
|
||||
|
||||
if (workingProxies.length === 0) {
|
||||
logger.warn('No working proxies available');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Return random proxy with preference for recently successful ones
|
||||
const sortedProxies = workingProxies.sort((a, b) => {
|
||||
// Prefer proxies with better success rates
|
||||
const aRate = a.successRate || 0;
|
||||
const bRate = b.successRate || 0;
|
||||
return bRate - aRate;
|
||||
});
|
||||
|
||||
// Take from top 50% of best performing proxies
|
||||
const topProxies = sortedProxies.slice(0, Math.max(1, Math.floor(sortedProxies.length * 0.5)));
|
||||
const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)];
|
||||
|
||||
if (!selectedProxy) {
|
||||
logger.warn('No proxy selected from available pool');
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.debug('Selected proxy', {
|
||||
host: selectedProxy.host,
|
||||
port: selectedProxy.port,
|
||||
successRate: selectedProxy.successRate,
|
||||
totalAvailable: workingProxies.length,
|
||||
});
|
||||
|
||||
return selectedProxy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all working proxies (synchronous)
|
||||
*/
|
||||
getWorkingProxies(): ProxyInfo[] {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('ProxyManager not initialized');
|
||||
}
|
||||
|
||||
return this.proxies.filter(proxy => proxy.isWorking !== false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all proxies (working and non-working)
|
||||
*/
|
||||
getAllProxies(): ProxyInfo[] {
|
||||
if (!this.isInitialized) {
|
||||
throw new Error('ProxyManager not initialized');
|
||||
}
|
||||
|
||||
return [...this.proxies];
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the proxy pool with new proxies
|
||||
*/
|
||||
async updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
||||
try {
|
||||
logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
|
||||
|
||||
this.proxies = proxies;
|
||||
this.lastUpdate = new Date();
|
||||
|
||||
// Store to cache
|
||||
await this.cache.set('active-proxies', proxies);
|
||||
await this.cache.set('last-update', this.lastUpdate.toISOString());
|
||||
|
||||
const workingCount = proxies.filter(p => p.isWorking !== false).length;
|
||||
logger.info('Proxy pool updated successfully', {
|
||||
totalProxies: proxies.length,
|
||||
workingProxies: workingCount,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to update proxy pool', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add or update a single proxy in the pool
|
||||
*/
|
||||
async updateProxy(proxy: ProxyInfo): Promise<void> {
|
||||
const existingIndex = this.proxies.findIndex(
|
||||
p => p.host === proxy.host && p.port === proxy.port && p.protocol === proxy.protocol
|
||||
);
|
||||
|
||||
if (existingIndex >= 0) {
|
||||
this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy };
|
||||
logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
|
||||
} else {
|
||||
this.proxies.push(proxy);
|
||||
logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
|
||||
}
|
||||
|
||||
// Update cache
|
||||
await this.updateProxies(this.proxies);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a proxy from the pool
|
||||
*/
|
||||
async removeProxy(host: string, port: number, protocol: string): Promise<void> {
|
||||
const initialLength = this.proxies.length;
|
||||
this.proxies = this.proxies.filter(
|
||||
p => !(p.host === host && p.port === port && p.protocol === protocol)
|
||||
);
|
||||
|
||||
if (this.proxies.length < initialLength) {
|
||||
await this.updateProxies(this.proxies);
|
||||
logger.debug('Removed proxy', { host, port, protocol });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all proxies from memory and cache
|
||||
*/
|
||||
async clearProxies(): Promise<void> {
|
||||
this.proxies = [];
|
||||
this.lastUpdate = null;
|
||||
|
||||
await this.cache.del('active-proxies');
|
||||
await this.cache.del('last-update');
|
||||
|
||||
logger.info('Cleared all proxies');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if proxy manager is ready
|
||||
*/
|
||||
isReady(): boolean {
|
||||
return this.isInitialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load proxies from cache storage
|
||||
*/
|
||||
private async loadFromCache(): Promise<void> {
|
||||
try {
|
||||
const cachedProxies = await this.cache.get<ProxyInfo[]>('active-proxies');
|
||||
const lastUpdateStr = await this.cache.get<string>('last-update');
|
||||
|
||||
if (cachedProxies && Array.isArray(cachedProxies)) {
|
||||
this.proxies = cachedProxies;
|
||||
this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null;
|
||||
|
||||
logger.debug('Loaded proxies from cache', {
|
||||
count: this.proxies.length,
|
||||
lastUpdate: this.lastUpdate,
|
||||
});
|
||||
} else {
|
||||
logger.debug('No cached proxies found');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to load proxies from cache', { error });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the singleton instance
|
||||
*/
|
||||
static async initialize(): Promise<void> {
|
||||
if (!ProxyManager.instance) {
|
||||
ProxyManager.instance = new ProxyManager();
|
||||
await ProxyManager.instance.initializeInternal();
|
||||
|
||||
// Perform initial sync with proxy:active:* storage
|
||||
try {
|
||||
const { syncProxiesOnce } = await import('./proxy-sync');
|
||||
await syncProxiesOnce();
|
||||
logger.info('Initial proxy sync completed');
|
||||
} catch (error) {
|
||||
logger.error('Failed to perform initial proxy sync', { error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the singleton instance (must be initialized first)
|
||||
*/
|
||||
static getInstance(): ProxyManager {
|
||||
if (!ProxyManager.instance) {
|
||||
throw new Error('ProxyManager not initialized. Call ProxyManager.initialize() first.');
|
||||
}
|
||||
return ProxyManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the singleton instance (for testing)
|
||||
*/
|
||||
static reset(): void {
|
||||
ProxyManager.instance = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Export the class as default
|
||||
export default ProxyManager;
|
||||
|
||||
// Convenience functions for easier imports
|
||||
export function getProxy(): ProxyInfo | null {
|
||||
return ProxyManager.getInstance().getRandomProxy();
|
||||
}
|
||||
|
||||
export function getRandomProxy(): ProxyInfo | null {
|
||||
return ProxyManager.getInstance().getRandomProxy();
|
||||
}
|
||||
|
||||
export function getAllProxies(): ProxyInfo[] {
|
||||
return ProxyManager.getInstance().getAllProxies();
|
||||
}
|
||||
|
||||
export function getWorkingProxies(): ProxyInfo[] {
|
||||
return ProxyManager.getInstance().getWorkingProxies();
|
||||
}
|
||||
|
||||
export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
||||
return ProxyManager.getInstance().updateProxies(proxies);
|
||||
}
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
/**
|
||||
* Proxy Storage Synchronization Service
|
||||
*
|
||||
* This service bridges the gap between two proxy storage systems:
|
||||
* 1. proxy:active:* keys (used by proxy tasks for individual proxy storage)
|
||||
* 2. proxies:active-proxies (used by ProxyManager for centralized storage)
|
||||
*/
|
||||
|
||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
||||
import { getDatabaseConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { ProxyInfo } from '@stock-bot/http';
|
||||
import { ProxyManager } from './proxy-manager';
|
||||
|
||||
const logger = getLogger('proxy-sync');
|
||||
|
||||
export class ProxySyncService {
|
||||
private cache: CacheProvider;
|
||||
private syncInterval: Timer | null = null;
|
||||
private isRunning = false;
|
||||
|
||||
constructor() {
|
||||
const databaseConfig = getDatabaseConfig();
|
||||
this.cache = createCache({
|
||||
redisConfig: databaseConfig.dragonfly,
|
||||
keyPrefix: '', // No prefix to access all keys
|
||||
ttl: 86400,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the synchronization service
|
||||
* @param intervalMs - Sync interval in milliseconds (default: 5 minutes)
|
||||
*/
|
||||
async start(intervalMs: number = 300000): Promise<void> {
|
||||
if (this.isRunning) {
|
||||
logger.warn('Proxy sync service is already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
logger.info('Starting proxy sync service', { intervalMs });
|
||||
|
||||
// Wait for cache to be ready before initial sync
|
||||
await this.cache.waitForReady(10000);
|
||||
|
||||
// Initial sync
|
||||
await this.syncProxies();
|
||||
|
||||
// Set up periodic sync
|
||||
this.syncInterval = setInterval(async () => {
|
||||
try {
|
||||
await this.syncProxies();
|
||||
} catch (error) {
|
||||
logger.error('Error during periodic sync', { error });
|
||||
}
|
||||
}, intervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the synchronization service
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.syncInterval) {
|
||||
clearInterval(this.syncInterval);
|
||||
this.syncInterval = null;
|
||||
}
|
||||
this.isRunning = false;
|
||||
logger.info('Stopped proxy sync service');
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a one-time synchronization
|
||||
*/
|
||||
async syncProxies(): Promise<void> {
|
||||
try {
|
||||
logger.debug('Starting proxy synchronization');
|
||||
|
||||
// Wait for cache to be ready
|
||||
await this.cache.waitForReady(5000);
|
||||
|
||||
// Collect all proxies from proxy:active:* storage
|
||||
const proxyKeys = await this.cache.keys('proxy:active:*');
|
||||
|
||||
if (proxyKeys.length === 0) {
|
||||
logger.debug('No proxies found in proxy:active:* storage');
|
||||
return;
|
||||
}
|
||||
|
||||
const allProxies: ProxyInfo[] = [];
|
||||
|
||||
// Fetch all proxies in parallel for better performance
|
||||
const proxyPromises = proxyKeys.map(key => this.cache.get<ProxyInfo>(key));
|
||||
const proxyResults = await Promise.all(proxyPromises);
|
||||
|
||||
for (const proxy of proxyResults) {
|
||||
if (proxy) {
|
||||
allProxies.push(proxy);
|
||||
}
|
||||
}
|
||||
|
||||
const workingCount = allProxies.filter(p => p.isWorking).length;
|
||||
|
||||
logger.info('Collected proxies from storage', {
|
||||
total: allProxies.length,
|
||||
working: workingCount,
|
||||
});
|
||||
|
||||
// Update ProxyManager with all proxies
|
||||
const manager = ProxyManager.getInstance();
|
||||
await manager.updateProxies(allProxies);
|
||||
|
||||
logger.info('Proxy synchronization completed', {
|
||||
synchronized: allProxies.length,
|
||||
working: workingCount,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to sync proxies', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get synchronization status
|
||||
*/
|
||||
getStatus(): { isRunning: boolean; lastSync?: Date } {
|
||||
return {
|
||||
isRunning: this.isRunning,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
let syncServiceInstance: ProxySyncService | null = null;
|
||||
|
||||
export function getProxySyncService(): ProxySyncService {
|
||||
if (!syncServiceInstance) {
|
||||
syncServiceInstance = new ProxySyncService();
|
||||
}
|
||||
return syncServiceInstance;
|
||||
}
|
||||
|
||||
// Convenience functions
|
||||
export async function startProxySync(intervalMs?: number): Promise<void> {
|
||||
const service = getProxySyncService();
|
||||
await service.start(intervalMs);
|
||||
}
|
||||
|
||||
export function stopProxySync(): void {
|
||||
const service = getProxySyncService();
|
||||
service.stop();
|
||||
}
|
||||
|
||||
export async function syncProxiesOnce(): Promise<void> {
|
||||
const service = getProxySyncService();
|
||||
await service.syncProxies();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue