boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

clean up proxyManager

Browse source
This commit is contained in:
Boki 2025-06-20 12:31:49 -04:00
parent da916222c1
commit 76d55fe35f
10 changed files with 38 additions and 438 deletions
Download patch file Download diff file Expand all files Collapse all files

6
apps/data-service/src/providers/qm.tasks.ts
View file

@ -2,7 +2,7 @@ import { getRandomUserAgent } from '@stock-bot/http';
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient } from '@stock-bot/mongodb-client';
import { QueueManager } from '@stock-bot/queue';
import { ProxyManager } from '@stock-bot/utils';
import { getRandomProxy } from '@stock-bot/utils';
// Shared instances (module-scoped, not global)
let isInitialized = false; // Track if resources are initialized
@ -90,7 +90,7 @@ export async function createSessions(): Promise<void> {
while (sessionCache[sessionId].length < 50) {
logger.info(`Creating new session for ${sessionId}`);
const proxyInfo = await ProxyManager.getInstance().getRandomProxy();
const proxyInfo = await getRandomProxy();
if (!proxyInfo) {
logger.error('No proxy available for QM session creation');
break; // Skip session creation if no proxy is available
@ -294,7 +294,7 @@ async function searchAndSpawnJobs(
// API call function to search symbols via QM
async function searchQMSymbolsAPI(query: string): Promise<string[]> {
const proxyInfo = await ProxyManager.getInstance().getRandomProxy();
const proxyInfo = await getRandomProxy();
if (!proxyInfo) {
throw new Error('No proxy available for QM API call');

67
apps/data-service/src/providers/webshare.provider.ts
View file

@ -7,7 +7,7 @@ import {
handlerRegistry,
type HandlerConfigWithSchedule,
} from '@stock-bot/queue';
import { ProxyManager } from '@stock-bot/utils';
import { updateProxies } from '@stock-bot/utils';
const logger = getLogger('webshare-provider');
@ -28,7 +28,7 @@ export function initializeWebShareProvider() {
if (proxies.length > 0) {
// Update the centralized proxy manager
await ProxyManager.getInstance().updateProxies(proxies);
await updateProxies(proxies);
logger.info('Updated proxy manager with WebShare proxies', {
count: proxies.length,
@ -57,40 +57,6 @@ export function initializeWebShareProvider() {
};
}
}),
'validate-proxies': createJobHandler(async () => {
logger.info('Validating existing proxies');
const { validateStoredProxies } = await import('./webshare.tasks');
try {
const validationResults = await validateStoredProxies();
// Update proxy manager with validated proxies
await ProxyManager.getInstance().updateProxies(validationResults.workingProxies);
logger.info('Proxy validation completed', {
totalChecked: validationResults.totalChecked,
workingCount: validationResults.workingCount,
successRate: ((validationResults.workingCount / validationResults.totalChecked) * 100).toFixed(1) + '%',
});
return validationResults;
} catch (error) {
logger.error('Failed to validate proxies', { error });
return {
workingProxies: [],
totalChecked: 0,
workingCount: 0,
error: error instanceof Error ? error.message : 'Unknown error',
};
}
}),
'get-stats': createJobHandler(async () => {
const stats = ProxyManager.getInstance().getStats();
logger.info('Proxy manager statistics', stats);
return stats;
}),
},
scheduledJobs: [
@ -103,22 +69,6 @@ export function initializeWebShareProvider() {
description: 'Fetch fresh proxies from WebShare API',
immediately: true, // Run on startup
},
{
type: 'webshare-validate',
operation: 'validate-proxies',
payload: {},
cronPattern: '0 */2 * * *', // Every 2 hours
priority: 4,
description: 'Validate and clean existing proxies',
},
{
type: 'webshare-stats',
operation: 'get-stats',
payload: {},
cronPattern: '0 * * * *', // Every hour
priority: 5,
description: 'Log proxy manager statistics',
},
],
};
@ -126,20 +76,7 @@ export function initializeWebShareProvider() {
logger.debug('WebShare provider registered successfully');
}
// Legacy function for backward compatibility - now uses centralized proxy manager
export async function getProxy(): Promise<string | null> {
const proxy = ProxyManager.getInstance().getRandomProxy();
if (!proxy) {
return null;
}
// Convert ProxyInfo back to string format for backward compatibility
const auth = proxy.username && proxy.password ? `${proxy.username}:${proxy.password}@` : '';
return `${proxy.protocol}://${auth}${proxy.host}:${proxy.port}`;
}
export const webShareProvider = {
initialize: initializeWebShareProvider,
getProxy,
};

177
apps/data-service/src/providers/webshare.tasks.ts
View file

@ -1,9 +1,8 @@
/**
* WebShare Tasks - API integration and proxy validation
* WebShare Tasks - API integration
*/
import { getLogger } from '@stock-bot/logger';
import { HttpClient, type ProxyInfo } from '@stock-bot/http';
import { ProxyManager } from '@stock-bot/utils';
import { type ProxyInfo } from '@stock-bot/http';
const logger = getLogger('webshare-tasks');
@ -65,7 +64,7 @@ export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
port: proxy.port,
username: proxy.username,
password: proxy.password,
isWorking: true, // Assume working until proven otherwise
isWorking: true, // WebShare provides working proxies
firstSeen: new Date(),
lastChecked: new Date(),
}));
@ -80,174 +79,4 @@ export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
logger.error('Failed to fetch proxies from WebShare', { error });
return [];
}
}
/**
* Validate stored proxies by testing connectivity
*/
export async function validateStoredProxies(): Promise<{
workingProxies: ProxyInfo[];
totalChecked: number;
workingCount: number;
}> {
const httpClient = new HttpClient({ timeout: 10000 }, logger);
const testUrl = 'https://httpbin.org/ip'; // Simple IP echo service
// Get all proxies from proxy manager
const allProxies = ProxyManager.getInstance().getAllProxies();
if (allProxies.length === 0) {
logger.warn('No proxies available for validation');
return {
workingProxies: [],
totalChecked: 0,
workingCount: 0,
};
}
logger.info('Starting proxy validation', { totalProxies: allProxies.length });
const workingProxies: ProxyInfo[] = [];
const validationPromises = allProxies.map(async (proxy) => {
try {
const startTime = Date.now();
const response = await httpClient.get(testUrl, {
proxy,
timeout: 10000,
});
const responseTime = Date.now() - startTime;
const isWorking = response.status === 200;
if (isWorking) {
// Update proxy with success metrics
const updatedProxy: ProxyInfo = {
...proxy,
isWorking: true,
lastChecked: new Date(),
responseTime,
total: (proxy.total || 0) + 1,
working: (proxy.working || 0) + 1,
averageResponseTime: proxy.averageResponseTime
? (proxy.averageResponseTime + responseTime) / 2
: responseTime,
};
// Calculate success rate
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: 100;
workingProxies.push(updatedProxy);
logger.debug('Proxy validation successful', {
host: proxy.host,
port: proxy.port,
responseTime,
successRate: updatedProxy.successRate?.toFixed(1) + '%',
});
} else {
logger.debug('Proxy validation failed', {
host: proxy.host,
port: proxy.port,
status: response.status,
});
}
} catch (error) {
// Update proxy with failure metrics
const updatedProxy: ProxyInfo = {
...proxy,
isWorking: false,
lastChecked: new Date(),
error: error instanceof Error ? error.message : 'Unknown error',
total: (proxy.total || 0) + 1,
working: proxy.working || 0,
};
// Calculate success rate
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: 0;
logger.debug('Proxy validation error', {
host: proxy.host,
port: proxy.port,
error: error instanceof Error ? error.message : 'Unknown error',
});
}
});
// Wait for all validations to complete
await Promise.all(validationPromises);
const results = {
workingProxies,
totalChecked: allProxies.length,
workingCount: workingProxies.length,
};
logger.info('Proxy validation completed', {
totalChecked: results.totalChecked,
workingCount: results.workingCount,
successRate: ((results.workingCount / results.totalChecked) * 100).toFixed(1) + '%',
});
return results;
}
/**
* Test a single proxy for connectivity
*/
export async function testProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
const httpClient = new HttpClient({ timeout: 10000 }, logger);
const testUrl = 'https://httpbin.org/ip';
try {
const startTime = Date.now();
const response = await httpClient.get(testUrl, {
proxy,
timeout: 10000,
});
const responseTime = Date.now() - startTime;
const isWorking = response.status === 200;
const updatedProxy: ProxyInfo = {
...proxy,
isWorking,
lastChecked: new Date(),
responseTime,
total: (proxy.total || 0) + 1,
working: isWorking ? (proxy.working || 0) + 1 : (proxy.working || 0),
};
// Calculate success rate
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: (isWorking ? 100 : 0);
// Update average response time
if (isWorking && responseTime) {
updatedProxy.averageResponseTime = proxy.averageResponseTime
? (proxy.averageResponseTime + responseTime) / 2
: responseTime;
}
return updatedProxy;
} catch (error) {
const updatedProxy: ProxyInfo = {
...proxy,
isWorking: false,
lastChecked: new Date(),
error: error instanceof Error ? error.message : 'Unknown error',
total: (proxy.total || 0) + 1,
working: proxy.working || 0,
};
updatedProxy.successRate = updatedProxy.total > 0
? (updatedProxy.working / updatedProxy.total) * 100
: 0;
return updatedProxy;
}
}