refactored out proxymanager from webshare to make it reusable
This commit is contained in:
parent
98aa414231
commit
84cb14680b
8 changed files with 622 additions and 109 deletions
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* WebShare Provider for proxy management
|
||||
* WebShare Provider for proxy management with scheduled updates
|
||||
*/
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import {
|
||||
|
|
@ -7,24 +7,10 @@ import {
|
|||
handlerRegistry,
|
||||
type HandlerConfigWithSchedule,
|
||||
} from '@stock-bot/queue';
|
||||
import { proxyManager } from '@stock-bot/utils';
|
||||
|
||||
const logger = getLogger('webshare-provider');
|
||||
|
||||
// In-memory proxy storage
|
||||
let proxies: string[] = [];
|
||||
let lastFetchTime: Date | null = null;
|
||||
let currentProxyIndex = 0;
|
||||
|
||||
export function getProxy(): string | null {
|
||||
if (proxies.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const proxy = proxies[currentProxyIndex];
|
||||
currentProxyIndex = (currentProxyIndex + 1) % proxies.length;
|
||||
return proxy ?? null;
|
||||
}
|
||||
|
||||
// Initialize and register the WebShare provider
|
||||
export function initializeWebShareProvider() {
|
||||
logger.debug('Registering WebShare provider with scheduled jobs...');
|
||||
|
|
@ -34,128 +20,126 @@ export function initializeWebShareProvider() {
|
|||
|
||||
operations: {
|
||||
'fetch-proxies': createJobHandler(async () => {
|
||||
logger.debug('Fetching proxies from WebShare API');
|
||||
|
||||
logger.info('Fetching proxies from WebShare API');
|
||||
const { fetchWebShareProxies } = await import('./webshare.tasks');
|
||||
|
||||
try {
|
||||
const fetchedProxies = await fetchProxiesFromWebShare();
|
||||
|
||||
if (fetchedProxies && fetchedProxies.length > 0) {
|
||||
proxies = fetchedProxies;
|
||||
lastFetchTime = new Date();
|
||||
|
||||
logger.info('Successfully updated proxy list', {
|
||||
const proxies = await fetchWebShareProxies();
|
||||
|
||||
if (proxies.length > 0) {
|
||||
// Update the centralized proxy manager
|
||||
await proxyManager.updateProxies(proxies);
|
||||
|
||||
logger.info('Updated proxy manager with WebShare proxies', {
|
||||
count: proxies.length,
|
||||
lastFetchTime: lastFetchTime.toISOString(),
|
||||
workingCount: proxies.filter(p => p.isWorking !== false).length,
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
return {
|
||||
success: true,
|
||||
count: proxies.length,
|
||||
lastFetchTime: lastFetchTime.toISOString(),
|
||||
proxiesUpdated: proxies.length,
|
||||
workingProxies: proxies.filter(p => p.isWorking !== false).length,
|
||||
};
|
||||
} else {
|
||||
logger.warn('No proxies fetched from WebShare API');
|
||||
return {
|
||||
success: false,
|
||||
count: 0,
|
||||
proxiesUpdated: 0,
|
||||
error: 'No proxies returned from API',
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch proxies from WebShare', { error });
|
||||
logger.error('Failed to fetch and update proxies', { error });
|
||||
return {
|
||||
success: false,
|
||||
count: proxies.length,
|
||||
proxiesUpdated: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
}
|
||||
}),
|
||||
|
||||
'validate-proxies': createJobHandler(async () => {
|
||||
logger.info('Validating existing proxies');
|
||||
const { validateStoredProxies } = await import('./webshare.tasks');
|
||||
|
||||
try {
|
||||
const validationResults = await validateStoredProxies();
|
||||
|
||||
// Update proxy manager with validated proxies
|
||||
await proxyManager.updateProxies(validationResults.workingProxies);
|
||||
|
||||
logger.info('Proxy validation completed', {
|
||||
totalChecked: validationResults.totalChecked,
|
||||
workingCount: validationResults.workingCount,
|
||||
successRate: ((validationResults.workingCount / validationResults.totalChecked) * 100).toFixed(1) + '%',
|
||||
});
|
||||
|
||||
return validationResults;
|
||||
} catch (error) {
|
||||
logger.error('Failed to validate proxies', { error });
|
||||
return {
|
||||
workingProxies: [],
|
||||
totalChecked: 0,
|
||||
workingCount: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
}
|
||||
}),
|
||||
|
||||
'get-stats': createJobHandler(async () => {
|
||||
const stats = proxyManager.getStats();
|
||||
logger.info('Proxy manager statistics', stats);
|
||||
return stats;
|
||||
}),
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'fetch-proxies',
|
||||
type: 'webshare-fetch',
|
||||
operation: 'fetch-proxies',
|
||||
payload: {},
|
||||
description: 'Fetch proxies from WebShare API',
|
||||
cronPattern: '*/5 * * * *', // Every 5 minutes
|
||||
priority: 2,
|
||||
immediately: true, // Fetch immediately on startup
|
||||
cronPattern: '0 */6 * * *', // Every 6 hours
|
||||
priority: 3,
|
||||
description: 'Fetch fresh proxies from WebShare API',
|
||||
immediately: true, // Run on startup
|
||||
},
|
||||
{
|
||||
type: 'webshare-validate',
|
||||
operation: 'validate-proxies',
|
||||
payload: {},
|
||||
cronPattern: '0 */2 * * *', // Every 2 hours
|
||||
priority: 4,
|
||||
description: 'Validate and clean existing proxies',
|
||||
},
|
||||
{
|
||||
type: 'webshare-stats',
|
||||
operation: 'get-stats',
|
||||
payload: {},
|
||||
cronPattern: '0 * * * *', // Every hour
|
||||
priority: 5,
|
||||
description: 'Log proxy manager statistics',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Register the provider
|
||||
handlerRegistry.registerWithSchedule(webShareProviderConfig);
|
||||
|
||||
logger.debug('WebShare provider registered successfully');
|
||||
}
|
||||
|
||||
// Legacy function for backward compatibility - now uses centralized proxy manager
|
||||
export async function getProxy(): Promise<string | null> {
|
||||
const proxy = await proxyManager.getRandomProxy();
|
||||
if (!proxy) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Convert ProxyInfo back to string format for backward compatibility
|
||||
const auth = proxy.username && proxy.password ? `${proxy.username}:${proxy.password}@` : '';
|
||||
return `${proxy.protocol}://${auth}${proxy.host}:${proxy.port}`;
|
||||
}
|
||||
|
||||
export const webShareProvider = {
|
||||
initialize: initializeWebShareProvider,
|
||||
getProxy,
|
||||
};
|
||||
|
||||
async function fetchProxiesFromWebShare(): Promise<string[] | null> {
|
||||
try {
|
||||
// Get configuration from config system
|
||||
const { getConfig } = await import('@stock-bot/config');
|
||||
const config = getConfig();
|
||||
|
||||
// Get configuration from config system
|
||||
const apiKey = config.webshare?.apiKey;
|
||||
const apiUrl = config.webshare?.apiUrl;
|
||||
|
||||
if (!apiKey || !apiUrl) {
|
||||
logger.error('Missing WebShare configuration', {
|
||||
hasApiKey: !!apiKey,
|
||||
hasApiUrl: !!apiUrl,
|
||||
configApiKey: apiKey?.substring(0, 10) + '...',
|
||||
configApiUrl: apiUrl,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.info('Fetching proxies from WebShare API');
|
||||
|
||||
const response = await fetch(`${apiUrl}proxy/list/?mode=direct&page=1&page_size=100`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Token ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error('WebShare API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.results || !Array.isArray(data.results)) {
|
||||
logger.error('Invalid response format from WebShare API', { data });
|
||||
return null;
|
||||
}
|
||||
|
||||
// Transform proxy data to the format: http://username:password@host:port
|
||||
const fetchedProxies = data.results.map(
|
||||
(proxy: { username: string; password: string; proxy_address: string; port: number }) => {
|
||||
return `http://${proxy.username}:${proxy.password}@${proxy.proxy_address}:${proxy.port}`;
|
||||
}
|
||||
);
|
||||
|
||||
logger.info('Successfully fetched proxies from WebShare', {
|
||||
count: fetchedProxies.length,
|
||||
total: data.count || fetchedProxies.length,
|
||||
});
|
||||
// console.log('Fetched Proxies:', fetchedProxies);
|
||||
return fetchedProxies;
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch proxies from WebShare', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue