boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

added a smart queue manager and moved proxy logic to proxy manager to make handler just schedule a call to it

Browse source
This commit is contained in:
Boki 2025-06-23 10:45:06 -04:00
parent da1c52a841
commit e7c0fe2798
19 changed files with 903 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

102
apps/stock/data-ingestion/src/handlers/webshare/operations/fetch.operations.ts
View file

@ -1,102 +0,0 @@
/**
* WebShare Fetch Operations - API integration
*/
import { OperationContext } from '@stock-bot/di';
import type { ProxyInfo } from '@stock-bot/proxy';
import { WEBSHARE_CONFIG } from '../shared/config';
/**
* Fetch proxies from WebShare API and convert to ProxyInfo format
*/
export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
const ctx = OperationContext.create('webshare', 'fetch-proxies');
try {
// Get configuration from stock config system - ensure it's initialized
const { getStockConfig, initializeStockConfig } = await import('@stock-bot/stock-config');
// Try to get existing config, or initialize if needed
let config;
try {
config = getStockConfig();
} catch (error) {
// Config not initialized yet, initialize it
config = initializeStockConfig('dataIngestion');
}
const apiKey = config.webshare?.apiKey;
const apiUrl = config.webshare?.apiUrl;
ctx.logger.debug('WebShare config loaded', {
hasConfig: !!config,
hasWebshare: !!config.webshare,
webshareConfig: config.webshare,
apiKeyLength: apiKey?.length || 0,
apiUrl: apiUrl,
envApiKey: process.env.WEBSHARE_API_KEY ? 'SET' : 'NOT_SET',
});
if (!apiKey || !apiUrl) {
ctx.logger.error('Missing WebShare configuration', {
hasApiKey: !!apiKey,
hasApiUrl: !!apiUrl,
apiKeyValue: apiKey ? `${apiKey.substring(0, 5)}...` : 'NOT_SET',
});
return [];
}
ctx.logger.info('Fetching proxies from WebShare API', { apiUrl });
const response = await fetch(
`${apiUrl}proxy/list/?mode=${WEBSHARE_CONFIG.DEFAULT_MODE}&page=${WEBSHARE_CONFIG.DEFAULT_PAGE}&page_size=${WEBSHARE_CONFIG.DEFAULT_PAGE_SIZE}`,
{
method: 'GET',
headers: {
Authorization: `Token ${apiKey}`,
'Content-Type': 'application/json',
},
signal: AbortSignal.timeout(WEBSHARE_CONFIG.TIMEOUT),
}
);
if (!response.ok) {
ctx.logger.error('WebShare API request failed', {
status: response.status,
statusText: response.statusText,
});
return [];
}
const data = await response.json();
if (!data.results || !Array.isArray(data.results)) {
ctx.logger.error('Invalid response format from WebShare API', { data });
return [];
}
// Transform proxy data to ProxyInfo format
const proxies: ProxyInfo[] = data.results.map(
(proxy: { username: string; password: string; proxy_address: string; port: number }) => ({
source: 'webshare',
protocol: 'http' as const,
host: proxy.proxy_address,
port: proxy.port,
username: proxy.username,
password: proxy.password,
isWorking: true, // WebShare provides working proxies
firstSeen: new Date(),
lastChecked: new Date(),
})
);
ctx.logger.info('Successfully fetched proxies from WebShare', {
count: proxies.length,
total: data.count || proxies.length,
});
return proxies;
} catch (error) {
ctx.logger.error('Failed to fetch proxies from WebShare', { error });
return [];
}
}

79
apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts
View file

@ -14,59 +14,52 @@ export class WebShareHandler extends BaseHandler {
}
@Operation('fetch-proxies')
@QueueSchedule('0 */6 * * *', { // once a month
@QueueSchedule('0 */6 * * *', { // every 6 hours
priority: 3,
immediately: true,
description: 'Fetch fresh proxies from WebShare API',
immediately: false, // Don't run immediately since ProxyManager fetches on startup
description: 'Refresh proxies from WebShare API',
})
async fetchProxies(_input: unknown, _context: ExecutionContext): Promise<unknown> {
this.logger.info('Fetching proxies from WebShare API');
this.logger.info('Refreshing proxies from WebShare API');
try {
const { fetchWebShareProxies } = await import('./operations/fetch.operations');
const proxies = await fetchWebShareProxies();
if (proxies.length > 0) {
// Update the centralized proxy manager using the injected service
if (!this.proxy) {
this.logger.warn('Proxy manager is not initialized, cannot update proxies');
return {
success: false,
proxiesUpdated: 0,
error: 'Proxy manager not initialized',
};
}
await this.proxy.updateProxies(proxies);
this.logger.info('Updated proxy manager with WebShare proxies', {
count: proxies.length,
workingCount: proxies.filter(p => p.isWorking !== false).length,
});
// Cache proxy stats for monitoring using handler's cache methods
await this.cacheSet('proxy-count', proxies.length, 3600);
await this.cacheSet(
'working-count',
proxies.filter(p => p.isWorking !== false).length,
3600
);
await this.cacheSet('last-fetch', new Date().toISOString(), 1800);
return {
success: true,
proxiesUpdated: proxies.length,
workingProxies: proxies.filter(p => p.isWorking !== false).length,
};
} else {
this.logger.warn('No proxies fetched from WebShare API');
// Check if proxy manager is available
if (!this.proxy) {
this.logger.warn('Proxy manager is not initialized, cannot refresh proxies');
return {
success: false,
proxiesUpdated: 0,
error: 'No proxies returned from API',
error: 'Proxy manager not initialized',
};
}
// Use the proxy manager's refresh method
await this.proxy.refreshProxies();
// Get stats after refresh
const stats = this.proxy.getStats();
const lastFetchTime = this.proxy.getLastFetchTime();
this.logger.info('Successfully refreshed proxies', {
total: stats.total,
working: stats.working,
failed: stats.failed,
lastFetchTime,
});
// Cache proxy stats for monitoring using handler's cache methods
await this.cacheSet('proxy-count', stats.total, 3600);
await this.cacheSet('working-count', stats.working, 3600);
await this.cacheSet('last-fetch', lastFetchTime?.toISOString() || 'unknown', 1800);
return {
success: true,
proxiesUpdated: stats.total,
workingProxies: stats.working,
failedProxies: stats.failed,
lastFetchTime,
};
} catch (error) {
this.logger.error('Failed to fetch and update proxies', { error });
this.logger.error('Failed to refresh proxies', { error });
throw error;
}
}