renaming services to more suitable names
This commit is contained in:
parent
3ae9de8376
commit
be6afef832
69 changed files with 41 additions and 2956 deletions
|
|
@ -0,0 +1,175 @@
|
|||
/**
|
||||
* Proxy Check Operations - Checking proxy functionality
|
||||
*/
|
||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
import { ProxyStatsManager } from '../shared/proxy-manager';
|
||||
|
||||
// Shared HTTP client
|
||||
let httpClient: HttpClient;
|
||||
|
||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
||||
if (!httpClient) {
|
||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
||||
}
|
||||
return httpClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a proxy is working
|
||||
*/
|
||||
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||
const ctx = OperationContext.create('proxy', 'check');
|
||||
|
||||
let success = false;
|
||||
ctx.logger.debug(`Checking Proxy:`, {
|
||||
protocol: proxy.protocol,
|
||||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
});
|
||||
|
||||
try {
|
||||
// Test the proxy
|
||||
const client = getHttpClient(ctx);
|
||||
const response = await client.get(PROXY_CONFIG.CHECK_URL, {
|
||||
proxy,
|
||||
timeout: PROXY_CONFIG.CHECK_TIMEOUT,
|
||||
});
|
||||
|
||||
const isWorking = response.status >= 200 && response.status < 300;
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking,
|
||||
lastChecked: new Date(),
|
||||
responseTime: response.responseTime,
|
||||
};
|
||||
|
||||
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
|
||||
success = true;
|
||||
await updateProxyInCache(result, true, ctx);
|
||||
} else {
|
||||
await updateProxyInCache(result, false, ctx);
|
||||
}
|
||||
|
||||
if (proxy.source) {
|
||||
updateProxyStats(proxy.source, success, ctx);
|
||||
}
|
||||
|
||||
ctx.logger.debug('Proxy check completed', {
|
||||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
isWorking,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking: false,
|
||||
error: errorMessage,
|
||||
lastChecked: new Date(),
|
||||
};
|
||||
|
||||
// Update cache for failed proxy (increment total, don't update TTL)
|
||||
await updateProxyInCache(result, false, ctx);
|
||||
|
||||
if (proxy.source) {
|
||||
updateProxyStats(proxy.source, success, ctx);
|
||||
}
|
||||
|
||||
ctx.logger.debug('Proxy check failed', {
|
||||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
error: errorMessage,
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update proxy data in cache with working/total stats and average response time
|
||||
*/
|
||||
async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean, ctx: OperationContext): Promise<void> {
|
||||
const cacheKey = `${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||
|
||||
try {
|
||||
const existing: ProxyInfo | null = await ctx.cache.get(cacheKey);
|
||||
|
||||
// For failed proxies, only update if they already exist
|
||||
if (!isWorking && !existing) {
|
||||
ctx.logger.debug('Proxy not in cache, skipping failed update', {
|
||||
proxy: `${proxy.host}:${proxy.port}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate new average response time if we have a response time
|
||||
let newAverageResponseTime = existing?.averageResponseTime;
|
||||
if (proxy.responseTime !== undefined) {
|
||||
const existingAvg = existing?.averageResponseTime || 0;
|
||||
const existingTotal = existing?.total || 0;
|
||||
|
||||
// Calculate weighted average: (existing_avg * existing_count + new_response) / (existing_count + 1)
|
||||
newAverageResponseTime =
|
||||
existingTotal > 0
|
||||
? (existingAvg * existingTotal + proxy.responseTime) / (existingTotal + 1)
|
||||
: proxy.responseTime;
|
||||
}
|
||||
|
||||
// Build updated proxy data
|
||||
const updated = {
|
||||
...existing,
|
||||
...proxy, // Keep latest proxy info
|
||||
total: (existing?.total || 0) + 1,
|
||||
working: isWorking ? (existing?.working || 0) + 1 : existing?.working || 0,
|
||||
isWorking,
|
||||
lastChecked: new Date(),
|
||||
// Add firstSeen only for new entries
|
||||
...(existing ? {} : { firstSeen: new Date() }),
|
||||
// Update average response time if we calculated a new one
|
||||
...(newAverageResponseTime !== undefined
|
||||
? { averageResponseTime: newAverageResponseTime }
|
||||
: {}),
|
||||
};
|
||||
|
||||
// Calculate success rate
|
||||
updated.successRate = updated.total > 0 ? (updated.working / updated.total) * 100 : 0;
|
||||
|
||||
// Save to cache: reset TTL for working proxies, keep existing TTL for failed ones
|
||||
const cacheOptions = isWorking ? { ttl: PROXY_CONFIG.CACHE_TTL } : undefined;
|
||||
await ctx.cache.set(cacheKey, updated, cacheOptions);
|
||||
|
||||
ctx.logger.debug(`Updated ${isWorking ? 'working' : 'failed'} proxy in cache`, {
|
||||
proxy: `${proxy.host}:${proxy.port}`,
|
||||
working: updated.working,
|
||||
total: updated.total,
|
||||
successRate: updated.successRate.toFixed(1) + '%',
|
||||
avgResponseTime: updated.averageResponseTime
|
||||
? `${updated.averageResponseTime.toFixed(0)}ms`
|
||||
: 'N/A',
|
||||
});
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to update proxy in cache', {
|
||||
proxy: `${proxy.host}:${proxy.port}`,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function updateProxyStats(sourceId: string, success: boolean, ctx: OperationContext) {
|
||||
const statsManager = ProxyStatsManager.getInstance();
|
||||
const source = statsManager.updateSourceStats(sourceId, success);
|
||||
|
||||
if (!source) {
|
||||
ctx.logger.warn(`Unknown proxy source: ${sourceId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cache the updated stats
|
||||
ctx.cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, { ttl: PROXY_CONFIG.CACHE_TTL })
|
||||
.catch(error => ctx.logger.debug('Failed to cache proxy stats', { error }));
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
/**
|
||||
* Proxy Fetch Operations - Fetching proxies from sources
|
||||
*/
|
||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
import { ProxyStatsManager } from '../shared/proxy-manager';
|
||||
import type { ProxySource } from '../shared/types';
|
||||
|
||||
// Shared HTTP client
|
||||
let httpClient: HttpClient;
|
||||
|
||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
||||
if (!httpClient) {
|
||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
||||
}
|
||||
return httpClient;
|
||||
}
|
||||
|
||||
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
|
||||
const ctx = OperationContext.create('proxy', 'fetch-sources');
|
||||
|
||||
const statsManager = ProxyStatsManager.getInstance();
|
||||
statsManager.resetStats();
|
||||
|
||||
const fetchPromises = PROXY_CONFIG.PROXY_SOURCES.map(source => fetchProxiesFromSource(source, ctx));
|
||||
const results = await Promise.all(fetchPromises);
|
||||
let allProxies: ProxyInfo[] = results.flat();
|
||||
allProxies = removeDuplicateProxies(allProxies);
|
||||
|
||||
ctx.logger.info('Fetched proxies from all sources', { total: allProxies.length });
|
||||
return allProxies;
|
||||
}
|
||||
|
||||
export async function fetchProxiesFromSource(source: ProxySource, ctx?: OperationContext): Promise<ProxyInfo[]> {
|
||||
if (!ctx) {
|
||||
ctx = OperationContext.create('proxy', 'fetch-source');
|
||||
}
|
||||
|
||||
const allProxies: ProxyInfo[] = [];
|
||||
|
||||
try {
|
||||
ctx.logger.info(`Fetching proxies from ${source.url}`);
|
||||
|
||||
const client = getHttpClient(ctx);
|
||||
const response = await client.get(source.url, {
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
if (response.status !== 200) {
|
||||
ctx.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const text = response.data;
|
||||
const lines = text.split('\n').filter((line: string) => line.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
let trimmed = line.trim();
|
||||
trimmed = cleanProxyUrl(trimmed);
|
||||
if (!trimmed || trimmed.startsWith('#')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse formats like \"host:port\" or \"host:port:user:pass\"
|
||||
const parts = trimmed.split(':');
|
||||
if (parts.length >= 2) {
|
||||
const proxy: ProxyInfo = {
|
||||
source: source.id,
|
||||
protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5',
|
||||
host: parts[0],
|
||||
port: parseInt(parts[1]),
|
||||
};
|
||||
|
||||
if (!isNaN(proxy.port) && proxy.host) {
|
||||
allProxies.push(proxy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ctx.logger.info(`Parsed ${allProxies.length} proxies from ${source.url}`);
|
||||
} catch (error) {
|
||||
ctx.logger.error(`Error fetching proxies from ${source.url}`, error);
|
||||
return [];
|
||||
}
|
||||
|
||||
return allProxies;
|
||||
}
|
||||
|
||||
// Utility functions
|
||||
function cleanProxyUrl(url: string): string {
|
||||
return url
|
||||
.replace(/^https?:\/\//, '')
|
||||
.replace(/^0+/, '')
|
||||
.replace(/:0+(\d)/g, ':$1');
|
||||
}
|
||||
|
||||
function removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
|
||||
const seen = new Set<string>();
|
||||
const unique: ProxyInfo[] = [];
|
||||
|
||||
for (const proxy of proxies) {
|
||||
const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
unique.push(proxy);
|
||||
}
|
||||
}
|
||||
|
||||
return unique;
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Proxy Query Operations - Getting active proxies from cache
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
|
||||
/**
|
||||
* Get a random active proxy from the cache
|
||||
* @param protocol - Optional protocol filter ('http' | 'https' | 'socks4' | 'socks5')
|
||||
* @param minSuccessRate - Minimum success rate percentage (default: 50)
|
||||
* @returns A random working proxy or null if none found
|
||||
*/
|
||||
export async function getRandomActiveProxy(
|
||||
protocol?: 'http' | 'https' | 'socks4' | 'socks5',
|
||||
minSuccessRate: number = 50
|
||||
): Promise<ProxyInfo | null> {
|
||||
const ctx = OperationContext.create('proxy', 'get-random');
|
||||
|
||||
try {
|
||||
// Get all active proxy keys from cache
|
||||
const pattern = protocol
|
||||
? `${PROXY_CONFIG.CACHE_KEY}:${protocol}://*`
|
||||
: `${PROXY_CONFIG.CACHE_KEY}:*`;
|
||||
|
||||
const keys = await ctx.cache.keys(pattern);
|
||||
|
||||
if (keys.length === 0) {
|
||||
ctx.logger.debug('No active proxies found in cache', { pattern });
|
||||
return null;
|
||||
}
|
||||
|
||||
// Shuffle the keys for randomness
|
||||
const shuffledKeys = keys.sort(() => Math.random() - 0.5);
|
||||
|
||||
// Find a working proxy that meets the criteria
|
||||
for (const key of shuffledKeys) {
|
||||
try {
|
||||
const proxyData: ProxyInfo | null = await ctx.cache.get(key);
|
||||
|
||||
if (
|
||||
proxyData &&
|
||||
proxyData.isWorking &&
|
||||
(!proxyData.successRate || proxyData.successRate >= minSuccessRate)
|
||||
) {
|
||||
ctx.logger.debug('Random active proxy selected', {
|
||||
proxy: `${proxyData.host}:${proxyData.port}`,
|
||||
protocol: proxyData.protocol,
|
||||
successRate: proxyData.successRate?.toFixed(1) + '%',
|
||||
avgResponseTime: proxyData.averageResponseTime
|
||||
? `${proxyData.averageResponseTime.toFixed(0)}ms`
|
||||
: 'N/A',
|
||||
});
|
||||
|
||||
return proxyData;
|
||||
}
|
||||
} catch (error) {
|
||||
ctx.logger.debug('Error reading proxy from cache', { key, error: (error as Error).message });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
ctx.logger.debug('No working proxies found meeting criteria', {
|
||||
protocol,
|
||||
minSuccessRate,
|
||||
keysChecked: shuffledKeys.length,
|
||||
});
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
ctx.logger.error('Error getting random active proxy', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
protocol,
|
||||
minSuccessRate,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* Proxy Queue Operations - Queueing proxy operations
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
export async function queueProxyFetch(): Promise<string> {
|
||||
const ctx = OperationContext.create('proxy', 'queue-fetch');
|
||||
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('proxy');
|
||||
const job = await queue.add('proxy-fetch', {
|
||||
handler: 'proxy',
|
||||
operation: 'fetch-and-check',
|
||||
payload: {},
|
||||
priority: 5,
|
||||
});
|
||||
|
||||
const jobId = job.id || 'unknown';
|
||||
ctx.logger.info('Proxy fetch job queued', { jobId });
|
||||
return jobId;
|
||||
}
|
||||
|
||||
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
|
||||
const ctx = OperationContext.create('proxy', 'queue-check');
|
||||
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('proxy');
|
||||
const job = await queue.add('proxy-check', {
|
||||
handler: 'proxy',
|
||||
operation: 'check-specific',
|
||||
payload: { proxies },
|
||||
priority: 3,
|
||||
});
|
||||
|
||||
const jobId = job.id || 'unknown';
|
||||
ctx.logger.info('Proxy check job queued', { jobId, count: proxies.length });
|
||||
return jobId;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue