boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

refactor of data-service

Browse source
This commit is contained in:
Boki 2025-06-12 08:03:09 -04:00
parent 3097686849
commit 54314a0cde
49 changed files with 2394 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

177
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -1,4 +1,3 @@
import pLimit from 'p-limit';
import { createCache, type CacheProvider } from '@stock-bot/cache';
import { HttpClient, ProxyInfo } from '@stock-bot/http';
import { getLogger } from '@stock-bot/logger';
@ -22,7 +21,6 @@ const PROXY_CONFIG = {
CHECK_TIMEOUT: 7000,
CHECK_IP: '99.246.102.205',
CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
CONCURRENCY_LIMIT: 100,
PROXY_SOURCES: [
{
id: 'prxchk',
@ -154,10 +152,10 @@ const PROXY_CONFIG = {
};
// Shared instances (module-scoped, not global)
let isInitialized = false; // Track if resources are initialized
let logger: ReturnType<typeof getLogger>;
let cache: CacheProvider;
let httpClient: HttpClient;
let concurrencyLimit: ReturnType<typeof pLimit>;
let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
@ -167,6 +165,37 @@ let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
url: source.url,
}));
/**
* Initialize proxy resources (cache and shared dependencies)
* This should be called before any proxy operations
* @param waitForCache - Whether to wait for cache readiness (default: false for fallback mode)
*/
export async function initializeProxyResources(waitForCache = false): Promise<void> {
// Skip if already initialized
if (isInitialized) {
return;
}
logger = getLogger('proxy-tasks');
cache = createCache({
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true,
});
httpClient = new HttpClient({ timeout: 10000 }, logger);
if (waitForCache) {
logger.info('Initializing proxy cache...');
await cache.waitForReady(10000);
logger.info('Proxy cache initialized successfully');
logger.info('Proxy tasks initialized');
} else {
logger.info('Proxy tasks initialized (fallback mode)');
}
isInitialized = true;
}
// make a function that takes in source id and a boolean success and updates the proxyStats array
async function updateProxyStats(sourceId: string, success: boolean) {
const source = proxyStats.find(s => s.id === sourceId);
@ -278,50 +307,8 @@ async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean): Promise
}
}
/**
* Initialize proxy cache for use during application startup
* This should be called before any proxy operations
*/
export async function initializeProxyCache(): Promise<void> {
logger = getLogger('proxy-tasks');
cache = createCache({
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true,
});
logger.info('Initializing proxy cache...');
await cache.waitForReady(10000);
logger.info('Proxy cache initialized successfully');
// Initialize other shared resources that don't require cache
httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized');
}
async function initializeSharedResources() {
if (!logger) {
// If not initialized at startup, initialize with fallback mode
logger = getLogger('proxy-tasks');
cache = createCache({
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true,
});
httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized (fallback mode)');
}
}
// Individual task functions
export async function queueProxyFetch(): Promise<string> {
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
type: 'proxy-fetch',
@ -337,8 +324,6 @@ export async function queueProxyFetch(): Promise<string> {
}
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
type: 'proxy-check',
@ -354,35 +339,15 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
}
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
await initializeSharedResources();
await resetProxyStats();
// Ensure concurrencyLimit is available before using it
if (!concurrencyLimit) {
logger.error('concurrencyLimit not initialized, using sequential processing');
const result = [];
for (const source of PROXY_CONFIG.PROXY_SOURCES) {
const proxies = await fetchProxiesFromSource(source);
result.push(...proxies);
}
let allProxies: ProxyInfo[] = result;
allProxies = removeDuplicateProxies(allProxies);
return allProxies;
}
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
concurrencyLimit(() => fetchProxiesFromSource(source))
);
const result = await Promise.all(sources);
let allProxies: ProxyInfo[] = result.flat();
const fetchPromises = PROXY_CONFIG.PROXY_SOURCES.map(source => fetchProxiesFromSource(source));
const results = await Promise.all(fetchPromises);
let allProxies: ProxyInfo[] = results.flat();
allProxies = removeDuplicateProxies(allProxies);
// await checkProxies(allProxies);
return allProxies;
}
export async function fetchProxiesFromSource(source: ProxySource): Promise<ProxyInfo[]> {
await initializeSharedResources();
const allProxies: ProxyInfo[] = [];
try {
@ -436,8 +401,6 @@ export async function fetchProxiesFromSource(source: ProxySource): Promise<Proxy
* Check if a proxy is working
*/
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
await initializeSharedResources();
let success = false;
logger.debug(`Checking Proxy:`, {
protocol: proxy.protocol,
@ -504,6 +467,76 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
}
}
/**
* Get a random active proxy from the cache
* @param protocol - Optional protocol filter ('http' | 'https' | 'socks4' | 'socks5')
* @param minSuccessRate - Minimum success rate percentage (default: 50)
* @returns A random working proxy or null if none found
*/
export async function getRandomActiveProxy(
protocol?: 'http' | 'https' | 'socks4' | 'socks5',
minSuccessRate: number = 50
): Promise<ProxyInfo | null> {
try {
// Get all active proxy keys from cache
const pattern = protocol
? `${PROXY_CONFIG.CACHE_KEY}:${protocol}://*`
: `${PROXY_CONFIG.CACHE_KEY}:*`;
const keys = await cache.keys(pattern);
if (keys.length === 0) {
logger.debug('No active proxies found in cache', { pattern });
return null;
}
// Shuffle the keys for randomness
const shuffledKeys = keys.sort(() => Math.random() - 0.5);
// Find a working proxy that meets the criteria
for (const key of shuffledKeys) {
try {
const proxyData: ProxyInfo | null = await cache.get(key);
if (
proxyData &&
proxyData.isWorking &&
(!proxyData.successRate || proxyData.successRate >= minSuccessRate)
) {
logger.debug('Random active proxy selected', {
proxy: `${proxyData.host}:${proxyData.port}`,
protocol: proxyData.protocol,
successRate: proxyData.successRate?.toFixed(1) + '%',
avgResponseTime: proxyData.averageResponseTime
? `${proxyData.averageResponseTime.toFixed(0)}ms`
: 'N/A',
});
return proxyData;
}
} catch (error) {
logger.debug('Error reading proxy from cache', { key, error: (error as Error).message });
continue;
}
}
logger.debug('No working proxies found meeting criteria', {
protocol,
minSuccessRate,
keysChecked: shuffledKeys.length,
});
return null;
} catch (error) {
logger.error('Error getting random active proxy', {
error: error instanceof Error ? error.message : String(error),
protocol,
minSuccessRate,
});
return null;
}
}
// Utility functions
function cleanProxyUrl(url: string): string {
return url