boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

running prettier for cleanup

Browse source
This commit is contained in:
Boki 2025-06-11 10:13:25 -04:00
parent fe7733aeb5
commit d85cd58acd
151 changed files with 29158 additions and 27966 deletions
Download patch file Download diff file Expand all files Collapse all files

972
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -1,436 +1,536 @@
import { getLogger } from '@stock-bot/logger';
import { createCache, type CacheProvider } from '@stock-bot/cache';
import { HttpClient, ProxyInfo } from '@stock-bot/http';
import pLimit from 'p-limit';
// Type definitions
export interface ProxySource {
id: string;
url: string;
protocol: string;
working?: number; // Optional, used for stats
total?: number; // Optional, used for stats
percentWorking?: number; // Optional, used for stats
lastChecked?: Date; // Optional, used for stats
}
// Shared configuration and utilities
const PROXY_CONFIG = {
CACHE_KEY: 'active',
CACHE_STATS_KEY: 'stats',
CACHE_TTL: 86400, // 24 hours
CHECK_TIMEOUT: 7000,
CHECK_IP: '99.246.102.205',
CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
CONCURRENCY_LIMIT: 100,
PROXY_SOURCES: [
{id: 'prxchk', url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt', protocol: 'http'},
{id: 'casals', url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http', protocol: 'http'},
{id: 'sunny9577', url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', protocol: 'http'},
{id: 'themiralay', url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt', protocol: 'http'},
{id: 'casa-ls', url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http', protocol: 'http'},
{id: 'databay', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
{id: 'speedx', url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', protocol: 'http'},
{id: 'monosans', url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', protocol: 'http'},
{id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'},
{id: 'vakhov-fresh', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', protocol: 'http'},
{id: 'kangproxy', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', protocol: 'http'},
{id: 'gfpcom', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http'},
{id: 'dpangestuw', url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', protocol: 'http'},
{id: 'gitrecon', url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', protocol: 'http'},
{id: 'vakhov-master', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
{id: 'breaking-tech', url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http'},
{id: 'ercindedeoglu', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', protocol: 'http'},
{id: 'tuanminpay', url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', protocol: 'http'},
{id: 'r00tee-https', url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt', protocol: 'https'},
{id: 'ercindedeoglu-https', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt', protocol: 'https'},
{id: 'vakhov-fresh-https', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https'},
{id: 'databay-https', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt', protocol: 'https'},
{id: 'kangproxy-https', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt', protocol: 'https'},
{id: 'zloi-user-https', url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt', protocol: 'https'},
{id: 'gfpcom-https', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt', protocol: 'https'},
]
};
// Shared instances (module-scoped, not global)
let logger: ReturnType<typeof getLogger>;
let cache: CacheProvider;
let httpClient: HttpClient;
let concurrencyLimit: ReturnType<typeof pLimit>;
let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
working: 0,
lastChecked: new Date(),
protocol: source.protocol,
url: source.url,
}));
// make a function that takes in source id and a boolean success and updates the proxyStats array
async function updateProxyStats(sourceId: string, success: boolean) {
const source = proxyStats.find(s => s.id === sourceId);
if (source !== undefined) {
if(typeof source.working !== 'number')
source.working = 0;
if(typeof source.total !== 'number')
source.total = 0;
source.total += 1;
if (success) {
source.working += 1;
}
source.percentWorking = source.working / source.total * 100;
source.lastChecked = new Date();
await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
return source;
} else {
logger.warn(`Unknown proxy source: ${sourceId}`);
}
}
// make a function that resets proxyStats
async function resetProxyStats(): Promise<void> {
proxyStats = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
working: 0,
lastChecked: new Date(),
protocol: source.protocol,
url: source.url,
}));
for (const source of proxyStats) {
await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
}
return Promise.resolve();
}
/**
* Update proxy data in cache with working/total stats and average response time
* @param proxy - The proxy to update
* @param isWorking - Whether the proxy is currently working
*/
async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean): Promise<void> {
const cacheKey = `${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`;
try {
const existing: any = await cache.get(cacheKey);
// For failed proxies, only update if they already exist
if (!isWorking && !existing) {
logger.debug('Proxy not in cache, skipping failed update', {
proxy: `${proxy.host}:${proxy.port}`
});
return;
}
// Calculate new average response time if we have a response time
let newAverageResponseTime = existing?.averageResponseTime;
if (proxy.responseTime !== undefined) {
const existingAvg = existing?.averageResponseTime || 0;
const existingTotal = existing?.total || 0;
// Calculate weighted average: (existing_avg * existing_count + new_response) / (existing_count + 1)
newAverageResponseTime = existingTotal > 0
? ((existingAvg * existingTotal) + proxy.responseTime) / (existingTotal + 1)
: proxy.responseTime;
}
// Build updated proxy data
const updated = {
...existing,
...proxy, // Keep latest proxy info
total: (existing?.total || 0) + 1,
working: isWorking ? (existing?.working || 0) + 1 : (existing?.working || 0),
isWorking,
lastChecked: new Date(),
// Add firstSeen only for new entries
...(existing ? {} : { firstSeen: new Date() }),
// Update average response time if we calculated a new one
...(newAverageResponseTime !== undefined ? { averageResponseTime: newAverageResponseTime } : {})
};
// Calculate success rate
updated.successRate = updated.total > 0 ? (updated.working / updated.total) * 100 : 0;
// Save to cache: reset TTL for working proxies, keep existing TTL for failed ones
const cacheOptions = isWorking ? PROXY_CONFIG.CACHE_TTL : undefined;
await cache.set(cacheKey, updated, cacheOptions);
logger.debug(`Updated ${isWorking ? 'working' : 'failed'} proxy in cache`, {
proxy: `${proxy.host}:${proxy.port}`,
working: updated.working,
total: updated.total,
successRate: updated.successRate.toFixed(1) + '%',
avgResponseTime: updated.averageResponseTime ? `${updated.averageResponseTime.toFixed(0)}ms` : 'N/A'
});
} catch (error) {
logger.error('Failed to update proxy in cache', {
proxy: `${proxy.host}:${proxy.port}`,
error: error instanceof Error ? error.message : String(error)
});
}
}
/**
* Initialize proxy cache for use during application startup
* This should be called before any proxy operations
*/
export async function initializeProxyCache(): Promise<void> {
logger = getLogger('proxy-tasks');
cache = createCache({
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true
});
logger.info('Initializing proxy cache...');
await cache.waitForReady(10000);
logger.info('Proxy cache initialized successfully');
// Initialize other shared resources that don't require cache
httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized');
}
async function initializeSharedResources() {
if (!logger) {
// If not initialized at startup, initialize with fallback mode
logger = getLogger('proxy-tasks');
cache = createCache({
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true
});
httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized (fallback mode)');
}
}
// Individual task functions
export async function queueProxyFetch(): Promise<string> {
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
type: 'proxy-fetch',
provider: 'proxy-service',
operation: 'fetch-and-check',
payload: {},
priority: 5
});
const jobId = job.id || 'unknown';
logger.info('Proxy fetch job queued', { jobId });
return jobId;
}
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
type: 'proxy-check',
provider: 'proxy-service',
operation: 'check-specific',
payload: { proxies },
priority: 3
});
const jobId = job.id || 'unknown';
logger.info('Proxy check job queued', { jobId, count: proxies.length });
return jobId;
}
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
await initializeSharedResources();
await resetProxyStats();
// Ensure concurrencyLimit is available before using it
if (!concurrencyLimit) {
logger.error('concurrencyLimit not initialized, using sequential processing');
const result = [];
for (const source of PROXY_CONFIG.PROXY_SOURCES) {
const proxies = await fetchProxiesFromSource(source);
result.push(...proxies);
}
let allProxies: ProxyInfo[] = result;
allProxies = removeDuplicateProxies(allProxies);
return allProxies;
}
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
concurrencyLimit(() => fetchProxiesFromSource(source))
);
const result = await Promise.all(sources);
let allProxies: ProxyInfo[] = result.flat();
allProxies = removeDuplicateProxies(allProxies);
// await checkProxies(allProxies);
return allProxies;
}
export async function fetchProxiesFromSource(source: ProxySource): Promise<ProxyInfo[]> {
await initializeSharedResources();
const allProxies: ProxyInfo[] = [];
try {
logger.info(`Fetching proxies from ${source.url}`);
const response = await httpClient.get(source.url, {
timeout: 10000
});
if (response.status !== 200) {
logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
return [];
}
const text = response.data;
const lines = text.split('\n').filter((line: string) => line.trim());
for (const line of lines) {
let trimmed = line.trim();
trimmed = cleanProxyUrl(trimmed);
if (!trimmed || trimmed.startsWith('#')) continue;
// Parse formats like "host:port" or "host:port:user:pass"
const parts = trimmed.split(':');
if (parts.length >= 2) {
const proxy: ProxyInfo = {
source: source.id,
protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5',
host: parts[0],
port: parseInt(parts[1])
};
if (!isNaN(proxy.port) && proxy.host) {
allProxies.push(proxy);
}
}
}
logger.info(`Parsed ${allProxies.length} proxies from ${source.url}`);
} catch (error) {
logger.error(`Error fetching proxies from ${source.url}`, error);
return [];
}
return allProxies;
}
/**
* Check if a proxy is working
*/
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
await initializeSharedResources();
let success = false;
logger.debug(`Checking Proxy:`, {
protocol: proxy.protocol,
host: proxy.host,
port: proxy.port,
});
try {
// Test the proxy
const response = await httpClient.get(PROXY_CONFIG.CHECK_URL, {
proxy,
timeout: PROXY_CONFIG.CHECK_TIMEOUT
});
const isWorking = response.status >= 200 && response.status < 300;
const result: ProxyInfo = {
...proxy,
isWorking,
lastChecked: new Date(),
responseTime: response.responseTime,
};
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
success = true;
await updateProxyInCache(result, true);
} else {
await updateProxyInCache(result, false);
}
if( proxy.source ){
await updateProxyStats(proxy.source, success);
}
logger.debug('Proxy check completed', {
host: proxy.host,
port: proxy.port,
isWorking,
});
return result;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
const result: ProxyInfo = {
...proxy,
isWorking: false,
error: errorMessage,
lastChecked: new Date()
};
// Update cache for failed proxy (increment total, don't update TTL)
await updateProxyInCache(result, false);
if( proxy.source ){
await updateProxyStats(proxy.source, success);
}
logger.debug('Proxy check failed', {
host: proxy.host,
port: proxy.port,
error: errorMessage
});
return result;
}
}
// Utility functions
function cleanProxyUrl(url: string): string {
return url
.replace(/^https?:\/\//, '')
.replace(/^0+/, '')
.replace(/:0+(\d)/g, ':$1');
}
function removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
const seen = new Set<string>();
const unique: ProxyInfo[] = [];
for (const proxy of proxies) {
const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
if (!seen.has(key)) {
seen.add(key);
unique.push(proxy);
}
}
return unique;
}
// Optional: Export a convenience object that groups related tasks
export const proxyTasks = {
queueProxyFetch,
queueProxyCheck,
fetchProxiesFromSources,
fetchProxiesFromSource,
checkProxy,
};
// Export singleton instance for backward compatibility (optional)
// Remove this if you want to fully move to the task-based approach
export const proxyService = proxyTasks;
import pLimit from 'p-limit';
import { createCache, type CacheProvider } from '@stock-bot/cache';
import { HttpClient, ProxyInfo } from '@stock-bot/http';
import { getLogger } from '@stock-bot/logger';
// Type definitions
export interface ProxySource {
id: string;
url: string;
protocol: string;
working?: number; // Optional, used for stats
total?: number; // Optional, used for stats
percentWorking?: number; // Optional, used for stats
lastChecked?: Date; // Optional, used for stats
}
// Shared configuration and utilities
const PROXY_CONFIG = {
CACHE_KEY: 'active',
CACHE_STATS_KEY: 'stats',
CACHE_TTL: 86400, // 24 hours
CHECK_TIMEOUT: 7000,
CHECK_IP: '99.246.102.205',
CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
CONCURRENCY_LIMIT: 100,
PROXY_SOURCES: [
{
id: 'prxchk',
url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',
protocol: 'http',
},
{
id: 'casals',
url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',
protocol: 'http',
},
{
id: 'sunny9577',
url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',
protocol: 'http',
},
{
id: 'themiralay',
url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',
protocol: 'http',
},
{
id: 'casa-ls',
url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',
protocol: 'http',
},
{
id: 'databay',
url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',
protocol: 'http',
},
{
id: 'speedx',
url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',
protocol: 'http',
},
{
id: 'monosans',
url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',
protocol: 'http',
},
{
id: 'murong',
url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',
protocol: 'http',
},
{
id: 'vakhov-fresh',
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',
protocol: 'http',
},
{
id: 'kangproxy',
url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',
protocol: 'http',
},
{
id: 'gfpcom',
url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt',
protocol: 'http',
},
{
id: 'dpangestuw',
url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',
protocol: 'http',
},
{
id: 'gitrecon',
url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',
protocol: 'http',
},
{
id: 'vakhov-master',
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',
protocol: 'http',
},
{
id: 'breaking-tech',
url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt',
protocol: 'http',
},
{
id: 'ercindedeoglu',
url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',
protocol: 'http',
},
{
id: 'tuanminpay',
url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',
protocol: 'http',
},
{
id: 'r00tee-https',
url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',
protocol: 'https',
},
{
id: 'ercindedeoglu-https',
url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',
protocol: 'https',
},
{
id: 'vakhov-fresh-https',
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt',
protocol: 'https',
},
{
id: 'databay-https',
url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',
protocol: 'https',
},
{
id: 'kangproxy-https',
url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',
protocol: 'https',
},
{
id: 'zloi-user-https',
url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',
protocol: 'https',
},
{
id: 'gfpcom-https',
url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',
protocol: 'https',
},
],
};
// Shared instances (module-scoped, not global)
let logger: ReturnType<typeof getLogger>;
let cache: CacheProvider;
let httpClient: HttpClient;
let concurrencyLimit: ReturnType<typeof pLimit>;
let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
working: 0,
lastChecked: new Date(),
protocol: source.protocol,
url: source.url,
}));
// make a function that takes in source id and a boolean success and updates the proxyStats array
async function updateProxyStats(sourceId: string, success: boolean) {
const source = proxyStats.find(s => s.id === sourceId);
if (source !== undefined) {
if (typeof source.working !== 'number') source.working = 0;
if (typeof source.total !== 'number') source.total = 0;
source.total += 1;
if (success) {
source.working += 1;
}
source.percentWorking = (source.working / source.total) * 100;
source.lastChecked = new Date();
await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
return source;
} else {
logger.warn(`Unknown proxy source: ${sourceId}`);
}
}
// make a function that resets proxyStats
async function resetProxyStats(): Promise<void> {
proxyStats = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
working: 0,
lastChecked: new Date(),
protocol: source.protocol,
url: source.url,
}));
for (const source of proxyStats) {
await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
}
return Promise.resolve();
}
/**
* Update proxy data in cache with working/total stats and average response time
* @param proxy - The proxy to update
* @param isWorking - Whether the proxy is currently working
*/
async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean): Promise<void> {
const cacheKey = `${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`;
try {
const existing: any = await cache.get(cacheKey);
// For failed proxies, only update if they already exist
if (!isWorking && !existing) {
logger.debug('Proxy not in cache, skipping failed update', {
proxy: `${proxy.host}:${proxy.port}`,
});
return;
}
// Calculate new average response time if we have a response time
let newAverageResponseTime = existing?.averageResponseTime;
if (proxy.responseTime !== undefined) {
const existingAvg = existing?.averageResponseTime || 0;
const existingTotal = existing?.total || 0;
// Calculate weighted average: (existing_avg * existing_count + new_response) / (existing_count + 1)
newAverageResponseTime =
existingTotal > 0
? (existingAvg * existingTotal + proxy.responseTime) / (existingTotal + 1)
: proxy.responseTime;
}
// Build updated proxy data
const updated = {
...existing,
...proxy, // Keep latest proxy info
total: (existing?.total || 0) + 1,
working: isWorking ? (existing?.working || 0) + 1 : existing?.working || 0,
isWorking,
lastChecked: new Date(),
// Add firstSeen only for new entries
...(existing ? {} : { firstSeen: new Date() }),
// Update average response time if we calculated a new one
...(newAverageResponseTime !== undefined
? { averageResponseTime: newAverageResponseTime }
: {}),
};
// Calculate success rate
updated.successRate = updated.total > 0 ? (updated.working / updated.total) * 100 : 0;
// Save to cache: reset TTL for working proxies, keep existing TTL for failed ones
const cacheOptions = isWorking ? PROXY_CONFIG.CACHE_TTL : undefined;
await cache.set(cacheKey, updated, cacheOptions);
logger.debug(`Updated ${isWorking ? 'working' : 'failed'} proxy in cache`, {
proxy: `${proxy.host}:${proxy.port}`,
working: updated.working,
total: updated.total,
successRate: updated.successRate.toFixed(1) + '%',
avgResponseTime: updated.averageResponseTime
? `${updated.averageResponseTime.toFixed(0)}ms`
: 'N/A',
});
} catch (error) {
logger.error('Failed to update proxy in cache', {
proxy: `${proxy.host}:${proxy.port}`,
error: error instanceof Error ? error.message : String(error),
});
}
}
/**
* Initialize proxy cache for use during application startup
* This should be called before any proxy operations
*/
export async function initializeProxyCache(): Promise<void> {
logger = getLogger('proxy-tasks');
cache = createCache({
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true,
});
logger.info('Initializing proxy cache...');
await cache.waitForReady(10000);
logger.info('Proxy cache initialized successfully');
// Initialize other shared resources that don't require cache
httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized');
}
async function initializeSharedResources() {
if (!logger) {
// If not initialized at startup, initialize with fallback mode
logger = getLogger('proxy-tasks');
cache = createCache({
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true,
});
httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized (fallback mode)');
}
}
// Individual task functions
export async function queueProxyFetch(): Promise<string> {
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
type: 'proxy-fetch',
provider: 'proxy-service',
operation: 'fetch-and-check',
payload: {},
priority: 5,
});
const jobId = job.id || 'unknown';
logger.info('Proxy fetch job queued', { jobId });
return jobId;
}
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
type: 'proxy-check',
provider: 'proxy-service',
operation: 'check-specific',
payload: { proxies },
priority: 3,
});
const jobId = job.id || 'unknown';
logger.info('Proxy check job queued', { jobId, count: proxies.length });
return jobId;
}
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
await initializeSharedResources();
await resetProxyStats();
// Ensure concurrencyLimit is available before using it
if (!concurrencyLimit) {
logger.error('concurrencyLimit not initialized, using sequential processing');
const result = [];
for (const source of PROXY_CONFIG.PROXY_SOURCES) {
const proxies = await fetchProxiesFromSource(source);
result.push(...proxies);
}
let allProxies: ProxyInfo[] = result;
allProxies = removeDuplicateProxies(allProxies);
return allProxies;
}
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
concurrencyLimit(() => fetchProxiesFromSource(source))
);
const result = await Promise.all(sources);
let allProxies: ProxyInfo[] = result.flat();
allProxies = removeDuplicateProxies(allProxies);
// await checkProxies(allProxies);
return allProxies;
}
export async function fetchProxiesFromSource(source: ProxySource): Promise<ProxyInfo[]> {
await initializeSharedResources();
const allProxies: ProxyInfo[] = [];
try {
logger.info(`Fetching proxies from ${source.url}`);
const response = await httpClient.get(source.url, {
timeout: 10000,
});
if (response.status !== 200) {
logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
return [];
}
const text = response.data;
const lines = text.split('\n').filter((line: string) => line.trim());
for (const line of lines) {
let trimmed = line.trim();
trimmed = cleanProxyUrl(trimmed);
if (!trimmed || trimmed.startsWith('#')) continue;
// Parse formats like "host:port" or "host:port:user:pass"
const parts = trimmed.split(':');
if (parts.length >= 2) {
const proxy: ProxyInfo = {
source: source.id,
protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5',
host: parts[0],
port: parseInt(parts[1]),
};
if (!isNaN(proxy.port) && proxy.host) {
allProxies.push(proxy);
}
}
}
logger.info(`Parsed ${allProxies.length} proxies from ${source.url}`);
} catch (error) {
logger.error(`Error fetching proxies from ${source.url}`, error);
return [];
}
return allProxies;
}
/**
* Check if a proxy is working
*/
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
await initializeSharedResources();
let success = false;
logger.debug(`Checking Proxy:`, {
protocol: proxy.protocol,
host: proxy.host,
port: proxy.port,
});
try {
// Test the proxy
const response = await httpClient.get(PROXY_CONFIG.CHECK_URL, {
proxy,
timeout: PROXY_CONFIG.CHECK_TIMEOUT,
});
const isWorking = response.status >= 200 && response.status < 300;
const result: ProxyInfo = {
...proxy,
isWorking,
lastChecked: new Date(),
responseTime: response.responseTime,
};
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
success = true;
await updateProxyInCache(result, true);
} else {
await updateProxyInCache(result, false);
}
if (proxy.source) {
await updateProxyStats(proxy.source, success);
}
logger.debug('Proxy check completed', {
host: proxy.host,
port: proxy.port,
isWorking,
});
return result;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
const result: ProxyInfo = {
...proxy,
isWorking: false,
error: errorMessage,
lastChecked: new Date(),
};
// Update cache for failed proxy (increment total, don't update TTL)
await updateProxyInCache(result, false);
if (proxy.source) {
await updateProxyStats(proxy.source, success);
}
logger.debug('Proxy check failed', {
host: proxy.host,
port: proxy.port,
error: errorMessage,
});
return result;
}
}
// Utility functions
function cleanProxyUrl(url: string): string {
return url
.replace(/^https?:\/\//, '')
.replace(/^0+/, '')
.replace(/:0+(\d)/g, ':$1');
}
function removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
const seen = new Set<string>();
const unique: ProxyInfo[] = [];
for (const proxy of proxies) {
const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
if (!seen.has(key)) {
seen.add(key);
unique.push(proxy);
}
}
return unique;
}
// Optional: Export a convenience object that groups related tasks
export const proxyTasks = {
queueProxyFetch,
queueProxyCheck,
fetchProxiesFromSources,
fetchProxiesFromSource,
checkProxy,
};
// Export singleton instance for backward compatibility (optional)
// Remove this if you want to fully move to the task-based approach
export const proxyService = proxyTasks;