boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

test

Browse source
This commit is contained in:
Boki 2025-06-10 09:07:12 -04:00
parent 35df03d00b
commit 34fa4b8e26
2 changed files with 58 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

68
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -8,16 +8,21 @@ export interface ProxySource {
id: string;
url: string;
protocol: string;
working?: number; // Optional, used for stats
total?: number; // Optional, used for stats
lastChecked?: Date; // Optional, used for stats
}
// Shared configuration and utilities
const PROXY_CONFIG = {
CACHE_KEY: 'proxy',
CACHE_STATS_KEY: 'proxy:stats',
CACHE_TTL: 86400, // 24 hours
CHECK_TIMEOUT: 7000,
CHECK_IP: '99.246.102.205',
CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
CONCURRENCY_LIMIT: 100, PROXY_SOURCES: [
CONCURRENCY_LIMIT: 100,
PROXY_SOURCES: [
{id: 'prxchk', url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt', protocol: 'http'},
{id: 'casals', url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http', protocol: 'http'},
{id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'},
@ -52,6 +57,39 @@ let logger: ReturnType<typeof getLogger>;
let cache: CacheProvider;
let httpClient: HttpClient;
let concurrencyLimit: ReturnType<typeof pLimit>;
let proxyStats: ProxySource[] = []
// make a function that takes in source id and a boolean success and updates the proxyStats array
function updateProxyStats(sourceId: string, success: boolean) {
const source = proxyStats.find(s => s.id === sourceId);
if (source !== undefined && source !== null && source.working && source.total) {
source.total += 1;
if (success) {
source.working += 1;
}
return source;
} else {
logger.warn(`Unknown proxy source: ${sourceId}`);
}
}
// make a function that resets proxyStats
async function resetProxyStats(): Promise<void> {
proxyStats = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
working: 0,
lastChecked: new Date(),
protocol: source.protocol,
url: source.url,
}));
// for (const source of proxyStats) {
// await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
// }
return Promise.resolve();
}
// Initialize shared resources
function initializeSharedResources() {
@ -103,6 +141,7 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
initializeSharedResources();
await resetProxyStats();
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
concurrencyLimit(() => fetchProxiesFromSource(source))
@ -194,19 +233,23 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
responseTime: response.responseTime,
};
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
success = true;
await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
} else {
await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
}
// if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
// success = true;
// await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
// } else {
// await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
// }
// if( proxy.source ){
// updateProxyStats(proxy.source, success);
// }
logger.debug('Proxy check completed', {
host: proxy.host,
port: proxy.port,
isWorking,
});
return result;
} catch (error) {
@ -220,9 +263,12 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
};
// If the proxy check failed, remove it from cache - success is here cause i think abort signal fails sometimes
if (!success) {
await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
}
// if (!success) {
// await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result);
// }
// if( proxy.source ){
// updateProxyStats(proxy.source, success);
// }
logger.debug('Proxy check failed', {
host: proxy.host,

2
apps/data-service/src/services/queue.service.ts
View file

@ -43,7 +43,7 @@ export class QueueService {
enableReadyCheck: false,
lazyConnect: true,
// Disable Redis Cluster mode if you're using standalone Redis/Dragonfly
enableOfflineQueue: false
enableOfflineQueue: true
};
// Worker configuration