boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

added initial jobs for proxies, will prob have to be optimized

Browse source
This commit is contained in:
Bojan Kucera 2025-06-08 19:32:53 -04:00
parent 39f6c42044
commit 1ccdbddb71
2 changed files with 141 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

32
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -114,7 +114,7 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
return jobId;
}
export async function fetchProxiesFromSources(): Promise<number> {
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
initializeSharedResources();
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
@ -124,7 +124,7 @@ export async function fetchProxiesFromSources(): Promise<number> {
let allProxies: ProxyInfo[] = result.flat();
allProxies = removeDuplicateProxies(allProxies);
// await checkProxies(allProxies);
return allProxies.length;
return allProxies;
}
export async function fetchProxiesFromSource(source: { url: string; protocol: string }): Promise<ProxyInfo[]> {
@ -177,29 +177,6 @@ export async function fetchProxiesFromSource(source: { url: string; protocol: st
return allProxies;
}
/**
* Check multiple proxies concurrently
*/
export async function checkProxies(proxies: ProxyInfo[]): Promise<ProxyInfo[]> {
initializeSharedResources();
logger.info('Checking proxies', { count: proxies.length });
const checkPromises = proxies.map(proxy =>
concurrencyLimit(() => checkProxy(proxy))
);
const results = await Promise.all(checkPromises);
const workingCount = results.filter((r: ProxyInfo) => r.isWorking).length;
logger.info('Proxy check completed', {
total: proxies.length,
working: workingCount,
failed: proxies.length - workingCount
});
return results;
}
/**
* Check if a proxy is working
*/
@ -207,7 +184,7 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
initializeSharedResources();
let success = false;
logger.debug(`Checking Proxy with ${concurrencyLimit.pendingCount} pending:`, {
logger.debug(`Checking Proxy:`, {
protocol: proxy.protocol,
host: proxy.host,
port: proxy.port,
@ -269,8 +246,6 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
}
}
// Utility functions
function cleanProxyUrl(url: string): string {
return url
@ -301,7 +276,6 @@ export const proxyTasks = {
fetchProxiesFromSources,
fetchProxiesFromSource,
checkProxy,
checkProxies,
};
// Export singleton instance for backward compatibility (optional)