boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

starting to implement the queue job service

Browse source
This commit is contained in:
Bojan Kucera 2025-06-08 13:35:07 -04:00
parent c10a524aa8
commit 8681c34529
9 changed files with 458 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

30
apps/data-service/src/services/proxy.service.ts
View file

@ -2,15 +2,16 @@ import { Logger } from '@stock-bot/logger';
import createCache, { type CacheProvider } from '@stock-bot/cache';
import { HttpClient, ProxyInfo } from '@stock-bot/http';
import pLimit from 'p-limit';
import { queueService } from './queue.service';
export class ProxyService {
private logger = new Logger('proxy-service');
private cache: CacheProvider = createCache('hybrid');
private httpClient: HttpClient;
private readonly concurrencyLimit = pLimit(250);
private readonly concurrencyLimit = pLimit(100);
private readonly CACHE_KEY = 'proxy';
private readonly CACHE_TTL = 86400; // 24 hours
private readonly CHECK_TIMEOUT = 5000;
private readonly CHECK_TIMEOUT = 7000;
private readonly CHECK_IP = '99.246.102.205'
private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
private readonly PROXY_SOURCES = [
@ -20,13 +21,10 @@ export class ProxyService {
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',protocol: 'http', },
// {url: 'https://github.com/zloi-user/hideip.me/raw/refs/heads/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http' },
{url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', },
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
@ -35,17 +33,15 @@ export class ProxyService {
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/http.txt',protocol: 'http', },
{url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
{url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https' },
{url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', },
{url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',protocol: 'https', },
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', },
// {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', },
{url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', },
{url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', },
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
@ -68,12 +64,24 @@ export class ProxyService {
constructor() {
this.httpClient = new HttpClient({
timeout: this.CHECK_TIMEOUT,
timeout: 10000,
}, this.logger);
// Start scheduled tasks
this.initializeScheduling();
this.logger.info('ProxyService initialized');
}
private async initializeScheduling() {
try {
await queueService.scheduleRecurringTasks();
this.logger.info('Proxy scheduling initialized');
} catch (error) {
this.logger.error('Failed to initialize scheduling', { error });
}
}
async fetchProxiesFromSources() : Promise<number> {
const sources = this.PROXY_SOURCES.map(source =>