boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

made a mess

Browse source
This commit is contained in:
Boki 2025-06-10 12:34:12 -04:00
parent 1caa2d5168
commit f52e3332df
5 changed files with 55 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

71
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -10,6 +10,7 @@ export interface ProxySource {
protocol: string;
working?: number; // Optional, used for stats
total?: number; // Optional, used for stats
percentWorking?: number; // Optional, used for stats
lastChecked?: Date; // Optional, used for stats
}
@ -57,17 +58,31 @@ let logger: ReturnType<typeof getLogger>;
let cache: CacheProvider;
let httpClient: HttpClient;
let concurrencyLimit: ReturnType<typeof pLimit>;
let proxyStats: ProxySource[] = []
let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
working: 0,
lastChecked: new Date(),
protocol: source.protocol,
url: source.url,
}));
// make a function that takes in source id and a boolean success and updates the proxyStats array
function updateProxyStats(sourceId: string, success: boolean) {
async function updateProxyStats(sourceId: string, success: boolean) {
const source = proxyStats.find(s => s.id === sourceId);
if (source !== undefined && source !== null && source.working && source.total) {
if (source !== undefined) {
if(typeof source.working !== 'number')
source.working = 0;
if(typeof source.total !== 'number')
source.total = 0;
source.total += 1;
if (success) {
source.working += 1;
}
source.percentWorking = source.working / source.total * 100;
source.lastChecked = new Date();
await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
return source;
} else {
logger.warn(`Unknown proxy source: ${sourceId}`);
@ -84,18 +99,24 @@ async function resetProxyStats(): Promise<void> {
protocol: source.protocol,
url: source.url,
}));
// for (const source of proxyStats) {
// await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
// }
for (const source of proxyStats) {
await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
}
return Promise.resolve();
}
// Initialize shared resources
function initializeSharedResources() {
async function initializeSharedResources() {
if (!logger) {
logger = getLogger('proxy-tasks');
cache = createCache('hybrid');
cache = createCache('hybrid', {
name: 'proxy-tasks',
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true
});
await cache.waitForReady();
httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized');
@ -104,7 +125,7 @@ function initializeSharedResources() {
// Individual task functions
export async function queueProxyFetch(): Promise<string> {
initializeSharedResources();
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
@ -122,7 +143,7 @@ export async function queueProxyFetch(): Promise<string> {
}
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
initializeSharedResources();
await initializeSharedResources();
const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({
@ -140,7 +161,7 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
}
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
initializeSharedResources();
await initializeSharedResources();
await resetProxyStats();
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
@ -154,7 +175,7 @@ export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
}
export async function fetchProxiesFromSource(source: ProxySource): Promise<ProxyInfo[]> {
initializeSharedResources();
await initializeSharedResources();
const allProxies: ProxyInfo[] = [];
@ -208,7 +229,7 @@ export async function fetchProxiesFromSource(source: ProxySource): Promise<Proxy
* Check if a proxy is working
*/
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
initializeSharedResources();
await initializeSharedResources();
let success = false;
logger.debug(`Checking Proxy:`, {
@ -233,16 +254,16 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
responseTime: response.responseTime,
};
// if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
// success = true;
// await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
// } else {
// await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
// }
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
success = true;
await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
} else {
await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
}
// if( proxy.source ){
// updateProxyStats(proxy.source, success);
// }
if( proxy.source ){
await updateProxyStats(proxy.source, success);
}
logger.debug('Proxy check completed', {
host: proxy.host,
@ -266,9 +287,9 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
// if (!success) {
// await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result);
// }
// if( proxy.source ){
// updateProxyStats(proxy.source, success);
// }
if( proxy.source ){
await updateProxyStats(proxy.source, success);
}
logger.debug('Proxy check failed', {
host: proxy.host,

2
apps/data-service/src/utils/batch-processor.ts
View file

@ -23,7 +23,6 @@ export class BatchProcessor {
private cacheProvider: CacheProvider;
private isReady = false;
private keyPrefix: string = 'batch:'; // Default key prefix for batch payloads
constructor(
private queueManager: any,
private cacheOptions?: { keyPrefix?: string; ttl?: number } // Optional cache configuration
@ -31,6 +30,7 @@ export class BatchProcessor {
this.keyPrefix = cacheOptions?.keyPrefix || 'batch:';
// Initialize cache provider with batch-specific settings
this.cacheProvider = createCache('redis', {
name: 'batch-processor',
keyPrefix: this.keyPrefix,
ttl: cacheOptions?.ttl || 86400 * 2, // 48 hours default
enableMetrics: true