boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

made a mess

Browse source
This commit is contained in:
Boki 2025-06-10 12:34:12 -04:00
parent 1caa2d5168
commit f52e3332df
5 changed files with 55 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

71
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -10,6 +10,7 @@ export interface ProxySource {
protocol: string; protocol: string;
working?: number; // Optional, used for stats working?: number; // Optional, used for stats
total?: number; // Optional, used for stats total?: number; // Optional, used for stats
percentWorking?: number; // Optional, used for stats
lastChecked?: Date; // Optional, used for stats lastChecked?: Date; // Optional, used for stats
} }
@ -57,17 +58,31 @@ let logger: ReturnType<typeof getLogger>;
let cache: CacheProvider; let cache: CacheProvider;
let httpClient: HttpClient; let httpClient: HttpClient;
let concurrencyLimit: ReturnType<typeof pLimit>; let concurrencyLimit: ReturnType<typeof pLimit>;
let proxyStats: ProxySource[] = [] let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
id: source.id,
total: 0,
working: 0,
lastChecked: new Date(),
protocol: source.protocol,
url: source.url,
}));
// make a function that takes in source id and a boolean success and updates the proxyStats array // make a function that takes in source id and a boolean success and updates the proxyStats array
function updateProxyStats(sourceId: string, success: boolean) { async function updateProxyStats(sourceId: string, success: boolean) {
const source = proxyStats.find(s => s.id === sourceId); const source = proxyStats.find(s => s.id === sourceId);
if (source !== undefined && source !== null && source.working && source.total) { if (source !== undefined) {
if(typeof source.working !== 'number')
source.working = 0;
if(typeof source.total !== 'number')
source.total = 0;
source.total += 1; source.total += 1;
if (success) { if (success) {
source.working += 1; source.working += 1;
} }
source.percentWorking = source.working / source.total * 100;
source.lastChecked = new Date();
await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
return source; return source;
} else { } else {
logger.warn(`Unknown proxy source: ${sourceId}`); logger.warn(`Unknown proxy source: ${sourceId}`);
@ -84,18 +99,24 @@ async function resetProxyStats(): Promise<void> {
protocol: source.protocol, protocol: source.protocol,
url: source.url, url: source.url,
})); }));
// for (const source of proxyStats) { for (const source of proxyStats) {
// await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL); await cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, PROXY_CONFIG.CACHE_TTL);
// } }
return Promise.resolve(); return Promise.resolve();
} }
// Initialize shared resources // Initialize shared resources
function initializeSharedResources() { async function initializeSharedResources() {
if (!logger) { if (!logger) {
logger = getLogger('proxy-tasks'); logger = getLogger('proxy-tasks');
cache = createCache('hybrid'); cache = createCache('hybrid', {
name: 'proxy-tasks',
keyPrefix: 'proxy:',
ttl: PROXY_CONFIG.CACHE_TTL,
enableMetrics: true
});
await cache.waitForReady();
httpClient = new HttpClient({ timeout: 10000 }, logger); httpClient = new HttpClient({ timeout: 10000 }, logger);
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT); concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
logger.info('Proxy tasks initialized'); logger.info('Proxy tasks initialized');
@ -104,7 +125,7 @@ function initializeSharedResources() {
// Individual task functions // Individual task functions
export async function queueProxyFetch(): Promise<string> { export async function queueProxyFetch(): Promise<string> {
initializeSharedResources(); await initializeSharedResources();
const { queueManager } = await import('../services/queue.service'); const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({ const job = await queueManager.addJob({
@ -122,7 +143,7 @@ export async function queueProxyFetch(): Promise<string> {
} }
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> { export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
initializeSharedResources(); await initializeSharedResources();
const { queueManager } = await import('../services/queue.service'); const { queueManager } = await import('../services/queue.service');
const job = await queueManager.addJob({ const job = await queueManager.addJob({
@ -140,7 +161,7 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
} }
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> { export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
initializeSharedResources(); await initializeSharedResources();
await resetProxyStats(); await resetProxyStats();
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source => const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
@ -154,7 +175,7 @@ export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
} }
export async function fetchProxiesFromSource(source: ProxySource): Promise<ProxyInfo[]> { export async function fetchProxiesFromSource(source: ProxySource): Promise<ProxyInfo[]> {
initializeSharedResources(); await initializeSharedResources();
const allProxies: ProxyInfo[] = []; const allProxies: ProxyInfo[] = [];
@ -208,7 +229,7 @@ export async function fetchProxiesFromSource(source: ProxySource): Promise<Proxy
* Check if a proxy is working * Check if a proxy is working
*/ */
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> { export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
initializeSharedResources(); await initializeSharedResources();
let success = false; let success = false;
logger.debug(`Checking Proxy:`, { logger.debug(`Checking Proxy:`, {
@ -233,16 +254,16 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
responseTime: response.responseTime, responseTime: response.responseTime,
}; };
// if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) { if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
// success = true; success = true;
// await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL); await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
// } else { } else {
// await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
// } }
// if( proxy.source ){ if( proxy.source ){
// updateProxyStats(proxy.source, success); await updateProxyStats(proxy.source, success);
// } }
logger.debug('Proxy check completed', { logger.debug('Proxy check completed', {
host: proxy.host, host: proxy.host,
@ -266,9 +287,9 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
// if (!success) { // if (!success) {
// await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result); // await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result);
// } // }
// if( proxy.source ){ if( proxy.source ){
// updateProxyStats(proxy.source, success); await updateProxyStats(proxy.source, success);
// } }
logger.debug('Proxy check failed', { logger.debug('Proxy check failed', {
host: proxy.host, host: proxy.host,

2
apps/data-service/src/utils/batch-processor.ts
View file

@ -23,7 +23,6 @@ export class BatchProcessor {
private cacheProvider: CacheProvider; private cacheProvider: CacheProvider;
private isReady = false; private isReady = false;
private keyPrefix: string = 'batch:'; // Default key prefix for batch payloads private keyPrefix: string = 'batch:'; // Default key prefix for batch payloads
constructor( constructor(
private queueManager: any, private queueManager: any,
private cacheOptions?: { keyPrefix?: string; ttl?: number } // Optional cache configuration private cacheOptions?: { keyPrefix?: string; ttl?: number } // Optional cache configuration
@ -31,6 +30,7 @@ export class BatchProcessor {
this.keyPrefix = cacheOptions?.keyPrefix || 'batch:'; this.keyPrefix = cacheOptions?.keyPrefix || 'batch:';
// Initialize cache provider with batch-specific settings // Initialize cache provider with batch-specific settings
this.cacheProvider = createCache('redis', { this.cacheProvider = createCache('redis', {
name: 'batch-processor',
keyPrefix: this.keyPrefix, keyPrefix: this.keyPrefix,
ttl: cacheOptions?.ttl || 86400 * 2, // 48 hours default ttl: cacheOptions?.ttl || 86400 * 2, // 48 hours default
enableMetrics: true enableMetrics: true

9
libs/cache/src/providers/redis-cache.ts vendored
View file

@ -22,13 +22,15 @@ export class RedisCache implements CacheProvider {
hitRate: 0, hitRate: 0,
total: 0, total: 0,
uptime: 0 uptime: 0
}; }; constructor(options: CacheOptions = {}) {
constructor(options: CacheOptions = {}) {
this.defaultTTL = options.ttl ?? 3600; // 1 hour default this.defaultTTL = options.ttl ?? 3600; // 1 hour default
this.keyPrefix = options.keyPrefix ?? 'cache:'; this.keyPrefix = options.keyPrefix ?? 'cache:';
this.enableMetrics = options.enableMetrics ?? true; this.enableMetrics = options.enableMetrics ?? true;
// Generate a connection name for monitoring
const baseName = options.name || this.keyPrefix.replace(':', '-').replace(/[^a-zA-Z0-9-]/g, '');
const connectionName = `${baseName}-${Date.now()}`;
const redisConfig = { const redisConfig = {
host: dragonflyConfig.DRAGONFLY_HOST, host: dragonflyConfig.DRAGONFLY_HOST,
port: dragonflyConfig.DRAGONFLY_PORT, port: dragonflyConfig.DRAGONFLY_PORT,
@ -40,6 +42,7 @@ export class RedisCache implements CacheProvider {
connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT, connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT,
commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT, commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT,
keepAlive: dragonflyConfig.DRAGONFLY_ENABLE_KEEPALIVE ? dragonflyConfig.DRAGONFLY_KEEPALIVE_INTERVAL * 1000 : 0, keepAlive: dragonflyConfig.DRAGONFLY_ENABLE_KEEPALIVE ? dragonflyConfig.DRAGONFLY_KEEPALIVE_INTERVAL * 1000 : 0,
connectionName, // Add connection name for monitoring
...(dragonflyConfig.DRAGONFLY_TLS && { ...(dragonflyConfig.DRAGONFLY_TLS && {
tls: { tls: {
cert: dragonflyConfig.DRAGONFLY_TLS_CERT_FILE || undefined, cert: dragonflyConfig.DRAGONFLY_TLS_CERT_FILE || undefined,

1
libs/cache/src/types.ts vendored
View file

@ -26,6 +26,7 @@ export interface CacheOptions {
enableMetrics?: boolean; enableMetrics?: boolean;
maxMemoryItems?: number; maxMemoryItems?: number;
memoryTTL?: number; memoryTTL?: number;
name?: string; // Add name for connection identification
} }
export interface CacheStats { export interface CacheStats {

2
libs/http/src/client.ts
View file

@ -64,7 +64,7 @@ export class HttpClient {
return response; return response;
} catch (error) { } catch (error) {
if( this.logger?.getServiceName() === 'proxy-service' ) { if( this.logger?.getServiceName() === 'proxy-tasks' ) {
this.logger?.debug('HTTP request failed', { this.logger?.debug('HTTP request failed', {
method: finalConfig.method, method: finalConfig.method,
url: finalConfig.url, url: finalConfig.url,