integrated data-ingestion

This commit is contained in:
Boki 2025-06-21 19:42:20 -04:00
parent 9673ae70ef
commit 3227388d25
15 changed files with 226 additions and 133 deletions

View file

@ -4,15 +4,15 @@
import { OperationContext } from '@stock-bot/di';
import { isShutdownSignalReceived } from '@stock-bot/shutdown';
import { getRandomProxy } from '@stock-bot/di';
import type { ServiceContainer } from '@stock-bot/connection-factory';
import { getRandomProxy } from '@stock-bot/utils';
import type { ServiceContainer } from '@stock-bot/di';
import { QMSessionManager } from '../shared/session-manager';
import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG, getQmHeaders } from '../shared/config';
import type { QMSession } from '../shared/types';
export async function createSessions(container: ServiceContainer): Promise<void> {
const ctx = OperationContext.create('qm', 'session', { container });
const ctx = new OperationContext('qm-handler', 'create-sessions', container);
try {
ctx.logger.info('Creating QM sessions...');
@ -33,7 +33,8 @@ export async function createSessions(container: ServiceContainer): Promise<void>
// Cache session creation stats
const initialStats = sessionManager.getStats();
await ctx.cache.set('pre-creation-stats', initialStats, { ttl: 300 });
const cache = ctx.resolve<any>('cache');
await cache.set('pre-creation-stats', initialStats, { ttl: 300 });
// Create sessions for each session ID that needs them
for (const [sessionKey, sessionId] of Object.entries(QM_SESSION_IDS)) {
@ -56,9 +57,9 @@ export async function createSessions(container: ServiceContainer): Promise<void>
const finalStats = sessionManager.getStats();
const totalSessions = sessionManager.getSessionCount();
await ctx.cache.set('post-creation-stats', finalStats, { ttl: 3600 });
await ctx.cache.set('session-count', totalSessions, { ttl: 900 });
await ctx.cache.set('last-session-creation', new Date().toISOString());
await cache.set('post-creation-stats', finalStats, { ttl: 3600 });
await cache.set('session-count', totalSessions, { ttl: 900 });
await cache.set('last-session-creation', new Date().toISOString());
ctx.logger.info('QM session creation completed', {
totalSessions,
@ -68,8 +69,6 @@ export async function createSessions(container: ServiceContainer): Promise<void>
} catch (error) {
ctx.logger.error('Failed to create QM sessions', { error });
throw error;
} finally {
await ctx.dispose();
}
}
@ -134,7 +133,8 @@ async function createSingleSession(
sessionManager.addSession(sessionId, newSession);
// Cache successful session creation
await ctx.cache.set(
const cacheService = ctx.resolve<any>('cache');
await cacheService.set(
`successful-session:${sessionKey}:${Date.now()}`,
{ sessionId, proxy, tokenExists: !!sessionData.token },
{ ttl: 300 }
@ -156,7 +156,8 @@ async function createSingleSession(
}
// Cache failed session attempt for debugging
await ctx.cache.set(
const cacheService = ctx.resolve<any>('cache');
await cacheService.set(
`failed-session:${sessionKey}:${Date.now()}`,
{ sessionId, proxy, error: error.message },
{ ttl: 300 }
@ -165,25 +166,34 @@ async function createSingleSession(
}
export async function initializeQMResources(container?: ServiceContainer): Promise<void> {
const ctx = OperationContext.create('qm', 'init', container ? { container } : undefined);
// Check if already initialized
const alreadyInitialized = await ctx.cache.get('initialized');
if (alreadyInitialized) {
ctx.logger.debug('QM resources already initialized');
return;
if (!container) {
throw new Error('Service container is required for QM resource initialization');
}
const ctx = new OperationContext('qm-handler', 'initialize-resources', container);
try {
const cache = ctx.resolve<any>('cache');
// Check if already initialized
const alreadyInitialized = await cache.get('initialized');
if (alreadyInitialized) {
ctx.logger.debug('QM resources already initialized');
return;
}
ctx.logger.debug('Initializing QM resources...');
// Mark as initialized in cache and session manager
await ctx.cache.set('initialized', true, { ttl: 3600 });
await ctx.cache.set('initialization-time', new Date().toISOString());
const sessionManager = QMSessionManager.getInstance();
sessionManager.setInitialized(true);
ctx.logger.info('QM resources initialized successfully');
await ctx.dispose();
ctx.logger.debug('Initializing QM resources...');
// Mark as initialized in cache and session manager
await cache.set('initialized', true, { ttl: 3600 });
await cache.set('initialization-time', new Date().toISOString());
const sessionManager = QMSessionManager.getInstance();
sessionManager.setInitialized(true);
ctx.logger.info('QM resources initialized successfully');
} catch (error) {
ctx.logger.error('Failed to initialize QM resources', { error });
throw error;
}
}

View file

@ -1,5 +1,4 @@
import {
BaseHandler,
ScheduledHandler,
Handler,
Operation,