boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fixed proxy started working on new qm

Browse source
This commit is contained in:
Boki 2025-06-26 21:47:27 -04:00
parent d989c0c814
commit e5f505335c
12 changed files with 375 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

5
apps/stock/data-ingestion/src/handlers/qm/actions/index.ts Normal file
View file

@ -0,0 +1,5 @@
/**
* QM Action Exports
*/
export { checkSessions, createSession } from './session.action';

208
apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts
View file

@ -2,35 +2,74 @@
* QM Session Actions - Session management and creation
*/
import { BaseHandler } from '@stock-bot/core/handlers';
import { QM_SESSION_IDS, SESSION_CONFIG } from '../shared/config';
import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
import { BunRequestInit, getRandomUserAgent } from '@stock-bot/utils';
import { QM_CONFIG, QM_SESSION_IDS, SESSION_CONFIG } from '../shared/config';
import { QMSessionManager } from '../shared/session-manager';
/**
* Check existing sessions and queue creation jobs for needed sessions
* This is the main session management function that handles cleanup, maintenance, and initialization
*/
export async function checkSessions(handler: BaseHandler): Promise<{
export async function checkSessions(
this: BaseHandler,
_input: unknown,
_context: ExecutionContext
): Promise<{
cleaned: number;
queued: number;
message: string;
}> {
this.logger.info('Checking QM sessions');
const sessionManager = QMSessionManager.getInstance();
// Set cache provider if not already set
if (this.cache) {
sessionManager.setCacheProvider(this.cache);
}
// Load sessions from cache if not initialized
if (!sessionManager.getInitialized()) {
await sessionManager.loadFromCache();
sessionManager.setInitialized(true);
}
const cleanedCount = sessionManager.cleanupFailedSessions();
// Sync after cleanup
await sessionManager.syncToCache();
// Check which session IDs need more sessions and queue creation jobs
let queuedCount = 0;
for (const [sessionType, sessionId] of Object.entries(QM_SESSION_IDS)) {
handler.logger.debug(`Checking session ID: ${sessionId}`);
this.logger.debug(`Checking session ID: ${sessionId}`);
if (sessionManager.needsMoreSessions(sessionId)) {
const currentCount = sessionManager.getSessions(sessionId).length;
const neededSessions = SESSION_CONFIG.MAX_SESSIONS - currentCount;
for (let i = 0; i < neededSessions; i++) {
await handler.scheduleOperation('create-session', { sessionId, sessionType });
handler.logger.info(`Queued job to create session for ${sessionType}`);
const neededSessions = SESSION_CONFIG.MIN_SESSIONS - currentCount;
// Queue up to 10 at a time to avoid overwhelming the system
const toQueue = Math.min(neededSessions, 10);
for (let i = 0; i < toQueue; i++) {
await this.scheduleOperation('create-session', { sessionId, sessionType }, {
delay: i * 2000, // Stagger creation by 2 seconds
});
queuedCount++;
}
this.logger.info(`Queued ${toQueue} jobs to create sessions for ${sessionType}`, {
currentCount,
targetCount: SESSION_CONFIG.MIN_SESSIONS,
});
}
}
this.logger.info('QM session check completed', {
cleaned: cleanedCount,
queued: queuedCount,
});
return {
cleaned: cleanedCount,
queued: queuedCount,
@ -46,32 +85,143 @@ interface CreateSessionInput {
sessionType?: string;
}
export async function createSingleSession(
handler: BaseHandler,
export async function createSession(
this: BaseHandler,
input: CreateSessionInput
): Promise<{ sessionId: string; status: string; sessionType: string }> {
const { sessionId: _sessionId, sessionType } = input || {};
const _sessionManager = QMSessionManager.getInstance();
const { sessionId, sessionType = 'LOOKUP' } = input || {};
const sessionManager = QMSessionManager.getInstance();
// Get proxy from proxy service
const _proxyString = handler.proxy.getProxy();
// Get the actual session ID from config
const actualSessionId = sessionId || QM_SESSION_IDS[sessionType as keyof typeof QM_SESSION_IDS];
if (!actualSessionId) {
throw new Error(`Invalid session type: ${sessionType}`);
}
// const session = {
// proxy: proxyString || 'http://proxy:8080',
// headers: sessionManager.getQmHeaders(),
// successfulCalls: 0,
// failedCalls: 0,
// lastUsed: new Date()
// };
// Set cache provider if not already set
if (this.cache) {
sessionManager.setCacheProvider(this.cache);
}
handler.logger.info(`Creating session for ${sessionType}`);
try {
// Get proxy from proxy service
const proxyUrl: string | null = this.proxy ? this.proxy.getProxy() : null;
if (!proxyUrl) {
this.logger.warn(`No proxy available for session type ${sessionType}`);
throw new Error(`No proxy available for session type ${sessionType}`);
}
// Add session to manager
// sessionManager.addSession(sessionType, session);
const userAgent = getRandomUserAgent();
this.logger.debug(`Using User-Agent: ${userAgent}, proxy: ${proxyUrl || 'none'}`);
return {
sessionId: sessionType,
status: 'created',
sessionType,
};
// Authenticate with QM API inline
const authUrl = `${QM_CONFIG.BASE_URL}${QM_CONFIG.SESSION_PATH}`;
// Build request options
const requestOptions: BunRequestInit = {
method: 'GET',
proxy: proxyUrl || undefined,
headers: {
'User-Agent': userAgent,
Accept: '*/*',
'Accept-Language': 'en',
'Sec-Fetch-Mode': 'cors',
Origin: 'https://www.quotemedia.com',
Referer: 'https://www.quotemedia.com/',
},
redirect: 'manual', // Don't follow redirects automatically
};
this.logger.debug('Authenticating with QM API', { authUrl });
const response = await fetch(authUrl, requestOptions);
// Extract cookies from response headers
// const cookies: string[] = [];
// const setCookieHeaders = response.headers.getSetCookie();
// if (setCookieHeaders && setCookieHeaders.length > 0) {
// cookies.push(...setCookieHeaders);
// }
// // Check if authentication was successful
// if (response.status === 200 || response.status === 302) {
// this.logger.info('QM authentication successful', {
// status: response.status,
// cookieCount: cookies.length,
// });
// // Build headers with cookies
// const headers = sessionManager.getQmHeaders();
// if (cookies.length > 0) {
// headers['Cookie'] = buildCookieString(cookies);
// }
// // Create session object
// const session: QMSession = {
// proxy: proxyUrl || '',
// headers,
// successfulCalls: 0,
// failedCalls: 0,
// lastUsed: new Date(),
// };
// // Add session to manager
// sessionManager.addSession(actualSessionId, session);
// // Sync to cache
// await sessionManager.syncToCache();
// this.logger.info(`Successfully created session for ${sessionType}`, {
// sessionId: actualSessionId,
// hasProxy: !!proxyUrl,
// hasCookies: cookies.length > 0,
// });
// return {
// sessionId: actualSessionId,
// status: 'created',
// sessionType,
// };
// } else {
// this.logger.warn('QM authentication failed', {
// status: response.status,
// statusText: response.statusText,
// });
// return {
// sessionId: actualSessionId,
// status: 'failed',
// sessionType,
// };
// }
return {
sessionId: 'test',//actualSessionId,
status: 'created',
sessionType,
};
} catch (error) {
this.logger.error(`Failed to create session for ${sessionType}`, { error });
return {
sessionId: actualSessionId,
status: 'error',
sessionType,
};
}
}
/**
* Build cookie string from array of set-cookie headers
*/
function buildCookieString(cookies: string[]): string {
return cookies
.map(cookie => {
// Extract just the name=value part, ignore attributes
const match = cookie.match(/^([^=]+=[^;]+)/);
return match ? match[1] : '';
})
.filter(Boolean)
.join('; ');
}