boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

created qm session check

Browse source
This commit is contained in:
Boki 2025-06-26 22:38:53 -04:00
parent e5f505335c
commit b767689470
4 changed files with 235 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

164
apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts
View file

@ -3,9 +3,10 @@
*/
import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
import { BunRequestInit, getRandomUserAgent } from '@stock-bot/utils';
import { QM_CONFIG, QM_SESSION_IDS, SESSION_CONFIG } from '../shared/config';
import { BunRequestInit } from '@stock-bot/utils';
import { getQmHeaders, QM_CONFIG, QM_SESSION_IDS, SESSION_CONFIG } from '../shared/config';
import { QMSessionManager } from '../shared/session-manager';
import { QMSession } from '../shared/types';
/**
* Check existing sessions and queue creation jobs for needed sessions
@ -24,29 +25,22 @@ export async function checkSessions(
const sessionManager = QMSessionManager.getInstance();
// Set cache provider if not already set
if (this.cache) {
sessionManager.setCacheProvider(this.cache);
}
// Initialize with cache provider and logger
sessionManager.initialize(this.cache, this.logger);
// Load sessions from cache if not initialized
if (!sessionManager.getInitialized()) {
await sessionManager.loadFromCache();
sessionManager.setInitialized(true);
}
// Always load fresh data from cache (don't rely on initialization flag)
await sessionManager.loadFromCache();
const cleanedCount = sessionManager.cleanupFailedSessions();
// Sync after cleanup
await sessionManager.syncToCache();
// Cleanup failed sessions (this now handles its own cache sync)
const cleanedCount = await sessionManager.cleanupFailedSessions();
// Check which session IDs need more sessions and queue creation jobs
let queuedCount = 0;
for (const [sessionType, sessionId] of Object.entries(QM_SESSION_IDS)) {
this.logger.debug(`Checking session ID: ${sessionId}`);
if (sessionManager.needsMoreSessions(sessionId)) {
if (await sessionManager.needsMoreSessions(sessionId)) {
const currentCount = sessionManager.getSessions(sessionId).length;
const neededSessions = SESSION_CONFIG.MIN_SESSIONS - currentCount;
const neededSessions = SESSION_CONFIG.MAX_SESSIONS - currentCount;
// Queue up to 10 at a time to avoid overwhelming the system
const toQueue = Math.min(neededSessions, 10);
@ -60,7 +54,7 @@ export async function checkSessions(
this.logger.info(`Queued ${toQueue} jobs to create sessions for ${sessionType}`, {
currentCount,
targetCount: SESSION_CONFIG.MIN_SESSIONS,
targetCount: SESSION_CONFIG.MAX_SESSIONS,
});
}
}
@ -88,7 +82,7 @@ interface CreateSessionInput {
export async function createSession(
this: BaseHandler,
input: CreateSessionInput
): Promise<{ sessionId: string; status: string; sessionType: string }> {
): Promise<{ sessionId: string; status: string; sessionType: string, session?: QMSession }> {
const { sessionId, sessionType = 'LOOKUP' } = input || {};
const sessionManager = QMSessionManager.getInstance();
@ -99,10 +93,8 @@ export async function createSession(
throw new Error(`Invalid session type: ${sessionType}`);
}
// Set cache provider if not already set
if (this.cache) {
sessionManager.setCacheProvider(this.cache);
}
// Initialize with cache provider and logger
sessionManager.initialize(this.cache, this.logger);
try {
// Get proxy from proxy service
@ -112,95 +104,59 @@ export async function createSession(
throw new Error(`No proxy available for session type ${sessionType}`);
}
const userAgent = getRandomUserAgent();
this.logger.debug(`Using User-Agent: ${userAgent}, proxy: ${proxyUrl || 'none'}`);
// Authenticate with QM API inline
const authUrl = `${QM_CONFIG.BASE_URL}${QM_CONFIG.SESSION_PATH}`;
const sessionUrl = `${QM_CONFIG.BASE_URL}${QM_CONFIG.SESSION_PATH}/${sessionId}`;
// Build request options
const requestOptions: BunRequestInit = {
method: 'GET',
const sessionRequest: BunRequestInit = {
proxy: proxyUrl || undefined,
headers: {
'User-Agent': userAgent,
Accept: '*/*',
'Accept-Language': 'en',
'Sec-Fetch-Mode': 'cors',
Origin: 'https://www.quotemedia.com',
Referer: 'https://www.quotemedia.com/',
},
redirect: 'manual', // Don't follow redirects automatically
headers: getQmHeaders(),
};
this.logger.debug('Authenticating with QM API', { authUrl });
this.logger.debug('Authenticating with QM API', { sessionUrl, sessionRequest });
const response = await fetch(authUrl, requestOptions);
const sessionResponse = await fetch(sessionUrl, sessionRequest);
// Extract cookies from response headers
// const cookies: string[] = [];
// const setCookieHeaders = response.headers.getSetCookie();
// Check if authentication was successful
if (sessionResponse.status === 200 || sessionResponse.status === 302) {
this.logger.info('QM authentication successful', {
status: sessionResponse.status,
});
}else{
this.logger.warn('QM authentication failed', {
status: sessionResponse.status,
statusText: sessionResponse.statusText,
});
throw new Error(`QM authentication failed with status ${sessionResponse.status}`);
}
const sessionData = await sessionResponse.json();
// if (setCookieHeaders && setCookieHeaders.length > 0) {
// cookies.push(...setCookieHeaders);
// }
// Add token to headers
sessionRequest.headers['Datatool-Token'] = sessionData.token;
// // Check if authentication was successful
// if (response.status === 200 || response.status === 302) {
// this.logger.info('QM authentication successful', {
// status: response.status,
// cookieCount: cookies.length,
// });
// Create session object with unique ID
const session: QMSession = {
uuid: `${sessionType.toLowerCase()}_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`,
proxy: proxyUrl,
headers: sessionRequest.headers,
successfulCalls: 0,
failedCalls: 0,
lastUsed: new Date(),
createdAt: new Date(),
};
// // Build headers with cookies
// const headers = sessionManager.getQmHeaders();
// if (cookies.length > 0) {
// headers['Cookie'] = buildCookieString(cookies);
// }
// Add session to manager (this now handles cache sync)
await sessionManager.addSession(actualSessionId, session);
// // Create session object
// const session: QMSession = {
// proxy: proxyUrl || '',
// headers,
// successfulCalls: 0,
// failedCalls: 0,
// lastUsed: new Date(),
// };
// // Add session to manager
// sessionManager.addSession(actualSessionId, session);
// // Sync to cache
// await sessionManager.syncToCache();
// this.logger.info(`Successfully created session for ${sessionType}`, {
// sessionId: actualSessionId,
// hasProxy: !!proxyUrl,
// hasCookies: cookies.length > 0,
// });
// return {
// sessionId: actualSessionId,
// status: 'created',
// sessionType,
// };
// } else {
// this.logger.warn('QM authentication failed', {
// status: response.status,
// statusText: response.statusText,
// });
// return {
// sessionId: actualSessionId,
// status: 'failed',
// sessionType,
// };
// }
this.logger.info(`Successfully created session for ${sessionType}`, { session });
return {
sessionId: 'test',//actualSessionId,
sessionId: actualSessionId,
status: 'created',
sessionType,
session,
};
} catch (error) {
this.logger.error(`Failed to create session for ${sessionType}`, { error });
@ -210,18 +166,4 @@ export async function createSession(
sessionType,
};
}
}
/**
* Build cookie string from array of set-cookie headers
*/
function buildCookieString(cookies: string[]): string {
return cookies
.map(cookie => {
// Extract just the name=value part, ignore attributes
const match = cookie.match(/^([^=]+=[^;]+)/);
return match ? match[1] : '';
})
.filter(Boolean)
.join('; ');
}
}