From 77bba3145628ab22ef07a4619a020522077c5d17 Mon Sep 17 00:00:00 2001 From: Boki Date: Sun, 29 Jun 2025 10:41:12 -0400 Subject: [PATCH] finished symbol info --- .../handlers/qm/actions/intraday.action.ts | 32 +++++-- .../handlers/qm/actions/symbol-info.action.ts | 88 ++++++++++--------- .../src/handlers/qm/qm.handler.ts | 2 +- .../src/handlers/qm/shared/config.ts | 5 +- 4 files changed, 73 insertions(+), 54 deletions(-) diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/intraday.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/intraday.action.ts index 0f8dc58..75900d3 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/intraday.action.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/intraday.action.ts @@ -106,6 +106,14 @@ export async function updateIntradayBars( ['symbol', 'timestamp'] // Unique keys ); + // Update operation tracking + const tracker = await getOperationTracker(this); + await tracker.updateSymbolOperation(qmSearchCode, 'intraday_bars', { + status: 'success', + lastRecordDate: targetDate, + recordCount: barsData.length + }); + this.logger.info('Intraday bars updated successfully', { symbol, date: targetDate, @@ -124,6 +132,15 @@ export async function updateIntradayBars( } else { // No data for this date (weekend, holiday, or no trading) this.logger.info('No intraday data for date', { symbol, date: targetDate }); + + // Still update operation tracking as successful (no data is a valid result) + const tracker = await getOperationTracker(this); + await tracker.updateSymbolOperation(qmSearchCode, 'intraday_bars', { + status: 'success', + lastRecordDate: targetDate, + recordCount: 0 + }); + return { success: true, symbol, @@ -146,6 +163,12 @@ export async function updateIntradayBars( error: error instanceof Error ? error.message : 'Unknown error' }); + // Update operation tracking for failure + const tracker = await getOperationTracker(this); + await tracker.updateSymbolOperation(qmSearchCode, 'intraday_bars', { + status: 'failure' + }); + return { success: false, symbol, @@ -256,14 +279,7 @@ export async function scheduleIntradayUpdates( jobsQueued++; } - // Update crawl state - await tracker.updateSymbolOperation(doc.qmSearchCode, 'intraday_bars', { - status: 'partial', - crawlState: { - finished: false, - oldestDateReached: new Date(startDate.getTime() - daysToFetch * 24 * 60 * 60 * 1000), - } - }); + // Note: Crawl state will be updated when the actual jobs run } else { // For update mode, just fetch today's data await this.scheduleOperation('update-intraday-bars', { diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/symbol-info.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/symbol-info.action.ts index d727005..ccf18dd 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/symbol-info.action.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/symbol-info.action.ts @@ -29,25 +29,24 @@ async function getOperationTracker(handler: QMHandler): Promise { - const { symbol, qmSearchCode } = input; + const { qmSearchCode } = input; - this.logger.info('Fetching symbol info', { symbol, qmSearchCode }); + this.logger.info(`Fetching symbol info ${qmSearchCode}`, { qmSearchCode }); const sessionManager = QMSessionManager.getInstance(); sessionManager.initialize(this.cache, this.logger); // Get a session - const sessionId = QM_SESSION_IDS.LOOKUP; + const sessionId = QM_SESSION_IDS.SYMBOL; const session = await sessionManager.getSession(sessionId); if (!session || !session.uuid) { @@ -57,13 +56,15 @@ export async function updateSymbolInfo( try { // Build API request for symbol info const searchParams = new URLSearchParams({ - q: qmSearchCode || symbol, - qmodTool: 'SymbolInfo', + fullDescription: 'true', + qmodTool: 'CompanyProfile', + symbols: qmSearchCode, + lang: 'en', + pathName: '/demo/portal/company-summary.php', webmasterId: '500', - includeExtended: 'true' - }); + } as Record); - const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`; + const apiUrl = `${QM_CONFIG.SYMBOL_URL}?${searchParams.toString()}`; const response = await fetch(apiUrl, { method: 'GET', @@ -81,43 +82,44 @@ export async function updateSymbolInfo( await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid); // Process and store symbol info - if (symbolData && (Array.isArray(symbolData) ? symbolData.length > 0 : true)) { - const symbolInfo = Array.isArray(symbolData) ? symbolData[0] : symbolData; + if (symbolData && Array.isArray(symbolData?.results?.company) && symbolData?.results?.company.length > 0) { // Update symbol in database with new metadata const updateData = { - ...symbolInfo, - symbol: symbol, - qmSearchCode: qmSearchCode || symbolInfo.symbol, - lastInfoUpdate: new Date(), - updated_at: new Date() - }; + qmSearchCode: qmSearchCode, + profile: symbolData?.results?.company[0].profile, + symbolInfo: symbolData?.results?.company[0].symbolinfo, + } await this.mongodb.updateOne( 'qmSymbols', - { symbol }, + { qmSearchCode, }, { $set: updateData }, { upsert: true } ); - this.logger.info('Symbol info updated successfully', { - symbol, - name: symbolInfo.name, - exchange: symbolInfo.exchange + // Update operation tracking + const tracker = await getOperationTracker(this); + await tracker.updateSymbolOperation(qmSearchCode, 'symbol_info', { + status: 'success', + lastRecordDate: new Date() + }); + + this.logger.info(`Symbol info updated successfully ${qmSearchCode}`, { + qmSearchCode, }); return { success: true, - symbol, - message: `Symbol info updated for ${symbol}`, - data: symbolInfo + qmSearchCode, + message: `Symbol info updated for ${qmSearchCode}`, }; } else { - this.logger.warn('No symbol data returned from API', { symbol }); + this.logger.warn('No symbol data returned from API', { qmSearchCode }); return { success: false, - symbol, - message: `No data found for symbol ${symbol}` + qmSearchCode, + message: `No data found for symbol ${qmSearchCode}` }; } @@ -128,13 +130,19 @@ export async function updateSymbolInfo( } this.logger.error('Error fetching symbol info', { - symbol, + qmSearchCode, error: error instanceof Error ? error.message : 'Unknown error' }); + // Update operation tracking for failure + const tracker = await getOperationTracker(this); + await tracker.updateSymbolOperation(qmSearchCode, 'symbol_info', { + status: 'failure' + }); + return { success: false, - symbol, + qmSearchCode, message: `Failed to fetch symbol info: ${error instanceof Error ? error.message : 'Unknown error'}` }; } @@ -156,7 +164,7 @@ export async function scheduleSymbolInfoUpdates( symbolsQueued: number; errors: number; }> { - const { limit = 100, forceUpdate = false } = input; + const { limit = 1, forceUpdate = false } = input; const tracker = await getOperationTracker(this); this.logger.info('Scheduling symbol info updates', { limit, forceUpdate }); @@ -181,9 +189,9 @@ export async function scheduleSymbolInfoUpdates( // Get full symbol data to include qmSearchCode const symbolDocs = await this.mongodb.find('qmSymbols', { - symbol: { $in: staleSymbols } + qmSearchCode: { $in: staleSymbols } }, { - projection: { symbol: 1, qmSearchCode: 1 } + projection: { qmSearchCode: 1 } }); let queued = 0; @@ -193,22 +201,16 @@ export async function scheduleSymbolInfoUpdates( for (const doc of symbolDocs) { try { await this.scheduleOperation('update-symbol-info', { - symbol: doc.symbol, - qmSearchCode: doc.qmSearchCode || doc.symbol + qmSearchCode: doc.qmSearchCode }, { // priority: 3, // Add some delay to avoid overwhelming the API // delay: queued * 1000 // 1 second between jobs }); - - // Track that we've scheduled this symbol - await tracker.updateSymbolOperation(doc.qmSearchCode, 'symbol_info', { - status: 'success' - }); - + queued++; } catch (error) { - this.logger.error(`Failed to schedule update for ${doc.symbol}`, { error }); + this.logger.error(`Failed to schedule update for ${doc.qmSearchCode}`, { error }); errors++; } } diff --git a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts index a84fc4c..7fd83c6 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts @@ -69,7 +69,7 @@ export class QMHandler extends BaseHandler { @Operation('update-symbol-info') updateSymbolInfo = updateSymbolInfo; - @Disabled() + // @Disabled() @ScheduledOperation('schedule-symbol-info-updates', '0 */6 * * *', { priority: 7, immediately: false, diff --git a/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts b/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts index d2c872f..e9b11f8 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts @@ -34,12 +34,13 @@ export const QM_CONFIG = { BASE_URL: 'https://app.quotemedia.com', SESSION_PATH: '/auth/g/authenticate/dataTool/v0/500', LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json', + SYMBOL_URL: 'https://app.quotemedia.com/datatool/getProfiles.json', } as const; // Session management settings export const SESSION_CONFIG = { - MIN_SESSIONS: 10, - MAX_SESSIONS: 10, + MIN_SESSIONS: 2, + MAX_SESSIONS: 2, MAX_FAILED_CALLS: 3, SESSION_TIMEOUT: 5000, // 10 seconds API_TIMEOUT: 30000, // 15 seconds