From a34ff3ed1b39e527169e88b75e5bb7038cc10c1b Mon Sep 17 00:00:00 2001 From: Boki Date: Sun, 29 Jun 2025 20:40:37 -0400 Subject: [PATCH] work on financials --- .../handlers/qm/actions/financials.action.ts | 49 ++++++++++--------- .../src/handlers/qm/shared/config.ts | 7 ++- .../handlers/qm/shared/operation-registry.ts | 10 +++- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/financials.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/financials.action.ts index d276eda..a9b8f1c 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/financials.action.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/financials.action.ts @@ -29,8 +29,9 @@ export async function updateFinancials( this: QMHandler, input: { symbol: string; - symbolId: number; + exchange: number; qmSearchCode: string; + reportType: 'Q' | 'A'; // Quarterly or Annual }, _context?: ExecutionContext ): Promise<{ @@ -39,15 +40,15 @@ export async function updateFinancials( message: string; data?: any; }> { - const { symbol, symbolId, qmSearchCode } = input; + const { symbol, exchange, qmSearchCode, reportType } = input; - this.logger.info('Fetching financials', { symbol, symbolId }); + this.logger.info('Fetching financials', { symbol, exchange, qmSearchCode }); const sessionManager = QMSessionManager.getInstance(); await sessionManager.initialize(this.cache, this.logger); // Get a session - you'll need to add the appropriate session ID for financials - const sessionId = QM_SESSION_IDS.LOOKUP; // TODO: Update with correct session ID + const sessionId = QM_SESSION_IDS.FINANCIALS; // TODO: Update with correct session ID const session = await sessionManager.getSession(sessionId); if (!session || !session.uuid) { @@ -57,14 +58,18 @@ export async function updateFinancials( try { // Build API request for financials const searchParams = new URLSearchParams({ - symbol: symbol, - symbolId: symbolId.toString(), + currency: 'true', + lang: 'en', + latestfiscaldate: 'true', + numberOfReports: '300', + pathName: '/demo/portal/company-research.php', qmodTool: 'Financials', - webmasterId: '500' + reportType: reportType, + symbol: 'AAPL', + webmasterId: '500', }); - // TODO: Update with correct financials endpoint - const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/financials.json?${searchParams.toString()}`; + const apiUrl = `${QM_CONFIG.FINANCIALS_URL}?${searchParams.toString()}`; const response = await fetch(apiUrl, { method: 'GET', @@ -89,7 +94,8 @@ export async function updateFinancials( financialData.map((statement: any) => ({ ...statement, symbol, - symbolId, + exchange, + qmSearchCode, updated_at: new Date() })), ['symbol', 'period', 'statementType'] // Unique keys @@ -170,12 +176,16 @@ export async function scheduleFinancialsUpdates( try { // Get symbols that need updating - const staleSymbols = await tracker.getStaleSymbols('financials_update', { + const staleSymbolsQ = await tracker.getStaleSymbols('financials_update_quarterly', { + minHoursSinceRun: forceUpdate ? 0 : 24 * 7, // Weekly by default + limit + }); + const staleSymbolsA = await tracker.getStaleSymbols('financials_update_annual', { minHoursSinceRun: forceUpdate ? 0 : 24 * 7, // Weekly by default limit }); - if (staleSymbols.length === 0) { + if (staleSymbolsQ.length === 0 && staleSymbolsA.length === 0) { this.logger.info('No symbols need financials updates'); return { message: 'No symbols need financials updates', @@ -190,7 +200,7 @@ export async function scheduleFinancialsUpdates( const symbolDocs = await this.mongodb.find('qmSymbols', { qmSearchCode: { $in: staleSymbols } }, { - projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 } + projection: { symbol: 1, exchange: 1, qmSearchCode: 1 } }); let queued = 0; @@ -199,23 +209,18 @@ export async function scheduleFinancialsUpdates( // Schedule individual update jobs for each symbol for (const doc of symbolDocs) { try { - if (!doc.symbolId) { - this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`); - continue; - } - await this.scheduleOperation('update-financials', { symbol: doc.symbol, - symbolId: doc.symbolId, - qmSearchCode: doc.qmSearchCode + exchange: doc.exchange, + qmSearchCode: doc.qmSearchCode, }, { priority: 4, - delay: queued * 2000 // 2 seconds between jobs + delay: queued * 1000 // 2 seconds between jobs }); queued++; } catch (error) { - this.logger.error(`Failed to schedule financials update for ${doc.symbol}`, { error }); + this.logger.error(`Failed to schedule financials update for ${doc.qmSearchCode}`, { error }); errors++; } } diff --git a/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts b/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts index 73c78ca..4733e6a 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts @@ -9,10 +9,8 @@ export const QM_SESSION_IDS = { LOOKUP: 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6', // lookup endpoint SYMBOL: '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6', // getProfiles PRICES: '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9', // getEnhancedChartData - // EDS: '', // + FINANCIALS: '4e4f1565fb7c9f2a8b4b32b9aa3137af684f3da8a2ce97799d3a7117b14f07be', // getFinancialsEnhancedBySymbol // FILINGS: '', // - // PRICES: '', // - // FINANCIALS: '', // // INTRADAY: '', // // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9' // getEhnachedChartData // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b @@ -38,7 +36,8 @@ export const QM_CONFIG = { LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json', SYMBOL_URL: 'https://app.quotemedia.com/datatool/getProfiles.json', PRICES_URL: 'https://app.quotemedia.com/datatool/getEnhancedChartData.json', - EVENTS_URL: 'https://app.quotemedia.com/datatool/getIndicatorsBySymbol.json' + EVENTS_URL: 'https://app.quotemedia.com/datatool/getIndicatorsBySymbol.json', + FINANCIALS_URL: 'https://app.quotemedia.com/datatool/getFinancialsEnhancedBySymbol.json', } as const; // Session management settings diff --git a/apps/stock/data-ingestion/src/handlers/qm/shared/operation-registry.ts b/apps/stock/data-ingestion/src/handlers/qm/shared/operation-registry.ts index 5326092..9d812ba 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/shared/operation-registry.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/shared/operation-registry.ts @@ -31,9 +31,15 @@ export const QM_OPERATIONS: QMOperationConfig[] = [ // Fundamental data operations { - name: 'financials_update', + name: 'financials_update_quarterly', type: 'standard', - description: 'Update financial statements', + description: 'Update quarterly financial statements', + defaultStaleHours: 24 * 7 // Weekly + }, + { + name: 'financials_update_annual', + type: 'standard', + description: 'Update annual financial statements', defaultStaleHours: 24 * 7 // Weekly }, // Corporate actions - fetched together in one API call