diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/filings.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/filings.action.ts index 15cf20e..3906f6e 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/filings.action.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/filings.action.ts @@ -45,22 +45,40 @@ export async function updateFilings( try { // Build API request for filings const searchParams = new URLSearchParams({ - symbol: qmSearchCode, + // symbol: qmSearchCode, + // webmasterId: "500", + // page: page ? page.toString() : "1", + // xbrlSubDoc: "true", + // inclIxbrl: "true", + // inclXbrl: "true", + // resultsPerPage: "25", webmasterId: "500", - page: "1", - xbrlSubDoc: "true", - inclIxbrl: "true", - inclXbrl: "true", - resultsPerPage: "25", + token: session.headers['Datatool-Token'] || '', }); + + delete session?.headers?.["Datatool-Token"] + + const formData = new FormData(); + formData.append('symbol', qmSearchCode); + formData.append('inclXbrl', 'true'); + formData.append('inclIxbrl', 'true'); + formData.append('oldestFilingYear', 'true'); + formData.append('resultsPerPage', '25'); + formData.append('page', page ? page.toString() : '1'); + formData.append('xbrlSubDoc', 'true'); + + // https://app.quotemedia.com/data/getCompanyFilings.json?page=1&webmasterId=500&symbol=AAPL&xbrlSubDoc=true&inclIxbrl=true&inclXbrl=true&resultsPerPage=25 // TODO: Update with correct filings endpoint const apiUrl = `${QM_CONFIG.FILING_URL}?${searchParams.toString()}`; + console.log('Fetching filings from:', apiUrl, formData, session.headers); + const response = await fetch(apiUrl, { - method: 'GET', + method: 'POST', headers: session.headers, proxy: session.proxy, + body: formData, }); if (!response.ok) { @@ -68,16 +86,29 @@ export async function updateFilings( } const filingsData = await response.json(); + + if( parseInt(filingsData.results.pagenumber) * filingsData.results.count >= filingsData.results.totalCount) { + await this.scheduleOperation('update-filings', { + symbol: symbol, + exchange: exchange, + qmSearchCode: qmSearchCode, + lastRecordDate: lastRecordDate || null, + page: parseInt(filingsData.results.pagenumber) + 1, + totalPages: (filingsData.results.totalCount / filingsData.results.count) + 1 + }, { + priority: 5, // Lower priority than financial data + }); + } // Update session success stats await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid); // Process and store filings data - if (filingsData && filingsData.length > 0) { + if (filingsData?.results?.filings?.filing[0] && filingsData.results.filings.filing[0] > 0) { // Store filings in a separate collection await this.mongodb.batchUpsert( 'qmFilings', - filingsData.map((filing: any) => ({ + filingsData.results.filings.filing[0].map((filing: any) => ({ ...filing, symbol, exchange, @@ -92,10 +123,10 @@ export async function updateFilings( recordCount: filingsData.length }); - - - this.logger.info('Filings updated successfully', { - symbol, + this.logger.info(`Filings updated successfully ${qmSearchCode} - ${page}/${totalPages}`, { + qmSearchCode, + page, + totalPages, filingsCount: filingsData.length }); @@ -172,7 +203,7 @@ export async function scheduleFilingsUpdates( // limit // }); - const staleSymbols = ['X:CA'] + const staleSymbols = ['AAPL'] if (staleSymbols.length === 0) { this.logger.info('No symbols need filings updates'); @@ -198,11 +229,6 @@ export async function scheduleFilingsUpdates( // Schedule individual update jobs for each symbol for (const doc of symbolDocs) { try { - if (!doc.symbolId) { - this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`); - continue; - } - await this.scheduleOperation('update-filings', { symbol: doc.symbol, exchange: doc.exchange, diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/intraday-crawl.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/intraday-crawl.action.ts index 9fc89c8..30ff2af 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/intraday-crawl.action.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/intraday-crawl.action.ts @@ -3,14 +3,14 @@ */ import type { ExecutionContext } from '@stock-bot/handlers'; -import type { QMHandler } from '../qm.handler'; import type { CrawlState } from '../../../shared/operation-manager/types'; -import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config'; +import type { QMHandler } from '../qm.handler'; +import { getWeekStart, QM_CONFIG, QM_SESSION_IDS } from '../shared/config'; import { QMSessionManager } from '../shared/session-manager'; interface IntradayCrawlInput { symbol: string; - symbolId: number; + exchange: string; qmSearchCode: string; targetOldestDate?: string; // ISO date string for how far back to crawl batchSize?: number; // Days per batch @@ -29,7 +29,7 @@ export async function processIntradayBatch( this: QMHandler, input: { symbol: string; - symbolId: number; + exchange: string; qmSearchCode: string; dateRange: DateRange; }, @@ -40,7 +40,8 @@ export async function processIntradayBatch( datesProcessed: number; errors: string[]; }> { - const { symbol, symbolId, qmSearchCode, dateRange } = input; + const { symbol, exchange, qmSearchCode, dateRange } = input; + console.log('Processing intraday batch for:', { symbol, exchange, qmSearchCode, dateRange }); const errors: string[] = []; let recordsProcessed = 0; let datesProcessed = 0; @@ -49,7 +50,7 @@ export async function processIntradayBatch( await sessionManager.initialize(this.cache, this.logger); // Get a session - const sessionId = QM_SESSION_IDS.LOOKUP; // TODO: Update with correct session ID + const sessionId = QM_SESSION_IDS.PRICES; // TODO: Update with correct session ID const session = await sessionManager.getSession(sessionId); if (!session || !session.uuid) { @@ -57,48 +58,67 @@ export async function processIntradayBatch( } // Process each date in the range - const currentDate = new Date(dateRange.start); + const currentWeek = getWeekStart(new Date(dateRange.start)); const endDate = new Date(dateRange.end); while ( - (dateRange.direction === 'backward' && currentDate >= endDate) || - (dateRange.direction === 'forward' && currentDate <= endDate) + (dateRange.direction === 'backward' && currentWeek >= endDate) || + (dateRange.direction === 'forward' && currentWeek <= endDate) ) { try { // Skip weekends - if (currentDate.getDay() === 0 || currentDate.getDay() === 6) { + if (currentWeek.getDay() === 0 || currentWeek.getDay() === 6) { if (dateRange.direction === 'backward') { - currentDate.setDate(currentDate.getDate() - 1); + currentWeek.setDate(currentWeek.getDate() - 1); } else { - currentDate.setDate(currentDate.getDate() + 1); + currentWeek.setDate(currentWeek.getDate() + 1); } continue; } + getWeekStart(currentWeek); // Ensure we are at the start of the week // Build API request const searchParams = new URLSearchParams({ - symbol: symbol, - symbolId: symbolId.toString(), - qmodTool: 'IntradayBars', - webmasterId: '500', - date: currentDate.toISOString().split('T')[0], - interval: '1' // 1-minute bars + adjType:'none', + adjusted:'true', + freq:'day', + interval:'1', + marketSession:'mkt', + pathName:'/demo/portal/company-quotes.php', + qmodTool:'InteractiveChart', + start: currentWeek.toISOString().split('T')[0], + symbol: qmSearchCode, + unadjusted:'false', + webmasterId:'500', + zeroTradeDays:'false', } as Record); - const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/intraday.json?${searchParams.toString()}`; + console.log('Fetching intraday data for:', searchParams.toString()); + console.log(test) + const apiUrl = `${QM_CONFIG.PRICES_URL}?${searchParams.toString()}`; const response = await fetch(apiUrl, { method: 'GET', headers: session.headers, proxy: session.proxy, }); + //https://app.quotemedia.com/datatool/getEnhancedChartData.json?zeroTradeDays=false&start=2025-06-24&interval=1&marketSession=mkt&freq=day&adjusted=true&adjustmentType=none&unadjusted=false&datatype=int&symbol=X:CA if (!response.ok) { throw new Error(`API request failed: ${response.status}`); } - const barsData = await response.json(); + const barsResults = await response.json(); + console.log('Bars results:', barsResults); + + const barsData = barsResults.results.intraday[0].interval || []; + this.logger.info(`Fetched ${barsData.length} bars for ${qmSearchCode} on ${currentWeek.toISOString().split('T')[0]}`, { + qmSearchCode, + date: currentWeek.toISOString().split('T')[0], + records: barsData.length + }); + // Update session success stats await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid); @@ -106,17 +126,16 @@ export async function processIntradayBatch( if (barsData && barsData.length > 0) { const processedBars = barsData.map((bar: any) => ({ ...bar, + qmSearchCode, symbol, - symbolId, - timestamp: new Date(bar.timestamp), - date: new Date(currentDate), - updated_at: new Date() + exchange, + timestamp: new Date(bar.startdatetime), })); await this.mongodb.batchUpsert( - 'qmIntradayBars', + 'qmIntraday', processedBars, - ['symbol', 'timestamp'] + ['qmSearchCode', 'timestamp'] ); recordsProcessed += barsData.length; @@ -125,7 +144,7 @@ export async function processIntradayBatch( datesProcessed++; } catch (error) { - const errorMsg = `Failed to fetch ${symbol} for ${currentDate.toISOString().split('T')[0]}: ${error}`; + const errorMsg = `Failed to fetch ${qmSearchCode} for ${currentWeek.toISOString().split('T')[0]}: ${error}`; errors.push(errorMsg); this.logger.error(errorMsg); @@ -137,9 +156,9 @@ export async function processIntradayBatch( // Move to next date if (dateRange.direction === 'backward') { - currentDate.setDate(currentDate.getDate() - 1); + currentWeek.setDate(currentWeek.getDate() - 1); } else { - currentDate.setDate(currentDate.getDate() + 1); + currentWeek.setDate(currentWeek.getDate() + 1); } } @@ -161,12 +180,14 @@ export async function crawlIntradayData( ): Promise<{ success: boolean; symbol: string; + exchange: string; + qmSearchCode: string; message: string; data?: any; }> { const { symbol, - symbolId, + exchange, qmSearchCode, targetOldestDate = '2020-01-01', // Default to ~5 years of data batchSize = 7 // Process a week at a time @@ -174,7 +195,7 @@ export async function crawlIntradayData( this.logger.info('Starting intraday crawl', { symbol, - symbolId, + exchange, targetOldestDate, batchSize }); @@ -254,7 +275,9 @@ export async function crawlIntradayData( this.logger.info('Intraday crawl already complete', { symbol }); return { success: true, + qmSearchCode, symbol, + exchange, message: 'Intraday crawl already complete' }; } @@ -274,7 +297,7 @@ export async function crawlIntradayData( const result = await processIntradayBatch.call(this, { symbol, - symbolId, + exchange, qmSearchCode, dateRange: range }); @@ -329,6 +352,8 @@ export async function crawlIntradayData( return { success: allErrors.length === 0, symbol, + exchange, + qmSearchCode, message, data: { datesProcessed: totalDates, @@ -352,6 +377,8 @@ export async function crawlIntradayData( return { success: false, symbol, + exchange, + qmSearchCode, message: `Intraday crawl failed: ${error instanceof Error ? error.message : 'Unknown error'}` }; } @@ -374,8 +401,8 @@ export async function scheduleIntradayCrawls( errors: number; }> { const { - limit = 50, - targetOldestDate = '2020-01-01', + limit = 1, + targetOldestDate = '1960-01-01', priorityMode = 'all' } = input; @@ -398,7 +425,7 @@ export async function scheduleIntradayCrawls( active: { $ne: false } }, { limit, - projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 } + projection: { symbol: 1, exchange: 1, qmSearchCode: 1, operations: 1 } }); break; @@ -436,6 +463,7 @@ export async function scheduleIntradayCrawls( errors: 0 }; } + symbolsToProcess = [{symbol: 'X:CA'}] // Get full symbol data if needed if (priorityMode !== 'never_run') { @@ -443,7 +471,7 @@ export async function scheduleIntradayCrawls( const fullSymbols = await this.mongodb.find('qmSymbols', { qmSearchCode: { $in: qmSearchCodes } }, { - projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 } + projection: { symbol: 1, exchange: 1, qmSearchCode: 1, operations: 1 } }); // Map back the full data @@ -456,22 +484,17 @@ export async function scheduleIntradayCrawls( let symbolsQueued = 0; let errors = 0; + // Schedule crawl jobs for (const doc of symbolsToProcess) { try { - if (!doc.symbolId) { - this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`); - continue; - } - await this.scheduleOperation('crawl-intraday-data', { symbol: doc.symbol, - symbolId: doc.symbolId, + exchange: doc.exchange, qmSearchCode: doc.qmSearchCode, targetOldestDate }, { priority: priorityMode === 'stale' ? 9 : 5, // Higher priority for updates - delay: symbolsQueued * 2000 // 2 seconds between jobs }); symbolsQueued++; diff --git a/apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts b/apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts index a67bb9a..f53a1df 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts @@ -107,7 +107,7 @@ export async function createSession( // Build request options const sessionRequest = { proxy: proxyUrl || undefined, - headers: getQmHeaders(), + headers: getQmHeaders(sessionType), }; this.logger.debug('Authenticating with QM API', { sessionUrl, sessionRequest }); diff --git a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts index d00853f..6196490 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts @@ -12,7 +12,6 @@ import { createSession, deduplicateSymbols, scheduleEventsUpdates, - scheduleFilingsUpdates, scheduleFinancialsUpdates, scheduleInsidersUpdates, scheduleIntradayUpdates, @@ -24,7 +23,6 @@ import { updateEvents, updateExchangeStats, updateExchangeStatsAndDeduplicate, - updateFilings, updateFinancials, updateGeneralNews, updateInsiders, @@ -141,20 +139,6 @@ export class QMHandler extends BaseHandler { }) scheduleEventsUpdates = scheduleEventsUpdates; - /** - * FILINGS - */ - @Operation('update-filings') - updateFilings = updateFilings; - - @Disabled() - @ScheduledOperation('schedule-filings-updates', '0 */8 * * *', { - priority: 5, - immediately: false, - description: 'Check for symbols needing filings updates every 8 hours' - }) - scheduleFilingsUpdates = scheduleFilingsUpdates; - /** * PRICE DATA */ @@ -189,10 +173,11 @@ export class QMHandler extends BaseHandler { }) scheduleIntradayUpdates = scheduleIntradayUpdates; - @ScheduledOperation('schedule-intraday-crawls-batch', '0 */4 * * *', { + // @Disabled() + @ScheduledOperation('schedule-intraday-crawls-batch', '0 */12 * * *', { priority: 5, immediately: false, - description: 'Schedule intraday crawls for incomplete symbols every 4 hours' + description: 'Schedule intraday crawls for incomplete symbols every 12 hours' }) scheduleIntradayCrawlsBatch = async () => { return scheduleIntradayCrawls.call(this, { @@ -244,4 +229,18 @@ export class QMHandler extends BaseHandler { lookbackMinutes: 5 // Only look back 5 minutes to avoid duplicates }); }; + + /** + * FILINGS + */ + // @Operation('update-filings') + // updateFilings = updateFilings; + + // // @Disabled() + // @ScheduledOperation('schedule-filings-updates', '0 */8 * * *', { + // priority: 5, + // immediately: false, + // description: 'Check for symbols needing filings updates every 8 hours' + // }) + // scheduleFilingsUpdates = scheduleFilingsUpdates; } \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts b/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts index 9fc78f4..dc4cc8c 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/shared/config.ts @@ -10,7 +10,7 @@ export const QM_SESSION_IDS = { SYMBOL: '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6', // getProfiles PRICES: '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9', // getEnhancedChartData FINANCIALS: '4e4f1565fb7c9f2a8b4b32b9aa3137af684f3da8a2ce97799d3a7117b14f07be', // getFinancialsEnhancedBySymbol - FILINGS: 'a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd', // getCompanyFilings + // FILINGS: 'a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd', // getCompanyFilings // INTRADAY: '', // // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9' // getEhnachedChartData // '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b @@ -50,7 +50,19 @@ export const SESSION_CONFIG = { API_TIMEOUT: 30000, // 15 seconds } as const; -export function getQmHeaders(): Record { +export function getQmHeaders(type?: string): Record { + // if(type?.toUpperCase() === 'FILINGS') { + // return { + // 'User-Agent': getRandomUserAgent(), + // Accept: '*/*', + // 'Accept-Language': 'en-US,en;q=0.5', + // 'Sec-Fetch-Mode': 'cors', + // Origin: 'https://client.quotemedia.com', + // Referer: 'https://client.quotemedia.com/', + // 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + // }; + // } + return { 'User-Agent': getRandomUserAgent(), Accept: '*/*', @@ -60,3 +72,55 @@ export function getQmHeaders(): Record { Referer: 'https://www.quotemedia.com/', }; } + +function parseLocalDate(dateString: string): Date { + const [year, month, day] = dateString.split('-').map(Number); + return new Date(year || 0, (month || 0) - 1, day); +} + +// Get start of week (Monday) +export function getWeekStart(dateInput: Date | string): Date { + // Handle string input properly + let date: Date; + if (typeof dateInput === 'string') { + date = parseLocalDate(dateInput); + } else { + // Create new date with local time components + date = new Date(dateInput.getFullYear(), dateInput.getMonth(), dateInput.getDate()); + } + + const day = date.getDay(); + + if (day !== 1) { + const diff = date.getDate() - day + (day === 0 ? -6 : 1); + date.setDate(diff); + } + + date.setHours(0, 0, 0, 0); + return date; +} + +// Get end of week (Sunday) +export function getWeekEnd(dateInput: Date | string): Date { + let date: Date; + + // Handle string input properly + if (typeof dateInput === 'string') { + date = parseLocalDate(dateInput); + } else { + // Create new date with local time components + date = new Date(dateInput.getFullYear(), dateInput.getMonth(), dateInput.getDate()); + } + + const day = date.getDay(); + + // If not already Sunday, calculate days until Sunday + if (day !== 0) { + const daysToSunday = 7 - day; + date.setDate(date.getDate() + daysToSunday); + } + + // Set to end of day + date.setHours(23, 59, 59, 999); + return date; +} \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/qm/shared/types.ts b/apps/stock/data-ingestion/src/handlers/qm/shared/types.ts index 486c813..0563f16 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/shared/types.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/shared/types.ts @@ -5,7 +5,7 @@ export interface QMSession { uuid: string; // Unique identifier for the session proxy: string; - headers: HeadersInit; + headers: Record; // Headers to use for requests successfulCalls: number; failedCalls: number; lastUsed: Date; @@ -49,7 +49,7 @@ export interface QMAuthResponse { export interface CachedSession { uuid: string; proxy: string; - headers: HeadersInit; + headers: Record; successfulCalls: number; failedCalls: number; lastUsed: string; // ISO string for Redis storage