boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

work on intraday

Browse source
This commit is contained in:
Boki 2025-07-01 18:02:24 -04:00
parent 960daf4cad
commit c9a679d9a5
6 changed files with 197 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

64
apps/stock/data-ingestion/src/handlers/qm/actions/filings.action.ts
View file

@ -45,22 +45,40 @@ export async function updateFilings(
try {
// Build API request for filings
const searchParams = new URLSearchParams({
symbol: qmSearchCode,
// symbol: qmSearchCode,
// webmasterId: "500",
// page: page ? page.toString() : "1",
// xbrlSubDoc: "true",
// inclIxbrl: "true",
// inclXbrl: "true",
// resultsPerPage: "25",
webmasterId: "500",
page: "1",
xbrlSubDoc: "true",
inclIxbrl: "true",
inclXbrl: "true",
resultsPerPage: "25",
token: session.headers['Datatool-Token'] || '',
});
delete session?.headers?.["Datatool-Token"]
const formData = new FormData();
formData.append('symbol', qmSearchCode);
formData.append('inclXbrl', 'true');
formData.append('inclIxbrl', 'true');
formData.append('oldestFilingYear', 'true');
formData.append('resultsPerPage', '25');
formData.append('page', page ? page.toString() : '1');
formData.append('xbrlSubDoc', 'true');
// https://app.quotemedia.com/data/getCompanyFilings.json?page=1&webmasterId=500&symbol=AAPL&xbrlSubDoc=true&inclIxbrl=true&inclXbrl=true&resultsPerPage=25
// TODO: Update with correct filings endpoint
const apiUrl = `${QM_CONFIG.FILING_URL}?${searchParams.toString()}`;
console.log('Fetching filings from:', apiUrl, formData, session.headers);
const response = await fetch(apiUrl, {
method: 'GET',
method: 'POST',
headers: session.headers,
proxy: session.proxy,
body: formData,
});
if (!response.ok) {
@ -69,15 +87,28 @@ export async function updateFilings(
const filingsData = await response.json();
if( parseInt(filingsData.results.pagenumber) * filingsData.results.count >= filingsData.results.totalCount) {
await this.scheduleOperation('update-filings', {
symbol: symbol,
exchange: exchange,
qmSearchCode: qmSearchCode,
lastRecordDate: lastRecordDate || null,
page: parseInt(filingsData.results.pagenumber) + 1,
totalPages: (filingsData.results.totalCount / filingsData.results.count) + 1
}, {
priority: 5, // Lower priority than financial data
});
}
// Update session success stats
await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
// Process and store filings data
if (filingsData && filingsData.length > 0) {
if (filingsData?.results?.filings?.filing[0] && filingsData.results.filings.filing[0] > 0) {
// Store filings in a separate collection
await this.mongodb.batchUpsert(
'qmFilings',
filingsData.map((filing: any) => ({
filingsData.results.filings.filing[0].map((filing: any) => ({
...filing,
symbol,
exchange,
@ -92,10 +123,10 @@ export async function updateFilings(
recordCount: filingsData.length
});
this.logger.info('Filings updated successfully', {
symbol,
this.logger.info(`Filings updated successfully ${qmSearchCode} - ${page}/${totalPages}`, {
qmSearchCode,
page,
totalPages,
filingsCount: filingsData.length
});
@ -172,7 +203,7 @@ export async function scheduleFilingsUpdates(
// limit
// });
const staleSymbols = ['X:CA']
const staleSymbols = ['AAPL']
if (staleSymbols.length === 0) {
this.logger.info('No symbols need filings updates');
@ -198,11 +229,6 @@ export async function scheduleFilingsUpdates(
// Schedule individual update jobs for each symbol
for (const doc of symbolDocs) {
try {
if (!doc.symbolId) {
this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`);
continue;
}
await this.scheduleOperation('update-filings', {
symbol: doc.symbol,
exchange: doc.exchange,

109
apps/stock/data-ingestion/src/handlers/qm/actions/intraday-crawl.action.ts
View file

@ -3,14 +3,14 @@
*/
import type { ExecutionContext } from '@stock-bot/handlers';
import type { QMHandler } from '../qm.handler';
import type { CrawlState } from '../../../shared/operation-manager/types';
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import type { QMHandler } from '../qm.handler';
import { getWeekStart, QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
import { QMSessionManager } from '../shared/session-manager';
interface IntradayCrawlInput {
symbol: string;
symbolId: number;
exchange: string;
qmSearchCode: string;
targetOldestDate?: string; // ISO date string for how far back to crawl
batchSize?: number; // Days per batch
@ -29,7 +29,7 @@ export async function processIntradayBatch(
this: QMHandler,
input: {
symbol: string;
symbolId: number;
exchange: string;
qmSearchCode: string;
dateRange: DateRange;
},
@ -40,7 +40,8 @@ export async function processIntradayBatch(
datesProcessed: number;
errors: string[];
}> {
const { symbol, symbolId, qmSearchCode, dateRange } = input;
const { symbol, exchange, qmSearchCode, dateRange } = input;
console.log('Processing intraday batch for:', { symbol, exchange, qmSearchCode, dateRange });
const errors: string[] = [];
let recordsProcessed = 0;
let datesProcessed = 0;
@ -49,7 +50,7 @@ export async function processIntradayBatch(
await sessionManager.initialize(this.cache, this.logger);
// Get a session
const sessionId = QM_SESSION_IDS.LOOKUP; // TODO: Update with correct session ID
const sessionId = QM_SESSION_IDS.PRICES; // TODO: Update with correct session ID
const session = await sessionManager.getSession(sessionId);
if (!session || !session.uuid) {
@ -57,47 +58,66 @@ export async function processIntradayBatch(
}
// Process each date in the range
const currentDate = new Date(dateRange.start);
const currentWeek = getWeekStart(new Date(dateRange.start));
const endDate = new Date(dateRange.end);
while (
(dateRange.direction === 'backward' && currentDate >= endDate) ||
(dateRange.direction === 'forward' && currentDate <= endDate)
(dateRange.direction === 'backward' && currentWeek >= endDate) ||
(dateRange.direction === 'forward' && currentWeek <= endDate)
) {
try {
// Skip weekends
if (currentDate.getDay() === 0 || currentDate.getDay() === 6) {
if (currentWeek.getDay() === 0 || currentWeek.getDay() === 6) {
if (dateRange.direction === 'backward') {
currentDate.setDate(currentDate.getDate() - 1);
currentWeek.setDate(currentWeek.getDate() - 1);
} else {
currentDate.setDate(currentDate.getDate() + 1);
currentWeek.setDate(currentWeek.getDate() + 1);
}
continue;
}
getWeekStart(currentWeek); // Ensure we are at the start of the week
// Build API request
const searchParams = new URLSearchParams({
symbol: symbol,
symbolId: symbolId.toString(),
qmodTool: 'IntradayBars',
webmasterId: '500',
date: currentDate.toISOString().split('T')[0],
interval: '1' // 1-minute bars
adjType:'none',
adjusted:'true',
freq:'day',
interval:'1',
marketSession:'mkt',
pathName:'/demo/portal/company-quotes.php',
qmodTool:'InteractiveChart',
start: currentWeek.toISOString().split('T')[0],
symbol: qmSearchCode,
unadjusted:'false',
webmasterId:'500',
zeroTradeDays:'false',
} as Record<string, string>);
const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/intraday.json?${searchParams.toString()}`;
console.log('Fetching intraday data for:', searchParams.toString());
console.log(test)
const apiUrl = `${QM_CONFIG.PRICES_URL}?${searchParams.toString()}`;
const response = await fetch(apiUrl, {
method: 'GET',
headers: session.headers,
proxy: session.proxy,
});
//https://app.quotemedia.com/datatool/getEnhancedChartData.json?zeroTradeDays=false&start=2025-06-24&interval=1&marketSession=mkt&freq=day&adjusted=true&adjustmentType=none&unadjusted=false&datatype=int&symbol=X:CA
if (!response.ok) {
throw new Error(`API request failed: ${response.status}`);
}
const barsData = await response.json();
const barsResults = await response.json();
console.log('Bars results:', barsResults);
const barsData = barsResults.results.intraday[0].interval || [];
this.logger.info(`Fetched ${barsData.length} bars for ${qmSearchCode} on ${currentWeek.toISOString().split('T')[0]}`, {
qmSearchCode,
date: currentWeek.toISOString().split('T')[0],
records: barsData.length
});
// Update session success stats
await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
@ -106,17 +126,16 @@ export async function processIntradayBatch(
if (barsData && barsData.length > 0) {
const processedBars = barsData.map((bar: any) => ({
...bar,
qmSearchCode,
symbol,
symbolId,
timestamp: new Date(bar.timestamp),
date: new Date(currentDate),
updated_at: new Date()
exchange,
timestamp: new Date(bar.startdatetime),
}));
await this.mongodb.batchUpsert(
'qmIntradayBars',
'qmIntraday',
processedBars,
['symbol', 'timestamp']
['qmSearchCode', 'timestamp']
);
recordsProcessed += barsData.length;
@ -125,7 +144,7 @@ export async function processIntradayBatch(
datesProcessed++;
} catch (error) {
const errorMsg = `Failed to fetch ${symbol} for ${currentDate.toISOString().split('T')[0]}: ${error}`;
const errorMsg = `Failed to fetch ${qmSearchCode} for ${currentWeek.toISOString().split('T')[0]}: ${error}`;
errors.push(errorMsg);
this.logger.error(errorMsg);
@ -137,9 +156,9 @@ export async function processIntradayBatch(
// Move to next date
if (dateRange.direction === 'backward') {
currentDate.setDate(currentDate.getDate() - 1);
currentWeek.setDate(currentWeek.getDate() - 1);
} else {
currentDate.setDate(currentDate.getDate() + 1);
currentWeek.setDate(currentWeek.getDate() + 1);
}
}
@ -161,12 +180,14 @@ export async function crawlIntradayData(
): Promise<{
success: boolean;
symbol: string;
exchange: string;
qmSearchCode: string;
message: string;
data?: any;
}> {
const {
symbol,
symbolId,
exchange,
qmSearchCode,
targetOldestDate = '2020-01-01', // Default to ~5 years of data
batchSize = 7 // Process a week at a time
@ -174,7 +195,7 @@ export async function crawlIntradayData(
this.logger.info('Starting intraday crawl', {
symbol,
symbolId,
exchange,
targetOldestDate,
batchSize
});
@ -254,7 +275,9 @@ export async function crawlIntradayData(
this.logger.info('Intraday crawl already complete', { symbol });
return {
success: true,
qmSearchCode,
symbol,
exchange,
message: 'Intraday crawl already complete'
};
}
@ -274,7 +297,7 @@ export async function crawlIntradayData(
const result = await processIntradayBatch.call(this, {
symbol,
symbolId,
exchange,
qmSearchCode,
dateRange: range
});
@ -329,6 +352,8 @@ export async function crawlIntradayData(
return {
success: allErrors.length === 0,
symbol,
exchange,
qmSearchCode,
message,
data: {
datesProcessed: totalDates,
@ -352,6 +377,8 @@ export async function crawlIntradayData(
return {
success: false,
symbol,
exchange,
qmSearchCode,
message: `Intraday crawl failed: ${error instanceof Error ? error.message : 'Unknown error'}`
};
}
@ -374,8 +401,8 @@ export async function scheduleIntradayCrawls(
errors: number;
}> {
const {
limit = 50,
targetOldestDate = '2020-01-01',
limit = 1,
targetOldestDate = '1960-01-01',
priorityMode = 'all'
} = input;
@ -398,7 +425,7 @@ export async function scheduleIntradayCrawls(
active: { $ne: false }
}, {
limit,
projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 }
projection: { symbol: 1, exchange: 1, qmSearchCode: 1, operations: 1 }
});
break;
@ -436,6 +463,7 @@ export async function scheduleIntradayCrawls(
errors: 0
};
}
symbolsToProcess = [{symbol: 'X:CA'}]
// Get full symbol data if needed
if (priorityMode !== 'never_run') {
@ -443,7 +471,7 @@ export async function scheduleIntradayCrawls(
const fullSymbols = await this.mongodb.find('qmSymbols', {
qmSearchCode: { $in: qmSearchCodes }
}, {
projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 }
projection: { symbol: 1, exchange: 1, qmSearchCode: 1, operations: 1 }
});
// Map back the full data
@ -456,22 +484,17 @@ export async function scheduleIntradayCrawls(
let symbolsQueued = 0;
let errors = 0;
// Schedule crawl jobs
for (const doc of symbolsToProcess) {
try {
if (!doc.symbolId) {
this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`);
continue;
}
await this.scheduleOperation('crawl-intraday-data', {
symbol: doc.symbol,
symbolId: doc.symbolId,
exchange: doc.exchange,
qmSearchCode: doc.qmSearchCode,
targetOldestDate
}, {
priority: priorityMode === 'stale' ? 9 : 5, // Higher priority for updates
delay: symbolsQueued * 2000 // 2 seconds between jobs
});
symbolsQueued++;

2
apps/stock/data-ingestion/src/handlers/qm/actions/session.action.ts
View file

@ -107,7 +107,7 @@ export async function createSession(
// Build request options
const sessionRequest = {
proxy: proxyUrl || undefined,
headers: getQmHeaders(),
headers: getQmHeaders(sessionType),
};
this.logger.debug('Authenticating with QM API', { sessionUrl, sessionRequest });

35
apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts
View file

@ -12,7 +12,6 @@ import {
createSession,
deduplicateSymbols,
scheduleEventsUpdates,
scheduleFilingsUpdates,
scheduleFinancialsUpdates,
scheduleInsidersUpdates,
scheduleIntradayUpdates,
@ -24,7 +23,6 @@ import {
updateEvents,
updateExchangeStats,
updateExchangeStatsAndDeduplicate,
updateFilings,
updateFinancials,
updateGeneralNews,
updateInsiders,
@ -141,20 +139,6 @@ export class QMHandler extends BaseHandler<DataIngestionServices> {
})
scheduleEventsUpdates = scheduleEventsUpdates;
/**
* FILINGS
*/
@Operation('update-filings')
updateFilings = updateFilings;
@Disabled()
@ScheduledOperation('schedule-filings-updates', '0 */8 * * *', {
priority: 5,
immediately: false,
description: 'Check for symbols needing filings updates every 8 hours'
})
scheduleFilingsUpdates = scheduleFilingsUpdates;
/**
* PRICE DATA
*/
@ -189,10 +173,11 @@ export class QMHandler extends BaseHandler<DataIngestionServices> {
})
scheduleIntradayUpdates = scheduleIntradayUpdates;
@ScheduledOperation('schedule-intraday-crawls-batch', '0 */4 * * *', {
// @Disabled()
@ScheduledOperation('schedule-intraday-crawls-batch', '0 */12 * * *', {
priority: 5,
immediately: false,
description: 'Schedule intraday crawls for incomplete symbols every 4 hours'
description: 'Schedule intraday crawls for incomplete symbols every 12 hours'
})
scheduleIntradayCrawlsBatch = async () => {
return scheduleIntradayCrawls.call(this, {
@ -244,4 +229,18 @@ export class QMHandler extends BaseHandler<DataIngestionServices> {
lookbackMinutes: 5 // Only look back 5 minutes to avoid duplicates
});
};
/**
* FILINGS
*/
// @Operation('update-filings')
// updateFilings = updateFilings;
// // @Disabled()
// @ScheduledOperation('schedule-filings-updates', '0 */8 * * *', {
// priority: 5,
// immediately: false,
// description: 'Check for symbols needing filings updates every 8 hours'
// })
// scheduleFilingsUpdates = scheduleFilingsUpdates;
}

68
apps/stock/data-ingestion/src/handlers/qm/shared/config.ts
View file

@ -10,7 +10,7 @@ export const QM_SESSION_IDS = {
SYMBOL: '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6', // getProfiles
PRICES: '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9', // getEnhancedChartData
FINANCIALS: '4e4f1565fb7c9f2a8b4b32b9aa3137af684f3da8a2ce97799d3a7117b14f07be', // getFinancialsEnhancedBySymbol
FILINGS: 'a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd', // getCompanyFilings
// FILINGS: 'a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd', // getCompanyFilings
// INTRADAY: '', //
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9' // getEhnachedChartData
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
@ -50,7 +50,19 @@ export const SESSION_CONFIG = {
API_TIMEOUT: 30000, // 15 seconds
} as const;
export function getQmHeaders(): Record<string, string> {
export function getQmHeaders(type?: string): Record<string, string> {
// if(type?.toUpperCase() === 'FILINGS') {
// return {
// 'User-Agent': getRandomUserAgent(),
// Accept: '*/*',
// 'Accept-Language': 'en-US,en;q=0.5',
// 'Sec-Fetch-Mode': 'cors',
// Origin: 'https://client.quotemedia.com',
// Referer: 'https://client.quotemedia.com/',
// 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
// };
// }
return {
'User-Agent': getRandomUserAgent(),
Accept: '*/*',
@ -60,3 +72,55 @@ export function getQmHeaders(): Record<string, string> {
Referer: 'https://www.quotemedia.com/',
};
}
function parseLocalDate(dateString: string): Date {
const [year, month, day] = dateString.split('-').map(Number);
return new Date(year || 0, (month || 0) - 1, day);
}
// Get start of week (Monday)
export function getWeekStart(dateInput: Date | string): Date {
// Handle string input properly
let date: Date;
if (typeof dateInput === 'string') {
date = parseLocalDate(dateInput);
} else {
// Create new date with local time components
date = new Date(dateInput.getFullYear(), dateInput.getMonth(), dateInput.getDate());
}
const day = date.getDay();
if (day !== 1) {
const diff = date.getDate() - day + (day === 0 ? -6 : 1);
date.setDate(diff);
}
date.setHours(0, 0, 0, 0);
return date;
}
// Get end of week (Sunday)
export function getWeekEnd(dateInput: Date | string): Date {
let date: Date;
// Handle string input properly
if (typeof dateInput === 'string') {
date = parseLocalDate(dateInput);
} else {
// Create new date with local time components
date = new Date(dateInput.getFullYear(), dateInput.getMonth(), dateInput.getDate());
}
const day = date.getDay();
// If not already Sunday, calculate days until Sunday
if (day !== 0) {
const daysToSunday = 7 - day;
date.setDate(date.getDate() + daysToSunday);
}
// Set to end of day
date.setHours(23, 59, 59, 999);
return date;
}

4
apps/stock/data-ingestion/src/handlers/qm/shared/types.ts
View file

@ -5,7 +5,7 @@
export interface QMSession {
uuid: string; // Unique identifier for the session
proxy: string;
headers: HeadersInit;
headers: Record<string, string>; // Headers to use for requests
successfulCalls: number;
failedCalls: number;
lastUsed: Date;
@ -49,7 +49,7 @@ export interface QMAuthResponse {
export interface CachedSession {
uuid: string;
proxy: string;
headers: HeadersInit;
headers: Record<string, string>;
successfulCalls: number;
failedCalls: number;
lastUsed: string; // ISO string for Redis storage