From 0b636905007bf136cfa61d212fb820e5c3d661f8 Mon Sep 17 00:00:00 2001 From: Boki Date: Sun, 6 Jul 2025 20:53:21 -0400 Subject: [PATCH] added initial price eod, still need to test --- apps/stock/config/config/default.json | 2 +- .../{fetch-exchanges.ts => exchanges.ts} | 9 +- .../src/handlers/eod/actions/index.ts | 5 +- .../src/handlers/eod/actions/prices.ts | 152 ++++++++++++++++++ .../actions/{fetch-symbols.ts => symbols.ts} | 9 +- .../src/handlers/eod/eod.handler.ts | 27 +++- .../src/handlers/eod/shared/config.ts | 1 + 7 files changed, 198 insertions(+), 7 deletions(-) rename apps/stock/data-ingestion/src/handlers/eod/actions/{fetch-exchanges.ts => exchanges.ts} (81%) create mode 100644 apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts rename apps/stock/data-ingestion/src/handlers/eod/actions/{fetch-symbols.ts => symbols.ts} (90%) diff --git a/apps/stock/config/config/default.json b/apps/stock/config/config/default.json index e8cef5f..86cbb20 100644 --- a/apps/stock/config/config/default.json +++ b/apps/stock/config/config/default.json @@ -150,7 +150,7 @@ "name": "eod", "enabled": false, "priority": 4, - "apiKey": "", + "apiKey": "657fe003583a32.85708911", "baseUrl": "https://eodhistoricaldata.com/api", "tier": "free" } diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/fetch-exchanges.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/exchanges.ts similarity index 81% rename from apps/stock/data-ingestion/src/handlers/eod/actions/fetch-exchanges.ts rename to apps/stock/data-ingestion/src/handlers/eod/actions/exchanges.ts index 157b1de..3c31191 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/fetch-exchanges.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/exchanges.ts @@ -1,5 +1,6 @@ import type { BaseHandler } from '@stock-bot/handlers'; import type { DataIngestionServices } from '../../../types'; +import { EOD_CONFIG } from '../shared'; export async function fetchExchanges( this: BaseHandler @@ -9,9 +10,15 @@ export async function fetchExchanges( try { logger.info('Fetching EOD exchanges list'); + // Get API key from config + const apiKey = EOD_CONFIG.API_TOKEN; + if (!apiKey) { + throw new Error('EOD API key not configured'); + } + // Build URL with query parameters const url = new URL('https://eodhd.com/api/exchanges-list/'); - url.searchParams.append('api_token', '657fe003583a32.85708911'); + url.searchParams.append('api_token', apiKey); url.searchParams.append('fmt', 'json'); // Fetch exchanges from EOD API using Bun fetch diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/index.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/index.ts index f019a42..3aac827 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/index.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/index.ts @@ -1,2 +1,3 @@ -export { fetchExchanges } from './fetch-exchanges'; -export { fetchSymbols, scheduleFetchSymbols } from './fetch-symbols'; +export { fetchExchanges } from './exchanges'; +export { fetchSymbols, scheduleFetchSymbols } from './symbols'; +export { fetchPrices, scheduleFetchPrices } from './prices'; diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts new file mode 100644 index 0000000..f4d43e7 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts @@ -0,0 +1,152 @@ +import type { BaseHandler } from '@stock-bot/handlers'; +import type { DataIngestionServices } from '../../../types'; +import { EOD_CONFIG } from '../shared'; + +interface FetchPricesInput { + symbol: string; + exchange: string; +} + +export async function scheduleFetchPrices( + this: BaseHandler +): Promise<{ success: boolean; jobsScheduled: number }> { + const logger = this.logger; + + try { + logger.info('Scheduling price fetch jobs for Canadian symbols'); + + // Calculate date one week ago + const oneWeekAgo = new Date(); + oneWeekAgo.setDate(oneWeekAgo.getDate() - 7); + + // Get Canadian exchanges (TSX, TSV, CNQ, NEO) + const canadianExchanges = ['TO', 'V', 'CN', 'NEO']; + + // Find symbols that haven't been updated in the last week + const symbols = await this.mongodb.collection('eodSymbols').find({ + Exchange: { $in: canadianExchanges }, + delisted: false, + $or: [ + { lastPriceUpdate: { $lt: oneWeekAgo } }, + { lastPriceUpdate: { $exists: false } } + ] + }).toArray(); + + if (!symbols || symbols.length === 0) { + logger.info('No Canadian symbols need price updates'); + return { success: true, jobsScheduled: 0 }; + } + + logger.info(`Found ${symbols.length} Canadian symbols needing price updates`); + + let jobsScheduled = 0; + + // Schedule jobs with staggered delays + for (let i = 0; i < symbols.length; i++) { + const symbol = symbols[i]; + await this.scheduleOperation('fetch-prices', { + symbol: symbol.Code, + exchange: symbol.Exchange + }, { + attempts: 3, + backoff: { + type: 'exponential', + delay: 5000 + }, + delay: i * 100 // Stagger jobs by 100ms per symbol to avoid rate limit spikes + }); + jobsScheduled++; + } + + logger.info(`Successfully scheduled ${jobsScheduled} price fetch jobs`); + + return { + success: true, + jobsScheduled + }; + } catch (error) { + logger.error('Failed to schedule price fetch jobs', { error }); + throw error; + } +} + +export async function fetchPrices( + this: BaseHandler, + input: FetchPricesInput +): Promise<{ success: boolean; priceCount: number }> { + const logger = this.logger; + const { symbol, exchange } = input; + + try { + logger.info('Fetching prices', { symbol, exchange }); + + // Get API key from config + const apiKey = EOD_CONFIG.API_TOKEN; + if (!apiKey) { + throw new Error('EOD API key not configured'); + } + + // Build URL for EOD price data + const url = new URL(`https://eodhd.com/api/eod/${symbol}.${exchange}`); + url.searchParams.append('api_token', apiKey); + url.searchParams.append('fmt', 'json'); + + // Fetch price data from EOD API + const response = await fetch(url.toString()); + + if (!response.ok) { + throw new Error(`EOD Prices API returned ${response.status}: ${response.statusText}`); + } + + const priceData = await response.json(); + + // EOD returns an array of historical prices + if (!Array.isArray(priceData)) { + throw new Error('Invalid response format from EOD API - expected array'); + } + + logger.info(`Fetched ${priceData.length} price records for ${symbol}.${exchange}`); + + // Add metadata to each price record + const pricesWithMetadata = priceData.map(price => ({ + symbol, + exchange, + symbolExchange: `${symbol}.${exchange}`, + date: price.date, + open: price.open, + high: price.high, + low: price.low, + close: price.close, + adjustedClose: price.adjusted_close, + volume: price.volume, + })); + + // Save to MongoDB - use date and symbol as unique identifier + const result = await this.mongodb.batchUpsert( + 'eodPrices', + pricesWithMetadata, + ['date', 'symbolExchange'] + ); + + // Update the symbol's last price update timestamp + await this.mongodb.collection('eodSymbols').updateOne( + { Code: symbol, Exchange: exchange }, + { + $set: { + lastPriceUpdate: new Date(), + lastPriceDate: priceData.length > 0 ? priceData[priceData.length - 1].date : null + } + } + ); + + logger.info(`Successfully saved ${result.insertedCount} price records for ${symbol}.${exchange}`); + + return { + success: true, + priceCount: result.insertedCount + }; + } catch (error) { + logger.error('Failed to fetch or save prices', { error, symbol, exchange }); + throw error; + } +} \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/fetch-symbols.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/symbols.ts similarity index 90% rename from apps/stock/data-ingestion/src/handlers/eod/actions/fetch-symbols.ts rename to apps/stock/data-ingestion/src/handlers/eod/actions/symbols.ts index 75e05a9..8080c3c 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/fetch-symbols.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/symbols.ts @@ -1,5 +1,6 @@ import type { BaseHandler } from '@stock-bot/handlers'; import type { DataIngestionServices } from '../../../types'; +import { EOD_CONFIG } from '../shared'; interface FetchSymbolsInput { exchangeCode: string; @@ -80,9 +81,15 @@ export async function fetchSymbols( try { logger.info('Fetching symbols for exchange', { exchangeCode, delisted }); + // Get API key from config + const apiKey = EOD_CONFIG.API_TOKEN; + if (!apiKey) { + throw new Error('EOD API key not configured'); + } + // Build URL with query parameters const url = new URL(`https://eodhd.com/api/exchange-symbol-list/${exchangeCode}`); - url.searchParams.append('api_token', '657fe003583a32.85708911'); + url.searchParams.append('api_token', apiKey); url.searchParams.append('fmt', 'json'); if (delisted) { diff --git a/apps/stock/data-ingestion/src/handlers/eod/eod.handler.ts b/apps/stock/data-ingestion/src/handlers/eod/eod.handler.ts index 9b04e1b..65cad75 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/eod.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/eod.handler.ts @@ -6,7 +6,13 @@ import { ScheduledOperation } from '@stock-bot/handlers'; import type { DataIngestionServices } from '../../types'; -import { fetchExchanges, fetchSymbols, scheduleFetchSymbols } from './actions'; +import { + fetchExchanges, + fetchSymbols, + scheduleFetchSymbols, + fetchPrices, + scheduleFetchPrices +} from './actions'; /** * EOD (End of Day) Handler demonstrating advanced rate limiting @@ -49,6 +55,23 @@ export class EodHandler extends BaseHandler { * Called by schedule-fetch-symbols for each exchange */ @Operation('fetch-symbols') - @RateLimit(1) // 10 points per exchange + @RateLimit(10) // 10 points per exchange fetchSymbols = fetchSymbols; + + /** + * Schedule price fetching for Canadian symbols not updated in last week + * Runs daily at 2 AM + */ + @Operation('schedule-fetch-prices') + @ScheduledOperation('schedule-fetch-prices', '0 2 * * *') + @RateLimit(1) // 1 point for scheduling + scheduleFetchPrices = scheduleFetchPrices; + + /** + * Fetch historical prices for a specific symbol + * Called by schedule-fetch-prices for each symbol + */ + @Operation('fetch-prices') + @RateLimit(10) // 10 points per price fetch + fetchPrices = fetchPrices; } \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/eod/shared/config.ts b/apps/stock/data-ingestion/src/handlers/eod/shared/config.ts index 4355f15..27fac75 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/shared/config.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/shared/config.ts @@ -1,4 +1,5 @@ export const EOD_CONFIG = { // API configuration API_BASE_URL: 'https://eodhd.com/api/', + API_TOKEN: '657fe003583a32.85708911"', }; \ No newline at end of file