From f69181a8bca99d2ebc552ae13f8ee8209fa27124 Mon Sep 17 00:00:00 2001 From: Boki Date: Mon, 7 Jul 2025 00:10:51 -0400 Subject: [PATCH] fixed up exchanges US --- .../handlers/eod/actions/corporate-actions.ts | 28 +++++++++++-- .../src/handlers/eod/actions/fundamentals.ts | 40 ++++++++++++++----- .../src/handlers/eod/actions/intraday.ts | 34 +++++++++++++--- .../src/handlers/eod/actions/prices.ts | 25 ++++++++++-- .../src/handlers/eod/shared/index.ts | 1 + .../src/handlers/eod/shared/utils.ts | 15 +++++++ 6 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 apps/stock/data-ingestion/src/handlers/eod/shared/utils.ts diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/corporate-actions.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/corporate-actions.ts index 0390c27..1938d61 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/corporate-actions.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/corporate-actions.ts @@ -6,6 +6,7 @@ interface FetchCorporateActionsInput { symbol: string; exchange: string; actionType: 'dividends' | 'splits'; + country?: string; } export async function scheduleFetchCorporateActions( @@ -58,7 +59,8 @@ export async function scheduleFetchCorporateActions( await this.scheduleOperation('fetch-corporate-actions', { symbol: symbol.Code, exchange: symbol.Exchange, - actionType: 'dividends' + actionType: 'dividends', + country: symbol.Country }, { attempts: 3, backoff: { @@ -73,7 +75,8 @@ export async function scheduleFetchCorporateActions( await this.scheduleOperation('fetch-corporate-actions', { symbol: symbol.Code, exchange: symbol.Exchange, - actionType: 'splits' + actionType: 'splits', + country: symbol.Country }, { attempts: 3, backoff: { @@ -102,7 +105,7 @@ export async function fetchCorporateActions( input: FetchCorporateActionsInput ): Promise<{ success: boolean; recordsCount: number }> { const logger = this.logger; - const { symbol, exchange, actionType } = input; + const { symbol, exchange, actionType, country } = input; try { logger.info(`Fetching ${actionType} for ${symbol}.${exchange}`); @@ -113,9 +116,26 @@ export async function fetchCorporateActions( throw new Error('EOD API key not configured'); } + // Get country if not provided + let symbolCountry = country; + if (!symbolCountry) { + const symbolDoc = await this.mongodb.collection('eodSymbols').findOne({ + Code: symbol, + Exchange: exchange + }); + + if (!symbolDoc) { + throw new Error(`Symbol ${symbol}.${exchange} not found in database`); + } + symbolCountry = symbolDoc.Country; + } + // Build URL based on action type + // For US symbols (Country: "USA"), use :US suffix instead of specific exchange code + const exchangeSuffix = symbolCountry === 'USA' ? 'US' : exchange; + const endpoint = actionType === 'dividends' ? 'div' : 'splits'; - const url = new URL(`https://eodhd.com/api/${endpoint}/${symbol}.${exchange}`); + const url = new URL(`https://eodhd.com/api/${endpoint}/${symbol}.${exchangeSuffix}`); url.searchParams.append('api_token', apiKey); url.searchParams.append('fmt', 'json'); diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/fundamentals.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/fundamentals.ts index e7c9c41..66eb33d 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/fundamentals.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/fundamentals.ts @@ -3,12 +3,13 @@ import type { DataIngestionServices } from '../../../types'; import { EOD_CONFIG } from '../shared'; interface BulkFundamentalsInput { - symbols: Array<{ symbol: string; exchange: string }>; + symbols: Array<{ symbol: string; exchange: string; country?: string }>; } interface FetchSingleFundamentalsInput { symbol: string; exchange: string; + country?: string; } export async function scheduleFetchFundamentals( @@ -68,7 +69,8 @@ export async function scheduleFetchFundamentals( const etf = etfs[i]; await this.scheduleOperation('fetch-single-fundamentals', { symbol: etf.Code, - exchange: etf.Exchange + exchange: etf.Exchange, + country: etf.Country }, { attempts: 3, backoff: { @@ -90,10 +92,11 @@ export async function scheduleFetchFundamentals( for (let i = 0; i < nonEtfs.length; i += batchSize) { const batch = nonEtfs.slice(i, i + batchSize); - // Convert to array of {symbol, exchange} objects + // Convert to array of {symbol, exchange, country} objects const symbolBatch = batch.map(s => ({ symbol: s.Code, - exchange: s.Exchange + exchange: s.Exchange, + country: s.Country })); await this.scheduleOperation('fetch-bulk-fundamentals', { @@ -139,12 +142,14 @@ export async function fetchBulkFundamentals( throw new Error('EOD API key not configured'); } - // Group symbols by exchange for the API call - const exchangeGroups = symbols.reduce((acc, { symbol, exchange }) => { + // Group symbols by actual exchange for API endpoint, but use country for symbol suffix + const exchangeGroups = symbols.reduce((acc, { symbol, exchange, country }) => { if (!acc[exchange]) { acc[exchange] = []; } - acc[exchange].push(`${symbol}.${exchange}`); + // For US symbols (Country: "USA"), use :US suffix + const exchangeSuffix = country === 'USA' ? 'US' : exchange; + acc[exchange].push(`${symbol}.${exchangeSuffix}`); return acc; }, {} as Record); @@ -250,11 +255,25 @@ export async function fetchSingleFundamentals( input: FetchSingleFundamentalsInput ): Promise<{ success: boolean; saved: boolean }> { const logger = this.logger; - const { symbol, exchange } = input; + const { symbol, exchange, country } = input; try { logger.info(`Fetching single fundamentals for ${symbol}.${exchange}`); + // Get country if not provided + let symbolCountry = country; + if (!symbolCountry) { + const symbolDoc = await this.mongodb.collection('eodSymbols').findOne({ + Code: symbol, + Exchange: exchange + }); + + if (!symbolDoc) { + throw new Error(`Symbol ${symbol}.${exchange} not found in database`); + } + symbolCountry = symbolDoc.Country; + } + // Get API key const apiKey = EOD_CONFIG.API_TOKEN; if (!apiKey) { @@ -262,7 +281,10 @@ export async function fetchSingleFundamentals( } // Build URL for single fundamentals endpoint - const url = new URL(`https://eodhd.com/api/fundamentals/${symbol}.${exchange}`); + // For US symbols (Country: "USA"), use :US suffix instead of specific exchange code + const exchangeSuffix = symbolCountry === 'USA' ? 'US' : exchange; + + const url = new URL(`https://eodhd.com/api/fundamentals/${symbol}.${exchangeSuffix}`); url.searchParams.append('api_token', apiKey); url.searchParams.append('fmt', 'json'); diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts index f14b683..04340a8 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts @@ -8,12 +8,14 @@ interface FetchIntradayInput { interval: '1m' | '5m' | '1h'; fromDate?: Date; toDate?: Date; + country?: string; } interface CrawlIntradayInput { symbol: string; exchange: string; interval: '1m' | '5m' | '1h'; + country?: string; } interface CrawlState { @@ -86,7 +88,8 @@ export async function scheduleIntradayCrawl( await this.scheduleOperation('crawl-intraday', { symbol: symbol.Code, exchange: symbol.Exchange, - interval + interval, + country: symbol.Country }, { attempts: 3, backoff: { @@ -116,7 +119,7 @@ export async function crawlIntraday( input: CrawlIntradayInput ): Promise<{ success: boolean; recordsProcessed: number; finished: boolean }> { const logger = this.logger; - const { symbol, exchange, interval } = input; + const { symbol, exchange, interval, country } = input; try { logger.info(`Starting intraday crawl for ${symbol}.${exchange} - ${interval}`); @@ -156,7 +159,8 @@ export async function crawlIntraday( exchange, interval, fromDate, - toDate + toDate, + country }); // Update crawl state @@ -204,7 +208,8 @@ export async function crawlIntraday( await this.scheduleOperation('crawl-intraday', { symbol, exchange, - interval + interval, + country }, { attempts: 3, backoff: { @@ -238,7 +243,7 @@ export async function fetchIntraday( input: FetchIntradayInput ): Promise<{ success: boolean; recordsSaved: number }> { const logger = this.logger; - const { symbol, exchange, interval, fromDate, toDate } = input; + const { symbol, exchange, interval, fromDate, toDate, country } = input; try { logger.info(`Fetching intraday data for ${symbol}.${exchange} - ${interval}`, { @@ -246,6 +251,20 @@ export async function fetchIntraday( to: toDate?.toISOString().split('T')[0] }); + // Get country if not provided + let symbolCountry = country; + if (!symbolCountry) { + const symbolDoc = await this.mongodb.collection('eodSymbols').findOne({ + Code: symbol, + Exchange: exchange + }); + + if (!symbolDoc) { + throw new Error(`Symbol ${symbol}.${exchange} not found in database`); + } + symbolCountry = symbolDoc.Country; + } + // Get API key const apiKey = EOD_CONFIG.API_TOKEN; if (!apiKey) { @@ -253,7 +272,10 @@ export async function fetchIntraday( } // Build URL - const url = new URL(`https://eodhd.com/api/intraday/${symbol}.${exchange}`); + // For US symbols (Country: "USA"), use :US suffix instead of specific exchange code + const exchangeSuffix = symbolCountry === 'USA' ? 'US' : exchange; + + const url = new URL(`https://eodhd.com/api/intraday/${symbol}.${exchangeSuffix}`); url.searchParams.append('api_token', apiKey); url.searchParams.append('fmt', 'json'); url.searchParams.append('interval', interval); diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts index 1a1c10d..0ae5f0a 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/prices.ts @@ -5,6 +5,7 @@ import { EOD_CONFIG } from '../shared'; interface FetchPricesInput { symbol: string; exchange: string; + country?: string; // Optional to maintain backward compatibility } export async function scheduleFetchPrices( @@ -55,7 +56,8 @@ export async function scheduleFetchPrices( await this.scheduleOperation('fetch-prices', { symbol: symbol.Code, - exchange: symbol.Exchange + exchange: symbol.Exchange, + country: symbol.Country }, { attempts: 3, backoff: { @@ -84,11 +86,25 @@ export async function fetchPrices( input: FetchPricesInput ): Promise<{ success: boolean; priceCount: number }> { const logger = this.logger; - const { symbol, exchange } = input; + const { symbol, exchange, country } = input; try { logger.info(`Fetching prices for ${symbol}.${exchange}`); + // Use provided country or fetch from database + let symbolCountry = country; + if (!symbolCountry) { + const symbolDoc = await this.mongodb.collection('eodSymbols').findOne({ + Code: symbol, + Exchange: exchange + }); + + if (!symbolDoc) { + throw new Error(`Symbol ${symbol}.${exchange} not found in database`); + } + symbolCountry = symbolDoc.Country; + } + // Get API key from config const apiKey = EOD_CONFIG.API_TOKEN; if (!apiKey) { @@ -96,7 +112,10 @@ export async function fetchPrices( } // Build URL for EOD price data - const url = new URL(`https://eodhd.com/api/eod/${symbol}.${exchange}`); + // For US symbols (Country: "USA"), use :US suffix instead of specific exchange code + const exchangeSuffix = symbolCountry === 'USA' ? 'US' : exchange; + + const url = new URL(`https://eodhd.com/api/eod/${symbol}.${exchangeSuffix}`); url.searchParams.append('api_token', apiKey); url.searchParams.append('fmt', 'json'); // Fetch price data from EOD API diff --git a/apps/stock/data-ingestion/src/handlers/eod/shared/index.ts b/apps/stock/data-ingestion/src/handlers/eod/shared/index.ts index 4226c05..34772c4 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/shared/index.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/shared/index.ts @@ -1 +1,2 @@ export * from './config'; +export * from './utils'; diff --git a/apps/stock/data-ingestion/src/handlers/eod/shared/utils.ts b/apps/stock/data-ingestion/src/handlers/eod/shared/utils.ts new file mode 100644 index 0000000..b0a8690 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/eod/shared/utils.ts @@ -0,0 +1,15 @@ +/** + * Get the exchange suffix for EOD API calls based on country + * US symbols use :US suffix, others use their actual exchange code + */ +export function getEodExchangeSuffix(exchange: string, country?: string): string { + return country === 'USA' ? 'US' : exchange; +} + +/** + * Build symbol.exchange format for EOD API + */ +export function buildEodSymbol(symbol: string, exchange: string, country?: string): string { + const suffix = getEodExchangeSuffix(exchange, country); + return `${symbol}.${suffix}`; +} \ No newline at end of file