From f6a47f7a8cd164c06454cd7c4e6b7d313ef9fffb Mon Sep 17 00:00:00 2001 From: Boki Date: Sat, 5 Jul 2025 10:13:38 -0400 Subject: [PATCH] initial tradingView handler --- .../data-ingestion/src/handlers/index.ts | 3 +- .../actions/fetch-exchanges.action.ts | 155 ++++++++++++++++++ .../src/handlers/tradingview/actions/index.ts | 1 + .../src/handlers/tradingview/shared/config.ts | 5 + .../src/handlers/tradingview/shared/index.ts | 2 + .../src/handlers/tradingview/shared/types.ts | 36 ++++ .../tradingview/tradingview.handler.ts | 31 ++++ tetest.ts | 23 +++ 8 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 apps/stock/data-ingestion/src/handlers/tradingview/actions/fetch-exchanges.action.ts create mode 100644 apps/stock/data-ingestion/src/handlers/tradingview/actions/index.ts create mode 100644 apps/stock/data-ingestion/src/handlers/tradingview/shared/config.ts create mode 100644 apps/stock/data-ingestion/src/handlers/tradingview/shared/index.ts create mode 100644 apps/stock/data-ingestion/src/handlers/tradingview/shared/types.ts create mode 100644 apps/stock/data-ingestion/src/handlers/tradingview/tradingview.handler.ts create mode 100644 tetest.ts diff --git a/apps/stock/data-ingestion/src/handlers/index.ts b/apps/stock/data-ingestion/src/handlers/index.ts index 13d0fb5..4a4fda5 100644 --- a/apps/stock/data-ingestion/src/handlers/index.ts +++ b/apps/stock/data-ingestion/src/handlers/index.ts @@ -11,6 +11,7 @@ import { CeoHandler } from './ceo/ceo.handler'; import { IbHandler } from './ib/ib.handler'; import { QMHandler } from './qm/qm.handler'; import { WebShareHandler } from './webshare/webshare.handler'; +import { TradingViewHandler } from './tradingview/tradingview.handler'; // Add more handler imports as needed @@ -26,7 +27,7 @@ export async function initializeAllHandlers(serviceContainer: IServiceContainer) // The HandlerScanner in the DI container will handle the actual registration // We just need to ensure handlers are imported so their decorators run - const handlers = [CeoHandler, IbHandler, QMHandler, WebShareHandler]; + const handlers = [CeoHandler, IbHandler, QMHandler, WebShareHandler, TradingViewHandler]; logger.info('Handler imports loaded', { count: handlers.length, diff --git a/apps/stock/data-ingestion/src/handlers/tradingview/actions/fetch-exchanges.action.ts b/apps/stock/data-ingestion/src/handlers/tradingview/actions/fetch-exchanges.action.ts new file mode 100644 index 0000000..4f99c24 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tradingview/actions/fetch-exchanges.action.ts @@ -0,0 +1,155 @@ +import type { TradingViewHandler } from '../tradingview.handler'; +import { TradingViewExchange } from '../shared/types'; +import { TRADINGVIEW_CONFIG } from '../shared/config'; + +export async function fetchExchanges(handler: TradingViewHandler): Promise { + const { logger, mongodb } = handler; + + if (!mongodb?.db) { + logger.error('MongoDB not available'); + return null; + } + + const db = mongodb.db; + + try { + // 1. Fetch the HTML page + const response = await fetch(TRADINGVIEW_CONFIG.DATA_COVERAGE_URL, { + headers: { + 'User-Agent': TRADINGVIEW_CONFIG.USER_AGENT, + }, + }); + + logger.debug('Response status:', { + status: response.status, + statusText: response.statusText, + url: response.url + }); + + if (!response.ok) { + throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`); + } + + const html = await response.text(); + + logger.info('Fetched HTML length:', { length: html.length }); + + // Save HTML for debugging + if (process.env.DEBUG_HTML) { + const fs = await import('fs/promises'); + await fs.writeFile('/tmp/tradingview-data-coverage.html', html); + logger.info('Saved HTML to /tmp/tradingview-data-coverage.html'); + } + + // 2. Parse HTML to extract exchange data from the table + const exchanges: TradingViewExchange[] = []; + + // Match table rows containing exchange data + const rowRegex = /]*class="row-qJcpoITA[^"]*"[^>]*>[\s\S]*?<\/tr>/g; + const rows = html.match(rowRegex) || []; + + logger.debug('Found table rows:', { count: rows.length }); + + for (const row of rows) { + try { + // Extract exchange ID + const exchangeMatch = row.match(/alt="([^"]+)"/); + const exchangeNameMatch = row.match(/]*class="exchangeName-qJcpoITA"[^>]*>([^<]+)<\/span>/); + const descriptionMatch = row.match(/]*class="exchangeDescName-qJcpoITA"[^>]*>([^<]+)<\/span>/); + const countryFlagMatch = row.match(/src="https:\/\/s3-symbol-logo\.tradingview\.com\/country\/([^.]+)\.svg"/); + const countryNameMatch = row.match(/]*class="[^"]*exchangeCountryFlag[^"]*"[^>]*\/>([^<]+)<\/span>/); + + // Extract data types from badges + const badgeMatches = row.matchAll(/]*class="badge-PlSmolIm[^"]*"[^>]*>[\s\S]*?]*class="content-PlSmolIm"[^>]*>([^<]+)<\/span>/g); + const dataTypes: string[] = []; + for (const match of badgeMatches) { + dataTypes.push(match[1].toLowerCase()); + } + + if (exchangeNameMatch && descriptionMatch && countryFlagMatch) { + const exchange: TradingViewExchange = { + exchange: descriptionMatch[1].toLowerCase().replace(/\s+/g, '_').replace(/[()]/g, ''), + name: exchangeNameMatch[1], + description: descriptionMatch[1], + dataType: dataTypes, + countryCode: countryFlagMatch[1].toUpperCase(), + countryName: '', // Will be filled from alt text or other sources + }; + + // Try to extract country name + if (countryNameMatch) { + exchange.countryName = countryNameMatch[1].trim(); + } else if (exchangeMatch) { + // Use alt text to extract country name + const parts = exchangeMatch[1].split(' '); + if (parts.length > 1) { + exchange.countryName = parts[0]; + } + } + + exchanges.push(exchange); + } + } catch (e) { + logger.debug('Failed to parse row', { error: e }); + } + } + + if (exchanges.length === 0) { + throw new Error('No exchanges found in HTML'); + } + + logger.info('Extracted exchanges from HTML', { count: exchanges.length }); + + // Batch update tvExchanges collection + try { + const batchOps = exchanges.map(exchange => ({ + updateOne: { + filter: { exchange: exchange.exchange }, + update: { + $set: { + ...exchange, + updated_at: new Date(), + }, + $setOnInsert: { + created_at: new Date(), + }, + }, + upsert: true, + }, + })); + + if (batchOps.length > 0) { + await db.collection('tvExchanges').bulkWrite(batchOps); + } + + logger.info('TradingView exchanges saved to MongoDB'); + } catch (dbError) { + logger.warn('Failed to save exchanges to MongoDB', { error: dbError }); + } + + logger.info('TradingView exchanges fetched', { + count: exchanges.length, + byType: groupExchangesByType(exchanges), + }); + + return exchanges; + } catch (error) { + logger.error('Failed to fetch TradingView exchanges', { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); + return null; + } +} + +function groupExchangesByType(exchanges: TradingViewExchange[]) { + const groups: Record = {}; + + for (const exchange of exchanges) { + for (const type of exchange.dataType) { + groups[type] = (groups[type] || 0) + 1; + } + } + + return groups; +} \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/tradingview/actions/index.ts b/apps/stock/data-ingestion/src/handlers/tradingview/actions/index.ts new file mode 100644 index 0000000..7f59fe9 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tradingview/actions/index.ts @@ -0,0 +1 @@ +export * from './fetch-exchanges.action'; \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/tradingview/shared/config.ts b/apps/stock/data-ingestion/src/handlers/tradingview/shared/config.ts new file mode 100644 index 0000000..7ecb71c --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tradingview/shared/config.ts @@ -0,0 +1,5 @@ +export const TRADINGVIEW_CONFIG = { + DATA_COVERAGE_URL: 'https://www.tradingview.com/data-coverage/', + REQUEST_TIMEOUT: 30000, + USER_AGENT: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', +}; \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/tradingview/shared/index.ts b/apps/stock/data-ingestion/src/handlers/tradingview/shared/index.ts new file mode 100644 index 0000000..c452253 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tradingview/shared/index.ts @@ -0,0 +1,2 @@ +export * from './types'; +export * from './config'; \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/tradingview/shared/types.ts b/apps/stock/data-ingestion/src/handlers/tradingview/shared/types.ts new file mode 100644 index 0000000..22052a7 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tradingview/shared/types.ts @@ -0,0 +1,36 @@ +export interface TradingViewExchange { + exchange: string; + name: string; + description: string; + dataType: string[]; + countryCode: string; // All caps + countryName: string; + popularityRank?: number; +} + +// Raw data from TradingView +export interface TradingViewRawExchange { + text_id?: string; + exchange: string; + provider_id?: string | null; + name: string; + description: string; + data_type: string[]; + country: string; + country_name: string; + popularity_rank?: number; + [key: string]: any; // Other fields we don't need +} + +export interface ExchangeGroup { + name: string; + exchanges: TradingViewRawExchange[]; +} + +export interface TradingViewData { + locale: string; + user: any; + is_authenticated: boolean; + exchanges: ExchangeGroup[]; + currency: string; +} \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/tradingview/tradingview.handler.ts b/apps/stock/data-ingestion/src/handlers/tradingview/tradingview.handler.ts new file mode 100644 index 0000000..029d8c9 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tradingview/tradingview.handler.ts @@ -0,0 +1,31 @@ +import { + BaseHandler, + Disabled, + Handler, + Operation, + ScheduledOperation, +} from '@stock-bot/handlers'; +import { fetchExchanges } from './actions'; + +@Disabled() +@Handler('tradingview') +export class TradingViewHandler extends BaseHandler { + constructor(services: any) { + super(services); + } + + @Operation('fetch-exchanges') + async fetchExchanges(): Promise { + return fetchExchanges(this); + } + + @ScheduledOperation('tradingview-exchanges', '0 0 * * 0', { + priority: 5, + description: 'Fetch and update TradingView exchanges data', + immediately: false, + }) + @Disabled() + async scheduledFetchExchanges(): Promise { + return this.fetchExchanges(); + } +} \ No newline at end of file diff --git a/tetest.ts b/tetest.ts new file mode 100644 index 0000000..50b6c1e --- /dev/null +++ b/tetest.ts @@ -0,0 +1,23 @@ +const pako = require('pako'); + +const value = "a/lpZGluZ2VjbBP9IAahQyHll2u4zSdk+eDEshxAXrKZAKBYXiuIYCAnNNRpnrEIBW5XO5a6x/6q21A6RLsYMsZ3law1yXl74NG7eMMEtJHauEYtK4TC8XsM24SQF8VZlYfF1mIU3dx2gJkGNBULa9wo3Dd4YOyA9mJfz5t+oPHPy80BQ5AAn/0II58Ndl+cyATaep28EaPY5NRd9jM+Z8U9scqGSACO3+3XbzEGaIqlY6tJV3Ajt4QYdsf7vVd5HiWzSQcLJGc45F/uT8/A9A9Qb8ABjlHkJva/HuDv9TezGyxKHvW/2/w0oljydYbJSCRKcAHBntf/QEBIPaARhqR57rwrCvp8lyHSyAJxT2I3UFMVwyuCdj9bwYBpnY0SrCNHbU+Po+KcrW2q3SWG988c1rcCY6XEMdoCLSgF/dry0ZHMQVrehI5d7wjATyvFnn9NM3htehPuqqckjgRh+fs/jSU5A5NyUdStfXBtZ2U=" + +const key = 'tradingeconomics-charts-core-api-key' + +const testFunc = function(e, k) { + console.log('testFunc called with:', e, k); + const a = atob(e) + , n = new Uint8Array(a.length); + console.log('Decoded base64 string:', a); + for (let e = 0; e < a.length; e++) + {n[e] = a.charCodeAt(e);} + const i = (new TextEncoder).encode(k); + console.log('Encoded key:', i); + for (let e = 0; e < n.length; e++) + {n[e] ^= i[e % i.length];} + console.log('XORed data:', n); + return pako.inflate(n, {to :'string'}); +} + +const result = testFunc(value, key); +console.log(JSON.parse(result.toString()));