initial tradingView handler

apps/stock/data-ingestion/src/handlers/index.ts

@@ -11,6 +11,7 @@ import { CeoHandler } from './ceo/ceo.handler';
 import { IbHandler } from './ib/ib.handler';
 import { QMHandler } from './qm/qm.handler';
 import { WebShareHandler } from './webshare/webshare.handler';
+import { TradingViewHandler } from './tradingview/tradingview.handler';
 
 // Add more handler imports as needed
 
@@ -26,7 +27,7 @@ export async function initializeAllHandlers(serviceContainer: IServiceContainer)
     // The HandlerScanner in the DI container will handle the actual registration
     // We just need to ensure handlers are imported so their decorators run
 
-    const handlers = [CeoHandler, IbHandler, QMHandler, WebShareHandler];
+    const handlers = [CeoHandler, IbHandler, QMHandler, WebShareHandler, TradingViewHandler];
 
     logger.info('Handler imports loaded', {
       count: handlers.length,

apps/stock/data-ingestion/src/handlers/tradingview/actions/fetch-exchanges.action.ts

@@ -0,0 +1,155 @@
+import type { TradingViewHandler } from '../tradingview.handler';
+import { TradingViewExchange } from '../shared/types';
+import { TRADINGVIEW_CONFIG } from '../shared/config';
+
+export async function fetchExchanges(handler: TradingViewHandler): Promise<TradingViewExchange[] | null> {
+  const { logger, mongodb } = handler;
+  
+  if (!mongodb?.db) {
+    logger.error('MongoDB not available');
+    return null;
+  }
+  
+  const db = mongodb.db;
+  
+  try {
+    // 1. Fetch the HTML page
+    const response = await fetch(TRADINGVIEW_CONFIG.DATA_COVERAGE_URL, {
+      headers: {
+        'User-Agent': TRADINGVIEW_CONFIG.USER_AGENT,
+      },
+    });
+    
+    logger.debug('Response status:', { 
+      status: response.status, 
+      statusText: response.statusText,
+      url: response.url 
+    });
+    
+    if (!response.ok) {
+      throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
+    }
+    
+    const html = await response.text();
+    
+    logger.info('Fetched HTML length:', { length: html.length });
+    
+    // Save HTML for debugging
+    if (process.env.DEBUG_HTML) {
+      const fs = await import('fs/promises');
+      await fs.writeFile('/tmp/tradingview-data-coverage.html', html);
+      logger.info('Saved HTML to /tmp/tradingview-data-coverage.html');
+    }
+    
+    // 2. Parse HTML to extract exchange data from the table
+    const exchanges: TradingViewExchange[] = [];
+    
+    // Match table rows containing exchange data
+    const rowRegex = /<tr[^>]*class="row-qJcpoITA[^"]*"[^>]*>[\s\S]*?<\/tr>/g;
+    const rows = html.match(rowRegex) || [];
+    
+    logger.debug('Found table rows:', { count: rows.length });
+    
+    for (const row of rows) {
+      try {
+        // Extract exchange ID
+        const exchangeMatch = row.match(/alt="([^"]+)"/);
+        const exchangeNameMatch = row.match(/<span[^>]*class="exchangeName-qJcpoITA"[^>]*>([^<]+)<\/span>/);
+        const descriptionMatch = row.match(/<span[^>]*class="exchangeDescName-qJcpoITA"[^>]*>([^<]+)<\/span>/);
+        const countryFlagMatch = row.match(/src="https:\/\/s3-symbol-logo\.tradingview\.com\/country\/([^.]+)\.svg"/);
+        const countryNameMatch = row.match(/<img[^>]*class="[^"]*exchangeCountryFlag[^"]*"[^>]*\/>([^<]+)<\/span>/);
+        
+        // Extract data types from badges
+        const badgeMatches = row.matchAll(/<span[^>]*class="badge-PlSmolIm[^"]*"[^>]*>[\s\S]*?<span[^>]*class="content-PlSmolIm"[^>]*>([^<]+)<\/span>/g);
+        const dataTypes: string[] = [];
+        for (const match of badgeMatches) {
+          dataTypes.push(match[1].toLowerCase());
+        }
+        
+        if (exchangeNameMatch && descriptionMatch && countryFlagMatch) {
+          const exchange: TradingViewExchange = {
+            exchange: descriptionMatch[1].toLowerCase().replace(/\s+/g, '_').replace(/[()]/g, ''),
+            name: exchangeNameMatch[1],
+            description: descriptionMatch[1],
+            dataType: dataTypes,
+            countryCode: countryFlagMatch[1].toUpperCase(),
+            countryName: '', // Will be filled from alt text or other sources
+          };
+          
+          // Try to extract country name
+          if (countryNameMatch) {
+            exchange.countryName = countryNameMatch[1].trim();
+          } else if (exchangeMatch) {
+            // Use alt text to extract country name
+            const parts = exchangeMatch[1].split(' ');
+            if (parts.length > 1) {
+              exchange.countryName = parts[0];
+            }
+          }
+          
+          exchanges.push(exchange);
+        }
+      } catch (e) {
+        logger.debug('Failed to parse row', { error: e });
+      }
+    }
+    
+    if (exchanges.length === 0) {
+      throw new Error('No exchanges found in HTML');
+    }
+    
+    logger.info('Extracted exchanges from HTML', { count: exchanges.length });
+    
+    // Batch update tvExchanges collection
+    try {
+      const batchOps = exchanges.map(exchange => ({
+        updateOne: {
+          filter: { exchange: exchange.exchange },
+          update: {
+            $set: {
+              ...exchange,
+              updated_at: new Date(),
+            },
+            $setOnInsert: {
+              created_at: new Date(),
+            },
+          },
+          upsert: true,
+        },
+      }));
+      
+      if (batchOps.length > 0) {
+        await db.collection('tvExchanges').bulkWrite(batchOps);
+      }
+      
+      logger.info('TradingView exchanges saved to MongoDB');
+    } catch (dbError) {
+      logger.warn('Failed to save exchanges to MongoDB', { error: dbError });
+    }
+    
+    logger.info('TradingView exchanges fetched', {
+      count: exchanges.length,
+      byType: groupExchangesByType(exchanges),
+    });
+    
+    return exchanges;
+  } catch (error) {
+    logger.error('Failed to fetch TradingView exchanges', { 
+      error: error instanceof Error ? error.message : String(error),
+      stack: error instanceof Error ? error.stack : undefined,
+    });
+    return null;
+  }
+}
+
+function groupExchangesByType(exchanges: TradingViewExchange[]) {
+  const groups: Record<string, number> = {};
+  
+  for (const exchange of exchanges) {
+    for (const type of exchange.dataType) {
+      groups[type] = (groups[type] || 0) + 1;
+    }
+  }
+  
+  return groups;
+}

apps/stock/data-ingestion/src/handlers/tradingview/actions/index.ts

@@ -0,0 +1 @@
+export * from './fetch-exchanges.action';

apps/stock/data-ingestion/src/handlers/tradingview/shared/config.ts

@@ -0,0 +1,5 @@
+export const TRADINGVIEW_CONFIG = {
+  DATA_COVERAGE_URL: 'https://www.tradingview.com/data-coverage/',
+  REQUEST_TIMEOUT: 30000,
+  USER_AGENT: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+};

apps/stock/data-ingestion/src/handlers/tradingview/shared/index.ts

@@ -0,0 +1,2 @@
+export * from './types';
+export * from './config';

apps/stock/data-ingestion/src/handlers/tradingview/shared/types.ts

@@ -0,0 +1,36 @@
+export interface TradingViewExchange {
+  exchange: string;
+  name: string;
+  description: string;
+  dataType: string[];
+  countryCode: string; // All caps
+  countryName: string;
+  popularityRank?: number;
+}
+
+// Raw data from TradingView
+export interface TradingViewRawExchange {
+  text_id?: string;
+  exchange: string;
+  provider_id?: string | null;
+  name: string;
+  description: string;
+  data_type: string[];
+  country: string;
+  country_name: string;
+  popularity_rank?: number;
+  [key: string]: any; // Other fields we don't need
+}
+
+export interface ExchangeGroup {
+  name: string;
+  exchanges: TradingViewRawExchange[];
+}
+
+export interface TradingViewData {
+  locale: string;
+  user: any;
+  is_authenticated: boolean;
+  exchanges: ExchangeGroup[];
+  currency: string;
+}

apps/stock/data-ingestion/src/handlers/tradingview/tradingview.handler.ts

@@ -0,0 +1,31 @@
+import {
+  BaseHandler,
+  Disabled,
+  Handler,
+  Operation,
+  ScheduledOperation,
+} from '@stock-bot/handlers';
+import { fetchExchanges } from './actions';
+
+@Disabled()
+@Handler('tradingview')
+export class TradingViewHandler extends BaseHandler {
+  constructor(services: any) {
+    super(services);
+  }
+
+  @Operation('fetch-exchanges')
+  async fetchExchanges(): Promise<unknown[] | null> {
+    return fetchExchanges(this);
+  }
+
+  @ScheduledOperation('tradingview-exchanges', '0 0 * * 0', {
+    priority: 5,
+    description: 'Fetch and update TradingView exchanges data',
+    immediately: false,
+  })
+  @Disabled()
+  async scheduledFetchExchanges(): Promise<unknown> {
+    return this.fetchExchanges();
+  }
+}

tetest.ts

@@ -0,0 +1,23 @@
+const pako = require('pako');
+
+const value = "a/lpZGluZ2VjbBP9IAahQyHll2u4zSdk+eDEshxAXrKZAKBYXiuIYCAnNNRpnrEIBW5XO5a6x/6q21A6RLsYMsZ3law1yXl74NG7eMMEtJHauEYtK4TC8XsM24SQF8VZlYfF1mIU3dx2gJkGNBULa9wo3Dd4YOyA9mJfz5t+oPHPy80BQ5AAn/0II58Ndl+cyATaep28EaPY5NRd9jM+Z8U9scqGSACO3+3XbzEGaIqlY6tJV3Ajt4QYdsf7vVd5HiWzSQcLJGc45F/uT8/A9A9Qb8ABjlHkJva/HuDv9TezGyxKHvW/2/w0oljydYbJSCRKcAHBntf/QEBIPaARhqR57rwrCvp8lyHSyAJxT2I3UFMVwyuCdj9bwYBpnY0SrCNHbU+Po+KcrW2q3SWG988c1rcCY6XEMdoCLSgF/dry0ZHMQVrehI5d7wjATyvFnn9NM3htehPuqqckjgRh+fs/jSU5A5NyUdStfXBtZ2U="
+
+const key = 'tradingeconomics-charts-core-api-key'
+
+const testFunc = function(e, k) {
+  console.log('testFunc called with:', e, k);
+    const a = atob(e)
+        , n = new Uint8Array(a.length);
+    console.log('Decoded base64 string:', a);
+    for (let e = 0; e < a.length; e++)
+        {n[e] = a.charCodeAt(e);}
+    const i = (new TextEncoder).encode(k);
+    console.log('Encoded key:', i);
+    for (let e = 0; e < n.length; e++)
+        {n[e] ^= i[e % i.length];}
+    console.log('XORed data:', n);
+    return pako.inflate(n, {to :'string'});
+}
+
+const result = testFunc(value, key);
+console.log(JSON.parse(result.toString()));