work on spider for te

apps/stock/config/config/default.json

@@ -78,7 +78,7 @@
       "db": 1
     },
     "workers": 5,
-    "concurrency": 5,
+    "concurrency": 2,
     "enableScheduledJobs": true,
     "defaultJobOptions": {
       "attempts": 3,

apps/stock/data-ingestion/node_modules

@@ -0,0 +1 @@
+../../../node_modules

apps/stock/data-ingestion/package.json

@@ -27,7 +27,8 @@
     "@stock-bot/browser": "*",
     "@stock-bot/proxy": "*",
     "hono": "^4.0.0",
-    "pako": "^2.1.0"
+    "pako": "^2.1.0",
+    "cheerio": "^1.0.0"
   },
   "devDependencies": {
     "typescript": "^5.0.0"

apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts

@@ -33,6 +33,7 @@ import { crawlIntradayData, scheduleIntradayCrawls } from './actions/intraday-cr
 import { createQMOperationRegistry } from './shared/operation-provider';
 
 @Handler('qm')
+@Disabled() // Disable by default, enable specific operations as needed
 export class QMHandler extends BaseHandler<DataIngestionServices> {
   public operationRegistry: OperationRegistry;
 

apps/stock/data-ingestion/src/handlers/te/actions/fetch-countries.action.ts

@@ -0,0 +1,227 @@
+import { getRandomUserAgent } from '@stock-bot/utils';
+import * as cheerio from 'cheerio';
+import { TE_CONFIG } from '../shared/config';
+import type { TeCountry } from '../shared/types';
+import type { TeHandler } from '../te.handler';
+
+export async function fetchCountries(this: TeHandler): Promise<TeCountry[] | null> {
+  const { logger, mongodb } = this;
+  
+  try {
+    // 1. Fetch the HTML page
+    const reqInfo = {
+      proxy: this.proxy.getProxy(),
+      headers: {
+        'User-Agent': getRandomUserAgent(),
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate, br',
+      },
+    }
+    const response = await fetch(TE_CONFIG.COUNTRIES_URL, reqInfo);
+    
+    logger.debug('Response status:', { 
+      status: response.status, 
+      statusText: response.statusText,
+      url: response.url 
+    });
+    
+    if (!response.ok) {
+      throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
+    }
+    
+    const html = await response.text();
+    
+    logger.info('Fetched HTML length:', { length: html.length });
+  
+    
+    // 2. Parse HTML to extract country data
+    const $ = cheerio.load(html);
+    const countries: TeCountry[] = [];
+    
+    // Look for country links - they typically have a pattern like /country-name
+    // Trading Economics groups countries by region in the page
+    $('.list-group-item, a[href^="/"]').each((_, element) => {
+      const $el = $(element);
+      
+      // Try to extract country information
+      let name: string | undefined;
+      let url: string | undefined;
+      let region: string | undefined;
+      
+      // Check if it's a direct link
+      if ($el.is('a')) {
+        const href = $el.attr('href');
+        const text = $el.text().trim();
+        console.log(href)
+        // Filter for country URLs (they don't contain special paths like /indicators, /calendar, etc.)
+        if (href && href.startsWith('/') && !href.includes('/') && text) {
+          name = text;
+          url = href;
+        }
+      } else {
+        // Check for links within table rows
+        const $link = $el.find('a[href^="/"]').first();
+        if ($link.length) {
+          const href = $link.attr('href');
+          const text = $link.text().trim();
+          
+          if (href && text && !href.includes('/indicators') && !href.includes('/calendar')) {
+            name = text;
+            url = href;
+            
+            // Try to get region from parent elements
+            const $regionHeader = $el.closest('.region-section, .country-group').find('h2, h3, .region-title').first();
+            if ($regionHeader.length) {
+              region = $regionHeader.text().trim();
+            }
+          }
+        }
+      }
+      
+      // Add to countries array if we found valid data
+      if (name && url) {
+        // Extract country code from URL if possible (e.g., /united-states -> US)
+        const code = extractCountryCode(url, name);
+        
+        countries.push({
+          name,
+          code,
+          url: `https://tradingeconomics.com${url}`,
+          region,
+          updated_at: new Date(),
+        });
+      }
+    });
+    
+    // Remove duplicates based on name
+    const uniqueCountries = Array.from(
+      new Map(countries.map(c => [c.name, c])).values()
+    );
+    
+    if (uniqueCountries.length === 0) {
+      throw new Error('No countries found in HTML');
+    }
+    
+    logger.info('Extracted countries from HTML', { 
+      count: uniqueCountries.length,
+      byRegion: groupCountriesByRegion(uniqueCountries),
+    });
+    
+    // 3. Save to MongoDB
+    try {
+      console.log( uniqueCountries)
+      if (uniqueCountries.length > 0) {
+        const result = await mongodb?.batchUpsert('teCountries', uniqueCountries, ['code']);
+        logger.info('Countries saved to MongoDB', {
+          matched: result.matchedCount,
+          modified: result.modifiedCount,
+          upserted: result.upsertedCount,
+        });
+      }
+    } catch (dbError) {
+      logger.error('Failed to save countries to MongoDB', { error: dbError });
+      throw dbError;
+    }
+    
+    return uniqueCountries;
+  } catch (error) {
+    logger.error('Failed to fetch Trading Economics countries', { 
+      error: error instanceof Error ? error.message : String(error),
+      stack: error instanceof Error ? error.stack : undefined,
+    });
+    return null;
+  }
+}
+
+function extractCountryCode(url: string, name: string): string | undefined {
+  // Common country code mappings
+  const countryCodeMap: Record<string, string> = {
+    'united-states': 'US',
+    'united-kingdom': 'GB',
+    'euro-area': 'EU',
+    'china': 'CN',
+    'japan': 'JP',
+    'germany': 'DE',
+    'france': 'FR',
+    'italy': 'IT',
+    'spain': 'ES',
+    'canada': 'CA',
+    'australia': 'AU',
+    'south-korea': 'KR',
+    'india': 'IN',
+    'brazil': 'BR',
+    'russia': 'RU',
+    'mexico': 'MX',
+    'indonesia': 'ID',
+    'netherlands': 'NL',
+    'saudi-arabia': 'SA',
+    'turkey': 'TR',
+    'switzerland': 'CH',
+    'poland': 'PL',
+    'sweden': 'SE',
+    'belgium': 'BE',
+    'argentina': 'AR',
+    'ireland': 'IE',
+    'austria': 'AT',
+    'norway': 'NO',
+    'israel': 'IL',
+    'singapore': 'SG',
+    'denmark': 'DK',
+    'egypt': 'EG',
+    'philippines': 'PH',
+    'finland': 'FI',
+    'chile': 'CL',
+    'pakistan': 'PK',
+    'romania': 'RO',
+    'new-zealand': 'NZ',
+    'greece': 'GR',
+    'iraq': 'IQ',
+    'portugal': 'PT',
+    'czech-republic': 'CZ',
+    'vietnam': 'VN',
+    'peru': 'PE',
+    'colombia': 'CO',
+    'malaysia': 'MY',
+    'ukraine': 'UA',
+    'hungary': 'HU',
+    'kuwait': 'KW',
+    'morocco': 'MA',
+    'slovakia': 'SK',
+    'kenya': 'KE',
+    'puerto-rico': 'PR',
+    'ecuador': 'EC',
+    'ethiopia': 'ET',
+    'dominican-republic': 'DO',
+    'luxembourg': 'LU',
+    'oman': 'OM',
+    'guatemala': 'GT',
+    'bulgaria': 'BG',
+    'ghana': 'GH',
+    'tanzania': 'TZ',
+    'turkmenistan': 'TM',
+    'croatia': 'HR',
+    'costa-rica': 'CR',
+    'lebanon': 'LB',
+    'slovenia': 'SI',
+    'lithuania': 'LT',
+    'serbia': 'RS',
+    'panama': 'PA',
+  };
+  
+  // Clean URL to get country slug
+  const slug = url.replace(/^\//, '').toLowerCase();
+  
+  return countryCodeMap[slug];
+}
+
+function groupCountriesByRegion(countries: TeCountry[]): Record<string, number> {
+  const groups: Record<string, number> = {};
+  
+  for (const country of countries) {
+    const region = country.region || 'Unknown';
+    groups[region] = (groups[region] || 0) + 1;
+  }
+  
+  return groups;
+}

apps/stock/data-ingestion/src/handlers/te/actions/index.ts

@@ -1,2 +1,4 @@
 // Export all action functions here
-// export * from './example.action';
+export * from './fetch-countries.action';
+export * from './spider.action';
+

apps/stock/data-ingestion/src/handlers/te/actions/spider.action.ts

@@ -0,0 +1,140 @@
+import { getRandomUserAgent } from '@stock-bot/utils';
+import * as cheerio from 'cheerio';
+import { TE_CONFIG } from '../shared/config';
+import type { TeHandler } from '../te.handler';
+
+export async function spiderUrl(this: TeHandler, payload: { url: string }): Promise<string[] | null> {
+  const { logger, mongodb } = this;
+  const reqUrl = payload && payload.url ? TE_CONFIG.MAIN_URL + payload.url : TE_CONFIG.MAIN_URL;
+  this.logger.info(`Spiderring URL: ${reqUrl}`, {reqUrl});
+
+  const mongoRecord = await mongodb?.findOne('teUrls', { url: payload?.url || '/' });
+  if(payload && payload.url && mongoRecord && mongoRecord.lastCrawled && mongoRecord.lastCrawled.getTime() > Date.now() - 7* 24 * 60 * 60 * 1000) {
+    this.logger.info(`Skipping URL ${reqUrl} as it was already crawled in the last 24 hours`);
+    return null; // Skip if already crawled in the last 24 hours
+  }
+
+  if (!payload) {
+    const oneDayAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
+    const records = await mongodb?.find('teUrls', {
+      $or: [
+        { lastCrawled: { $lt: oneDayAgo } },  // Crawled more than 24 hours ago
+        { lastCrawled: { $exists: false } }   // Never crawled
+      ]
+    });
+    this.logger.info(`Found ${records?.length || 0} records to process`);
+    for (const record of records || []) {
+      await this.scheduleOperation('te-spider', {
+        url: record.url,
+      }, {
+        jobId: `te-spider-${record.url}`,
+        priority: 5, // Lower priority than financial data
+      });
+    }
+  }
+  
+  try {
+    // 1. Fetch the HTML page
+    const reqInfo = {
+      proxy: 'http://5.79.66.2:13010',//this.proxy.getProxy(),
+      headers: {
+        'User-Agent': getRandomUserAgent(),
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate, br',
+      },
+    }
+    const response = await fetch(reqUrl, reqInfo);
+    
+    logger.debug('Response status:', { 
+      status: response.status, 
+      statusText: response.statusText,
+      url: response.url 
+    });
+    
+    if (!response.ok) {
+      throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    
+
+    let match = html.match(/TESymbol = '([^']+)'/);
+    const teSymbol = match ? match[1] : undefined;
+    match = html.match(/;TELastUpdate = '([^']+)'/);
+    const teLastUpdate = match ? match[1] : undefined;
+    match = html.match(/; var TEChartsDatasource = '([^']+)'/);
+    const teChartUrl =   match ? match[1] : undefined;
+    match = html.match(/; var TEChartsToken = '([^']+)'/);
+    const teChartToken = match ? match[1] : undefined;
+
+    console.log(teSymbol, teLastUpdate, teChartUrl, teChartToken);
+
+    const $ = cheerio.load(html);
+    const urls: string[] = [];
+    
+    $('.list-group-item, a[href^="/"]').each((_, element) => {
+      const $el = $(element);
+      let url: string | undefined;
+      if ($el.is('a')) {
+        const href = $el.attr('href');
+        if (href && href.startsWith('/') && !href.includes('.aspx')) {
+          url = href;
+        }
+      }
+
+      if (url && urls.indexOf(url) === -1) {
+        urls.push(url);
+      }
+    });
+    
+    if (urls.length === 0) {
+      throw new Error('No urls found in HTML');
+    }
+    
+    // 3. Save to MongoDB
+    try {
+      if (urls.length > 0) {
+        const urlMap: {url: string, lastCrawled?: Date, teSymbol? : string, teLastUpdate? : string, teChartUrl? : string, teChartToken? : string}[] = urls.map(url => ({url}));
+        if( payload && payload.url) {
+          urlMap.push({
+            url: payload.url,
+            lastCrawled: new Date(),
+            teSymbol,
+            teLastUpdate,
+            teChartUrl,
+            teChartToken,})
+        }else {
+          urlMap.push({url: '/', lastCrawled: new Date()})
+        }
+
+        const result = await mongodb?.batchUpsert('teUrls', urlMap, ['url']);
+        logger.info('TE URLs saved to MongoDB', {
+          matched: result.matchedCount,
+          modified: result.modifiedCount,
+          upserted: result.upsertedCount,
+        });
+      }
+    } catch (dbError) {
+      logger.error('Failed to save urls to MongoDB', { error: dbError });
+      throw dbError;
+    }
+
+    for (const url of urls) {
+      this.scheduleOperation('te-spider', {
+        url: url,
+      }, {
+        jobId: `te-spider-${url}`,
+        priority: 5, // Lower priority than financial data
+      })
+    }
+    
+    return urls;
+  } catch (error) {
+    logger.error(`Failed to fetch Trading Economics URLs ${reqUrl}`, { 
+      error: error instanceof Error ? error.message : String(error),
+      stack: error instanceof Error ? error.stack : undefined,
+    });
+    return null;
+  }
+}

apps/stock/data-ingestion/src/handlers/te/shared/config.ts

@@ -1,6 +1,9 @@
 export const TE_CONFIG = {
   // Add configuration constants here
   API_URL: 'https://api.example.com',
+  MAIN_URL: 'https://tradingeconomics.com',
+  COUNTRIES_URL: 'https://tradingeconomics.com/countries',
   REQUEST_TIMEOUT: 30000,
+  USER_AGENT: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
   KEY: 'tradingeconomics-charts-core-api-key',
 };

apps/stock/data-ingestion/src/handlers/te/shared/types.ts

@@ -10,4 +10,13 @@ export interface TeResponse {
   data: TeData[];
   status: string;
   timestamp: Date;
+}
+
+export interface TeCountry {
+  name: string;
+  code?: string;
+  region?: string;
+  url?: string;
+  updated_at: Date;
+  created_at?: Date;
 }

apps/stock/data-ingestion/src/handlers/te/te.handler.ts

@@ -1,29 +1,31 @@
 import {
   BaseHandler,
+  Disabled,
   Handler,
-  Operation,
+  ScheduledOperation
-  ScheduledOperation,
 } from '@stock-bot/handlers';
+import type { DataIngestionServices } from '../../types';
+import { fetchCountries, spiderUrl } from './actions';
 
 @Handler('te')
-export class TeHandler extends BaseHandler {
+@Disabled()
+export class TeHandler extends BaseHandler<DataIngestionServices> {
   constructor(services: any) {
     super(services);
   }
 
-  @Operation('example-operation')
+  @ScheduledOperation('te-countries', '0 0 * * 0', {
-  async exampleOperation(): Promise<unknown> {
-    this.logger.info('TE handler example operation executed');
-    return { success: true };
-  }
-
-  @ScheduledOperation('te-scheduled-job', '0 0 * * *', {
     priority: 5,
-    description: 'Daily TE handler scheduled job',
+    description: 'Fetch and update Trading Economics countries data',
-    immediately: false,
+    immediately: true,
   })
-  async scheduledJob(): Promise<unknown> {
+  @Disabled()
-    this.logger.info('TE handler scheduled job executed');
+  fetchCountries = fetchCountries;
-    return this.exampleOperation();
+
-  }
+  @ScheduledOperation('te-spider', '0 0 * * 0', {
+    priority: 5,
+    description: 'Fetch and update Trading Economics countries data',
+    immediately: true,
+  })
+  spiderUrlSchedule = spiderUrl;
 }

apps/stock/tsconfig.json

@@ -14,7 +14,6 @@
     { "path": "./data-pipeline" },
     { "path": "./web-api" },
     { "path": "./web-app" },
-    { "path": "./engine"},
     { "path": "./orchestrator"}
   ]
 }

apps/stock/web-app/tsconfig.json

@@ -7,10 +7,11 @@
     "module": "ESNext",
     "skipLibCheck": true,
     "moduleResolution": "bundler",
-    "allowImportingTsExtensions": true,
+    // "allowImportingTsExtensions": true,
     "resolveJsonModule": true,
     "isolatedModules": true,
-    "noEmit": true,
+    // "noEmit": true,
+    "composite": true,
     "jsx": "react-jsx",
     "strict": true,
     "noUnusedLocals": true,

bun.lock

@@ -77,6 +77,7 @@
         "@stock-bot/shutdown": "*",
         "@stock-bot/stock-config": "*",
         "@stock-bot/utils": "*",
+        "cheerio": "^1.0.0",
         "hono": "^4.0.0",
         "pako": "^2.1.0",
       },
@@ -1171,6 +1172,8 @@
 
     "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="],
 
+    "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
+
     "bowser": ["bowser@2.11.0", "", {}, "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA=="],
 
     "bplist-parser": ["bplist-parser@0.2.0", "", { "dependencies": { "big-integer": "^1.6.44" } }, "sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw=="],
@@ -1223,6 +1226,10 @@
 
     "chalk": ["chalk@5.4.1", "", {}, "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="],
 
+    "cheerio": ["cheerio@1.1.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.0", "htmlparser2": "^10.0.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.10.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-+0hMx9eYhJvWbgpKV9hN7jg0JcwydpopZE4hgi+KvQtByZXPp04NiCWU0LzcAbP63abZckIHkTQaXVF52mX3xQ=="],
+
+    "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="],
+
     "chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="],
 
     "chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="],
@@ -1283,6 +1290,10 @@
 
     "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
 
+    "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
+
+    "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
+
     "cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="],
 
     "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
@@ -1375,6 +1386,14 @@
 
     "dom-helpers": ["dom-helpers@5.2.1", "", { "dependencies": { "@babel/runtime": "^7.8.7", "csstype": "^3.0.2" } }, "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA=="],
 
+    "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
+
+    "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
+
+    "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
+
+    "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
+
     "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
 
     "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="],
@@ -1387,6 +1406,8 @@
 
     "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
 
+    "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="],
+
     "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="],
 
     "engine.io": ["engine.io@6.6.4", "", { "dependencies": { "@types/cors": "^2.8.12", "@types/node": ">=10.0.0", "accepts": "~1.3.4", "base64id": "2.0.0", "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.3.1", "engine.io-parser": "~5.2.1", "ws": "~8.17.1" } }, "sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g=="],
@@ -1395,6 +1416,8 @@
 
     "engine.io-parser": ["engine.io-parser@5.2.3", "", {}, "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q=="],
 
+    "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
+
     "es-abstract": ["es-abstract@1.24.0", "", { "dependencies": { "array-buffer-byte-length": "^1.0.2", "arraybuffer.prototype.slice": "^1.0.4", "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", "call-bound": "^1.0.4", "data-view-buffer": "^1.0.2", "data-view-byte-length": "^1.0.2", "data-view-byte-offset": "^1.0.1", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "es-set-tostringtag": "^2.1.0", "es-to-primitive": "^1.3.0", "function.prototype.name": "^1.1.8", "get-intrinsic": "^1.3.0", "get-proto": "^1.0.1", "get-symbol-description": "^1.1.0", "globalthis": "^1.0.4", "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", "has-proto": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "internal-slot": "^1.1.0", "is-array-buffer": "^3.0.5", "is-callable": "^1.2.7", "is-data-view": "^1.0.2", "is-negative-zero": "^2.0.3", "is-regex": "^1.2.1", "is-set": "^2.0.3", "is-shared-array-buffer": "^1.0.4", "is-string": "^1.1.1", "is-typed-array": "^1.1.15", "is-weakref": "^1.1.1", "math-intrinsics": "^1.1.0", "object-inspect": "^1.13.4", "object-keys": "^1.1.1", "object.assign": "^4.1.7", "own-keys": "^1.0.1", "regexp.prototype.flags": "^1.5.4", "safe-array-concat": "^1.1.3", "safe-push-apply": "^1.0.0", "safe-regex-test": "^1.1.0", "set-proto": "^1.0.0", "stop-iteration-iterator": "^1.1.0", "string.prototype.trim": "^1.2.10", "string.prototype.trimend": "^1.0.9", "string.prototype.trimstart": "^1.0.8", "typed-array-buffer": "^1.0.3", "typed-array-byte-length": "^1.0.3", "typed-array-byte-offset": "^1.0.4", "typed-array-length": "^1.0.7", "unbox-primitive": "^1.1.0", "which-typed-array": "^1.1.19" } }, "sha512-WSzPgsdLtTcQwm4CROfS5ju2Wa1QQcVeT37jFjYzdFz1r9ahadC8B8/a4qxJxM+09F18iumCdRmlr96ZYkQvEg=="],
 
     "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
@@ -1621,6 +1644,8 @@
 
     "hono": ["hono@4.8.4", "", {}, "sha512-KOIBp1+iUs0HrKztM4EHiB2UtzZDTBihDtOF5K6+WaJjCPeaW4Q92R8j63jOhvJI5+tZSMuKD9REVEXXY9illg=="],
 
+    "htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="],
+
     "http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="],
 
     "http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="],
@@ -1931,6 +1956,8 @@
 
     "npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="],
 
+    "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
+
     "numeral": ["numeral@2.0.6", "", {}, "sha512-qaKRmtYPZ5qdw4jWJD6bxEf1FJEqllJrwxCLIm0sQU/A7v2/czigzOb+C2uSiFsa9lBUzeH7M1oK+Q+OLxL3kA=="],
 
     "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
@@ -1995,6 +2022,12 @@
 
     "parent-module": ["parent-module@1.0.1", "", { "dependencies": { "callsites": "^3.0.0" } }, "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g=="],
 
+    "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
+
+    "parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="],
+
+    "parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="],
+
     "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
 
     "pascal-case": ["pascal-case@3.1.2", "", { "dependencies": { "no-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g=="],
@@ -2445,7 +2478,7 @@
 
     "unbox-primitive": ["unbox-primitive@1.1.0", "", { "dependencies": { "call-bound": "^1.0.3", "has-bigints": "^1.0.2", "has-symbols": "^1.1.0", "which-boxed-primitive": "^1.1.1" } }, "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw=="],
 
-    "undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="],
+    "undici": ["undici@7.11.0", "", {}, "sha512-heTSIac3iLhsmZhUCjyS3JQEkZELateufzZuBaVM5RHXdSBMb1LPMQf5x+FH7qjsZYDP0ttAc3nnVpUB+wYbOg=="],
 
     "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
 
@@ -2475,6 +2508,10 @@
 
     "webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="],
 
+    "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
+
+    "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
+
     "whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="],
 
     "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
@@ -2659,6 +2696,8 @@
 
     "got/type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="],
 
+    "htmlparser2/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
+
     "is-wsl/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="],
 
     "joi/@hapi/hoek": ["@hapi/hoek@9.3.0", "", {}, "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ=="],
@@ -2687,6 +2726,8 @@
 
     "os-dns-native/node-addon-api": ["node-addon-api@4.3.0", "", {}, "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="],
 
+    "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
+
     "path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
 
     "pg-mem/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="],
@@ -2735,6 +2776,8 @@
 
     "tailwindcss/object-hash": ["object-hash@3.0.0", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="],
 
+    "testcontainers/undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="],
+
     "ts-unused-exports/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
 
     "type-is/mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="],

docs/data-sources.md

@@ -40,4 +40,11 @@
 
 ## Additional Resources
 - **Marketstack**: https://marketstack.com/
-- **Alpaca**: https://alpaca.markets/ - Future hopefully or US connection
+- **Alpaca**: https://alpaca.markets/ - Future hopefully or US connection
+- **MacroMicro**: https://en.macromicro.me/
+- **Norgate**: https://norgatedata.com/stockmarketpackages.php#castocks
+- **FINRA**: https://www.finra.org
+https://otce.finra.org/otce/EquityShortInterest/staticArchives?startDate=2014-01-01&endDate=2024-11-03&dateDisplay=November%203,%202018
+https://www.finra.org/finra-data/browse-catalog/equity-short-interest/files
+https://www.cboe.com/us/equities/market_statistics/short_interest/?year=2025&month=06&mkt=bzx
+- **MORNINGSTAR**: https://finra-markets.morningstar.com/MarketData/EquityOptions/detail.jsp?query=14:0P0001BUHA  PRICES/SHORT INTEREST

docs/todo.md

@@ -0,0 +1,8 @@
+Data Ingestion
+- QM scrape all timeframes add timeframe column to collection
+- EOD Fundementals / Prices / Intraday / Delisted ++
+- DataBento - 125 credit - use for symbols / second / minute intraday
+- Work on TE Indicators
+- CEO - watch all and save them to posts. do it every minute
+- Test our handler specific rate limits
+- Fix up handler worker counts

libs/core/handlers/src/base/BaseHandler.ts

@@ -1,4 +1,3 @@
-import type { Collection } from 'mongodb';
 import { createNamespacedCache } from '@stock-bot/cache';
 import { getLogger } from '@stock-bot/logger';
 import type {
@@ -11,6 +10,7 @@ import type {
   ServiceTypes,
 } from '@stock-bot/types';
 import { fetch } from '@stock-bot/utils';
+import type { Collection } from 'mongodb';
 // Handler registry is now injected, not imported
 import { createJobHandler } from '../utils/create-job-handler';
 
@@ -18,6 +18,7 @@ import { createJobHandler } from '../utils/create-job-handler';
  * Job scheduling options
  */
 export interface JobScheduleOptions {
+  jobId?: string; // Optional job ID for custom identification
   delay?: number;
   priority?: number;
   attempts?: number;

tetest.ts

@@ -1,6 +1,8 @@
 const pako = require('pako');
 
-const value = "a/lpZGluZ2VjbBP9IAahQyHll2u4zSdk+eDEshxAXrKZAKBYXiuIYCAnNNRpnrEIBW5XO5a6x/6q21A6RLsYMsZ3law1yXl74NG7eMMEtJHauEYtK4TC8XsM24SQF8VZlYfF1mIU3dx2gJkGNBULa9wo3Dd4YOyA9mJfz5t+oPHPy80BQ5AAn/0II58Ndl+cyATaep28EaPY5NRd9jM+Z8U9scqGSACO3+3XbzEGaIqlY6tJV3Ajt4QYdsf7vVd5HiWzSQcLJGc45F/uT8/A9A9Qb8ABjlHkJva/HuDv9TezGyxKHvW/2/w0oljydYbJSCRKcAHBntf/QEBIPaARhqR57rwrCvp8lyHSyAJxT2I3UFMVwyuCdj9bwYBpnY0SrCNHbU+Po+KcrW2q3SWG988c1rcCY6XEMdoCLSgF/dry0ZHMQVrehI5d7wjATyvFnn9NM3htehPuqqckjgRh+fs/jSU5A5NyUdStfXBtZ2U="
+
+const value = "a/lpZGluZ2VjbAM1pgcHtCCefSKpgAeH6QtP0aHicG8/E3R+LXZixUTbVGiIXr8M4eCCDCL834tYqtuIDcXYS1QZCKHHNMoXO8X+qlYyeas0uDlKeJPzqaonV4UIeZ6Xl0yU84SAbzuXJYyTBAebk1o1gV6dmouchDgdjAcMt75NiQFeGhMPW/D8zjdw1FvG6eUWq6YxKZIBlK7Z+v6MrfS+FltPy43ImpPt0t6CXdZW3ki9z+7+S475zI4ckhSan5yCtAo6jrrNkphC3JBpGtcZovpSkJkPA4yHn6LTm6bEyIm4shCc1JqYk/qFmwzerHDMhuKPPsYKtYcmzedlx5/6adh/bnyQn6Gc1BJddpUBmCPJkKPM9OJelArjhpVYaXXe+7Xkjc3V5XbKLlMVn9HQ5ggh3VWeyqoeTy2e1Ayw7WC00qoPjeHNjS69w5bYMYQvUIHD+t/WsdlavkBvPtjyEytQa/8vzVOblwjy2626ldpGWBzQotgGSx8pKwN3YlmDMcam2kJQ6wZoXwy/XsQE9SqmZQ1CJROVwYE2L/BvL06jWxT1w6LRKNwbH6IP0JMGqyqTXQjZ1TOZRrQOifUkTm5/9d++VCAzM2KsqH3b/os/yPk6Ubt5x6AJW7Ng4mQB3m+R+sc1fwfZfkLwHQEld1HLrmMiNoFgvBeL/y2AOAn7lLZ3zCBvL4MHChEcDZx7LppmG4wVu9inrH+dJqLgREyZF8ONRW89+Q4wf/qszwqMi5yDWyCZF/MT7LHRrLvSokxddwH+XrnuMXygH/5gH0WrJNQCYuSTSzAyRUbI+rwyETTkOXbGtE2r7AnHd/XfKyCTtSf9rJL9u9obwLVWW65+ocVG0z/K4jZUdWNRVAb3iqDy2pj5TwqHJxIyBVPfcX0L5fJqzj9nlyrNIr5zXr5gPhtHz//vsu/nCwqbHarK950T4hflFmFrJUPyaXwB/1eT7vlYB6YrUHV/wkvVzIQ2WSTqYy5OeEffhKyHTJWkaOuNCKKPovqrtjYCwIiElaGG99HeQP/tZ4yWfeIIuN8deiBlEa1OMRcgqhRxEXtY1mR3MDBAs8M1M7WRT0dGcnvJPlL5ihxZal2nY7qN4yb6/MNdFJ3xwk/5IEG8W4J4DqioZ7IJxFkOKLixxAQiyS3hOqD9NZh0KPg0OkUkPiMUul0mTRYnNgPf79grkM0w+BvUfaP1274L4e01tA+CmrfYSvkfHk8wNTvI+0XI+O8XFYMYX9ewa1L4yWHpVZj8HlD0/oNLp/JBHr7n+AMXAGZfvtPy6YXT4NrYBO1IAsRr0MJZThDd14VTzbkMh4pVwn467fGKLYLXOVgj8pia9YY54AE28YZ7ox5/G2Cy6vbnZYnJdUF0mBBvijkhzNchHVtzyw7BMehKMF5TAEN6ndcH5fW7KtV4IKyv/ZRvxOZcgKDBRx/LYbFri8uGs03sXvBC4CmOpGsHmtaBCvZ1G8fwxK/jHt+ZoxlNfsbaS9mxiRdVDdlFRmobRTzA5+W4c+1HLvlTNMRIW0auXQWGjSfYeFQfpICzWAiTKw1w0gcC7YLAvot+ivrinZFORkSyCvFtETc/gok/ajprIF97eoeEY+2yEqaeTOdo3G04wyOz27EOjvy8XUst4Na3d8j+e0LPTh93ZaCxjAYUdv5DwjkktyM8nQyyTmfiXTJlUUKWuvbedG9lNnn/9Oq5T+4cGSm9tfsMz8tK8VD/68FIiu2pq/o2h6yzV8RX0PgJu6q3Oub2143YgzQy7Xp+fCslI3BJ+kbEEWGgtxgV6WIAbeMHgJvyizjfZsk/CHzModCYWF3KnXzWDbwe8FL+rhC0GGuvZbleThSx9IYw0/MI5YQs3+1fAcUkkgN01F1/qxCkAF3q2fXq2Ctbm8Uim1OBOsMg3zi1sDXbdUMeTQs7MislQdMiX3VLVHqHGm61jynPf5X8OTLkFl6LGy2HHj8LYGg74bLcPIoyxONilLHWe48FtJMaJdqmCKK+v2WrhQ/uhvyIEf2Lrij/jd8JXjhg1Q4vugu/DXLCzETrFxJU+XbFyifAa0kQ/p1X0jfH/8f6vkO7zyfUE1d1N46iudYeSRpZNCLZlIZZY/EfVj4XAEM46M5E1xlyDLrWXvxf6ewF8oLMBarSDqzg2MbgH2q0SPIN1jfE2RYpJDzGlp1hZ8YTEESlpu+1ofbJqSnRfu6A3YtqXJ9aImkqpAo4WzVZoN/Ppn+DpsvMRO37kNmqPapC1+MBRP+cR0nYKPuNx92NrrgY4WBTGR0F4tnCC4bxEFimd0kJgTeksjRC0htizYQYNGLKrSYG3Nt9NFkrSBn6c+HYgAH3u86l2U1H+W7JmDW3h5kkt9L2n5Ml+OtSwit0uP95BGtgD/ZVWx13K9WkqF7l7SckA3kTtGN/n3AlPnzklR0JZpItGuB2S6Joe8oBWltEMRQjDS7Li688Lv0Q8fVHKvRt9ct2ZgibPmu4vbAWqhbv9tvVP2A5GZk9MIJEOG10owdhPcayBiK08DhLyLxY2/C7xKEj2XlHB/I13beb8u+oGT9umT1H29jN1qyKEAxRPapPT5TzEAATVi5kPPvWvxw5/x6Xg6koQ4WkN1gIWByOuX/hS+Y+jrfLE8Y5FOOeDguqigQXfgULWi44VcpF9DLFVMbT9uQlRr4Kal2GvDtFxb1PsdYQRN9KrRmCMQU+ZFFyP0CF1EEZ73hQ2KOXdT7rpLW8Q+z26dVbmyipAwXcxDxOFwSLEGJlCYgaB0/TQtj3b/aa9qjFM8iZH++pch7Jb8RY4TOfyviaOWsUJXAvwowQqB6lS5UQXj6wqKY9sq/D4iQdgqzXLjyaD7S3ZSMPWxE28qyP4o6Blrnhnv6UyvbFasIy/6/Dej3FNS6fFaU+6Lo3nuCjXczeASIDa5JqPQXqJcaW5ZcOrBIFizTkgHCQV+eBNllfufva7Gzaei51Hy98mCWwVxuNXuGql3EyTKF8f6vloesS6rjMGdsbUiZVWSWgwTCUTbzT9i+i+PK70gm+9nSJhRcyqOYC3nC9xuF6UImmGyUf4ocCSowW+s3GKzoV+HQ67mAf9iM5qq8NaH12SznH3CnZThUyOqBQ/wpRMF8sBK7gvAh3NVQ/UvXfOyPEh1QogRRARzT2C1X2wKc7yqVMKNOuJjy+wGi/1+1otom3BVCeqZwCmHSpXlTbrXEQzt0FSObtyDr2yGDADOrvyDL59/81hatpfxDcZlABQtXJc7NhtdiCXwRH4f3zJpGhkaTY1Dw6RrUY4TeePOCvBYa9aQegzaLZBYFUfFV8el+CpksZYtnwKSrYCgeBErXb/ILnmwj+gTmYaqWzeCSDy4gvjVan+K7Ndp06ziZFMuZ9uad9p0G1hk5p5WjQnCU2qV6V7HZooODkxwNRV+75SxOrhGK6hmz7fO9AXSh0DnudJLEI3Opcb3ABB1Uelbd5zwjt7qbS0Ko9wuYJbnLd4E8RJnbjcYddPgtLa6HSVnizFXGIbE8z+PKHNs6iq/XK1UBPhFswZ60bbPl56ucquQfmzqKxy0gepgNVPyypmsRS1mLed+K2g7NGUCRbMwilWmAwLF644S9hEcHQ1Y8MdSPcj+umluyEWGfeXrRVrRpkmzfAjckbzLc/EHJuBBMEsFmX2KZ7Q4o63Tk4vGrh1Jm7PIorHoW2LXKX24ahvI773rGwgC80WURYumc4voF1iYfKOamEwUgJQyHPvTvKUsAO37mF/Mi509CNlEU7FLtbTu8YTqzT2z6XWZcvVmFXVI2FO7dGxPKES3Rhe7C3n5m03JxuHaWyEN53326liRK8UZ0bX9Y8tQc8PDQBAGiSM9qmECOMzHaYSqEE3xfntRtMADQOxLfj6eDfcTABPTPkfPseQ1v/zBDZxvCmd91BOx+Asda6HHCGxkCC+j+7bzVk9yDkR89Zk8Z7D9yg4jJ54e3DSvZeTa5ta/gbpzaPGnkM158IqSBLKGB2HzWjUhe1VfSh0ruuFkbUulO0apI00iia2s8yCyVdf5tb8DHieW3HcXkWeWkEHq4TNdzb12cDRpou3UcUlPNU1aC+xf2F8pcID7WxyZADLOuqWD1/+9TFtvvOG8h5HyXOPDcEeUlp7jUNazvel6pAYbqbpzah24GftKNZ4cdy3TrWt31efsJtzARHKTDqFue1nT8f3gnU0aje3jTW9z++HH/pt3m4mMfDVnlt0lbtocIPQTOfuVG6nEQk7ltTtnOCwyeSMesX2Mjw2ilZoqW6ULWvyFfGpqOjAVD2Nqp9Cnb/W6hOQJ6su6+5uOUDi7+s1oEVtLVHfASfjVDawwzhcSbNkbXmRhSmkr7m0JiBFfqfo/d8GtLDILpWgAcJ0ivqPyL4Tf2ruPHRmE2PgeKf585jY24iXlrD/cIwrH6A/j4EKg5HtfqaGsXMpgVaPuv3mfiS1qGW0z2gtsF/0FHxGMSb6YwF6bGLyN60mR9qgGGgAbwH3NgtMA90D3sI0pb0JUAMjmff/uVpPPi7BqWaaBvLwceSstkZpc3ZKYzIWLPLlpGs77kX1X8kz9xPMhsh7ftlfses0Pygpw3AObDJRHqCQFO4tJxxor6IndwRhZ9VEtjDyoimu5aSoFqeR7H211K8LTMtn04lowOVmYaF2o2UipyeF8CTgpuAhYCAld8edIITGhoYCZHY/04draSRPbFMGlMbDowi9apWO4UOy/7JIJsz0A4amBMaS9KUgZBuCwwX3rOu+AOWzLJiF+eKKAcf+6VCnrLUgrEB8nfc71jstbqF0+T62ryfqsqGiILDlxeCCItNAzDZmYAfkZeRhmRcs+Cgu4KZ05YxkZ6RtuB2DmvUWbyLmo6f/9YXHtkYFBcSk3NITHF9d//MsCwChLSlwsBB7/YC3Bnvp0/OrVND6+6cGleczlReL0M8Br/swxT6DKGvPSWF7bYTlZNG0DIiUb1JZ2U="
+const URL = 'https://d3ii0wo49og5mi.cloudfront.net/economics/szlttr?span=max&v=20250619074600&key=20240229:nazare'
 
 const key = 'tradingeconomics-charts-core-api-key'
 
@@ -19,5 +21,15 @@ const testFunc = function(e, k) {
     return pako.inflate(n, {to :'string'});
 }
 
-const result = testFunc(value, key);
+fetch(URL, {
-console.log(JSON.parse(result.toString()));
+  proxy: 'http://5.79.66.2:13010',
+  headers: {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome'
+  }
+}).then(async response => {
+  const resp = await response.text();
+  const result = testFunc(resp.replaceAll('"', ""), key);
+  // console.log(result.toString());
+  console.log(JSON.stringify(JSON.parse(result.toString())));
+})
+