diff --git a/apps/stock/config/config/default.json b/apps/stock/config/config/default.json index 6f127f8..e8e112d 100644 --- a/apps/stock/config/config/default.json +++ b/apps/stock/config/config/default.json @@ -78,7 +78,7 @@ "db": 1 }, "workers": 5, - "concurrency": 5, + "concurrency": 2, "enableScheduledJobs": true, "defaultJobOptions": { "attempts": 3, diff --git a/apps/stock/data-ingestion/node_modules b/apps/stock/data-ingestion/node_modules new file mode 120000 index 0000000..6c57164 --- /dev/null +++ b/apps/stock/data-ingestion/node_modules @@ -0,0 +1 @@ +../../../node_modules \ No newline at end of file diff --git a/apps/stock/data-ingestion/package.json b/apps/stock/data-ingestion/package.json index 3fd76d4..9cb6fdd 100644 --- a/apps/stock/data-ingestion/package.json +++ b/apps/stock/data-ingestion/package.json @@ -27,7 +27,8 @@ "@stock-bot/browser": "*", "@stock-bot/proxy": "*", "hono": "^4.0.0", - "pako": "^2.1.0" + "pako": "^2.1.0", + "cheerio": "^1.0.0" }, "devDependencies": { "typescript": "^5.0.0" diff --git a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts index 213667c..8314504 100644 --- a/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts @@ -33,6 +33,7 @@ import { crawlIntradayData, scheduleIntradayCrawls } from './actions/intraday-cr import { createQMOperationRegistry } from './shared/operation-provider'; @Handler('qm') +@Disabled() // Disable by default, enable specific operations as needed export class QMHandler extends BaseHandler { public operationRegistry: OperationRegistry; diff --git a/apps/stock/data-ingestion/src/handlers/te/actions/fetch-countries.action.ts b/apps/stock/data-ingestion/src/handlers/te/actions/fetch-countries.action.ts new file mode 100644 index 0000000..dd08b7a --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/te/actions/fetch-countries.action.ts @@ -0,0 +1,227 @@ +import { getRandomUserAgent } from '@stock-bot/utils'; +import * as cheerio from 'cheerio'; +import { TE_CONFIG } from '../shared/config'; +import type { TeCountry } from '../shared/types'; +import type { TeHandler } from '../te.handler'; + +export async function fetchCountries(this: TeHandler): Promise { + const { logger, mongodb } = this; + + try { + // 1. Fetch the HTML page + const reqInfo = { + proxy: this.proxy.getProxy(), + headers: { + 'User-Agent': getRandomUserAgent(), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br', + }, + } + const response = await fetch(TE_CONFIG.COUNTRIES_URL, reqInfo); + + logger.debug('Response status:', { + status: response.status, + statusText: response.statusText, + url: response.url + }); + + if (!response.ok) { + throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`); + } + + const html = await response.text(); + + logger.info('Fetched HTML length:', { length: html.length }); + + + // 2. Parse HTML to extract country data + const $ = cheerio.load(html); + const countries: TeCountry[] = []; + + // Look for country links - they typically have a pattern like /country-name + // Trading Economics groups countries by region in the page + $('.list-group-item, a[href^="/"]').each((_, element) => { + const $el = $(element); + + // Try to extract country information + let name: string | undefined; + let url: string | undefined; + let region: string | undefined; + + // Check if it's a direct link + if ($el.is('a')) { + const href = $el.attr('href'); + const text = $el.text().trim(); + console.log(href) + // Filter for country URLs (they don't contain special paths like /indicators, /calendar, etc.) + if (href && href.startsWith('/') && !href.includes('/') && text) { + name = text; + url = href; + } + } else { + // Check for links within table rows + const $link = $el.find('a[href^="/"]').first(); + if ($link.length) { + const href = $link.attr('href'); + const text = $link.text().trim(); + + if (href && text && !href.includes('/indicators') && !href.includes('/calendar')) { + name = text; + url = href; + + // Try to get region from parent elements + const $regionHeader = $el.closest('.region-section, .country-group').find('h2, h3, .region-title').first(); + if ($regionHeader.length) { + region = $regionHeader.text().trim(); + } + } + } + } + + // Add to countries array if we found valid data + if (name && url) { + // Extract country code from URL if possible (e.g., /united-states -> US) + const code = extractCountryCode(url, name); + + countries.push({ + name, + code, + url: `https://tradingeconomics.com${url}`, + region, + updated_at: new Date(), + }); + } + }); + + // Remove duplicates based on name + const uniqueCountries = Array.from( + new Map(countries.map(c => [c.name, c])).values() + ); + + if (uniqueCountries.length === 0) { + throw new Error('No countries found in HTML'); + } + + logger.info('Extracted countries from HTML', { + count: uniqueCountries.length, + byRegion: groupCountriesByRegion(uniqueCountries), + }); + + // 3. Save to MongoDB + try { + console.log( uniqueCountries) + if (uniqueCountries.length > 0) { + const result = await mongodb?.batchUpsert('teCountries', uniqueCountries, ['code']); + logger.info('Countries saved to MongoDB', { + matched: result.matchedCount, + modified: result.modifiedCount, + upserted: result.upsertedCount, + }); + } + } catch (dbError) { + logger.error('Failed to save countries to MongoDB', { error: dbError }); + throw dbError; + } + + return uniqueCountries; + } catch (error) { + logger.error('Failed to fetch Trading Economics countries', { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); + return null; + } +} + +function extractCountryCode(url: string, name: string): string | undefined { + // Common country code mappings + const countryCodeMap: Record = { + 'united-states': 'US', + 'united-kingdom': 'GB', + 'euro-area': 'EU', + 'china': 'CN', + 'japan': 'JP', + 'germany': 'DE', + 'france': 'FR', + 'italy': 'IT', + 'spain': 'ES', + 'canada': 'CA', + 'australia': 'AU', + 'south-korea': 'KR', + 'india': 'IN', + 'brazil': 'BR', + 'russia': 'RU', + 'mexico': 'MX', + 'indonesia': 'ID', + 'netherlands': 'NL', + 'saudi-arabia': 'SA', + 'turkey': 'TR', + 'switzerland': 'CH', + 'poland': 'PL', + 'sweden': 'SE', + 'belgium': 'BE', + 'argentina': 'AR', + 'ireland': 'IE', + 'austria': 'AT', + 'norway': 'NO', + 'israel': 'IL', + 'singapore': 'SG', + 'denmark': 'DK', + 'egypt': 'EG', + 'philippines': 'PH', + 'finland': 'FI', + 'chile': 'CL', + 'pakistan': 'PK', + 'romania': 'RO', + 'new-zealand': 'NZ', + 'greece': 'GR', + 'iraq': 'IQ', + 'portugal': 'PT', + 'czech-republic': 'CZ', + 'vietnam': 'VN', + 'peru': 'PE', + 'colombia': 'CO', + 'malaysia': 'MY', + 'ukraine': 'UA', + 'hungary': 'HU', + 'kuwait': 'KW', + 'morocco': 'MA', + 'slovakia': 'SK', + 'kenya': 'KE', + 'puerto-rico': 'PR', + 'ecuador': 'EC', + 'ethiopia': 'ET', + 'dominican-republic': 'DO', + 'luxembourg': 'LU', + 'oman': 'OM', + 'guatemala': 'GT', + 'bulgaria': 'BG', + 'ghana': 'GH', + 'tanzania': 'TZ', + 'turkmenistan': 'TM', + 'croatia': 'HR', + 'costa-rica': 'CR', + 'lebanon': 'LB', + 'slovenia': 'SI', + 'lithuania': 'LT', + 'serbia': 'RS', + 'panama': 'PA', + }; + + // Clean URL to get country slug + const slug = url.replace(/^\//, '').toLowerCase(); + + return countryCodeMap[slug]; +} + +function groupCountriesByRegion(countries: TeCountry[]): Record { + const groups: Record = {}; + + for (const country of countries) { + const region = country.region || 'Unknown'; + groups[region] = (groups[region] || 0) + 1; + } + + return groups; +} \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/te/actions/index.ts b/apps/stock/data-ingestion/src/handlers/te/actions/index.ts index 0acc02f..018c6ed 100644 --- a/apps/stock/data-ingestion/src/handlers/te/actions/index.ts +++ b/apps/stock/data-ingestion/src/handlers/te/actions/index.ts @@ -1,2 +1,4 @@ // Export all action functions here -// export * from './example.action'; \ No newline at end of file +export * from './fetch-countries.action'; +export * from './spider.action'; + diff --git a/apps/stock/data-ingestion/src/handlers/te/actions/spider.action.ts b/apps/stock/data-ingestion/src/handlers/te/actions/spider.action.ts new file mode 100644 index 0000000..d5f27c6 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/te/actions/spider.action.ts @@ -0,0 +1,140 @@ +import { getRandomUserAgent } from '@stock-bot/utils'; +import * as cheerio from 'cheerio'; +import { TE_CONFIG } from '../shared/config'; +import type { TeHandler } from '../te.handler'; + +export async function spiderUrl(this: TeHandler, payload: { url: string }): Promise { + const { logger, mongodb } = this; + const reqUrl = payload && payload.url ? TE_CONFIG.MAIN_URL + payload.url : TE_CONFIG.MAIN_URL; + this.logger.info(`Spiderring URL: ${reqUrl}`, {reqUrl}); + + const mongoRecord = await mongodb?.findOne('teUrls', { url: payload?.url || '/' }); + if(payload && payload.url && mongoRecord && mongoRecord.lastCrawled && mongoRecord.lastCrawled.getTime() > Date.now() - 7* 24 * 60 * 60 * 1000) { + this.logger.info(`Skipping URL ${reqUrl} as it was already crawled in the last 24 hours`); + return null; // Skip if already crawled in the last 24 hours + } + + if (!payload) { + const oneDayAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + const records = await mongodb?.find('teUrls', { + $or: [ + { lastCrawled: { $lt: oneDayAgo } }, // Crawled more than 24 hours ago + { lastCrawled: { $exists: false } } // Never crawled + ] + }); + this.logger.info(`Found ${records?.length || 0} records to process`); + for (const record of records || []) { + await this.scheduleOperation('te-spider', { + url: record.url, + }, { + jobId: `te-spider-${record.url}`, + priority: 5, // Lower priority than financial data + }); + } + } + + try { + // 1. Fetch the HTML page + const reqInfo = { + proxy: 'http://5.79.66.2:13010',//this.proxy.getProxy(), + headers: { + 'User-Agent': getRandomUserAgent(), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br', + }, + } + const response = await fetch(reqUrl, reqInfo); + + logger.debug('Response status:', { + status: response.status, + statusText: response.statusText, + url: response.url + }); + + if (!response.ok) { + throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`); + } + + const html = await response.text(); + + + let match = html.match(/TESymbol = '([^']+)'/); + const teSymbol = match ? match[1] : undefined; + match = html.match(/;TELastUpdate = '([^']+)'/); + const teLastUpdate = match ? match[1] : undefined; + match = html.match(/; var TEChartsDatasource = '([^']+)'/); + const teChartUrl = match ? match[1] : undefined; + match = html.match(/; var TEChartsToken = '([^']+)'/); + const teChartToken = match ? match[1] : undefined; + + console.log(teSymbol, teLastUpdate, teChartUrl, teChartToken); + + const $ = cheerio.load(html); + const urls: string[] = []; + + $('.list-group-item, a[href^="/"]').each((_, element) => { + const $el = $(element); + let url: string | undefined; + if ($el.is('a')) { + const href = $el.attr('href'); + if (href && href.startsWith('/') && !href.includes('.aspx')) { + url = href; + } + } + + if (url && urls.indexOf(url) === -1) { + urls.push(url); + } + }); + + if (urls.length === 0) { + throw new Error('No urls found in HTML'); + } + + // 3. Save to MongoDB + try { + if (urls.length > 0) { + const urlMap: {url: string, lastCrawled?: Date, teSymbol? : string, teLastUpdate? : string, teChartUrl? : string, teChartToken? : string}[] = urls.map(url => ({url})); + if( payload && payload.url) { + urlMap.push({ + url: payload.url, + lastCrawled: new Date(), + teSymbol, + teLastUpdate, + teChartUrl, + teChartToken,}) + }else { + urlMap.push({url: '/', lastCrawled: new Date()}) + } + + const result = await mongodb?.batchUpsert('teUrls', urlMap, ['url']); + logger.info('TE URLs saved to MongoDB', { + matched: result.matchedCount, + modified: result.modifiedCount, + upserted: result.upsertedCount, + }); + } + } catch (dbError) { + logger.error('Failed to save urls to MongoDB', { error: dbError }); + throw dbError; + } + + for (const url of urls) { + this.scheduleOperation('te-spider', { + url: url, + }, { + jobId: `te-spider-${url}`, + priority: 5, // Lower priority than financial data + }) + } + + return urls; + } catch (error) { + logger.error(`Failed to fetch Trading Economics URLs ${reqUrl}`, { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }); + return null; + } +} \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/te/shared/config.ts b/apps/stock/data-ingestion/src/handlers/te/shared/config.ts index f49b9bb..fe7c630 100644 --- a/apps/stock/data-ingestion/src/handlers/te/shared/config.ts +++ b/apps/stock/data-ingestion/src/handlers/te/shared/config.ts @@ -1,6 +1,9 @@ export const TE_CONFIG = { // Add configuration constants here API_URL: 'https://api.example.com', + MAIN_URL: 'https://tradingeconomics.com', + COUNTRIES_URL: 'https://tradingeconomics.com/countries', REQUEST_TIMEOUT: 30000, + USER_AGENT: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', KEY: 'tradingeconomics-charts-core-api-key', }; \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/te/shared/types.ts b/apps/stock/data-ingestion/src/handlers/te/shared/types.ts index cee4327..52f9b7f 100644 --- a/apps/stock/data-ingestion/src/handlers/te/shared/types.ts +++ b/apps/stock/data-ingestion/src/handlers/te/shared/types.ts @@ -10,4 +10,13 @@ export interface TeResponse { data: TeData[]; status: string; timestamp: Date; +} + +export interface TeCountry { + name: string; + code?: string; + region?: string; + url?: string; + updated_at: Date; + created_at?: Date; } \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/te/te.handler.ts b/apps/stock/data-ingestion/src/handlers/te/te.handler.ts index 8043516..d5fe07c 100644 --- a/apps/stock/data-ingestion/src/handlers/te/te.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/te/te.handler.ts @@ -1,29 +1,31 @@ import { BaseHandler, + Disabled, Handler, - Operation, - ScheduledOperation, + ScheduledOperation } from '@stock-bot/handlers'; +import type { DataIngestionServices } from '../../types'; +import { fetchCountries, spiderUrl } from './actions'; @Handler('te') -export class TeHandler extends BaseHandler { +@Disabled() +export class TeHandler extends BaseHandler { constructor(services: any) { super(services); } - @Operation('example-operation') - async exampleOperation(): Promise { - this.logger.info('TE handler example operation executed'); - return { success: true }; - } - - @ScheduledOperation('te-scheduled-job', '0 0 * * *', { + @ScheduledOperation('te-countries', '0 0 * * 0', { priority: 5, - description: 'Daily TE handler scheduled job', - immediately: false, + description: 'Fetch and update Trading Economics countries data', + immediately: true, }) - async scheduledJob(): Promise { - this.logger.info('TE handler scheduled job executed'); - return this.exampleOperation(); - } + @Disabled() + fetchCountries = fetchCountries; + + @ScheduledOperation('te-spider', '0 0 * * 0', { + priority: 5, + description: 'Fetch and update Trading Economics countries data', + immediately: true, + }) + spiderUrlSchedule = spiderUrl; } \ No newline at end of file diff --git a/apps/stock/tsconfig.json b/apps/stock/tsconfig.json index d609f08..6be8e35 100644 --- a/apps/stock/tsconfig.json +++ b/apps/stock/tsconfig.json @@ -14,7 +14,6 @@ { "path": "./data-pipeline" }, { "path": "./web-api" }, { "path": "./web-app" }, - { "path": "./engine"}, { "path": "./orchestrator"} ] } \ No newline at end of file diff --git a/apps/stock/web-app/tsconfig.json b/apps/stock/web-app/tsconfig.json index 145593e..0764cd9 100644 --- a/apps/stock/web-app/tsconfig.json +++ b/apps/stock/web-app/tsconfig.json @@ -7,10 +7,11 @@ "module": "ESNext", "skipLibCheck": true, "moduleResolution": "bundler", - "allowImportingTsExtensions": true, + // "allowImportingTsExtensions": true, "resolveJsonModule": true, "isolatedModules": true, - "noEmit": true, + // "noEmit": true, + "composite": true, "jsx": "react-jsx", "strict": true, "noUnusedLocals": true, diff --git a/bun.lock b/bun.lock index 213d4d8..ae9f5d7 100644 --- a/bun.lock +++ b/bun.lock @@ -77,6 +77,7 @@ "@stock-bot/shutdown": "*", "@stock-bot/stock-config": "*", "@stock-bot/utils": "*", + "cheerio": "^1.0.0", "hono": "^4.0.0", "pako": "^2.1.0", }, @@ -1171,6 +1172,8 @@ "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="], + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + "bowser": ["bowser@2.11.0", "", {}, "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA=="], "bplist-parser": ["bplist-parser@0.2.0", "", { "dependencies": { "big-integer": "^1.6.44" } }, "sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw=="], @@ -1223,6 +1226,10 @@ "chalk": ["chalk@5.4.1", "", {}, "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="], + "cheerio": ["cheerio@1.1.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.0", "htmlparser2": "^10.0.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.10.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-+0hMx9eYhJvWbgpKV9hN7jg0JcwydpopZE4hgi+KvQtByZXPp04NiCWU0LzcAbP63abZckIHkTQaXVF52mX3xQ=="], + + "cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="], + "chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], "chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], @@ -1283,6 +1290,10 @@ "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], + + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + "cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="], "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], @@ -1375,6 +1386,14 @@ "dom-helpers": ["dom-helpers@5.2.1", "", { "dependencies": { "@babel/runtime": "^7.8.7", "csstype": "^3.0.2" } }, "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA=="], + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], + + "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], + + "domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="], + + "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="], @@ -1387,6 +1406,8 @@ "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], "engine.io": ["engine.io@6.6.4", "", { "dependencies": { "@types/cors": "^2.8.12", "@types/node": ">=10.0.0", "accepts": "~1.3.4", "base64id": "2.0.0", "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.3.1", "engine.io-parser": "~5.2.1", "ws": "~8.17.1" } }, "sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g=="], @@ -1395,6 +1416,8 @@ "engine.io-parser": ["engine.io-parser@5.2.3", "", {}, "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q=="], + "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "es-abstract": ["es-abstract@1.24.0", "", { "dependencies": { "array-buffer-byte-length": "^1.0.2", "arraybuffer.prototype.slice": "^1.0.4", "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", "call-bound": "^1.0.4", "data-view-buffer": "^1.0.2", "data-view-byte-length": "^1.0.2", "data-view-byte-offset": "^1.0.1", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "es-set-tostringtag": "^2.1.0", "es-to-primitive": "^1.3.0", "function.prototype.name": "^1.1.8", "get-intrinsic": "^1.3.0", "get-proto": "^1.0.1", "get-symbol-description": "^1.1.0", "globalthis": "^1.0.4", "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", "has-proto": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "internal-slot": "^1.1.0", "is-array-buffer": "^3.0.5", "is-callable": "^1.2.7", "is-data-view": "^1.0.2", "is-negative-zero": "^2.0.3", "is-regex": "^1.2.1", "is-set": "^2.0.3", "is-shared-array-buffer": "^1.0.4", "is-string": "^1.1.1", "is-typed-array": "^1.1.15", "is-weakref": "^1.1.1", "math-intrinsics": "^1.1.0", "object-inspect": "^1.13.4", "object-keys": "^1.1.1", "object.assign": "^4.1.7", "own-keys": "^1.0.1", "regexp.prototype.flags": "^1.5.4", "safe-array-concat": "^1.1.3", "safe-push-apply": "^1.0.0", "safe-regex-test": "^1.1.0", "set-proto": "^1.0.0", "stop-iteration-iterator": "^1.1.0", "string.prototype.trim": "^1.2.10", "string.prototype.trimend": "^1.0.9", "string.prototype.trimstart": "^1.0.8", "typed-array-buffer": "^1.0.3", "typed-array-byte-length": "^1.0.3", "typed-array-byte-offset": "^1.0.4", "typed-array-length": "^1.0.7", "unbox-primitive": "^1.1.0", "which-typed-array": "^1.1.19" } }, "sha512-WSzPgsdLtTcQwm4CROfS5ju2Wa1QQcVeT37jFjYzdFz1r9ahadC8B8/a4qxJxM+09F18iumCdRmlr96ZYkQvEg=="], "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], @@ -1621,6 +1644,8 @@ "hono": ["hono@4.8.4", "", {}, "sha512-KOIBp1+iUs0HrKztM4EHiB2UtzZDTBihDtOF5K6+WaJjCPeaW4Q92R8j63jOhvJI5+tZSMuKD9REVEXXY9illg=="], + "htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="], + "http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="], "http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="], @@ -1931,6 +1956,8 @@ "npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="], + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + "numeral": ["numeral@2.0.6", "", {}, "sha512-qaKRmtYPZ5qdw4jWJD6bxEf1FJEqllJrwxCLIm0sQU/A7v2/czigzOb+C2uSiFsa9lBUzeH7M1oK+Q+OLxL3kA=="], "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], @@ -1995,6 +2022,12 @@ "parent-module": ["parent-module@1.0.1", "", { "dependencies": { "callsites": "^3.0.0" } }, "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g=="], + "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + + "parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="], + + "parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="], + "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], "pascal-case": ["pascal-case@3.1.2", "", { "dependencies": { "no-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g=="], @@ -2445,7 +2478,7 @@ "unbox-primitive": ["unbox-primitive@1.1.0", "", { "dependencies": { "call-bound": "^1.0.3", "has-bigints": "^1.0.2", "has-symbols": "^1.1.0", "which-boxed-primitive": "^1.1.1" } }, "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw=="], - "undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], + "undici": ["undici@7.11.0", "", {}, "sha512-heTSIac3iLhsmZhUCjyS3JQEkZELateufzZuBaVM5RHXdSBMb1LPMQf5x+FH7qjsZYDP0ttAc3nnVpUB+wYbOg=="], "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], @@ -2475,6 +2508,10 @@ "webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="], + "whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="], + + "whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="], + "whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="], "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], @@ -2659,6 +2696,8 @@ "got/type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="], + "htmlparser2/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + "is-wsl/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="], "joi/@hapi/hoek": ["@hapi/hoek@9.3.0", "", {}, "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ=="], @@ -2687,6 +2726,8 @@ "os-dns-native/node-addon-api": ["node-addon-api@4.3.0", "", {}, "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="], + "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + "path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], "pg-mem/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], @@ -2735,6 +2776,8 @@ "tailwindcss/object-hash": ["object-hash@3.0.0", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="], + "testcontainers/undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], + "ts-unused-exports/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], "type-is/mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="], diff --git a/docs/data-sources.md b/docs/data-sources.md index 658c492..8bb20db 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -40,4 +40,11 @@ ## Additional Resources - **Marketstack**: https://marketstack.com/ -- **Alpaca**: https://alpaca.markets/ - Future hopefully or US connection \ No newline at end of file +- **Alpaca**: https://alpaca.markets/ - Future hopefully or US connection +- **MacroMicro**: https://en.macromicro.me/ +- **Norgate**: https://norgatedata.com/stockmarketpackages.php#castocks +- **FINRA**: https://www.finra.org +https://otce.finra.org/otce/EquityShortInterest/staticArchives?startDate=2014-01-01&endDate=2024-11-03&dateDisplay=November%203,%202018 +https://www.finra.org/finra-data/browse-catalog/equity-short-interest/files +https://www.cboe.com/us/equities/market_statistics/short_interest/?year=2025&month=06&mkt=bzx +- **MORNINGSTAR**: https://finra-markets.morningstar.com/MarketData/EquityOptions/detail.jsp?query=14:0P0001BUHA PRICES/SHORT INTEREST \ No newline at end of file diff --git a/docs/todo.md b/docs/todo.md new file mode 100644 index 0000000..5f7b857 --- /dev/null +++ b/docs/todo.md @@ -0,0 +1,8 @@ +Data Ingestion +- QM scrape all timeframes add timeframe column to collection +- EOD Fundementals / Prices / Intraday / Delisted ++ +- DataBento - 125 credit - use for symbols / second / minute intraday +- Work on TE Indicators +- CEO - watch all and save them to posts. do it every minute +- Test our handler specific rate limits +- Fix up handler worker counts diff --git a/libs/core/handlers/src/base/BaseHandler.ts b/libs/core/handlers/src/base/BaseHandler.ts index 898589c..0dba5c7 100644 --- a/libs/core/handlers/src/base/BaseHandler.ts +++ b/libs/core/handlers/src/base/BaseHandler.ts @@ -1,4 +1,3 @@ -import type { Collection } from 'mongodb'; import { createNamespacedCache } from '@stock-bot/cache'; import { getLogger } from '@stock-bot/logger'; import type { @@ -11,6 +10,7 @@ import type { ServiceTypes, } from '@stock-bot/types'; import { fetch } from '@stock-bot/utils'; +import type { Collection } from 'mongodb'; // Handler registry is now injected, not imported import { createJobHandler } from '../utils/create-job-handler'; @@ -18,6 +18,7 @@ import { createJobHandler } from '../utils/create-job-handler'; * Job scheduling options */ export interface JobScheduleOptions { + jobId?: string; // Optional job ID for custom identification delay?: number; priority?: number; attempts?: number; diff --git a/tetest.ts b/tetest.ts index 50b6c1e..48d7183 100644 --- a/tetest.ts +++ b/tetest.ts @@ -1,6 +1,8 @@ const pako = require('pako'); -const value = "a/lpZGluZ2VjbBP9IAahQyHll2u4zSdk+eDEshxAXrKZAKBYXiuIYCAnNNRpnrEIBW5XO5a6x/6q21A6RLsYMsZ3law1yXl74NG7eMMEtJHauEYtK4TC8XsM24SQF8VZlYfF1mIU3dx2gJkGNBULa9wo3Dd4YOyA9mJfz5t+oPHPy80BQ5AAn/0II58Ndl+cyATaep28EaPY5NRd9jM+Z8U9scqGSACO3+3XbzEGaIqlY6tJV3Ajt4QYdsf7vVd5HiWzSQcLJGc45F/uT8/A9A9Qb8ABjlHkJva/HuDv9TezGyxKHvW/2/w0oljydYbJSCRKcAHBntf/QEBIPaARhqR57rwrCvp8lyHSyAJxT2I3UFMVwyuCdj9bwYBpnY0SrCNHbU+Po+KcrW2q3SWG988c1rcCY6XEMdoCLSgF/dry0ZHMQVrehI5d7wjATyvFnn9NM3htehPuqqckjgRh+fs/jSU5A5NyUdStfXBtZ2U=" + +const value = "a/lpZGluZ2VjbAM1pgcHtCCefSKpgAeH6QtP0aHicG8/E3R+LXZixUTbVGiIXr8M4eCCDCL834tYqtuIDcXYS1QZCKHHNMoXO8X+qlYyeas0uDlKeJPzqaonV4UIeZ6Xl0yU84SAbzuXJYyTBAebk1o1gV6dmouchDgdjAcMt75NiQFeGhMPW/D8zjdw1FvG6eUWq6YxKZIBlK7Z+v6MrfS+FltPy43ImpPt0t6CXdZW3ki9z+7+S475zI4ckhSan5yCtAo6jrrNkphC3JBpGtcZovpSkJkPA4yHn6LTm6bEyIm4shCc1JqYk/qFmwzerHDMhuKPPsYKtYcmzedlx5/6adh/bnyQn6Gc1BJddpUBmCPJkKPM9OJelArjhpVYaXXe+7Xkjc3V5XbKLlMVn9HQ5ggh3VWeyqoeTy2e1Ayw7WC00qoPjeHNjS69w5bYMYQvUIHD+t/WsdlavkBvPtjyEytQa/8vzVOblwjy2626ldpGWBzQotgGSx8pKwN3YlmDMcam2kJQ6wZoXwy/XsQE9SqmZQ1CJROVwYE2L/BvL06jWxT1w6LRKNwbH6IP0JMGqyqTXQjZ1TOZRrQOifUkTm5/9d++VCAzM2KsqH3b/os/yPk6Ubt5x6AJW7Ng4mQB3m+R+sc1fwfZfkLwHQEld1HLrmMiNoFgvBeL/y2AOAn7lLZ3zCBvL4MHChEcDZx7LppmG4wVu9inrH+dJqLgREyZF8ONRW89+Q4wf/qszwqMi5yDWyCZF/MT7LHRrLvSokxddwH+XrnuMXygH/5gH0WrJNQCYuSTSzAyRUbI+rwyETTkOXbGtE2r7AnHd/XfKyCTtSf9rJL9u9obwLVWW65+ocVG0z/K4jZUdWNRVAb3iqDy2pj5TwqHJxIyBVPfcX0L5fJqzj9nlyrNIr5zXr5gPhtHz//vsu/nCwqbHarK950T4hflFmFrJUPyaXwB/1eT7vlYB6YrUHV/wkvVzIQ2WSTqYy5OeEffhKyHTJWkaOuNCKKPovqrtjYCwIiElaGG99HeQP/tZ4yWfeIIuN8deiBlEa1OMRcgqhRxEXtY1mR3MDBAs8M1M7WRT0dGcnvJPlL5ihxZal2nY7qN4yb6/MNdFJ3xwk/5IEG8W4J4DqioZ7IJxFkOKLixxAQiyS3hOqD9NZh0KPg0OkUkPiMUul0mTRYnNgPf79grkM0w+BvUfaP1274L4e01tA+CmrfYSvkfHk8wNTvI+0XI+O8XFYMYX9ewa1L4yWHpVZj8HlD0/oNLp/JBHr7n+AMXAGZfvtPy6YXT4NrYBO1IAsRr0MJZThDd14VTzbkMh4pVwn467fGKLYLXOVgj8pia9YY54AE28YZ7ox5/G2Cy6vbnZYnJdUF0mBBvijkhzNchHVtzyw7BMehKMF5TAEN6ndcH5fW7KtV4IKyv/ZRvxOZcgKDBRx/LYbFri8uGs03sXvBC4CmOpGsHmtaBCvZ1G8fwxK/jHt+ZoxlNfsbaS9mxiRdVDdlFRmobRTzA5+W4c+1HLvlTNMRIW0auXQWGjSfYeFQfpICzWAiTKw1w0gcC7YLAvot+ivrinZFORkSyCvFtETc/gok/ajprIF97eoeEY+2yEqaeTOdo3G04wyOz27EOjvy8XUst4Na3d8j+e0LPTh93ZaCxjAYUdv5DwjkktyM8nQyyTmfiXTJlUUKWuvbedG9lNnn/9Oq5T+4cGSm9tfsMz8tK8VD/68FIiu2pq/o2h6yzV8RX0PgJu6q3Oub2143YgzQy7Xp+fCslI3BJ+kbEEWGgtxgV6WIAbeMHgJvyizjfZsk/CHzModCYWF3KnXzWDbwe8FL+rhC0GGuvZbleThSx9IYw0/MI5YQs3+1fAcUkkgN01F1/qxCkAF3q2fXq2Ctbm8Uim1OBOsMg3zi1sDXbdUMeTQs7MislQdMiX3VLVHqHGm61jynPf5X8OTLkFl6LGy2HHj8LYGg74bLcPIoyxONilLHWe48FtJMaJdqmCKK+v2WrhQ/uhvyIEf2Lrij/jd8JXjhg1Q4vugu/DXLCzETrFxJU+XbFyifAa0kQ/p1X0jfH/8f6vkO7zyfUE1d1N46iudYeSRpZNCLZlIZZY/EfVj4XAEM46M5E1xlyDLrWXvxf6ewF8oLMBarSDqzg2MbgH2q0SPIN1jfE2RYpJDzGlp1hZ8YTEESlpu+1ofbJqSnRfu6A3YtqXJ9aImkqpAo4WzVZoN/Ppn+DpsvMRO37kNmqPapC1+MBRP+cR0nYKPuNx92NrrgY4WBTGR0F4tnCC4bxEFimd0kJgTeksjRC0htizYQYNGLKrSYG3Nt9NFkrSBn6c+HYgAH3u86l2U1H+W7JmDW3h5kkt9L2n5Ml+OtSwit0uP95BGtgD/ZVWx13K9WkqF7l7SckA3kTtGN/n3AlPnzklR0JZpItGuB2S6Joe8oBWltEMRQjDS7Li688Lv0Q8fVHKvRt9ct2ZgibPmu4vbAWqhbv9tvVP2A5GZk9MIJEOG10owdhPcayBiK08DhLyLxY2/C7xKEj2XlHB/I13beb8u+oGT9umT1H29jN1qyKEAxRPapPT5TzEAATVi5kPPvWvxw5/x6Xg6koQ4WkN1gIWByOuX/hS+Y+jrfLE8Y5FOOeDguqigQXfgULWi44VcpF9DLFVMbT9uQlRr4Kal2GvDtFxb1PsdYQRN9KrRmCMQU+ZFFyP0CF1EEZ73hQ2KOXdT7rpLW8Q+z26dVbmyipAwXcxDxOFwSLEGJlCYgaB0/TQtj3b/aa9qjFM8iZH++pch7Jb8RY4TOfyviaOWsUJXAvwowQqB6lS5UQXj6wqKY9sq/D4iQdgqzXLjyaD7S3ZSMPWxE28qyP4o6Blrnhnv6UyvbFasIy/6/Dej3FNS6fFaU+6Lo3nuCjXczeASIDa5JqPQXqJcaW5ZcOrBIFizTkgHCQV+eBNllfufva7Gzaei51Hy98mCWwVxuNXuGql3EyTKF8f6vloesS6rjMGdsbUiZVWSWgwTCUTbzT9i+i+PK70gm+9nSJhRcyqOYC3nC9xuF6UImmGyUf4ocCSowW+s3GKzoV+HQ67mAf9iM5qq8NaH12SznH3CnZThUyOqBQ/wpRMF8sBK7gvAh3NVQ/UvXfOyPEh1QogRRARzT2C1X2wKc7yqVMKNOuJjy+wGi/1+1otom3BVCeqZwCmHSpXlTbrXEQzt0FSObtyDr2yGDADOrvyDL59/81hatpfxDcZlABQtXJc7NhtdiCXwRH4f3zJpGhkaTY1Dw6RrUY4TeePOCvBYa9aQegzaLZBYFUfFV8el+CpksZYtnwKSrYCgeBErXb/ILnmwj+gTmYaqWzeCSDy4gvjVan+K7Ndp06ziZFMuZ9uad9p0G1hk5p5WjQnCU2qV6V7HZooODkxwNRV+75SxOrhGK6hmz7fO9AXSh0DnudJLEI3Opcb3ABB1Uelbd5zwjt7qbS0Ko9wuYJbnLd4E8RJnbjcYddPgtLa6HSVnizFXGIbE8z+PKHNs6iq/XK1UBPhFswZ60bbPl56ucquQfmzqKxy0gepgNVPyypmsRS1mLed+K2g7NGUCRbMwilWmAwLF644S9hEcHQ1Y8MdSPcj+umluyEWGfeXrRVrRpkmzfAjckbzLc/EHJuBBMEsFmX2KZ7Q4o63Tk4vGrh1Jm7PIorHoW2LXKX24ahvI773rGwgC80WURYumc4voF1iYfKOamEwUgJQyHPvTvKUsAO37mF/Mi509CNlEU7FLtbTu8YTqzT2z6XWZcvVmFXVI2FO7dGxPKES3Rhe7C3n5m03JxuHaWyEN53326liRK8UZ0bX9Y8tQc8PDQBAGiSM9qmECOMzHaYSqEE3xfntRtMADQOxLfj6eDfcTABPTPkfPseQ1v/zBDZxvCmd91BOx+Asda6HHCGxkCC+j+7bzVk9yDkR89Zk8Z7D9yg4jJ54e3DSvZeTa5ta/gbpzaPGnkM158IqSBLKGB2HzWjUhe1VfSh0ruuFkbUulO0apI00iia2s8yCyVdf5tb8DHieW3HcXkWeWkEHq4TNdzb12cDRpou3UcUlPNU1aC+xf2F8pcID7WxyZADLOuqWD1/+9TFtvvOG8h5HyXOPDcEeUlp7jUNazvel6pAYbqbpzah24GftKNZ4cdy3TrWt31efsJtzARHKTDqFue1nT8f3gnU0aje3jTW9z++HH/pt3m4mMfDVnlt0lbtocIPQTOfuVG6nEQk7ltTtnOCwyeSMesX2Mjw2ilZoqW6ULWvyFfGpqOjAVD2Nqp9Cnb/W6hOQJ6su6+5uOUDi7+s1oEVtLVHfASfjVDawwzhcSbNkbXmRhSmkr7m0JiBFfqfo/d8GtLDILpWgAcJ0ivqPyL4Tf2ruPHRmE2PgeKf585jY24iXlrD/cIwrH6A/j4EKg5HtfqaGsXMpgVaPuv3mfiS1qGW0z2gtsF/0FHxGMSb6YwF6bGLyN60mR9qgGGgAbwH3NgtMA90D3sI0pb0JUAMjmff/uVpPPi7BqWaaBvLwceSstkZpc3ZKYzIWLPLlpGs77kX1X8kz9xPMhsh7ftlfses0Pygpw3AObDJRHqCQFO4tJxxor6IndwRhZ9VEtjDyoimu5aSoFqeR7H211K8LTMtn04lowOVmYaF2o2UipyeF8CTgpuAhYCAld8edIITGhoYCZHY/04draSRPbFMGlMbDowi9apWO4UOy/7JIJsz0A4amBMaS9KUgZBuCwwX3rOu+AOWzLJiF+eKKAcf+6VCnrLUgrEB8nfc71jstbqF0+T62ryfqsqGiILDlxeCCItNAzDZmYAfkZeRhmRcs+Cgu4KZ05YxkZ6RtuB2DmvUWbyLmo6f/9YXHtkYFBcSk3NITHF9d//MsCwChLSlwsBB7/YC3Bnvp0/OrVND6+6cGleczlReL0M8Br/swxT6DKGvPSWF7bYTlZNG0DIiUb1JZ2U=" +const URL = 'https://d3ii0wo49og5mi.cloudfront.net/economics/szlttr?span=max&v=20250619074600&key=20240229:nazare' const key = 'tradingeconomics-charts-core-api-key' @@ -19,5 +21,15 @@ const testFunc = function(e, k) { return pako.inflate(n, {to :'string'}); } -const result = testFunc(value, key); -console.log(JSON.parse(result.toString())); +fetch(URL, { + proxy: 'http://5.79.66.2:13010', + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome' + } +}).then(async response => { + const resp = await response.text(); + const result = testFunc(resp.replaceAll('"', ""), key); + // console.log(result.toString()); + console.log(JSON.stringify(JSON.parse(result.toString()))); +}) +