work on spider for te
This commit is contained in:
parent
505565a09e
commit
95b1381480
17 changed files with 485 additions and 28 deletions
|
|
@ -78,7 +78,7 @@
|
|||
"db": 1
|
||||
},
|
||||
"workers": 5,
|
||||
"concurrency": 5,
|
||||
"concurrency": 2,
|
||||
"enableScheduledJobs": true,
|
||||
"defaultJobOptions": {
|
||||
"attempts": 3,
|
||||
|
|
|
|||
1
apps/stock/data-ingestion/node_modules
Symbolic link
1
apps/stock/data-ingestion/node_modules
Symbolic link
|
|
@ -0,0 +1 @@
|
|||
../../../node_modules
|
||||
|
|
@ -27,7 +27,8 @@
|
|||
"@stock-bot/browser": "*",
|
||||
"@stock-bot/proxy": "*",
|
||||
"hono": "^4.0.0",
|
||||
"pako": "^2.1.0"
|
||||
"pako": "^2.1.0",
|
||||
"cheerio": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.0.0"
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import { crawlIntradayData, scheduleIntradayCrawls } from './actions/intraday-cr
|
|||
import { createQMOperationRegistry } from './shared/operation-provider';
|
||||
|
||||
@Handler('qm')
|
||||
@Disabled() // Disable by default, enable specific operations as needed
|
||||
export class QMHandler extends BaseHandler<DataIngestionServices> {
|
||||
public operationRegistry: OperationRegistry;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,227 @@
|
|||
import { getRandomUserAgent } from '@stock-bot/utils';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { TE_CONFIG } from '../shared/config';
|
||||
import type { TeCountry } from '../shared/types';
|
||||
import type { TeHandler } from '../te.handler';
|
||||
|
||||
export async function fetchCountries(this: TeHandler): Promise<TeCountry[] | null> {
|
||||
const { logger, mongodb } = this;
|
||||
|
||||
try {
|
||||
// 1. Fetch the HTML page
|
||||
const reqInfo = {
|
||||
proxy: this.proxy.getProxy(),
|
||||
headers: {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
},
|
||||
}
|
||||
const response = await fetch(TE_CONFIG.COUNTRIES_URL, reqInfo);
|
||||
|
||||
logger.debug('Response status:', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
url: response.url
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
|
||||
logger.info('Fetched HTML length:', { length: html.length });
|
||||
|
||||
|
||||
// 2. Parse HTML to extract country data
|
||||
const $ = cheerio.load(html);
|
||||
const countries: TeCountry[] = [];
|
||||
|
||||
// Look for country links - they typically have a pattern like /country-name
|
||||
// Trading Economics groups countries by region in the page
|
||||
$('.list-group-item, a[href^="/"]').each((_, element) => {
|
||||
const $el = $(element);
|
||||
|
||||
// Try to extract country information
|
||||
let name: string | undefined;
|
||||
let url: string | undefined;
|
||||
let region: string | undefined;
|
||||
|
||||
// Check if it's a direct link
|
||||
if ($el.is('a')) {
|
||||
const href = $el.attr('href');
|
||||
const text = $el.text().trim();
|
||||
console.log(href)
|
||||
// Filter for country URLs (they don't contain special paths like /indicators, /calendar, etc.)
|
||||
if (href && href.startsWith('/') && !href.includes('/') && text) {
|
||||
name = text;
|
||||
url = href;
|
||||
}
|
||||
} else {
|
||||
// Check for links within table rows
|
||||
const $link = $el.find('a[href^="/"]').first();
|
||||
if ($link.length) {
|
||||
const href = $link.attr('href');
|
||||
const text = $link.text().trim();
|
||||
|
||||
if (href && text && !href.includes('/indicators') && !href.includes('/calendar')) {
|
||||
name = text;
|
||||
url = href;
|
||||
|
||||
// Try to get region from parent elements
|
||||
const $regionHeader = $el.closest('.region-section, .country-group').find('h2, h3, .region-title').first();
|
||||
if ($regionHeader.length) {
|
||||
region = $regionHeader.text().trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add to countries array if we found valid data
|
||||
if (name && url) {
|
||||
// Extract country code from URL if possible (e.g., /united-states -> US)
|
||||
const code = extractCountryCode(url, name);
|
||||
|
||||
countries.push({
|
||||
name,
|
||||
code,
|
||||
url: `https://tradingeconomics.com${url}`,
|
||||
region,
|
||||
updated_at: new Date(),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Remove duplicates based on name
|
||||
const uniqueCountries = Array.from(
|
||||
new Map(countries.map(c => [c.name, c])).values()
|
||||
);
|
||||
|
||||
if (uniqueCountries.length === 0) {
|
||||
throw new Error('No countries found in HTML');
|
||||
}
|
||||
|
||||
logger.info('Extracted countries from HTML', {
|
||||
count: uniqueCountries.length,
|
||||
byRegion: groupCountriesByRegion(uniqueCountries),
|
||||
});
|
||||
|
||||
// 3. Save to MongoDB
|
||||
try {
|
||||
console.log( uniqueCountries)
|
||||
if (uniqueCountries.length > 0) {
|
||||
const result = await mongodb?.batchUpsert('teCountries', uniqueCountries, ['code']);
|
||||
logger.info('Countries saved to MongoDB', {
|
||||
matched: result.matchedCount,
|
||||
modified: result.modifiedCount,
|
||||
upserted: result.upsertedCount,
|
||||
});
|
||||
}
|
||||
} catch (dbError) {
|
||||
logger.error('Failed to save countries to MongoDB', { error: dbError });
|
||||
throw dbError;
|
||||
}
|
||||
|
||||
return uniqueCountries;
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch Trading Economics countries', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractCountryCode(url: string, name: string): string | undefined {
|
||||
// Common country code mappings
|
||||
const countryCodeMap: Record<string, string> = {
|
||||
'united-states': 'US',
|
||||
'united-kingdom': 'GB',
|
||||
'euro-area': 'EU',
|
||||
'china': 'CN',
|
||||
'japan': 'JP',
|
||||
'germany': 'DE',
|
||||
'france': 'FR',
|
||||
'italy': 'IT',
|
||||
'spain': 'ES',
|
||||
'canada': 'CA',
|
||||
'australia': 'AU',
|
||||
'south-korea': 'KR',
|
||||
'india': 'IN',
|
||||
'brazil': 'BR',
|
||||
'russia': 'RU',
|
||||
'mexico': 'MX',
|
||||
'indonesia': 'ID',
|
||||
'netherlands': 'NL',
|
||||
'saudi-arabia': 'SA',
|
||||
'turkey': 'TR',
|
||||
'switzerland': 'CH',
|
||||
'poland': 'PL',
|
||||
'sweden': 'SE',
|
||||
'belgium': 'BE',
|
||||
'argentina': 'AR',
|
||||
'ireland': 'IE',
|
||||
'austria': 'AT',
|
||||
'norway': 'NO',
|
||||
'israel': 'IL',
|
||||
'singapore': 'SG',
|
||||
'denmark': 'DK',
|
||||
'egypt': 'EG',
|
||||
'philippines': 'PH',
|
||||
'finland': 'FI',
|
||||
'chile': 'CL',
|
||||
'pakistan': 'PK',
|
||||
'romania': 'RO',
|
||||
'new-zealand': 'NZ',
|
||||
'greece': 'GR',
|
||||
'iraq': 'IQ',
|
||||
'portugal': 'PT',
|
||||
'czech-republic': 'CZ',
|
||||
'vietnam': 'VN',
|
||||
'peru': 'PE',
|
||||
'colombia': 'CO',
|
||||
'malaysia': 'MY',
|
||||
'ukraine': 'UA',
|
||||
'hungary': 'HU',
|
||||
'kuwait': 'KW',
|
||||
'morocco': 'MA',
|
||||
'slovakia': 'SK',
|
||||
'kenya': 'KE',
|
||||
'puerto-rico': 'PR',
|
||||
'ecuador': 'EC',
|
||||
'ethiopia': 'ET',
|
||||
'dominican-republic': 'DO',
|
||||
'luxembourg': 'LU',
|
||||
'oman': 'OM',
|
||||
'guatemala': 'GT',
|
||||
'bulgaria': 'BG',
|
||||
'ghana': 'GH',
|
||||
'tanzania': 'TZ',
|
||||
'turkmenistan': 'TM',
|
||||
'croatia': 'HR',
|
||||
'costa-rica': 'CR',
|
||||
'lebanon': 'LB',
|
||||
'slovenia': 'SI',
|
||||
'lithuania': 'LT',
|
||||
'serbia': 'RS',
|
||||
'panama': 'PA',
|
||||
};
|
||||
|
||||
// Clean URL to get country slug
|
||||
const slug = url.replace(/^\//, '').toLowerCase();
|
||||
|
||||
return countryCodeMap[slug];
|
||||
}
|
||||
|
||||
function groupCountriesByRegion(countries: TeCountry[]): Record<string, number> {
|
||||
const groups: Record<string, number> = {};
|
||||
|
||||
for (const country of countries) {
|
||||
const region = country.region || 'Unknown';
|
||||
groups[region] = (groups[region] || 0) + 1;
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
|
@ -1,2 +1,4 @@
|
|||
// Export all action functions here
|
||||
// export * from './example.action';
|
||||
export * from './fetch-countries.action';
|
||||
export * from './spider.action';
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
import { getRandomUserAgent } from '@stock-bot/utils';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { TE_CONFIG } from '../shared/config';
|
||||
import type { TeHandler } from '../te.handler';
|
||||
|
||||
export async function spiderUrl(this: TeHandler, payload: { url: string }): Promise<string[] | null> {
|
||||
const { logger, mongodb } = this;
|
||||
const reqUrl = payload && payload.url ? TE_CONFIG.MAIN_URL + payload.url : TE_CONFIG.MAIN_URL;
|
||||
this.logger.info(`Spiderring URL: ${reqUrl}`, {reqUrl});
|
||||
|
||||
const mongoRecord = await mongodb?.findOne('teUrls', { url: payload?.url || '/' });
|
||||
if(payload && payload.url && mongoRecord && mongoRecord.lastCrawled && mongoRecord.lastCrawled.getTime() > Date.now() - 7* 24 * 60 * 60 * 1000) {
|
||||
this.logger.info(`Skipping URL ${reqUrl} as it was already crawled in the last 24 hours`);
|
||||
return null; // Skip if already crawled in the last 24 hours
|
||||
}
|
||||
|
||||
if (!payload) {
|
||||
const oneDayAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
||||
const records = await mongodb?.find('teUrls', {
|
||||
$or: [
|
||||
{ lastCrawled: { $lt: oneDayAgo } }, // Crawled more than 24 hours ago
|
||||
{ lastCrawled: { $exists: false } } // Never crawled
|
||||
]
|
||||
});
|
||||
this.logger.info(`Found ${records?.length || 0} records to process`);
|
||||
for (const record of records || []) {
|
||||
await this.scheduleOperation('te-spider', {
|
||||
url: record.url,
|
||||
}, {
|
||||
jobId: `te-spider-${record.url}`,
|
||||
priority: 5, // Lower priority than financial data
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// 1. Fetch the HTML page
|
||||
const reqInfo = {
|
||||
proxy: 'http://5.79.66.2:13010',//this.proxy.getProxy(),
|
||||
headers: {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
},
|
||||
}
|
||||
const response = await fetch(reqUrl, reqInfo);
|
||||
|
||||
logger.debug('Response status:', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
url: response.url
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
|
||||
|
||||
let match = html.match(/TESymbol = '([^']+)'/);
|
||||
const teSymbol = match ? match[1] : undefined;
|
||||
match = html.match(/;TELastUpdate = '([^']+)'/);
|
||||
const teLastUpdate = match ? match[1] : undefined;
|
||||
match = html.match(/; var TEChartsDatasource = '([^']+)'/);
|
||||
const teChartUrl = match ? match[1] : undefined;
|
||||
match = html.match(/; var TEChartsToken = '([^']+)'/);
|
||||
const teChartToken = match ? match[1] : undefined;
|
||||
|
||||
console.log(teSymbol, teLastUpdate, teChartUrl, teChartToken);
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
const urls: string[] = [];
|
||||
|
||||
$('.list-group-item, a[href^="/"]').each((_, element) => {
|
||||
const $el = $(element);
|
||||
let url: string | undefined;
|
||||
if ($el.is('a')) {
|
||||
const href = $el.attr('href');
|
||||
if (href && href.startsWith('/') && !href.includes('.aspx')) {
|
||||
url = href;
|
||||
}
|
||||
}
|
||||
|
||||
if (url && urls.indexOf(url) === -1) {
|
||||
urls.push(url);
|
||||
}
|
||||
});
|
||||
|
||||
if (urls.length === 0) {
|
||||
throw new Error('No urls found in HTML');
|
||||
}
|
||||
|
||||
// 3. Save to MongoDB
|
||||
try {
|
||||
if (urls.length > 0) {
|
||||
const urlMap: {url: string, lastCrawled?: Date, teSymbol? : string, teLastUpdate? : string, teChartUrl? : string, teChartToken? : string}[] = urls.map(url => ({url}));
|
||||
if( payload && payload.url) {
|
||||
urlMap.push({
|
||||
url: payload.url,
|
||||
lastCrawled: new Date(),
|
||||
teSymbol,
|
||||
teLastUpdate,
|
||||
teChartUrl,
|
||||
teChartToken,})
|
||||
}else {
|
||||
urlMap.push({url: '/', lastCrawled: new Date()})
|
||||
}
|
||||
|
||||
const result = await mongodb?.batchUpsert('teUrls', urlMap, ['url']);
|
||||
logger.info('TE URLs saved to MongoDB', {
|
||||
matched: result.matchedCount,
|
||||
modified: result.modifiedCount,
|
||||
upserted: result.upsertedCount,
|
||||
});
|
||||
}
|
||||
} catch (dbError) {
|
||||
logger.error('Failed to save urls to MongoDB', { error: dbError });
|
||||
throw dbError;
|
||||
}
|
||||
|
||||
for (const url of urls) {
|
||||
this.scheduleOperation('te-spider', {
|
||||
url: url,
|
||||
}, {
|
||||
jobId: `te-spider-${url}`,
|
||||
priority: 5, // Lower priority than financial data
|
||||
})
|
||||
}
|
||||
|
||||
return urls;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to fetch Trading Economics URLs ${reqUrl}`, {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
stack: error instanceof Error ? error.stack : undefined,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,9 @@
|
|||
export const TE_CONFIG = {
|
||||
// Add configuration constants here
|
||||
API_URL: 'https://api.example.com',
|
||||
MAIN_URL: 'https://tradingeconomics.com',
|
||||
COUNTRIES_URL: 'https://tradingeconomics.com/countries',
|
||||
REQUEST_TIMEOUT: 30000,
|
||||
USER_AGENT: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
KEY: 'tradingeconomics-charts-core-api-key',
|
||||
};
|
||||
|
|
@ -11,3 +11,12 @@ export interface TeResponse {
|
|||
status: string;
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
export interface TeCountry {
|
||||
name: string;
|
||||
code?: string;
|
||||
region?: string;
|
||||
url?: string;
|
||||
updated_at: Date;
|
||||
created_at?: Date;
|
||||
}
|
||||
|
|
@ -1,29 +1,31 @@
|
|||
import {
|
||||
BaseHandler,
|
||||
Disabled,
|
||||
Handler,
|
||||
Operation,
|
||||
ScheduledOperation,
|
||||
ScheduledOperation
|
||||
} from '@stock-bot/handlers';
|
||||
import type { DataIngestionServices } from '../../types';
|
||||
import { fetchCountries, spiderUrl } from './actions';
|
||||
|
||||
@Handler('te')
|
||||
export class TeHandler extends BaseHandler {
|
||||
@Disabled()
|
||||
export class TeHandler extends BaseHandler<DataIngestionServices> {
|
||||
constructor(services: any) {
|
||||
super(services);
|
||||
}
|
||||
|
||||
@Operation('example-operation')
|
||||
async exampleOperation(): Promise<unknown> {
|
||||
this.logger.info('TE handler example operation executed');
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
@ScheduledOperation('te-scheduled-job', '0 0 * * *', {
|
||||
@ScheduledOperation('te-countries', '0 0 * * 0', {
|
||||
priority: 5,
|
||||
description: 'Daily TE handler scheduled job',
|
||||
immediately: false,
|
||||
description: 'Fetch and update Trading Economics countries data',
|
||||
immediately: true,
|
||||
})
|
||||
async scheduledJob(): Promise<unknown> {
|
||||
this.logger.info('TE handler scheduled job executed');
|
||||
return this.exampleOperation();
|
||||
}
|
||||
@Disabled()
|
||||
fetchCountries = fetchCountries;
|
||||
|
||||
@ScheduledOperation('te-spider', '0 0 * * 0', {
|
||||
priority: 5,
|
||||
description: 'Fetch and update Trading Economics countries data',
|
||||
immediately: true,
|
||||
})
|
||||
spiderUrlSchedule = spiderUrl;
|
||||
}
|
||||
|
|
@ -14,7 +14,6 @@
|
|||
{ "path": "./data-pipeline" },
|
||||
{ "path": "./web-api" },
|
||||
{ "path": "./web-app" },
|
||||
{ "path": "./engine"},
|
||||
{ "path": "./orchestrator"}
|
||||
]
|
||||
}
|
||||
|
|
@ -7,10 +7,11 @@
|
|||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
// "allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
// "noEmit": true,
|
||||
"composite": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
|
|
|
|||
45
bun.lock
45
bun.lock
|
|
@ -77,6 +77,7 @@
|
|||
"@stock-bot/shutdown": "*",
|
||||
"@stock-bot/stock-config": "*",
|
||||
"@stock-bot/utils": "*",
|
||||
"cheerio": "^1.0.0",
|
||||
"hono": "^4.0.0",
|
||||
"pako": "^2.1.0",
|
||||
},
|
||||
|
|
@ -1171,6 +1172,8 @@
|
|||
|
||||
"body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"bowser": ["bowser@2.11.0", "", {}, "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA=="],
|
||||
|
||||
"bplist-parser": ["bplist-parser@0.2.0", "", { "dependencies": { "big-integer": "^1.6.44" } }, "sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw=="],
|
||||
|
|
@ -1223,6 +1226,10 @@
|
|||
|
||||
"chalk": ["chalk@5.4.1", "", {}, "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="],
|
||||
|
||||
"cheerio": ["cheerio@1.1.0", "", { "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "encoding-sniffer": "^0.2.0", "htmlparser2": "^10.0.0", "parse5": "^7.3.0", "parse5-htmlparser2-tree-adapter": "^7.1.0", "parse5-parser-stream": "^7.1.2", "undici": "^7.10.0", "whatwg-mimetype": "^4.0.0" } }, "sha512-+0hMx9eYhJvWbgpKV9hN7jg0JcwydpopZE4hgi+KvQtByZXPp04NiCWU0LzcAbP63abZckIHkTQaXVF52mX3xQ=="],
|
||||
|
||||
"cheerio-select": ["cheerio-select@2.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-select": "^5.1.0", "css-what": "^6.1.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1" } }, "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g=="],
|
||||
|
||||
"chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="],
|
||||
|
||||
"chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="],
|
||||
|
|
@ -1283,6 +1290,10 @@
|
|||
|
||||
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
||||
|
||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||
|
||||
"cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="],
|
||||
|
||||
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
|
||||
|
|
@ -1375,6 +1386,14 @@
|
|||
|
||||
"dom-helpers": ["dom-helpers@5.2.1", "", { "dependencies": { "@babel/runtime": "^7.8.7", "csstype": "^3.0.2" } }, "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA=="],
|
||||
|
||||
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||
|
||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||
|
||||
"domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
|
||||
|
||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
|
||||
"eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="],
|
||||
|
|
@ -1387,6 +1406,8 @@
|
|||
|
||||
"encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
|
||||
|
||||
"encoding-sniffer": ["encoding-sniffer@0.2.1", "", { "dependencies": { "iconv-lite": "^0.6.3", "whatwg-encoding": "^3.1.1" } }, "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw=="],
|
||||
|
||||
"end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="],
|
||||
|
||||
"engine.io": ["engine.io@6.6.4", "", { "dependencies": { "@types/cors": "^2.8.12", "@types/node": ">=10.0.0", "accepts": "~1.3.4", "base64id": "2.0.0", "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.3.1", "engine.io-parser": "~5.2.1", "ws": "~8.17.1" } }, "sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g=="],
|
||||
|
|
@ -1395,6 +1416,8 @@
|
|||
|
||||
"engine.io-parser": ["engine.io-parser@5.2.3", "", {}, "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q=="],
|
||||
|
||||
"entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
|
||||
"es-abstract": ["es-abstract@1.24.0", "", { "dependencies": { "array-buffer-byte-length": "^1.0.2", "arraybuffer.prototype.slice": "^1.0.4", "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", "call-bound": "^1.0.4", "data-view-buffer": "^1.0.2", "data-view-byte-length": "^1.0.2", "data-view-byte-offset": "^1.0.1", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "es-set-tostringtag": "^2.1.0", "es-to-primitive": "^1.3.0", "function.prototype.name": "^1.1.8", "get-intrinsic": "^1.3.0", "get-proto": "^1.0.1", "get-symbol-description": "^1.1.0", "globalthis": "^1.0.4", "gopd": "^1.2.0", "has-property-descriptors": "^1.0.2", "has-proto": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "internal-slot": "^1.1.0", "is-array-buffer": "^3.0.5", "is-callable": "^1.2.7", "is-data-view": "^1.0.2", "is-negative-zero": "^2.0.3", "is-regex": "^1.2.1", "is-set": "^2.0.3", "is-shared-array-buffer": "^1.0.4", "is-string": "^1.1.1", "is-typed-array": "^1.1.15", "is-weakref": "^1.1.1", "math-intrinsics": "^1.1.0", "object-inspect": "^1.13.4", "object-keys": "^1.1.1", "object.assign": "^4.1.7", "own-keys": "^1.0.1", "regexp.prototype.flags": "^1.5.4", "safe-array-concat": "^1.1.3", "safe-push-apply": "^1.0.0", "safe-regex-test": "^1.1.0", "set-proto": "^1.0.0", "stop-iteration-iterator": "^1.1.0", "string.prototype.trim": "^1.2.10", "string.prototype.trimend": "^1.0.9", "string.prototype.trimstart": "^1.0.8", "typed-array-buffer": "^1.0.3", "typed-array-byte-length": "^1.0.3", "typed-array-byte-offset": "^1.0.4", "typed-array-length": "^1.0.7", "unbox-primitive": "^1.1.0", "which-typed-array": "^1.1.19" } }, "sha512-WSzPgsdLtTcQwm4CROfS5ju2Wa1QQcVeT37jFjYzdFz1r9ahadC8B8/a4qxJxM+09F18iumCdRmlr96ZYkQvEg=="],
|
||||
|
||||
"es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
|
||||
|
|
@ -1621,6 +1644,8 @@
|
|||
|
||||
"hono": ["hono@4.8.4", "", {}, "sha512-KOIBp1+iUs0HrKztM4EHiB2UtzZDTBihDtOF5K6+WaJjCPeaW4Q92R8j63jOhvJI5+tZSMuKD9REVEXXY9illg=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="],
|
||||
|
||||
"http-cache-semantics": ["http-cache-semantics@4.2.0", "", {}, "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ=="],
|
||||
|
||||
"http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="],
|
||||
|
|
@ -1931,6 +1956,8 @@
|
|||
|
||||
"npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="],
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"numeral": ["numeral@2.0.6", "", {}, "sha512-qaKRmtYPZ5qdw4jWJD6bxEf1FJEqllJrwxCLIm0sQU/A7v2/czigzOb+C2uSiFsa9lBUzeH7M1oK+Q+OLxL3kA=="],
|
||||
|
||||
"object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
|
||||
|
|
@ -1995,6 +2022,12 @@
|
|||
|
||||
"parent-module": ["parent-module@1.0.1", "", { "dependencies": { "callsites": "^3.0.0" } }, "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g=="],
|
||||
|
||||
"parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="],
|
||||
|
||||
"parse5-htmlparser2-tree-adapter": ["parse5-htmlparser2-tree-adapter@7.1.0", "", { "dependencies": { "domhandler": "^5.0.3", "parse5": "^7.0.0" } }, "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g=="],
|
||||
|
||||
"parse5-parser-stream": ["parse5-parser-stream@7.1.2", "", { "dependencies": { "parse5": "^7.0.0" } }, "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow=="],
|
||||
|
||||
"parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
|
||||
|
||||
"pascal-case": ["pascal-case@3.1.2", "", { "dependencies": { "no-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g=="],
|
||||
|
|
@ -2445,7 +2478,7 @@
|
|||
|
||||
"unbox-primitive": ["unbox-primitive@1.1.0", "", { "dependencies": { "call-bound": "^1.0.3", "has-bigints": "^1.0.2", "has-symbols": "^1.1.0", "which-boxed-primitive": "^1.1.1" } }, "sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw=="],
|
||||
|
||||
"undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="],
|
||||
"undici": ["undici@7.11.0", "", {}, "sha512-heTSIac3iLhsmZhUCjyS3JQEkZELateufzZuBaVM5RHXdSBMb1LPMQf5x+FH7qjsZYDP0ttAc3nnVpUB+wYbOg=="],
|
||||
|
||||
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
|
||||
|
||||
|
|
@ -2475,6 +2508,10 @@
|
|||
|
||||
"webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="],
|
||||
|
||||
"whatwg-encoding": ["whatwg-encoding@3.1.1", "", { "dependencies": { "iconv-lite": "0.6.3" } }, "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@4.0.0", "", {}, "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@14.2.0", "", { "dependencies": { "tr46": "^5.1.0", "webidl-conversions": "^7.0.0" } }, "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw=="],
|
||||
|
||||
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
|
||||
|
|
@ -2659,6 +2696,8 @@
|
|||
|
||||
"got/type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="],
|
||||
|
||||
"htmlparser2/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||
|
||||
"is-wsl/is-docker": ["is-docker@2.2.1", "", { "bin": { "is-docker": "cli.js" } }, "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="],
|
||||
|
||||
"joi/@hapi/hoek": ["@hapi/hoek@9.3.0", "", {}, "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ=="],
|
||||
|
|
@ -2687,6 +2726,8 @@
|
|||
|
||||
"os-dns-native/node-addon-api": ["node-addon-api@4.3.0", "", {}, "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="],
|
||||
|
||||
"parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||
|
||||
"path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
|
||||
|
||||
"pg-mem/lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="],
|
||||
|
|
@ -2735,6 +2776,8 @@
|
|||
|
||||
"tailwindcss/object-hash": ["object-hash@3.0.0", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="],
|
||||
|
||||
"testcontainers/undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="],
|
||||
|
||||
"ts-unused-exports/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
|
||||
|
||||
"type-is/mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="],
|
||||
|
|
|
|||
|
|
@ -41,3 +41,10 @@
|
|||
## Additional Resources
|
||||
- **Marketstack**: https://marketstack.com/
|
||||
- **Alpaca**: https://alpaca.markets/ - Future hopefully or US connection
|
||||
- **MacroMicro**: https://en.macromicro.me/
|
||||
- **Norgate**: https://norgatedata.com/stockmarketpackages.php#castocks
|
||||
- **FINRA**: https://www.finra.org
|
||||
https://otce.finra.org/otce/EquityShortInterest/staticArchives?startDate=2014-01-01&endDate=2024-11-03&dateDisplay=November%203,%202018
|
||||
https://www.finra.org/finra-data/browse-catalog/equity-short-interest/files
|
||||
https://www.cboe.com/us/equities/market_statistics/short_interest/?year=2025&month=06&mkt=bzx
|
||||
- **MORNINGSTAR**: https://finra-markets.morningstar.com/MarketData/EquityOptions/detail.jsp?query=14:0P0001BUHA PRICES/SHORT INTEREST
|
||||
8
docs/todo.md
Normal file
8
docs/todo.md
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
Data Ingestion
|
||||
- QM scrape all timeframes add timeframe column to collection
|
||||
- EOD Fundementals / Prices / Intraday / Delisted ++
|
||||
- DataBento - 125 credit - use for symbols / second / minute intraday
|
||||
- Work on TE Indicators
|
||||
- CEO - watch all and save them to posts. do it every minute
|
||||
- Test our handler specific rate limits
|
||||
- Fix up handler worker counts
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
import type { Collection } from 'mongodb';
|
||||
import { createNamespacedCache } from '@stock-bot/cache';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type {
|
||||
|
|
@ -11,6 +10,7 @@ import type {
|
|||
ServiceTypes,
|
||||
} from '@stock-bot/types';
|
||||
import { fetch } from '@stock-bot/utils';
|
||||
import type { Collection } from 'mongodb';
|
||||
// Handler registry is now injected, not imported
|
||||
import { createJobHandler } from '../utils/create-job-handler';
|
||||
|
||||
|
|
@ -18,6 +18,7 @@ import { createJobHandler } from '../utils/create-job-handler';
|
|||
* Job scheduling options
|
||||
*/
|
||||
export interface JobScheduleOptions {
|
||||
jobId?: string; // Optional job ID for custom identification
|
||||
delay?: number;
|
||||
priority?: number;
|
||||
attempts?: number;
|
||||
|
|
|
|||
18
tetest.ts
18
tetest.ts
|
|
@ -1,6 +1,8 @@
|
|||
const pako = require('pako');
|
||||
|
||||
const value = "a/lpZGluZ2VjbBP9IAahQyHll2u4zSdk+eDEshxAXrKZAKBYXiuIYCAnNNRpnrEIBW5XO5a6x/6q21A6RLsYMsZ3law1yXl74NG7eMMEtJHauEYtK4TC8XsM24SQF8VZlYfF1mIU3dx2gJkGNBULa9wo3Dd4YOyA9mJfz5t+oPHPy80BQ5AAn/0II58Ndl+cyATaep28EaPY5NRd9jM+Z8U9scqGSACO3+3XbzEGaIqlY6tJV3Ajt4QYdsf7vVd5HiWzSQcLJGc45F/uT8/A9A9Qb8ABjlHkJva/HuDv9TezGyxKHvW/2/w0oljydYbJSCRKcAHBntf/QEBIPaARhqR57rwrCvp8lyHSyAJxT2I3UFMVwyuCdj9bwYBpnY0SrCNHbU+Po+KcrW2q3SWG988c1rcCY6XEMdoCLSgF/dry0ZHMQVrehI5d7wjATyvFnn9NM3htehPuqqckjgRh+fs/jSU5A5NyUdStfXBtZ2U="
|
||||
|
||||
const value = "a/lpZGluZ2VjbAM1pgcHtCCefSKpgAeH6QtP0aHicG8/E3R+LXZixUTbVGiIXr8M4eCCDCL834tYqtuIDcXYS1QZCKHHNMoXO8X+qlYyeas0uDlKeJPzqaonV4UIeZ6Xl0yU84SAbzuXJYyTBAebk1o1gV6dmouchDgdjAcMt75NiQFeGhMPW/D8zjdw1FvG6eUWq6YxKZIBlK7Z+v6MrfS+FltPy43ImpPt0t6CXdZW3ki9z+7+S475zI4ckhSan5yCtAo6jrrNkphC3JBpGtcZovpSkJkPA4yHn6LTm6bEyIm4shCc1JqYk/qFmwzerHDMhuKPPsYKtYcmzedlx5/6adh/bnyQn6Gc1BJddpUBmCPJkKPM9OJelArjhpVYaXXe+7Xkjc3V5XbKLlMVn9HQ5ggh3VWeyqoeTy2e1Ayw7WC00qoPjeHNjS69w5bYMYQvUIHD+t/WsdlavkBvPtjyEytQa/8vzVOblwjy2626ldpGWBzQotgGSx8pKwN3YlmDMcam2kJQ6wZoXwy/XsQE9SqmZQ1CJROVwYE2L/BvL06jWxT1w6LRKNwbH6IP0JMGqyqTXQjZ1TOZRrQOifUkTm5/9d++VCAzM2KsqH3b/os/yPk6Ubt5x6AJW7Ng4mQB3m+R+sc1fwfZfkLwHQEld1HLrmMiNoFgvBeL/y2AOAn7lLZ3zCBvL4MHChEcDZx7LppmG4wVu9inrH+dJqLgREyZF8ONRW89+Q4wf/qszwqMi5yDWyCZF/MT7LHRrLvSokxddwH+XrnuMXygH/5gH0WrJNQCYuSTSzAyRUbI+rwyETTkOXbGtE2r7AnHd/XfKyCTtSf9rJL9u9obwLVWW65+ocVG0z/K4jZUdWNRVAb3iqDy2pj5TwqHJxIyBVPfcX0L5fJqzj9nlyrNIr5zXr5gPhtHz//vsu/nCwqbHarK950T4hflFmFrJUPyaXwB/1eT7vlYB6YrUHV/wkvVzIQ2WSTqYy5OeEffhKyHTJWkaOuNCKKPovqrtjYCwIiElaGG99HeQP/tZ4yWfeIIuN8deiBlEa1OMRcgqhRxEXtY1mR3MDBAs8M1M7WRT0dGcnvJPlL5ihxZal2nY7qN4yb6/MNdFJ3xwk/5IEG8W4J4DqioZ7IJxFkOKLixxAQiyS3hOqD9NZh0KPg0OkUkPiMUul0mTRYnNgPf79grkM0w+BvUfaP1274L4e01tA+CmrfYSvkfHk8wNTvI+0XI+O8XFYMYX9ewa1L4yWHpVZj8HlD0/oNLp/JBHr7n+AMXAGZfvtPy6YXT4NrYBO1IAsRr0MJZThDd14VTzbkMh4pVwn467fGKLYLXOVgj8pia9YY54AE28YZ7ox5/G2Cy6vbnZYnJdUF0mBBvijkhzNchHVtzyw7BMehKMF5TAEN6ndcH5fW7KtV4IKyv/ZRvxOZcgKDBRx/LYbFri8uGs03sXvBC4CmOpGsHmtaBCvZ1G8fwxK/jHt+ZoxlNfsbaS9mxiRdVDdlFRmobRTzA5+W4c+1HLvlTNMRIW0auXQWGjSfYeFQfpICzWAiTKw1w0gcC7YLAvot+ivrinZFORkSyCvFtETc/gok/ajprIF97eoeEY+2yEqaeTOdo3G04wyOz27EOjvy8XUst4Na3d8j+e0LPTh93ZaCxjAYUdv5DwjkktyM8nQyyTmfiXTJlUUKWuvbedG9lNnn/9Oq5T+4cGSm9tfsMz8tK8VD/68FIiu2pq/o2h6yzV8RX0PgJu6q3Oub2143YgzQy7Xp+fCslI3BJ+kbEEWGgtxgV6WIAbeMHgJvyizjfZsk/CHzModCYWF3KnXzWDbwe8FL+rhC0GGuvZbleThSx9IYw0/MI5YQs3+1fAcUkkgN01F1/qxCkAF3q2fXq2Ctbm8Uim1OBOsMg3zi1sDXbdUMeTQs7MislQdMiX3VLVHqHGm61jynPf5X8OTLkFl6LGy2HHj8LYGg74bLcPIoyxONilLHWe48FtJMaJdqmCKK+v2WrhQ/uhvyIEf2Lrij/jd8JXjhg1Q4vugu/DXLCzETrFxJU+XbFyifAa0kQ/p1X0jfH/8f6vkO7zyfUE1d1N46iudYeSRpZNCLZlIZZY/EfVj4XAEM46M5E1xlyDLrWXvxf6ewF8oLMBarSDqzg2MbgH2q0SPIN1jfE2RYpJDzGlp1hZ8YTEESlpu+1ofbJqSnRfu6A3YtqXJ9aImkqpAo4WzVZoN/Ppn+DpsvMRO37kNmqPapC1+MBRP+cR0nYKPuNx92NrrgY4WBTGR0F4tnCC4bxEFimd0kJgTeksjRC0htizYQYNGLKrSYG3Nt9NFkrSBn6c+HYgAH3u86l2U1H+W7JmDW3h5kkt9L2n5Ml+OtSwit0uP95BGtgD/ZVWx13K9WkqF7l7SckA3kTtGN/n3AlPnzklR0JZpItGuB2S6Joe8oBWltEMRQjDS7Li688Lv0Q8fVHKvRt9ct2ZgibPmu4vbAWqhbv9tvVP2A5GZk9MIJEOG10owdhPcayBiK08DhLyLxY2/C7xKEj2XlHB/I13beb8u+oGT9umT1H29jN1qyKEAxRPapPT5TzEAATVi5kPPvWvxw5/x6Xg6koQ4WkN1gIWByOuX/hS+Y+jrfLE8Y5FOOeDguqigQXfgULWi44VcpF9DLFVMbT9uQlRr4Kal2GvDtFxb1PsdYQRN9KrRmCMQU+ZFFyP0CF1EEZ73hQ2KOXdT7rpLW8Q+z26dVbmyipAwXcxDxOFwSLEGJlCYgaB0/TQtj3b/aa9qjFM8iZH++pch7Jb8RY4TOfyviaOWsUJXAvwowQqB6lS5UQXj6wqKY9sq/D4iQdgqzXLjyaD7S3ZSMPWxE28qyP4o6Blrnhnv6UyvbFasIy/6/Dej3FNS6fFaU+6Lo3nuCjXczeASIDa5JqPQXqJcaW5ZcOrBIFizTkgHCQV+eBNllfufva7Gzaei51Hy98mCWwVxuNXuGql3EyTKF8f6vloesS6rjMGdsbUiZVWSWgwTCUTbzT9i+i+PK70gm+9nSJhRcyqOYC3nC9xuF6UImmGyUf4ocCSowW+s3GKzoV+HQ67mAf9iM5qq8NaH12SznH3CnZThUyOqBQ/wpRMF8sBK7gvAh3NVQ/UvXfOyPEh1QogRRARzT2C1X2wKc7yqVMKNOuJjy+wGi/1+1otom3BVCeqZwCmHSpXlTbrXEQzt0FSObtyDr2yGDADOrvyDL59/81hatpfxDcZlABQtXJc7NhtdiCXwRH4f3zJpGhkaTY1Dw6RrUY4TeePOCvBYa9aQegzaLZBYFUfFV8el+CpksZYtnwKSrYCgeBErXb/ILnmwj+gTmYaqWzeCSDy4gvjVan+K7Ndp06ziZFMuZ9uad9p0G1hk5p5WjQnCU2qV6V7HZooODkxwNRV+75SxOrhGK6hmz7fO9AXSh0DnudJLEI3Opcb3ABB1Uelbd5zwjt7qbS0Ko9wuYJbnLd4E8RJnbjcYddPgtLa6HSVnizFXGIbE8z+PKHNs6iq/XK1UBPhFswZ60bbPl56ucquQfmzqKxy0gepgNVPyypmsRS1mLed+K2g7NGUCRbMwilWmAwLF644S9hEcHQ1Y8MdSPcj+umluyEWGfeXrRVrRpkmzfAjckbzLc/EHJuBBMEsFmX2KZ7Q4o63Tk4vGrh1Jm7PIorHoW2LXKX24ahvI773rGwgC80WURYumc4voF1iYfKOamEwUgJQyHPvTvKUsAO37mF/Mi509CNlEU7FLtbTu8YTqzT2z6XWZcvVmFXVI2FO7dGxPKES3Rhe7C3n5m03JxuHaWyEN53326liRK8UZ0bX9Y8tQc8PDQBAGiSM9qmECOMzHaYSqEE3xfntRtMADQOxLfj6eDfcTABPTPkfPseQ1v/zBDZxvCmd91BOx+Asda6HHCGxkCC+j+7bzVk9yDkR89Zk8Z7D9yg4jJ54e3DSvZeTa5ta/gbpzaPGnkM158IqSBLKGB2HzWjUhe1VfSh0ruuFkbUulO0apI00iia2s8yCyVdf5tb8DHieW3HcXkWeWkEHq4TNdzb12cDRpou3UcUlPNU1aC+xf2F8pcID7WxyZADLOuqWD1/+9TFtvvOG8h5HyXOPDcEeUlp7jUNazvel6pAYbqbpzah24GftKNZ4cdy3TrWt31efsJtzARHKTDqFue1nT8f3gnU0aje3jTW9z++HH/pt3m4mMfDVnlt0lbtocIPQTOfuVG6nEQk7ltTtnOCwyeSMesX2Mjw2ilZoqW6ULWvyFfGpqOjAVD2Nqp9Cnb/W6hOQJ6su6+5uOUDi7+s1oEVtLVHfASfjVDawwzhcSbNkbXmRhSmkr7m0JiBFfqfo/d8GtLDILpWgAcJ0ivqPyL4Tf2ruPHRmE2PgeKf585jY24iXlrD/cIwrH6A/j4EKg5HtfqaGsXMpgVaPuv3mfiS1qGW0z2gtsF/0FHxGMSb6YwF6bGLyN60mR9qgGGgAbwH3NgtMA90D3sI0pb0JUAMjmff/uVpPPi7BqWaaBvLwceSstkZpc3ZKYzIWLPLlpGs77kX1X8kz9xPMhsh7ftlfses0Pygpw3AObDJRHqCQFO4tJxxor6IndwRhZ9VEtjDyoimu5aSoFqeR7H211K8LTMtn04lowOVmYaF2o2UipyeF8CTgpuAhYCAld8edIITGhoYCZHY/04draSRPbFMGlMbDowi9apWO4UOy/7JIJsz0A4amBMaS9KUgZBuCwwX3rOu+AOWzLJiF+eKKAcf+6VCnrLUgrEB8nfc71jstbqF0+T62ryfqsqGiILDlxeCCItNAzDZmYAfkZeRhmRcs+Cgu4KZ05YxkZ6RtuB2DmvUWbyLmo6f/9YXHtkYFBcSk3NITHF9d//MsCwChLSlwsBB7/YC3Bnvp0/OrVND6+6cGleczlReL0M8Br/swxT6DKGvPSWF7bYTlZNG0DIiUb1JZ2U="
|
||||
const URL = 'https://d3ii0wo49og5mi.cloudfront.net/economics/szlttr?span=max&v=20250619074600&key=20240229:nazare'
|
||||
|
||||
const key = 'tradingeconomics-charts-core-api-key'
|
||||
|
||||
|
|
@ -19,5 +21,15 @@ const testFunc = function(e, k) {
|
|||
return pako.inflate(n, {to :'string'});
|
||||
}
|
||||
|
||||
const result = testFunc(value, key);
|
||||
console.log(JSON.parse(result.toString()));
|
||||
fetch(URL, {
|
||||
proxy: 'http://5.79.66.2:13010',
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome'
|
||||
}
|
||||
}).then(async response => {
|
||||
const resp = await response.text();
|
||||
const result = testFunc(resp.replaceAll('"', ""), key);
|
||||
// console.log(result.toString());
|
||||
console.log(JSON.stringify(JSON.parse(result.toString())));
|
||||
})
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue