diff --git a/apps/stock/data-ingestion/src/handlers/index.ts b/apps/stock/data-ingestion/src/handlers/index.ts index db9a896..357c19d 100644 --- a/apps/stock/data-ingestion/src/handlers/index.ts +++ b/apps/stock/data-ingestion/src/handlers/index.ts @@ -14,6 +14,7 @@ import { WebShareHandler } from './webshare/webshare.handler'; import { TradingViewHandler } from './tradingview/tradingview.handler'; import { TeHandler } from './te/te.handler'; import { EodHandler } from './eod/eod.handler'; +import { TtHandler } from './tt/tt.handler'; import { createEODOperationRegistry } from './eod/shared'; import { createQMOperationRegistry } from './qm/shared/operation-provider'; @@ -62,7 +63,7 @@ export async function initializeAllHandlers(serviceContainer: IServiceContainer) // The HandlerScanner in the DI container will handle the actual registration // We just need to ensure handlers are imported so their decorators run - const handlers = [CeoHandler, IbHandler, QMHandler, WebShareHandler, TradingViewHandler, TeHandler, EodHandler]; + const handlers = [CeoHandler, IbHandler, QMHandler, WebShareHandler, TradingViewHandler, TeHandler, EodHandler, TtHandler]; logger.info('Handler imports loaded', { count: handlers.length, diff --git a/apps/stock/data-ingestion/src/handlers/tt/actions/import.action.ts b/apps/stock/data-ingestion/src/handlers/tt/actions/import.action.ts new file mode 100644 index 0000000..d902e95 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tt/actions/import.action.ts @@ -0,0 +1,147 @@ +import * as cheerio from 'cheerio'; +import { getRandomUserAgent } from '@stock-bot/utils'; +import { existsSync, mkdirSync, rmSync, readdirSync, readFileSync } from 'fs'; +import { execSync } from 'child_process'; +import { join } from 'path'; +import type { TtHandler } from '../tt.handler'; + +const BASE_URL = 'https://www.turtletrader.com'; +const TMP_DIR = '/tmp/tt'; + +const MONTH_MAP: Record = { + F: '01', G: '02', H: '03', J: '04', K: '05', M: '06', + N: '07', Q: '08', U: '09', V: '10', X: '11', Z: '12', +}; + +function parseDate(yymmdd: string): string { + const yy = parseInt(yymmdd.slice(0, 2)); + const mm = yymmdd.slice(2, 4); + const dd = yymmdd.slice(4, 6); + const year = yy >= 50 ? 1900 + yy : 2000 + yy; + return `${year}-${mm}-${dd}`; +} + +export async function importData(this: TtHandler): Promise<{ totalRecords: number }> { + const { logger, mongodb, http } = this; + + // 1. Fetch page and extract zip links + const response = await http.get(`${BASE_URL}/hpd/`, { + headers: { 'User-Agent': getRandomUserAgent() }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch TurtleTrader page: ${response.status}`); + } + + const html = await response.text(); + const $ = cheerio.load(html); + + const zipLinks: Array<{ href: string; name: string }> = []; + $('a[href$=".zip"]').each((_, el) => { + const href = $(el).attr('href') || ''; + const name = $(el).text().trim(); + if (href) zipLinks.push({ href, name }); + }); + + if (!zipLinks.length) { + throw new Error('No zip links found on page'); + } + + logger.info(`Found ${zipLinks.length} zip files to download`); + + // 2. Setup temp dir + if (existsSync(TMP_DIR)) { + rmSync(TMP_DIR, { recursive: true }); + } + mkdirSync(TMP_DIR, { recursive: true }); + + let totalRecords = 0; + + try { + for (const { href, name } of zipLinks) { + const zipUrl = href.startsWith('http') ? href : `${BASE_URL}${href}`; + const symbol = href.split('/').pop()!.replace('.zip', '').toUpperCase(); + const zipPath = join(TMP_DIR, `${symbol}.zip`); + const extractDir = join(TMP_DIR, symbol); + + // Download + logger.info(`Downloading ${name} (${symbol})`); + const zipResp = await http.get(zipUrl, { + headers: { 'User-Agent': getRandomUserAgent() }, + }); + + if (!zipResp.ok) { + logger.error(`Failed to download ${zipUrl}: ${zipResp.status}`); + continue; + } + + const buf = Buffer.from(await zipResp.arrayBuffer()); + require('fs').writeFileSync(zipPath, buf); + + // Extract + mkdirSync(extractDir, { recursive: true }); + execSync(`unzip -o "${zipPath}" -d "${extractDir}"`, { stdio: 'pipe' }); + + // Parse each txt file + const files = readdirSync(extractDir).filter(f => f.endsWith('.txt')); + const records: any[] = []; + + for (const file of files) { + const contract = file.replace('.txt', ''); + // Parse contract code: e.g. CL83M → symbol=CL, year=83, month=M + const match = contract.match(/^([A-Z]+)(\d{2})([A-Z])$/); + if (!match) { + logger.warn(`Skipping unrecognized contract format: ${contract}`); + continue; + } + + const [, sym, year, month] = match; + const filePath = join(extractDir, file); + const content = readFileSync(filePath, 'utf-8'); + + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) continue; + + const parts = trimmed.split(','); + if (parts.length < 7) continue; + + const [dateStr, o, h, l, c, v, oi] = parts; + + records.push({ + contract, + symbol: sym, + name, + month, + year, + date: parseDate(dateStr), + o: parseFloat(o), + h: parseFloat(h), + l: parseFloat(l), + c: parseFloat(c), + v: parseInt(v), + oi: parseInt(oi), + }); + } + } + + if (records.length) { + await mongodb?.batchUpsert('ttFutures', records, ['contract', 'date']); + totalRecords += records.length; + logger.info(`Imported ${records.length} records for ${name} (${symbol}, ${files.length} contracts)`); + } + + // Clean up this zip + rmSync(zipPath, { force: true }); + rmSync(extractDir, { recursive: true, force: true }); + } + } finally { + // Clean up temp dir + if (existsSync(TMP_DIR)) { + rmSync(TMP_DIR, { recursive: true, force: true }); + } + } + + logger.info(`TurtleTrader import complete: ${totalRecords} total records`); + return { totalRecords }; +} diff --git a/apps/stock/data-ingestion/src/handlers/tt/actions/index.ts b/apps/stock/data-ingestion/src/handlers/tt/actions/index.ts new file mode 100644 index 0000000..b547b00 --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tt/actions/index.ts @@ -0,0 +1 @@ +export * from './import.action'; diff --git a/apps/stock/data-ingestion/src/handlers/tt/tt.handler.ts b/apps/stock/data-ingestion/src/handlers/tt/tt.handler.ts new file mode 100644 index 0000000..f306d8d --- /dev/null +++ b/apps/stock/data-ingestion/src/handlers/tt/tt.handler.ts @@ -0,0 +1,48 @@ +import { + BaseHandler, + Handler, + ScheduledOperation, +} from '@stock-bot/handlers'; +import type { DataIngestionServices } from '../../types'; +import { importData } from './actions'; + +@Handler('tt') +export class TtHandler extends BaseHandler { + constructor(services: any) { + super(services); + } + + async onInit(): Promise { + if (!this.mongodb) return; + + const indexes = [ + { + indexSpec: { contract: 1, date: 1 }, + options: { name: 'contract_date_unique_idx', unique: true, background: true }, + }, + { + indexSpec: { symbol: 1, date: 1 }, + options: { name: 'symbol_date_idx', background: true }, + }, + { + indexSpec: { date: 1 }, + options: { name: 'date_idx', background: true }, + }, + ]; + + for (const index of indexes) { + try { + await this.mongodb.createIndex('ttFutures', index.indexSpec, index.options); + } catch (error) { + this.logger.debug(`ttFutures index ${index.options.name} may already exist:`, error); + } + } + } + + @ScheduledOperation('tt-import', '0 0 * * 0', { + priority: 5, + description: 'Import TurtleTrader futures data', + immediately: true, + }) + importData = importData; +}