boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

work up marketdatagateway

Browse source
This commit is contained in:
Bojan Kucera 2025-06-03 19:43:22 -04:00
parent 58ae897e90
commit d22f7aafa0
17 changed files with 653 additions and 494 deletions
Download patch file Download diff file Expand all files Collapse all files

12
apps/data-services/data-processor/src/services/DataIngestionService.ts
View file

@ -1,7 +1,9 @@
import { logger } from '@stock-bot/utils';
import { getLogger } from '@stock-bot/logger';
const logger = getLogger('DataIngestionService');
import { IngestionStep, ProcessingResult, DataSource } from '../types/DataPipeline';
import axios from 'axios';
import * as csv from 'csv-parser';
import csv from 'csv-parser';
import * as fs from 'fs';
export class DataIngestionService {
@ -112,11 +114,9 @@ export class DataIngestionService {
return new Promise((resolve, reject) => {
const records: any[] = [];
const errors: any[] = [];
let recordCount = 0;
fs.createReadStream(filePath)
let recordCount = 0; fs.createReadStream(filePath)
.pipe(csv())
.on('data', (data) => {
.on('data', (data: any) => {
recordCount++;
try {
records.push(data);