diff --git a/apps/stock/data-ingestion/scripts/migrate-price-tracking-mcp.ts b/apps/stock/data-ingestion/scripts/migrate-price-tracking-mcp.ts deleted file mode 100644 index c4ff49e..0000000 --- a/apps/stock/data-ingestion/scripts/migrate-price-tracking-mcp.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { MongoClient } from 'mongodb'; - -async function migratePriceTracking() { - const client = new MongoClient('mongodb://localhost:27017'); - await client.connect(); - - const mongodb = client.db('stock'); - - try { - console.log('Starting price tracking migration...'); - - const collection = mongodb.collection('eodSymbols'); - const batchSize = 100; - let processedCount = 0; - let hasMore = true; - - while (hasMore) { - // Find documents that need migration - const documents = await collection.find({ - lastPriceUpdate: { $exists: true }, - 'operations.price_update': { $exists: false } - }).limit(batchSize).toArray(); - - if (documents.length === 0) { - hasMore = false; - break; - } - - // Process each document - for (const doc of documents) { - // Normalize date to 00:00:00 UTC - const lastPriceUpdate = new Date(doc.lastPriceUpdate); - const normalizedDate = new Date(Date.UTC( - lastPriceUpdate.getUTCFullYear(), - lastPriceUpdate.getUTCMonth(), - lastPriceUpdate.getUTCDate(), - 0, 0, 0, 0 - )); - - // Parse lastPriceDate if it exists - let lastRecordDate = null; - if (doc.lastPriceDate) { - try { - lastRecordDate = new Date(doc.lastPriceDate); - } catch (e) { - console.warn(`Failed to parse lastPriceDate for ${doc.Code}: ${doc.lastPriceDate}`); - } - } - - // Update the document - await collection.updateOne( - { _id: doc._id }, - { - $set: { - 'operations.price_update': { - lastRunAt: normalizedDate, - lastSuccessAt: normalizedDate, - status: 'success', - ...(lastRecordDate && { lastRecordDate }) - } - } - } - ); - - processedCount++; - if (processedCount % 1000 === 0) { - console.log(`Processed ${processedCount} documents...`); - } - } - } - - console.log(`Migration completed. Total documents migrated: ${processedCount}`); - - // Optional: Remove old fields - const removeOldFields = false; // Set to true to remove old fields - if (removeOldFields) { - console.log('Removing old fields...'); - const result = await collection.updateMany( - {}, - { $unset: { lastPriceUpdate: '', lastPriceDate: '' } } - ); - console.log(`Removed old fields from ${result.modifiedCount} documents`); - } - - } catch (error) { - console.error('Migration failed:', error); - } finally { - await client.close(); - } -} - -// Run the migration -migratePriceTracking().catch(console.error); \ No newline at end of file diff --git a/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts b/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts index e5a871d..737f636 100644 --- a/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts +++ b/apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts @@ -329,7 +329,6 @@ export async function fetchIntraday( symbol, exchange, symbolExchange: `${symbol}.${exchange}`, - interval, datetime: bar.datetime, timestamp: bar.timestamp, gmtoffset: bar.gmtoffset, @@ -341,12 +340,12 @@ export async function fetchIntraday( source: 'eod' })); - // Save to MongoDB - use timestamp, symbol, and interval as unique identifier + // Save to MongoDB - use timestamp and symbolExchange as unique identifier const collectionName = `eodIntraday${interval.toUpperCase()}`; const result = await this.mongodb.batchUpsert( collectionName, recordsWithMetadata, - ['timestamp', 'symbolExchange', 'interval'] + ['timestamp', 'symbolExchange'] ); logger.info(`Saved ${result.insertedCount} intraday records`, { diff --git a/price-migration.mongodb.js b/price-migration.mongodb.js deleted file mode 100644 index e15d60f..0000000 --- a/price-migration.mongodb.js +++ /dev/null @@ -1,73 +0,0 @@ -// MongoDB Shell Script for Price Tracking Migration -// Run with: mongosh mongodb://username:password@localhost:27017/stock price-migration.mongodb.js - -print("Starting price tracking migration..."); - -const batchSize = 1000; -let processedCount = 0; -let cursor = db.eodSymbols.find({ - lastPriceUpdate: { $exists: true }, - 'operations.price_update': { $exists: false } -}); - -let batch = []; - -cursor.forEach(doc => { - // Normalize date to 00:00:00 UTC - const normalizedDate = new Date(doc.lastPriceUpdate); - normalizedDate.setUTCHours(0, 0, 0, 0); - - // Parse lastPriceDate if it exists - let lastRecordDate = null; - if (doc.lastPriceDate) { - try { - lastRecordDate = new Date(doc.lastPriceDate); - } catch (e) { - print(`Failed to parse lastPriceDate for ${doc.Code}: ${doc.lastPriceDate}`); - } - } - - batch.push({ - updateOne: { - filter: { _id: doc._id }, - update: { - $set: { - 'operations.price_update': { - lastRunAt: normalizedDate, - lastSuccessAt: normalizedDate, - status: 'success', - ...(lastRecordDate && { lastRecordDate }) - } - } - } - } - }); - - if (batch.length >= batchSize) { - db.eodSymbols.bulkWrite(batch); - processedCount += batch.length; - print(`Processed ${processedCount} documents...`); - batch = []; - } -}); - -// Process remaining batch -if (batch.length > 0) { - db.eodSymbols.bulkWrite(batch); - processedCount += batch.length; -} - -print(`Migration completed. Processed ${processedCount} documents.`); - -// Verify migration -const sampleDoc = db.eodSymbols.findOne({ 'operations.price_update': { $exists: true } }); -print("\nSample migrated document:"); -printjson(sampleDoc.operations); - -// Count remaining documents with old fields -const remainingCount = db.eodSymbols.countDocuments({ lastPriceUpdate: { $exists: true } }); -print(`\nDocuments still having lastPriceUpdate field: ${remainingCount}`); - -// Optional: Remove old fields (uncomment to execute) -// print("\nRemoving old fields..."); -// db.eodSymbols.updateMany({}, { $unset: { lastPriceUpdate: '', lastPriceDate: '' } }); \ No newline at end of file diff --git a/run-price-migration.js b/run-price-migration.js deleted file mode 100644 index 0a13c55..0000000 --- a/run-price-migration.js +++ /dev/null @@ -1,78 +0,0 @@ -// This script performs the migration of lastPriceUpdate and lastPriceDate fields -// to the operations tracking system - -// Since authentication is required, you'll need to run this script with proper MongoDB credentials -// Update the connection string below with your authentication details - -const MONGODB_URI = 'mongodb://username:password@localhost:27017/stock?authSource=admin'; -// Or use environment variable: process.env.MONGODB_URI - -async function runMigration() { - console.log(` -======================================== -PRICE TRACKING MIGRATION SCRIPT -======================================== - -This script will migrate lastPriceUpdate and lastPriceDate fields -from eodSymbols collection to the operations tracking system. - -To run this migration: - -1. Update the MONGODB_URI in this file with your MongoDB credentials - OR set the MONGODB_URI environment variable - -2. Run the script: - node run-price-migration.js - -The script will: -- Find all documents with lastPriceUpdate field -- Convert dates to normalized format (00:00:00 UTC) -- Create operations.price_update structure -- Process in batches of 1000 documents - -Total documents to migrate: ~199,175 - -Migration query that will be executed: -`); - - console.log(` -db.eodSymbols.find({ - lastPriceUpdate: { $exists: true }, - 'operations.price_update': { $exists: false } -}).forEach(doc => { - const normalizedDate = new Date(doc.lastPriceUpdate); - normalizedDate.setUTCHours(0, 0, 0, 0); - - let lastRecordDate = null; - if (doc.lastPriceDate) { - try { - lastRecordDate = new Date(doc.lastPriceDate); - } catch (e) {} - } - - db.eodSymbols.updateOne( - { _id: doc._id }, - { - $set: { - 'operations.price_update': { - lastRunAt: normalizedDate, - lastSuccessAt: normalizedDate, - status: 'success', - ...(lastRecordDate && { lastRecordDate }) - } - } - } - ); -}); -`); - - console.log(` -After migration, you can optionally remove old fields: -db.eodSymbols.updateMany({}, { $unset: { lastPriceUpdate: '', lastPriceDate: '' } }) - -To verify migration: -db.eodSymbols.findOne({ 'operations.price_update': { $exists: true } }) -`); -} - -runMigration(); \ No newline at end of file