removed some files
This commit is contained in:
parent
b87a931a2b
commit
cbf002a31a
4 changed files with 2 additions and 247 deletions
|
|
@ -1,93 +0,0 @@
|
||||||
import { MongoClient } from 'mongodb';
|
|
||||||
|
|
||||||
async function migratePriceTracking() {
|
|
||||||
const client = new MongoClient('mongodb://localhost:27017');
|
|
||||||
await client.connect();
|
|
||||||
|
|
||||||
const mongodb = client.db('stock');
|
|
||||||
|
|
||||||
try {
|
|
||||||
console.log('Starting price tracking migration...');
|
|
||||||
|
|
||||||
const collection = mongodb.collection('eodSymbols');
|
|
||||||
const batchSize = 100;
|
|
||||||
let processedCount = 0;
|
|
||||||
let hasMore = true;
|
|
||||||
|
|
||||||
while (hasMore) {
|
|
||||||
// Find documents that need migration
|
|
||||||
const documents = await collection.find({
|
|
||||||
lastPriceUpdate: { $exists: true },
|
|
||||||
'operations.price_update': { $exists: false }
|
|
||||||
}).limit(batchSize).toArray();
|
|
||||||
|
|
||||||
if (documents.length === 0) {
|
|
||||||
hasMore = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process each document
|
|
||||||
for (const doc of documents) {
|
|
||||||
// Normalize date to 00:00:00 UTC
|
|
||||||
const lastPriceUpdate = new Date(doc.lastPriceUpdate);
|
|
||||||
const normalizedDate = new Date(Date.UTC(
|
|
||||||
lastPriceUpdate.getUTCFullYear(),
|
|
||||||
lastPriceUpdate.getUTCMonth(),
|
|
||||||
lastPriceUpdate.getUTCDate(),
|
|
||||||
0, 0, 0, 0
|
|
||||||
));
|
|
||||||
|
|
||||||
// Parse lastPriceDate if it exists
|
|
||||||
let lastRecordDate = null;
|
|
||||||
if (doc.lastPriceDate) {
|
|
||||||
try {
|
|
||||||
lastRecordDate = new Date(doc.lastPriceDate);
|
|
||||||
} catch (e) {
|
|
||||||
console.warn(`Failed to parse lastPriceDate for ${doc.Code}: ${doc.lastPriceDate}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update the document
|
|
||||||
await collection.updateOne(
|
|
||||||
{ _id: doc._id },
|
|
||||||
{
|
|
||||||
$set: {
|
|
||||||
'operations.price_update': {
|
|
||||||
lastRunAt: normalizedDate,
|
|
||||||
lastSuccessAt: normalizedDate,
|
|
||||||
status: 'success',
|
|
||||||
...(lastRecordDate && { lastRecordDate })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
processedCount++;
|
|
||||||
if (processedCount % 1000 === 0) {
|
|
||||||
console.log(`Processed ${processedCount} documents...`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Migration completed. Total documents migrated: ${processedCount}`);
|
|
||||||
|
|
||||||
// Optional: Remove old fields
|
|
||||||
const removeOldFields = false; // Set to true to remove old fields
|
|
||||||
if (removeOldFields) {
|
|
||||||
console.log('Removing old fields...');
|
|
||||||
const result = await collection.updateMany(
|
|
||||||
{},
|
|
||||||
{ $unset: { lastPriceUpdate: '', lastPriceDate: '' } }
|
|
||||||
);
|
|
||||||
console.log(`Removed old fields from ${result.modifiedCount} documents`);
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Migration failed:', error);
|
|
||||||
} finally {
|
|
||||||
await client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the migration
|
|
||||||
migratePriceTracking().catch(console.error);
|
|
||||||
|
|
@ -329,7 +329,6 @@ export async function fetchIntraday(
|
||||||
symbol,
|
symbol,
|
||||||
exchange,
|
exchange,
|
||||||
symbolExchange: `${symbol}.${exchange}`,
|
symbolExchange: `${symbol}.${exchange}`,
|
||||||
interval,
|
|
||||||
datetime: bar.datetime,
|
datetime: bar.datetime,
|
||||||
timestamp: bar.timestamp,
|
timestamp: bar.timestamp,
|
||||||
gmtoffset: bar.gmtoffset,
|
gmtoffset: bar.gmtoffset,
|
||||||
|
|
@ -341,12 +340,12 @@ export async function fetchIntraday(
|
||||||
source: 'eod'
|
source: 'eod'
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Save to MongoDB - use timestamp, symbol, and interval as unique identifier
|
// Save to MongoDB - use timestamp and symbolExchange as unique identifier
|
||||||
const collectionName = `eodIntraday${interval.toUpperCase()}`;
|
const collectionName = `eodIntraday${interval.toUpperCase()}`;
|
||||||
const result = await this.mongodb.batchUpsert(
|
const result = await this.mongodb.batchUpsert(
|
||||||
collectionName,
|
collectionName,
|
||||||
recordsWithMetadata,
|
recordsWithMetadata,
|
||||||
['timestamp', 'symbolExchange', 'interval']
|
['timestamp', 'symbolExchange']
|
||||||
);
|
);
|
||||||
|
|
||||||
logger.info(`Saved ${result.insertedCount} intraday records`, {
|
logger.info(`Saved ${result.insertedCount} intraday records`, {
|
||||||
|
|
|
||||||
|
|
@ -1,73 +0,0 @@
|
||||||
// MongoDB Shell Script for Price Tracking Migration
|
|
||||||
// Run with: mongosh mongodb://username:password@localhost:27017/stock price-migration.mongodb.js
|
|
||||||
|
|
||||||
print("Starting price tracking migration...");
|
|
||||||
|
|
||||||
const batchSize = 1000;
|
|
||||||
let processedCount = 0;
|
|
||||||
let cursor = db.eodSymbols.find({
|
|
||||||
lastPriceUpdate: { $exists: true },
|
|
||||||
'operations.price_update': { $exists: false }
|
|
||||||
});
|
|
||||||
|
|
||||||
let batch = [];
|
|
||||||
|
|
||||||
cursor.forEach(doc => {
|
|
||||||
// Normalize date to 00:00:00 UTC
|
|
||||||
const normalizedDate = new Date(doc.lastPriceUpdate);
|
|
||||||
normalizedDate.setUTCHours(0, 0, 0, 0);
|
|
||||||
|
|
||||||
// Parse lastPriceDate if it exists
|
|
||||||
let lastRecordDate = null;
|
|
||||||
if (doc.lastPriceDate) {
|
|
||||||
try {
|
|
||||||
lastRecordDate = new Date(doc.lastPriceDate);
|
|
||||||
} catch (e) {
|
|
||||||
print(`Failed to parse lastPriceDate for ${doc.Code}: ${doc.lastPriceDate}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
batch.push({
|
|
||||||
updateOne: {
|
|
||||||
filter: { _id: doc._id },
|
|
||||||
update: {
|
|
||||||
$set: {
|
|
||||||
'operations.price_update': {
|
|
||||||
lastRunAt: normalizedDate,
|
|
||||||
lastSuccessAt: normalizedDate,
|
|
||||||
status: 'success',
|
|
||||||
...(lastRecordDate && { lastRecordDate })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (batch.length >= batchSize) {
|
|
||||||
db.eodSymbols.bulkWrite(batch);
|
|
||||||
processedCount += batch.length;
|
|
||||||
print(`Processed ${processedCount} documents...`);
|
|
||||||
batch = [];
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Process remaining batch
|
|
||||||
if (batch.length > 0) {
|
|
||||||
db.eodSymbols.bulkWrite(batch);
|
|
||||||
processedCount += batch.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
print(`Migration completed. Processed ${processedCount} documents.`);
|
|
||||||
|
|
||||||
// Verify migration
|
|
||||||
const sampleDoc = db.eodSymbols.findOne({ 'operations.price_update': { $exists: true } });
|
|
||||||
print("\nSample migrated document:");
|
|
||||||
printjson(sampleDoc.operations);
|
|
||||||
|
|
||||||
// Count remaining documents with old fields
|
|
||||||
const remainingCount = db.eodSymbols.countDocuments({ lastPriceUpdate: { $exists: true } });
|
|
||||||
print(`\nDocuments still having lastPriceUpdate field: ${remainingCount}`);
|
|
||||||
|
|
||||||
// Optional: Remove old fields (uncomment to execute)
|
|
||||||
// print("\nRemoving old fields...");
|
|
||||||
// db.eodSymbols.updateMany({}, { $unset: { lastPriceUpdate: '', lastPriceDate: '' } });
|
|
||||||
|
|
@ -1,78 +0,0 @@
|
||||||
// This script performs the migration of lastPriceUpdate and lastPriceDate fields
|
|
||||||
// to the operations tracking system
|
|
||||||
|
|
||||||
// Since authentication is required, you'll need to run this script with proper MongoDB credentials
|
|
||||||
// Update the connection string below with your authentication details
|
|
||||||
|
|
||||||
const MONGODB_URI = 'mongodb://username:password@localhost:27017/stock?authSource=admin';
|
|
||||||
// Or use environment variable: process.env.MONGODB_URI
|
|
||||||
|
|
||||||
async function runMigration() {
|
|
||||||
console.log(`
|
|
||||||
========================================
|
|
||||||
PRICE TRACKING MIGRATION SCRIPT
|
|
||||||
========================================
|
|
||||||
|
|
||||||
This script will migrate lastPriceUpdate and lastPriceDate fields
|
|
||||||
from eodSymbols collection to the operations tracking system.
|
|
||||||
|
|
||||||
To run this migration:
|
|
||||||
|
|
||||||
1. Update the MONGODB_URI in this file with your MongoDB credentials
|
|
||||||
OR set the MONGODB_URI environment variable
|
|
||||||
|
|
||||||
2. Run the script:
|
|
||||||
node run-price-migration.js
|
|
||||||
|
|
||||||
The script will:
|
|
||||||
- Find all documents with lastPriceUpdate field
|
|
||||||
- Convert dates to normalized format (00:00:00 UTC)
|
|
||||||
- Create operations.price_update structure
|
|
||||||
- Process in batches of 1000 documents
|
|
||||||
|
|
||||||
Total documents to migrate: ~199,175
|
|
||||||
|
|
||||||
Migration query that will be executed:
|
|
||||||
`);
|
|
||||||
|
|
||||||
console.log(`
|
|
||||||
db.eodSymbols.find({
|
|
||||||
lastPriceUpdate: { $exists: true },
|
|
||||||
'operations.price_update': { $exists: false }
|
|
||||||
}).forEach(doc => {
|
|
||||||
const normalizedDate = new Date(doc.lastPriceUpdate);
|
|
||||||
normalizedDate.setUTCHours(0, 0, 0, 0);
|
|
||||||
|
|
||||||
let lastRecordDate = null;
|
|
||||||
if (doc.lastPriceDate) {
|
|
||||||
try {
|
|
||||||
lastRecordDate = new Date(doc.lastPriceDate);
|
|
||||||
} catch (e) {}
|
|
||||||
}
|
|
||||||
|
|
||||||
db.eodSymbols.updateOne(
|
|
||||||
{ _id: doc._id },
|
|
||||||
{
|
|
||||||
$set: {
|
|
||||||
'operations.price_update': {
|
|
||||||
lastRunAt: normalizedDate,
|
|
||||||
lastSuccessAt: normalizedDate,
|
|
||||||
status: 'success',
|
|
||||||
...(lastRecordDate && { lastRecordDate })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
});
|
|
||||||
`);
|
|
||||||
|
|
||||||
console.log(`
|
|
||||||
After migration, you can optionally remove old fields:
|
|
||||||
db.eodSymbols.updateMany({}, { $unset: { lastPriceUpdate: '', lastPriceDate: '' } })
|
|
||||||
|
|
||||||
To verify migration:
|
|
||||||
db.eodSymbols.findOne({ 'operations.price_update': { $exists: true } })
|
|
||||||
`);
|
|
||||||
}
|
|
||||||
|
|
||||||
runMigration();
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue