boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

removed some files

Browse source
This commit is contained in:
Boki 2025-07-09 23:51:10 -04:00
parent b87a931a2b
commit cbf002a31a
4 changed files with 2 additions and 247 deletions
Download patch file Download diff file Expand all files Collapse all files

93
apps/stock/data-ingestion/scripts/migrate-price-tracking-mcp.ts
View file

@ -1,93 +0,0 @@
import { MongoClient } from 'mongodb';
async function migratePriceTracking() {
const client = new MongoClient('mongodb://localhost:27017');
await client.connect();
const mongodb = client.db('stock');
try {
console.log('Starting price tracking migration...');
const collection = mongodb.collection('eodSymbols');
const batchSize = 100;
let processedCount = 0;
let hasMore = true;
while (hasMore) {
// Find documents that need migration
const documents = await collection.find({
lastPriceUpdate: { $exists: true },
'operations.price_update': { $exists: false }
}).limit(batchSize).toArray();
if (documents.length === 0) {
hasMore = false;
break;
}
// Process each document
for (const doc of documents) {
// Normalize date to 00:00:00 UTC
const lastPriceUpdate = new Date(doc.lastPriceUpdate);
const normalizedDate = new Date(Date.UTC(
lastPriceUpdate.getUTCFullYear(),
lastPriceUpdate.getUTCMonth(),
lastPriceUpdate.getUTCDate(),
0, 0, 0, 0
));
// Parse lastPriceDate if it exists
let lastRecordDate = null;
if (doc.lastPriceDate) {
try {
lastRecordDate = new Date(doc.lastPriceDate);
} catch (e) {
console.warn(`Failed to parse lastPriceDate for ${doc.Code}: ${doc.lastPriceDate}`);
}
}
// Update the document
await collection.updateOne(
{ _id: doc._id },
{
$set: {
'operations.price_update': {
lastRunAt: normalizedDate,
lastSuccessAt: normalizedDate,
status: 'success',
...(lastRecordDate && { lastRecordDate })
}
}
}
);
processedCount++;
if (processedCount % 1000 === 0) {
console.log(`Processed ${processedCount} documents...`);
}
}
}
console.log(`Migration completed. Total documents migrated: ${processedCount}`);
// Optional: Remove old fields
const removeOldFields = false; // Set to true to remove old fields
if (removeOldFields) {
console.log('Removing old fields...');
const result = await collection.updateMany(
{},
{ $unset: { lastPriceUpdate: '', lastPriceDate: '' } }
);
console.log(`Removed old fields from ${result.modifiedCount} documents`);
}
} catch (error) {
console.error('Migration failed:', error);
} finally {
await client.close();
}
}
// Run the migration
migratePriceTracking().catch(console.error);

5
apps/stock/data-ingestion/src/handlers/eod/actions/intraday.ts
View file

@ -329,7 +329,6 @@ export async function fetchIntraday(
symbol,
exchange,
symbolExchange: `${symbol}.${exchange}`,
interval,
datetime: bar.datetime,
timestamp: bar.timestamp,
gmtoffset: bar.gmtoffset,
@ -341,12 +340,12 @@ export async function fetchIntraday(
source: 'eod'
}));
// Save to MongoDB - use timestamp, symbol, and interval as unique identifier
// Save to MongoDB - use timestamp and symbolExchange as unique identifier
const collectionName = `eodIntraday${interval.toUpperCase()}`;
const result = await this.mongodb.batchUpsert(
collectionName,
recordsWithMetadata,
['timestamp', 'symbolExchange', 'interval']
['timestamp', 'symbolExchange']
);
logger.info(`Saved ${result.insertedCount} intraday records`, {

73
price-migration.mongodb.js
View file

@ -1,73 +0,0 @@
// MongoDB Shell Script for Price Tracking Migration
// Run with: mongosh mongodb://username:password@localhost:27017/stock price-migration.mongodb.js
print("Starting price tracking migration...");
const batchSize = 1000;
let processedCount = 0;
let cursor = db.eodSymbols.find({
lastPriceUpdate: { $exists: true },
'operations.price_update': { $exists: false }
});
let batch = [];
cursor.forEach(doc => {
// Normalize date to 00:00:00 UTC
const normalizedDate = new Date(doc.lastPriceUpdate);
normalizedDate.setUTCHours(0, 0, 0, 0);
// Parse lastPriceDate if it exists
let lastRecordDate = null;
if (doc.lastPriceDate) {
try {
lastRecordDate = new Date(doc.lastPriceDate);
} catch (e) {
print(`Failed to parse lastPriceDate for ${doc.Code}: ${doc.lastPriceDate}`);
}
}
batch.push({
updateOne: {
filter: { _id: doc._id },
update: {
$set: {
'operations.price_update': {
lastRunAt: normalizedDate,
lastSuccessAt: normalizedDate,
status: 'success',
...(lastRecordDate && { lastRecordDate })
}
}
}
}
});
if (batch.length >= batchSize) {
db.eodSymbols.bulkWrite(batch);
processedCount += batch.length;
print(`Processed ${processedCount} documents...`);
batch = [];
}
});
// Process remaining batch
if (batch.length > 0) {
db.eodSymbols.bulkWrite(batch);
processedCount += batch.length;
}
print(`Migration completed. Processed ${processedCount} documents.`);
// Verify migration
const sampleDoc = db.eodSymbols.findOne({ 'operations.price_update': { $exists: true } });
print("\nSample migrated document:");
printjson(sampleDoc.operations);
// Count remaining documents with old fields
const remainingCount = db.eodSymbols.countDocuments({ lastPriceUpdate: { $exists: true } });
print(`\nDocuments still having lastPriceUpdate field: ${remainingCount}`);
// Optional: Remove old fields (uncomment to execute)
// print("\nRemoving old fields...");
// db.eodSymbols.updateMany({}, { $unset: { lastPriceUpdate: '', lastPriceDate: '' } });

78
run-price-migration.js
View file

@ -1,78 +0,0 @@
// This script performs the migration of lastPriceUpdate and lastPriceDate fields
// to the operations tracking system
// Since authentication is required, you'll need to run this script with proper MongoDB credentials
// Update the connection string below with your authentication details
const MONGODB_URI = 'mongodb://username:password@localhost:27017/stock?authSource=admin';
// Or use environment variable: process.env.MONGODB_URI
async function runMigration() {
console.log(`
========================================
PRICE TRACKING MIGRATION SCRIPT
========================================
This script will migrate lastPriceUpdate and lastPriceDate fields
from eodSymbols collection to the operations tracking system.
To run this migration:
1. Update the MONGODB_URI in this file with your MongoDB credentials
OR set the MONGODB_URI environment variable
2. Run the script:
node run-price-migration.js
The script will:
- Find all documents with lastPriceUpdate field
- Convert dates to normalized format (00:00:00 UTC)
- Create operations.price_update structure
- Process in batches of 1000 documents
Total documents to migrate: ~199,175
Migration query that will be executed:
`);
console.log(`
db.eodSymbols.find({
lastPriceUpdate: { $exists: true },
'operations.price_update': { $exists: false }
}).forEach(doc => {
const normalizedDate = new Date(doc.lastPriceUpdate);
normalizedDate.setUTCHours(0, 0, 0, 0);
let lastRecordDate = null;
if (doc.lastPriceDate) {
try {
lastRecordDate = new Date(doc.lastPriceDate);
} catch (e) {}
}
db.eodSymbols.updateOne(
{ _id: doc._id },
{
$set: {
'operations.price_update': {
lastRunAt: normalizedDate,
lastSuccessAt: normalizedDate,
status: 'success',
...(lastRecordDate && { lastRecordDate })
}
}
}
);
});
`);
console.log(`
After migration, you can optionally remove old fields:
db.eodSymbols.updateMany({}, { $unset: { lastPriceUpdate: '', lastPriceDate: '' } })
To verify migration:
db.eodSymbols.findOne({ 'operations.price_update': { $exists: true } })
`);
}
runMigration();