boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

starting data-sync service

Browse source
This commit is contained in:
Boki 2025-06-18 19:08:51 -04:00
parent 96f515a76b
commit bd8a5bfe9e
13 changed files with 43 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

10
apps/data-sync-service/config/default.json
View file

@ -2,6 +2,14 @@
"service": {
"name": "data-sync-service",
"port": 3005,
"environment": "development"
"host": "0.0.0.0",
"healthCheckPath": "/health",
"metricsPath": "/metrics",
"shutdownTimeout": 30000,
"cors": {
"enabled": true,
"origin": "*",
"credentials": false
}
}
}

23
apps/data-sync-service/src/index.ts
View file

@ -3,7 +3,7 @@
*/
import { Hono } from 'hono';
import { cors } from 'hono/cors';
import { initializeConfig, getServiceConfig, getLoggingConfig } from '@stock-bot/config-new';
import { initializeConfig, getServiceConfig, getLoggingConfig, getDatabaseConfig } from '@stock-bot/config-new';
import { getLogger, shutdownLoggers, setLoggerConfig } from '@stock-bot/logger';
import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client';
import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client';
@ -13,8 +13,9 @@ import { syncManager } from './services/sync-manager';
import { setPostgreSQLClient, setMongoDBClient } from './clients';
// Initialize configuration
await initializeConfig();
await initializeConfig('./config');
const serviceConfig = getServiceConfig();
const databaseConfig = getDatabaseConfig();
// Initialize logger with config
const loggingConfig = getLoggingConfig();
@ -191,16 +192,16 @@ async function initializeServices() {
try {
// Initialize MongoDB client
logger.info('Connecting to MongoDB...');
const mongoConfig = serviceConfig.database.mongodb;
const mongoConfig = databaseConfig.mongodb;
mongoClient = await createAndConnectMongoDBClient({
uri: mongoConfig.uri,
database: mongoConfig.database,
host: 'localhost',
port: 27017,
host: mongoConfig.host || 'localhost',
port: mongoConfig.port || 27017,
timeouts: {
connectTimeout: mongoConfig.connectionTimeout || 30000,
connectTimeout: 30000,
socketTimeout: 30000,
serverSelectionTimeout: mongoConfig.serverSelectionTimeout || 5000,
serverSelectionTimeout: 5000,
},
});
setMongoDBClient(mongoClient);
@ -208,17 +209,17 @@ async function initializeServices() {
// Initialize PostgreSQL client
logger.info('Connecting to PostgreSQL...');
const pgConfig = serviceConfig.database.postgres;
const pgConfig = databaseConfig.postgres;
postgresClient = await createAndConnectPostgreSQLClient({
host: pgConfig.host,
port: pgConfig.port,
database: pgConfig.database,
username: pgConfig.username,
username: pgConfig.user,
password: pgConfig.password,
poolSettings: {
min: 2,
max: pgConfig.maxConnections || 10,
idleTimeoutMillis: 30000,
max: pgConfig.poolSize || 10,
idleTimeoutMillis: pgConfig.idleTimeout || 30000,
},
});
setPostgreSQLClient(postgresClient);

44
apps/web-api/config/default.json
View file

@ -11,49 +11,5 @@
"origin": ["http://localhost:4200", "http://localhost:3000", "http://localhost:3002"],
"credentials": true
}
},
"logging": {
"level": "info",
"format": "json"
},
"database": {
"postgres": {
"host": "localhost",
"port": 5432,
"database": "trading_bot",
"user": "trading_user",
"password": "trading_pass_dev",
"ssl": false,
"poolSize": 10,
"connectionTimeout": 30000,
"idleTimeout": 10000
},
"questdb": {
"host": "localhost",
"ilpPort": 9009,
"httpPort": 9000,
"pgPort": 8812,
"database": "questdb",
"user": "admin",
"password": "quest",
"bufferSize": 65536,
"flushInterval": 1000
},
"mongodb": {
"host": "localhost",
"port": 27017,
"database": "stock",
"user": "trading_admin",
"password": "trading_mongo_dev",
"authSource": "admin",
"poolSize": 10
},
"dragonfly": {
"host": "localhost",
"port": 6379,
"db": 0,
"maxRetries": 3,
"retryDelay": 100
}
}
}

12
bun.lock
View file

@ -47,7 +47,6 @@
"version": "1.0.0",
"dependencies": {
"@stock-bot/cache": "*",
"@stock-bot/config": "*",
"@stock-bot/event-bus": "*",
"@stock-bot/http": "*",
"@stock-bot/logger": "*",
@ -71,7 +70,6 @@
"name": "@stock-bot/data-sync-service",
"version": "1.0.0",
"dependencies": {
"@stock-bot/config-new": "*",
"@stock-bot/logger": "*",
"@stock-bot/mongodb-client": "*",
"@stock-bot/postgres-client": "*",
@ -153,6 +151,7 @@
"name": "@stock-bot/cache",
"version": "1.0.0",
"dependencies": {
"@stock-bot/logger": "*",
"ioredis": "^5.3.2",
},
"devDependencies": {
@ -212,7 +211,6 @@
"name": "@stock-bot/event-bus",
"version": "1.0.0",
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"eventemitter3": "^5.0.1",
"ioredis": "^5.3.2",
@ -249,7 +247,6 @@
"name": "@stock-bot/logger",
"version": "1.0.0",
"dependencies": {
"@stock-bot/config": "*",
"got": "^14.4.7",
"pino": "^9.7.0",
"pino-loki": "^2.6.0",
@ -265,12 +262,10 @@
"name": "@stock-bot/mongodb-client",
"version": "1.0.0",
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"@stock-bot/types": "*",
"@types/mongodb": "^4.0.7",
"mongodb": "^6.17.0",
"yup": "^1.6.1",
},
"devDependencies": {
"@types/node": "^20.11.0",
@ -285,11 +280,9 @@
"name": "@stock-bot/postgres-client",
"version": "1.0.0",
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"@stock-bot/types": "*",
"pg": "^8.11.3",
"yup": "^1.6.1",
},
"devDependencies": {
"@types/node": "^20.11.0",
@ -313,9 +306,9 @@
"name": "@stock-bot/questdb-client",
"version": "1.0.0",
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"@stock-bot/types": "*",
"pg": "^8.11.3",
},
"devDependencies": {
"@types/node": "^20.11.0",
@ -352,7 +345,6 @@
"name": "@stock-bot/strategy-engine",
"version": "1.0.0",
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/data-frame": "*",
"@stock-bot/event-bus": "*",
"@stock-bot/logger": "*",

27
config/default.json
View file

@ -13,28 +13,41 @@
"host": "localhost",
"port": 5432,
"database": "trading_bot",
"username": "trading_user",
"user": "trading_user",
"password": "trading_pass_dev",
"maxConnections": 10
"ssl": false,
"poolSize": 10,
"connectionTimeout": 30000,
"idleTimeout": 10000
},
"questdb": {
"host": "localhost",
"ilpPort": 9009,
"httpPort": 9000,
"pgPort": 8812,
"database": "questdb"
"database": "questdb",
"user": "admin",
"password": "quest",
"bufferSize": 65536,
"flushInterval": 1000
},
"mongodb": {
"uri": "mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin",
"host": "localhost",
"port": 27017,
"database": "stock",
"connectionTimeout": 30000,
"serverSelectionTimeout": 5000
"user": "trading_admin",
"password": "trading_mongo_dev",
"authSource": "admin",
"poolSize": 10
},
"dragonfly": {
"host": "localhost",
"port": 6379,
"password": "",
"db": 0,
"keyPrefix": "stock-bot:"
"keyPrefix": "stock-bot:",
"maxRetries": 3,
"retryDelay": 100
}
},
"logging": {

0
test-large-scale-performance.ts
View file

0
test-mongodb-batch.ts
View file

137
test-mongodb-multi-db.ts
View file

@ -1,137 +0,0 @@
#!/usr/bin/env bun
/**
* Test script for MongoDB Client with Multiple Database Support
*/
import { MongoDBClient } from './libs/mongodb-client/src/client';
interface TestDocument {
_id?: string;
name: string;
value: number;
created_at?: Date;
updated_at?: Date;
}
async function testMultiDatabaseSupport() {
const client = MongoDBClient.getInstance();
try {
console.log('🔌 Connecting to MongoDB...');
await client.connect();
console.log('✅ Connected successfully!');
// Test 1: Check default database
console.log('\n📊 Testing Default Database Operations');
console.log(`Default database: ${client.getDefaultDatabase()}`);
// Test 2: Insert into default database (stock)
const stockData: TestDocument[] = [
{ name: 'AAPL', value: 150.25 },
{ name: 'GOOGL', value: 2750.8 },
{ name: 'MSFT', value: 305.15 },
];
console.log('\n💾 Inserting into default database (stock)...');
const stockResult = await client.batchUpsert('test_symbols', stockData, 'name');
console.log('Stock database result:', stockResult);
// Test 3: Change default database
console.log('\n🔄 Changing default database to analytics...');
client.setDefaultDatabase('analytics');
console.log(`New default database: ${client.getDefaultDatabase()}`);
// Test 4: Insert into new default database
const analyticsData: TestDocument[] = [
{ name: 'daily_volume', value: 1000000 },
{ name: 'avg_price', value: 125.5 },
{ name: 'volatility', value: 0.25 },
];
console.log('\n📈 Inserting into new default database (analytics)...');
const analyticsResult = await client.batchUpsert('test_metrics', analyticsData, 'name');
console.log('Analytics database result:', analyticsResult);
// Test 5: Explicitly specify database (override default)
const tradingData: TestDocument[] = [
{ name: 'NYSE', value: 1 },
{ name: 'NASDAQ', value: 2 },
{ name: 'LSE', value: 3 },
];
console.log('\n🏦 Inserting into specific database (trading_documents)...');
const tradingResult = await client.batchUpsert('test_exchanges', tradingData, 'name', {
database: 'trading_documents',
});
console.log('Trading documents database result:', tradingResult);
// Test 6: Use convenience methods
console.log('\n🚀 Testing convenience methods...');
// Stock convenience method
const stockConvenienceResult = await client.batchUpsertStock(
'test_prices',
[{ name: 'TSLA', value: 800.25 }],
'name'
);
console.log('Stock convenience method result:', stockConvenienceResult);
// Analytics convenience method
const analyticsConvenienceResult = await client.batchUpsertAnalytics(
'test_performance',
[{ name: 'sharpe_ratio', value: 1.25 }],
'name'
);
console.log('Analytics convenience method result:', analyticsConvenienceResult);
// Trading convenience method
const tradingConvenienceResult = await client.batchUpsertTrading(
'test_orders',
[{ name: 'order_001', value: 100 }],
'name'
);
console.log('Trading convenience method result:', tradingConvenienceResult);
// Test 7: Direct database access
console.log('\n🎯 Testing direct database access...');
const stockDb = client.getDatabase('stock');
const analyticsDb = client.getDatabase('analytics');
const tradingDb = client.getDatabase('trading_documents');
console.log('Available databases:', {
stock: stockDb.databaseName,
analytics: analyticsDb.databaseName,
trading: tradingDb.databaseName,
});
// Test 8: Collection operations with database override
console.log('\n📋 Testing collection operations with database override...');
const stockCollection = client.getCollection('test_symbols', 'stock');
const stockCount = await stockCollection.countDocuments();
console.log(`Stock test_symbols count: ${stockCount}`);
const analyticsCollection = client.getCollection('test_metrics', 'analytics');
const analyticsCount = await analyticsCollection.countDocuments();
console.log(`Analytics test_metrics count: ${analyticsCount}`);
// Test 9: InsertOne with database override
console.log('\n➕ Testing insertOne with database override...');
const insertResult = await client.insertOne(
'test_single',
{ name: 'single_test', value: 999 },
'stock'
);
console.log('InsertOne result:', insertResult);
console.log('\n🎉 All multi-database tests completed successfully!');
} catch (error) {
console.error('❌ Test failed:', error);
} finally {
console.log('\n🔌 Disconnecting from MongoDB...');
await client.disconnect();
console.log('✅ Disconnected successfully!');
}
}
// Run the test
testMultiDatabaseSupport().catch(console.error);

0
test-mongodb-simplified.ts
View file

0
test-query-performance.ts
View file

0
test-shutdown-simple.ts
View file

0
test-shutdown.ts
View file

0
test-signals.ts
View file