starting data-sync service

apps/data-sync-service/config/default.json

@@ -2,6 +2,14 @@
   "service": {
     "name": "data-sync-service",
     "port": 3005,
-    "environment": "development"
+    "host": "0.0.0.0",
+    "healthCheckPath": "/health",
+    "metricsPath": "/metrics",
+    "shutdownTimeout": 30000,
+    "cors": {
+      "enabled": true,
+      "origin": "*",
+      "credentials": false
+    }
   }
 }

apps/data-sync-service/src/index.ts

@@ -3,7 +3,7 @@
  */
 import { Hono } from 'hono';
 import { cors } from 'hono/cors';
-import { initializeConfig, getServiceConfig, getLoggingConfig } from '@stock-bot/config-new';
+import { initializeConfig, getServiceConfig, getLoggingConfig, getDatabaseConfig } from '@stock-bot/config-new';
 import { getLogger, shutdownLoggers, setLoggerConfig } from '@stock-bot/logger';
 import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client';
 import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client';
@@ -13,8 +13,9 @@ import { syncManager } from './services/sync-manager';
 import { setPostgreSQLClient, setMongoDBClient } from './clients';
 
 // Initialize configuration
-await initializeConfig();
+await initializeConfig('./config');
 const serviceConfig = getServiceConfig();
+const databaseConfig = getDatabaseConfig();
 
 // Initialize logger with config
 const loggingConfig = getLoggingConfig();
@@ -191,16 +192,16 @@ async function initializeServices() {
   try {
     // Initialize MongoDB client
     logger.info('Connecting to MongoDB...');
-    const mongoConfig = serviceConfig.database.mongodb;
+    const mongoConfig = databaseConfig.mongodb;
     mongoClient = await createAndConnectMongoDBClient({
       uri: mongoConfig.uri,
       database: mongoConfig.database,
-      host: 'localhost',
+      host: mongoConfig.host || 'localhost',
-      port: 27017,
+      port: mongoConfig.port || 27017,
       timeouts: {
-        connectTimeout: mongoConfig.connectionTimeout || 30000,
+        connectTimeout: 30000,
         socketTimeout: 30000,
-        serverSelectionTimeout: mongoConfig.serverSelectionTimeout || 5000,
+        serverSelectionTimeout: 5000,
       },
     });
     setMongoDBClient(mongoClient);
@@ -208,17 +209,17 @@ async function initializeServices() {
 
     // Initialize PostgreSQL client
     logger.info('Connecting to PostgreSQL...');
-    const pgConfig = serviceConfig.database.postgres;
+    const pgConfig = databaseConfig.postgres;
     postgresClient = await createAndConnectPostgreSQLClient({
       host: pgConfig.host,
       port: pgConfig.port,
       database: pgConfig.database,
-      username: pgConfig.username,
+      username: pgConfig.user,
       password: pgConfig.password,
       poolSettings: {
         min: 2,
-        max: pgConfig.maxConnections || 10,
+        max: pgConfig.poolSize || 10,
-        idleTimeoutMillis: 30000,
+        idleTimeoutMillis: pgConfig.idleTimeout || 30000,
       },
     });
     setPostgreSQLClient(postgresClient);

apps/web-api/config/default.json

@@ -11,49 +11,5 @@
       "origin": ["http://localhost:4200", "http://localhost:3000", "http://localhost:3002"],
       "credentials": true
     }
-  },
-  "logging": {
-    "level": "info",
-    "format": "json"
-  },
-  "database": {
-    "postgres": {
-      "host": "localhost",
-      "port": 5432,
-      "database": "trading_bot",
-      "user": "trading_user",
-      "password": "trading_pass_dev",
-      "ssl": false,
-      "poolSize": 10,
-      "connectionTimeout": 30000,
-      "idleTimeout": 10000
-    },
-    "questdb": {
-      "host": "localhost",
-      "ilpPort": 9009,
-      "httpPort": 9000,
-      "pgPort": 8812,
-      "database": "questdb",
-      "user": "admin",
-      "password": "quest",
-      "bufferSize": 65536,
-      "flushInterval": 1000
-    },
-    "mongodb": {
-      "host": "localhost",
-      "port": 27017,
-      "database": "stock",
-      "user": "trading_admin",
-      "password": "trading_mongo_dev",
-      "authSource": "admin",
-      "poolSize": 10
-    },
-    "dragonfly": {
-      "host": "localhost",
-      "port": 6379,
-      "db": 0,
-      "maxRetries": 3,
-      "retryDelay": 100
-    }
   }
 }

bun.lock

@@ -47,7 +47,6 @@
       "version": "1.0.0",
       "dependencies": {
         "@stock-bot/cache": "*",
-        "@stock-bot/config": "*",
         "@stock-bot/event-bus": "*",
         "@stock-bot/http": "*",
         "@stock-bot/logger": "*",
@@ -71,7 +70,6 @@
       "name": "@stock-bot/data-sync-service",
       "version": "1.0.0",
       "dependencies": {
-        "@stock-bot/config-new": "*",
         "@stock-bot/logger": "*",
         "@stock-bot/mongodb-client": "*",
         "@stock-bot/postgres-client": "*",
@@ -153,6 +151,7 @@
       "name": "@stock-bot/cache",
       "version": "1.0.0",
       "dependencies": {
+        "@stock-bot/logger": "*",
         "ioredis": "^5.3.2",
       },
       "devDependencies": {
@@ -212,7 +211,6 @@
       "name": "@stock-bot/event-bus",
       "version": "1.0.0",
       "dependencies": {
-        "@stock-bot/config": "*",
         "@stock-bot/logger": "*",
         "eventemitter3": "^5.0.1",
         "ioredis": "^5.3.2",
@@ -249,7 +247,6 @@
       "name": "@stock-bot/logger",
       "version": "1.0.0",
       "dependencies": {
-        "@stock-bot/config": "*",
         "got": "^14.4.7",
         "pino": "^9.7.0",
         "pino-loki": "^2.6.0",
@@ -265,12 +262,10 @@
       "name": "@stock-bot/mongodb-client",
       "version": "1.0.0",
       "dependencies": {
-        "@stock-bot/config": "*",
         "@stock-bot/logger": "*",
         "@stock-bot/types": "*",
         "@types/mongodb": "^4.0.7",
         "mongodb": "^6.17.0",
-        "yup": "^1.6.1",
       },
       "devDependencies": {
         "@types/node": "^20.11.0",
@@ -285,11 +280,9 @@
       "name": "@stock-bot/postgres-client",
       "version": "1.0.0",
       "dependencies": {
-        "@stock-bot/config": "*",
         "@stock-bot/logger": "*",
         "@stock-bot/types": "*",
         "pg": "^8.11.3",
-        "yup": "^1.6.1",
       },
       "devDependencies": {
         "@types/node": "^20.11.0",
@@ -313,9 +306,9 @@
       "name": "@stock-bot/questdb-client",
       "version": "1.0.0",
       "dependencies": {
-        "@stock-bot/config": "*",
         "@stock-bot/logger": "*",
         "@stock-bot/types": "*",
+        "pg": "^8.11.3",
       },
       "devDependencies": {
         "@types/node": "^20.11.0",
@@ -352,7 +345,6 @@
       "name": "@stock-bot/strategy-engine",
       "version": "1.0.0",
       "dependencies": {
-        "@stock-bot/config": "*",
         "@stock-bot/data-frame": "*",
         "@stock-bot/event-bus": "*",
         "@stock-bot/logger": "*",

config/default.json

@@ -13,28 +13,41 @@
       "host": "localhost",
       "port": 5432,
       "database": "trading_bot",
-      "username": "trading_user",
+      "user": "trading_user",
       "password": "trading_pass_dev",
-      "maxConnections": 10
+      "ssl": false,
+      "poolSize": 10,
+      "connectionTimeout": 30000,
+      "idleTimeout": 10000
     },
     "questdb": {
       "host": "localhost",
+      "ilpPort": 9009,
       "httpPort": 9000,
       "pgPort": 8812,
-      "database": "questdb"
+      "database": "questdb",
+      "user": "admin",
+      "password": "quest",
+      "bufferSize": 65536,
+      "flushInterval": 1000
     },
     "mongodb": {
       "uri": "mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin",
+      "host": "localhost",
+      "port": 27017,
       "database": "stock",
-      "connectionTimeout": 30000,
+      "user": "trading_admin",
-      "serverSelectionTimeout": 5000
+      "password": "trading_mongo_dev",
+      "authSource": "admin",
+      "poolSize": 10
     },
     "dragonfly": {
       "host": "localhost",
       "port": 6379,
-      "password": "",
       "db": 0,
-      "keyPrefix": "stock-bot:"
+      "keyPrefix": "stock-bot:",
+      "maxRetries": 3,
+      "retryDelay": 100
     }
   },
   "logging": {

test-mongodb-multi-db.ts

@@ -1,137 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Test script for MongoDB Client with Multiple Database Support
- */
-import { MongoDBClient } from './libs/mongodb-client/src/client';
-
-interface TestDocument {
-  _id?: string;
-  name: string;
-  value: number;
-  created_at?: Date;
-  updated_at?: Date;
-}
-
-async function testMultiDatabaseSupport() {
-  const client = MongoDBClient.getInstance();
-
-  try {
-    console.log('🔌 Connecting to MongoDB...');
-    await client.connect();
-    console.log('✅ Connected successfully!');
-
-    // Test 1: Check default database
-    console.log('\n📊 Testing Default Database Operations');
-    console.log(`Default database: ${client.getDefaultDatabase()}`);
-
-    // Test 2: Insert into default database (stock)
-    const stockData: TestDocument[] = [
-      { name: 'AAPL', value: 150.25 },
-      { name: 'GOOGL', value: 2750.8 },
-      { name: 'MSFT', value: 305.15 },
-    ];
-
-    console.log('\n💾 Inserting into default database (stock)...');
-    const stockResult = await client.batchUpsert('test_symbols', stockData, 'name');
-    console.log('Stock database result:', stockResult);
-
-    // Test 3: Change default database
-    console.log('\n🔄 Changing default database to analytics...');
-    client.setDefaultDatabase('analytics');
-    console.log(`New default database: ${client.getDefaultDatabase()}`);
-
-    // Test 4: Insert into new default database
-    const analyticsData: TestDocument[] = [
-      { name: 'daily_volume', value: 1000000 },
-      { name: 'avg_price', value: 125.5 },
-      { name: 'volatility', value: 0.25 },
-    ];
-
-    console.log('\n📈 Inserting into new default database (analytics)...');
-    const analyticsResult = await client.batchUpsert('test_metrics', analyticsData, 'name');
-    console.log('Analytics database result:', analyticsResult);
-
-    // Test 5: Explicitly specify database (override default)
-    const tradingData: TestDocument[] = [
-      { name: 'NYSE', value: 1 },
-      { name: 'NASDAQ', value: 2 },
-      { name: 'LSE', value: 3 },
-    ];
-
-    console.log('\n🏦 Inserting into specific database (trading_documents)...');
-    const tradingResult = await client.batchUpsert('test_exchanges', tradingData, 'name', {
-      database: 'trading_documents',
-    });
-    console.log('Trading documents database result:', tradingResult);
-
-    // Test 6: Use convenience methods
-    console.log('\n🚀 Testing convenience methods...');
-
-    // Stock convenience method
-    const stockConvenienceResult = await client.batchUpsertStock(
-      'test_prices',
-      [{ name: 'TSLA', value: 800.25 }],
-      'name'
-    );
-    console.log('Stock convenience method result:', stockConvenienceResult);
-
-    // Analytics convenience method
-    const analyticsConvenienceResult = await client.batchUpsertAnalytics(
-      'test_performance',
-      [{ name: 'sharpe_ratio', value: 1.25 }],
-      'name'
-    );
-    console.log('Analytics convenience method result:', analyticsConvenienceResult);
-
-    // Trading convenience method
-    const tradingConvenienceResult = await client.batchUpsertTrading(
-      'test_orders',
-      [{ name: 'order_001', value: 100 }],
-      'name'
-    );
-    console.log('Trading convenience method result:', tradingConvenienceResult);
-
-    // Test 7: Direct database access
-    console.log('\n🎯 Testing direct database access...');
-    const stockDb = client.getDatabase('stock');
-    const analyticsDb = client.getDatabase('analytics');
-    const tradingDb = client.getDatabase('trading_documents');
-
-    console.log('Available databases:', {
-      stock: stockDb.databaseName,
-      analytics: analyticsDb.databaseName,
-      trading: tradingDb.databaseName,
-    });
-
-    // Test 8: Collection operations with database override
-    console.log('\n📋 Testing collection operations with database override...');
-
-    const stockCollection = client.getCollection('test_symbols', 'stock');
-    const stockCount = await stockCollection.countDocuments();
-    console.log(`Stock test_symbols count: ${stockCount}`);
-
-    const analyticsCollection = client.getCollection('test_metrics', 'analytics');
-    const analyticsCount = await analyticsCollection.countDocuments();
-    console.log(`Analytics test_metrics count: ${analyticsCount}`);
-
-    // Test 9: InsertOne with database override
-    console.log('\n➕ Testing insertOne with database override...');
-    const insertResult = await client.insertOne(
-      'test_single',
-      { name: 'single_test', value: 999 },
-      'stock'
-    );
-    console.log('InsertOne result:', insertResult);
-
-    console.log('\n🎉 All multi-database tests completed successfully!');
-  } catch (error) {
-    console.error('❌ Test failed:', error);
-  } finally {
-    console.log('\n🔌 Disconnecting from MongoDB...');
-    await client.disconnect();
-    console.log('✅ Disconnected successfully!');
-  }
-}
-
-// Run the test
-testMultiDatabaseSupport().catch(console.error);