From bd8a5bfe9e0ec9c6d8ae45e8e176a73294407e8d Mon Sep 17 00:00:00 2001 From: Boki Date: Wed, 18 Jun 2025 19:08:51 -0400 Subject: [PATCH] starting data-sync service --- apps/data-sync-service/config/default.json | 10 +- apps/data-sync-service/src/index.ts | 23 ++-- apps/web-api/config/default.json | 44 ------- bun.lock | 12 +- config/default.json | 27 ++-- test-large-scale-performance.ts | 0 test-mongodb-batch.ts | 0 test-mongodb-multi-db.ts | 137 --------------------- test-mongodb-simplified.ts | 0 test-query-performance.ts | 0 test-shutdown-simple.ts | 0 test-shutdown.ts | 0 test-signals.ts | 0 13 files changed, 43 insertions(+), 210 deletions(-) delete mode 100644 test-large-scale-performance.ts delete mode 100644 test-mongodb-batch.ts delete mode 100644 test-mongodb-multi-db.ts delete mode 100644 test-mongodb-simplified.ts delete mode 100644 test-query-performance.ts delete mode 100644 test-shutdown-simple.ts delete mode 100644 test-shutdown.ts delete mode 100644 test-signals.ts diff --git a/apps/data-sync-service/config/default.json b/apps/data-sync-service/config/default.json index b22641d..fa994d5 100644 --- a/apps/data-sync-service/config/default.json +++ b/apps/data-sync-service/config/default.json @@ -2,6 +2,14 @@ "service": { "name": "data-sync-service", "port": 3005, - "environment": "development" + "host": "0.0.0.0", + "healthCheckPath": "/health", + "metricsPath": "/metrics", + "shutdownTimeout": 30000, + "cors": { + "enabled": true, + "origin": "*", + "credentials": false + } } } \ No newline at end of file diff --git a/apps/data-sync-service/src/index.ts b/apps/data-sync-service/src/index.ts index 262d318..4a20573 100644 --- a/apps/data-sync-service/src/index.ts +++ b/apps/data-sync-service/src/index.ts @@ -3,7 +3,7 @@ */ import { Hono } from 'hono'; import { cors } from 'hono/cors'; -import { initializeConfig, getServiceConfig, getLoggingConfig } from '@stock-bot/config-new'; +import { initializeConfig, getServiceConfig, getLoggingConfig, getDatabaseConfig } from '@stock-bot/config-new'; import { getLogger, shutdownLoggers, setLoggerConfig } from '@stock-bot/logger'; import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client'; import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client'; @@ -13,8 +13,9 @@ import { syncManager } from './services/sync-manager'; import { setPostgreSQLClient, setMongoDBClient } from './clients'; // Initialize configuration -await initializeConfig(); +await initializeConfig('./config'); const serviceConfig = getServiceConfig(); +const databaseConfig = getDatabaseConfig(); // Initialize logger with config const loggingConfig = getLoggingConfig(); @@ -191,16 +192,16 @@ async function initializeServices() { try { // Initialize MongoDB client logger.info('Connecting to MongoDB...'); - const mongoConfig = serviceConfig.database.mongodb; + const mongoConfig = databaseConfig.mongodb; mongoClient = await createAndConnectMongoDBClient({ uri: mongoConfig.uri, database: mongoConfig.database, - host: 'localhost', - port: 27017, + host: mongoConfig.host || 'localhost', + port: mongoConfig.port || 27017, timeouts: { - connectTimeout: mongoConfig.connectionTimeout || 30000, + connectTimeout: 30000, socketTimeout: 30000, - serverSelectionTimeout: mongoConfig.serverSelectionTimeout || 5000, + serverSelectionTimeout: 5000, }, }); setMongoDBClient(mongoClient); @@ -208,17 +209,17 @@ async function initializeServices() { // Initialize PostgreSQL client logger.info('Connecting to PostgreSQL...'); - const pgConfig = serviceConfig.database.postgres; + const pgConfig = databaseConfig.postgres; postgresClient = await createAndConnectPostgreSQLClient({ host: pgConfig.host, port: pgConfig.port, database: pgConfig.database, - username: pgConfig.username, + username: pgConfig.user, password: pgConfig.password, poolSettings: { min: 2, - max: pgConfig.maxConnections || 10, - idleTimeoutMillis: 30000, + max: pgConfig.poolSize || 10, + idleTimeoutMillis: pgConfig.idleTimeout || 30000, }, }); setPostgreSQLClient(postgresClient); diff --git a/apps/web-api/config/default.json b/apps/web-api/config/default.json index c4e227c..dacdb89 100644 --- a/apps/web-api/config/default.json +++ b/apps/web-api/config/default.json @@ -11,49 +11,5 @@ "origin": ["http://localhost:4200", "http://localhost:3000", "http://localhost:3002"], "credentials": true } - }, - "logging": { - "level": "info", - "format": "json" - }, - "database": { - "postgres": { - "host": "localhost", - "port": 5432, - "database": "trading_bot", - "user": "trading_user", - "password": "trading_pass_dev", - "ssl": false, - "poolSize": 10, - "connectionTimeout": 30000, - "idleTimeout": 10000 - }, - "questdb": { - "host": "localhost", - "ilpPort": 9009, - "httpPort": 9000, - "pgPort": 8812, - "database": "questdb", - "user": "admin", - "password": "quest", - "bufferSize": 65536, - "flushInterval": 1000 - }, - "mongodb": { - "host": "localhost", - "port": 27017, - "database": "stock", - "user": "trading_admin", - "password": "trading_mongo_dev", - "authSource": "admin", - "poolSize": 10 - }, - "dragonfly": { - "host": "localhost", - "port": 6379, - "db": 0, - "maxRetries": 3, - "retryDelay": 100 - } } } \ No newline at end of file diff --git a/bun.lock b/bun.lock index c9ae2f9..4e94000 100644 --- a/bun.lock +++ b/bun.lock @@ -47,7 +47,6 @@ "version": "1.0.0", "dependencies": { "@stock-bot/cache": "*", - "@stock-bot/config": "*", "@stock-bot/event-bus": "*", "@stock-bot/http": "*", "@stock-bot/logger": "*", @@ -71,7 +70,6 @@ "name": "@stock-bot/data-sync-service", "version": "1.0.0", "dependencies": { - "@stock-bot/config-new": "*", "@stock-bot/logger": "*", "@stock-bot/mongodb-client": "*", "@stock-bot/postgres-client": "*", @@ -153,6 +151,7 @@ "name": "@stock-bot/cache", "version": "1.0.0", "dependencies": { + "@stock-bot/logger": "*", "ioredis": "^5.3.2", }, "devDependencies": { @@ -212,7 +211,6 @@ "name": "@stock-bot/event-bus", "version": "1.0.0", "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", "eventemitter3": "^5.0.1", "ioredis": "^5.3.2", @@ -249,7 +247,6 @@ "name": "@stock-bot/logger", "version": "1.0.0", "dependencies": { - "@stock-bot/config": "*", "got": "^14.4.7", "pino": "^9.7.0", "pino-loki": "^2.6.0", @@ -265,12 +262,10 @@ "name": "@stock-bot/mongodb-client", "version": "1.0.0", "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", "@stock-bot/types": "*", "@types/mongodb": "^4.0.7", "mongodb": "^6.17.0", - "yup": "^1.6.1", }, "devDependencies": { "@types/node": "^20.11.0", @@ -285,11 +280,9 @@ "name": "@stock-bot/postgres-client", "version": "1.0.0", "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", "@stock-bot/types": "*", "pg": "^8.11.3", - "yup": "^1.6.1", }, "devDependencies": { "@types/node": "^20.11.0", @@ -313,9 +306,9 @@ "name": "@stock-bot/questdb-client", "version": "1.0.0", "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", "@stock-bot/types": "*", + "pg": "^8.11.3", }, "devDependencies": { "@types/node": "^20.11.0", @@ -352,7 +345,6 @@ "name": "@stock-bot/strategy-engine", "version": "1.0.0", "dependencies": { - "@stock-bot/config": "*", "@stock-bot/data-frame": "*", "@stock-bot/event-bus": "*", "@stock-bot/logger": "*", diff --git a/config/default.json b/config/default.json index 51ab131..87e1d7b 100644 --- a/config/default.json +++ b/config/default.json @@ -13,28 +13,41 @@ "host": "localhost", "port": 5432, "database": "trading_bot", - "username": "trading_user", + "user": "trading_user", "password": "trading_pass_dev", - "maxConnections": 10 + "ssl": false, + "poolSize": 10, + "connectionTimeout": 30000, + "idleTimeout": 10000 }, "questdb": { "host": "localhost", + "ilpPort": 9009, "httpPort": 9000, "pgPort": 8812, - "database": "questdb" + "database": "questdb", + "user": "admin", + "password": "quest", + "bufferSize": 65536, + "flushInterval": 1000 }, "mongodb": { "uri": "mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin", + "host": "localhost", + "port": 27017, "database": "stock", - "connectionTimeout": 30000, - "serverSelectionTimeout": 5000 + "user": "trading_admin", + "password": "trading_mongo_dev", + "authSource": "admin", + "poolSize": 10 }, "dragonfly": { "host": "localhost", "port": 6379, - "password": "", "db": 0, - "keyPrefix": "stock-bot:" + "keyPrefix": "stock-bot:", + "maxRetries": 3, + "retryDelay": 100 } }, "logging": { diff --git a/test-large-scale-performance.ts b/test-large-scale-performance.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test-mongodb-batch.ts b/test-mongodb-batch.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test-mongodb-multi-db.ts b/test-mongodb-multi-db.ts deleted file mode 100644 index ff4add9..0000000 --- a/test-mongodb-multi-db.ts +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bun -/** - * Test script for MongoDB Client with Multiple Database Support - */ -import { MongoDBClient } from './libs/mongodb-client/src/client'; - -interface TestDocument { - _id?: string; - name: string; - value: number; - created_at?: Date; - updated_at?: Date; -} - -async function testMultiDatabaseSupport() { - const client = MongoDBClient.getInstance(); - - try { - console.log('šŸ”Œ Connecting to MongoDB...'); - await client.connect(); - console.log('āœ… Connected successfully!'); - - // Test 1: Check default database - console.log('\nšŸ“Š Testing Default Database Operations'); - console.log(`Default database: ${client.getDefaultDatabase()}`); - - // Test 2: Insert into default database (stock) - const stockData: TestDocument[] = [ - { name: 'AAPL', value: 150.25 }, - { name: 'GOOGL', value: 2750.8 }, - { name: 'MSFT', value: 305.15 }, - ]; - - console.log('\nšŸ’¾ Inserting into default database (stock)...'); - const stockResult = await client.batchUpsert('test_symbols', stockData, 'name'); - console.log('Stock database result:', stockResult); - - // Test 3: Change default database - console.log('\nšŸ”„ Changing default database to analytics...'); - client.setDefaultDatabase('analytics'); - console.log(`New default database: ${client.getDefaultDatabase()}`); - - // Test 4: Insert into new default database - const analyticsData: TestDocument[] = [ - { name: 'daily_volume', value: 1000000 }, - { name: 'avg_price', value: 125.5 }, - { name: 'volatility', value: 0.25 }, - ]; - - console.log('\nšŸ“ˆ Inserting into new default database (analytics)...'); - const analyticsResult = await client.batchUpsert('test_metrics', analyticsData, 'name'); - console.log('Analytics database result:', analyticsResult); - - // Test 5: Explicitly specify database (override default) - const tradingData: TestDocument[] = [ - { name: 'NYSE', value: 1 }, - { name: 'NASDAQ', value: 2 }, - { name: 'LSE', value: 3 }, - ]; - - console.log('\nšŸ¦ Inserting into specific database (trading_documents)...'); - const tradingResult = await client.batchUpsert('test_exchanges', tradingData, 'name', { - database: 'trading_documents', - }); - console.log('Trading documents database result:', tradingResult); - - // Test 6: Use convenience methods - console.log('\nšŸš€ Testing convenience methods...'); - - // Stock convenience method - const stockConvenienceResult = await client.batchUpsertStock( - 'test_prices', - [{ name: 'TSLA', value: 800.25 }], - 'name' - ); - console.log('Stock convenience method result:', stockConvenienceResult); - - // Analytics convenience method - const analyticsConvenienceResult = await client.batchUpsertAnalytics( - 'test_performance', - [{ name: 'sharpe_ratio', value: 1.25 }], - 'name' - ); - console.log('Analytics convenience method result:', analyticsConvenienceResult); - - // Trading convenience method - const tradingConvenienceResult = await client.batchUpsertTrading( - 'test_orders', - [{ name: 'order_001', value: 100 }], - 'name' - ); - console.log('Trading convenience method result:', tradingConvenienceResult); - - // Test 7: Direct database access - console.log('\nšŸŽÆ Testing direct database access...'); - const stockDb = client.getDatabase('stock'); - const analyticsDb = client.getDatabase('analytics'); - const tradingDb = client.getDatabase('trading_documents'); - - console.log('Available databases:', { - stock: stockDb.databaseName, - analytics: analyticsDb.databaseName, - trading: tradingDb.databaseName, - }); - - // Test 8: Collection operations with database override - console.log('\nšŸ“‹ Testing collection operations with database override...'); - - const stockCollection = client.getCollection('test_symbols', 'stock'); - const stockCount = await stockCollection.countDocuments(); - console.log(`Stock test_symbols count: ${stockCount}`); - - const analyticsCollection = client.getCollection('test_metrics', 'analytics'); - const analyticsCount = await analyticsCollection.countDocuments(); - console.log(`Analytics test_metrics count: ${analyticsCount}`); - - // Test 9: InsertOne with database override - console.log('\nāž• Testing insertOne with database override...'); - const insertResult = await client.insertOne( - 'test_single', - { name: 'single_test', value: 999 }, - 'stock' - ); - console.log('InsertOne result:', insertResult); - - console.log('\nšŸŽ‰ All multi-database tests completed successfully!'); - } catch (error) { - console.error('āŒ Test failed:', error); - } finally { - console.log('\nšŸ”Œ Disconnecting from MongoDB...'); - await client.disconnect(); - console.log('āœ… Disconnected successfully!'); - } -} - -// Run the test -testMultiDatabaseSupport().catch(console.error); diff --git a/test-mongodb-simplified.ts b/test-mongodb-simplified.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test-query-performance.ts b/test-query-performance.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test-shutdown-simple.ts b/test-shutdown-simple.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test-shutdown.ts b/test-shutdown.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test-signals.ts b/test-signals.ts deleted file mode 100644 index e69de29..0000000