diff --git a/CLAUDE.md b/CLAUDE.md index 1e77f7e..648162c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co - **strategy-service** - Trading strategies and backtesting (multi-mode: live, event-driven, vectorized, hybrid) - **execution-service** - Order management and risk controls - **portfolio-service** - Position tracking and performance analytics -- **web** - React dashboard with real-time updates +- **web-app** - React dashboard with real-time updates ### Shared Libraries (`libs/`) - **config** - Environment configuration with Zod validation diff --git a/apps/data-sync-service/src/clients.ts b/apps/data-sync-service/src/clients.ts new file mode 100644 index 0000000..5dfe266 --- /dev/null +++ b/apps/data-sync-service/src/clients.ts @@ -0,0 +1,27 @@ +import { PostgreSQLClient } from '@stock-bot/postgres-client'; +import { MongoDBClient } from '@stock-bot/mongodb-client'; + +let postgresClient: PostgreSQLClient | null = null; +let mongodbClient: MongoDBClient | null = null; + +export function setPostgreSQLClient(client: PostgreSQLClient): void { + postgresClient = client; +} + +export function getPostgreSQLClient(): PostgreSQLClient { + if (!postgresClient) { + throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); + } + return postgresClient; +} + +export function setMongoDBClient(client: MongoDBClient): void { + mongodbClient = client; +} + +export function getMongoDBClient(): MongoDBClient { + if (!mongodbClient) { + throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); + } + return mongodbClient; +} \ No newline at end of file diff --git a/apps/data-sync-service/src/index.ts b/apps/data-sync-service/src/index.ts index c689fb5..262d318 100644 --- a/apps/data-sync-service/src/index.ts +++ b/apps/data-sync-service/src/index.ts @@ -3,16 +3,29 @@ */ import { Hono } from 'hono'; import { cors } from 'hono/cors'; -import { initializeConfig, getServiceConfig } from '@stock-bot/config-new'; -import { getLogger, shutdownLoggers } from '@stock-bot/logger'; -import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client'; -import { connectPostgreSQL, disconnectPostgreSQL } from '@stock-bot/postgres-client'; +import { initializeConfig, getServiceConfig, getLoggingConfig } from '@stock-bot/config-new'; +import { getLogger, shutdownLoggers, setLoggerConfig } from '@stock-bot/logger'; +import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client'; +import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client'; import { Shutdown } from '@stock-bot/shutdown'; import { enhancedSyncManager } from './services/enhanced-sync-manager'; import { syncManager } from './services/sync-manager'; +import { setPostgreSQLClient, setMongoDBClient } from './clients'; // Initialize configuration await initializeConfig(); +const serviceConfig = getServiceConfig(); + +// Initialize logger with config +const loggingConfig = getLoggingConfig(); +if (loggingConfig) { + setLoggerConfig({ + logLevel: loggingConfig.level, + logConsole: true, + logFile: false, + environment: serviceConfig.environment, + }); +} const app = new Hono(); @@ -28,9 +41,10 @@ app.use( ); const logger = getLogger('data-sync-service'); -const serviceConfig = getServiceConfig(); const PORT = serviceConfig.port; let server: ReturnType | null = null; +let postgresClient: PostgreSQLClient | null = null; +let mongoClient: MongoDBClient | null = null; // Initialize shutdown manager const shutdown = Shutdown.getInstance({ timeout: 15000 }); @@ -177,12 +191,37 @@ async function initializeServices() { try { // Initialize MongoDB client logger.info('Connecting to MongoDB...'); - await connectMongoDB(); + const mongoConfig = serviceConfig.database.mongodb; + mongoClient = await createAndConnectMongoDBClient({ + uri: mongoConfig.uri, + database: mongoConfig.database, + host: 'localhost', + port: 27017, + timeouts: { + connectTimeout: mongoConfig.connectionTimeout || 30000, + socketTimeout: 30000, + serverSelectionTimeout: mongoConfig.serverSelectionTimeout || 5000, + }, + }); + setMongoDBClient(mongoClient); logger.info('MongoDB connected'); // Initialize PostgreSQL client logger.info('Connecting to PostgreSQL...'); - await connectPostgreSQL(); + const pgConfig = serviceConfig.database.postgres; + postgresClient = await createAndConnectPostgreSQLClient({ + host: pgConfig.host, + port: pgConfig.port, + database: pgConfig.database, + username: pgConfig.username, + password: pgConfig.password, + poolSettings: { + min: 2, + max: pgConfig.maxConnections || 10, + idleTimeoutMillis: 30000, + }, + }); + setPostgreSQLClient(postgresClient); logger.info('PostgreSQL connected'); // Initialize sync managers @@ -238,8 +277,12 @@ shutdown.onShutdown(async () => { shutdown.onShutdown(async () => { logger.info('Disconnecting from databases...'); try { - await disconnectMongoDB(); - await disconnectPostgreSQL(); + if (mongoClient) { + await mongoClient.disconnect(); + } + if (postgresClient) { + await postgresClient.disconnect(); + } logger.info('Database connections closed'); } catch (error) { logger.error('Error closing database connections', { error }); diff --git a/apps/data-sync-service/src/services/enhanced-sync-manager.ts b/apps/data-sync-service/src/services/enhanced-sync-manager.ts index 82fdff6..fe47b5d 100644 --- a/apps/data-sync-service/src/services/enhanced-sync-manager.ts +++ b/apps/data-sync-service/src/services/enhanced-sync-manager.ts @@ -2,8 +2,7 @@ * Enhanced Sync Manager - Improved syncing with comprehensive exchange data */ import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient } from '@stock-bot/mongodb-client'; -import { getPostgreSQLClient } from '@stock-bot/postgres-client'; +import { getMongoDBClient, getPostgreSQLClient } from '../clients'; const logger = getLogger('enhanced-sync-manager'); diff --git a/apps/data-sync-service/src/services/sync-manager.ts b/apps/data-sync-service/src/services/sync-manager.ts index 3f9c7d5..7336321 100644 --- a/apps/data-sync-service/src/services/sync-manager.ts +++ b/apps/data-sync-service/src/services/sync-manager.ts @@ -2,8 +2,7 @@ * Sync Manager - Handles syncing raw MongoDB data to PostgreSQL master records */ import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient } from '@stock-bot/mongodb-client'; -import { getPostgreSQLClient } from '@stock-bot/postgres-client'; +import { getMongoDBClient, getPostgreSQLClient } from '../clients'; const logger = getLogger('sync-manager'); diff --git a/apps/web-api/src/clients.ts b/apps/web-api/src/clients.ts new file mode 100644 index 0000000..5dfe266 --- /dev/null +++ b/apps/web-api/src/clients.ts @@ -0,0 +1,27 @@ +import { PostgreSQLClient } from '@stock-bot/postgres-client'; +import { MongoDBClient } from '@stock-bot/mongodb-client'; + +let postgresClient: PostgreSQLClient | null = null; +let mongodbClient: MongoDBClient | null = null; + +export function setPostgreSQLClient(client: PostgreSQLClient): void { + postgresClient = client; +} + +export function getPostgreSQLClient(): PostgreSQLClient { + if (!postgresClient) { + throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.'); + } + return postgresClient; +} + +export function setMongoDBClient(client: MongoDBClient): void { + mongodbClient = client; +} + +export function getMongoDBClient(): MongoDBClient { + if (!mongodbClient) { + throw new Error('MongoDB client not initialized. Call setMongoDBClient first.'); + } + return mongodbClient; +} \ No newline at end of file diff --git a/apps/web-api/src/index.ts b/apps/web-api/src/index.ts index 9effdce..2e7daec 100644 --- a/apps/web-api/src/index.ts +++ b/apps/web-api/src/index.ts @@ -3,17 +3,30 @@ */ import { Hono } from 'hono'; import { cors } from 'hono/cors'; -import { initializeConfig, getServiceConfig } from '@stock-bot/config-new'; -import { getLogger, shutdownLoggers } from '@stock-bot/logger'; -import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client'; -import { connectPostgreSQL, disconnectPostgreSQL } from '@stock-bot/postgres-client'; +import { initializeConfig, getServiceConfig, getLoggingConfig } from '@stock-bot/config-new'; +import { getLogger, shutdownLoggers, setLoggerConfig } from '@stock-bot/logger'; +import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client'; +import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client'; import { Shutdown } from '@stock-bot/shutdown'; +import { setPostgreSQLClient, setMongoDBClient } from './clients'; // Import routes import { exchangeRoutes } from './routes/exchange.routes'; import { healthRoutes } from './routes/health.routes'; // Initialize configuration await initializeConfig(); +const serviceConfig = getServiceConfig(); + +// Initialize logger with config +const loggingConfig = getLoggingConfig(); +if (loggingConfig) { + setLoggerConfig({ + logLevel: loggingConfig.level, + logConsole: true, + logFile: false, + environment: serviceConfig.environment, + }); +} const app = new Hono(); @@ -29,9 +42,10 @@ app.use( ); const logger = getLogger('web-api'); -const serviceConfig = getServiceConfig(); const PORT = serviceConfig.port; let server: ReturnType | null = null; +let postgresClient: PostgreSQLClient | null = null; +let mongoClient: MongoDBClient | null = null; // Initialize shutdown manager const shutdown = Shutdown.getInstance({ timeout: 15000 }); @@ -61,12 +75,37 @@ async function initializeServices() { try { // Initialize MongoDB client logger.info('Connecting to MongoDB...'); - await connectMongoDB(); + const mongoConfig = serviceConfig.database.mongodb; + mongoClient = await createAndConnectMongoDBClient({ + uri: mongoConfig.uri, + database: mongoConfig.database, + host: 'localhost', + port: 27017, + timeouts: { + connectTimeout: mongoConfig.connectionTimeout || 30000, + socketTimeout: 30000, + serverSelectionTimeout: mongoConfig.serverSelectionTimeout || 5000, + }, + }); + setMongoDBClient(mongoClient); logger.info('MongoDB connected'); // Initialize PostgreSQL client logger.info('Connecting to PostgreSQL...'); - await connectPostgreSQL(); + const pgConfig = serviceConfig.database.postgres; + postgresClient = await createAndConnectPostgreSQLClient({ + host: pgConfig.host, + port: pgConfig.port, + database: pgConfig.database, + username: pgConfig.username, + password: pgConfig.password, + poolSettings: { + min: 2, + max: pgConfig.maxConnections || 10, + idleTimeoutMillis: 30000, + }, + }); + setPostgreSQLClient(postgresClient); logger.info('PostgreSQL connected'); logger.info('All services initialized successfully'); @@ -105,8 +144,12 @@ shutdown.onShutdown(async () => { shutdown.onShutdown(async () => { logger.info('Disconnecting from databases...'); try { - await disconnectMongoDB(); - await disconnectPostgreSQL(); + if (mongoClient) { + await mongoClient.disconnect(); + } + if (postgresClient) { + await postgresClient.disconnect(); + } logger.info('Database connections closed'); } catch (error) { logger.error('Error closing database connections', { error }); diff --git a/apps/web-api/src/routes/health.routes.ts b/apps/web-api/src/routes/health.routes.ts index 93ea162..9ace743 100644 --- a/apps/web-api/src/routes/health.routes.ts +++ b/apps/web-api/src/routes/health.routes.ts @@ -3,8 +3,7 @@ */ import { Hono } from 'hono'; import { getLogger } from '@stock-bot/logger'; -import { getMongoDBClient } from '@stock-bot/mongodb-client'; -import { getPostgreSQLClient } from '@stock-bot/postgres-client'; +import { getPostgreSQLClient, getMongoDBClient } from '../clients'; const logger = getLogger('health-routes'); export const healthRoutes = new Hono(); diff --git a/apps/web-api/src/services/exchange.service.ts b/apps/web-api/src/services/exchange.service.ts index e58ccf2..3ab04f1 100644 --- a/apps/web-api/src/services/exchange.service.ts +++ b/apps/web-api/src/services/exchange.service.ts @@ -1,6 +1,5 @@ import { getLogger } from '@stock-bot/logger'; -import { getPostgreSQLClient } from '@stock-bot/postgres-client'; -import { getMongoDBClient } from '@stock-bot/mongodb-client'; +import { getPostgreSQLClient, getMongoDBClient } from '../clients'; import { Exchange, ExchangeWithMappings, diff --git a/libs/cache/package.json b/libs/cache/package.json index 1689588..3daaf61 100644 --- a/libs/cache/package.json +++ b/libs/cache/package.json @@ -11,7 +11,6 @@ "test": "bun test" }, "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", "ioredis": "^5.3.2" }, diff --git a/libs/cache/src/connection-manager.ts b/libs/cache/src/connection-manager.ts index a3856ad..e8dcb8e 100644 --- a/libs/cache/src/connection-manager.ts +++ b/libs/cache/src/connection-manager.ts @@ -1,11 +1,12 @@ import Redis from 'ioredis'; -import { dragonflyConfig } from '@stock-bot/config'; import { getLogger } from '@stock-bot/logger'; +import type { RedisConfig } from './types'; interface ConnectionConfig { name: string; singleton?: boolean; db?: number; + redisConfig: RedisConfig; } /** @@ -32,12 +33,12 @@ export class RedisConnectionManager { * @returns Redis connection instance */ getConnection(config: ConnectionConfig): Redis { - const { name, singleton = false, db } = config; + const { name, singleton = false, db, redisConfig } = config; if (singleton) { // Use shared connection across all instances if (!RedisConnectionManager.sharedConnections.has(name)) { - const connection = this.createConnection(name, db); + const connection = this.createConnection(name, redisConfig, db); RedisConnectionManager.sharedConnections.set(name, connection); this.logger.info(`Created shared Redis connection: ${name}`); } @@ -45,7 +46,7 @@ export class RedisConnectionManager { } else { // Create unique connection per instance const uniqueName = `${name}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; - const connection = this.createConnection(uniqueName, db); + const connection = this.createConnection(uniqueName, redisConfig, db); this.connections.set(uniqueName, connection); this.logger.info(`Created unique Redis connection: ${uniqueName}`); return connection; @@ -55,33 +56,31 @@ export class RedisConnectionManager { /** * Create a new Redis connection with configuration */ - private createConnection(name: string, db?: number): Redis { - const redisConfig = { - host: dragonflyConfig.DRAGONFLY_HOST, - port: dragonflyConfig.DRAGONFLY_PORT, - password: dragonflyConfig.DRAGONFLY_PASSWORD || undefined, - username: dragonflyConfig.DRAGONFLY_USERNAME || undefined, - db: db ?? dragonflyConfig.DRAGONFLY_DATABASE, - maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES, - retryDelayOnFailover: dragonflyConfig.DRAGONFLY_RETRY_DELAY, - connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT, - commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT, - keepAlive: dragonflyConfig.DRAGONFLY_ENABLE_KEEPALIVE - ? dragonflyConfig.DRAGONFLY_KEEPALIVE_INTERVAL * 1000 - : 0, + private createConnection(name: string, config: RedisConfig, db?: number): Redis { + const redisOptions = { + host: config.host, + port: config.port, + password: config.password || undefined, + username: config.username || undefined, + db: db ?? config.db ?? 0, + maxRetriesPerRequest: config.maxRetriesPerRequest ?? 3, + retryDelayOnFailover: config.retryDelayOnFailover ?? 100, + connectTimeout: config.connectTimeout ?? 10000, + commandTimeout: config.commandTimeout ?? 5000, + keepAlive: config.keepAlive ?? 0, connectionName: name, lazyConnect: false, // Connect immediately instead of waiting for first command - ...(dragonflyConfig.DRAGONFLY_TLS && { + ...(config.tls && { tls: { - cert: dragonflyConfig.DRAGONFLY_TLS_CERT_FILE || undefined, - key: dragonflyConfig.DRAGONFLY_TLS_KEY_FILE || undefined, - ca: dragonflyConfig.DRAGONFLY_TLS_CA_FILE || undefined, - rejectUnauthorized: !dragonflyConfig.DRAGONFLY_TLS_SKIP_VERIFY, + cert: config.tls.cert || undefined, + key: config.tls.key || undefined, + ca: config.tls.ca || undefined, + rejectUnauthorized: config.tls.rejectUnauthorized ?? true, }, }), }; - const redis = new Redis(redisConfig); + const redis = new Redis(redisOptions); // Setup event handlers redis.on('connect', () => { diff --git a/libs/cache/src/index.ts b/libs/cache/src/index.ts index f0fddcb..be863ae 100644 --- a/libs/cache/src/index.ts +++ b/libs/cache/src/index.ts @@ -1,6 +1,6 @@ import { RedisConnectionManager } from './connection-manager'; import { RedisCache } from './redis-cache'; -import type { CacheOptions, CacheProvider } from './types'; +import type { CacheOptions, CacheProvider, RedisConfig } from './types'; // Cache instances registry to prevent multiple instances with same prefix const cacheInstances = new Map(); @@ -8,7 +8,7 @@ const cacheInstances = new Map(); /** * Create a Redis cache instance with trading-optimized defaults */ -export function createCache(options: Partial = {}): CacheProvider { +export function createCache(options: CacheOptions): CacheProvider { const defaultOptions: CacheOptions = { keyPrefix: 'cache:', ttl: 3600, // 1 hour default @@ -37,39 +37,42 @@ export function createCache(options: Partial = {}): CacheProvider /** * Create a cache instance for trading data */ -export function createTradingCache(options: Partial = {}): CacheProvider { +export function createTradingCache(redisConfig: RedisConfig, options?: Partial>): CacheProvider { return createCache({ keyPrefix: 'trading:', ttl: 3600, // 1 hour default enableMetrics: true, shared: true, ...options, + redisConfig, }); } /** * Create a cache for market data with shorter TTL */ -export function createMarketDataCache(options: Partial = {}): CacheProvider { +export function createMarketDataCache(redisConfig: RedisConfig, options?: Partial>): CacheProvider { return createCache({ keyPrefix: 'market:', ttl: 300, // 5 minutes for market data enableMetrics: true, shared: true, ...options, + redisConfig, }); } /** * Create a cache for indicators with longer TTL */ -export function createIndicatorCache(options: Partial = {}): CacheProvider { +export function createIndicatorCache(redisConfig: RedisConfig, options?: Partial>): CacheProvider { return createCache({ keyPrefix: 'indicators:', ttl: 1800, // 30 minutes for indicators enableMetrics: true, shared: true, ...options, + redisConfig, }); } @@ -81,6 +84,7 @@ export type { CacheStats, CacheKey, SerializationOptions, + RedisConfig, } from './types'; export { RedisCache } from './redis-cache'; diff --git a/libs/cache/src/redis-cache.ts b/libs/cache/src/redis-cache.ts index 02756e2..52747d0 100644 --- a/libs/cache/src/redis-cache.ts +++ b/libs/cache/src/redis-cache.ts @@ -25,7 +25,7 @@ export class RedisCache implements CacheProvider { uptime: 0, }; - constructor(options: CacheOptions = {}) { + constructor(options: CacheOptions) { this.defaultTTL = options.ttl ?? 3600; // 1 hour default this.keyPrefix = options.keyPrefix ?? 'cache:'; this.enableMetrics = options.enableMetrics ?? true; @@ -46,6 +46,7 @@ export class RedisCache implements CacheProvider { this.redis = this.connectionManager.getConnection({ name: `${baseName}-SERVICE`, singleton: options.shared ?? true, // Default to shared connection for cache + redisConfig: options.redisConfig, }); // Only setup event handlers for non-shared connections to avoid memory leaks diff --git a/libs/cache/src/types.ts b/libs/cache/src/types.ts index f19c8cb..cdaaca2 100644 --- a/libs/cache/src/types.ts +++ b/libs/cache/src/types.ts @@ -1,3 +1,23 @@ +export interface RedisConfig { + host: string; + port: number; + password?: string; + username?: string; + db?: number; + keyPrefix?: string; + maxRetriesPerRequest?: number; + retryDelayOnFailover?: number; + connectTimeout?: number; + commandTimeout?: number; + keepAlive?: number; + tls?: { + cert?: string; + key?: string; + ca?: string; + rejectUnauthorized?: boolean; + }; +} + export interface CacheProvider { get(key: string): Promise; set( @@ -64,6 +84,7 @@ export interface CacheOptions { enableMetrics?: boolean; name?: string; // Name for connection identification shared?: boolean; // Whether to use shared connection + redisConfig: RedisConfig; } export interface CacheStats { diff --git a/libs/event-bus/package.json b/libs/event-bus/package.json index 9aeb808..e6595b6 100644 --- a/libs/event-bus/package.json +++ b/libs/event-bus/package.json @@ -12,7 +12,6 @@ }, "dependencies": { "@stock-bot/logger": "*", - "@stock-bot/config": "*", "ioredis": "^5.3.2", "eventemitter3": "^5.0.1" }, diff --git a/libs/event-bus/src/index.ts b/libs/event-bus/src/index.ts index b26c26f..085712d 100644 --- a/libs/event-bus/src/index.ts +++ b/libs/event-bus/src/index.ts @@ -1,6 +1,5 @@ import { EventEmitter } from 'eventemitter3'; import Redis from 'ioredis'; -import { dragonflyConfig } from '@stock-bot/config'; import { getLogger } from '@stock-bot/logger'; export interface EventBusMessage { @@ -16,12 +15,21 @@ export interface EventHandler { (message: EventBusMessage & { data: T }): Promise | void; } +export interface RedisConfig { + host: string; + port: number; + password?: string; + db?: number; + maxRetriesPerRequest?: number; +} + export interface EventBusOptions { serviceName: string; enablePersistence?: boolean; useStreams?: boolean; maxRetries?: number; retryDelay?: number; + redisConfig: RedisConfig; } export interface StreamConsumerInfo { @@ -53,21 +61,22 @@ export class EventBus extends EventEmitter { this.retryDelay = options.retryDelay ?? 1000; this.logger = getLogger(`event-bus:${this.serviceName}`); + const { redisConfig } = options; this.redis = new Redis({ - host: dragonflyConfig.DRAGONFLY_HOST, - port: dragonflyConfig.DRAGONFLY_PORT, - password: dragonflyConfig.DRAGONFLY_PASSWORD, - db: dragonflyConfig.DRAGONFLY_DATABASE || 0, - maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES, + host: redisConfig.host, + port: redisConfig.port, + password: redisConfig.password, + db: redisConfig.db || 0, + maxRetriesPerRequest: redisConfig.maxRetriesPerRequest || 3, lazyConnect: false, }); if (!this.useStreams) { this.subscriber = new Redis({ - host: dragonflyConfig.DRAGONFLY_HOST, - port: dragonflyConfig.DRAGONFLY_PORT, - password: dragonflyConfig.DRAGONFLY_PASSWORD, - db: dragonflyConfig.DRAGONFLY_DATABASE || 0, + host: redisConfig.host, + port: redisConfig.port, + password: redisConfig.password, + db: redisConfig.db || 0, }); this.subscriber.on('message', this.handleRedisMessage.bind(this)); } diff --git a/libs/logger/package.json b/libs/logger/package.json index 4ad2ee4..fabcc7e 100644 --- a/libs/logger/package.json +++ b/libs/logger/package.json @@ -11,7 +11,6 @@ "test": "bun test" }, "dependencies": { - "@stock-bot/config": "*", "got": "^14.4.7", "pino": "^9.7.0", "pino-loki": "^2.6.0", diff --git a/libs/logger/src/index.ts b/libs/logger/src/index.ts index 3876b0d..db74377 100644 --- a/libs/logger/src/index.ts +++ b/libs/logger/src/index.ts @@ -5,10 +5,10 @@ */ // Core logger classes and functions -export { Logger, getLogger, shutdownLoggers } from './logger'; +export { Logger, getLogger, shutdownLoggers, setLoggerConfig } from './logger'; // Type definitions -export type { LogLevel, LogContext, LogMetadata } from './types'; +export type { LogLevel, LogContext, LogMetadata, LoggerConfig } from './types'; // Default export export { getLogger as default } from './logger'; diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index da5a459..194c23d 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -9,23 +9,41 @@ */ import pino from 'pino'; -import { loggingConfig, lokiConfig } from '@stock-bot/config'; -import type { LogContext, LogLevel, LogMetadata } from './types'; +import type { LogContext, LogLevel, LogMetadata, LoggerConfig } from './types'; // Simple cache for logger instances const loggerCache = new Map(); -console.log('Logger cache initialized: ', loggingConfig.LOG_LEVEL); + +// Global config that can be set +let globalConfig: LoggerConfig = { + logLevel: 'info', + logConsole: true, + logFile: false, + logFilePath: './logs', + logLoki: false, + environment: 'development', +}; + +/** + * Set global logger configuration + */ +export function setLoggerConfig(config: LoggerConfig): void { + globalConfig = { ...globalConfig, ...config }; + // Clear cache to force recreation with new config + loggerCache.clear(); + console.log('Logger config updated:', globalConfig.logLevel); +} /** * Create transport configuration */ -function createTransports(serviceName: string): any { +function createTransports(serviceName: string, config: LoggerConfig = globalConfig): any { const targets: any[] = []; - // const isDev = loggingConfig.LOG_ENVIRONMENT === 'development'; + // Console transport - if (loggingConfig.LOG_CONSOLE) { + if (config.logConsole) { targets.push({ target: 'pino-pretty', - level: loggingConfig.LOG_LEVEL, // Only show errors on console + level: config.logLevel || 'info', options: { colorize: true, translateTime: 'yyyy-mm-dd HH:MM:ss.l', @@ -40,29 +58,35 @@ function createTransports(serviceName: string): any { } // File transport - if (loggingConfig.LOG_FILE) { + if (config.logFile) { targets.push({ target: 'pino/file', - level: loggingConfig.LOG_LEVEL, + level: config.logLevel || 'info', options: { - destination: `${loggingConfig.LOG_FILE_PATH}/${serviceName}.log`, + destination: `${config.logFilePath}/${serviceName}.log`, mkdir: true, }, }); } // Loki transport - if (lokiConfig.LOKI_HOST) { + if (config.logLoki && config.lokiHost) { targets.push({ target: 'pino-loki', - level: loggingConfig.LOG_LEVEL, + level: config.logLevel || 'info', options: { - host: lokiConfig.LOKI_URL || `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`, + host: config.lokiHost, labels: { service: serviceName, - environment: lokiConfig.LOKI_ENVIRONMENT_LABEL, + environment: config.environment || 'development', }, ignore: 'childName', + ...(config.lokiUser && config.lokiPassword ? { + basicAuth: { + username: config.lokiUser, + password: config.lokiPassword, + }, + } : {}), }, }); } @@ -73,27 +97,28 @@ function createTransports(serviceName: string): any { /** * Get or create pino logger */ -function getPinoLogger(serviceName: string): pino.Logger { - if (!loggerCache.has(serviceName)) { - const transport = createTransports(serviceName); +function getPinoLogger(serviceName: string, config: LoggerConfig = globalConfig): pino.Logger { + const cacheKey = `${serviceName}-${JSON.stringify(config)}`; + if (!loggerCache.has(cacheKey)) { + const transport = createTransports(serviceName, config); - const config: pino.LoggerOptions = { - level: loggingConfig.LOG_LEVEL, + const loggerOptions: pino.LoggerOptions = { + level: config.logLevel || 'info', base: { service: serviceName, - environment: loggingConfig.LOG_ENVIRONMENT, - version: loggingConfig.LOG_SERVICE_VERSION, + environment: config.environment || 'development', + version: '1.0.0', }, }; if (transport) { - config.transport = transport; + loggerOptions.transport = transport; } - loggerCache.set(serviceName, pino(config)); + loggerCache.set(cacheKey, pino(loggerOptions)); } - return loggerCache.get(serviceName)!; + return loggerCache.get(cacheKey)!; } /** @@ -105,8 +130,8 @@ export class Logger { private serviceName: string; private childName?: string; - constructor(serviceName: string, context: LogContext = {}) { - this.pino = getPinoLogger(serviceName); + constructor(serviceName: string, context: LogContext = {}, config?: LoggerConfig) { + this.pino = getPinoLogger(serviceName, config); this.context = context; this.serviceName = serviceName; } @@ -232,8 +257,8 @@ export class Logger { /** * Main factory function */ -export function getLogger(serviceName: string, context?: LogContext): Logger { - return new Logger(serviceName, context); +export function getLogger(serviceName: string, context?: LogContext, config?: LoggerConfig): Logger { + return new Logger(serviceName, context, config); } /** diff --git a/libs/logger/src/types.ts b/libs/logger/src/types.ts index 5d4e298..3cde37a 100644 --- a/libs/logger/src/types.ts +++ b/libs/logger/src/types.ts @@ -14,3 +14,16 @@ export interface LogContext { export interface LogMetadata { [key: string]: any; } + +// Logger configuration +export interface LoggerConfig { + logLevel?: LogLevel; + logConsole?: boolean; + logFile?: boolean; + logFilePath?: string; + logLoki?: boolean; + lokiHost?: string; + lokiUser?: string; + lokiPassword?: string; + environment?: string; +} diff --git a/libs/mongodb-client/package.json b/libs/mongodb-client/package.json index 473a66a..ee9957a 100644 --- a/libs/mongodb-client/package.json +++ b/libs/mongodb-client/package.json @@ -13,12 +13,10 @@ "clean": "rimraf dist" }, "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", "@stock-bot/types": "*", "@types/mongodb": "^4.0.7", - "mongodb": "^6.17.0", - "yup": "^1.6.1" + "mongodb": "^6.17.0" }, "devDependencies": { "@types/node": "^20.11.0", diff --git a/libs/mongodb-client/src/client.ts b/libs/mongodb-client/src/client.ts index 8eee075..b49be3f 100644 --- a/libs/mongodb-client/src/client.ts +++ b/libs/mongodb-client/src/client.ts @@ -1,32 +1,24 @@ import { Collection, Db, MongoClient, OptionalUnlessRequiredId } from 'mongodb'; -import { mongodbConfig } from '@stock-bot/config'; import { getLogger } from '@stock-bot/logger'; -import type { DocumentBase } from './types'; +import type { DocumentBase, MongoDBClientConfig } from './types'; /** - * Simplified MongoDB Client for Stock Bot Data Service + * MongoDB Client for Stock Bot Data Service * - * A singleton MongoDB client focused solely on batch upsert operations + * MongoDB client focused on batch upsert operations * with minimal configuration and no health monitoring complexity. */ export class MongoDBClient { - private static instance: MongoDBClient | null = null; private client: MongoClient | null = null; private db: Db | null = null; - private defaultDatabase: string = 'stock'; // Default database name - private readonly logger = getLogger('mongodb-client-simple'); + private readonly config: MongoDBClientConfig; + private defaultDatabase: string; + private readonly logger = getLogger('mongodb-client'); private isConnected = false; - private constructor() {} - - /** - * Get singleton instance - */ - static getInstance(): MongoDBClient { - if (!MongoDBClient.instance) { - MongoDBClient.instance = new MongoDBClient(); - } - return MongoDBClient.instance; + constructor(config: MongoDBClientConfig) { + this.config = config; + this.defaultDatabase = config.database || 'stock'; } /** @@ -42,18 +34,17 @@ export class MongoDBClient { this.logger.info('Connecting to MongoDB...'); this.client = new MongoClient(uri, { - maxPoolSize: 10, - minPoolSize: 1, - connectTimeoutMS: 10000, - socketTimeoutMS: 30000, - serverSelectionTimeoutMS: 5000, + maxPoolSize: this.config.poolSettings?.maxPoolSize || 10, + minPoolSize: this.config.poolSettings?.minPoolSize || 1, + connectTimeoutMS: this.config.timeouts?.connectTimeout || 10000, + socketTimeoutMS: this.config.timeouts?.socketTimeout || 30000, + serverSelectionTimeoutMS: this.config.timeouts?.serverSelectionTimeout || 5000, }); await this.client.connect(); - await this.client.db(mongodbConfig.MONGODB_DATABASE).admin().ping(); + await this.client.db(this.defaultDatabase).admin().ping(); // Set default database from config - this.defaultDatabase = mongodbConfig.MONGODB_DATABASE; this.db = this.client.db(this.defaultDatabase); this.isConnected = true; @@ -347,18 +338,11 @@ export class MongoDBClient { } private buildConnectionUri(): string { - if (mongodbConfig.MONGODB_URI) { - return mongodbConfig.MONGODB_URI; + if (this.config.uri) { + return this.config.uri; } - const { - MONGODB_HOST: host, - MONGODB_PORT: port, - MONGODB_USERNAME: username, - MONGODB_PASSWORD: password, - MONGODB_DATABASE: database, - MONGODB_AUTH_SOURCE: authSource, - } = mongodbConfig; + const { host, port, username, password, database, authSource } = this.config; // Build URI components const auth = username && password ? `${username}:${password}@` : ''; diff --git a/libs/mongodb-client/src/factory.ts b/libs/mongodb-client/src/factory.ts index 27ab638..8134baa 100644 --- a/libs/mongodb-client/src/factory.ts +++ b/libs/mongodb-client/src/factory.ts @@ -1,53 +1,20 @@ import { MongoDBClient } from './client'; +import type { MongoDBClientConfig } from './types'; /** - * Get the singleton MongoDB client instance + * Factory function to create a MongoDB client instance */ -export function getMongoDBClient(): MongoDBClient { - return MongoDBClient.getInstance(); +export function createMongoDBClient(config: MongoDBClientConfig): MongoDBClient { + return new MongoDBClient(config); } /** - * Connect to MongoDB using the singleton client + * Create and connect a MongoDB client */ -export async function connectMongoDB(): Promise { - const client = getMongoDBClient(); - if (!client.connected) { - await client.connect(); - } +export async function createAndConnectMongoDBClient( + config: MongoDBClientConfig +): Promise { + const client = createMongoDBClient(config); + await client.connect(); return client; -} - -/** - * Disconnect from MongoDB - */ -export async function disconnectMongoDB(): Promise { - const client = getMongoDBClient(); - if (client.connected) { - await client.disconnect(); - } -} - -/** - * Set the default database for all operations - */ -export function setDefaultDatabase(databaseName: string): void { - const client = getMongoDBClient(); - client.setDefaultDatabase(databaseName); -} - -/** - * Get the current default database name - */ -export function getCurrentDatabase(): string { - const client = getMongoDBClient(); - return client.getDefaultDatabase(); -} - -/** - * Get a database instance by name - */ -export function getDatabase(databaseName?: string) { - const client = getMongoDBClient(); - return client.getDatabase(databaseName); -} +} \ No newline at end of file diff --git a/libs/mongodb-client/src/index.ts b/libs/mongodb-client/src/index.ts index 72f385d..2ba0765 100644 --- a/libs/mongodb-client/src/index.ts +++ b/libs/mongodb-client/src/index.ts @@ -1,7 +1,7 @@ /** - * Simplified MongoDB Client Library for Stock Bot Data Service + * MongoDB Client Library for Stock Bot Data Service * - * Provides a singleton MongoDB client focused on batch upsert operations + * Provides a MongoDB client focused on batch upsert operations * for high-performance data ingestion. */ @@ -14,6 +14,8 @@ export type { EarningsTranscript, ExchangeSourceMapping, MasterExchange, + MongoDBClientConfig, + MongoDBConnectionOptions, NewsArticle, RawDocument, SecFiling, @@ -22,10 +24,6 @@ export type { // Factory functions export { - connectMongoDB, - disconnectMongoDB, - getCurrentDatabase, - getDatabase, - getMongoDBClient, - setDefaultDatabase, + createMongoDBClient, + createAndConnectMongoDBClient, } from './factory'; diff --git a/libs/postgres-client/package.json b/libs/postgres-client/package.json index 80517e1..781e696 100644 --- a/libs/postgres-client/package.json +++ b/libs/postgres-client/package.json @@ -13,11 +13,9 @@ "clean": "rimraf dist" }, "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", "@stock-bot/types": "*", - "pg": "^8.11.3", - "yup": "^1.6.1" + "pg": "^8.11.3" }, "devDependencies": { "@types/node": "^20.11.0", diff --git a/libs/postgres-client/src/client.ts b/libs/postgres-client/src/client.ts index 8bc7697..5526f65 100644 --- a/libs/postgres-client/src/client.ts +++ b/libs/postgres-client/src/client.ts @@ -1,5 +1,4 @@ import { Pool, QueryResultRow } from 'pg'; -import { postgresConfig } from '@stock-bot/config'; import { getLogger } from '@stock-bot/logger'; import { PostgreSQLHealthMonitor } from './health'; import { PostgreSQLQueryBuilder } from './query-builder'; @@ -26,8 +25,8 @@ export class PostgreSQLClient { private readonly transactionManager: PostgreSQLTransactionManager; private isConnected = false; - constructor(config?: Partial, options?: PostgreSQLConnectionOptions) { - this.config = this.buildConfig(config); + constructor(config: PostgreSQLClientConfig, options?: PostgreSQLConnectionOptions) { + this.config = config; this.options = { retryAttempts: 3, retryDelay: 1000, @@ -367,34 +366,6 @@ export class PostgreSQLClient { return this.pool; } - private buildConfig(config?: Partial): PostgreSQLClientConfig { - return { - host: config?.host || postgresConfig.POSTGRES_HOST, - port: config?.port || postgresConfig.POSTGRES_PORT, - database: config?.database || postgresConfig.POSTGRES_DATABASE, - username: config?.username || postgresConfig.POSTGRES_USERNAME, - password: config?.password || postgresConfig.POSTGRES_PASSWORD, - poolSettings: { - min: postgresConfig.POSTGRES_POOL_MIN, - max: postgresConfig.POSTGRES_POOL_MAX, - idleTimeoutMillis: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT, - ...config?.poolSettings, - }, - ssl: { - enabled: postgresConfig.POSTGRES_SSL, - rejectUnauthorized: postgresConfig.POSTGRES_SSL_REJECT_UNAUTHORIZED, - ...config?.ssl, - }, - timeouts: { - query: postgresConfig.POSTGRES_QUERY_TIMEOUT, - connection: postgresConfig.POSTGRES_CONNECTION_TIMEOUT, - statement: postgresConfig.POSTGRES_STATEMENT_TIMEOUT, - lock: postgresConfig.POSTGRES_LOCK_TIMEOUT, - idleInTransaction: postgresConfig.POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT, - ...config?.timeouts, - }, - }; - } private buildPoolConfig(): any { return { diff --git a/libs/postgres-client/src/factory.ts b/libs/postgres-client/src/factory.ts index 5cfe9b2..81158f6 100644 --- a/libs/postgres-client/src/factory.ts +++ b/libs/postgres-client/src/factory.ts @@ -1,4 +1,3 @@ -import { postgresConfig } from '@stock-bot/config'; import { PostgreSQLClient } from './client'; import type { PostgreSQLClientConfig, PostgreSQLConnectionOptions } from './types'; @@ -6,59 +5,21 @@ import type { PostgreSQLClientConfig, PostgreSQLConnectionOptions } from './type * Factory function to create a PostgreSQL client instance */ export function createPostgreSQLClient( - config?: Partial, + config: PostgreSQLClientConfig, options?: PostgreSQLConnectionOptions ): PostgreSQLClient { return new PostgreSQLClient(config, options); } /** - * Create a PostgreSQL client with default configuration + * Create and connect a PostgreSQL client */ -export function createDefaultPostgreSQLClient(): PostgreSQLClient { - const config: Partial = { - host: postgresConfig.POSTGRES_HOST, - port: postgresConfig.POSTGRES_PORT, - database: postgresConfig.POSTGRES_DATABASE, - username: postgresConfig.POSTGRES_USERNAME, - password: postgresConfig.POSTGRES_PASSWORD, - }; - - return new PostgreSQLClient(config); -} - -/** - * Singleton PostgreSQL client instance - */ -let defaultClient: PostgreSQLClient | null = null; - -/** - * Get or create the default PostgreSQL client instance - */ -export function getPostgreSQLClient(): PostgreSQLClient { - if (!defaultClient) { - defaultClient = createDefaultPostgreSQLClient(); - } - return defaultClient; -} - -/** - * Connect to PostgreSQL using the default client - */ -export async function connectPostgreSQL(): Promise { - const client = getPostgreSQLClient(); - if (!client.connected) { - await client.connect(); - } +export async function createAndConnectPostgreSQLClient( + config: PostgreSQLClientConfig, + options?: PostgreSQLConnectionOptions +): Promise { + const client = createPostgreSQLClient(config, options); + await client.connect(); return client; } -/** - * Disconnect from PostgreSQL - */ -export async function disconnectPostgreSQL(): Promise { - if (defaultClient) { - await defaultClient.disconnect(); - defaultClient = null; - } -} diff --git a/libs/postgres-client/src/index.ts b/libs/postgres-client/src/index.ts index e1ab767..f90091b 100644 --- a/libs/postgres-client/src/index.ts +++ b/libs/postgres-client/src/index.ts @@ -33,7 +33,5 @@ export type { // Utils export { createPostgreSQLClient, - getPostgreSQLClient, - connectPostgreSQL, - disconnectPostgreSQL, + createAndConnectPostgreSQLClient, } from './factory'; diff --git a/libs/questdb-client/package.json b/libs/questdb-client/package.json index 8c5c3b6..a1ec85b 100644 --- a/libs/questdb-client/package.json +++ b/libs/questdb-client/package.json @@ -13,9 +13,9 @@ "clean": "rimraf dist" }, "dependencies": { - "@stock-bot/config": "*", "@stock-bot/logger": "*", - "@stock-bot/types": "*" + "@stock-bot/types": "*", + "pg": "^8.11.3" }, "devDependencies": { "@types/node": "^20.11.0", diff --git a/libs/questdb-client/src/client.ts b/libs/questdb-client/src/client.ts index f45240c..9080b42 100644 --- a/libs/questdb-client/src/client.ts +++ b/libs/questdb-client/src/client.ts @@ -1,5 +1,4 @@ import { Pool } from 'pg'; -import { questdbConfig } from '@stock-bot/config'; import { getLogger } from '@stock-bot/logger'; import { QuestDBHealthMonitor } from './health'; import { QuestDBInfluxWriter } from './influx-writer'; @@ -30,8 +29,8 @@ export class QuestDBClient { private readonly schemaManager: QuestDBSchemaManager; private isConnected = false; - constructor(config?: Partial, options?: QuestDBConnectionOptions) { - this.config = this.buildConfig(config); + constructor(config: QuestDBClientConfig, options?: QuestDBConnectionOptions) { + this.config = config; this.options = { protocol: 'pg', retryAttempts: 3, @@ -408,29 +407,6 @@ export class QuestDBClient { return { ...this.config }; } - private buildConfig(config?: Partial): QuestDBClientConfig { - return { - host: config?.host || questdbConfig.QUESTDB_HOST, - httpPort: config?.httpPort || questdbConfig.QUESTDB_HTTP_PORT, - pgPort: config?.pgPort || questdbConfig.QUESTDB_PG_PORT, - influxPort: config?.influxPort || questdbConfig.QUESTDB_INFLUX_PORT, - user: config?.user || questdbConfig.QUESTDB_USER, - password: config?.password || questdbConfig.QUESTDB_PASSWORD, - database: config?.database || questdbConfig.QUESTDB_DEFAULT_DATABASE, - tls: { - enabled: questdbConfig.QUESTDB_TLS_ENABLED, - verifyServerCert: questdbConfig.QUESTDB_TLS_VERIFY_SERVER_CERT, - ...config?.tls, - }, - timeouts: { - connection: questdbConfig.QUESTDB_CONNECTION_TIMEOUT, - request: questdbConfig.QUESTDB_REQUEST_TIMEOUT, - ...config?.timeouts, - }, - retryAttempts: questdbConfig.QUESTDB_RETRY_ATTEMPTS, - ...config, - }; - } private buildPgPoolConfig(): any { return { diff --git a/libs/questdb-client/src/factory.ts b/libs/questdb-client/src/factory.ts index dd11d4f..d349810 100644 --- a/libs/questdb-client/src/factory.ts +++ b/libs/questdb-client/src/factory.ts @@ -1,4 +1,3 @@ -import { questdbConfig } from '@stock-bot/config'; import { QuestDBClient } from './client'; import type { QuestDBClientConfig, QuestDBConnectionOptions } from './types'; @@ -6,58 +5,20 @@ import type { QuestDBClientConfig, QuestDBConnectionOptions } from './types'; * Factory function to create a QuestDB client instance */ export function createQuestDBClient( - config?: Partial, + config: QuestDBClientConfig, options?: QuestDBConnectionOptions ): QuestDBClient { return new QuestDBClient(config, options); } /** - * Create a QuestDB client with default configuration + * Create and connect a QuestDB client */ -export function createDefaultQuestDBClient(): QuestDBClient { - const config: Partial = { - host: questdbConfig.QUESTDB_HOST, - httpPort: questdbConfig.QUESTDB_HTTP_PORT, - pgPort: questdbConfig.QUESTDB_PG_PORT, - influxPort: questdbConfig.QUESTDB_INFLUX_PORT, - user: questdbConfig.QUESTDB_USER, - password: questdbConfig.QUESTDB_PASSWORD, - }; - - return new QuestDBClient(config); -} - -/** - * Singleton QuestDB client instance - */ -let defaultClient: QuestDBClient | null = null; - -/** - * Get or create the default QuestDB client instance - */ -export function getQuestDBClient(): QuestDBClient { - if (!defaultClient) { - defaultClient = createDefaultQuestDBClient(); - } - return defaultClient; -} - -/** - * Connect to QuestDB using the default client - */ -export async function connectQuestDB(): Promise { - const client = getQuestDBClient(); +export async function createAndConnectQuestDBClient( + config: QuestDBClientConfig, + options?: QuestDBConnectionOptions +): Promise { + const client = createQuestDBClient(config, options); await client.connect(); return client; -} - -/** - * Disconnect from QuestDB - */ -export async function disconnectQuestDB(): Promise { - if (defaultClient) { - await defaultClient.disconnect(); - defaultClient = null; - } -} +} \ No newline at end of file diff --git a/libs/questdb-client/src/index.ts b/libs/questdb-client/src/index.ts index f51f9d8..1add108 100644 --- a/libs/questdb-client/src/index.ts +++ b/libs/questdb-client/src/index.ts @@ -29,4 +29,4 @@ export type { } from './types'; // Utils -export { createQuestDBClient, getQuestDBClient } from './factory'; +export { createQuestDBClient, createAndConnectQuestDBClient } from './factory';