boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

removed configs from all libs and will inject them within the services themselves

Browse source
This commit is contained in:
Boki 2025-06-18 14:50:47 -04:00
parent fd28162811
commit 6cc5b339bc
32 changed files with 366 additions and 349 deletions
Download patch file Download diff file Expand all files Collapse all files

2
CLAUDE.md
View file

@ -45,7 +45,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
- **strategy-service** - Trading strategies and backtesting (multi-mode: live, event-driven, vectorized, hybrid)
- **execution-service** - Order management and risk controls
- **portfolio-service** - Position tracking and performance analytics
- **web** - React dashboard with real-time updates
- **web-app** - React dashboard with real-time updates
### Shared Libraries (`libs/`)
- **config** - Environment configuration with Zod validation

27
apps/data-sync-service/src/clients.ts Normal file
View file

@ -0,0 +1,27 @@
import { PostgreSQLClient } from '@stock-bot/postgres-client';
import { MongoDBClient } from '@stock-bot/mongodb-client';
let postgresClient: PostgreSQLClient | null = null;
let mongodbClient: MongoDBClient | null = null;
export function setPostgreSQLClient(client: PostgreSQLClient): void {
postgresClient = client;
}
export function getPostgreSQLClient(): PostgreSQLClient {
if (!postgresClient) {
throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.');
}
return postgresClient;
}
export function setMongoDBClient(client: MongoDBClient): void {
mongodbClient = client;
}
export function getMongoDBClient(): MongoDBClient {
if (!mongodbClient) {
throw new Error('MongoDB client not initialized. Call setMongoDBClient first.');
}
return mongodbClient;
}

61
apps/data-sync-service/src/index.ts
View file

@ -3,16 +3,29 @@
*/
import { Hono } from 'hono';
import { cors } from 'hono/cors';
import { initializeConfig, getServiceConfig } from '@stock-bot/config-new';
import { getLogger, shutdownLoggers } from '@stock-bot/logger';
import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client';
import { connectPostgreSQL, disconnectPostgreSQL } from '@stock-bot/postgres-client';
import { initializeConfig, getServiceConfig, getLoggingConfig } from '@stock-bot/config-new';
import { getLogger, shutdownLoggers, setLoggerConfig } from '@stock-bot/logger';
import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client';
import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client';
import { Shutdown } from '@stock-bot/shutdown';
import { enhancedSyncManager } from './services/enhanced-sync-manager';
import { syncManager } from './services/sync-manager';
import { setPostgreSQLClient, setMongoDBClient } from './clients';
// Initialize configuration
await initializeConfig();
const serviceConfig = getServiceConfig();
// Initialize logger with config
const loggingConfig = getLoggingConfig();
if (loggingConfig) {
setLoggerConfig({
logLevel: loggingConfig.level,
logConsole: true,
logFile: false,
environment: serviceConfig.environment,
});
}
const app = new Hono();
@ -28,9 +41,10 @@ app.use(
);
const logger = getLogger('data-sync-service');
const serviceConfig = getServiceConfig();
const PORT = serviceConfig.port;
let server: ReturnType<typeof Bun.serve> | null = null;
let postgresClient: PostgreSQLClient | null = null;
let mongoClient: MongoDBClient | null = null;
// Initialize shutdown manager
const shutdown = Shutdown.getInstance({ timeout: 15000 });
@ -177,12 +191,37 @@ async function initializeServices() {
try {
// Initialize MongoDB client
logger.info('Connecting to MongoDB...');
await connectMongoDB();
const mongoConfig = serviceConfig.database.mongodb;
mongoClient = await createAndConnectMongoDBClient({
uri: mongoConfig.uri,
database: mongoConfig.database,
host: 'localhost',
port: 27017,
timeouts: {
connectTimeout: mongoConfig.connectionTimeout || 30000,
socketTimeout: 30000,
serverSelectionTimeout: mongoConfig.serverSelectionTimeout || 5000,
},
});
setMongoDBClient(mongoClient);
logger.info('MongoDB connected');
// Initialize PostgreSQL client
logger.info('Connecting to PostgreSQL...');
await connectPostgreSQL();
const pgConfig = serviceConfig.database.postgres;
postgresClient = await createAndConnectPostgreSQLClient({
host: pgConfig.host,
port: pgConfig.port,
database: pgConfig.database,
username: pgConfig.username,
password: pgConfig.password,
poolSettings: {
min: 2,
max: pgConfig.maxConnections || 10,
idleTimeoutMillis: 30000,
},
});
setPostgreSQLClient(postgresClient);
logger.info('PostgreSQL connected');
// Initialize sync managers
@ -238,8 +277,12 @@ shutdown.onShutdown(async () => {
shutdown.onShutdown(async () => {
logger.info('Disconnecting from databases...');
try {
await disconnectMongoDB();
await disconnectPostgreSQL();
if (mongoClient) {
await mongoClient.disconnect();
}
if (postgresClient) {
await postgresClient.disconnect();
}
logger.info('Database connections closed');
} catch (error) {
logger.error('Error closing database connections', { error });

3
apps/data-sync-service/src/services/enhanced-sync-manager.ts
View file

@ -2,8 +2,7 @@
* Enhanced Sync Manager - Improved syncing with comprehensive exchange data
*/
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient } from '@stock-bot/mongodb-client';
import { getPostgreSQLClient } from '@stock-bot/postgres-client';
import { getMongoDBClient, getPostgreSQLClient } from '../clients';
const logger = getLogger('enhanced-sync-manager');

3
apps/data-sync-service/src/services/sync-manager.ts
View file

@ -2,8 +2,7 @@
* Sync Manager - Handles syncing raw MongoDB data to PostgreSQL master records
*/
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient } from '@stock-bot/mongodb-client';
import { getPostgreSQLClient } from '@stock-bot/postgres-client';
import { getMongoDBClient, getPostgreSQLClient } from '../clients';
const logger = getLogger('sync-manager');

27
apps/web-api/src/clients.ts Normal file
View file

@ -0,0 +1,27 @@
import { PostgreSQLClient } from '@stock-bot/postgres-client';
import { MongoDBClient } from '@stock-bot/mongodb-client';
let postgresClient: PostgreSQLClient | null = null;
let mongodbClient: MongoDBClient | null = null;
export function setPostgreSQLClient(client: PostgreSQLClient): void {
postgresClient = client;
}
export function getPostgreSQLClient(): PostgreSQLClient {
if (!postgresClient) {
throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.');
}
return postgresClient;
}
export function setMongoDBClient(client: MongoDBClient): void {
mongodbClient = client;
}
export function getMongoDBClient(): MongoDBClient {
if (!mongodbClient) {
throw new Error('MongoDB client not initialized. Call setMongoDBClient first.');
}
return mongodbClient;
}

61
apps/web-api/src/index.ts
View file

@ -3,17 +3,30 @@
*/
import { Hono } from 'hono';
import { cors } from 'hono/cors';
import { initializeConfig, getServiceConfig } from '@stock-bot/config-new';
import { getLogger, shutdownLoggers } from '@stock-bot/logger';
import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client';
import { connectPostgreSQL, disconnectPostgreSQL } from '@stock-bot/postgres-client';
import { initializeConfig, getServiceConfig, getLoggingConfig } from '@stock-bot/config-new';
import { getLogger, shutdownLoggers, setLoggerConfig } from '@stock-bot/logger';
import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client';
import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client';
import { Shutdown } from '@stock-bot/shutdown';
import { setPostgreSQLClient, setMongoDBClient } from './clients';
// Import routes
import { exchangeRoutes } from './routes/exchange.routes';
import { healthRoutes } from './routes/health.routes';
// Initialize configuration
await initializeConfig();
const serviceConfig = getServiceConfig();
// Initialize logger with config
const loggingConfig = getLoggingConfig();
if (loggingConfig) {
setLoggerConfig({
logLevel: loggingConfig.level,
logConsole: true,
logFile: false,
environment: serviceConfig.environment,
});
}
const app = new Hono();
@ -29,9 +42,10 @@ app.use(
);
const logger = getLogger('web-api');
const serviceConfig = getServiceConfig();
const PORT = serviceConfig.port;
let server: ReturnType<typeof Bun.serve> | null = null;
let postgresClient: PostgreSQLClient | null = null;
let mongoClient: MongoDBClient | null = null;
// Initialize shutdown manager
const shutdown = Shutdown.getInstance({ timeout: 15000 });
@ -61,12 +75,37 @@ async function initializeServices() {
try {
// Initialize MongoDB client
logger.info('Connecting to MongoDB...');
await connectMongoDB();
const mongoConfig = serviceConfig.database.mongodb;
mongoClient = await createAndConnectMongoDBClient({
uri: mongoConfig.uri,
database: mongoConfig.database,
host: 'localhost',
port: 27017,
timeouts: {
connectTimeout: mongoConfig.connectionTimeout || 30000,
socketTimeout: 30000,
serverSelectionTimeout: mongoConfig.serverSelectionTimeout || 5000,
},
});
setMongoDBClient(mongoClient);
logger.info('MongoDB connected');
// Initialize PostgreSQL client
logger.info('Connecting to PostgreSQL...');
await connectPostgreSQL();
const pgConfig = serviceConfig.database.postgres;
postgresClient = await createAndConnectPostgreSQLClient({
host: pgConfig.host,
port: pgConfig.port,
database: pgConfig.database,
username: pgConfig.username,
password: pgConfig.password,
poolSettings: {
min: 2,
max: pgConfig.maxConnections || 10,
idleTimeoutMillis: 30000,
},
});
setPostgreSQLClient(postgresClient);
logger.info('PostgreSQL connected');
logger.info('All services initialized successfully');
@ -105,8 +144,12 @@ shutdown.onShutdown(async () => {
shutdown.onShutdown(async () => {
logger.info('Disconnecting from databases...');
try {
await disconnectMongoDB();
await disconnectPostgreSQL();
if (mongoClient) {
await mongoClient.disconnect();
}
if (postgresClient) {
await postgresClient.disconnect();
}
logger.info('Database connections closed');
} catch (error) {
logger.error('Error closing database connections', { error });

3
apps/web-api/src/routes/health.routes.ts
View file

@ -3,8 +3,7 @@
*/
import { Hono } from 'hono';
import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient } from '@stock-bot/mongodb-client';
import { getPostgreSQLClient } from '@stock-bot/postgres-client';
import { getPostgreSQLClient, getMongoDBClient } from '../clients';
const logger = getLogger('health-routes');
export const healthRoutes = new Hono();

3
apps/web-api/src/services/exchange.service.ts
View file

@ -1,6 +1,5 @@
import { getLogger } from '@stock-bot/logger';
import { getPostgreSQLClient } from '@stock-bot/postgres-client';
import { getMongoDBClient } from '@stock-bot/mongodb-client';
import { getPostgreSQLClient, getMongoDBClient } from '../clients';
import {
Exchange,
ExchangeWithMappings,

1
libs/cache/package.json vendored
View file

@ -11,7 +11,6 @@
"test": "bun test"
},
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"ioredis": "^5.3.2"
},

47
libs/cache/src/connection-manager.ts vendored
View file

@ -1,11 +1,12 @@
import Redis from 'ioredis';
import { dragonflyConfig } from '@stock-bot/config';
import { getLogger } from '@stock-bot/logger';
import type { RedisConfig } from './types';
interface ConnectionConfig {
name: string;
singleton?: boolean;
db?: number;
redisConfig: RedisConfig;
}
/**
@ -32,12 +33,12 @@ export class RedisConnectionManager {
* @returns Redis connection instance
*/
getConnection(config: ConnectionConfig): Redis {
const { name, singleton = false, db } = config;
const { name, singleton = false, db, redisConfig } = config;
if (singleton) {
// Use shared connection across all instances
if (!RedisConnectionManager.sharedConnections.has(name)) {
const connection = this.createConnection(name, db);
const connection = this.createConnection(name, redisConfig, db);
RedisConnectionManager.sharedConnections.set(name, connection);
this.logger.info(`Created shared Redis connection: ${name}`);
}
@ -45,7 +46,7 @@ export class RedisConnectionManager {
} else {
// Create unique connection per instance
const uniqueName = `${name}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
const connection = this.createConnection(uniqueName, db);
const connection = this.createConnection(uniqueName, redisConfig, db);
this.connections.set(uniqueName, connection);
this.logger.info(`Created unique Redis connection: ${uniqueName}`);
return connection;
@ -55,33 +56,31 @@ export class RedisConnectionManager {
/**
* Create a new Redis connection with configuration
*/
private createConnection(name: string, db?: number): Redis {
const redisConfig = {
host: dragonflyConfig.DRAGONFLY_HOST,
port: dragonflyConfig.DRAGONFLY_PORT,
password: dragonflyConfig.DRAGONFLY_PASSWORD || undefined,
username: dragonflyConfig.DRAGONFLY_USERNAME || undefined,
db: db ?? dragonflyConfig.DRAGONFLY_DATABASE,
maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES,
retryDelayOnFailover: dragonflyConfig.DRAGONFLY_RETRY_DELAY,
connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT,
commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT,
keepAlive: dragonflyConfig.DRAGONFLY_ENABLE_KEEPALIVE
? dragonflyConfig.DRAGONFLY_KEEPALIVE_INTERVAL * 1000
: 0,
private createConnection(name: string, config: RedisConfig, db?: number): Redis {
const redisOptions = {
host: config.host,
port: config.port,
password: config.password || undefined,
username: config.username || undefined,
db: db ?? config.db ?? 0,
maxRetriesPerRequest: config.maxRetriesPerRequest ?? 3,
retryDelayOnFailover: config.retryDelayOnFailover ?? 100,
connectTimeout: config.connectTimeout ?? 10000,
commandTimeout: config.commandTimeout ?? 5000,
keepAlive: config.keepAlive ?? 0,
connectionName: name,
lazyConnect: false, // Connect immediately instead of waiting for first command
...(dragonflyConfig.DRAGONFLY_TLS && {
...(config.tls && {
tls: {
cert: dragonflyConfig.DRAGONFLY_TLS_CERT_FILE || undefined,
key: dragonflyConfig.DRAGONFLY_TLS_KEY_FILE || undefined,
ca: dragonflyConfig.DRAGONFLY_TLS_CA_FILE || undefined,
rejectUnauthorized: !dragonflyConfig.DRAGONFLY_TLS_SKIP_VERIFY,
cert: config.tls.cert || undefined,
key: config.tls.key || undefined,
ca: config.tls.ca || undefined,
rejectUnauthorized: config.tls.rejectUnauthorized ?? true,
},
}),
};
const redis = new Redis(redisConfig);
const redis = new Redis(redisOptions);
// Setup event handlers
redis.on('connect', () => {

14
libs/cache/src/index.ts vendored
View file

@ -1,6 +1,6 @@
import { RedisConnectionManager } from './connection-manager';
import { RedisCache } from './redis-cache';
import type { CacheOptions, CacheProvider } from './types';
import type { CacheOptions, CacheProvider, RedisConfig } from './types';
// Cache instances registry to prevent multiple instances with same prefix
const cacheInstances = new Map<string, CacheProvider>();
@ -8,7 +8,7 @@ const cacheInstances = new Map<string, CacheProvider>();
/**
* Create a Redis cache instance with trading-optimized defaults
*/
export function createCache(options: Partial<CacheOptions> = {}): CacheProvider {
export function createCache(options: CacheOptions): CacheProvider {
const defaultOptions: CacheOptions = {
keyPrefix: 'cache:',
ttl: 3600, // 1 hour default
@ -37,39 +37,42 @@ export function createCache(options: Partial<CacheOptions> = {}): CacheProvider
/**
* Create a cache instance for trading data
*/
export function createTradingCache(options: Partial<CacheOptions> = {}): CacheProvider {
export function createTradingCache(redisConfig: RedisConfig, options?: Partial<Omit<CacheOptions, 'redisConfig'>>): CacheProvider {
return createCache({
keyPrefix: 'trading:',
ttl: 3600, // 1 hour default
enableMetrics: true,
shared: true,
...options,
redisConfig,
});
}
/**
* Create a cache for market data with shorter TTL
*/
export function createMarketDataCache(options: Partial<CacheOptions> = {}): CacheProvider {
export function createMarketDataCache(redisConfig: RedisConfig, options?: Partial<Omit<CacheOptions, 'redisConfig'>>): CacheProvider {
return createCache({
keyPrefix: 'market:',
ttl: 300, // 5 minutes for market data
enableMetrics: true,
shared: true,
...options,
redisConfig,
});
}
/**
* Create a cache for indicators with longer TTL
*/
export function createIndicatorCache(options: Partial<CacheOptions> = {}): CacheProvider {
export function createIndicatorCache(redisConfig: RedisConfig, options?: Partial<Omit<CacheOptions, 'redisConfig'>>): CacheProvider {
return createCache({
keyPrefix: 'indicators:',
ttl: 1800, // 30 minutes for indicators
enableMetrics: true,
shared: true,
...options,
redisConfig,
});
}
@ -81,6 +84,7 @@ export type {
CacheStats,
CacheKey,
SerializationOptions,
RedisConfig,
} from './types';
export { RedisCache } from './redis-cache';

3
libs/cache/src/redis-cache.ts vendored
View file

@ -25,7 +25,7 @@ export class RedisCache implements CacheProvider {
uptime: 0,
};
constructor(options: CacheOptions = {}) {
constructor(options: CacheOptions) {
this.defaultTTL = options.ttl ?? 3600; // 1 hour default
this.keyPrefix = options.keyPrefix ?? 'cache:';
this.enableMetrics = options.enableMetrics ?? true;
@ -46,6 +46,7 @@ export class RedisCache implements CacheProvider {
this.redis = this.connectionManager.getConnection({
name: `${baseName}-SERVICE`,
singleton: options.shared ?? true, // Default to shared connection for cache
redisConfig: options.redisConfig,
});
// Only setup event handlers for non-shared connections to avoid memory leaks

21
libs/cache/src/types.ts vendored
View file

@ -1,3 +1,23 @@
export interface RedisConfig {
host: string;
port: number;
password?: string;
username?: string;
db?: number;
keyPrefix?: string;
maxRetriesPerRequest?: number;
retryDelayOnFailover?: number;
connectTimeout?: number;
commandTimeout?: number;
keepAlive?: number;
tls?: {
cert?: string;
key?: string;
ca?: string;
rejectUnauthorized?: boolean;
};
}
export interface CacheProvider {
get<T>(key: string): Promise<T | null>;
set<T>(
@ -64,6 +84,7 @@ export interface CacheOptions {
enableMetrics?: boolean;
name?: string; // Name for connection identification
shared?: boolean; // Whether to use shared connection
redisConfig: RedisConfig;
}
export interface CacheStats {

1
libs/event-bus/package.json
View file

@ -12,7 +12,6 @@
},
"dependencies": {
"@stock-bot/logger": "*",
"@stock-bot/config": "*",
"ioredis": "^5.3.2",
"eventemitter3": "^5.0.1"
},

29
libs/event-bus/src/index.ts
View file

@ -1,6 +1,5 @@
import { EventEmitter } from 'eventemitter3';
import Redis from 'ioredis';
import { dragonflyConfig } from '@stock-bot/config';
import { getLogger } from '@stock-bot/logger';
export interface EventBusMessage {
@ -16,12 +15,21 @@ export interface EventHandler<T = any> {
(message: EventBusMessage & { data: T }): Promise<void> | void;
}
export interface RedisConfig {
host: string;
port: number;
password?: string;
db?: number;
maxRetriesPerRequest?: number;
}
export interface EventBusOptions {
serviceName: string;
enablePersistence?: boolean;
useStreams?: boolean;
maxRetries?: number;
retryDelay?: number;
redisConfig: RedisConfig;
}
export interface StreamConsumerInfo {
@ -53,21 +61,22 @@ export class EventBus extends EventEmitter {
this.retryDelay = options.retryDelay ?? 1000;
this.logger = getLogger(`event-bus:${this.serviceName}`);
const { redisConfig } = options;
this.redis = new Redis({
host: dragonflyConfig.DRAGONFLY_HOST,
port: dragonflyConfig.DRAGONFLY_PORT,
password: dragonflyConfig.DRAGONFLY_PASSWORD,
db: dragonflyConfig.DRAGONFLY_DATABASE || 0,
maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES,
host: redisConfig.host,
port: redisConfig.port,
password: redisConfig.password,
db: redisConfig.db || 0,
maxRetriesPerRequest: redisConfig.maxRetriesPerRequest || 3,
lazyConnect: false,
});
if (!this.useStreams) {
this.subscriber = new Redis({
host: dragonflyConfig.DRAGONFLY_HOST,
port: dragonflyConfig.DRAGONFLY_PORT,
password: dragonflyConfig.DRAGONFLY_PASSWORD,
db: dragonflyConfig.DRAGONFLY_DATABASE || 0,
host: redisConfig.host,
port: redisConfig.port,
password: redisConfig.password,
db: redisConfig.db || 0,
});
this.subscriber.on('message', this.handleRedisMessage.bind(this));
}

1
libs/logger/package.json
View file

@ -11,7 +11,6 @@
"test": "bun test"
},
"dependencies": {
"@stock-bot/config": "*",
"got": "^14.4.7",
"pino": "^9.7.0",
"pino-loki": "^2.6.0",

4
libs/logger/src/index.ts
View file

@ -5,10 +5,10 @@
*/
// Core logger classes and functions
export { Logger, getLogger, shutdownLoggers } from './logger';
export { Logger, getLogger, shutdownLoggers, setLoggerConfig } from './logger';
// Type definitions
export type { LogLevel, LogContext, LogMetadata } from './types';
export type { LogLevel, LogContext, LogMetadata, LoggerConfig } from './types';
// Default export
export { getLogger as default } from './logger';

81
libs/logger/src/logger.ts
View file

@ -9,23 +9,41 @@
*/
import pino from 'pino';
import { loggingConfig, lokiConfig } from '@stock-bot/config';
import type { LogContext, LogLevel, LogMetadata } from './types';
import type { LogContext, LogLevel, LogMetadata, LoggerConfig } from './types';
// Simple cache for logger instances
const loggerCache = new Map<string, pino.Logger>();
console.log('Logger cache initialized: ', loggingConfig.LOG_LEVEL);
// Global config that can be set
let globalConfig: LoggerConfig = {
logLevel: 'info',
logConsole: true,
logFile: false,
logFilePath: './logs',
logLoki: false,
environment: 'development',
};
/**
* Set global logger configuration
*/
export function setLoggerConfig(config: LoggerConfig): void {
globalConfig = { ...globalConfig, ...config };
// Clear cache to force recreation with new config
loggerCache.clear();
console.log('Logger config updated:', globalConfig.logLevel);
}
/**
* Create transport configuration
*/
function createTransports(serviceName: string): any {
function createTransports(serviceName: string, config: LoggerConfig = globalConfig): any {
const targets: any[] = [];
// const isDev = loggingConfig.LOG_ENVIRONMENT === 'development';
// Console transport
if (loggingConfig.LOG_CONSOLE) {
if (config.logConsole) {
targets.push({
target: 'pino-pretty',
level: loggingConfig.LOG_LEVEL, // Only show errors on console
level: config.logLevel || 'info',
options: {
colorize: true,
translateTime: 'yyyy-mm-dd HH:MM:ss.l',
@ -40,29 +58,35 @@ function createTransports(serviceName: string): any {
}
// File transport
if (loggingConfig.LOG_FILE) {
if (config.logFile) {
targets.push({
target: 'pino/file',
level: loggingConfig.LOG_LEVEL,
level: config.logLevel || 'info',
options: {
destination: `${loggingConfig.LOG_FILE_PATH}/${serviceName}.log`,
destination: `${config.logFilePath}/${serviceName}.log`,
mkdir: true,
},
});
}
// Loki transport
if (lokiConfig.LOKI_HOST) {
if (config.logLoki && config.lokiHost) {
targets.push({
target: 'pino-loki',
level: loggingConfig.LOG_LEVEL,
level: config.logLevel || 'info',
options: {
host: lokiConfig.LOKI_URL || `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`,
host: config.lokiHost,
labels: {
service: serviceName,
environment: lokiConfig.LOKI_ENVIRONMENT_LABEL,
environment: config.environment || 'development',
},
ignore: 'childName',
...(config.lokiUser && config.lokiPassword ? {
basicAuth: {
username: config.lokiUser,
password: config.lokiPassword,
},
} : {}),
},
});
}
@ -73,27 +97,28 @@ function createTransports(serviceName: string): any {
/**
* Get or create pino logger
*/
function getPinoLogger(serviceName: string): pino.Logger {
if (!loggerCache.has(serviceName)) {
const transport = createTransports(serviceName);
function getPinoLogger(serviceName: string, config: LoggerConfig = globalConfig): pino.Logger {
const cacheKey = `${serviceName}-${JSON.stringify(config)}`;
if (!loggerCache.has(cacheKey)) {
const transport = createTransports(serviceName, config);
const config: pino.LoggerOptions = {
level: loggingConfig.LOG_LEVEL,
const loggerOptions: pino.LoggerOptions = {
level: config.logLevel || 'info',
base: {
service: serviceName,
environment: loggingConfig.LOG_ENVIRONMENT,
version: loggingConfig.LOG_SERVICE_VERSION,
environment: config.environment || 'development',
version: '1.0.0',
},
};
if (transport) {
config.transport = transport;
loggerOptions.transport = transport;
}
loggerCache.set(serviceName, pino(config));
loggerCache.set(cacheKey, pino(loggerOptions));
}
return loggerCache.get(serviceName)!;
return loggerCache.get(cacheKey)!;
}
/**
@ -105,8 +130,8 @@ export class Logger {
private serviceName: string;
private childName?: string;
constructor(serviceName: string, context: LogContext = {}) {
this.pino = getPinoLogger(serviceName);
constructor(serviceName: string, context: LogContext = {}, config?: LoggerConfig) {
this.pino = getPinoLogger(serviceName, config);
this.context = context;
this.serviceName = serviceName;
}
@ -232,8 +257,8 @@ export class Logger {
/**
* Main factory function
*/
export function getLogger(serviceName: string, context?: LogContext): Logger {
return new Logger(serviceName, context);
export function getLogger(serviceName: string, context?: LogContext, config?: LoggerConfig): Logger {
return new Logger(serviceName, context, config);
}
/**

13
libs/logger/src/types.ts
View file

@ -14,3 +14,16 @@ export interface LogContext {
export interface LogMetadata {
[key: string]: any;
}
// Logger configuration
export interface LoggerConfig {
logLevel?: LogLevel;
logConsole?: boolean;
logFile?: boolean;
logFilePath?: string;
logLoki?: boolean;
lokiHost?: string;
lokiUser?: string;
lokiPassword?: string;
environment?: string;
}

4
libs/mongodb-client/package.json
View file

@ -13,12 +13,10 @@
"clean": "rimraf dist"
},
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"@stock-bot/types": "*",
"@types/mongodb": "^4.0.7",
"mongodb": "^6.17.0",
"yup": "^1.6.1"
"mongodb": "^6.17.0"
},
"devDependencies": {
"@types/node": "^20.11.0",

52
libs/mongodb-client/src/client.ts
View file

@ -1,32 +1,24 @@
import { Collection, Db, MongoClient, OptionalUnlessRequiredId } from 'mongodb';
import { mongodbConfig } from '@stock-bot/config';
import { getLogger } from '@stock-bot/logger';
import type { DocumentBase } from './types';
import type { DocumentBase, MongoDBClientConfig } from './types';
/**
* Simplified MongoDB Client for Stock Bot Data Service
* MongoDB Client for Stock Bot Data Service
*
* A singleton MongoDB client focused solely on batch upsert operations
* MongoDB client focused on batch upsert operations
* with minimal configuration and no health monitoring complexity.
*/
export class MongoDBClient {
private static instance: MongoDBClient | null = null;
private client: MongoClient | null = null;
private db: Db | null = null;
private defaultDatabase: string = 'stock'; // Default database name
private readonly logger = getLogger('mongodb-client-simple');
private readonly config: MongoDBClientConfig;
private defaultDatabase: string;
private readonly logger = getLogger('mongodb-client');
private isConnected = false;
private constructor() {}
/**
* Get singleton instance
*/
static getInstance(): MongoDBClient {
if (!MongoDBClient.instance) {
MongoDBClient.instance = new MongoDBClient();
}
return MongoDBClient.instance;
constructor(config: MongoDBClientConfig) {
this.config = config;
this.defaultDatabase = config.database || 'stock';
}
/**
@ -42,18 +34,17 @@ export class MongoDBClient {
this.logger.info('Connecting to MongoDB...');
this.client = new MongoClient(uri, {
maxPoolSize: 10,
minPoolSize: 1,
connectTimeoutMS: 10000,
socketTimeoutMS: 30000,
serverSelectionTimeoutMS: 5000,
maxPoolSize: this.config.poolSettings?.maxPoolSize || 10,
minPoolSize: this.config.poolSettings?.minPoolSize || 1,
connectTimeoutMS: this.config.timeouts?.connectTimeout || 10000,
socketTimeoutMS: this.config.timeouts?.socketTimeout || 30000,
serverSelectionTimeoutMS: this.config.timeouts?.serverSelectionTimeout || 5000,
});
await this.client.connect();
await this.client.db(mongodbConfig.MONGODB_DATABASE).admin().ping();
await this.client.db(this.defaultDatabase).admin().ping();
// Set default database from config
this.defaultDatabase = mongodbConfig.MONGODB_DATABASE;
this.db = this.client.db(this.defaultDatabase);
this.isConnected = true;
@ -347,18 +338,11 @@ export class MongoDBClient {
}
private buildConnectionUri(): string {
if (mongodbConfig.MONGODB_URI) {
return mongodbConfig.MONGODB_URI;
if (this.config.uri) {
return this.config.uri;
}
const {
MONGODB_HOST: host,
MONGODB_PORT: port,
MONGODB_USERNAME: username,
MONGODB_PASSWORD: password,
MONGODB_DATABASE: database,
MONGODB_AUTH_SOURCE: authSource,
} = mongodbConfig;
const { host, port, username, password, database, authSource } = this.config;
// Build URI components
const auth = username && password ? `${username}:${password}@` : '';

53
libs/mongodb-client/src/factory.ts
View file

@ -1,53 +1,20 @@
import { MongoDBClient } from './client';
import type { MongoDBClientConfig } from './types';
/**
* Get the singleton MongoDB client instance
* Factory function to create a MongoDB client instance
*/
export function getMongoDBClient(): MongoDBClient {
return MongoDBClient.getInstance();
export function createMongoDBClient(config: MongoDBClientConfig): MongoDBClient {
return new MongoDBClient(config);
}
/**
* Connect to MongoDB using the singleton client
* Create and connect a MongoDB client
*/
export async function connectMongoDB(): Promise<MongoDBClient> {
const client = getMongoDBClient();
if (!client.connected) {
await client.connect();
}
export async function createAndConnectMongoDBClient(
config: MongoDBClientConfig
): Promise<MongoDBClient> {
const client = createMongoDBClient(config);
await client.connect();
return client;
}
/**
* Disconnect from MongoDB
*/
export async function disconnectMongoDB(): Promise<void> {
const client = getMongoDBClient();
if (client.connected) {
await client.disconnect();
}
}
/**
* Set the default database for all operations
*/
export function setDefaultDatabase(databaseName: string): void {
const client = getMongoDBClient();
client.setDefaultDatabase(databaseName);
}
/**
* Get the current default database name
*/
export function getCurrentDatabase(): string {
const client = getMongoDBClient();
return client.getDefaultDatabase();
}
/**
* Get a database instance by name
*/
export function getDatabase(databaseName?: string) {
const client = getMongoDBClient();
return client.getDatabase(databaseName);
}

14
libs/mongodb-client/src/index.ts
View file

@ -1,7 +1,7 @@
/**
* Simplified MongoDB Client Library for Stock Bot Data Service
* MongoDB Client Library for Stock Bot Data Service
*
* Provides a singleton MongoDB client focused on batch upsert operations
* Provides a MongoDB client focused on batch upsert operations
* for high-performance data ingestion.
*/
@ -14,6 +14,8 @@ export type {
EarningsTranscript,
ExchangeSourceMapping,
MasterExchange,
MongoDBClientConfig,
MongoDBConnectionOptions,
NewsArticle,
RawDocument,
SecFiling,
@ -22,10 +24,6 @@ export type {
// Factory functions
export {
connectMongoDB,
disconnectMongoDB,
getCurrentDatabase,
getDatabase,
getMongoDBClient,
setDefaultDatabase,
createMongoDBClient,
createAndConnectMongoDBClient,
} from './factory';

4
libs/postgres-client/package.json
View file

@ -13,11 +13,9 @@
"clean": "rimraf dist"
},
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"@stock-bot/types": "*",
"pg": "^8.11.3",
"yup": "^1.6.1"
"pg": "^8.11.3"
},
"devDependencies": {
"@types/node": "^20.11.0",

33
libs/postgres-client/src/client.ts
View file

@ -1,5 +1,4 @@
import { Pool, QueryResultRow } from 'pg';
import { postgresConfig } from '@stock-bot/config';
import { getLogger } from '@stock-bot/logger';
import { PostgreSQLHealthMonitor } from './health';
import { PostgreSQLQueryBuilder } from './query-builder';
@ -26,8 +25,8 @@ export class PostgreSQLClient {
private readonly transactionManager: PostgreSQLTransactionManager;
private isConnected = false;
constructor(config?: Partial<PostgreSQLClientConfig>, options?: PostgreSQLConnectionOptions) {
this.config = this.buildConfig(config);
constructor(config: PostgreSQLClientConfig, options?: PostgreSQLConnectionOptions) {
this.config = config;
this.options = {
retryAttempts: 3,
retryDelay: 1000,
@ -367,34 +366,6 @@ export class PostgreSQLClient {
return this.pool;
}
private buildConfig(config?: Partial<PostgreSQLClientConfig>): PostgreSQLClientConfig {
return {
host: config?.host || postgresConfig.POSTGRES_HOST,
port: config?.port || postgresConfig.POSTGRES_PORT,
database: config?.database || postgresConfig.POSTGRES_DATABASE,
username: config?.username || postgresConfig.POSTGRES_USERNAME,
password: config?.password || postgresConfig.POSTGRES_PASSWORD,
poolSettings: {
min: postgresConfig.POSTGRES_POOL_MIN,
max: postgresConfig.POSTGRES_POOL_MAX,
idleTimeoutMillis: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT,
...config?.poolSettings,
},
ssl: {
enabled: postgresConfig.POSTGRES_SSL,
rejectUnauthorized: postgresConfig.POSTGRES_SSL_REJECT_UNAUTHORIZED,
...config?.ssl,
},
timeouts: {
query: postgresConfig.POSTGRES_QUERY_TIMEOUT,
connection: postgresConfig.POSTGRES_CONNECTION_TIMEOUT,
statement: postgresConfig.POSTGRES_STATEMENT_TIMEOUT,
lock: postgresConfig.POSTGRES_LOCK_TIMEOUT,
idleInTransaction: postgresConfig.POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT,
...config?.timeouts,
},
};
}
private buildPoolConfig(): any {
return {

55
libs/postgres-client/src/factory.ts
View file

@ -1,4 +1,3 @@
import { postgresConfig } from '@stock-bot/config';
import { PostgreSQLClient } from './client';
import type { PostgreSQLClientConfig, PostgreSQLConnectionOptions } from './types';
@ -6,59 +5,21 @@ import type { PostgreSQLClientConfig, PostgreSQLConnectionOptions } from './type
* Factory function to create a PostgreSQL client instance
*/
export function createPostgreSQLClient(
config?: Partial<PostgreSQLClientConfig>,
config: PostgreSQLClientConfig,
options?: PostgreSQLConnectionOptions
): PostgreSQLClient {
return new PostgreSQLClient(config, options);
}
/**
* Create a PostgreSQL client with default configuration
* Create and connect a PostgreSQL client
*/
export function createDefaultPostgreSQLClient(): PostgreSQLClient {
const config: Partial<PostgreSQLClientConfig> = {
host: postgresConfig.POSTGRES_HOST,
port: postgresConfig.POSTGRES_PORT,
database: postgresConfig.POSTGRES_DATABASE,
username: postgresConfig.POSTGRES_USERNAME,
password: postgresConfig.POSTGRES_PASSWORD,
};
return new PostgreSQLClient(config);
}
/**
* Singleton PostgreSQL client instance
*/
let defaultClient: PostgreSQLClient | null = null;
/**
* Get or create the default PostgreSQL client instance
*/
export function getPostgreSQLClient(): PostgreSQLClient {
if (!defaultClient) {
defaultClient = createDefaultPostgreSQLClient();
}
return defaultClient;
}
/**
* Connect to PostgreSQL using the default client
*/
export async function connectPostgreSQL(): Promise<PostgreSQLClient> {
const client = getPostgreSQLClient();
if (!client.connected) {
await client.connect();
}
export async function createAndConnectPostgreSQLClient(
config: PostgreSQLClientConfig,
options?: PostgreSQLConnectionOptions
): Promise<PostgreSQLClient> {
const client = createPostgreSQLClient(config, options);
await client.connect();
return client;
}
/**
* Disconnect from PostgreSQL
*/
export async function disconnectPostgreSQL(): Promise<void> {
if (defaultClient) {
await defaultClient.disconnect();
defaultClient = null;
}
}

4
libs/postgres-client/src/index.ts
View file

@ -33,7 +33,5 @@ export type {
// Utils
export {
createPostgreSQLClient,
getPostgreSQLClient,
connectPostgreSQL,
disconnectPostgreSQL,
createAndConnectPostgreSQLClient,
} from './factory';

4
libs/questdb-client/package.json
View file

@ -13,9 +13,9 @@
"clean": "rimraf dist"
},
"dependencies": {
"@stock-bot/config": "*",
"@stock-bot/logger": "*",
"@stock-bot/types": "*"
"@stock-bot/types": "*",
"pg": "^8.11.3"
},
"devDependencies": {
"@types/node": "^20.11.0",

28
libs/questdb-client/src/client.ts
View file

@ -1,5 +1,4 @@
import { Pool } from 'pg';
import { questdbConfig } from '@stock-bot/config';
import { getLogger } from '@stock-bot/logger';
import { QuestDBHealthMonitor } from './health';
import { QuestDBInfluxWriter } from './influx-writer';
@ -30,8 +29,8 @@ export class QuestDBClient {
private readonly schemaManager: QuestDBSchemaManager;
private isConnected = false;
constructor(config?: Partial<QuestDBClientConfig>, options?: QuestDBConnectionOptions) {
this.config = this.buildConfig(config);
constructor(config: QuestDBClientConfig, options?: QuestDBConnectionOptions) {
this.config = config;
this.options = {
protocol: 'pg',
retryAttempts: 3,
@ -408,29 +407,6 @@ export class QuestDBClient {
return { ...this.config };
}
private buildConfig(config?: Partial<QuestDBClientConfig>): QuestDBClientConfig {
return {
host: config?.host || questdbConfig.QUESTDB_HOST,
httpPort: config?.httpPort || questdbConfig.QUESTDB_HTTP_PORT,
pgPort: config?.pgPort || questdbConfig.QUESTDB_PG_PORT,
influxPort: config?.influxPort || questdbConfig.QUESTDB_INFLUX_PORT,
user: config?.user || questdbConfig.QUESTDB_USER,
password: config?.password || questdbConfig.QUESTDB_PASSWORD,
database: config?.database || questdbConfig.QUESTDB_DEFAULT_DATABASE,
tls: {
enabled: questdbConfig.QUESTDB_TLS_ENABLED,
verifyServerCert: questdbConfig.QUESTDB_TLS_VERIFY_SERVER_CERT,
...config?.tls,
},
timeouts: {
connection: questdbConfig.QUESTDB_CONNECTION_TIMEOUT,
request: questdbConfig.QUESTDB_REQUEST_TIMEOUT,
...config?.timeouts,
},
retryAttempts: questdbConfig.QUESTDB_RETRY_ATTEMPTS,
...config,
};
}
private buildPgPoolConfig(): any {
return {

53
libs/questdb-client/src/factory.ts
View file

@ -1,4 +1,3 @@
import { questdbConfig } from '@stock-bot/config';
import { QuestDBClient } from './client';
import type { QuestDBClientConfig, QuestDBConnectionOptions } from './types';
@ -6,58 +5,20 @@ import type { QuestDBClientConfig, QuestDBConnectionOptions } from './types';
* Factory function to create a QuestDB client instance
*/
export function createQuestDBClient(
config?: Partial<QuestDBClientConfig>,
config: QuestDBClientConfig,
options?: QuestDBConnectionOptions
): QuestDBClient {
return new QuestDBClient(config, options);
}
/**
* Create a QuestDB client with default configuration
* Create and connect a QuestDB client
*/
export function createDefaultQuestDBClient(): QuestDBClient {
const config: Partial<QuestDBClientConfig> = {
host: questdbConfig.QUESTDB_HOST,
httpPort: questdbConfig.QUESTDB_HTTP_PORT,
pgPort: questdbConfig.QUESTDB_PG_PORT,
influxPort: questdbConfig.QUESTDB_INFLUX_PORT,
user: questdbConfig.QUESTDB_USER,
password: questdbConfig.QUESTDB_PASSWORD,
};
return new QuestDBClient(config);
}
/**
* Singleton QuestDB client instance
*/
let defaultClient: QuestDBClient | null = null;
/**
* Get or create the default QuestDB client instance
*/
export function getQuestDBClient(): QuestDBClient {
if (!defaultClient) {
defaultClient = createDefaultQuestDBClient();
}
return defaultClient;
}
/**
* Connect to QuestDB using the default client
*/
export async function connectQuestDB(): Promise<QuestDBClient> {
const client = getQuestDBClient();
export async function createAndConnectQuestDBClient(
config: QuestDBClientConfig,
options?: QuestDBConnectionOptions
): Promise<QuestDBClient> {
const client = createQuestDBClient(config, options);
await client.connect();
return client;
}
/**
* Disconnect from QuestDB
*/
export async function disconnectQuestDB(): Promise<void> {
if (defaultClient) {
await defaultClient.disconnect();
defaultClient = null;
}
}

2
libs/questdb-client/src/index.ts
View file

@ -29,4 +29,4 @@ export type {
} from './types';
// Utils
export { createQuestDBClient, getQuestDBClient } from './factory';
export { createQuestDBClient, createAndConnectQuestDBClient } from './factory';