diff --git a/libs/config/src/config-manager.ts b/libs/config/src/config-manager.ts index 717c308..c2449a9 100644 --- a/libs/config/src/config-manager.ts +++ b/libs/config/src/config-manager.ts @@ -1,15 +1,15 @@ import { join } from 'path'; import { z } from 'zod'; -import { - ConfigManagerOptions, - Environment, - ConfigLoader, - DeepPartial, - ConfigSchema -} from './types'; -import { ConfigError, ConfigValidationError } from './errors'; import { EnvLoader } from './loaders/env.loader'; import { FileLoader } from './loaders/file.loader'; +import { ConfigError, ConfigValidationError } from './errors'; +import { + ConfigLoader, + ConfigManagerOptions, + ConfigSchema, + DeepPartial, + Environment, +} from './types'; export class ConfigManager> { private config: T | null = null; @@ -19,7 +19,7 @@ export class ConfigManager> { constructor(options: ConfigManagerOptions = {}) { this.environment = options.environment || this.detectEnvironment(); - + // Default loaders if none provided if (options.loaders) { this.loaders = options.loaders; @@ -59,7 +59,11 @@ export class ConfigManager> { const mergedConfig = this.deepMerge(...configs) as T; // Add environment if not present - if (typeof mergedConfig === 'object' && mergedConfig !== null && !('environment' in mergedConfig)) { + if ( + typeof mergedConfig === 'object' && + mergedConfig !== null && + !('environment' in mergedConfig) + ) { (mergedConfig as Record)['environment'] = this.environment; } @@ -69,10 +73,7 @@ export class ConfigManager> { this.config = this.schema.parse(mergedConfig) as T; } catch (error) { if (error instanceof z.ZodError) { - throw new ConfigValidationError( - 'Configuration validation failed', - error.errors - ); + throw new ConfigValidationError('Configuration validation failed', error.errors); } throw error; } @@ -132,7 +133,10 @@ export class ConfigManager> { throw new ConfigError('Configuration not initialized. Call initialize() first.'); } - const updated = this.deepMerge(this.config as Record, updates as Record) as T; + const updated = this.deepMerge( + this.config as Record, + updates as Record + ) as T; // Re-validate if schema is present if (this.schema) { @@ -204,13 +208,13 @@ export class ConfigManager> { if (value === null || value === undefined) { result[key] = value; } else if ( - typeof value === 'object' && + typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date) && !(value instanceof RegExp) ) { result[key] = this.deepMerge( - (result[key] as Record) || {} as Record, + (result[key] as Record) || ({} as Record), value as Record ); } else { @@ -221,4 +225,4 @@ export class ConfigManager> { return result; } -} \ No newline at end of file +} diff --git a/libs/config/src/loaders/env.loader.ts b/libs/config/src/loaders/env.loader.ts index 50ed7da..a3eb6ba 100644 --- a/libs/config/src/loaders/env.loader.ts +++ b/libs/config/src/loaders/env.loader.ts @@ -1,6 +1,6 @@ -import { ConfigLoader } from '../types'; -import { ConfigLoaderError } from '../errors'; import { readFileSync } from 'fs'; +import { ConfigLoaderError } from '../errors'; +import { ConfigLoader } from '../types'; export interface EnvLoaderOptions { convertCase?: boolean; @@ -21,7 +21,7 @@ export class EnvLoader implements ConfigLoader { parseJson: true, parseValues: true, nestedDelimiter: '_', - ...options + ...options, }; } @@ -32,7 +32,7 @@ export class EnvLoader implements ConfigLoader { for (const path of possiblePaths) { this.loadEnvFile(path); } - + const config: Record = {}; const envVars = process.env; @@ -41,9 +41,7 @@ export class EnvLoader implements ConfigLoader { continue; } - const configKey = this.prefix - ? key.slice(this.prefix.length) - : key; + const configKey = this.prefix ? key.slice(this.prefix.length) : key; if (!this.options.convertCase && !this.options.nestedDelimiter) { // Simple case - just keep the key as is @@ -56,10 +54,7 @@ export class EnvLoader implements ConfigLoader { return config; } catch (error) { - throw new ConfigLoaderError( - `Failed to load environment variables: ${error}`, - 'EnvLoader' - ); + throw new ConfigLoaderError(`Failed to load environment variables: ${error}`, 'EnvLoader'); } } @@ -80,7 +75,11 @@ export class EnvLoader implements ConfigLoader { // Convert to camelCase const camelKey = this.toCamelCase(key); config[camelKey] = parsedValue; - } else if (this.options.nestedDelimiter && this.options.nestedDelimiter !== '_' && key.includes(this.options.nestedDelimiter)) { + } else if ( + this.options.nestedDelimiter && + this.options.nestedDelimiter !== '_' && + key.includes(this.options.nestedDelimiter) + ) { // Handle nested delimiter (e.g., APP__NAME -> { APP: { NAME: value } }) const parts = key.split(this.options.nestedDelimiter); this.setNestedValue(config, parts, parsedValue); @@ -108,7 +107,7 @@ export class EnvLoader implements ConfigLoader { if (!lastKey) { return false; // This should never happen due to length check above } - + try { const target = path.reduce((acc, key) => { if (!acc[key] || typeof acc[key] !== 'object') { @@ -116,7 +115,7 @@ export class EnvLoader implements ConfigLoader { } return acc[key] as Record; }, obj); - + (target as Record)[lastKey] = value; return true; } catch { @@ -126,63 +125,61 @@ export class EnvLoader implements ConfigLoader { } private toCamelCase(str: string): string { - return str - .toLowerCase() - .replace(/_([a-z])/g, (_, char) => char.toUpperCase()); + return str.toLowerCase().replace(/_([a-z])/g, (_, char) => char.toUpperCase()); } private getProviderMapping(envKey: string): { path: string[] } | null { // Provider-specific and special environment variable mappings const providerMappings: Record = { // WebShare provider mappings - 'WEBSHARE_API_KEY': ['webshare', 'apiKey'], - 'WEBSHARE_API_URL': ['webshare', 'apiUrl'], - 'WEBSHARE_ENABLED': ['webshare', 'enabled'], - - // EOD provider mappings - 'EOD_API_KEY': ['providers', 'eod', 'apiKey'], - 'EOD_BASE_URL': ['providers', 'eod', 'baseUrl'], - 'EOD_TIER': ['providers', 'eod', 'tier'], - 'EOD_ENABLED': ['providers', 'eod', 'enabled'], - 'EOD_PRIORITY': ['providers', 'eod', 'priority'], - + WEBSHARE_API_KEY: ['webshare', 'apiKey'], + WEBSHARE_API_URL: ['webshare', 'apiUrl'], + WEBSHARE_ENABLED: ['webshare', 'enabled'], + + // EOD provider mappings + EOD_API_KEY: ['providers', 'eod', 'apiKey'], + EOD_BASE_URL: ['providers', 'eod', 'baseUrl'], + EOD_TIER: ['providers', 'eod', 'tier'], + EOD_ENABLED: ['providers', 'eod', 'enabled'], + EOD_PRIORITY: ['providers', 'eod', 'priority'], + // Interactive Brokers provider mappings - 'IB_GATEWAY_HOST': ['providers', 'ib', 'gateway', 'host'], - 'IB_GATEWAY_PORT': ['providers', 'ib', 'gateway', 'port'], - 'IB_CLIENT_ID': ['providers', 'ib', 'gateway', 'clientId'], - 'IB_ACCOUNT': ['providers', 'ib', 'account'], - 'IB_MARKET_DATA_TYPE': ['providers', 'ib', 'marketDataType'], - 'IB_ENABLED': ['providers', 'ib', 'enabled'], - 'IB_PRIORITY': ['providers', 'ib', 'priority'], - + IB_GATEWAY_HOST: ['providers', 'ib', 'gateway', 'host'], + IB_GATEWAY_PORT: ['providers', 'ib', 'gateway', 'port'], + IB_CLIENT_ID: ['providers', 'ib', 'gateway', 'clientId'], + IB_ACCOUNT: ['providers', 'ib', 'account'], + IB_MARKET_DATA_TYPE: ['providers', 'ib', 'marketDataType'], + IB_ENABLED: ['providers', 'ib', 'enabled'], + IB_PRIORITY: ['providers', 'ib', 'priority'], + // QuoteMedia provider mappings - 'QM_USERNAME': ['providers', 'qm', 'username'], - 'QM_PASSWORD': ['providers', 'qm', 'password'], - 'QM_BASE_URL': ['providers', 'qm', 'baseUrl'], - 'QM_WEBMASTER_ID': ['providers', 'qm', 'webmasterId'], - 'QM_ENABLED': ['providers', 'qm', 'enabled'], - 'QM_PRIORITY': ['providers', 'qm', 'priority'], - + QM_USERNAME: ['providers', 'qm', 'username'], + QM_PASSWORD: ['providers', 'qm', 'password'], + QM_BASE_URL: ['providers', 'qm', 'baseUrl'], + QM_WEBMASTER_ID: ['providers', 'qm', 'webmasterId'], + QM_ENABLED: ['providers', 'qm', 'enabled'], + QM_PRIORITY: ['providers', 'qm', 'priority'], + // Yahoo Finance provider mappings - 'YAHOO_BASE_URL': ['providers', 'yahoo', 'baseUrl'], - 'YAHOO_COOKIE_JAR': ['providers', 'yahoo', 'cookieJar'], - 'YAHOO_CRUMB': ['providers', 'yahoo', 'crumb'], - 'YAHOO_ENABLED': ['providers', 'yahoo', 'enabled'], - 'YAHOO_PRIORITY': ['providers', 'yahoo', 'priority'], - + YAHOO_BASE_URL: ['providers', 'yahoo', 'baseUrl'], + YAHOO_COOKIE_JAR: ['providers', 'yahoo', 'cookieJar'], + YAHOO_CRUMB: ['providers', 'yahoo', 'crumb'], + YAHOO_ENABLED: ['providers', 'yahoo', 'enabled'], + YAHOO_PRIORITY: ['providers', 'yahoo', 'priority'], + // General application config mappings - 'NAME': ['name'], - 'VERSION': ['version'], - + NAME: ['name'], + VERSION: ['version'], + // Log mappings (using LOG_ prefix for all) - 'LOG_LEVEL': ['log', 'level'], - 'LOG_FORMAT': ['log', 'format'], - 'LOG_LOKI_ENABLED': ['log', 'loki', 'enabled'], - 'LOG_LOKI_HOST': ['log', 'loki', 'host'], - 'LOG_LOKI_PORT': ['log', 'loki', 'port'], - + LOG_LEVEL: ['log', 'level'], + LOG_FORMAT: ['log', 'format'], + LOG_LOKI_ENABLED: ['log', 'loki', 'enabled'], + LOG_LOKI_HOST: ['log', 'loki', 'host'], + LOG_LOKI_PORT: ['log', 'loki', 'port'], + // Special mappings to avoid conflicts - 'DEBUG_MODE': ['debug'], + DEBUG_MODE: ['debug'], }; const mapping = providerMappings[envKey]; @@ -208,16 +205,26 @@ export class EnvLoader implements ConfigLoader { } // Handle booleans - if (value.toLowerCase() === 'true') {return true;} - if (value.toLowerCase() === 'false') {return false;} + if (value.toLowerCase() === 'true') { + return true; + } + if (value.toLowerCase() === 'false') { + return false; + } // Handle numbers const num = Number(value); - if (!isNaN(num) && value !== '') {return num;} + if (!isNaN(num) && value !== '') { + return num; + } // Handle null/undefined - if (value.toLowerCase() === 'null') {return null;} - if (value.toLowerCase() === 'undefined') {return undefined;} + if (value.toLowerCase() === 'null') { + return null; + } + if (value.toLowerCase() === 'undefined') { + return undefined; + } // Return as string return value; @@ -227,27 +234,29 @@ export class EnvLoader implements ConfigLoader { try { const envContent = readFileSync(filePath, 'utf-8'); const lines = envContent.split('\n'); - + for (const line of lines) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith('#')) { continue; // Skip empty lines and comments } - + const equalIndex = trimmed.indexOf('='); if (equalIndex === -1) { continue; // Skip lines without = } - + const key = trimmed.substring(0, equalIndex).trim(); let value = trimmed.substring(equalIndex + 1).trim(); - + // Remove surrounding quotes if present - if ((value.startsWith('"') && value.endsWith('"')) || - (value.startsWith("'") && value.endsWith("'"))) { + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { value = value.slice(1, -1); } - + // Only set if not already set (allows override precedence) if (!(key in process.env)) { process.env[key] = value; @@ -257,8 +266,11 @@ export class EnvLoader implements ConfigLoader { // File not found is not an error (env files are optional) if (error && typeof error === 'object' && 'code' in error && error.code !== 'ENOENT') { // eslint-disable-next-line no-console - console.warn(`Warning: Could not load env file ${filePath}:`, error instanceof Error ? error.message : String(error)); + console.warn( + `Warning: Could not load env file ${filePath}:`, + error instanceof Error ? error.message : String(error) + ); } } } -} \ No newline at end of file +} diff --git a/libs/config/src/loaders/file.loader.ts b/libs/config/src/loaders/file.loader.ts index a6484f3..0306054 100644 --- a/libs/config/src/loaders/file.loader.ts +++ b/libs/config/src/loaders/file.loader.ts @@ -1,7 +1,7 @@ -import { readFileSync, existsSync } from 'fs'; +import { existsSync, readFileSync } from 'fs'; import { join } from 'path'; -import { ConfigLoader } from '../types'; import { ConfigLoaderError } from '../errors'; +import { ConfigLoader } from '../types'; export class FileLoader implements ConfigLoader { readonly priority = 50; // Medium priority @@ -30,10 +30,7 @@ export class FileLoader implements ConfigLoader { // Merge configs (later configs override earlier ones) return this.deepMerge(...configs); } catch (error) { - throw new ConfigLoaderError( - `Failed to load configuration files: ${error}`, - 'FileLoader' - ); + throw new ConfigLoaderError(`Failed to load configuration files: ${error}`, 'FileLoader'); } } @@ -56,7 +53,10 @@ export class FileLoader implements ConfigLoader { if (value === null || value === undefined) { result[key] = value; } else if (typeof value === 'object' && !Array.isArray(value) && value !== null) { - result[key] = this.deepMerge(result[key] as Record || {}, value as Record); + result[key] = this.deepMerge( + (result[key] as Record) || {}, + value as Record + ); } else { result[key] = value; } @@ -65,4 +65,4 @@ export class FileLoader implements ConfigLoader { return result; } -} \ No newline at end of file +} diff --git a/libs/config/src/types/index.ts b/libs/config/src/types/index.ts index 3f7472a..2290b3b 100644 --- a/libs/config/src/types/index.ts +++ b/libs/config/src/types/index.ts @@ -25,4 +25,4 @@ export interface ProviderConfig { name: string; enabled: boolean; [key: string]: unknown; -} \ No newline at end of file +} diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 6f30299..559c91c 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -9,6 +9,7 @@ */ import pino from 'pino'; +import pretty from 'pino-pretty'; import type { LogContext, LoggerConfig, LogLevel, LogMetadata } from './types'; // Simple cache for logger instances @@ -35,68 +36,73 @@ export function setLoggerConfig(config: LoggerConfig): void { console.log('Logger config updated:', globalConfig.logLevel); } /** - * Create transport configuration + * Create logger destination using multistream approach: + * - Console: In-process pretty stream (fast shutdown, disabled in production) + * - File/Loki: Worker transports (default timeout, ok to wait) */ -function createTransports(serviceName: string, config: LoggerConfig = globalConfig): any { - const targets: any[] = []; +function createDestination( + serviceName: string, + config: LoggerConfig = globalConfig +): pino.DestinationStream | null { + const streams: pino.StreamEntry[] = []; - // Console transport - if (config.logConsole) { - targets.push({ - target: 'pino-pretty', - level: config.logLevel || 'info', - options: { - colorize: true, - translateTime: 'yyyy-mm-dd HH:MM:ss.l', - messageFormat: '[{service}{childName}] {msg}', - singleLine: true, - hideObject: false, - ignore: 'pid,hostname,service,environment,version,childName', - errorLikeObjectKeys: ['err', 'error'], - errorProps: 'message,stack,name,code', - // Ensure the transport respects the configured level - minimumLevel: config.logLevel || 'info', - }, + // Console: In-process pretty stream for dev (fast shutdown) + if (config.logConsole && config.environment !== 'production') { + const prettyStream = pretty({ + sync: true, + colorize: true, + translateTime: 'yyyy-mm-dd HH:MM:ss.l', + messageFormat: '[{service}{childName}] {msg}', + singleLine: true, + hideObject: false, + ignore: 'pid,hostname,service,environment,version,childName', + errorLikeObjectKeys: ['err', 'error'], + errorProps: 'message,stack,name,code', }); + streams.push({ stream: prettyStream }); } - // File transport + // File: Worker transport (has timeout but acceptable) if (config.logFile) { - targets.push({ - target: 'pino/file', - level: config.logLevel || 'info', - options: { - destination: `${config.logFilePath}/${serviceName}.log`, - mkdir: true, - }, - }); - } - - // Loki transport - if (config.logLoki && config.lokiHost) { - targets.push({ - target: 'pino-loki', - level: config.logLevel || 'info', - options: { - host: config.lokiHost, - labels: { - service: serviceName, - environment: config.environment || 'development', + streams.push( + pino.transport({ + target: 'pino/file', + level: config.logLevel || 'info', + options: { + destination: `${config.logFilePath}/${serviceName}.log`, + mkdir: true, }, - ignore: 'childName', - ...(config.lokiUser && config.lokiPassword - ? { - basicAuth: { - username: config.lokiUser, - password: config.lokiPassword, - }, - } - : {}), - }, - }); + }) + ); } - return targets.length > 0 ? { targets } : null; + // Loki: Worker transport (has timeout but acceptable) + if (config.logLoki && config.lokiHost) { + streams.push( + pino.transport({ + target: 'pino-loki', + level: config.logLevel || 'info', + options: { + host: config.lokiHost, + labels: { + service: serviceName, + environment: config.environment || 'development', + }, + ignore: 'childName', + ...(config.lokiUser && config.lokiPassword + ? { + basicAuth: { + username: config.lokiUser, + password: config.lokiPassword, + }, + } + : {}), + }, + }) + ); + } + + return streams.length > 0 ? pino.multistream(streams) : null; } /** @@ -105,7 +111,7 @@ function createTransports(serviceName: string, config: LoggerConfig = globalConf function getPinoLogger(serviceName: string, config: LoggerConfig = globalConfig): pino.Logger { const cacheKey = `${serviceName}-${JSON.stringify(config)}`; if (!loggerCache.has(cacheKey)) { - const transport = createTransports(serviceName, config); + const destination = createDestination(serviceName, config); const loggerOptions: pino.LoggerOptions = { level: config.logLevel || 'info', @@ -116,11 +122,8 @@ function getPinoLogger(serviceName: string, config: LoggerConfig = globalConfig) }, }; - if (transport) { - loggerOptions.transport = transport; - } + const logger = destination ? pino(loggerOptions, destination) : pino(loggerOptions); - const logger = pino(loggerOptions); loggerCache.set(cacheKey, logger); } @@ -356,5 +359,35 @@ export async function shutdownLoggers(): Promise { } } +/** + * Graceful shutdown - flush all logger transports quickly + * Use this in your application shutdown handlers + */ +export async function gracefulShutdown(): Promise { + const flushPromises: Promise[] = []; + + for (const logger of loggerCache.values()) { + // Use pino v9's flush() method - this is much faster than the complex shutdown + flushPromises.push( + new Promise((resolve, reject) => { + logger.flush((err?: Error) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }) + ); + } + + try { + await Promise.all(flushPromises); + } catch (error) { + // eslint-disable-next-line no-console + console.error('Logger graceful shutdown failed:', error); + } +} + // Export types for convenience export type { LogContext, LogLevel, LogMetadata } from './types';