From 7c2e055dd4827136ec4b891c8e0a896ae4f954af Mon Sep 17 00:00:00 2001 From: Bojan Kucera Date: Thu, 5 Jun 2025 09:05:40 -0400 Subject: [PATCH] fixed logging --- .env.complete | 1 - .env.example | 1 - .env.prod | 1 - apps/data-service/src/index.ts | 1 - docs/loki-logging.md | 2 +- libs/config/src/loki.ts | 5 +--- libs/logger/README.md | 2 -- libs/logger/src/logger.ts | 51 ++++++++++++---------------------- 8 files changed, 19 insertions(+), 45 deletions(-) diff --git a/.env.complete b/.env.complete index b970009..4aea194 100644 --- a/.env.complete +++ b/.env.complete @@ -121,7 +121,6 @@ LOKI_USERNAME= LOKI_PASSWORD= LOKI_TENANT_ID= LOKI_PUSH_TIMEOUT=10000 -LOKI_FLUSH_INTERVAL_MS=5000 LOKI_BATCH_SIZE=1024 LOKI_BATCH_WAIT=1000 LOKI_RETENTION_PERIOD=30d diff --git a/.env.example b/.env.example index aa9c632..3415413 100644 --- a/.env.example +++ b/.env.example @@ -37,7 +37,6 @@ LOKI_PASSWORD= LOKI_RETENTION_DAYS=30 LOKI_LABELS=environment=development,service=stock-bot LOKI_BATCH_SIZE=100 -LOKI_FLUSH_INTERVAL_MS=5000 # Feature Flags ENABLE_ML_SIGNALS=false diff --git a/.env.prod b/.env.prod index 255aaa8..2aa8d1d 100644 --- a/.env.prod +++ b/.env.prod @@ -116,7 +116,6 @@ LOKI_USERNAME=${LOKI_USERNAME} LOKI_PASSWORD=${LOKI_PASSWORD} LOKI_TENANT_ID=${LOKI_TENANT_ID} LOKI_PUSH_TIMEOUT=30000 -LOKI_FLUSH_INTERVAL_MS=5000 LOKI_BATCH_SIZE=2048 LOKI_BATCH_WAIT=5000 LOKI_RETENTION_PERIOD=90d diff --git a/apps/data-service/src/index.ts b/apps/data-service/src/index.ts index 013ead6..a2f9fa7 100644 --- a/apps/data-service/src/index.ts +++ b/apps/data-service/src/index.ts @@ -12,7 +12,6 @@ loadEnvVariables(); const app = new Hono(); const logger = createLogger('data-service'); const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002'); -logger.info(`Data Service starting on port ${PORT}`); // Health check endpoint app.get('/health', (c) => { return c.json({ diff --git a/docs/loki-logging.md b/docs/loki-logging.md index 19170d4..8184bf0 100644 --- a/docs/loki-logging.md +++ b/docs/loki-logging.md @@ -78,7 +78,7 @@ LOKI_PORT=3100 # Loki server port LOKI_RETENTION_DAYS=30 # Days to retain logs LOKI_LABELS=environment=development,service=stock-bot # Default labels LOKI_BATCH_SIZE=100 # Number of logs to batch before sending -LOKI_FLUSH_INTERVAL_MS=5000 # Max time to wait before sending logs +LOKI_BATCH_WAIT=5 # Max time to wait before sending logs ``` ## Useful Loki Queries diff --git a/libs/config/src/loki.ts b/libs/config/src/loki.ts index 1c8e54e..659c263 100644 --- a/libs/config/src/loki.ts +++ b/libs/config/src/loki.ts @@ -21,7 +21,7 @@ export const lokiConfig = cleanEnv(process.env, { // Push Configuration LOKI_PUSH_TIMEOUT: num({ default: 10000, desc: 'Push timeout in ms' }), LOKI_BATCH_SIZE: num({ default: 1024, desc: 'Batch size for log entries' }), - LOKI_BATCH_WAIT: num({ default: 1000, desc: 'Batch wait time in ms' }), + LOKI_BATCH_WAIT: num({ default: 5, desc: 'Batch wait time in ms' }), // Retention Settings LOKI_RETENTION_PERIOD: str({ default: '30d', desc: 'Log retention period' }), @@ -35,8 +35,6 @@ export const lokiConfig = cleanEnv(process.env, { LOKI_DEFAULT_LABELS: str({ default: '', desc: 'Default labels for all log entries (JSON format)' }), LOKI_SERVICE_LABEL: str({ default: 'stock-bot', desc: 'Service label for log entries' }), LOKI_ENVIRONMENT_LABEL: str({ default: 'development', desc: 'Environment label for log entries' }), - - LOKI_FLUSH_INTERVAL_MS: num({ default: 5000, desc: 'Flush interval ms' }), }); // Export typed configuration object @@ -60,5 +58,4 @@ export const { LOKI_DEFAULT_LABELS, LOKI_SERVICE_LABEL, LOKI_ENVIRONMENT_LABEL, - LOKI_FLUSH_INTERVAL_MS, } = lokiConfig; diff --git a/libs/logger/README.md b/libs/logger/README.md index 8c44bc8..24a5f2a 100644 --- a/libs/logger/README.md +++ b/libs/logger/README.md @@ -163,7 +163,6 @@ LOG_FILE_PATH=./logs LOKI_HOST=localhost LOKI_PORT=3100 LOKI_BATCH_SIZE=1024 -LOKI_FLUSH_INTERVAL_MS=5000 ``` ## Advanced Usage @@ -332,7 +331,6 @@ To use in your service: ### High memory usage - Reduce `LOKI_BATCH_SIZE` if batching too many logs -- Decrease `LOKI_FLUSH_INTERVAL_MS` to flush more frequently - Disable file logging if not needed ### Missing logs diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 1157d10..cf0bc2a 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -19,17 +19,6 @@ import type { LogLevel, LogContext, LogMetadata } from './types'; // Global logger instances cache const loggerInstances = new Map(); -// Pino log level mapping from string to number -const PINO_LEVELS: Record = { - silly: 10, - debug: 20, - verbose: 25, - http: 30, - info: 30, - warn: 40, - error: 50 -}; - /** * Create transport configuration for Pino based on options */ @@ -50,16 +39,11 @@ function createTransports(serviceName: string, options?: { if (enableConsole) { targets.push({ target: 'pino-pretty', - level: loggingConfig.LOG_LEVEL, - options: { + level: loggingConfig.LOG_LEVEL, options: { colorize: true, translateTime: 'yyyy-mm-dd HH:MM:ss.l', ignore: 'pid,hostname', - messageFormat: '[{service}] {msg}', - customPrettifiers: { - service: (service: string) => `${service}`, - level: (level: string) => `[${level.toUpperCase()}]` - } + messageFormat: '[{service}] {msg}' } }); } @@ -95,7 +79,7 @@ function createTransports(serviceName: string, options?: { level: loggingConfig.LOG_LEVEL, options: { batching: true, - interval: lokiConfig.LOKI_FLUSH_INTERVAL_MS, + interval: lokiConfig.LOKI_BATCH_WAIT, host: lokiConfig.LOKI_URL || `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`, basicAuth: lokiConfig.LOKI_USERNAME && lokiConfig.LOKI_PASSWORD ? { @@ -109,11 +93,13 @@ function createTransports(serviceName: string, options?: { ...(lokiConfig.LOKI_DEFAULT_LABELS ? JSON.parse(lokiConfig.LOKI_DEFAULT_LABELS) : {}) }, timeout: lokiConfig.LOKI_PUSH_TIMEOUT || 10000, - silenceErrors: false + silenceErrors: false, + // Better JSON handling + replaceTimestamp: false, } }); } - + return { targets }; @@ -161,27 +147,25 @@ function buildLogger(serviceName: string, options?: { enableFile, enableLoki }); - const loggerConfig: pino.LoggerOptions = { - // level: PINO_LEVELS[level] ? level : 'info', - // customLevels: PINO_LEVELS, - useOnlyCustomLevels: false, + level: level, timestamp: () => `,"timestamp":"${new Date().toISOString()}"`, - // formatters: { - // level: (label: string) => ({ level: label }), - // bindings: () => ({}) - // }, base: { service: serviceName, environment: loggingConfig.LOG_ENVIRONMENT, version: loggingConfig.LOG_SERVICE_VERSION - }, - transport + } }; + // Only add transport if targets exist to avoid worker thread issues + if (transport && transport.targets && transport.targets.length > 0) { + loggerConfig.transport = transport; + } + return pino(loggerConfig); } + /** * Enhanced Logger class with convenience methods and flexible message handling */ @@ -199,8 +183,7 @@ export class Logger { this.serviceName = serviceName; this.context = context; this.pino = createLogger(serviceName, options); - } - /** + } /** * Flexible log method that accepts string or object messages */ log(level: LogLevel, message: string | object, metadata?: LogMetadata): void { @@ -221,7 +204,7 @@ export class Logger { } /** - * Map custom log levels to Pino levels + * Map custom log levels to standard Pino levels */ private mapToPinoLevel(level: LogLevel): string { switch (level) {