diff --git a/apps/data-service/src/index.ts b/apps/data-service/src/index.ts index ffcbeee..c2cd8b0 100644 --- a/apps/data-service/src/index.ts +++ b/apps/data-service/src/index.ts @@ -1,7 +1,7 @@ /** * Data Service - Combined live and historical data ingestion */ -import { createLogger, GracefulShutdownManager } from '@stock-bot/logger'; +import { createLogger } from '@stock-bot/logger'; import { loadEnvVariables } from '@stock-bot/config'; import { Hono } from 'hono'; import { serve } from '@hono/node-server'; @@ -11,7 +11,6 @@ loadEnvVariables(); const app = new Hono(); const logger = createLogger('data-service'); -const shutdownManager = new GracefulShutdownManager(logger); const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002'); // Health check endpoint diff --git a/apps/data-service/src/proxy-demo.ts b/apps/data-service/src/proxy-demo.ts index 7a4ff51..30adb08 100644 --- a/apps/data-service/src/proxy-demo.ts +++ b/apps/data-service/src/proxy-demo.ts @@ -107,7 +107,8 @@ async function demonstrateCustomProxySource() { sourceCount: customSources.length }); } catch (error) { - logger.error('❌ Custom source scraping failed', error, { + logger.error('❌ Custom source scraping failed',{ + error: error as Error, sourceUrls: customSources.map(s => s.url) }); } diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index 2b1822b..d935a21 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -97,7 +97,7 @@ export class ProxyService { try { await this.scrapeProxies(); } catch (error) { - this.logger.error('Error in periodic proxy refresh', { error }); + this.logger.error('Error in periodic proxy refresh', {error}); } }, intervalMs); } @@ -169,7 +169,7 @@ export class ProxyService { } catch (error) { this.logger.error('Error scraping from source', { url: source.url, - error: error instanceof Error ? error.message : String(error) + error: error }); return []; } @@ -318,7 +318,7 @@ export class ProxyService { } } } catch (error) { - this.logger.error('Error updating proxy status', { error }); + this.logger.error('Error updating proxy status', {error}); } } /** @@ -346,7 +346,7 @@ export class ProxyService { this.logger.warn('No working proxies available'); return null; } catch (error) { - this.logger.error('Error getting working proxy', { error }); + this.logger.error('Error getting working proxy', {error}); return null; } } @@ -371,7 +371,7 @@ export class ProxyService { return workingProxies; } catch (error) { - this.logger.error('Error getting working proxies', { error }); + this.logger.error('Error getting working proxies', {error}); return []; } } @@ -391,7 +391,7 @@ export class ProxyService { this.logger.warn('getAllProxies not fully implemented - Redis cache provider limitations'); return proxies; } catch (error) { - this.logger.error('Error getting all proxies', { error }); + this.logger.error('Error getting all proxies', {error}); return []; } } @@ -424,7 +424,7 @@ export class ProxyService { return stats; } catch (error) { - this.logger.error('Error getting proxy stats', { error }); + this.logger.error('Error getting proxy stats', {error}); return { total: 0, working: 0, @@ -485,7 +485,7 @@ export class ProxyService { stillWorking: successCount }); } catch (error) { - this.logger.error('Error in health check', { error }); + this.logger.error('Error in health check', {error}); } }, intervalMs); } @@ -502,7 +502,7 @@ export class ProxyService { this.logger.info('Cleared proxy stats from cache'); this.logger.warn('Full proxy data clearing not implemented due to cache provider limitations'); } catch (error) { - this.logger.error('Error clearing proxy data', { error }); + this.logger.error('Error clearing proxy data', {error}); } } diff --git a/libs/logger/src/gracefulShutdown.ts b/libs/logger/src/gracefulShutdown.ts deleted file mode 100644 index ef2d783..0000000 --- a/libs/logger/src/gracefulShutdown.ts +++ /dev/null @@ -1,116 +0,0 @@ -import pino from 'pino'; - -interface ShutdownHandler { - name: string; - handler: () => Promise | void; - timeout?: number; -} - -export class GracefulShutdownManager { - private logger: pino.Logger; - private handlers: ShutdownHandler[] = []; - private isShuttingDown = false; - private startTime = Date.now(); - - constructor(logger: pino.Logger) { - this.logger = logger; - this.setupSignalHandlers(); - } - - /** - * Register a shutdown handler - */ - addHandler(handler: ShutdownHandler): void { - this.handlers.push(handler); - } - - /** - * Setup signal handlers for graceful shutdown - */ - private setupSignalHandlers(): void { - // Graceful shutdown signals - process.on('SIGTERM', () => this.shutdown('SIGTERM', 0)); - process.on('SIGINT', () => this.shutdown('SIGINT', 0)); - process.on('SIGUSR2', () => this.shutdown('SIGUSR2', 0)); // nodemon - - // Fatal error handlers - process.on('uncaughtException', (error) => { - this.logger.error('Uncaught exception', error, { fatal: true }); - this.shutdown('uncaughtException', 1); - }); - - process.on('unhandledRejection', (reason, promise) => { - this.logger.error('Unhandled promise rejection', { - reason: reason instanceof Error ? reason.message : reason, - stack: reason instanceof Error ? reason.stack : undefined, - fatal: true - }); - this.shutdown('unhandledRejection', 1); - }); - - // Process warnings - process.on('warning', (warning) => { - this.logger.warn('Process warning', { - name: warning.name, - message: warning.message, - stack: warning.stack - }); - }); - - // Final exit - process.on('exit', (code) => { - const uptime = Date.now() - this.startTime; - console.log(`Process exiting with code ${code} after ${uptime}ms`); - }); - } - - /** - * Execute graceful shutdown - */ - private async shutdown(signal: string, exitCode: number): Promise { - if (this.isShuttingDown) { - this.logger.warn('Force shutdown - multiple signals received'); - process.exit(1); - } - - this.isShuttingDown = true; - const shutdownStart = Date.now(); - - this.logger.info('Graceful shutdown initiated', { - signal, - exitCode, - uptime: Date.now() - this.startTime, - memoryUsage: process.memoryUsage(), - handlersCount: this.handlers.length - }); - - // Execute shutdown handlers - for (const handler of this.handlers) { - try { - this.logger.info(`Executing shutdown handler: ${handler.name}`); - const timeout = handler.timeout || 5000; - - await Promise.race([ - Promise.resolve(handler.handler()), - new Promise((_, reject) => - setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout) - ) - ]); - - this.logger.info(`Shutdown handler completed: ${handler.name}`); - } catch (error) { - this.logger.error(`Shutdown handler failed: ${handler.name}`, error); - } - } - - const shutdownDuration = Date.now() - shutdownStart; - this.logger.info('Graceful shutdown completed', { - signal, - shutdownDuration, - totalUptime: Date.now() - this.startTime - }); - - // Give logs time to flush - setTimeout(() => process.exit(exitCode), 100); - } -} \ No newline at end of file diff --git a/libs/logger/src/index.ts b/libs/logger/src/index.ts index af7bfd4..e5ab7f6 100644 --- a/libs/logger/src/index.ts +++ b/libs/logger/src/index.ts @@ -1,49 +1,14 @@ /** - * @stock-bot/logger - Enhanced logging library with Loki integration + * @stock-bot/logger - Simplified logging library * * Main exports for the logger library */ // Core logger classes and functions -export { Logger, createLogger, getLogger, shutdownLoggers } from './logger'; +export { Logger, createLogger, getLogger } from './logger'; -// Utility functions -export { - createTimer, - formatError, - sanitizeMetadata, - generateCorrelationId, - extractHttpMetadata, - createBusinessEvent, - createSecurityEvent, - maskSensitiveData, - calculateLogSize, - LogThrottle -} from './utils'; +// Type definitions +export type { LogLevel, LogContext, LogMetadata } from './types'; -// Hono middleware -export { - loggingMiddleware, - errorLoggingMiddleware, - createRequestLogger, - performanceMiddleware, - securityMiddleware, - businessEventMiddleware, - comprehensiveLoggingMiddleware -} from './middleware'; - -// Type exports -export type { - LogLevel, - LogContext, - LogMetadata, - LoggerOptions, - LokiTransportOptions, - PerformanceTimer, - LokiLogEntry, - StructuredLog -} from './types'; - -export type { LoggingMiddlewareOptions } from './middleware'; - -export { GracefulShutdownManager } from './gracefulShutdown'; +// Default export +export { getLogger as default } from './logger'; diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index f20694a..bb32aaa 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -1,59 +1,47 @@ /** - * Enhanced Pino-based logger with Loki integration for Stock Bot platform + * Simplified Pino-based logger for Stock Bot platform * * Features: * - High performance JSON logging with Pino - * - Multiple log levels (debug, info, warn, error, http, verbose, silly) - * - Console and file logging with pino-pretty formatting - * - Loki integration for centralized logging + * - Console, file, and Loki transports * - Structured logging with metadata - * - Flexible message handling (string or object) * - Service-specific context - * - TypeScript-friendly interfaces */ import pino from 'pino'; import { loggingConfig, lokiConfig } from '@stock-bot/config'; import type { LogLevel, LogContext, LogMetadata } from './types'; -// Global logger instances cache -const loggerInstances = new Map(); +// Simple cache for logger instances +const loggerCache = new Map(); /** - * Create transport configuration for Pino based on options + * Create transport configuration */ -function createTransports(serviceName: string, options?: { - enableConsole?: boolean; - enableFile?: boolean; - enableLoki?: boolean; -}): any { - const { - enableConsole = loggingConfig.LOG_CONSOLE, - enableFile = loggingConfig.LOG_FILE, - enableLoki = true - } = options || {}; - +function createTransports(serviceName: string): any { const targets: any[] = []; - // Console transport with pretty formatting - if (enableConsole) { + // const isDev = loggingConfig.LOG_ENVIRONMENT === 'development'; + + // Console transport + if (loggingConfig.LOG_CONSOLE) { targets.push({ target: 'pino-pretty', level: loggingConfig.LOG_LEVEL, options: { colorize: true, translateTime: 'yyyy-mm-dd HH:MM:ss.l', - ignore: 'pid,hostname,environment,version,service', messageFormat: '[{service}] {msg}', + singleLine: true, + hideObject: false, + ignore: 'pid,hostname,service,environment,version', errorLikeObjectKeys: ['err', 'error'], errorProps: 'message,stack,name,code', - singleLine: false, // Allow multiline for better metadata display - hideObject: false, // Show metadata objects } }); } - // File transport for general logs - if (enableFile) { + // File transport + if (loggingConfig.LOG_FILE) { targets.push({ target: 'pino/file', level: loggingConfig.LOG_LEVEL, @@ -62,335 +50,153 @@ function createTransports(serviceName: string, options?: { mkdir: true } }); - - // Separate error file if enabled - if (loggingConfig.LOG_ERROR_FILE) { - targets.push({ - target: 'pino/file', - level: 'error', - options: { - destination: `${loggingConfig.LOG_FILE_PATH}/${serviceName}-error.log`, - mkdir: true - } - }); - } } - // Loki transport for centralized logging - if (enableLoki && lokiConfig.LOKI_HOST) { + // Loki transport + if (lokiConfig.LOKI_HOST) { targets.push({ target: 'pino-loki', level: loggingConfig.LOG_LEVEL, options: { - batching: true, - interval: lokiConfig.LOKI_BATCH_WAIT, host: lokiConfig.LOKI_URL || `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`, - basicAuth: lokiConfig.LOKI_USERNAME && lokiConfig.LOKI_PASSWORD - ? { - username: lokiConfig.LOKI_USERNAME, - password: lokiConfig.LOKI_PASSWORD - } - : undefined, labels: { service: serviceName, - environment: lokiConfig.LOKI_ENVIRONMENT_LABEL, - ...(lokiConfig.LOKI_DEFAULT_LABELS ? JSON.parse(lokiConfig.LOKI_DEFAULT_LABELS) : {}) - }, - timeout: lokiConfig.LOKI_PUSH_TIMEOUT || 10000, - silenceErrors: false, - // Better JSON handling - replaceTimestamp: false, + environment: lokiConfig.LOKI_ENVIRONMENT_LABEL + } } }); } - - return { - targets - }; + + return targets.length > 0 ? { targets } : null; } /** - * Create or retrieve a logger instance for a specific service + * Get or create pino logger */ -export function createLogger(serviceName: string, options?: { - level?: LogLevel; - enableLoki?: boolean; - enableFile?: boolean; - enableConsole?: boolean; -}): pino.Logger { - const key = `${serviceName}-${JSON.stringify(options || {})}`; - - if (loggerInstances.has(key)) { - return loggerInstances.get(key)!; - } +function getPinoLogger(serviceName: string): pino.Logger { + if (!loggerCache.has(serviceName)) { + const transport = createTransports(serviceName); + + const config: pino.LoggerOptions = { + level: loggingConfig.LOG_LEVEL, + base: { + service: serviceName, + environment: loggingConfig.LOG_ENVIRONMENT, + version: loggingConfig.LOG_SERVICE_VERSION + } + }; - const logger = buildLogger(serviceName, options); - loggerInstances.set(key, logger); - - return logger; -} - -/** - * Build a Pino logger with all configured transports - */ -function buildLogger(serviceName: string, options?: { - level?: LogLevel; - enableLoki?: boolean; - enableFile?: boolean; - enableConsole?: boolean; -}): pino.Logger { - const { - level = loggingConfig.LOG_LEVEL as LogLevel, - enableLoki = true, - enableFile = loggingConfig.LOG_FILE, - enableConsole = loggingConfig.LOG_CONSOLE - } = options || {}; - - const transport = createTransports(serviceName, { - enableConsole, - enableFile, - enableLoki - }); - const loggerConfig: pino.LoggerOptions = { - level: level, - timestamp: () => `,"timestamp":"${new Date().toISOString()}"`, - base: { - service: serviceName, - environment: loggingConfig.LOG_ENVIRONMENT, - version: loggingConfig.LOG_SERVICE_VERSION + if (transport) { + config.transport = transport; } - }; - // Only add transport if targets exist to avoid worker thread issues - if (transport && transport.targets && transport.targets.length > 0) { - loggerConfig.transport = transport; + loggerCache.set(serviceName, pino(config)); } - return pino(loggerConfig); + return loggerCache.get(serviceName)!; } /** - * Enhanced Logger class with convenience methods and flexible message handling + * Simplified Logger class */ export class Logger { private pino: pino.Logger; - private serviceName: string; private context: LogContext; - constructor(serviceName: string, context: LogContext = {}, options?: { - level?: LogLevel; - enableLoki?: boolean; - enableFile?: boolean; - enableConsole?: boolean; - }) { - this.serviceName = serviceName; + + constructor(serviceName: string, context: LogContext = {}) { + this.pino = getPinoLogger(serviceName); this.context = context; - this.pino = createLogger(serviceName, options); } /** - * Flexible log method that accepts string or object messages + * Core log method */ - log(level: LogLevel, message: string | object, metadata?: LogMetadata): void { - const logData = { - ...this.context, - ...metadata, - }; + private log(level: LogLevel, message: string | object, metadata?: LogMetadata): void { + const data = { ...this.context, ...metadata }; if (typeof message === 'string') { - (this.pino as any)[level](logData, message); + (this.pino as any)[level](data, message); } else { - // For object messages, use the message object as the main log data - // and add metadata without overwriting properties from the message - const messageObj = message as Record; - const combinedData = { ...logData }; - - // Add message properties that don't conflict with metadata - Object.keys(messageObj).forEach(key => { - if (!(key in combinedData)) { - combinedData[key] = messageObj[key]; - } - }); - - (this.pino as any)[level](combinedData, messageObj.msg || messageObj.message || 'Object log'); + (this.pino as any)[level]({ ...data, data: message }, 'Object logged'); } } - /** - * Debug level logging - */ + // Simple log level methods debug(message: string | object, metadata?: LogMetadata): void { this.log('debug', message, metadata); } - /** - * Verbose level logging - */ - verbose(message: string | object, metadata?: LogMetadata): void { - // Map verbose to debug level since Pino doesn't have verbose by default - this.log('debug', message, { ...metadata, originalLevel: 'verbose' }); - } - - /** - * Silly level logging - */ - silly(message: string | object, metadata?: LogMetadata): void { - // Map silly to debug level since Pino doesn't have silly by default - this.log('debug', message, { ...metadata, originalLevel: 'silly' }); - } - - /** - * Info level logging - */ info(message: string | object, metadata?: LogMetadata): void { this.log('info', message, metadata); } - /** - * Warning level logging - */ warn(message: string | object, metadata?: LogMetadata): void { this.log('warn', message, metadata); } - /** - * Error level logging with proper pino-pretty formatting - */ - error(message: string | object, error?: Error | any, metadata?: LogMetadata): void { - const logData: LogMetadata = { ...metadata }; + + error(message: string | object, metadata?: LogMetadata & { error?: any }): void { + const data = { ...metadata }; - if (error) { - if (error instanceof Error) { - // Use 'err' key for pino-pretty compatibility - logData.err = error; - } else { - logData.error = error; - } + // Handle any type of error automatically + if (data.error) { + data.err = this.normalizeError(data.error); + delete data.error; } - - this.log('error', message, logData); + + this.log('error', message, data); } /** - * HTTP request logging + * Normalize any error type to a structured format */ - http(message: string | object, requestData?: { - method?: string; - url?: string; - statusCode?: number; - responseTime?: number; - userAgent?: string; - ip?: string; - }): void { - if (!loggingConfig.LOG_HTTP_REQUESTS) return; - - this.log('info', message, { - request: requestData, - type: 'http_request' - }); + private normalizeError(error: any): any { + if (error instanceof Error) { + return { + name: error.name, + message: error.message, + stack: error.stack, + }; + } + + if (error && typeof error === 'object') { + // Handle error-like objects + return { + name: error.name || 'UnknownError', + message: error.message || error.toString(), + ...(error.stack && { stack: error.stack }), + ...(error.code && { code: error.code }), + ...(error.status && { status: error.status }) + }; + } + + // Handle primitives (string, number, etc.) + return { + name: 'UnknownError', + message: String(error) + }; } /** - * Performance/timing logging - */ - performance(message: string | object, timing: { - operation: string; - duration: number; - startTime?: number; - endTime?: number; - }): void { - if (!loggingConfig.LOG_PERFORMANCE) return; - - this.log('info', message, { - performance: timing, - type: 'performance' - }); - } - - /** - * Business event logging - */ - business(message: string | object, event: { - type: string; - entity?: string; - action?: string; - result?: 'success' | 'failure' | 'partial'; - amount?: number; - symbol?: string; - }): void { - this.log('info', message, { - business: event, - type: 'business_event' - }); - } - - /** - * Security event logging - */ - security(message: string | object, event: { - type: 'authentication' | 'authorization' | 'access' | 'vulnerability'; - user?: string; - resource?: string; - action?: string; - result?: 'success' | 'failure'; - ip?: string; - severity?: 'low' | 'medium' | 'high' | 'critical'; - }): void { - this.log('warn', message, { - security: event, - type: 'security_event' - }); - } - - /** - * Create a child logger with additional context + * Create child logger with additional context */ child(context: LogContext): Logger { - return new Logger(this.serviceName, { ...this.context, ...context }); - } - - /** - * Add persistent context to this logger - */ - addContext(context: LogContext): void { - this.context = { ...this.context, ...context }; - } - - /** - * Get the underlying Pino logger - */ - getPinoLogger(): pino.Logger { - return this.pino; - } - - /** - * Gracefully close all transports - */ - async close(): Promise { - // Pino doesn't require explicit closing like Winston - // But we can flush any pending logs - this.pino.flush(); + return new Logger((this.pino.bindings() as any).service, { ...this.context, ...context }); } } /** - * Create a default logger instance for a service + * Main factory function */ export function getLogger(serviceName: string, context?: LogContext): Logger { return new Logger(serviceName, context); } /** - * Shutdown all logger instances gracefully + * Keep backward compatibility */ -export async function shutdownLoggers(): Promise { - // Flush all logger instances - const flushPromises = Array.from(loggerInstances.values()).map(logger => { - logger.flush(); - return Promise.resolve(); - }); - - await Promise.all(flushPromises); - loggerInstances.clear(); +export function createLogger(serviceName: string): pino.Logger { + return getPinoLogger(serviceName); } // Export types for convenience diff --git a/libs/logger/src/middleware.ts b/libs/logger/src/middleware.ts deleted file mode 100644 index ce10e73..0000000 --- a/libs/logger/src/middleware.ts +++ /dev/null @@ -1,462 +0,0 @@ -/** - * Comprehensive Hono middleware for request logging with Pino integration - */ - -import type { Context, Next } from 'hono'; -import { Logger } from './logger'; -import { generateCorrelationId, createTimer, sanitizeMetadata, LogThrottle } from './utils'; - -export interface LoggingMiddlewareOptions { - logger?: Logger; - serviceName?: string; - skipPaths?: string[]; - skipSuccessfulRequests?: boolean; - logRequestBody?: boolean; - logResponseBody?: boolean; - maxBodySize?: number; - enablePerformanceMetrics?: boolean; - enableThrottling?: boolean; - throttleConfig?: { - maxLogs?: number; - windowMs?: number; - }; - sensitiveHeaders?: string[]; - redactSensitiveData?: boolean; -} - -/** - * Hono middleware for HTTP request/response logging with enhanced features - */ -export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { - const { - serviceName = 'unknown-service', - skipPaths = ['/health', '/metrics', '/favicon.ico'], - skipSuccessfulRequests = false, - logRequestBody = false, - logResponseBody = false, - maxBodySize = 1024, - enablePerformanceMetrics = true, - enableThrottling = false, - throttleConfig = { maxLogs: 100, windowMs: 60000 }, - sensitiveHeaders = ['authorization', 'cookie', 'x-api-key', 'x-auth-token'], - redactSensitiveData = true - } = options; - - // Create logger if not provided - const logger = options.logger || new Logger(serviceName); - - // Create throttle instance if enabled - const throttle = enableThrottling ? new LogThrottle(throttleConfig.maxLogs, throttleConfig.windowMs) : null; - - return async (c: Context, next: Next) => { - const url = new URL(c.req.url); - const path = url.pathname; - - // Skip certain paths - if (skipPaths.some(skipPath => path.startsWith(skipPath))) { - return next(); - } - - // Check throttling - const throttleKey = `${c.req.method}-${path}`; - if (throttle && !throttle.shouldLog(throttleKey)) { - return next(); - } - - // Generate correlation ID - const correlationId = generateCorrelationId(); - // Store correlation ID in context for later use - c.set('correlationId', correlationId); - // Set correlation ID as response header - c.header('x-correlation-id', correlationId); - - // Start timer - const timer = createTimer(`${c.req.method} ${path}`); - - // Extract request headers (sanitized) - const rawHeaders = Object.fromEntries( - Array.from(c.req.raw.headers.entries()) - ); - - const headers = redactSensitiveData - ? Object.fromEntries( - Object.entries(rawHeaders).map(([key, value]) => [ - key, - sensitiveHeaders.some(sensitive => key.toLowerCase().includes(sensitive.toLowerCase())) - ? '[REDACTED]' - : value - ]) - ) - : rawHeaders; // Extract request metadata following LogMetadata.request structure - const requestMetadata: any = { - method: c.req.method, - url: c.req.url, - path: path, - userAgent: c.req.header('user-agent'), - contentType: c.req.header('content-type'), - contentLength: c.req.header('content-length'), - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown', - headers: headers - }; - // Add request body if enabled - if (logRequestBody) { - try { - const body = await c.req.text(); - if (body) { - requestMetadata.body = body.length > maxBodySize - ? body.substring(0, maxBodySize) + '...[truncated]' - : body; - } - } catch (error) { - // Body might not be available or already consumed - requestMetadata.bodyError = 'Unable to read request body'; - } - } // Log request start with structured data - logger.http('HTTP Request started', { - method: c.req.method, - url: c.req.url, - userAgent: c.req.header('user-agent'), - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' - }); - - let error: Error | null = null; - let responseTime: number = 0; - - try { - // Process request - await next(); - } catch (err) { - error = err instanceof Error ? err : new Error(String(err)); - throw error; // Re-throw to maintain error handling flow - } finally { - // Calculate response time - const timing = timer.end(); - responseTime = timing.duration; - - // Get response information - const response = c.res; - const status = response.status; - - // Determine log level based on status code - const isError = status >= 400 || error !== null; - const isSuccess = status >= 200 && status < 300; - const isRedirect = status >= 300 && status < 400; - - // Skip successful requests if configured - if (skipSuccessfulRequests && isSuccess && !error) { - return; - } - - // Extract response headers (sanitized) - const rawResponseHeaders = Object.fromEntries( - Array.from(response.headers.entries()) - ); - - const responseHeaders = redactSensitiveData - ? Object.fromEntries( - Object.entries(rawResponseHeaders).map(([key, value]) => [ - key, - sensitiveHeaders.some(sensitive => key.toLowerCase().includes(sensitive.toLowerCase())) - ? '[REDACTED]' - : value - ]) - ) - : rawResponseHeaders; - - const responseMetadata: any = { - statusCode: status, - statusText: response.statusText, - headers: responseHeaders, - contentLength: response.headers.get('content-length'), - contentType: response.headers.get('content-type') - }; - - // Add response body if enabled and not an error - if (logResponseBody && !isError) { - try { - const responseBody = await response.clone().text(); - if (responseBody) { - responseMetadata.body = responseBody.length > maxBodySize - ? responseBody.substring(0, maxBodySize) + '...[truncated]' - : responseBody; - } - } catch (bodyError) { - responseMetadata.bodyError = 'Unable to read response body'; - } - } // Performance metrics matching LogMetadata.performance structure - const performanceMetrics = { - operation: timing.operation, - duration: timing.duration, - startTime: enablePerformanceMetrics ? timing.startTime : undefined, - endTime: enablePerformanceMetrics ? timing.endTime : undefined - }; // Create comprehensive log entry - const logEntry: any = { - correlationId, - request: requestMetadata, - response: responseMetadata, - performance: performanceMetrics, - type: error ? 'custom' : isError ? 'custom' : 'http_request' - }; - - // Sanitize if needed - const finalLogEntry = redactSensitiveData ? sanitizeMetadata(logEntry) : logEntry; // Log based on status code and errors - if (error) { - logger.error('HTTP Request error', error, finalLogEntry); - } else if (isError) { - logger.warn('HTTP Request failed', finalLogEntry); - } else if (isRedirect) { - logger.info('HTTP Request redirected', finalLogEntry); - } else { - logger.http('HTTP Request completed', { - method: c.req.method, - url: c.req.url, - statusCode: status, - responseTime: timing.duration, - userAgent: c.req.header('user-agent'), - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' - }); - } - } - }; -} - -/** - * Error logging middleware for Hono - */ -export function errorLoggingMiddleware(logger?: Logger) { - return async (c: Context, next: Next) => { - const errorLogger = logger || new Logger('error-handler'); - - try { - await next(); - } catch (err) { - const correlationId = c.get('correlationId') || c.req.header('x-correlation-id'); - const url = new URL(c.req.url); - - const requestMetadata = { - method: c.req.method, - url: c.req.url, - path: url.pathname, - userAgent: c.req.header('user-agent'), - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' - }; - - errorLogger.error('Unhandled HTTP error', err instanceof Error ? err : new Error(String(err)), { - correlationId, - request: requestMetadata, - response: { - statusCode: c.res.status - } - }); - - // Re-throw the error so Hono can handle it - throw err; - } - }; -} - -/** - * Create a child logger with request context for Hono - */ -export function createRequestLogger(c: Context, baseLogger: Logger): Logger { - const correlationId = c.get('correlationId') || c.req.header('x-correlation-id'); - const url = new URL(c.req.url); - - return baseLogger.child({ - correlationId, - requestId: correlationId, - method: c.req.method, - path: url.pathname, - userAgent: c.req.header('user-agent'), - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' - }); -} - -/** - * Performance monitoring middleware for specific operations - */ -export function performanceMiddleware(operationName?: string, logger?: Logger) { - return async (c: Context, next: Next) => { - const perfLogger = logger || new Logger('performance'); - const operation = operationName || `${c.req.method} ${new URL(c.req.url).pathname}`; - const timer = createTimer(operation); - - try { - await next(); - - const timing = timer.end(); - perfLogger.info('Operation completed', { - type: 'performance', - performance: timing, - correlationId: c.get('correlationId') - }); - } catch (error) { - const timing = timer.end(); - perfLogger.warn('Operation failed', { - type: 'performance', - performance: timing, - correlationId: c.get('correlationId'), - error: error instanceof Error ? { - name: error.name, - message: error.message, - stack: error.stack - } : { message: String(error) } - }); - throw error; - } - }; -} - -/** - * Security event logging middleware - */ -export function securityMiddleware(logger?: Logger) { - return async (c: Context, next: Next) => { - const secLogger = logger || new Logger('security'); - const url = new URL(c.req.url); - - // Log authentication attempts - const authHeader = c.req.header('authorization'); - if (authHeader) { - secLogger.info('Authentication attempt', { - type: 'security_event', - security: { - type: 'authentication', - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown', - resource: url.pathname, - action: 'access_attempt' - }, - correlationId: c.get('correlationId') - }); - } - - await next(); - - // Log access control - const status = c.res.status; - if (status === 401 || status === 403) { - secLogger.warn('Access denied', { - type: 'security_event', - security: { - type: 'authorization', - result: 'failure', - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown', - resource: url.pathname, - action: c.req.method, - severity: status === 401 ? 'medium' : 'high' - }, - correlationId: c.get('correlationId') - }); - } - }; -} - -/** - * Business event logging middleware for trading operations - */ -export function businessEventMiddleware(logger?: Logger) { - return async (c: Context, next: Next) => { - const bizLogger = logger || new Logger('business'); - const url = new URL(c.req.url); - const path = url.pathname; - - // Check if this is a business-critical endpoint - const businessEndpoints = [ - '/api/orders', - '/api/trades', - '/api/portfolio', - '/api/strategies', - '/api/signals' - ]; - - const isBusinessEndpoint = businessEndpoints.some(endpoint => path.startsWith(endpoint)); - - if (isBusinessEndpoint) { - const timer = createTimer(`business_${c.req.method}_${path}`); - - try { - await next(); - - const timing = timer.end(); - const status = c.res.status; - - bizLogger.info('Business operation completed', { - type: 'business_event', - business: { - type: 'trading_operation', - action: c.req.method, - result: status >= 200 && status < 300 ? 'success' : 'failure' - }, - performance: timing, - correlationId: c.get('correlationId') - }); - } catch (error) { - const timing = timer.end(); - - bizLogger.error('Business operation failed', error, { - type: 'business_event', - business: { - type: 'trading_operation', - action: c.req.method, - result: 'failure' - }, - performance: timing, - correlationId: c.get('correlationId') - }); - throw error; - } - } else { - await next(); - } - }; -} - -/** - * Comprehensive logging middleware that combines all logging features - */ -export function comprehensiveLoggingMiddleware(options: LoggingMiddlewareOptions & { - enableSecurity?: boolean; - enableBusiness?: boolean; - enablePerformance?: boolean; -} = {}) { - const { - enableSecurity = true, - enableBusiness = true, - enablePerformance = true, - ...loggingOptions - } = options; - return async (c: Context, next: Next) => { - const middlewares: Array<(c: Context, next: Next) => Promise> = []; - - // Add security middleware - if (enableSecurity) { - middlewares.push(securityMiddleware(options.logger)); - } - - // Add performance middleware - if (enablePerformance) { - middlewares.push(performanceMiddleware(undefined, options.logger)); - } - - // Add business event middleware - if (enableBusiness) { - middlewares.push(businessEventMiddleware(options.logger)); - } - - // Add main logging middleware - middlewares.push(loggingMiddleware(loggingOptions)); - - // Execute middleware chain - let index = 0; - async function dispatch(i: number): Promise { - if (i >= middlewares.length) { - return next(); - } - - const middleware = middlewares[i]; - return middleware(c, () => dispatch(i + 1)); - } - - return dispatch(0); - }; -} diff --git a/libs/logger/src/types.ts b/libs/logger/src/types.ts index 8f545b3..5d4e298 100644 --- a/libs/logger/src/types.ts +++ b/libs/logger/src/types.ts @@ -1,119 +1,16 @@ /** - * Type definitions for the logger library + * Simplified type definitions for the logger library */ -// Standard log levels -export type LogLevel = 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly'; +// Standard log levels (simplified to pino defaults) +export type LogLevel = 'debug' | 'info' | 'warn' | 'error'; // Context that persists across log calls export interface LogContext { [key: string]: any; - requestId?: string; - sessionId?: string; - userId?: string; - traceId?: string; - spanId?: string; - correlationId?: string; } // Metadata for individual log entries export interface LogMetadata { [key: string]: any; - correlationId?: string; - error?: { - name: string; - message: string; - stack?: string; - } | any;request?: { - method?: string; - url?: string; - path?: string; - statusCode?: number; - responseTime?: number; - userAgent?: string; - ip?: string; - body?: string; - headers?: Record; - contentType?: string; - contentLength?: string; - }; - performance?: { - operation: string; - duration: number; - startTime?: number; - endTime?: number; - }; - business?: { - type: string; - entity?: string; - action?: string; - result?: 'success' | 'failure' | 'partial'; - amount?: number; - symbol?: string; - }; - security?: { - type: 'authentication' | 'authorization' | 'access' | 'vulnerability'; - user?: string; - resource?: string; - action?: string; - result?: 'success' | 'failure'; - ip?: string; - severity?: 'low' | 'medium' | 'high' | 'critical'; - }; - type?: 'http_request' | 'performance' | 'business_event' | 'security_event' | 'system' | 'custom'; -} - -// Logger configuration options -export interface LoggerOptions { - level?: LogLevel; - enableLoki?: boolean; - enableFile?: boolean; - enableConsole?: boolean; -} - -// Loki-specific configuration -export interface LokiTransportOptions { - host: string; - port?: number; - url?: string; - username?: string; - password?: string; - tenantId?: string; - labels?: Record; - batchSize?: number; - interval?: number; - timeout?: number; - json?: boolean; - batching?: boolean; - basicAuth?: string; -} - -// Performance timer utility type -export interface PerformanceTimer { - operation: string; - startTime: number; - end(): { operation: string; duration: number; startTime: number; endTime: number }; -} - -// Log entry structure for Loki -export interface LokiLogEntry { - timestamp: string; - level: LogLevel; - message: string; - service: string; - environment?: string; - version?: string; - metadata?: LogMetadata; -} - -// Structured log format -export interface StructuredLog { - timestamp: string; - level: LogLevel; - service: string; - message: string; - context?: LogContext; - metadata?: LogMetadata; - environment?: string; - version?: string; } diff --git a/libs/logger/src/utils.ts b/libs/logger/src/utils.ts deleted file mode 100644 index 33d5837..0000000 --- a/libs/logger/src/utils.ts +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Utility functions for logging operations - */ - -import type { PerformanceTimer, LogMetadata } from './types'; - -/** - * Create a performance timer to measure operation duration - */ -export function createTimer(operation: string): PerformanceTimer { - const startTime = Date.now(); - - return { - operation, - startTime, - end() { - const endTime = Date.now(); - return { - operation, - duration: endTime - startTime, - startTime, - endTime - }; - } - }; -} - -/** - * Format error for logging - */ -export function formatError(error: Error | any): LogMetadata['error'] { - if (error instanceof Error) { - return { - name: error.name, - message: error.message, - stack: error.stack - }; - } - - if (typeof error === 'object' && error !== null) { - return { - name: error.constructor?.name || 'UnknownError', - message: error.message || error.toString(), - ...error - }; - } - - return { - name: 'UnknownError', - message: String(error) - }; -} - -/** - * Sanitize sensitive data from log metadata - */ -export function sanitizeMetadata(metadata: LogMetadata): LogMetadata { - const sensitiveKeys = [ - 'password', 'token', 'secret', 'key', 'apiKey', 'api_key', - 'authorization', 'auth', 'credential', 'credentials' - ]; - - function sanitizeValue(value: any): any { - if (typeof value === 'string') { - return '[REDACTED]'; - } - if (Array.isArray(value)) { - return value.map(sanitizeValue); - } - if (typeof value === 'object' && value !== null) { - const sanitized: any = {}; - for (const [key, val] of Object.entries(value)) { - const lowerKey = key.toLowerCase(); - if (sensitiveKeys.some(sensitive => lowerKey.includes(sensitive))) { - sanitized[key] = '[REDACTED]'; - } else { - sanitized[key] = sanitizeValue(val); - } - } - return sanitized; - } - return value; - } - - const sanitized: LogMetadata = {}; - for (const [key, value] of Object.entries(metadata)) { - const lowerKey = key.toLowerCase(); - if (sensitiveKeys.some(sensitive => lowerKey.includes(sensitive))) { - sanitized[key] = '[REDACTED]'; - } else { - sanitized[key] = sanitizeValue(value); - } - } - - return sanitized; -} - -/** - * Generate a correlation ID for request tracking - */ -export function generateCorrelationId(): string { - return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; -} - -/** - * Extract relevant HTTP request information for logging - */ -export function extractHttpMetadata(req: any): LogMetadata['request'] { - if (!req) return undefined; - - return { - method: req.method, - url: req.url || req.originalUrl, - userAgent: req.get?.('User-Agent') || req.headers?.['user-agent'], - ip: req.ip || req.connection?.remoteAddress || req.headers?.['x-forwarded-for'] - }; -} - -/** - * Create standardized business event metadata - */ -export function createBusinessEvent( - type: string, - action: string, - options?: { - entity?: string; - result?: 'success' | 'failure' | 'partial'; - amount?: number; - symbol?: string; - [key: string]: any; - } -): LogMetadata { - return { - business: { - type, - action, - ...options - }, - type: 'business_event' - }; -} - -/** - * Create standardized security event metadata - */ -export function createSecurityEvent( - type: 'authentication' | 'authorization' | 'access' | 'vulnerability', - options?: { - user?: string; - resource?: string; - action?: string; - result?: 'success' | 'failure'; - ip?: string; - severity?: 'low' | 'medium' | 'high' | 'critical'; - [key: string]: any; - } -): LogMetadata { - return { - security: { - type, - ...options - }, - type: 'security_event' - }; -} - -/** - * Mask sensitive data in strings - */ -export function maskSensitiveData(str: string): string { - // Credit card numbers - str = str.replace(/\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/g, '****-****-****-****'); - - // Email addresses - str = str.replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, '***@***.***'); - - // API keys (common patterns) - str = str.replace(/\b[A-Za-z0-9]{32,}\b/g, '[API_KEY]'); - - // JWT tokens - str = str.replace(/eyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]*\.[A-Za-z0-9_-]*/g, '[JWT_TOKEN]'); - - return str; -} - -/** - * Calculate log entry size for batching - */ -export function calculateLogSize(entry: any): number { - return JSON.stringify(entry).length; -} - -/** - * Throttle logging to prevent spam - */ -export class LogThrottle { - private counters = new Map(); - private readonly maxLogs: number; - private readonly windowMs: number; - - constructor(maxLogs = 100, windowMs = 60000) { - this.maxLogs = maxLogs; - this.windowMs = windowMs; - } - - shouldLog(key: string): boolean { - const now = Date.now(); - const counter = this.counters.get(key); - - if (!counter || now - counter.lastReset > this.windowMs) { - this.counters.set(key, { count: 1, lastReset: now }); - return true; - } - - if (counter.count >= this.maxLogs) { - return false; - } - - counter.count++; - return true; - } - - reset(key?: string): void { - if (key) { - this.counters.delete(key); - } else { - this.counters.clear(); - } - } -}