From d0e8fd9e3f44a1d1c3ddb154cc81a682707fe04b Mon Sep 17 00:00:00 2001 From: Bojan Kucera Date: Thu, 5 Jun 2025 08:27:06 -0400 Subject: [PATCH] added cache and started fixing data-service --- apps/data-service/src/index.ts | 2 +- apps/data-service/src/providers/unified.ts | 13 +- bun.lock | 16 + docs/cache-library-usage.md | 532 +++++++++++++++++++++ libs/cache/src/decorators/cacheable.ts | 265 ++++++++++ libs/cache/src/index.ts | 114 ++++- libs/cache/src/key-generator.ts | 73 +++ libs/cache/src/providers/hybrid-cache.ts | 261 ++++++++++ libs/cache/src/providers/memory-cache.ts | 245 +++++++++- libs/cache/src/providers/redis-cache.ts | 258 ++++++++-- libs/cache/src/types.ts | 52 +- libs/cache/tsconfig.json | 10 +- libs/logger/src/logger.ts | 12 +- libs/questdb-client/src/client.ts | 5 +- tsconfig.json | 1 - 15 files changed, 1761 insertions(+), 98 deletions(-) create mode 100644 docs/cache-library-usage.md create mode 100644 libs/cache/src/decorators/cacheable.ts create mode 100644 libs/cache/src/key-generator.ts create mode 100644 libs/cache/src/providers/hybrid-cache.ts diff --git a/apps/data-service/src/index.ts b/apps/data-service/src/index.ts index e6d9de1..013ead6 100644 --- a/apps/data-service/src/index.ts +++ b/apps/data-service/src/index.ts @@ -12,7 +12,7 @@ loadEnvVariables(); const app = new Hono(); const logger = createLogger('data-service'); const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002'); - +logger.info(`Data Service starting on port ${PORT}`); // Health check endpoint app.get('/health', (c) => { return c.json({ diff --git a/apps/data-service/src/providers/unified.ts b/apps/data-service/src/providers/unified.ts index ddc4c78..9b04220 100644 --- a/apps/data-service/src/providers/unified.ts +++ b/apps/data-service/src/providers/unified.ts @@ -51,10 +51,7 @@ export class UnifiedDataProvider implements DataProvider { // Initialize QuestDB client if (config.questdb) { - this.questdb = new QuestDBClient({ - host: config.questdb.host, - port: config.questdb.port - }); + this.questdb = new QuestDBClient(); } this.initializeEventSubscriptions(); @@ -82,7 +79,7 @@ export class UnifiedDataProvider implements DataProvider { await this.eventBus.publish('market.data.response', { requestId, symbol, - error: error.message, + error: (error as Error).message, status: 'error' }); } @@ -111,7 +108,7 @@ export class UnifiedDataProvider implements DataProvider { } // Publish live data event - await this.eventBus.publishMarketData(symbol, liveData); + await this.eventBus.publish('marketData', liveData); return liveData; @@ -304,7 +301,7 @@ export class UnifiedDataProvider implements DataProvider { try { const liveData = this.generateSimulatedData(symbol); this.handleLiveData(liveData); - await this.eventBus.publishMarketData(symbol, liveData); + await this.eventBus.publish('marketData', liveData); } catch (error) { logger.error('Error in live data simulation', { symbol, error }); } @@ -353,7 +350,7 @@ export class UnifiedDataProvider implements DataProvider { async close(): Promise { if (this.questdb) { - await this.questdb.close(); + await this.questdb.disconnect(); } this.cache.clear(); this.liveSubscriptions.clear(); diff --git a/bun.lock b/bun.lock index e01550d..d0e59f9 100644 --- a/bun.lock +++ b/bun.lock @@ -22,6 +22,20 @@ "typescript": "^5.4.5", }, }, + "libs/cache": { + "name": "@stock-bot/cache", + "version": "1.0.0", + "dependencies": { + "ioredis": "^5.3.2", + }, + "devDependencies": { + "@types/node": "^20.10.0", + "typescript": "^5.3.0", + }, + "peerDependencies": { + "bun-types": "*", + }, + }, "libs/config": { "name": "@stock-bot/config", "version": "1.0.0", @@ -269,6 +283,8 @@ "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + "@stock-bot/cache": ["@stock-bot/cache@workspace:libs/cache"], + "@stock-bot/config": ["@stock-bot/config@workspace:libs/config"], "@stock-bot/data-frame": ["@stock-bot/data-frame@workspace:libs/data-frame"], diff --git a/docs/cache-library-usage.md b/docs/cache-library-usage.md new file mode 100644 index 0000000..d91c07d --- /dev/null +++ b/docs/cache-library-usage.md @@ -0,0 +1,532 @@ +# Cache Library Usage Guide + +The `@stock-bot/cache` library provides a powerful, flexible caching solution designed specifically for trading bot applications. It supports multiple cache providers including Redis/Dragonfly, in-memory caching, and hybrid caching strategies. + +## Table of Contents + +1. [Installation](#installation) +2. [Quick Start](#quick-start) +3. [Cache Providers](#cache-providers) +4. [Factory Functions](#factory-functions) +5. [Cache Decorators](#cache-decorators) +6. [Trading-Specific Usage](#trading-specific-usage) +7. [Configuration](#configuration) +8. [Monitoring & Metrics](#monitoring--metrics) +9. [Error Handling](#error-handling) +10. [Best Practices](#best-practices) + +## Installation + +The cache library is already included in the monorepo. To use it in your service: + +```json +{ + "dependencies": { + "@stock-bot/cache": "workspace:*" + } +} +``` + +## Quick Start + +### Basic Usage + +```typescript +import { createCache } from '@stock-bot/cache'; + +// Auto-detect best cache type (hybrid if Redis available, otherwise memory) +const cache = createCache('auto'); + +// Basic operations +await cache.set('user:123', { name: 'John', balance: 1000 }, 3600); +const user = await cache.get<{ name: string; balance: number }>('user:123'); +await cache.delete('user:123'); +``` + +### Trading-Optimized Cache + +```typescript +import { createTradingCache } from '@stock-bot/cache'; + +const cache = createTradingCache({ + keyPrefix: 'trading:', + ttl: 300, // 5 minutes default + enableMetrics: true +}); + +// Cache market data +await cache.set('market:AAPL:price', { price: 150.25, timestamp: Date.now() }); +``` + +## Cache Providers + +### 1. Redis Cache (Dragonfly) + +Uses Redis/Dragonfly for distributed caching with persistence and high performance. + +```typescript +import { RedisCache } from '@stock-bot/cache'; + +const redisCache = new RedisCache({ + keyPrefix: 'app:', + ttl: 3600, + enableMetrics: true +}); + +// Automatic connection to Dragonfly using config +await redisCache.set('key', 'value'); +``` + +### 2. Memory Cache + +In-memory caching with LRU eviction and TTL support. + +```typescript +import { MemoryCache } from '@stock-bot/cache'; + +const memoryCache = new MemoryCache({ + maxSize: 1000, // Maximum 1000 entries + ttl: 300, // 5 minutes default TTL + cleanupInterval: 60 // Cleanup every minute +}); +``` + +### 3. Hybrid Cache + +Two-tier caching combining fast memory cache (L1) with persistent Redis cache (L2). + +```typescript +import { HybridCache } from '@stock-bot/cache'; + +const hybridCache = new HybridCache({ + memoryTTL: 60, // L1 cache TTL: 1 minute + redisTTL: 3600, // L2 cache TTL: 1 hour + memoryMaxSize: 500 // L1 cache max entries +}); + +// Data flows: Memory -> Redis -> Database +const data = await hybridCache.get('expensive:calculation'); +``` + +## Factory Functions + +### createCache() + +General-purpose cache factory with auto-detection. + +```typescript +import { createCache } from '@stock-bot/cache'; + +// Auto-detect (recommended) +const cache = createCache('auto'); + +// Specific provider +const redisCache = createCache('redis', { ttl: 1800 }); +const memoryCache = createCache('memory', { maxSize: 2000 }); +const hybridCache = createCache('hybrid'); +``` + +### createTradingCache() + +Optimized for trading operations with sensible defaults. + +```typescript +import { createTradingCache } from '@stock-bot/cache'; + +const tradingCache = createTradingCache({ + keyPrefix: 'trading:', + ttl: 300, // 5 minutes - good for price data + enableMetrics: true +}); +``` + +### createMarketDataCache() + +Specialized for market data with short TTLs. + +```typescript +import { createMarketDataCache } from '@stock-bot/cache'; + +const marketCache = createMarketDataCache({ + priceDataTTL: 30, // 30 seconds for price data + indicatorDataTTL: 300, // 5 minutes for indicators + newsDataTTL: 1800 // 30 minutes for news +}); +``` + +### createStrategyCache() + +For strategy computations and backtesting results. + +```typescript +import { createStrategyCache } from '@stock-bot/cache'; + +const strategyCache = createStrategyCache({ + backtestTTL: 86400, // 24 hours for backtest results + signalTTL: 300, // 5 minutes for signals + optimizationTTL: 3600 // 1 hour for optimization results +}); +``` + +## Cache Decorators + +### @Cacheable + +Automatically cache method results. + +```typescript +import { Cacheable } from '@stock-bot/cache'; + +class MarketDataService { + @Cacheable({ + keyGenerator: (symbol: string) => `price:${symbol}`, + ttl: 60 + }) + async getPrice(symbol: string): Promise { + // Expensive API call + return await this.fetchPriceFromAPI(symbol); + } +} +``` + +### @CacheEvict + +Invalidate cache entries when data changes. + +```typescript +import { CacheEvict } from '@stock-bot/cache'; + +class PortfolioService { + @CacheEvict({ + keyPattern: 'portfolio:*' + }) + async updatePosition(symbol: string, quantity: number): Promise { + // Update database + await this.savePosition(symbol, quantity); + // Cache automatically invalidated + } +} +``` + +### @CachePut + +Always execute method and update cache. + +```typescript +import { CachePut } from '@stock-bot/cache'; + +class StrategyService { + @CachePut({ + keyGenerator: (strategyId: string) => `strategy:${strategyId}:result` + }) + async runStrategy(strategyId: string): Promise { + const result = await this.executeStrategy(strategyId); + // Result always cached after execution + return result; + } +} +``` + +## Trading-Specific Usage + +### Market Data Caching + +```typescript +import { createMarketDataCache, CacheKeyGenerator } from '@stock-bot/cache'; + +const marketCache = createMarketDataCache(); +const keyGen = new CacheKeyGenerator(); + +// Cache price data +const priceKey = keyGen.priceKey('AAPL'); +await marketCache.set(priceKey, { price: 150.25, volume: 1000000 }, 30); + +// Cache technical indicators +const smaKey = keyGen.indicatorKey('AAPL', 'SMA', { period: 20 }); +await marketCache.set(smaKey, 148.50, 300); + +// Cache order book +const orderBookKey = keyGen.orderBookKey('AAPL'); +await marketCache.set(orderBookKey, orderBookData, 5); +``` + +### Strategy Result Caching + +```typescript +import { createStrategyCache, CacheKeyGenerator } from '@stock-bot/cache'; + +const strategyCache = createStrategyCache(); +const keyGen = new CacheKeyGenerator(); + +// Cache backtest results +const backtestKey = keyGen.backtestKey('momentum-strategy', { + startDate: '2024-01-01', + endDate: '2024-12-31', + symbol: 'AAPL' +}); +await strategyCache.set(backtestKey, backtestResults, 86400); + +// Cache trading signals +const signalKey = keyGen.signalKey('AAPL', 'momentum-strategy'); +await strategyCache.set(signalKey, { action: 'BUY', confidence: 0.85 }, 300); +``` + +### Portfolio Data Caching + +```typescript +import { createTradingCache, CacheKeyGenerator } from '@stock-bot/cache'; + +const portfolioCache = createTradingCache(); +const keyGen = new CacheKeyGenerator(); + +// Cache portfolio positions +const positionsKey = keyGen.portfolioKey('user123', 'positions'); +await portfolioCache.set(positionsKey, positions, 300); + +// Cache risk metrics +const riskKey = keyGen.riskKey('user123', 'VaR'); +await portfolioCache.set(riskKey, { var95: 1250.50 }, 3600); +``` + +## Configuration + +Cache configuration is handled through the `@stock-bot/config` package. Key settings: + +```typescript +// Dragonfly/Redis configuration +DRAGONFLY_HOST=localhost +DRAGONFLY_PORT=6379 +DRAGONFLY_PASSWORD=your_password +DRAGONFLY_DATABASE=0 +DRAGONFLY_MAX_RETRIES=3 +DRAGONFLY_RETRY_DELAY=100 +DRAGONFLY_CONNECT_TIMEOUT=10000 +DRAGONFLY_COMMAND_TIMEOUT=5000 + +// TLS settings (optional) +DRAGONFLY_TLS=true +DRAGONFLY_TLS_CERT_FILE=/path/to/cert.pem +DRAGONFLY_TLS_KEY_FILE=/path/to/key.pem +DRAGONFLY_TLS_CA_FILE=/path/to/ca.pem +``` + +## Monitoring & Metrics + +### Cache Statistics + +```typescript +const cache = createTradingCache({ enableMetrics: true }); + +// Get cache statistics +const stats = await cache.getStats(); +console.log(`Hit rate: ${stats.hitRate}%`); +console.log(`Total operations: ${stats.total}`); +console.log(`Uptime: ${stats.uptime} seconds`); +``` + +### Health Checks + +```typescript +const cache = createCache('hybrid'); + +// Check cache health +const isHealthy = await cache.isHealthy(); +if (!isHealthy) { + console.error('Cache is not healthy'); +} + +// Monitor connection status +cache.on('connect', () => console.log('Cache connected')); +cache.on('disconnect', () => console.error('Cache disconnected')); +cache.on('error', (error) => console.error('Cache error:', error)); +``` + +### Metrics Integration + +```typescript +// Export metrics to Prometheus/Grafana +const metrics = await cache.getStats(); + +// Custom metrics tracking +await cache.set('key', 'value', 300, { + tags: { service: 'trading-bot', operation: 'price-update' } +}); +``` + +## Error Handling + +The cache library implements graceful error handling: + +### Automatic Failover + +```typescript +// Hybrid cache automatically falls back to memory if Redis fails +const hybridCache = createCache('hybrid'); + +// If Redis is down, data is served from memory cache +const data = await hybridCache.get('key'); // Never throws, returns null if not found +``` + +### Circuit Breaker Pattern + +```typescript +const cache = createTradingCache({ + maxConsecutiveFailures: 5, // Open circuit after 5 failures + circuitBreakerTimeout: 30000 // Try again after 30 seconds +}); + +try { + await cache.set('key', 'value'); +} catch (error) { + // Handle cache unavailability + console.warn('Cache unavailable, falling back to direct data access'); +} +``` + +### Error Events + +```typescript +cache.on('error', (error) => { + if (error.code === 'CONNECTION_LOST') { + // Handle connection loss + await cache.reconnect(); + } +}); +``` + +## Best Practices + +### 1. Choose the Right Cache Type + +- **Memory Cache**: Fast access, limited by RAM, good for frequently accessed small data +- **Redis Cache**: Persistent, distributed, good for shared data across services +- **Hybrid Cache**: Best of both worlds, use for hot data with fallback + +### 2. Set Appropriate TTLs + +```typescript +// Trading data TTL guidelines +const TTL = { + PRICE_DATA: 30, // 30 seconds - very volatile + INDICATORS: 300, // 5 minutes - calculated values + NEWS: 1800, // 30 minutes - slower changing + BACKTEST_RESULTS: 86400, // 24 hours - expensive calculations + USER_PREFERENCES: 3600 // 1 hour - rarely change during session +}; +``` + +### 3. Use Proper Key Naming + +```typescript +// Good key naming convention +const keyGen = new CacheKeyGenerator(); +const key = keyGen.priceKey('AAPL'); // trading:price:AAPL:2024-01-01 + +// Avoid generic keys +// Bad: "data", "result", "temp" +// Good: "trading:price:AAPL", "strategy:momentum:signals" +``` + +### 4. Implement Cache Warming + +```typescript +// Pre-populate cache with frequently accessed data +async function warmupCache() { + const symbols = ['AAPL', 'GOOGL', 'MSFT']; + const cache = createMarketDataCache(); + + for (const symbol of symbols) { + const price = await fetchPrice(symbol); + await cache.set(keyGen.priceKey(symbol), price, 300); + } +} +``` + +### 5. Monitor Cache Performance + +```typescript +// Regular performance monitoring +setInterval(async () => { + const stats = await cache.getStats(); + if (stats.hitRate < 80) { + console.warn('Low cache hit rate:', stats.hitRate); + } +}, 60000); // Check every minute +``` + +### 6. Handle Cache Invalidation + +```typescript +// Invalidate related cache entries when data changes +class PositionService { + async updatePosition(symbol: string, quantity: number) { + await this.saveToDatabase(symbol, quantity); + + // Invalidate related cache entries + await cache.delete(`portfolio:positions`); + await cache.delete(`portfolio:risk:*`); + await cache.delete(`strategy:signals:${symbol}`); + } +} +``` + +## Advanced Examples + +### Custom Cache Provider + +```typescript +class DatabaseCache implements CacheProvider { + async get(key: string): Promise { + // Implement database-backed cache + } + + async set(key: string, value: T, ttl?: number): Promise { + // Store in database with expiration + } + + // ... implement other methods +} + +// Use with factory +const dbCache = new DatabaseCache(); +``` + +### Batch Operations + +```typescript +// Efficient batch operations +const keys = ['price:AAPL', 'price:GOOGL', 'price:MSFT']; +const values = await cache.mget(keys); + +const updates = new Map([ + ['price:AAPL', 150.25], + ['price:GOOGL', 2800.50], + ['price:MSFT', 350.75] +]); +await cache.mset(updates, 300); +``` + +### Conditional Caching + +```typescript +class SmartCache { + async getOrCompute( + key: string, + computeFn: () => Promise, + shouldCache: (value: T) => boolean = () => true + ): Promise { + let value = await this.cache.get(key); + + if (value === null) { + value = await computeFn(); + if (shouldCache(value)) { + await this.cache.set(key, value, this.defaultTTL); + } + } + + return value; + } +} +``` + +This cache library provides enterprise-grade caching capabilities specifically designed for trading bot applications, with built-in monitoring, error handling, and performance optimization. diff --git a/libs/cache/src/decorators/cacheable.ts b/libs/cache/src/decorators/cacheable.ts new file mode 100644 index 0000000..a0d7a43 --- /dev/null +++ b/libs/cache/src/decorators/cacheable.ts @@ -0,0 +1,265 @@ +import { createLogger } from '@stock-bot/logger'; +import { CacheProvider } from '../types'; +import { CacheKeyGenerator } from '../key-generator'; + +const logger = createLogger('cache-decorator'); + +/** + * Method decorator for automatic caching + */ +export function Cacheable( + cacheProvider: CacheProvider, + options: { + keyGenerator?: (args: any[], target?: any, methodName?: string) => string; + ttl?: number; + skipFirstArg?: boolean; // Skip 'this' if it's the first argument + } = {} +) { + return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + + descriptor.value = async function (...args: any[]) { + try { + // Generate cache key + const key = options.keyGenerator + ? options.keyGenerator(args, target, propertyName) + : generateDefaultKey(target.constructor.name, propertyName, args); + + // Try to get from cache + const cached = await cacheProvider.get(key); + if (cached !== null) { + logger.debug('Method cache hit', { + class: target.constructor.name, + method: propertyName, + key + }); + return cached; + } + + // Execute method and cache result + const result = await originalMethod.apply(this, args); + await cacheProvider.set(key, result, options.ttl); + + logger.debug('Method executed and cached', { + class: target.constructor.name, + method: propertyName, + key + }); + + return result; + } catch (error) { + logger.error('Cache decorator error', { + class: target.constructor.name, + method: propertyName, + error: error instanceof Error ? error.message : String(error) + }); + + // Fallback to original method if caching fails + return await originalMethod.apply(this, args); + } + }; + }; +} + +/** + * Cache invalidation decorator + */ +export function CacheEvict( + cacheProvider: CacheProvider, + options: { + keyGenerator?: (args: any[], target?: any, methodName?: string) => string | string[]; + evictBefore?: boolean; // Evict before method execution + } = {} +) { + return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + + descriptor.value = async function (...args: any[]) { + try { + const keys = options.keyGenerator + ? options.keyGenerator(args, target, propertyName) + : generateDefaultKey(target.constructor.name, propertyName, args); + + const keysArray = Array.isArray(keys) ? keys : [keys]; + + if (options.evictBefore) { + // Evict before method execution + for (const key of keysArray) { + await cacheProvider.del(key); + } + logger.debug('Cache evicted before method execution', { + class: target.constructor.name, + method: propertyName, + keys: keysArray + }); + } + + // Execute method + const result = await originalMethod.apply(this, args); + + if (!options.evictBefore) { + // Evict after method execution + for (const key of keysArray) { + await cacheProvider.del(key); + } + logger.debug('Cache evicted after method execution', { + class: target.constructor.name, + method: propertyName, + keys: keysArray + }); + } + + return result; + } catch (error) { + logger.error('Cache evict decorator error', { + class: target.constructor.name, + method: propertyName, + error: error instanceof Error ? error.message : String(error) + }); + + // Continue with original method execution even if eviction fails + return await originalMethod.apply(this, args); + } + }; + }; +} + +/** + * Cache warming decorator - pre-populate cache with method results + */ +export function CacheWarm( + cacheProvider: CacheProvider, + options: { + keyGenerator?: (args: any[], target?: any, methodName?: string) => string; + ttl?: number; + warmupArgs: any[][]; // Array of argument arrays to warm up + } +) { + return function (target: any, propertyName: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + + // Warmup cache when method is first accessed + let warmed = false; + + descriptor.value = async function (...args: any[]) { + // Perform warmup if not done yet + if (!warmed) { + warmed = true; + setImmediate(async () => { + try { + for (const warmupArgs of options.warmupArgs) { + const key = options.keyGenerator + ? options.keyGenerator(warmupArgs, target, propertyName) + : generateDefaultKey(target.constructor.name, propertyName, warmupArgs); + + // Check if already cached + const exists = await cacheProvider.exists(key); + if (!exists) { + const result = await originalMethod.apply(this, warmupArgs); + await cacheProvider.set(key, result, options.ttl); + } + } + logger.info('Cache warmed up', { + class: target.constructor.name, + method: propertyName, + count: options.warmupArgs.length + }); + } catch (error) { + logger.error('Cache warmup failed', { + class: target.constructor.name, + method: propertyName, + error + }); + } + }); + } + + // Execute normal cacheable logic + const key = options.keyGenerator + ? options.keyGenerator(args, target, propertyName) + : generateDefaultKey(target.constructor.name, propertyName, args); + + const cached = await cacheProvider.get(key); + if (cached !== null) { + return cached; + } + + const result = await originalMethod.apply(this, args); + await cacheProvider.set(key, result, options.ttl); + + return result; + }; + }; +} + +/** + * Trading-specific decorators + */ + +/** + * Cache market data with appropriate TTL + */ +export function CacheMarketData( + cacheProvider: CacheProvider, + ttl: number = 300 // 5 minutes default +) { + return Cacheable(cacheProvider, { + keyGenerator: (args) => { + const [symbol, timeframe, date] = args; + return CacheKeyGenerator.marketData(symbol, timeframe, date); + }, + ttl + }); +} + +/** + * Cache technical indicators + */ +export function CacheIndicator( + cacheProvider: CacheProvider, + ttl: number = 600 // 10 minutes default +) { + return Cacheable(cacheProvider, { + keyGenerator: (args) => { + const [symbol, indicator, period, data] = args; + const dataHash = hashArray(data); + return CacheKeyGenerator.indicator(symbol, indicator, period, dataHash); + }, + ttl + }); +} + +/** + * Cache strategy results + */ +export function CacheStrategy( + cacheProvider: CacheProvider, + ttl: number = 1800 // 30 minutes default +) { + return Cacheable(cacheProvider, { + keyGenerator: (args) => { + const [strategyName, symbol, timeframe] = args; + return CacheKeyGenerator.strategy(strategyName, symbol, timeframe); + }, + ttl + }); +} + +/** + * Helper functions + */ +function generateDefaultKey(className: string, methodName: string, args: any[]): string { + const argsHash = hashArray(args); + return `method:${className}:${methodName}:${argsHash}`; +} + +function hashArray(arr: any[]): string { + const str = JSON.stringify(arr); + let hash = 0; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32-bit integer + } + return Math.abs(hash).toString(36); +} diff --git a/libs/cache/src/index.ts b/libs/cache/src/index.ts index 205e474..6e48be3 100644 --- a/libs/cache/src/index.ts +++ b/libs/cache/src/index.ts @@ -1,26 +1,118 @@ +import { dragonflyConfig } from '@stock-bot/config'; import { RedisCache } from './providers/redis-cache'; import { MemoryCache } from './providers/memory-cache'; -import type { CacheProvider, CacheOptions } from './types'; +import { HybridCache } from './providers/hybrid-cache'; +import type { CacheProvider, CacheOptions, CacheConfig } from './types'; /** - * Factory for creating cache providers. + * Factory for creating cache providers with smart defaults * - * @param type 'redis' | 'memory' + * @param type 'redis' | 'memory' | 'hybrid' | 'auto' * @param options configuration for the cache */ export function createCache( - type: 'redis' | 'memory', + type: 'redis' | 'memory' | 'hybrid' | 'auto' = 'auto', options: CacheOptions = {} ): CacheProvider { - if (type === 'redis') { - return new RedisCache(options); + // Auto-detect best cache type based on environment + if (type === 'auto') { + try { + // Try to use hybrid cache if Redis/Dragonfly is configured + if (dragonflyConfig.DRAGONFLY_HOST) { + type = 'hybrid'; + } else { + type = 'memory'; + } + } catch { + // Fallback to memory if config is not available + type = 'memory'; + } + } + + switch (type) { + case 'redis': + return new RedisCache(options); + case 'memory': + return new MemoryCache(options); + case 'hybrid': + return new HybridCache(options); + default: + throw new Error(`Unknown cache type: ${type}`); } - return new MemoryCache(options); } -export { +/** + * Create a cache instance with trading-optimized defaults + */ +export function createTradingCache(options: Partial = {}): CacheProvider { + const defaultOptions: CacheOptions = { + keyPrefix: 'trading:', + ttl: 3600, // 1 hour default + memoryTTL: 300, // 5 minutes for memory cache + maxMemoryItems: 2000, // More items for trading data + enableMetrics: true, + ...options + }; + + return createCache('auto', defaultOptions); +} + +/** + * Create a cache for market data with appropriate settings + */ +export function createMarketDataCache(options: Partial = {}): CacheProvider { + const defaultOptions: CacheOptions = { + keyPrefix: 'market:', + ttl: 300, // 5 minutes for market data + memoryTTL: 60, // 1 minute in memory + maxMemoryItems: 5000, // Lots of market data + enableMetrics: true, + ...options + }; + + return createCache('auto', defaultOptions); +} + +/** + * Create a cache for indicators with longer TTL + */ +export function createIndicatorCache(options: Partial = {}): CacheProvider { + const defaultOptions: CacheOptions = { + keyPrefix: 'indicators:', + ttl: 1800, // 30 minutes for indicators + memoryTTL: 600, // 10 minutes in memory + maxMemoryItems: 1000, + enableMetrics: true, + ...options + }; + + return createCache('auto', defaultOptions); +} + +// Export types and classes +export type { CacheProvider, CacheOptions, - RedisCache, - MemoryCache -}; + CacheConfig, + CacheStats, + CacheKey, + SerializationOptions +} from './types'; + +export { RedisCache } from './providers/redis-cache'; +export { MemoryCache } from './providers/memory-cache'; +export { HybridCache } from './providers/hybrid-cache'; + +export { CacheKeyGenerator } from './key-generator'; + +export { + Cacheable, + CacheEvict, + CacheWarm, + CacheMarketData, + CacheIndicator, + CacheStrategy +} from './decorators/cacheable'; + +// Default export for convenience +export default createCache; \ No newline at end of file diff --git a/libs/cache/src/key-generator.ts b/libs/cache/src/key-generator.ts new file mode 100644 index 0000000..2c16cbe --- /dev/null +++ b/libs/cache/src/key-generator.ts @@ -0,0 +1,73 @@ +export class CacheKeyGenerator { + /** + * Generate cache key for market data + */ + static marketData(symbol: string, timeframe: string, date?: Date): string { + const dateStr = date ? date.toISOString().split('T')[0] : 'latest'; + return `market:${symbol.toLowerCase()}:${timeframe}:${dateStr}`; + } + + /** + * Generate cache key for technical indicators + */ + static indicator(symbol: string, indicator: string, period: number, dataHash: string): string { + return `indicator:${symbol.toLowerCase()}:${indicator}:${period}:${dataHash}`; + } + + /** + * Generate cache key for backtest results + */ + static backtest(strategyName: string, params: Record): string { + const paramHash = this.hashObject(params); + return `backtest:${strategyName}:${paramHash}`; + } + + /** + * Generate cache key for strategy results + */ + static strategy(strategyName: string, symbol: string, timeframe: string): string { + return `strategy:${strategyName}:${symbol.toLowerCase()}:${timeframe}`; + } + + /** + * Generate cache key for user sessions + */ + static userSession(userId: string): string { + return `session:${userId}`; + } + + /** + * Generate cache key for portfolio data + */ + static portfolio(userId: string, portfolioId: string): string { + return `portfolio:${userId}:${portfolioId}`; + } + + /** + * Generate cache key for real-time prices + */ + static realtimePrice(symbol: string): string { + return `price:realtime:${symbol.toLowerCase()}`; + } + + /** + * Generate cache key for order book data + */ + static orderBook(symbol: string, depth: number = 10): string { + return `orderbook:${symbol.toLowerCase()}:${depth}`; + } + + /** + * Create a simple hash from object for cache keys + */ + private static hashObject(obj: Record): string { + const str = JSON.stringify(obj, Object.keys(obj).sort()); + let hash = 0; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32-bit integer + } + return Math.abs(hash).toString(36); + } +} diff --git a/libs/cache/src/providers/hybrid-cache.ts b/libs/cache/src/providers/hybrid-cache.ts new file mode 100644 index 0000000..e9173fe --- /dev/null +++ b/libs/cache/src/providers/hybrid-cache.ts @@ -0,0 +1,261 @@ +import { createLogger } from '@stock-bot/logger'; +import { CacheProvider, CacheOptions, CacheStats } from '../types'; +import { RedisCache } from './redis-cache'; +import { MemoryCache } from './memory-cache'; + +/** + * Hybrid cache provider that uses memory as L1 cache and Redis as L2 cache + * Provides the best of both worlds: fast memory access and persistent Redis storage + */ +export class HybridCache implements CacheProvider { + private memoryCache: MemoryCache; + private redisCache: RedisCache; + private logger = createLogger('hybrid-cache'); + private enableMetrics: boolean; + private startTime = Date.now(); + + private stats: CacheStats = { + hits: 0, + misses: 0, + errors: 0, + hitRate: 0, + total: 0, + uptime: 0 + }; + + constructor(options: CacheOptions = {}) { + this.enableMetrics = options.enableMetrics ?? true; + + // Create L1 (memory) cache with shorter TTL + this.memoryCache = new MemoryCache({ + ...options, + ttl: options.memoryTTL ?? 300, // 5 minutes for memory + maxMemoryItems: options.maxMemoryItems ?? 1000, + enableMetrics: false // We'll handle metrics at hybrid level + }); + + // Create L2 (Redis) cache with longer TTL + this.redisCache = new RedisCache({ + ...options, + enableMetrics: false // We'll handle metrics at hybrid level + }); + + this.logger.info('Hybrid cache initialized', { + memoryTTL: options.memoryTTL ?? 300, + redisTTL: options.ttl ?? 3600, + maxMemoryItems: options.maxMemoryItems ?? 1000 + }); + } + + private updateStats(hit: boolean, error = false): void { + if (!this.enableMetrics) return; + + if (error) { + this.stats.errors++; + } else if (hit) { + this.stats.hits++; + } else { + this.stats.misses++; + } + + this.stats.total = this.stats.hits + this.stats.misses; + this.stats.hitRate = this.stats.total > 0 ? this.stats.hits / this.stats.total : 0; + this.stats.uptime = Date.now() - this.startTime; + } + + async get(key: string): Promise { + try { + // Try L1 cache first (memory) + const memoryValue = await this.memoryCache.get(key); + if (memoryValue !== null) { + this.updateStats(true); + this.logger.debug('L1 cache hit', { key, hitRate: this.stats.hitRate }); + return memoryValue; + } + + // Try L2 cache (Redis) + const redisValue = await this.redisCache.get(key); + if (redisValue !== null) { + // Populate L1 cache for next access + await this.memoryCache.set(key, redisValue); + this.updateStats(true); + this.logger.debug('L2 cache hit, populating L1', { key, hitRate: this.stats.hitRate }); + return redisValue; + } + + // Complete miss + this.updateStats(false); + this.logger.debug('Cache miss (both L1 and L2)', { key }); + return null; + + } catch (error) { + this.updateStats(false, true); + this.logger.error('Hybrid cache get error', { + key, + error: error instanceof Error ? error.message : String(error) + }); + return null; + } + } + + async set(key: string, value: T, ttl?: number): Promise { + try { + // Set in both caches + const memoryPromise = this.memoryCache.set(key, value, Math.min(ttl ?? 300, 300)); + const redisPromise = this.redisCache.set(key, value, ttl); + + await Promise.allSettled([memoryPromise, redisPromise]); + this.logger.debug('Cache set (both L1 and L2)', { key, ttl }); + + } catch (error) { + this.updateStats(false, true); + this.logger.error('Hybrid cache set error', { + key, + error: error instanceof Error ? error.message : String(error) + }); + } + } + + async del(key: string): Promise { + try { + // Delete from both caches + const memoryPromise = this.memoryCache.del(key); + const redisPromise = this.redisCache.del(key); + + await Promise.allSettled([memoryPromise, redisPromise]); + this.logger.debug('Cache delete (both L1 and L2)', { key }); + + } catch (error) { + this.updateStats(false, true); + this.logger.error('Hybrid cache delete error', { + key, + error: error instanceof Error ? error.message : String(error) + }); + } + } + + async exists(key: string): Promise { + try { + // Check memory first, then Redis + const memoryExists = await this.memoryCache.exists(key); + if (memoryExists) return true; + + return await this.redisCache.exists(key); + + } catch (error) { + this.updateStats(false, true); + this.logger.error('Hybrid cache exists error', { + key, + error: error instanceof Error ? error.message : String(error) + }); + return false; + } + } + + async clear(): Promise { + try { + // Clear both caches + const memoryPromise = this.memoryCache.clear(); + const redisPromise = this.redisCache.clear(); + + await Promise.allSettled([memoryPromise, redisPromise]); + this.logger.info('Cache cleared (both L1 and L2)'); + + } catch (error) { + this.updateStats(false, true); + this.logger.error('Hybrid cache clear error', { + error: error instanceof Error ? error.message : String(error) + }); + } + } + + async health(): Promise { + try { + const memoryHealthy = await this.memoryCache.health(); + const redisHealthy = await this.redisCache.health(); + + // Hybrid cache is healthy if at least one cache is working + const isHealthy = memoryHealthy || redisHealthy; + + this.logger.debug('Hybrid cache health check', { + memory: memoryHealthy, + redis: redisHealthy, + overall: isHealthy + }); + + return isHealthy; + } catch (error) { + this.logger.error('Hybrid cache health check failed', { error }); + return false; + } + } + + getStats(): CacheStats { + return { + ...this.stats, + uptime: Date.now() - this.startTime + }; + } + + /** + * Get detailed stats for both cache layers + */ + getDetailedStats() { + return { + hybrid: this.getStats(), + memory: this.memoryCache.getStats(), + redis: this.redisCache.getStats() + }; + } + + /** + * Warm up the memory cache with frequently accessed keys from Redis + */ + async warmupMemoryCache(keys: string[]): Promise { + this.logger.info('Starting memory cache warmup', { keyCount: keys.length }); + + let warmed = 0; + for (const key of keys) { + try { + const value = await this.redisCache.get(key); + if (value !== null) { + await this.memoryCache.set(key, value); + warmed++; + } + } catch (error) { + this.logger.warn('Failed to warm up key', { key, error }); + } + } + + this.logger.info('Memory cache warmup completed', { + requested: keys.length, + warmed + }); + } + + /** + * Sync memory cache with Redis for specific keys + */ + async syncCaches(keys: string[]): Promise { + for (const key of keys) { + try { + const redisValue = await this.redisCache.get(key); + if (redisValue !== null) { + await this.memoryCache.set(key, redisValue); + } else { + await this.memoryCache.del(key); + } + } catch (error) { + this.logger.warn('Failed to sync key', { key, error }); + } + } + } + + /** + * Close connections for both caches + */ + async disconnect(): Promise { + await this.redisCache.disconnect(); + this.logger.info('Hybrid cache disconnected'); + } +} diff --git a/libs/cache/src/providers/memory-cache.ts b/libs/cache/src/providers/memory-cache.ts index 867f798..e499f8f 100644 --- a/libs/cache/src/providers/memory-cache.ts +++ b/libs/cache/src/providers/memory-cache.ts @@ -1,48 +1,259 @@ -import { CacheProvider } from '../types'; +import { createLogger } from '@stock-bot/logger'; +import { CacheProvider, CacheOptions, CacheStats } from '../types'; + +interface CacheEntry { + value: T; + expiry: number; + accessed: number; +} /** - * Simple in-memory cache provider. + * In-memory cache provider with LRU eviction and comprehensive metrics */ export class MemoryCache implements CacheProvider { - private store = new Map(); + private store = new Map>(); + private logger = createLogger('memory-cache'); private defaultTTL: number; private keyPrefix: string; + private maxItems: number; + private enableMetrics: boolean; + private startTime = Date.now(); + + private stats: CacheStats = { + hits: 0, + misses: 0, + errors: 0, + hitRate: 0, + total: 0, + uptime: 0 + }; - constructor(options: { ttl?: number; keyPrefix?: string } = {}) { - this.defaultTTL = options.ttl ?? 3600; + constructor(options: CacheOptions = {}) { + this.defaultTTL = options.ttl ?? 3600; // 1 hour default this.keyPrefix = options.keyPrefix ?? 'cache:'; + this.maxItems = options.maxMemoryItems ?? 1000; + this.enableMetrics = options.enableMetrics ?? true; + + this.logger.info('Memory cache initialized', { + maxItems: this.maxItems, + defaultTTL: this.defaultTTL, + enableMetrics: this.enableMetrics + }); + + // Cleanup expired entries every 5 minutes + setInterval(() => this.cleanup(), 5 * 60 * 1000); } private getKey(key: string): string { return `${this.keyPrefix}${key}`; } + private updateStats(hit: boolean, error = false): void { + if (!this.enableMetrics) return; + + if (error) { + this.stats.errors++; + } else if (hit) { + this.stats.hits++; + } else { + this.stats.misses++; + } + + this.stats.total = this.stats.hits + this.stats.misses; + this.stats.hitRate = this.stats.total > 0 ? this.stats.hits / this.stats.total : 0; + this.stats.uptime = Date.now() - this.startTime; + } + + private cleanup(): void { + const now = Date.now(); + let cleaned = 0; + + for (const [key, entry] of this.store.entries()) { + if (entry.expiry < now) { + this.store.delete(key); + cleaned++; + } + } + + if (cleaned > 0) { + this.logger.debug('Cleaned expired entries', { + cleaned, + remaining: this.store.size + }); + } + } + + private evictLRU(): void { + if (this.store.size <= this.maxItems) return; + + // Find least recently accessed item + let oldestKey = ''; + let oldestAccess = Date.now(); + + for (const [key, entry] of this.store.entries()) { + if (entry.accessed < oldestAccess) { + oldestAccess = entry.accessed; + oldestKey = key; + } + } + + if (oldestKey) { + this.store.delete(oldestKey); + this.logger.debug('Evicted LRU entry', { key: oldestKey }); + } + } + async get(key: string): Promise { - const fullKey = this.getKey(key); - const entry = this.store.get(fullKey); - if (!entry) return null; - if (entry.expiry < Date.now()) { - this.store.delete(fullKey); + try { + const fullKey = this.getKey(key); + const entry = this.store.get(fullKey); + + if (!entry) { + this.updateStats(false); + this.logger.debug('Cache miss', { key }); + return null; + } + + const now = Date.now(); + if (entry.expiry < now) { + this.store.delete(fullKey); + this.updateStats(false); + this.logger.debug('Cache miss (expired)', { key }); + return null; + } + + // Update access time for LRU + entry.accessed = now; + this.updateStats(true); + this.logger.debug('Cache hit', { key, hitRate: this.stats.hitRate }); + + return entry.value; + } catch (error) { + this.updateStats(false, true); + this.logger.error('Cache get error', { + key, + error: error instanceof Error ? error.message : String(error) + }); return null; } - return entry.value; } async set(key: string, value: T, ttl?: number): Promise { - const fullKey = this.getKey(key); - const expiry = Date.now() + 1000 * (ttl ?? this.defaultTTL); - this.store.set(fullKey, { value, expiry }); + try { + const fullKey = this.getKey(key); + const now = Date.now(); + const expiry = now + 1000 * (ttl ?? this.defaultTTL); + + // Evict if necessary + this.evictLRU(); + + this.store.set(fullKey, { + value, + expiry, + accessed: now + }); + + this.logger.debug('Cache set', { key, ttl: ttl ?? this.defaultTTL }); + } catch (error) { + this.updateStats(false, true); + this.logger.error('Cache set error', { + key, + error: error instanceof Error ? error.message : String(error) + }); + } } async del(key: string): Promise { - this.store.delete(this.getKey(key)); + try { + const fullKey = this.getKey(key); + const deleted = this.store.delete(fullKey); + this.logger.debug('Cache delete', { key, deleted }); + } catch (error) { + this.updateStats(false, true); + this.logger.error('Cache delete error', { + key, + error: error instanceof Error ? error.message : String(error) + }); + } } async exists(key: string): Promise { - return (await this.get(key)) !== null; + try { + const fullKey = this.getKey(key); + const entry = this.store.get(fullKey); + + if (!entry) return false; + + // Check if expired + if (entry.expiry < Date.now()) { + this.store.delete(fullKey); + return false; + } + + return true; + } catch (error) { + this.updateStats(false, true); + this.logger.error('Cache exists error', { + key, + error: error instanceof Error ? error.message : String(error) + }); + return false; + } } async clear(): Promise { - this.store.clear(); + try { + const size = this.store.size; + this.store.clear(); + this.logger.info('Cache cleared', { entriesDeleted: size }); + } catch (error) { + this.updateStats(false, true); + this.logger.error('Cache clear error', { + error: error instanceof Error ? error.message : String(error) + }); + } + } + + async health(): Promise { + try { + // Simple health check - try to set and get a test value + await this.set('__health_check__', 'ok', 1); + const result = await this.get('__health_check__'); + await this.del('__health_check__'); + return result === 'ok'; + } catch (error) { + this.logger.error('Memory cache health check failed', { error }); + return false; + } + } + + getStats(): CacheStats { + return { + ...this.stats, + uptime: Date.now() - this.startTime + }; + } + + /** + * Get additional memory cache specific stats + */ + getMemoryStats() { + return { + ...this.getStats(), + entries: this.store.size, + maxItems: this.maxItems, + memoryUsage: this.estimateMemoryUsage() + }; + } + + private estimateMemoryUsage(): number { + // Rough estimation of memory usage in bytes + let bytes = 0; + for (const [key, entry] of this.store.entries()) { + bytes += key.length * 2; // UTF-16 characters + bytes += JSON.stringify(entry.value).length * 2; + bytes += 24; // Overhead for entry object + } + return bytes; } } diff --git a/libs/cache/src/providers/redis-cache.ts b/libs/cache/src/providers/redis-cache.ts index f5862c4..06bc0c8 100644 --- a/libs/cache/src/providers/redis-cache.ts +++ b/libs/cache/src/providers/redis-cache.ts @@ -1,59 +1,263 @@ -import Redis, { RedisOptions } from 'ioredis'; -import { CacheProvider, CacheOptions } from '../types'; +import Redis from 'ioredis'; +import { createLogger } from '@stock-bot/logger'; +import { dragonflyConfig } from '@stock-bot/config'; +import { CacheProvider, CacheOptions, CacheStats } from '../types'; /** - * Redis-based cache provider implementing CacheProvider interface. + * Redis-based cache provider with comprehensive error handling and metrics */ export class RedisCache implements CacheProvider { private redis: Redis; + private logger = createLogger('redis-cache'); private defaultTTL: number; private keyPrefix: string; + private enableMetrics: boolean; + private isConnected = false; + private startTime = Date.now(); + + private stats: CacheStats = { + hits: 0, + misses: 0, + errors: 0, + hitRate: 0, + total: 0, + uptime: 0 + }; constructor(options: CacheOptions = {}) { - if (options.redisUrl) { - this.redis = new Redis(options.redisUrl); - } else { - this.redis = new Redis(options.redisOptions as RedisOptions); - } - - this.defaultTTL = options.ttl ?? 3600; // default 1 hour + this.defaultTTL = options.ttl ?? 3600; // 1 hour default this.keyPrefix = options.keyPrefix ?? 'cache:'; + this.enableMetrics = options.enableMetrics ?? true; + + const redisConfig = { + host: dragonflyConfig.DRAGONFLY_HOST, + port: dragonflyConfig.DRAGONFLY_PORT, + password: dragonflyConfig.DRAGONFLY_PASSWORD || undefined, + username: dragonflyConfig.DRAGONFLY_USERNAME || undefined, + db: dragonflyConfig.DRAGONFLY_DATABASE, + maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES, + retryDelayOnFailover: dragonflyConfig.DRAGONFLY_RETRY_DELAY, + connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT, + commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT, + keepAlive: dragonflyConfig.DRAGONFLY_ENABLE_KEEPALIVE ? dragonflyConfig.DRAGONFLY_KEEPALIVE_INTERVAL * 1000 : 0, + ...(dragonflyConfig.DRAGONFLY_TLS && { + tls: { + cert: dragonflyConfig.DRAGONFLY_TLS_CERT_FILE || undefined, + key: dragonflyConfig.DRAGONFLY_TLS_KEY_FILE || undefined, + ca: dragonflyConfig.DRAGONFLY_TLS_CA_FILE || undefined, + rejectUnauthorized: !dragonflyConfig.DRAGONFLY_TLS_SKIP_VERIFY, + } + }) + }; + + this.redis = new Redis(redisConfig); + this.setupEventHandlers(); + } + + private setupEventHandlers(): void { + this.redis.on('connect', () => { + this.isConnected = true; + this.logger.info('Redis cache connected', { + host: dragonflyConfig.DRAGONFLY_HOST, + port: dragonflyConfig.DRAGONFLY_PORT, + db: dragonflyConfig.DRAGONFLY_DATABASE + }); + }); + + this.redis.on('ready', () => { + this.logger.info('Redis cache ready for commands'); + }); + + this.redis.on('error', (error) => { + this.isConnected = false; + this.stats.errors++; + this.logger.error('Redis cache connection error', { error: error.message }); + }); + + this.redis.on('close', () => { + this.isConnected = false; + this.logger.warn('Redis cache connection closed'); + }); + + this.redis.on('reconnecting', () => { + this.logger.info('Redis cache reconnecting...'); + }); } private getKey(key: string): string { return `${this.keyPrefix}${key}`; } - async get(key: string): Promise { - const fullKey = this.getKey(key); - const val = await this.redis.get(fullKey); - if (val === null) return null; + private updateStats(hit: boolean, error = false): void { + if (!this.enableMetrics) return; + + if (error) { + this.stats.errors++; + } else if (hit) { + this.stats.hits++; + } else { + this.stats.misses++; + } + + this.stats.total = this.stats.hits + this.stats.misses; + this.stats.hitRate = this.stats.total > 0 ? this.stats.hits / this.stats.total : 0; + this.stats.uptime = Date.now() - this.startTime; + } + + private async safeExecute( + operation: () => Promise, + fallback: T, + operationName: string + ): Promise { + if (!this.isConnected) { + this.logger.warn(`Redis not connected for ${operationName}, using fallback`); + this.updateStats(false, true); + return fallback; + } + try { - return JSON.parse(val) as T; - } catch { - return (val as unknown) as T; + return await operation(); + } catch (error) { + this.logger.error(`Redis ${operationName} failed`, { + error: error instanceof Error ? error.message : String(error) + }); + this.updateStats(false, true); + return fallback; } } + async get(key: string): Promise { + return this.safeExecute( + async () => { + const fullKey = this.getKey(key); + const value = await this.redis.get(fullKey); + + if (value === null) { + this.updateStats(false); + this.logger.debug('Cache miss', { key }); + return null; + } + + this.updateStats(true); + this.logger.debug('Cache hit', { key, hitRate: this.stats.hitRate }); + + try { + return JSON.parse(value) as T; + } catch { + // Return as-is if not valid JSON + return value as unknown as T; + } + }, + null, + 'get' + ); + } + async set(key: string, value: T, ttl?: number): Promise { - const fullKey = this.getKey(key); - const str = typeof value === 'string' ? (value as unknown as string) : JSON.stringify(value); - const expiry = ttl ?? this.defaultTTL; - await this.redis.set(fullKey, str, 'EX', expiry); + await this.safeExecute( + async () => { + const fullKey = this.getKey(key); + const serialized = typeof value === 'string' ? value : JSON.stringify(value); + const expiry = ttl ?? this.defaultTTL; + + await this.redis.setex(fullKey, expiry, serialized); + this.logger.debug('Cache set', { key, ttl: expiry }); + }, + undefined, + 'set' + ); } async del(key: string): Promise { - await this.redis.del(this.getKey(key)); + await this.safeExecute( + async () => { + const fullKey = this.getKey(key); + await this.redis.del(fullKey); + this.logger.debug('Cache delete', { key }); + }, + undefined, + 'del' + ); } async exists(key: string): Promise { - const exists = await this.redis.exists(this.getKey(key)); - return exists === 1; + return this.safeExecute( + async () => { + const fullKey = this.getKey(key); + const result = await this.redis.exists(fullKey); + return result === 1; + }, + false, + 'exists' + ); } async clear(): Promise { - const pattern = `${this.keyPrefix}*`; - const keys = await this.redis.keys(pattern); - if (keys.length) await this.redis.del(...keys); + await this.safeExecute( + async () => { + const pattern = `${this.keyPrefix}*`; + const keys = await this.redis.keys(pattern); + if (keys.length > 0) { + await this.redis.del(...keys); + this.logger.info('Cache cleared', { keysDeleted: keys.length }); + } + }, + undefined, + 'clear' + ); + } + + async health(): Promise { + try { + const pong = await this.redis.ping(); + return pong === 'PONG' && this.isConnected; + } catch (error) { + this.logger.error('Redis health check failed', { error }); + return false; + } + } + + getStats(): CacheStats { + return { + ...this.stats, + uptime: Date.now() - this.startTime + }; + } + + /** + * Trading-specific convenience methods + */ + async cacheMarketData(symbol: string, timeframe: string, data: any[], ttl = 300): Promise { + const key = `market:${symbol}:${timeframe}`; + await this.set(key, data, ttl); + } + + async getMarketData(symbol: string, timeframe: string): Promise { + const key = `market:${symbol}:${timeframe}`; + return this.get(key); + } + + async cacheIndicator( + symbol: string, + indicator: string, + period: number, + data: number[], + ttl = 600 + ): Promise { + const key = `indicator:${symbol}:${indicator}:${period}`; + await this.set(key, data, ttl); + } + + async getIndicator(symbol: string, indicator: string, period: number): Promise { + const key = `indicator:${symbol}:${indicator}:${period}`; + return this.get(key); + } + + /** + * Close the Redis connection + */ + async disconnect(): Promise { + await this.redis.quit(); + this.logger.info('Redis cache disconnected'); } } diff --git a/libs/cache/src/types.ts b/libs/cache/src/types.ts index 5613474..2551578 100644 --- a/libs/cache/src/types.ts +++ b/libs/cache/src/types.ts @@ -1,34 +1,42 @@ -import type { RedisOptions as IORedisOptions } from 'ioredis'; - -/** - * Interface for a generic cache provider. - */ export interface CacheProvider { get(key: string): Promise; set(key: string, value: T, ttl?: number): Promise; del(key: string): Promise; exists(key: string): Promise; clear(): Promise; + getStats(): CacheStats; + health(): Promise; } -/** - * Options for configuring the cache provider. - */ export interface CacheOptions { - /** - * Full Redis connection string (e.g., redis://localhost:6379) - */ - redisUrl?: string; - /** - * Raw ioredis connection options if not using a URL. - */ - redisOptions?: IORedisOptions; - /** - * Default time-to-live for cache entries (in seconds). - */ ttl?: number; - /** - * Prefix to use for all cache keys. - */ keyPrefix?: string; + enableMetrics?: boolean; + maxMemoryItems?: number; + memoryTTL?: number; +} + +export interface CacheStats { + hits: number; + misses: number; + errors: number; + hitRate: number; + total: number; + uptime: number; +} + +export interface CacheConfig { + type: 'redis' | 'memory' | 'hybrid'; + keyPrefix?: string; + defaultTTL?: number; + maxMemoryItems?: number; + enableMetrics?: boolean; + compression?: boolean; +} + +export type CacheKey = string | (() => string); + +export interface SerializationOptions { + compress?: boolean; + binary?: boolean; } diff --git a/libs/cache/tsconfig.json b/libs/cache/tsconfig.json index 2e0bad5..9af0434 100644 --- a/libs/cache/tsconfig.json +++ b/libs/cache/tsconfig.json @@ -5,6 +5,12 @@ "rootDir": "./src", "declaration": true }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"] + "include": [ + "src/**/*" + ], + "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"], + "references": [ + { "path": "../logger" }, + { "path": "../config" } + ] } diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 11b687e..1157d10 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -163,14 +163,14 @@ function buildLogger(serviceName: string, options?: { }); const loggerConfig: pino.LoggerOptions = { - level: PINO_LEVELS[level] ? level : 'info', - customLevels: PINO_LEVELS, + // level: PINO_LEVELS[level] ? level : 'info', + // customLevels: PINO_LEVELS, useOnlyCustomLevels: false, timestamp: () => `,"timestamp":"${new Date().toISOString()}"`, - formatters: { - level: (label: string) => ({ level: label }), - bindings: () => ({}) - }, + // formatters: { + // level: (label: string) => ({ level: label }), + // bindings: () => ({}) + // }, base: { service: serviceName, environment: loggingConfig.LOG_ENVIRONMENT, diff --git a/libs/questdb-client/src/client.ts b/libs/questdb-client/src/client.ts index 13710cd..2b7ef32 100644 --- a/libs/questdb-client/src/client.ts +++ b/libs/questdb-client/src/client.ts @@ -1,6 +1,6 @@ import { Pool } from 'pg'; import { questdbConfig } from '@stock-bot/config'; -import { Logger } from '@stock-bot/logger'; +import { createLogger, Logger } from '@stock-bot/logger'; import type { QuestDBClientConfig, QuestDBConnectionOptions, @@ -24,7 +24,7 @@ export class QuestDBClient { private pgPool: Pool | null = null; private readonly config: QuestDBClientConfig; private readonly options: QuestDBConnectionOptions; - private readonly logger: Logger; + private readonly logger = createLogger('QuestDBClient'); private readonly healthMonitor: QuestDBHealthMonitor; private readonly influxWriter: QuestDBInfluxWriter; private readonly schemaManager: QuestDBSchemaManager; @@ -43,7 +43,6 @@ export class QuestDBClient { ...options }; - this.logger = new Logger('QuestDBClient'); this.healthMonitor = new QuestDBHealthMonitor(this); this.influxWriter = new QuestDBInfluxWriter(this); this.schemaManager = new QuestDBSchemaManager(this); diff --git a/tsconfig.json b/tsconfig.json index 606b8fd..aae2e74 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -48,7 +48,6 @@ { "path": "./libs/questdb-client" }, { "path": "./libs/types" }, { "path": "./libs/cache" }, - { "path": "./libs/logger" }, { "path": "./libs/utils" }, { "path": "./libs/event-bus" }, { "path": "./libs/data-frame" },