diff --git a/.gitignore b/.gitignore index 0d59aaf..4702ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -99,9 +99,12 @@ ehthumbs.db Thumbs.db # Trading bot specific -data/ -backtest-results/ -logs/ +.data/ +.backtest-results/ +.logs/ +.old/ +.mongo/ +.chat/ *.db *.sqlite *.sqlite3 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index 8838558..0000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,703 +0,0 @@ -# ๐Ÿ—๏ธ Stock Bot Trading System - Architecture Documentation - -## ๐Ÿ“‹ Table of Contents -- [System Overview](#system-overview) -- [Current Architecture](#current-architecture) -- [Service Breakdown](#service-breakdown) -- [Data Flow](#data-flow) -- [Technology Stack](#technology-stack) -- [Future Architecture](#future-architecture) -- [Improvement Recommendations](#improvement-recommendations) -- [Deployment Architecture](#deployment-architecture) -- [Security Architecture](#security-architecture) -- [Monitoring & Observability](#monitoring--observability) - ---- - -## ๐ŸŽฏ System Overview - -The Stock Bot Trading System is a **microservice-based**, **event-driven** trading platform built for **real-time market analysis**, **strategy execution**, and **risk management**. The system follows a **service-oriented architecture (SOA)** with **clear separation of concerns** and **horizontal scalability**. - -### Core Principles -- **Microservices Architecture**: Independent, deployable services -- **Event-Driven Communication**: WebSocket and Redis pub/sub -- **Real-Time Processing**: Sub-second latency requirements -- **Scalable Design**: Horizontal scaling capabilities -- **Fault Tolerance**: Circuit breakers and graceful degradation -- **Type Safety**: Full TypeScript implementation - ---- - -## ๐Ÿ—๏ธ Current Architecture - -```mermaid -graph TB - subgraph "Frontend Layer" - UI[Angular Trading Dashboard] - end - - subgraph "API Gateway Layer" - GW[API Gateway - Future] - end - - subgraph "Core Services" - MDG[Market Data Gateway
Port 3001] - RG[Risk Guardian
Port 3002] - SO[Strategy Orchestrator
Port 4001] - end - - subgraph "Data Layer" - Redis[(Redis Cache)] - PG[(PostgreSQL)] - QDB[(QuestDB)] - Mongo[(MongoDB)] - end - - subgraph "External APIs" - Alpha[Alpha Vantage] - IEX[IEX Cloud] - Yahoo[Yahoo Finance] - end - - UI -->|WebSocket/HTTP| MDG - UI -->|WebSocket/HTTP| RG - UI -->|WebSocket/HTTP| SO - - MDG --> Redis - MDG --> QDB - MDG -->|Fetch Data| Alpha - MDG -->|Fetch Data| IEX - MDG -->|Fetch Data| Yahoo - - RG --> Redis - RG --> PG - - SO --> Redis - SO --> PG - SO --> Mongo -``` - ---- - -## ๐Ÿ”ง Service Breakdown - -### **1. Interface Services** - -#### **Trading Dashboard** (`apps/interface-services/trading-dashboard`) -- **Framework**: Angular 20 + Angular Material + Tailwind CSS -- **Port**: 4200 (development) -- **Purpose**: Real-time trading interface and strategy management -- **Key Features**: - - Real-time market data visualization - - Strategy creation and backtesting UI - - Risk management dashboard - - Portfolio monitoring - - WebSocket integration for live updates - -**Current Structure:** -``` -trading-dashboard/ -โ”œโ”€โ”€ src/ -โ”‚ โ”œโ”€โ”€ app/ -โ”‚ โ”‚ โ”œโ”€โ”€ components/ # Reusable UI components -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ sidebar/ # Navigation sidebar -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ notifications/ # Alert system -โ”‚ โ”‚ โ”œโ”€โ”€ pages/ # Route-based pages -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ dashboard/ # Main trading dashboard -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ market-data/ # Market data visualization -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ portfolio/ # Portfolio management -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ strategies/ # Strategy management -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ risk-management/ # Risk controls -โ”‚ โ”‚ โ”œโ”€โ”€ services/ # Angular services -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ api.service.ts # HTTP API communication -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ websocket.service.ts # WebSocket management -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ strategy.service.ts # Strategy operations -โ”‚ โ”‚ โ””โ”€โ”€ shared/ # Shared utilities -โ”‚ โ””โ”€โ”€ styles.css # Global styles -``` - -### **2. Core Services** - -#### **Market Data Gateway** (`apps/core-services/market-data-gateway`) -- **Framework**: Hono + Bun -- **Port**: 3001 -- **Purpose**: Market data aggregation and real-time distribution -- **Database**: QuestDB (time-series), Redis (caching) - -**Responsibilities:** -- Aggregate data from multiple market data providers -- Real-time WebSocket streaming to clients -- Historical data storage and retrieval -- Rate limiting and API management -- Data normalization and validation - -**API Endpoints:** -``` -GET /health # Health check -GET /api/market-data/:symbol # Get latest market data -GET /api/historical/:symbol # Get historical data -WS /ws # WebSocket for real-time data -``` - -#### **Risk Guardian** (`apps/core-services/risk-guardian`) -- **Framework**: Hono + Bun -- **Port**: 3002 -- **Purpose**: Real-time risk monitoring and controls -- **Database**: PostgreSQL (persistent), Redis (real-time) - -**Responsibilities:** -- Position size monitoring -- Daily loss tracking -- Portfolio risk assessment -- Volatility monitoring -- Real-time risk alerts -- Risk threshold management - -**API Endpoints:** -``` -GET /health # Health check -GET /api/risk/thresholds # Get risk thresholds -PUT /api/risk/thresholds # Update risk thresholds -POST /api/risk/evaluate # Evaluate position risk -GET /api/risk/history # Risk evaluation history -WS /ws # WebSocket for risk alerts -``` - -#### **Strategy Orchestrator** (`apps/intelligence-services/strategy-orchestrator`) -- **Framework**: Hono + Bun -- **Port**: 4001 -- **Purpose**: Strategy lifecycle management and execution -- **Database**: MongoDB (strategies), PostgreSQL (trades), Redis (signals) - -**Responsibilities:** -- Strategy creation and management -- Backtesting engine (vectorized & event-based) -- Real-time strategy execution -- Signal generation and broadcasting -- Performance analytics -- Strategy optimization - -**Current Structure:** -``` -strategy-orchestrator/ -โ”œโ”€โ”€ src/ -โ”‚ โ”œโ”€โ”€ core/ -โ”‚ โ”‚ โ”œโ”€โ”€ backtesting/ -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ BacktestEngine.ts # Main backtesting engine -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ BacktestService.ts # Backtesting service layer -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ MarketDataFeed.ts # Historical data provider -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ PerformanceAnalytics.ts # Performance metrics -โ”‚ โ”‚ โ”œโ”€โ”€ execution/ -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ StrategyExecutionService.ts # Real-time execution -โ”‚ โ”‚ โ”œโ”€โ”€ strategies/ -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ Strategy.ts # Base strategy interface -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ StrategyRegistry.ts # Strategy management -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ BaseStrategy.ts # Abstract base class -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ VectorizedStrategy.ts # Vectorized base class -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ MovingAverageCrossover.ts # MA strategy -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ MeanReversionStrategy.ts # Mean reversion -โ”‚ โ”‚ โ””โ”€โ”€ indicators/ -โ”‚ โ”‚ โ””โ”€โ”€ TechnicalIndicators.ts # Technical analysis -โ”‚ โ”œโ”€โ”€ controllers/ -โ”‚ โ”‚ โ””โ”€โ”€ StrategyController.ts # API endpoints -โ”‚ โ””โ”€โ”€ index.ts # Main entry point -``` - -**API Endpoints:** -``` -GET /health # Health check -GET /api/strategies # List strategies -POST /api/strategies # Create strategy -PUT /api/strategies/:id # Update strategy -POST /api/strategies/:id/:action # Start/stop/pause strategy -GET /api/strategies/:id/signals # Get strategy signals -POST /api/strategies/:id/backtest # Run backtest -GET /api/strategies/:id/performance # Get performance metrics -WS /ws # WebSocket for strategy updates -``` - -### **3. Shared Packages** - -#### **Shared Types** (`packages/types`) -```typescript -export interface MarketData { - symbol: string; - price: number; - volume: number; - timestamp: Date; - bid: number; - ask: number; -} - -export interface Strategy { - id: string; - name: string; - symbols: string[]; - parameters: Record; - status: 'ACTIVE' | 'PAUSED' | 'STOPPED'; -} - -export interface BacktestResult { - totalReturn: number; - sharpeRatio: number; - maxDrawdown: number; - winRate: number; - totalTrades: number; -} -``` - -#### **Configuration** (`packages/config`) -```typescript -export const config = { - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379') - }, - database: { - postgres: process.env.POSTGRES_URL, - questdb: process.env.QUESTDB_URL, - mongodb: process.env.MONGODB_URL - }, - marketData: { - alphaVantageKey: process.env.ALPHA_VANTAGE_KEY, - iexKey: process.env.IEX_KEY - } -}; -``` - ---- - -## ๐Ÿ”„ Data Flow - -### **Real-Time Market Data Flow** -```mermaid -sequenceDiagram - participant EXT as External APIs - participant MDG as Market Data Gateway - participant Redis as Redis Cache - participant QDB as QuestDB - participant UI as Trading Dashboard - - EXT->>MDG: Market data feed - MDG->>Redis: Cache latest prices - MDG->>QDB: Store historical data - MDG->>UI: WebSocket broadcast - UI->>UI: Update charts/tables -``` - -### **Strategy Execution Flow** -```mermaid -sequenceDiagram - participant UI as Trading Dashboard - participant SO as Strategy Orchestrator - participant MDG as Market Data Gateway - participant RG as Risk Guardian - participant Redis as Redis - - UI->>SO: Start strategy - SO->>MDG: Subscribe to market data - MDG->>SO: Real-time price updates - SO->>SO: Generate trading signals - SO->>RG: Risk evaluation - RG->>SO: Risk approval/rejection - SO->>Redis: Store signals - SO->>UI: WebSocket signal broadcast -``` - ---- - -## ๐Ÿ’ป Technology Stack - -### **Backend** -- **Runtime**: Bun (ultra-fast JavaScript runtime) -- **Web Framework**: Hono (lightweight, fast web framework) -- **Language**: TypeScript (type safety) -- **Build Tool**: Turbo (monorepo management) - -### **Frontend** -- **Framework**: Angular 20 (latest stable) -- **UI Library**: Angular Material + Tailwind CSS -- **State Management**: Angular Signals (reactive programming) -- **WebSocket**: Native WebSocket API - -### **Databases** -- **Time-Series**: QuestDB (market data storage) -- **Relational**: PostgreSQL (structured data) -- **Document**: MongoDB (strategy configurations) -- **Cache/Pub-Sub**: Redis (real-time data) - -### **Infrastructure** -- **Containerization**: Docker + Docker Compose -- **Process Management**: PM2 (production) -- **Monitoring**: Built-in health checks -- **Development**: Hot reload, TypeScript compilation - ---- - -## ๐Ÿš€ Future Architecture - -### **Phase 1: Enhanced Microservices (Q2 2025)** -```mermaid -graph TB - subgraph "API Gateway Layer" - GW[Kong/Envoy API Gateway] - LB[Load Balancer] - end - - subgraph "Authentication" - AUTH[Auth Service
JWT + OAuth] - end - - subgraph "Core Services" - MDG[Market Data Gateway] - RG[Risk Guardian] - SO[Strategy Orchestrator] - OE[Order Execution Engine] - NS[Notification Service] - end - - subgraph "Analytics Services" - BA[Backtest Analytics] - PA[Performance Analytics] - ML[ML Prediction Service] - end - - subgraph "Message Queue" - NATS[NATS/Apache Kafka] - end -``` - -### **Phase 2: Machine Learning Integration (Q3 2025)** -- **ML Pipeline**: Python-based ML services -- **Feature Engineering**: Real-time feature computation -- **Model Training**: Automated model retraining -- **Prediction API**: Real-time predictions - -### **Phase 3: Multi-Asset Support (Q4 2025)** -- **Crypto Trading**: Binance, Coinbase integration -- **Forex Trading**: OANDA, FXCM integration -- **Options Trading**: Interactive Brokers integration -- **Futures Trading**: CME, ICE integration - ---- - -## ๐Ÿ“ˆ Improvement Recommendations - -### **1. High Priority Improvements** - -#### **API Gateway Implementation** -```typescript -// Implement Kong or Envoy for: -- Rate limiting per service -- Authentication/authorization -- Request/response transformation -- Circuit breaker patterns -- Load balancing -``` - -#### **Enhanced Error Handling** -```typescript -// Implement structured error handling: -interface ServiceError { - code: string; - message: string; - service: string; - timestamp: Date; - correlationId: string; -} -``` - -#### **Comprehensive Logging** -```typescript -// Implement structured logging: -interface LogEntry { - level: 'debug' | 'info' | 'warn' | 'error'; - service: string; - message: string; - metadata: Record; - timestamp: Date; - correlationId: string; -} -``` - -### **2. Medium Priority Improvements** - -#### **Database Optimization** -```sql --- QuestDB optimizations for market data: -CREATE TABLE market_data ( - symbol SYMBOL, - timestamp TIMESTAMP, - price DOUBLE, - volume DOUBLE, - bid DOUBLE, - ask DOUBLE -) timestamp(timestamp) PARTITION BY DAY; - --- Add indexes for fast queries: -CREATE INDEX idx_symbol_timestamp ON market_data (symbol, timestamp); -``` - -#### **Caching Strategy** -```typescript -// Implement multi-layer caching: -interface CacheStrategy { - L1: Map; // In-memory cache - L2: Redis; // Distributed cache - L3: Database; // Persistent storage -} -``` - -#### **WebSocket Optimization** -```typescript -// Implement WebSocket connection pooling: -interface WSConnectionPool { - connections: Map; - balancer: RoundRobinBalancer; - heartbeat: HeartbeatManager; -} -``` - -### **3. Low Priority Improvements** - -#### **Code Quality** -- Implement comprehensive unit tests (>90% coverage) -- Add integration tests for all services -- Implement E2E tests for critical user flows -- Add performance benchmarks - -#### **Documentation** -- API documentation with OpenAPI/Swagger -- Developer onboarding guide -- Deployment runbooks -- Architecture decision records (ADRs) - ---- - -## ๐Ÿ—๏ธ Deployment Architecture - -### **Development Environment** -```yaml -# docker-compose.dev.yml -version: '3.8' -services: - # Databases - postgres: - image: postgres:15 - ports: ["5432:5432"] - - redis: - image: redis:7-alpine - ports: ["6379:6379"] - - questdb: - image: questdb/questdb:latest - ports: ["9000:9000", "8812:8812"] - - mongodb: - image: mongo:6 - ports: ["27017:27017"] - - # Services - market-data-gateway: - build: ./apps/core-services/market-data-gateway - ports: ["3001:3001"] - depends_on: [redis, questdb] - - risk-guardian: - build: ./apps/core-services/risk-guardian - ports: ["3002:3002"] - depends_on: [postgres, redis] - - strategy-orchestrator: - build: ./apps/intelligence-services/strategy-orchestrator - ports: ["4001:4001"] - depends_on: [mongodb, postgres, redis] - - trading-dashboard: - build: ./apps/interface-services/trading-dashboard - ports: ["4200:4200"] -``` - -### **Production Environment** -```yaml -# kubernetes deployment example -apiVersion: apps/v1 -kind: Deployment -metadata: - name: market-data-gateway -spec: - replicas: 3 - selector: - matchLabels: - app: market-data-gateway - template: - metadata: - labels: - app: market-data-gateway - spec: - containers: - - name: market-data-gateway - image: stockbot/market-data-gateway:latest - ports: - - containerPort: 3001 - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "512Mi" - cpu: "500m" -``` - ---- - -## ๐Ÿ”’ Security Architecture - -### **Authentication & Authorization** -```typescript -// JWT-based authentication -interface AuthToken { - userId: string; - roles: string[]; - permissions: string[]; - expiresAt: Date; -} - -// Role-based access control -enum UserRole { - TRADER = 'TRADER', - ADMIN = 'ADMIN', - VIEWER = 'VIEWER' -} - -enum Permission { - READ_MARKET_DATA = 'READ_MARKET_DATA', - EXECUTE_TRADES = 'EXECUTE_TRADES', - MANAGE_STRATEGIES = 'MANAGE_STRATEGIES', - CONFIGURE_RISK = 'CONFIGURE_RISK' -} -``` - -### **API Security** -```typescript -// Rate limiting configuration -interface RateLimit { - windowMs: number; // 15 minutes - maxRequests: number; // 100 requests per window - skipIf: (req: Request) => boolean; -} - -// Input validation -interface ApiValidation { - schema: JSONSchema; - sanitization: SanitizationRules; - authentication: AuthenticationMiddleware; -} -``` - -### **Data Security** -- **Encryption at Rest**: AES-256 for sensitive data -- **Encryption in Transit**: TLS 1.3 for all communications -- **Secrets Management**: Kubernetes secrets or HashiCorp Vault -- **Network Security**: VPC, security groups, firewalls - ---- - -## ๐Ÿ“Š Monitoring & Observability - -### **Metrics Collection** -```typescript -interface ServiceMetrics { - // Performance metrics - requestLatency: Histogram; - requestRate: Counter; - errorRate: Counter; - - // Business metrics - tradesExecuted: Counter; - strategiesActive: Gauge; - portfolioValue: Gauge; - - // System metrics - memoryUsage: Gauge; - cpuUsage: Gauge; - dbConnections: Gauge; -} -``` - -### **Health Checks** -```typescript -interface HealthCheck { - service: string; - status: 'healthy' | 'degraded' | 'unhealthy'; - checks: { - database: boolean; - redis: boolean; - externalApis: boolean; - webSocket: boolean; - }; - uptime: number; - version: string; -} -``` - -### **Alerting Rules** -```yaml -# Prometheus alerting rules -groups: -- name: stockbot - rules: - - alert: HighErrorRate - expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1 - for: 2m - - - alert: HighLatency - expr: http_request_duration_seconds{quantile="0.95"} > 1 - for: 2m - - - alert: ServiceDown - expr: up{job="stockbot"} == 0 - for: 30s -``` - ---- - -## ๐Ÿ“‹ Migration Plan - -### **Phase 1: Current โ†’ Enhanced (1-2 months)** -1. **Week 1-2**: Implement API Gateway and authentication -2. **Week 3-4**: Add comprehensive logging and monitoring -3. **Week 5-6**: Enhance error handling and resilience -4. **Week 7-8**: Performance optimization and testing - -### **Phase 2: Enhanced โ†’ ML-Ready (2-3 months)** -1. **Month 1**: Implement ML pipeline infrastructure -2. **Month 2**: Develop feature engineering services -3. **Month 3**: Integrate ML predictions into strategies - -### **Phase 3: ML-Ready โ†’ Multi-Asset (3-4 months)** -1. **Month 1**: Abstract market data interfaces -2. **Month 2**: Implement crypto trading support -3. **Month 3**: Add forex and options trading -4. **Month 4**: Performance optimization and testing - ---- - -## ๐ŸŽฏ Success Metrics - -### **Technical KPIs** -- **Latency**: < 100ms for market data updates -- **Throughput**: > 10,000 requests/second -- **Availability**: 99.9% uptime -- **Error Rate**: < 0.1% of requests - -### **Business KPIs** -- **Strategy Performance**: Sharpe ratio > 1.5 -- **Risk Management**: Max drawdown < 5% -- **Execution Quality**: Slippage < 0.01% -- **System Adoption**: > 90% user satisfaction - ---- - -This architecture document serves as a living blueprint for the Stock Bot Trading System, providing clear guidance for current development and future scaling decisions. \ No newline at end of file diff --git a/CONTEXT.md b/CONTEXT.md deleted file mode 100644 index 52b8c6b..0000000 --- a/CONTEXT.md +++ /dev/null @@ -1,229 +0,0 @@ -# ๐Ÿ“‹ Stock Bot Trading System - Architecture Context - -## ๐Ÿ—๏ธ System Overview - -A comprehensive, microservice-based trading bot system built with **Bun**, **TypeScript**, and **Turborepo**. The system features a service-oriented architecture designed for real-time market data processing, strategy execution, and risk management. - -## ๐ŸŽฏ Current System Status - -### โœ… **Operational Services** -- **Market Data Gateway** (`apps/core-services/market-data-gateway`) - Port 3004 - - **UNIFIED IMPLEMENTATION** - Merged from duplicate services - - Real-time market data ingestion and processing - - WebSocket server for live data streaming (Port 3005) - - REST API for market data queries and configuration - - Mock data implementation for testing - - Full TypeScript implementation with resolved compilation errors - -- **Trading Dashboard** (`apps/interface-services/trading-dashboard`) - Port 5173 - - React + TypeScript frontend with Tremor UI - - Real-time data visualization - - WebSocket client for live updates - - Professional financial dashboard components - -### ๐Ÿšง **Ready for Implementation** -- **Strategy Orchestrator** (`apps/intelligence-services/strategy-orchestrator`) - Port 4001 - - Package structure created, implementation needed - - Strategy execution and management - - Signal generation coordination - -- **Risk Guardian** (`apps/core-services/risk-guardian`) - Port 3002 - - Package structure created, implementation needed - - Real-time risk monitoring and alerts - - Position and exposure limits - -## ๐Ÿ—๏ธ Service Architecture - -### **Service Categories** -``` -apps/ -โ”œโ”€โ”€ core-services/ # Essential trading infrastructure -โ”‚ โ”œโ”€โ”€ market-data-gateway/ โœ… Operational (UNIFIED) -โ”‚ โ””โ”€โ”€ risk-guardian/ ๐Ÿ“‹ Ready to implement -โ”œโ”€โ”€ intelligence-services/ # Strategy and signal generation -โ”‚ โ””โ”€โ”€ strategy-orchestrator/ ๐Ÿ“‹ Ready to implement -โ”œโ”€โ”€ interface-services/ # User interfaces and APIs -โ”‚ โ”œโ”€โ”€ trading-dashboard/ โœ… Operational -โ”‚ โ””โ”€โ”€ trading-dashboard-react/ ๐Ÿ“‹ Alternative implementation -โ”œโ”€โ”€ data-services/ # Data processing and analytics -โ”œโ”€โ”€ execution-services/ # Order management and execution -โ”œโ”€โ”€ integration-services/ # External system integrations -โ””โ”€โ”€ platform-services/ # Infrastructure and monitoring -``` - -### **Shared Packages** -``` -packages/ -โ”œโ”€โ”€ types/ # TypeScript type definitions -โ”œโ”€โ”€ config/ # Configuration management -โ”œโ”€โ”€ database/ # Database utilities (planned) -โ””โ”€โ”€ trading-core/ # Core trading logic (planned) -``` - -## ๐Ÿ”„ Communication Architecture - -### **Event-Driven Core (Dragonfly)** -- **Primary Event Bus**: Dragonfly Redis-compatible streams -- **Event Types**: Market data, trading signals, risk alerts, portfolio updates -- **Communication Pattern**: Publish/Subscribe for loose coupling -- **Real-time Processing**: Sub-millisecond event propagation - -### **Data Flow Patterns** -1. **Market Data Flow** - ``` - External APIs โ†’ Market Data Gateway โ†’ Dragonfly Events โ†’ Dashboard/Services - ``` - -2. **Trading Signal Flow** - ``` - Market Data โ†’ Strategy Orchestrator โ†’ Signals โ†’ Risk Guardian โ†’ Execution - ``` - -3. **Real-time Updates** - ``` - Services โ†’ WebSocket Server โ†’ Dashboard (Live UI Updates) - ``` - -### **API Communication** -- **REST APIs**: Service-to-service and client-to-service communication -- **WebSocket**: Real-time bidirectional communication -- **HTTP Health Checks**: Service monitoring and discovery - -## ๐Ÿ—„๏ธ Data Infrastructure - -### **Storage Systems** -- **๐Ÿ‰ Dragonfly** (Port 6379): Redis-compatible event streaming and caching -- **๐Ÿ˜ PostgreSQL** (Port 5432): Operational data with trading schemas -- **๐Ÿ“Š QuestDB** (Ports 9000/8812): Time-series market data and analytics -- **๐Ÿƒ MongoDB** (Port 27017): Document storage for sentiment analysis and raw documents - -### **Database Schemas** -- `trading.*` - Orders, positions, executions, accounts -- `strategy.*` - Strategies, signals, performance metrics -- `risk.*` - Risk limits, events, monitoring -- `audit.*` - System events, health checks, configuration - -### **MongoDB Collections** -- `sentiment_analysis` - Market sentiment scores and analysis -- `raw_documents` - News articles, social media posts, research reports -- `market_events` - Significant market events and their impact - -### **Admin Interfaces** -- **Redis Insight** (Port 8001): Dragonfly management -- **PgAdmin** (Port 8080): PostgreSQL administration -- **QuestDB Console** (Port 9000): Time-series data management -- **Mongo Express** (Port 8081): MongoDB document browser and editor - -## ๐Ÿ›ก๏ธ Infrastructure & DevOps - -### **Container Management** -- **Docker Compose**: Multi-service orchestration -- **Development Environment**: `npm run dev:full` -- **Infrastructure Management**: PowerShell scripts in `scripts/docker.ps1` - -### **Monitoring Stack** -- **Prometheus** (Port 9090): Metrics collection -- **Grafana** (Port 3000): Dashboards and alerting -- **Health Checks**: Each service exposes `/health` endpoints - -### **Development Workflow** -```bash -# Start infrastructure -npm run infra:up - -# Start admin tools -npm run docker:admin - -# Start development services -npm run dev:full -``` - -## ๐ŸŒ Port Allocation - -| Service Category | Port Range | Current Services | -|-----------------|------------|------------------| -| **Core Services** | 3001-3099 | Market Data Gateway (3001), Risk Guardian (3002) | -| **Intelligence** | 4001-4099 | Strategy Orchestrator (4001) | -| **Data Services** | 5001-5099 | (Future expansion) | -| **Interface** | 5173, 8001+ | Trading Dashboard (5173) | -| **Infrastructure** | 6379, 5432, 9000+, 27017 | Dragonfly, PostgreSQL, QuestDB, MongoDB | - -## ๐ŸŽฏ Implementation Roadmap - -### **Phase 1 - Foundation** โœ… Complete -- [x] Monorepo setup with Turborepo -- [x] Market Data Gateway with real-time streaming -- [x] Professional React dashboard with Tremor UI -- [x] Docker infrastructure with Dragonfly/PostgreSQL/QuestDB -- [x] WebSocket real-time communication - -### **Phase 2 - Trading Logic** ๐Ÿ“‹ Next Priority -- [ ] Strategy Orchestrator implementation -- [ ] Risk Guardian implementation -- [ ] Event-driven strategy execution -- [ ] Portfolio position tracking - -### **Phase 3 - Advanced Features** โณ Future -- [ ] Execution Engine with broker integration -- [ ] Advanced analytics and backtesting -- [ ] Machine learning signal generation -- [ ] Multi-broker support - -## ๐Ÿ”ง Technology Stack - -### **Backend Services** -- **Runtime**: Bun (fast JavaScript runtime) -- **Framework**: Hono (lightweight web framework) -- **Language**: TypeScript (type safety) -- **Events**: Dragonfly Redis Streams -- **WebSocket**: Native WebSocket implementation - -### **Frontend** -- **Framework**: React with TypeScript -- **Build Tool**: Vite (fast development) -- **UI Library**: Tremor UI (financial components) -- **Styling**: Modern CSS with responsive design - -### **Infrastructure** -- **Monorepo**: Turborepo for build orchestration -- **Containers**: Docker & Docker Compose -- **Databases**: PostgreSQL, QuestDB, Dragonfly -- **Monitoring**: Prometheus + Grafana - -## ๐Ÿ”’ Security & Configuration - -### **Environment Management** -- Environment-specific configurations -- Secure credential management -- Development vs production separation - -### **Service Security** -- Inter-service authentication -- API rate limiting -- Database connection security -- External API key management - -## ๐Ÿš€ Getting Started - -1. **Prerequisites**: Docker Desktop, Bun runtime -2. **Infrastructure**: `npm run infra:up` -3. **Admin Tools**: `npm run docker:admin` -4. **Development**: `npm run dev:full` -5. **Access**: Dashboard at http://localhost:5173 - -## ๐Ÿ“ Key Configuration Files - -- `turbo.json` - Monorepo build configuration -- `docker-compose.yml` - Infrastructure orchestration -- `packages/config/` - Shared configuration management -- `database/postgres/init/` - Database schema definitions - ---- - -**Architecture Design Principles:** -- **Microservices**: Independent, scalable services -- **Event-Driven**: Loose coupling via Dragonfly events -- **Type Safety**: TypeScript across all services -- **Real-time**: WebSocket and event streaming -- **Observability**: Comprehensive monitoring and logging -- **Developer Experience**: Fast development with hot reload diff --git a/README.md b/README.md deleted file mode 100644 index 6269de6..0000000 --- a/README.md +++ /dev/null @@ -1,180 +0,0 @@ -# ๐Ÿค– Stock Bot Trading System - -A comprehensive trading bot built with Bun and Turborepo, featuring a service-oriented architecture for real-time market data processing and strategy execution. - -## ๐Ÿš€ Quick Start - -### Prerequisites -- [Bun](https://bun.sh/) runtime -- Node.js 18+ (for compatibility) - -### Installation -```bash -# Clone and install dependencies -git clone -cd stock-bot -bun install -``` - -### Running the System - -#### Option 1: VS Code Tasks (Recommended) -1. Open the project in VS Code -2. Press `Ctrl+Shift+P` (or `Cmd+Shift+P` on Mac) -3. Type "Tasks: Run Task" and select it -4. Choose "Start All Services" - -#### Option 2: Manual Startup -```bash -# Terminal 1: Start Market Data Gateway -cd apps/core-services/market-data-gateway -bun run dev - -# Terminal 2: Start Trading Dashboard -cd apps/interface-services/trading-dashboard -bun run dev -``` - -### Access Points -- **Trading Dashboard**: http://localhost:5173 -- **Market Data API**: http://localhost:3001 -- **Health Check**: http://localhost:3001/health - -## ๐Ÿ“Š Dashboard Features - -### Real-time Market Data -- Live price feeds for AAPL, GOOGL, MSFT, TSLA, AMZN -- WebSocket connections for real-time updates -- Service health monitoring - -### Professional UI Components -- Built with Tremor UI for financial visualizations -- Interactive charts and metrics -- Responsive design for all devices - -### Dashboard Tabs -1. **Market Data**: Live prices, volume, bid/ask spreads -2. **Portfolio**: Holdings allocation and performance -3. **Charts**: Price and volume analysis -4. **Performance**: Trading metrics and statistics - -## ๐Ÿ—๏ธ Architecture - -### Service-Oriented Design -``` -apps/ -โ”œโ”€โ”€ core-services/ -โ”‚ โ””โ”€โ”€ market-data-gateway/ # Market data ingestion -โ”œโ”€โ”€ interface-services/ -โ”‚ โ””โ”€โ”€ trading-dashboard/ # React dashboard -โ”œโ”€โ”€ data-services/ # (Future) Data processing -โ”œโ”€โ”€ execution-services/ # (Future) Order management -โ”œโ”€โ”€ intelligence-services/ # (Future) Strategy engine -โ”œโ”€โ”€ platform-services/ # (Future) Infrastructure -โ””โ”€โ”€ integration-services/ # (Future) External APIs -``` - -### Shared Packages -``` -packages/ -โ”œโ”€โ”€ types/ # TypeScript definitions -โ”œโ”€โ”€ config/ # Configuration management -โ”œโ”€โ”€ database/ # (Future) Database utilities -โ””โ”€โ”€ trading-core/ # (Future) Core trading logic -``` - -## ๐Ÿ”ง Development - -### Project Structure -- **Turborepo**: Monorepo management -- **Bun**: Package manager and runtime -- **TypeScript**: Type safety across all services -- **React + Vite**: Modern frontend development -- **Tremor UI**: Financial dashboard components - -### Key Technologies -- **Backend**: Hono framework, WebSockets, Redis -- **Frontend**: React, TypeScript, Tremor UI -- **Data**: QuestDB (planned), PostgreSQL (planned) -- **Deployment**: Docker, Kubernetes (planned) - -## ๐Ÿ“ˆ Current Status - -### โœ… Completed -- [x] Monorepo setup with Turborepo -- [x] Market Data Gateway service -- [x] Real-time WebSocket connections -- [x] Professional React dashboard -- [x] Tremor UI integration -- [x] TypeScript type system -- [x] Service health monitoring - -### ๐Ÿšง In Progress -- [ ] Strategy execution engine -- [ ] Risk management system -- [ ] Portfolio tracking -- [ ] Real broker integration - -### ๐Ÿ”ฎ Planned -- [ ] Advanced charting -- [ ] Backtesting framework -- [ ] Machine learning signals -- [ ] Multi-broker support -- [ ] Mobile application - -## ๐Ÿ› ๏ธ API Endpoints - -### Market Data Gateway (Port 3001) -``` -GET /health # Service health check -GET /api/market-data/:symbol # Current market data -GET /api/ohlcv/:symbol # Historical OHLCV data -WS ws://localhost:3001 # Real-time data stream -``` - -### Data Format -```typescript -interface MarketData { - symbol: string; - price: number; - bid: number; - ask: number; - volume: number; - timestamp: string; -} -``` - -## ๐Ÿ”ง Configuration - -Environment variables are managed in `.env`: -```bash -# Database Configuration -DATABASE_URL=postgresql://... -QUESTDB_URL=http://localhost:9000 - -# External APIs -ALPHA_VANTAGE_API_KEY=your_key_here -ALPACA_API_KEY=your_key_here - -# Service Configuration -NODE_ENV=development -LOG_LEVEL=info -``` - -## ๐Ÿค Contributing - -1. Fork the repository -2. Create a feature branch: `git checkout -b feature/amazing-feature` -3. Commit changes: `git commit -m 'Add amazing feature'` -4. Push to branch: `git push origin feature/amazing-feature` -5. Open a Pull Request - -## ๐Ÿ“ License - -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. - -## ๐Ÿ™ Acknowledgments - -- [Tremor UI](https://tremor.so/) for beautiful financial components -- [Bun](https://bun.sh/) for fast runtime and package management -- [Turborepo](https://turbo.build/) for monorepo tooling diff --git a/REFACTORING.md b/REFACTORING.md deleted file mode 100644 index 0f679fc..0000000 --- a/REFACTORING.md +++ /dev/null @@ -1,62 +0,0 @@ -# Stock Bot Project Structure Refactoring - -This document outlines the changes made to improve separation of concerns in the stock-bot project architecture. - -## Directory Structure Changes - -1. Created a dedicated `libs/` directory (replacing the previous `packages/` approach) -2. Split monolithic type definitions into domain-specific modules -3. Created specialized libraries for cross-cutting concerns - -## New Libraries - -### 1. `@stock-bot/types` -Domain-specific type definitions organized by functional area: -- `market/` - Market data structures (OHLCV, OrderBook, etc.) -- `trading/` - Trading types (Orders, Positions, etc.) -- `strategy/` - Strategy and signal types -- `events/` - Event definitions for the event bus -- `api/` - Common API request/response types -- `config/` - Configuration type definitions - -### 2. `@stock-bot/event-bus` -A Redis-based event bus implementation for inter-service communication: -- Publish/subscribe pattern for asynchronous messaging -- Support for typed events -- Reliable message delivery - -### 3. `@stock-bot/utils` -Common utility functions shared across services: -- `dateUtils` - Date/time helpers for market data -- `financialUtils` - Financial calculations (Sharpe ratio, drawdown) -- `logger` - Standardized logging service - -### 4. `@stock-bot/api-client` -Type-safe API clients for inter-service communication: -- `BaseApiClient` - Common HTTP client functionality -- Service-specific clients (BacktestClient, StrategyClient) - -## Benefits of the New Architecture - -1. **Better Separation of Concerns** - - Each library has a clear, focused purpose - - Domain types are logically grouped - -2. **Improved Maintainability** - - Smaller, focused modules - - Clear dependencies between modules - -3. **Type Safety** - - Consistent types across the entire system - - Better IDE autocompletion and error checking - -4. **Reduced Duplication** - - Shared utilities in a central location - - Consistent implementation of common patterns - -## Next Steps - -1. Update service implementations to use the new libraries -2. Migrate from the old packages directory to the new libs structure -3. Implement domain-specific validations in each type module -4. Add unit tests for each library diff --git a/apps/core-services/market-data-gateway/README.md b/apps/core-services/market-data-gateway/README.md deleted file mode 100644 index 8a351bf..0000000 --- a/apps/core-services/market-data-gateway/README.md +++ /dev/null @@ -1,196 +0,0 @@ -# Market Data Gateway - Unified Implementation - -## Overview - -The Market Data Gateway is a unified service that consolidates real-time market data ingestion, processing, and distribution capabilities. This service has been created by merging the previous core-services and data-services market-data-gateway implementations into a single, comprehensive solution. - -## Architecture - -### Unified Design -- **Single Service**: Combines data ingestion, processing, and distribution in one service -- **HTTP API**: RESTful endpoints for configuration and data retrieval -- **WebSocket Server**: Real-time data streaming capabilities -- **Type Safety**: Full TypeScript implementation with comprehensive type definitions - -### Key Components -- **Data Source Management**: Configure and manage multiple market data sources -- **Real-time Processing**: Stream processing pipelines for market data -- **WebSocket Streaming**: Real-time data distribution to clients -- **Health Monitoring**: Comprehensive health checks and metrics -- **Cache Management**: Redis-based caching for performance optimization - -## Features - -### HTTP Endpoints - -#### Health & Status -- `GET /health` - Basic health check -- `GET /health/readiness` - Readiness probe -- `GET /health/liveness` - Liveness probe -- `GET /api/v1/gateway/status` - Gateway status and metrics -- `GET /api/v1/gateway/config` - Current configuration - -#### Data Sources -- `GET /api/v1/sources` - List configured data sources -- `POST /api/v1/sources` - Add new data source -- `PUT /api/v1/sources/:sourceId` - Update data source -- `DELETE /api/v1/sources/:sourceId` - Remove data source - -#### Market Data -- `GET /api/v1/data/tick/:symbol` - Latest tick data for symbol -- `GET /api/v1/data/candles/:symbol` - Historical candle data -- `GET /api/v1/subscriptions` - List active subscriptions -- `POST /api/v1/subscriptions` - Create new subscription - -#### Metrics -- `GET /api/v1/metrics` - System and gateway metrics - -### WebSocket Streaming - -Connect to `ws://localhost:3005/ws` for real-time data streaming. - -#### Message Types - -**Subscribe to symbols:** -```json -{ - "type": "subscribe", - "symbols": ["AAPL", "GOOGL", "MSFT"] -} -``` - -**Unsubscribe:** -```json -{ - "type": "unsubscribe", - "subscriptionId": "sub_1234567890" -} -``` - -**Receive tick data:** -```json -{ - "type": "tick", - "data": { - "symbol": "AAPL", - "price": 150.25, - "volume": 1000, - "timestamp": "2025-06-03T13:01:49.638Z", - "bid": 150.20, - "ask": 150.30 - } -} -``` - -## Configuration - -The service is configured through environment variables and the `GatewayConfig` interface: - -### Environment Variables -- `PORT` - HTTP server port (default: 3004) -- `HOST` - Server host (default: 0.0.0.0) -- `REDIS_HOST` - Redis host (default: localhost) -- `REDIS_PORT` - Redis port (default: 6379) -- `REDIS_PASSWORD` - Redis password (optional) -- `REDIS_DB` - Redis database number (default: 0) -- `METRICS_PORT` - Metrics port (default: 9090) - -### Configuration Structure -```typescript -interface GatewayConfig { - server: ServerConfig; - dataSources: DataSourceConfig[]; - processing: ProcessingConfig; - cache: CacheConfig; - monitoring: MonitoringConfig; -} -``` - -## Development - -### Prerequisites -- Bun runtime -- Redis server -- TypeScript - -### Setup -```bash -cd apps/core-services/market-data-gateway -bun install -``` - -### Development Mode -```bash -bun run dev -``` - -### Build -```bash -bun run build -``` - -### Testing -The service includes mock data for testing purposes. When running, it will: -- Respond to health checks -- Provide mock tick and candle data -- Accept WebSocket connections -- Send simulated real-time data every 5 seconds - -## Deployment - -The service can be deployed using: -- Docker containers -- Kubernetes -- Direct Node.js/Bun deployment - -### Docker -```dockerfile -FROM oven/bun:latest -WORKDIR /app -COPY package.json . -COPY src/ ./src/ -RUN bun install -RUN bun run build -EXPOSE 3004 3005 -CMD ["bun", "run", "start"] -``` - -## Migration Notes - -This unified implementation replaces both: -- `apps/core-services/market-data-gateway` (original) -- `apps/data-services/market-data-gateway` (duplicate) - -### Changes Made -1. **Consolidated Architecture**: Merged real-time and storage capabilities -2. **Fixed Type Issues**: Resolved all TypeScript compilation errors -3. **Simplified Configuration**: Aligned with `GatewayConfig` interface -4. **Working WebSocket**: Functional real-time streaming -5. **Comprehensive API**: Full REST API implementation -6. **Mock Data**: Testing capabilities with simulated data - -### Removed Duplicates -- Removed `apps/data-services/market-data-gateway` directory -- Consolidated type definitions -- Unified configuration structure - -## Future Enhancements - -1. **Real Data Sources**: Replace mock data with actual market data feeds -2. **Advanced Processing**: Implement complex processing pipelines -3. **Persistence Layer**: Add database storage for historical data -4. **Authentication**: Add API authentication and authorization -5. **Rate Limiting**: Implement request rate limiting -6. **Monitoring**: Enhanced metrics and alerting -7. **Load Balancing**: Support for horizontal scaling - -## Status - -โœ… **COMPLETED**: TypeScript compilation errors resolved -โœ… **COMPLETED**: Unified service architecture -โœ… **COMPLETED**: Working HTTP and WebSocket servers -โœ… **COMPLETED**: Mock data implementation -โœ… **COMPLETED**: Health and metrics endpoints -โœ… **COMPLETED**: Duplicate service removal - -The market data gateway merge is now complete and the service is fully operational. diff --git a/apps/core-services/market-data-gateway/package.json b/apps/core-services/market-data-gateway/package.json deleted file mode 100644 index 413a8ab..0000000 --- a/apps/core-services/market-data-gateway/package.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "name": "@stock-bot/market-data-gateway", - "version": "1.0.0", - "description": "Unified market data gateway - real-time processing and historical storage", - "main": "src/index.ts", - "type": "module", - "scripts": { - "dev": "bun --watch src/index.ts", - "build": "tsc", - "start": "bun src/index.ts", - "test": "bun test", - "lint": "eslint src/**/*.ts", - "type-check": "tsc --noEmit" - }, - "dependencies": { - "hono": "^4.6.3", - "@hono/node-server": "^1.8.0", - "ws": "^8.18.0", - "bull": "^4.12.0", - "ioredis": "^5.4.1", - "zod": "^3.22.0", - "uuid": "^9.0.0", - "compression": "^1.7.4", - "helmet": "^7.1.0", - "rate-limiter-flexible": "^5.0.0", - "node-cron": "^3.0.3", - "eventemitter3": "^5.0.1", - "fast-json-stringify": "^5.10.0", - "pino": "^8.17.0", - "dotenv": "^16.3.0", - "@stock-bot/http-client": "*", - "@stock-bot/config": "*", - "@stock-bot/types": "*", - "@stock-bot/event-bus": "*", - "@stock-bot/utils": "*" - }, - "devDependencies": { - "@types/node": "^20.11.0", - "@types/ws": "^8.5.12", - "@types/uuid": "^9.0.0", - "@types/compression": "^1.7.5", - "@types/node-cron": "^3.0.11", - "typescript": "^5.3.0", - "eslint": "^8.56.0", - "@typescript-eslint/eslint-plugin": "^6.19.0", - "@typescript-eslint/parser": "^6.19.0", - "bun-types": "^1.2.15" - }, - "keywords": [ - "market-data", - "gateway", - "real-time", - "websocket", - "historical", - "stock-bot", - "core-services" - ] -} diff --git a/apps/core-services/market-data-gateway/src/config/DataProviderConfig.ts b/apps/core-services/market-data-gateway/src/config/DataProviderConfig.ts deleted file mode 100644 index 60d11cb..0000000 --- a/apps/core-services/market-data-gateway/src/config/DataProviderConfig.ts +++ /dev/null @@ -1,114 +0,0 @@ -// Data Provider Configuration -export interface DataProviderConfig { - name: string; - type: 'rest' | 'websocket' | 'both'; - enabled: boolean; - endpoints: { - quotes?: string; - candles?: string; - trades?: string; - websocket?: string; - }; - authentication?: { - type: 'api_key' | 'bearer' | 'basic'; - key?: string; - secret?: string; - token?: string; - }; - rateLimits: { - requestsPerSecond: number; - requestsPerMinute: number; - requestsPerHour: number; - }; - retryPolicy: { - maxRetries: number; - backoffMultiplier: number; - initialDelayMs: number; - }; - timeout: number; - priority: number; // 1-10, higher is better -} - -export const dataProviderConfigs: Record = { - 'alpha-vantage': { - name: 'Alpha Vantage', - type: 'rest', - enabled: true, - endpoints: { - quotes: 'https://www.alphavantage.co/query', - candles: 'https://www.alphavantage.co/query', - }, - authentication: { - type: 'api_key', - key: process.env.ALPHA_VANTAGE_API_KEY, - }, - rateLimits: { - requestsPerSecond: 5, - requestsPerMinute: 500, - requestsPerHour: 25000, - }, - retryPolicy: { - maxRetries: 3, - backoffMultiplier: 2, - initialDelayMs: 1000, - }, - timeout: 10000, - priority: 7, - }, - 'yahoo-finance': { - name: 'Yahoo Finance', - type: 'rest', - enabled: true, - endpoints: { - quotes: 'https://query1.finance.yahoo.com/v8/finance/chart', - candles: 'https://query1.finance.yahoo.com/v8/finance/chart', - }, - rateLimits: { - requestsPerSecond: 10, - requestsPerMinute: 2000, - requestsPerHour: 100000, - }, - retryPolicy: { - maxRetries: 3, - backoffMultiplier: 1.5, - initialDelayMs: 500, - }, - timeout: 8000, - priority: 8, - }, - 'polygon': { - name: 'Polygon.io', - type: 'both', - enabled: false, - endpoints: { - quotes: 'https://api.polygon.io/v2/last/nbbo', - candles: 'https://api.polygon.io/v2/aggs/ticker', - trades: 'https://api.polygon.io/v3/trades', - websocket: 'wss://socket.polygon.io/stocks', - }, - authentication: { - type: 'api_key', - key: process.env.POLYGON_API_KEY, - }, - rateLimits: { - requestsPerSecond: 100, - requestsPerMinute: 5000, - requestsPerHour: 100000, - }, - retryPolicy: { - maxRetries: 5, - backoffMultiplier: 2, - initialDelayMs: 200, - }, - timeout: 5000, - priority: 9, - }, -}; - -export function getEnabledProviders(): DataProviderConfig[] { - return Object.values(dataProviderConfigs).filter(config => config.enabled); -} - -export function getProviderByPriority(): DataProviderConfig[] { - return getEnabledProviders().sort((a, b) => b.priority - a.priority); -} diff --git a/apps/core-services/market-data-gateway/src/controllers/GatewayController.ts b/apps/core-services/market-data-gateway/src/controllers/GatewayController.ts deleted file mode 100644 index a6f6fa7..0000000 --- a/apps/core-services/market-data-gateway/src/controllers/GatewayController.ts +++ /dev/null @@ -1,449 +0,0 @@ -import { Context } from 'hono'; -import { MarketDataGatewayService } from '../services/MarketDataGatewayService'; -import { - DataSourceConfig, - SubscriptionRequest, - ProcessingPipelineConfig, - Logger -} from '../types/MarketDataGateway'; - -export class GatewayController { - constructor( - private gatewayService: MarketDataGatewayService, - private logger: Logger - ) {} - - // Gateway status and control - async getStatus(c: Context) { - try { - const status = this.gatewayService.getStatus(); - return c.json(status); - } catch (error) { - this.logger.error('Failed to get gateway status:', error); - return c.json({ error: 'Failed to get gateway status' }, 500); - } - } - - async getHealth(c: Context) { - try { - const health = this.gatewayService.getHealth(); - const statusCode = health.status === 'healthy' ? 200 : - health.status === 'degraded' ? 200 : 503; - return c.json(health, statusCode); - } catch (error) { - this.logger.error('Failed to get gateway health:', error); - return c.json({ - status: 'unhealthy', - message: 'Health check failed', - timestamp: new Date().toISOString() - }, 503); - } - } - - async getMetrics(c: Context) { - try { - const metrics = this.gatewayService.getMetrics(); - return c.json(metrics); - } catch (error) { - this.logger.error('Failed to get gateway metrics:', error); - return c.json({ error: 'Failed to get gateway metrics' }, 500); - } - } - - async getConfiguration(c: Context) { - try { - const config = this.gatewayService.getConfiguration(); - return c.json(config); - } catch (error) { - this.logger.error('Failed to get gateway configuration:', error); - return c.json({ error: 'Failed to get gateway configuration' }, 500); - } - } - - async updateConfiguration(c: Context) { - try { - const updates = await c.req.json(); - await this.gatewayService.updateConfiguration(updates); - - return c.json({ - message: 'Configuration updated successfully', - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to update gateway configuration:', error); - return c.json({ error: 'Failed to update gateway configuration' }, 500); - } - } - - // Data source management - async getDataSources(c: Context) { - try { - const sources = this.gatewayService.getDataSources(); - return c.json({ dataSources: Array.from(sources.values()) }); - } catch (error) { - this.logger.error('Failed to get data sources:', error); - return c.json({ error: 'Failed to get data sources' }, 500); - } - } - - async getDataSource(c: Context) { - try { - const sourceId = c.req.param('sourceId'); - const source = this.gatewayService.getDataSource(sourceId); - - if (!source) { - return c.json({ error: 'Data source not found' }, 404); - } - - return c.json(source); - } catch (error) { - this.logger.error('Failed to get data source:', error); - return c.json({ error: 'Failed to get data source' }, 500); - } - } - - async addDataSource(c: Context) { - try { - const sourceConfig: DataSourceConfig = await c.req.json(); - - // Validate required fields - if (!sourceConfig.id || !sourceConfig.type || !sourceConfig.provider) { - return c.json({ - error: 'Missing required fields: id, type, provider' - }, 400); - } - - await this.gatewayService.addDataSource(sourceConfig); - - return c.json({ - message: 'Data source added successfully', - sourceId: sourceConfig.id, - timestamp: new Date().toISOString() - }, 201); - } catch (error) { - this.logger.error('Failed to add data source:', error); - return c.json({ error: 'Failed to add data source' }, 500); - } - } - - async updateDataSource(c: Context) { - try { - const sourceId = c.req.param('sourceId'); - const updates = await c.req.json(); - - await this.gatewayService.updateDataSource(sourceId, updates); - - return c.json({ - message: 'Data source updated successfully', - sourceId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to update data source:', error); - return c.json({ error: 'Failed to update data source' }, 500); - } - } - - async removeDataSource(c: Context) { - try { - const sourceId = c.req.param('sourceId'); - await this.gatewayService.removeDataSource(sourceId); - - return c.json({ - message: 'Data source removed successfully', - sourceId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to remove data source:', error); - return c.json({ error: 'Failed to remove data source' }, 500); - } - } - - async startDataSource(c: Context) { - try { - const sourceId = c.req.param('sourceId'); - await this.gatewayService.startDataSource(sourceId); - - return c.json({ - message: 'Data source started successfully', - sourceId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to start data source:', error); - return c.json({ error: 'Failed to start data source' }, 500); - } - } - - async stopDataSource(c: Context) { - try { - const sourceId = c.req.param('sourceId'); - await this.gatewayService.stopDataSource(sourceId); - - return c.json({ - message: 'Data source stopped successfully', - sourceId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to stop data source:', error); - return c.json({ error: 'Failed to stop data source' }, 500); - } - } - - // Subscription management - async getSubscriptions(c: Context) { - try { - const subscriptions = this.gatewayService.getSubscriptions(); - return c.json({ subscriptions: Array.from(subscriptions.values()) }); - } catch (error) { - this.logger.error('Failed to get subscriptions:', error); - return c.json({ error: 'Failed to get subscriptions' }, 500); - } - } - - async getSubscription(c: Context) { - try { - const subscriptionId = c.req.param('subscriptionId'); - const subscription = this.gatewayService.getSubscription(subscriptionId); - - if (!subscription) { - return c.json({ error: 'Subscription not found' }, 404); - } - - return c.json(subscription); - } catch (error) { - this.logger.error('Failed to get subscription:', error); - return c.json({ error: 'Failed to get subscription' }, 500); - } - } - - async createSubscription(c: Context) { - try { - const subscriptionRequest: SubscriptionRequest = await c.req.json(); - - // Validate required fields - if (!subscriptionRequest.clientId || !subscriptionRequest.symbols || subscriptionRequest.symbols.length === 0) { - return c.json({ - error: 'Missing required fields: clientId, symbols' - }, 400); - } - - const subscriptionId = await this.gatewayService.subscribe(subscriptionRequest); - - return c.json({ - message: 'Subscription created successfully', - subscriptionId, - clientId: subscriptionRequest.clientId, - symbols: subscriptionRequest.symbols, - timestamp: new Date().toISOString() - }, 201); - } catch (error) { - this.logger.error('Failed to create subscription:', error); - return c.json({ error: 'Failed to create subscription' }, 500); - } - } - - async updateSubscription(c: Context) { - try { - const subscriptionId = c.req.param('subscriptionId'); - const updates = await c.req.json(); - - await this.gatewayService.updateSubscription(subscriptionId, updates); - - return c.json({ - message: 'Subscription updated successfully', - subscriptionId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to update subscription:', error); - return c.json({ error: 'Failed to update subscription' }, 500); - } - } - - async deleteSubscription(c: Context) { - try { - const subscriptionId = c.req.param('subscriptionId'); - await this.gatewayService.unsubscribe(subscriptionId); - - return c.json({ - message: 'Subscription deleted successfully', - subscriptionId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to delete subscription:', error); - return c.json({ error: 'Failed to delete subscription' }, 500); - } - } - - // Processing pipeline management - async getProcessingPipelines(c: Context) { - try { - const pipelines = this.gatewayService.getProcessingPipelines(); - return c.json({ pipelines: Array.from(pipelines.values()) }); - } catch (error) { - this.logger.error('Failed to get processing pipelines:', error); - return c.json({ error: 'Failed to get processing pipelines' }, 500); - } - } - - async getProcessingPipeline(c: Context) { - try { - const pipelineId = c.req.param('pipelineId'); - const pipeline = this.gatewayService.getProcessingPipeline(pipelineId); - - if (!pipeline) { - return c.json({ error: 'Processing pipeline not found' }, 404); - } - - return c.json(pipeline); - } catch (error) { - this.logger.error('Failed to get processing pipeline:', error); - return c.json({ error: 'Failed to get processing pipeline' }, 500); - } - } - - async createProcessingPipeline(c: Context) { - try { - const pipelineConfig: ProcessingPipelineConfig = await c.req.json(); - - // Validate required fields - if (!pipelineConfig.id || !pipelineConfig.name || !pipelineConfig.processors) { - return c.json({ - error: 'Missing required fields: id, name, processors' - }, 400); - } - - await this.gatewayService.addProcessingPipeline(pipelineConfig); - - return c.json({ - message: 'Processing pipeline created successfully', - pipelineId: pipelineConfig.id, - timestamp: new Date().toISOString() - }, 201); - } catch (error) { - this.logger.error('Failed to create processing pipeline:', error); - return c.json({ error: 'Failed to create processing pipeline' }, 500); - } - } - - async updateProcessingPipeline(c: Context) { - try { - const pipelineId = c.req.param('pipelineId'); - const updates = await c.req.json(); - - await this.gatewayService.updateProcessingPipeline(pipelineId, updates); - - return c.json({ - message: 'Processing pipeline updated successfully', - pipelineId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to update processing pipeline:', error); - return c.json({ error: 'Failed to update processing pipeline' }, 500); - } - } - - async deleteProcessingPipeline(c: Context) { - try { - const pipelineId = c.req.param('pipelineId'); - await this.gatewayService.removeProcessingPipeline(pipelineId); - - return c.json({ - message: 'Processing pipeline deleted successfully', - pipelineId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to delete processing pipeline:', error); - return c.json({ error: 'Failed to delete processing pipeline' }, 500); - } - } - - // Market data queries - async getLatestTick(c: Context) { - try { - const symbol = c.req.param('symbol'); - if (!symbol) { - return c.json({ error: 'Symbol parameter is required' }, 400); - } - - const tick = await this.gatewayService.getLatestTick(symbol); - - if (!tick) { - return c.json({ error: 'No data available for symbol' }, 404); - } - - return c.json(tick); - } catch (error) { - this.logger.error('Failed to get latest tick:', error); - return c.json({ error: 'Failed to get latest tick' }, 500); - } - } - - async getCandles(c: Context) { - try { - const symbol = c.req.param('symbol'); - const timeframe = c.req.query('timeframe') || '1m'; - const startTime = c.req.query('startTime'); - const endTime = c.req.query('endTime'); - const limit = parseInt(c.req.query('limit') || '100'); - - if (!symbol) { - return c.json({ error: 'Symbol parameter is required' }, 400); - } - - const candles = await this.gatewayService.getCandles( - symbol, - timeframe, - startTime ? parseInt(startTime) : undefined, - endTime ? parseInt(endTime) : undefined, - limit - ); - - return c.json({ - symbol, - timeframe, - candles, - count: candles.length, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to get candles:', error); - return c.json({ error: 'Failed to get candles' }, 500); - } - } - - // System operations - async flushCache(c: Context) { - try { - await this.gatewayService.flushCache(); - - return c.json({ - message: 'Cache flushed successfully', - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to flush cache:', error); - return c.json({ error: 'Failed to flush cache' }, 500); - } - } - - async restart(c: Context) { - try { - await this.gatewayService.restart(); - - return c.json({ - message: 'Gateway restart initiated', - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to restart gateway:', error); - return c.json({ error: 'Failed to restart gateway' }, 500); - } - } -} diff --git a/apps/core-services/market-data-gateway/src/controllers/HealthController.ts b/apps/core-services/market-data-gateway/src/controllers/HealthController.ts deleted file mode 100644 index 34e96ed..0000000 --- a/apps/core-services/market-data-gateway/src/controllers/HealthController.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { Context } from 'hono'; -import { MarketDataGatewayService } from '../services/MarketDataGatewayService'; -import { Logger } from '../types/MarketDataGateway'; - -export class HealthController { - constructor( - private gatewayService: MarketDataGatewayService, - private logger: Logger - ) {} - - async getHealth(c: Context) { - try { - const health = this.gatewayService.getHealth(); - const statusCode = health.status === 'healthy' ? 200 : - health.status === 'degraded' ? 200 : 503; - - return c.json(health, statusCode); - } catch (error) { - this.logger.error('Health check failed:', error); - return c.json({ - status: 'unhealthy', - message: 'Health check failed', - timestamp: new Date().toISOString(), - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - } - - async getReadiness(c: Context) { - try { - const health = this.gatewayService.getHealth(); - const components = health.details?.components || {}; - - // Check if all critical components are ready - const criticalComponents = [ - 'dataSourceManager', - 'processingEngine', - 'subscriptionManager' - ]; - - const allReady = criticalComponents.every(component => { - const componentHealth = components[component]; - return componentHealth && componentHealth.status === 'healthy'; - }); - - const readinessStatus = { - status: allReady ? 'ready' : 'not-ready', - message: allReady ? 'All critical components are ready' : 'Some critical components are not ready', - timestamp: new Date().toISOString(), - components: Object.fromEntries( - criticalComponents.map(component => [ - component, - components[component]?.status || 'unknown' - ]) - ) - }; - - const statusCode = allReady ? 200 : 503; - return c.json(readinessStatus, statusCode); - } catch (error) { - this.logger.error('Readiness check failed:', error); - return c.json({ - status: 'not-ready', - message: 'Readiness check failed', - timestamp: new Date().toISOString(), - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - } - - async getLiveness(c: Context) { - try { - // Basic liveness check - just verify the service is responding - const uptime = process.uptime(); - const memoryUsage = process.memoryUsage(); - - return c.json({ - status: 'alive', - message: 'Service is alive and responding', - timestamp: new Date().toISOString(), - uptime: Math.floor(uptime), - memory: { - rss: Math.round(memoryUsage.rss / 1024 / 1024), // MB - heapUsed: Math.round(memoryUsage.heapUsed / 1024 / 1024), // MB - heapTotal: Math.round(memoryUsage.heapTotal / 1024 / 1024), // MB - } - }); - } catch (error) { - this.logger.error('Liveness check failed:', error); - return c.json({ - status: 'dead', - message: 'Liveness check failed', - timestamp: new Date().toISOString(), - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - } - - async getComponentHealth(c: Context) { - try { - const component = c.req.param('component'); - const health = this.gatewayService.getHealth(); - const components = health.details?.components || {}; - - if (!components[component]) { - return c.json({ - error: 'Component not found', - availableComponents: Object.keys(components) - }, 404); - } - - return c.json({ - component, - ...components[component], - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Component health check failed:', error); - return c.json({ - error: 'Component health check failed', - message: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getDetailedHealth(c: Context) { - try { - const health = this.gatewayService.getHealth(); - const metrics = this.gatewayService.getMetrics(); - const status = this.gatewayService.getStatus(); - - return c.json({ - health, - metrics, - status, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Detailed health check failed:', error); - return c.json({ - error: 'Detailed health check failed', - message: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } -} diff --git a/apps/core-services/market-data-gateway/src/controllers/MetricsController.ts b/apps/core-services/market-data-gateway/src/controllers/MetricsController.ts deleted file mode 100644 index 9635eb5..0000000 --- a/apps/core-services/market-data-gateway/src/controllers/MetricsController.ts +++ /dev/null @@ -1,330 +0,0 @@ -import { Context } from 'hono'; -import { MetricsCollector } from '../services/MetricsCollector'; -import { Logger } from '../types/MarketDataGateway'; - -export class MetricsController { - constructor( - private metricsCollector: MetricsCollector, - private logger: Logger - ) {} - - async getMetrics(c: Context) { - try { - const format = c.req.query('format') || 'json'; - - if (format === 'prometheus') { - const prometheusMetrics = this.metricsCollector.exportMetrics('prometheus'); - return c.text(prometheusMetrics, 200, { - 'Content-Type': 'text/plain; charset=utf-8' - }); - } - - const metrics = this.metricsCollector.getAggregatedMetrics(); - return c.json(metrics); - } catch (error) { - this.logger.error('Failed to get metrics:', error); - return c.json({ error: 'Failed to get metrics' }, 500); - } - } - - async getMetric(c: Context) { - try { - const metricName = c.req.param('metricName'); - const duration = c.req.query('duration'); - - if (!metricName) { - return c.json({ error: 'Metric name is required' }, 400); - } - - const durationMs = duration ? parseInt(duration) * 1000 : undefined; - const metricData = this.metricsCollector.getMetric(metricName, durationMs); - - return c.json({ - metric: metricName, - duration: duration || 'all', - data: metricData, - count: metricData.length - }); - } catch (error) { - this.logger.error('Failed to get metric:', error); - return c.json({ error: 'Failed to get metric' }, 500); - } - } - - async getMetricAverage(c: Context) { - try { - const metricName = c.req.param('metricName'); - const duration = c.req.query('duration'); - - if (!metricName) { - return c.json({ error: 'Metric name is required' }, 400); - } - - const durationMs = duration ? parseInt(duration) * 1000 : undefined; - const average = this.metricsCollector.getAverageMetric(metricName, durationMs); - - return c.json({ - metric: metricName, - duration: duration || 'all', - average, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to get metric average:', error); - return c.json({ error: 'Failed to get metric average' }, 500); - } - } - - async getMetricLatest(c: Context) { - try { - const metricName = c.req.param('metricName'); - - if (!metricName) { - return c.json({ error: 'Metric name is required' }, 400); - } - - const latest = this.metricsCollector.getLatestMetric(metricName); - - if (latest === null) { - return c.json({ error: 'Metric not found or no data available' }, 404); - } - - return c.json({ - metric: metricName, - value: latest, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to get latest metric:', error); - return c.json({ error: 'Failed to get latest metric' }, 500); - } - } - - async getMetricRate(c: Context) { - try { - const metricName = c.req.param('metricName'); - const duration = c.req.query('duration') || '60'; // Default 60 seconds - - if (!metricName) { - return c.json({ error: 'Metric name is required' }, 400); - } - - const durationMs = parseInt(duration) * 1000; - const rate = this.metricsCollector.getRate(metricName, durationMs); - - return c.json({ - metric: metricName, - duration: duration, - rate: rate, - unit: 'per second', - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to get metric rate:', error); - return c.json({ error: 'Failed to get metric rate' }, 500); - } - } - - async getMetricPercentile(c: Context) { - try { - const metricName = c.req.param('metricName'); - const percentile = c.req.query('percentile') || '95'; - const duration = c.req.query('duration'); - - if (!metricName) { - return c.json({ error: 'Metric name is required' }, 400); - } - - const percentileValue = parseFloat(percentile); - if (isNaN(percentileValue) || percentileValue < 0 || percentileValue > 100) { - return c.json({ error: 'Percentile must be a number between 0 and 100' }, 400); - } - - const durationMs = duration ? parseInt(duration) * 1000 : undefined; - const value = this.metricsCollector.getPercentile(metricName, percentileValue, durationMs); - - return c.json({ - metric: metricName, - percentile: percentileValue, - value, - duration: duration || 'all', - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to get metric percentile:', error); - return c.json({ error: 'Failed to get metric percentile' }, 500); - } - } - - async getAlerts(c: Context) { - try { - const activeOnly = c.req.query('active') === 'true'; - - if (activeOnly) { - const alerts = this.metricsCollector.getActiveAlerts(); - return c.json({ alerts, count: alerts.length }); - } - - const rules = this.metricsCollector.getAlertRules(); - const alerts = this.metricsCollector.getActiveAlerts(); - - return c.json({ - alertRules: rules, - activeAlerts: alerts, - rulesCount: rules.length, - activeCount: alerts.length - }); - } catch (error) { - this.logger.error('Failed to get alerts:', error); - return c.json({ error: 'Failed to get alerts' }, 500); - } - } - - async addAlertRule(c: Context) { - try { - const rule = await c.req.json(); - - // Validate required fields - if (!rule.id || !rule.metric || !rule.condition || rule.threshold === undefined) { - return c.json({ - error: 'Missing required fields: id, metric, condition, threshold' - }, 400); - } - - // Validate condition - const validConditions = ['gt', 'lt', 'eq', 'gte', 'lte']; - if (!validConditions.includes(rule.condition)) { - return c.json({ - error: `Invalid condition. Must be one of: ${validConditions.join(', ')}` - }, 400); - } - - this.metricsCollector.addAlertRule(rule); - - return c.json({ - message: 'Alert rule added successfully', - ruleId: rule.id, - timestamp: new Date().toISOString() - }, 201); - } catch (error) { - this.logger.error('Failed to add alert rule:', error); - return c.json({ error: 'Failed to add alert rule' }, 500); - } - } - - async removeAlertRule(c: Context) { - try { - const ruleId = c.req.param('ruleId'); - - if (!ruleId) { - return c.json({ error: 'Rule ID is required' }, 400); - } - - this.metricsCollector.removeAlertRule(ruleId); - - return c.json({ - message: 'Alert rule removed successfully', - ruleId, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to remove alert rule:', error); - return c.json({ error: 'Failed to remove alert rule' }, 500); - } - } - - async recordCustomMetric(c: Context) { - try { - const { name, value, labels, type = 'gauge' } = await c.req.json(); - - if (!name || value === undefined) { - return c.json({ - error: 'Missing required fields: name, value' - }, 400); - } - - if (typeof value !== 'number') { - return c.json({ - error: 'Value must be a number' - }, 400); - } - - switch (type) { - case 'gauge': - this.metricsCollector.setGauge(name, value, labels); - break; - case 'counter': - this.metricsCollector.incrementCounter(name, value, labels); - break; - case 'histogram': - this.metricsCollector.recordHistogram(name, value, labels); - break; - default: - return c.json({ - error: 'Invalid metric type. Must be one of: gauge, counter, histogram' - }, 400); - } - - return c.json({ - message: 'Custom metric recorded successfully', - name, - value, - type, - timestamp: new Date().toISOString() - }); - } catch (error) { - this.logger.error('Failed to record custom metric:', error); - return c.json({ error: 'Failed to record custom metric' }, 500); - } - } - - async getSystemMetrics(c: Context) { - try { - const process = require('process'); - const uptime = process.uptime(); - const memoryUsage = process.memoryUsage(); - const cpuUsage = process.cpuUsage(); - - const systemMetrics = { - uptime: Math.floor(uptime), - memory: { - rss: Math.round(memoryUsage.rss / 1024 / 1024), // MB - heapUsed: Math.round(memoryUsage.heapUsed / 1024 / 1024), // MB - heapTotal: Math.round(memoryUsage.heapTotal / 1024 / 1024), // MB - external: Math.round(memoryUsage.external / 1024 / 1024), // MB - }, - cpu: { - user: cpuUsage.user, - system: cpuUsage.system, - }, - timestamp: new Date().toISOString() - }; - - return c.json(systemMetrics); - } catch (error) { - this.logger.error('Failed to get system metrics:', error); - return c.json({ error: 'Failed to get system metrics' }, 500); - } - } - - async exportMetrics(c: Context) { - try { - const format = c.req.query('format') || 'json'; - const exported = this.metricsCollector.exportMetrics(format); - - if (format === 'prometheus') { - return c.text(exported, 200, { - 'Content-Type': 'text/plain; charset=utf-8' - }); - } - - return c.text(exported, 200, { - 'Content-Type': 'application/json', - 'Content-Disposition': `attachment; filename="metrics-${new Date().toISOString()}.json"` - }); - } catch (error) { - this.logger.error('Failed to export metrics:', error); - return c.json({ error: 'Failed to export metrics' }, 500); - } - } -} diff --git a/apps/core-services/market-data-gateway/src/index.ts b/apps/core-services/market-data-gateway/src/index.ts deleted file mode 100644 index 0ed49f4..0000000 --- a/apps/core-services/market-data-gateway/src/index.ts +++ /dev/null @@ -1,651 +0,0 @@ -// Market Data Gateway - Enhanced Implementation -import { Hono } from 'hono'; -import { cors } from 'hono/cors'; -import { logger } from 'hono/logger'; -import { prettyJSON } from 'hono/pretty-json'; -import { WebSocketServer } from 'ws'; - -// Types -import { GatewayConfig } from './types/MarketDataGateway'; - -// Services -import { DataNormalizer, DataNormalizationResult } from './services/DataNormalizer'; -import { MarketDataCache } from './services/AdvancedCache'; -import { ConnectionPoolManager } from './services/ConnectionPoolManager'; -import { dataProviderConfigs, getEnabledProviders } from './config/DataProviderConfig'; - -// Simple logger interface -interface Logger { - info: (message: string, ...args: any[]) => void; - error: (message: string, ...args: any[]) => void; - warn: (message: string, ...args: any[]) => void; - debug: (message: string, ...args: any[]) => void; -} - -// Create application logger -const appLogger: Logger = { - info: (message: string, ...args: any[]) => console.log(`[MDG-ENHANCED] [INFO] ${message}`, ...args), - error: (message: string, ...args: any[]) => console.error(`[MDG-ENHANCED] [ERROR] ${message}`, ...args), - warn: (message: string, ...args: any[]) => console.warn(`[MDG-ENHANCED] [WARN] ${message}`, ...args), - debug: (message: string, ...args: any[]) => console.debug(`[MDG-ENHANCED] [DEBUG] ${message}`, ...args), -}; - -// Initialize services -const dataNormalizer = new DataNormalizer(); -const marketDataCache = new MarketDataCache(); -const connectionPool = new ConnectionPoolManager({ - maxConnections: 100, - maxConnectionsPerHost: 20, - connectionTimeout: 10000, - requestTimeout: 30000, - retryAttempts: 3, - retryDelay: 1000, - keepAlive: true, - maxIdleTime: 300000, // 5 minutes -}); - -// Configuration matching the GatewayConfig interface -const config: GatewayConfig = { - server: { - port: parseInt(process.env.PORT || '3004'), - host: process.env.HOST || '0.0.0.0', - maxConnections: 1000, - cors: { - origins: ['http://localhost:3000', 'http://localhost:3001', 'http://localhost:8080'], - methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], - headers: ['Content-Type', 'Authorization'], - }, - }, dataSources: getEnabledProviders().map(provider => ({ - id: provider.name.toLowerCase().replace(/\s+/g, '-'), - name: provider.name, - type: provider.type === 'both' ? 'websocket' : provider.type as any, - provider: provider.name, - enabled: provider.enabled, - priority: provider.priority, - rateLimit: { - requestsPerSecond: provider.rateLimits.requestsPerSecond, - burstLimit: provider.rateLimits.requestsPerMinute, - }, - connection: { - url: provider.endpoints.quotes || provider.endpoints.websocket || '', - authentication: provider.authentication ? { - type: provider.authentication.type === 'api_key' ? 'apikey' as const : 'basic' as const, - credentials: { - apiKey: provider.authentication.key || '', - secret: provider.authentication.secret || '', - token: provider.authentication.token || '', - }, - } : undefined, - }, - subscriptions: { - quotes: true, - trades: true, - orderbook: provider.endpoints.websocket ? true : false, - candles: true, - news: false, - }, - symbols: ['AAPL', 'GOOGL', 'MSFT', 'AMZN', 'TSLA'], // Default symbols - retryPolicy: { - maxRetries: provider.retryPolicy.maxRetries, - backoffMultiplier: provider.retryPolicy.backoffMultiplier, - maxBackoffMs: provider.retryPolicy.initialDelayMs * 10, - }, - healthCheck: { - intervalMs: 30000, - timeoutMs: provider.timeout, - expectedLatencyMs: 1000, - }, - })), - processing: { - pipelines: [], - bufferSize: 10000, - batchSize: 100, - flushIntervalMs: 1000, - }, cache: { - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - password: process.env.REDIS_PASSWORD, - db: parseInt(process.env.REDIS_DB || '0'), - }, - ttl: { - quotes: 60000, // 1 minute - trades: 300000, // 5 minutes - candles: 86400000, // 24 hours - orderbook: 30000, // 30 seconds - }, - }, monitoring: { - metrics: { - enabled: true, - port: parseInt(process.env.METRICS_PORT || '9090'), - intervalMs: 5000, - retention: '24h', - }, - alerts: { - enabled: true, - thresholds: { - latency: 1000, - latencyMs: 1000, - errorRate: 0.05, - connectionLoss: 3, - }, - }, - }, -}; - -// Global state variables -let webSocketServer: WebSocketServer | null = null; -let isShuttingDown = false; - -// Create Hono application -const app = new Hono(); - -// Middleware setup -app.use('*', logger()); -app.use('*', prettyJSON()); -app.use('*', cors({ - origin: config.server.cors.origins, - allowMethods: config.server.cors.methods, - allowHeaders: config.server.cors.headers, -})); - -// Mock data for testing -const mockTickData = { - symbol: 'AAPL', - price: 150.25, - volume: 1000, - timestamp: new Date().toISOString(), - bid: 150.20, - ask: 150.30, -}; - -const mockCandleData = { - symbol: 'AAPL', - timeframe: '1m', - timestamp: new Date().toISOString(), - open: 150.00, - high: 150.50, - low: 149.75, - close: 150.25, - volume: 5000, -}; - -// Health endpoints -app.get('/health', async (c) => { - return c.json({ - status: 'healthy', - timestamp: new Date().toISOString(), - service: 'market-data-gateway', - version: '1.0.0', - }); -}); - -app.get('/health/readiness', async (c) => { - return c.json({ - status: 'ready', - timestamp: new Date().toISOString(), - checks: { - webSocket: webSocketServer ? 'connected' : 'disconnected', - cache: 'available', - }, - }); -}); - -app.get('/health/liveness', async (c) => { - return c.json({ - status: 'alive', - timestamp: new Date().toISOString(), - uptime: process.uptime(), - }); -}); - -// Gateway status endpoints -app.get('/api/v1/gateway/status', async (c) => { - return c.json({ - status: 'running', - dataSources: config.dataSources.length, - activeConnections: webSocketServer ? webSocketServer.clients.size : 0, - timestamp: new Date().toISOString(), - }); -}); - -app.get('/api/v1/gateway/config', async (c) => { - return c.json({ - server: config.server, - dataSourcesCount: config.dataSources.length, - processingConfig: config.processing, - monitoring: config.monitoring, - }); -}); - -// Data source management endpoints -app.get('/api/v1/sources', async (c) => { - return c.json({ - dataSources: config.dataSources, - total: config.dataSources.length, - }); -}); - -app.post('/api/v1/sources', async (c) => { - try { - const newSource = await c.req.json(); - // In a real implementation, validate and add the data source - return c.json({ - message: 'Data source configuration received', - source: newSource, - status: 'pending_validation', - }); - } catch (error) { - return c.json({ error: 'Invalid data source configuration' }, 400); - } -}); - -// Subscription management endpoints -app.get('/api/v1/subscriptions', async (c) => { - return c.json({ - subscriptions: [], - total: 0, - message: 'Subscription management not yet implemented', - }); -}); - -app.post('/api/v1/subscriptions', async (c) => { - try { - const subscription = await c.req.json(); - return c.json({ - subscriptionId: `sub_${Date.now()}`, - status: 'active', - subscription, - }); - } catch (error) { - return c.json({ error: 'Invalid subscription request' }, 400); - } -}); - -// Market data endpoints with enhanced functionality -app.get('/api/v1/data/tick/:symbol', async (c) => { - const symbol = c.req.param('symbol').toUpperCase(); - const source = c.req.query('source') || 'yahoo-finance'; - - try { - // Check cache first - const cacheKey = marketDataCache.getQuoteKey(symbol); - const cachedData = marketDataCache.get(cacheKey); - - if (cachedData) { - appLogger.debug(`Cache hit for ${symbol}`); - return c.json({ - ...cachedData, - cached: true, - timestamp: new Date().toISOString(), - }); - } - - // Fetch from provider - const provider = dataProviderConfigs[source]; - if (!provider || !provider.enabled) { - return c.json({ error: 'Data source not available' }, 400); - } - - // Mock data for now (replace with actual API calls) - const mockData = { - symbol, - price: 150.25 + (Math.random() - 0.5) * 10, - volume: Math.floor(Math.random() * 100000), - timestamp: new Date().toISOString(), - bid: 150.20, - ask: 150.30, - source, - }; - - // Normalize the data - const normalizedResult = dataNormalizer.normalizeMarketData(mockData, source); - - if (!normalizedResult.success) { - return c.json({ error: normalizedResult.error }, 500); - } - - // Cache the result - marketDataCache.setQuote(symbol, normalizedResult.data); - - return c.json({ - ...normalizedResult.data, - cached: false, - processingTimeMs: normalizedResult.processingTimeMs, - }); - - } catch (error) { - appLogger.error(`Error fetching tick data for ${symbol}:`, error); - return c.json({ error: 'Internal server error' }, 500); - } -}); - -app.get('/api/v1/data/candles/:symbol', async (c) => { - const symbol = c.req.param('symbol').toUpperCase(); - const timeframe = c.req.query('timeframe') || '1m'; - const limit = Math.min(parseInt(c.req.query('limit') || '100'), 1000); // Max 1000 - const source = c.req.query('source') || 'yahoo-finance'; - - try { - // Generate cache key - const cacheKey = `candles:${symbol}:${timeframe}:${limit}`; - const cachedData = marketDataCache.get(cacheKey); - - if (cachedData) { - appLogger.debug(`Cache hit for candles ${symbol}:${timeframe}`); - return c.json({ - candles: cachedData, - cached: true, - count: cachedData.length, - }); - } - - // Mock candle data generation (replace with actual API calls) - const candles = Array.from({ length: limit }, (_, i) => { - const timestamp = new Date(Date.now() - i * 60000); - const basePrice = 150 + (Math.random() - 0.5) * 20; - const variation = (Math.random() - 0.5) * 2; - - return { - symbol, - timeframe, - timestamp: timestamp.toISOString(), - open: basePrice + variation, - high: basePrice + variation + Math.random() * 2, - low: basePrice + variation - Math.random() * 2, - close: basePrice + variation + (Math.random() - 0.5), - volume: Math.floor(Math.random() * 10000), - source, - }; - }).reverse(); // Oldest first - - // Normalize OHLCV data - const normalizedResult = dataNormalizer.normalizeOHLCV( - { candles: candles.map(c => ({ ...c, timestamp: new Date(c.timestamp) })) }, - source - ); - - if (!normalizedResult.success) { - return c.json({ error: normalizedResult.error }, 500); - } - - // Cache the result - marketDataCache.set(cacheKey, normalizedResult.data, marketDataCache['getCandleTTL'](timeframe)); - - return c.json({ - candles: normalizedResult.data, - cached: false, - count: normalizedResult.data?.length || 0, - processingTimeMs: normalizedResult.processingTimeMs, - }); - - } catch (error) { - appLogger.error(`Error fetching candles for ${symbol}:`, error); - return c.json({ error: 'Internal server error' }, 500); - } -}); - -// Enhanced metrics endpoints -app.get('/api/v1/metrics', async (c) => { - const cacheStats = marketDataCache.getStats(); - const connectionStats = connectionPool.getStats(); - - return c.json({ - system: { - uptime: process.uptime(), - memoryUsage: process.memoryUsage(), - cpuUsage: process.cpuUsage(), - }, - gateway: { - activeConnections: webSocketServer ? webSocketServer.clients.size : 0, - dataSourcesCount: config.dataSources.filter(ds => ds.enabled).length, - messagesProcessed: 0, - }, - cache: cacheStats, - connectionPool: connectionStats, - timestamp: new Date().toISOString(), - }); -}); - -// Data quality assessment endpoint -app.get('/api/v1/data/quality/:symbol', async (c) => { - const symbol = c.req.param('symbol').toUpperCase(); - const source = c.req.query('source') || 'yahoo-finance'; - - try { - // Get recent data for quality assessment (mock for now) - const recentData = Array.from({ length: 10 }, (_, i) => ({ - symbol, - price: 150 + (Math.random() - 0.5) * 10, - bid: 149.5, - ask: 150.5, - volume: Math.floor(Math.random() * 10000), - timestamp: new Date(Date.now() - i * 60000), - })); - - const qualityMetrics = dataNormalizer.assessDataQuality(recentData, source); - - return c.json({ - symbol, - source, - dataPoints: recentData.length, - qualityMetrics, - timestamp: new Date().toISOString(), - }); - - } catch (error) { - appLogger.error(`Error assessing data quality for ${symbol}:`, error); - return c.json({ error: 'Internal server error' }, 500); - } -}); - -// Cache management endpoints -app.get('/api/v1/cache/stats', async (c) => { - return c.json({ - stats: marketDataCache.getStats(), - keys: marketDataCache.keys().slice(0, 100), // Limit to first 100 keys - timestamp: new Date().toISOString(), - }); -}); - -app.delete('/api/v1/cache/clear', async (c) => { - marketDataCache.clear(); - return c.json({ - message: 'Cache cleared successfully', - timestamp: new Date().toISOString(), - }); -}); - -app.delete('/api/v1/cache/key/:key', async (c) => { - const key = c.req.param('key'); - const deleted = marketDataCache.delete(key); - - return c.json({ - message: deleted ? 'Key deleted successfully' : 'Key not found', - key, - deleted, - timestamp: new Date().toISOString(), - }); -}); - -// Data providers status endpoint -app.get('/api/v1/providers', async (c) => { - const providers = Object.values(dataProviderConfigs).map(provider => ({ - name: provider.name, - enabled: provider.enabled, - type: provider.type, - priority: provider.priority, - rateLimits: provider.rateLimits, - endpoints: Object.keys(provider.endpoints), - })); - - return c.json({ - providers, - enabled: providers.filter(p => p.enabled).length, - total: providers.length, - timestamp: new Date().toISOString(), - }); -}); - -// WebSocket server setup -function setupWebSocketServer(): void { - const wsPort = config.server.port + 1; // Use port + 1 for WebSocket - - webSocketServer = new WebSocketServer({ - port: wsPort, - perMessageDeflate: false, - }); - - webSocketServer.on('connection', (ws, request) => { - appLogger.info(`New WebSocket connection from ${request.socket.remoteAddress}`); - - ws.on('message', async (message) => { - try { - const data = JSON.parse(message.toString()); - - switch (data.type) { - case 'subscribe': - if (data.symbols && Array.isArray(data.symbols)) { - const subscriptionId = `sub_${Date.now()}`; - ws.send(JSON.stringify({ - type: 'subscription_confirmed', - subscriptionId, - symbols: data.symbols, - timestamp: new Date().toISOString(), - })); - - // Send mock data every 5 seconds - const interval = setInterval(() => { - if (ws.readyState === ws.OPEN) { - data.symbols.forEach((symbol: string) => { - ws.send(JSON.stringify({ - type: 'tick', - data: { - ...mockTickData, - symbol: symbol.toUpperCase(), - price: mockTickData.price + (Math.random() - 0.5) * 2, - timestamp: new Date().toISOString(), - }, - })); - }); - } else { - clearInterval(interval); - } - }, 5000); - } - break; - - case 'unsubscribe': - if (data.subscriptionId) { - ws.send(JSON.stringify({ - type: 'unsubscription_confirmed', - subscriptionId: data.subscriptionId, - timestamp: new Date().toISOString(), - })); - } - break; - - default: - ws.send(JSON.stringify({ - type: 'error', - message: 'Unknown message type', - })); - } - } catch (error) { - ws.send(JSON.stringify({ - type: 'error', - message: 'Invalid message format', - })); - } - }); - - ws.on('close', () => { - appLogger.info('WebSocket connection closed'); - }); - - ws.on('error', (error) => { - appLogger.error('WebSocket error:', error); - }); - }); - - appLogger.info(`WebSocket server listening on port ${wsPort}`); -} - -// Enhanced graceful shutdown handler -async function gracefulShutdown(): Promise { - if (isShuttingDown) return; - isShuttingDown = true; - - appLogger.info('Initiating graceful shutdown...'); - - try { - // Close WebSocket server - if (webSocketServer) { - webSocketServer.clients.forEach((client) => { - client.terminate(); - }); - webSocketServer.close(); - appLogger.info('WebSocket server closed'); - } - - // Close connection pool - await connectionPool.close(); - appLogger.info('Connection pool closed'); - - // Clean up cache - marketDataCache.destroy(); - appLogger.info('Cache destroyed'); - - appLogger.info('Graceful shutdown completed'); - process.exit(0); - } catch (error) { - appLogger.error('Error during shutdown:', error); - process.exit(1); - } -} - -// Enhanced start server function -async function startServer(): Promise { - try { - appLogger.info('Starting Enhanced Market Data Gateway...'); - - // Initialize cache event listeners - marketDataCache.on('hit', (key) => appLogger.debug(`Cache hit: ${key}`)); - marketDataCache.on('miss', (key) => appLogger.debug(`Cache miss: ${key}`)); - marketDataCache.on('evict', (key) => appLogger.debug(`Cache evict: ${key}`)); - - // Initialize connection pool event listeners - connectionPool.on('connectionCreated', (host) => appLogger.debug(`Connection created for: ${host}`)); - connectionPool.on('error', ({ host, error }) => appLogger.warn(`Connection error for ${host}: ${error}`)); - - // Setup WebSocket server - setupWebSocketServer(); - - // Setup graceful shutdown handlers - process.on('SIGTERM', gracefulShutdown); - process.on('SIGINT', gracefulShutdown); - - // Log service status - appLogger.info(`HTTP server starting on ${config.server.host}:${config.server.port}`); - appLogger.info(`WebSocket server running on port ${config.server.port + 1}`); - appLogger.info(`Data sources configured: ${config.dataSources.length}`); - appLogger.info(`Enabled providers: ${config.dataSources.filter(ds => ds.enabled).length}`); - appLogger.info(`Cache max size: ${marketDataCache['config'].maxSize}`); - appLogger.info(`Connection pool max connections: ${connectionPool['config'].maxConnections}`); - appLogger.info('Enhanced Market Data Gateway started successfully'); - - } catch (error) { - appLogger.error('Failed to start server:', error); - process.exit(1); - } -} - -// Start the application -if (require.main === module) { - startServer(); -} - -export default { - port: config.server.port, - fetch: app.fetch, -}; diff --git a/apps/core-services/market-data-gateway/src/index_clean.ts b/apps/core-services/market-data-gateway/src/index_clean.ts deleted file mode 100644 index 126d00e..0000000 --- a/apps/core-services/market-data-gateway/src/index_clean.ts +++ /dev/null @@ -1,386 +0,0 @@ -// Market Data Gateway - Unified Implementation -import { Hono } from 'hono'; -import { cors } from 'hono/cors'; -import { logger } from 'hono/logger'; -import { prettyJSON } from 'hono/pretty-json'; -import { WebSocketServer } from 'ws'; - -// Types -import { GatewayConfig } from './types/MarketDataGateway'; - -// Simple logger interface -interface Logger { - info: (message: string, ...args: any[]) => void; - error: (message: string, ...args: any[]) => void; - warn: (message: string, ...args: any[]) => void; - debug: (message: string, ...args: any[]) => void; -} - -// Create application logger -const appLogger: Logger = { - info: (message: string, ...args: any[]) => console.log(`[MDG-UNIFIED] [INFO] ${message}`, ...args), - error: (message: string, ...args: any[]) => console.error(`[MDG-UNIFIED] [ERROR] ${message}`, ...args), - warn: (message: string, ...args: any[]) => console.warn(`[MDG-UNIFIED] [WARN] ${message}`, ...args), - debug: (message: string, ...args: any[]) => console.debug(`[MDG-UNIFIED] [DEBUG] ${message}`, ...args), -}; - -// Configuration matching the GatewayConfig interface -const config: GatewayConfig = { - server: { - port: parseInt(process.env.PORT || '3004'), - host: process.env.HOST || '0.0.0.0', - maxConnections: 1000, - cors: { - origins: ['http://localhost:3000', 'http://localhost:3001', 'http://localhost:8080'], - methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], - headers: ['Content-Type', 'Authorization'], - }, - }, - dataSources: [], // Array of DataSourceConfig, initially empty - processing: { - pipelines: [], - bufferSize: 10000, - batchSize: 100, - flushIntervalMs: 1000, - }, cache: { - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - password: process.env.REDIS_PASSWORD, - db: parseInt(process.env.REDIS_DB || '0'), - }, - ttl: { - quotes: 60000, // 1 minute - trades: 300000, // 5 minutes - candles: 86400000, // 24 hours - orderbook: 30000, // 30 seconds - }, - }, monitoring: { - metrics: { - enabled: true, - port: parseInt(process.env.METRICS_PORT || '9090'), - intervalMs: 5000, - retention: '24h', - }, - alerts: { - enabled: true, - thresholds: { - latency: 1000, - latencyMs: 1000, - errorRate: 0.05, - connectionLoss: 3, - }, - }, - }, -}; - -// Global state variables -let webSocketServer: WebSocketServer | null = null; -let isShuttingDown = false; - -// Create Hono application -const app = new Hono(); - -// Middleware setup -app.use('*', logger()); -app.use('*', prettyJSON()); -app.use('*', cors({ - origin: config.server.cors.origins, - allowMethods: config.server.cors.methods, - allowHeaders: config.server.cors.headers, -})); - -// Mock data for testing -const mockTickData = { - symbol: 'AAPL', - price: 150.25, - volume: 1000, - timestamp: new Date().toISOString(), - bid: 150.20, - ask: 150.30, -}; - -const mockCandleData = { - symbol: 'AAPL', - timeframe: '1m', - timestamp: new Date().toISOString(), - open: 150.00, - high: 150.50, - low: 149.75, - close: 150.25, - volume: 5000, -}; - -// Health endpoints -app.get('/health', async (c) => { - return c.json({ - status: 'healthy', - timestamp: new Date().toISOString(), - service: 'market-data-gateway', - version: '1.0.0', - }); -}); - -app.get('/health/readiness', async (c) => { - return c.json({ - status: 'ready', - timestamp: new Date().toISOString(), - checks: { - webSocket: webSocketServer ? 'connected' : 'disconnected', - cache: 'available', - }, - }); -}); - -app.get('/health/liveness', async (c) => { - return c.json({ - status: 'alive', - timestamp: new Date().toISOString(), - uptime: process.uptime(), - }); -}); - -// Gateway status endpoints -app.get('/api/v1/gateway/status', async (c) => { - return c.json({ - status: 'running', - dataSources: config.dataSources.length, - activeConnections: webSocketServer ? webSocketServer.clients.size : 0, - timestamp: new Date().toISOString(), - }); -}); - -app.get('/api/v1/gateway/config', async (c) => { - return c.json({ - server: config.server, - dataSourcesCount: config.dataSources.length, - processingConfig: config.processing, - monitoring: config.monitoring, - }); -}); - -// Data source management endpoints -app.get('/api/v1/sources', async (c) => { - return c.json({ - dataSources: config.dataSources, - total: config.dataSources.length, - }); -}); - -app.post('/api/v1/sources', async (c) => { - try { - const newSource = await c.req.json(); - // In a real implementation, validate and add the data source - return c.json({ - message: 'Data source configuration received', - source: newSource, - status: 'pending_validation', - }); - } catch (error) { - return c.json({ error: 'Invalid data source configuration' }, 400); - } -}); - -// Subscription management endpoints -app.get('/api/v1/subscriptions', async (c) => { - return c.json({ - subscriptions: [], - total: 0, - message: 'Subscription management not yet implemented', - }); -}); - -app.post('/api/v1/subscriptions', async (c) => { - try { - const subscription = await c.req.json(); - return c.json({ - subscriptionId: `sub_${Date.now()}`, - status: 'active', - subscription, - }); - } catch (error) { - return c.json({ error: 'Invalid subscription request' }, 400); - } -}); - -// Market data endpoints -app.get('/api/v1/data/tick/:symbol', async (c) => { - const symbol = c.req.param('symbol'); - return c.json({ - ...mockTickData, - symbol: symbol.toUpperCase(), - }); -}); - -app.get('/api/v1/data/candles/:symbol', async (c) => { - const symbol = c.req.param('symbol'); - const timeframe = c.req.query('timeframe') || '1m'; - const limit = parseInt(c.req.query('limit') || '100'); - - const candles = Array.from({ length: limit }, (_, i) => ({ - ...mockCandleData, - symbol: symbol.toUpperCase(), - timeframe, - timestamp: new Date(Date.now() - i * 60000).toISOString(), - })); - - return c.json({ candles }); -}); - -// Metrics endpoints -app.get('/api/v1/metrics', async (c) => { - return c.json({ - system: { - uptime: process.uptime(), - memoryUsage: process.memoryUsage(), - cpuUsage: process.cpuUsage(), - }, - gateway: { - activeConnections: webSocketServer ? webSocketServer.clients.size : 0, - dataSourcesCount: config.dataSources.length, - messagesProcessed: 0, - }, - timestamp: new Date().toISOString(), - }); -}); - -// WebSocket server setup -function setupWebSocketServer(): void { - const wsPort = config.server.port + 1; // Use port + 1 for WebSocket - - webSocketServer = new WebSocketServer({ - port: wsPort, - perMessageDeflate: false, - }); - - webSocketServer.on('connection', (ws, request) => { - appLogger.info(`New WebSocket connection from ${request.socket.remoteAddress}`); - - ws.on('message', async (message) => { - try { - const data = JSON.parse(message.toString()); - - switch (data.type) { - case 'subscribe': - if (data.symbols && Array.isArray(data.symbols)) { - const subscriptionId = `sub_${Date.now()}`; - ws.send(JSON.stringify({ - type: 'subscription_confirmed', - subscriptionId, - symbols: data.symbols, - timestamp: new Date().toISOString(), - })); - - // Send mock data every 5 seconds - const interval = setInterval(() => { - if (ws.readyState === ws.OPEN) { - data.symbols.forEach((symbol: string) => { - ws.send(JSON.stringify({ - type: 'tick', - data: { - ...mockTickData, - symbol: symbol.toUpperCase(), - price: mockTickData.price + (Math.random() - 0.5) * 2, - timestamp: new Date().toISOString(), - }, - })); - }); - } else { - clearInterval(interval); - } - }, 5000); - } - break; - - case 'unsubscribe': - if (data.subscriptionId) { - ws.send(JSON.stringify({ - type: 'unsubscription_confirmed', - subscriptionId: data.subscriptionId, - timestamp: new Date().toISOString(), - })); - } - break; - - default: - ws.send(JSON.stringify({ - type: 'error', - message: 'Unknown message type', - })); - } - } catch (error) { - ws.send(JSON.stringify({ - type: 'error', - message: 'Invalid message format', - })); - } - }); - - ws.on('close', () => { - appLogger.info('WebSocket connection closed'); - }); - - ws.on('error', (error) => { - appLogger.error('WebSocket error:', error); - }); - }); - - appLogger.info(`WebSocket server listening on port ${wsPort}`); -} - -// Graceful shutdown handler -async function gracefulShutdown(): Promise { - if (isShuttingDown) return; - isShuttingDown = true; - - appLogger.info('Initiating graceful shutdown...'); - - try { - // Close WebSocket server - if (webSocketServer) { - webSocketServer.clients.forEach((client) => { - client.terminate(); - }); - webSocketServer.close(); - appLogger.info('WebSocket server closed'); - } - - appLogger.info('Graceful shutdown completed'); - process.exit(0); - } catch (error) { - appLogger.error('Error during shutdown:', error); - process.exit(1); - } -} - -// Start server function -async function startServer(): Promise { - try { - appLogger.info('Starting Market Data Gateway...'); - - // Setup WebSocket server - setupWebSocketServer(); - - // Setup graceful shutdown handlers - process.on('SIGTERM', gracefulShutdown); - process.on('SIGINT', gracefulShutdown); - - appLogger.info(`HTTP server starting on ${config.server.host}:${config.server.port}`); - appLogger.info(`WebSocket server running on port ${config.server.port + 1}`); - appLogger.info('Market Data Gateway started successfully'); - - } catch (error) { - appLogger.error('Failed to start server:', error); - process.exit(1); - } -} - -// Start the application -if (require.main === module) { - startServer(); -} - -export default { - port: config.server.port, - fetch: app.fetch, -}; diff --git a/apps/core-services/market-data-gateway/src/realtime/index.ts b/apps/core-services/market-data-gateway/src/realtime/index.ts deleted file mode 100644 index d21ca34..0000000 --- a/apps/core-services/market-data-gateway/src/realtime/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -// Real-time market data processing components -export { SubscriptionManager } from '../services/SubscriptionManager'; -export { DataSourceManager } from '../services/DataSourceManager'; -export { EventPublisher } from '../services/EventPublisher'; -export { ProcessingEngine } from '../services/ProcessingEngine'; -export { MarketDataService } from '../services/MarketDataService'; -export { MarketDataGatewayService } from '../services/MarketDataGatewayService'; diff --git a/apps/core-services/market-data-gateway/src/services/AdvancedCache.ts b/apps/core-services/market-data-gateway/src/services/AdvancedCache.ts deleted file mode 100644 index e52bb34..0000000 --- a/apps/core-services/market-data-gateway/src/services/AdvancedCache.ts +++ /dev/null @@ -1,361 +0,0 @@ -import { EventEmitter } from 'events'; - -export interface CacheEntry { - data: T; - timestamp: number; - ttl: number; - hits: number; - lastAccessed: number; -} - -export interface CacheStats { - totalEntries: number; - memoryUsage: number; - hitRate: number; - totalHits: number; - totalMisses: number; - averageAccessTime: number; -} - -export interface CacheConfig { - maxSize: number; - defaultTtl: number; - cleanupInterval: number; - enableStats: boolean; - compressionEnabled: boolean; -} - -export class AdvancedCache extends EventEmitter { - private cache = new Map>(); - private stats = { - hits: 0, - misses: 0, - totalAccessTime: 0, - accessCount: 0, - }; - private cleanupTimer: NodeJS.Timeout | null = null; - - constructor(private config: CacheConfig) { - super(); - this.startCleanupTimer(); - } - - /** - * Get value from cache - */ - get(key: string): T | null { - const startTime = Date.now(); - const entry = this.cache.get(key); - - if (!entry) { - this.stats.misses++; - this.emit('miss', key); - return null; - } - - // Check if entry has expired - if (Date.now() > entry.timestamp + entry.ttl) { - this.cache.delete(key); - this.stats.misses++; - this.emit('expired', key, entry); - return null; - } - - // Update access statistics - entry.hits++; - entry.lastAccessed = Date.now(); - this.stats.hits++; - - if (this.config.enableStats) { - this.stats.totalAccessTime += Date.now() - startTime; - this.stats.accessCount++; - } - - this.emit('hit', key, entry); - return entry.data; - } - - /** - * Set value in cache - */ - set(key: string, value: T, ttl?: number): void { - const effectiveTtl = ttl || this.config.defaultTtl; - - // Check cache size limits - if (this.cache.size >= this.config.maxSize && !this.cache.has(key)) { - this.evictLeastUsed(); - } - - const entry: CacheEntry = { - data: value, - timestamp: Date.now(), - ttl: effectiveTtl, - hits: 0, - lastAccessed: Date.now(), - }; - - this.cache.set(key, entry); - this.emit('set', key, entry); - } - - /** - * Delete value from cache - */ - delete(key: string): boolean { - const deleted = this.cache.delete(key); - if (deleted) { - this.emit('delete', key); - } - return deleted; - } - - /** - * Check if key exists in cache - */ - has(key: string): boolean { - const entry = this.cache.get(key); - if (!entry) return false; - - // Check if expired - if (Date.now() > entry.timestamp + entry.ttl) { - this.cache.delete(key); - return false; - } - - return true; - } - - /** - * Clear all cache entries - */ - clear(): void { - this.cache.clear(); - this.resetStats(); - this.emit('clear'); - } - - /** - * Get cache statistics - */ - getStats(): CacheStats { - const memoryUsage = this.estimateMemoryUsage(); - const hitRate = this.stats.hits + this.stats.misses > 0 - ? this.stats.hits / (this.stats.hits + this.stats.misses) - : 0; - const averageAccessTime = this.stats.accessCount > 0 - ? this.stats.totalAccessTime / this.stats.accessCount - : 0; - - return { - totalEntries: this.cache.size, - memoryUsage, - hitRate, - totalHits: this.stats.hits, - totalMisses: this.stats.misses, - averageAccessTime, - }; - } - - /** - * Get all cache keys - */ - keys(): string[] { - return Array.from(this.cache.keys()); - } - - /** - * Get cache size - */ - size(): number { - return this.cache.size; - } - - /** - * Get or set with async loader function - */ - async getOrSet( - key: string, - loader: () => Promise, - ttl?: number - ): Promise { - const cached = this.get(key) as K; - if (cached !== null) { - return cached; - } - - try { - const value = await loader(); - this.set(key, value as any, ttl); - return value; - } catch (error) { - this.emit('error', key, error); - throw error; - } - } - - /** - * Batch get multiple keys - */ - mget(keys: string[]): Map { - const result = new Map(); - for (const key of keys) { - result.set(key, this.get(key)); - } - return result; - } - - /** - * Batch set multiple key-value pairs - */ - mset(entries: Map, ttl?: number): void { - for (const [key, value] of entries) { - this.set(key, value, ttl); - } - } - - /** - * Clean up expired entries - */ - cleanup(): number { - const now = Date.now(); - let removedCount = 0; - - for (const [key, entry] of this.cache.entries()) { - if (now > entry.timestamp + entry.ttl) { - this.cache.delete(key); - removedCount++; - this.emit('expired', key, entry); - } - } - - return removedCount; - } - - /** - * Evict least recently used entries - */ - private evictLeastUsed(): void { - let oldestKey: string | null = null; - let oldestTime = Date.now(); - - for (const [key, entry] of this.cache.entries()) { - if (entry.lastAccessed < oldestTime) { - oldestTime = entry.lastAccessed; - oldestKey = key; - } - } - - if (oldestKey) { - this.cache.delete(oldestKey); - this.emit('evict', oldestKey); - } - } - - /** - * Estimate memory usage in bytes - */ - private estimateMemoryUsage(): number { - let totalSize = 0; - - for (const [key, entry] of this.cache.entries()) { - // Rough estimation: key size + data size (as JSON string) - totalSize += key.length * 2; // UTF-16 encoding - totalSize += JSON.stringify(entry.data).length * 2; - totalSize += 64; // Overhead for entry metadata - } - - return totalSize; - } - - /** - * Reset statistics - */ - private resetStats(): void { - this.stats = { - hits: 0, - misses: 0, - totalAccessTime: 0, - accessCount: 0, - }; - } - - /** - * Start cleanup timer - */ - private startCleanupTimer(): void { - if (this.cleanupTimer) { - clearInterval(this.cleanupTimer); - } - - this.cleanupTimer = setInterval(() => { - const removed = this.cleanup(); - if (removed > 0) { - this.emit('cleanup', removed); - } - }, this.config.cleanupInterval); - } - - /** - * Stop cleanup timer and close cache - */ - destroy(): void { - if (this.cleanupTimer) { - clearInterval(this.cleanupTimer); - this.cleanupTimer = null; - } - this.clear(); - this.removeAllListeners(); - } -} - -// Specialized cache for market data -export class MarketDataCache extends AdvancedCache { - constructor() { - super({ - maxSize: 10000, - defaultTtl: 60000, // 1 minute - cleanupInterval: 30000, // 30 seconds - enableStats: true, - compressionEnabled: false, - }); - } - - // Market data specific cache keys - getQuoteKey(symbol: string): string { - return `quote:${symbol}`; - } - - getCandleKey(symbol: string, timeframe: string, timestamp: Date): string { - return `candle:${symbol}:${timeframe}:${timestamp.getTime()}`; - } - - getOrderBookKey(symbol: string): string { - return `orderbook:${symbol}`; - } - - // Market data specific TTLs - setQuote(symbol: string, data: any): void { - this.set(this.getQuoteKey(symbol), data, 60000); // 1 minute - } - - setCandle(symbol: string, timeframe: string, timestamp: Date, data: any): void { - const ttl = this.getCandleTTL(timeframe); - this.set(this.getCandleKey(symbol, timeframe, timestamp), data, ttl); - } - - setOrderBook(symbol: string, data: any): void { - this.set(this.getOrderBookKey(symbol), data, 30000); // 30 seconds - } - - private getCandleTTL(timeframe: string): number { - const ttlMap: Record = { - '1m': 60000, // 1 minute - '5m': 300000, // 5 minutes - '15m': 900000, // 15 minutes - '1h': 3600000, // 1 hour - '1d': 86400000, // 24 hours - }; - - return ttlMap[timeframe] || 300000; // Default 5 minutes - } -} diff --git a/apps/core-services/market-data-gateway/src/services/CacheManager.ts b/apps/core-services/market-data-gateway/src/services/CacheManager.ts deleted file mode 100644 index 03fd8da..0000000 --- a/apps/core-services/market-data-gateway/src/services/CacheManager.ts +++ /dev/null @@ -1,372 +0,0 @@ -import Redis from 'ioredis'; -import { EventEmitter } from 'events'; -import { - MarketDataTick, - MarketDataCandle, - CacheConfig, - Logger, - HealthStatus, - GatewayMetrics -} from '../types/MarketDataGateway'; - -interface CacheMetrics { - hits: number; - misses: number; - sets: number; - deletes: number; - errors: number; - totalRequests: number; - hitRate: number; -} - -interface CacheEntry { - data: T; - timestamp: number; - ttl: number; -} - -export class CacheManager extends EventEmitter { - private redis: Redis; - private config: CacheConfig; - private logger: Logger; - private metrics: CacheMetrics; - private isInitialized: boolean = false; - - constructor(config: CacheConfig, logger: Logger) { - super(); - this.config = config; - this.logger = logger; - this.redis = new Redis({ - host: config.redis.host, - port: config.redis.port, - password: config.redis.password, - db: config.redis.database || 0, - retryDelayOnFailover: 100, - maxRetriesPerRequest: 3, - lazyConnect: true, - }); - - this.metrics = { - hits: 0, - misses: 0, - sets: 0, - deletes: 0, - errors: 0, - totalRequests: 0, - hitRate: 0, - }; - - this.setupEventHandlers(); - } - - private setupEventHandlers(): void { - this.redis.on('connect', () => { - this.logger.info('Cache manager connected to Redis'); - this.isInitialized = true; - this.emit('connected'); - }); - - this.redis.on('error', (error) => { - this.logger.error('Redis connection error:', error); - this.metrics.errors++; - this.emit('error', error); - }); - - this.redis.on('close', () => { - this.logger.warn('Redis connection closed'); - this.isInitialized = false; - this.emit('disconnected'); - }); - } - - async initialize(): Promise { - try { - await this.redis.connect(); - this.logger.info('Cache manager initialized successfully'); - } catch (error) { - this.logger.error('Failed to initialize cache manager:', error); - throw error; - } - } - - async shutdown(): Promise { - try { - await this.redis.quit(); - this.logger.info('Cache manager shut down successfully'); - } catch (error) { - this.logger.error('Error shutting down cache manager:', error); - } - } - - // Market data caching methods - async cacheTick(symbol: string, tick: MarketDataTick, ttl?: number): Promise { - try { - const key = this.getTickKey(symbol); - const cacheEntry: CacheEntry = { - data: tick, - timestamp: Date.now(), - ttl: ttl || this.config.tickTtl, - }; - - await this.redis.setex(key, cacheEntry.ttl, JSON.stringify(cacheEntry)); - this.metrics.sets++; - - // Also cache latest price for quick access - await this.redis.setex( - this.getLatestPriceKey(symbol), - this.config.tickTtl, - tick.price.toString() - ); - - this.emit('tick-cached', { symbol, tick }); - } catch (error) { - this.logger.error(`Failed to cache tick for ${symbol}:`, error); - this.metrics.errors++; - throw error; - } - } - - async getLatestTick(symbol: string): Promise { - try { - this.metrics.totalRequests++; - const key = this.getTickKey(symbol); - const cached = await this.redis.get(key); - - if (cached) { - this.metrics.hits++; - const entry: CacheEntry = JSON.parse(cached); - return entry.data; - } - - this.metrics.misses++; - return null; - } catch (error) { - this.logger.error(`Failed to get latest tick for ${symbol}:`, error); - this.metrics.errors++; - return null; - } finally { - this.updateHitRate(); - } - } - - async cacheCandle(symbol: string, timeframe: string, candle: MarketDataCandle, ttl?: number): Promise { - try { - const key = this.getCandleKey(symbol, timeframe, candle.timestamp); - const cacheEntry: CacheEntry = { - data: candle, - timestamp: Date.now(), - ttl: ttl || this.config.candleTtl, - }; - - await this.redis.setex(key, cacheEntry.ttl, JSON.stringify(cacheEntry)); - this.metrics.sets++; - - // Also add to sorted set for range queries - await this.redis.zadd( - this.getCandleSetKey(symbol, timeframe), - candle.timestamp, - key - ); - - this.emit('candle-cached', { symbol, timeframe, candle }); - } catch (error) { - this.logger.error(`Failed to cache candle for ${symbol}:`, error); - this.metrics.errors++; - throw error; - } - } - - async getCandleRange( - symbol: string, - timeframe: string, - startTime: number, - endTime: number - ): Promise { - try { - this.metrics.totalRequests++; - const setKey = this.getCandleSetKey(symbol, timeframe); - const candleKeys = await this.redis.zrangebyscore(setKey, startTime, endTime); - - if (candleKeys.length === 0) { - this.metrics.misses++; - return []; - } - - const pipeline = this.redis.pipeline(); - candleKeys.forEach(key => pipeline.get(key)); - const results = await pipeline.exec(); - - const candles: MarketDataCandle[] = []; - let hasData = false; - - results?.forEach((result) => { - if (result && result[1]) { - hasData = true; - try { - const entry: CacheEntry = JSON.parse(result[1] as string); - candles.push(entry.data); - } catch (parseError) { - this.logger.error('Failed to parse cached candle:', parseError); - } - } - }); - - if (hasData) { - this.metrics.hits++; - } else { - this.metrics.misses++; - } - - return candles.sort((a, b) => a.timestamp - b.timestamp); - } catch (error) { - this.logger.error(`Failed to get candle range for ${symbol}:`, error); - this.metrics.errors++; - return []; - } finally { - this.updateHitRate(); - } - } - - // Generic caching methods - async set(key: string, value: T, ttl?: number): Promise { - try { - const cacheEntry: CacheEntry = { - data: value, - timestamp: Date.now(), - ttl: ttl || this.config.defaultTtl, - }; - - await this.redis.setex(key, cacheEntry.ttl, JSON.stringify(cacheEntry)); - this.metrics.sets++; - } catch (error) { - this.logger.error(`Failed to set cache key ${key}:`, error); - this.metrics.errors++; - throw error; - } - } - - async get(key: string): Promise { - try { - this.metrics.totalRequests++; - const cached = await this.redis.get(key); - - if (cached) { - this.metrics.hits++; - const entry: CacheEntry = JSON.parse(cached); - return entry.data; - } - - this.metrics.misses++; - return null; - } catch (error) { - this.logger.error(`Failed to get cache key ${key}:`, error); - this.metrics.errors++; - return null; - } finally { - this.updateHitRate(); - } - } - - async delete(key: string): Promise { - try { - await this.redis.del(key); - this.metrics.deletes++; - } catch (error) { - this.logger.error(`Failed to delete cache key ${key}:`, error); - this.metrics.errors++; - throw error; - } - } - - async deletePattern(pattern: string): Promise { - try { - const keys = await this.redis.keys(pattern); - if (keys.length > 0) { - const deleted = await this.redis.del(...keys); - this.metrics.deletes += deleted; - return deleted; - } - return 0; - } catch (error) { - this.logger.error(`Failed to delete pattern ${pattern}:`, error); - this.metrics.errors++; - throw error; - } - } - - // Cache management - async flush(): Promise { - try { - await this.redis.flushdb(); - this.logger.info('Cache flushed successfully'); - } catch (error) { - this.logger.error('Failed to flush cache:', error); - this.metrics.errors++; - throw error; - } - } - - async getSize(): Promise { - try { - return await this.redis.dbsize(); - } catch (error) { - this.logger.error('Failed to get cache size:', error); - return 0; - } - } - - async getMemoryUsage(): Promise<{ used: number; peak: number }> { - try { - const info = await this.redis.memory('usage'); - const stats = await this.redis.memory('stats'); - - return { - used: parseInt(info as string) || 0, - peak: parseInt(stats['peak.allocated'] as string) || 0, - }; - } catch (error) { - this.logger.error('Failed to get memory usage:', error); - return { used: 0, peak: 0 }; - } - } - - // Health and metrics - getHealth(): HealthStatus { - return { - status: this.isInitialized ? 'healthy' : 'unhealthy', - message: this.isInitialized ? 'Cache manager is operational' : 'Cache manager not connected', - timestamp: new Date().toISOString(), - details: { - connected: this.isInitialized, - metrics: this.metrics, - }, - }; - } - - getMetrics(): CacheMetrics { - return { ...this.metrics }; - } - - private updateHitRate(): void { - this.metrics.hitRate = this.metrics.totalRequests > 0 - ? this.metrics.hits / this.metrics.totalRequests - : 0; - } - - // Key generation methods - private getTickKey(symbol: string): string { - return `tick:${symbol}:latest`; - } - - private getLatestPriceKey(symbol: string): string { - return `price:${symbol}:latest`; - } - - private getCandleKey(symbol: string, timeframe: string, timestamp: number): string { - return `candle:${symbol}:${timeframe}:${timestamp}`; - } - - private getCandleSetKey(symbol: string, timeframe: string): string { - return `candles:${symbol}:${timeframe}`; - } -} diff --git a/apps/core-services/market-data-gateway/src/services/ConnectionPoolManager.ts b/apps/core-services/market-data-gateway/src/services/ConnectionPoolManager.ts deleted file mode 100644 index 97e1773..0000000 --- a/apps/core-services/market-data-gateway/src/services/ConnectionPoolManager.ts +++ /dev/null @@ -1,346 +0,0 @@ -import { EventEmitter } from 'eventemitter3'; -import { - BunHttpClient, - RequestConfig, - HttpResponse, - ConnectionStats, - HttpClientConfig -} from '@stock-bot/http-client'; - -export interface ConnectionPoolConfig { - maxConnections: number; - maxConnectionsPerHost: number; - connectionTimeout: number; - requestTimeout: number; - retryAttempts: number; - retryDelay: number; - keepAlive: boolean; - maxIdleTime: number; -} - -export interface QueuedRequest { - id: string; - config: RequestConfig; - resolve: (value: any) => void; - reject: (error: any) => void; - timestamp: number; - retryCount: number; -} - -export class ConnectionPoolManager extends EventEmitter { - private clients = new Map(); - private activeRequests = new Map(); // host -> count - private requestQueue: QueuedRequest[] = []; - private stats = { - totalConnections: 0, - successfulRequests: 0, - failedRequests: 0, - totalResponseTime: 0, - requestCount: 0, - }; - private isProcessingQueue = false; - private queueProcessor?: NodeJS.Timeout; - - constructor(private config: ConnectionPoolConfig) { - super(); - this.startQueueProcessor(); - } - - /** - * Get or create a client for a host - */ - private getClient(host: string): BunHttpClient { - if (!this.clients.has(host)) { - const client = new BunHttpClient({ - baseURL: `https://${host}`, - timeout: this.config.requestTimeout, - retries: this.config.retryAttempts, - retryDelay: this.config.retryDelay, - keepAlive: this.config.keepAlive, - headers: { - 'User-Agent': 'StockBot-MarketDataGateway/1.0', - 'Accept': 'application/json', - }, - validateStatus: (status: number) => status < 500, - }); - - // Listen for events from the client - client.on('response', (data) => { - const responseTime = data.response.timing.duration; - this.updateStats(true, responseTime); - this.emit('response', { - host, - responseTime, - status: data.response.status - }); - }); - - client.on('error', (data) => { - const responseTime = data.error?.config?.metadata?.startTime - ? Date.now() - data.error.config.metadata.startTime - : 0; - - this.updateStats(false, responseTime); - this.emit('error', { - host, - error: data.error.message, - responseTime - }); - }); - - this.clients.set(host, client); - this.activeRequests.set(host, 0); - this.stats.totalConnections++; - - this.emit('connectionCreated', host); - } - - return this.clients.get(host)!; - } - - /** - * Make an HTTP request with connection pooling - */ - async request(config: RequestConfig): Promise { - return new Promise((resolve, reject) => { - const requestId = this.generateRequestId(); - const queuedRequest: QueuedRequest = { - id: requestId, - config, - resolve, - reject, - timestamp: Date.now(), - retryCount: 0, - }; - - this.requestQueue.push(queuedRequest); - this.processQueue(); - }); - } - - /** - * Process the request queue - */ - private async processQueue(): Promise { - if (this.isProcessingQueue || this.requestQueue.length === 0) { - return; - } - - this.isProcessingQueue = true; - - while (this.requestQueue.length > 0) { - const request = this.requestQueue.shift()!; - - try { - const host = this.extractHost(request.config.url || ''); - const currentConnections = this.activeRequests.get(host) || 0; - - // Check connection limits - if (currentConnections >= this.config.maxConnectionsPerHost) { - // Put request back in queue - this.requestQueue.unshift(request); - break; - } - - // Check global connection limit - const totalActive = Array.from(this.activeRequests.values()).reduce((sum, count) => sum + count, 0); - if (totalActive >= this.config.maxConnections) { - this.requestQueue.unshift(request); - break; - } - - // Execute the request - this.executeRequest(request, host); - - } catch (error) { - request.reject(error); - } - } - - this.isProcessingQueue = false; - } - - /** - * Execute a single request - */ - private async executeRequest(request: QueuedRequest, host: string): Promise { - const client = this.getClient(host); - - // Increment active connections - this.activeRequests.set(host, (this.activeRequests.get(host) || 0) + 1); - - try { - // Add metadata to track timing - if (!request.config.metadata) { - request.config.metadata = {}; - } - request.config.metadata.startTime = Date.now(); - - // Execute request using our client - const response = await client.request(request.config); - request.resolve(response.data); - - } catch (error: any) { - // No need to handle retries explicitly as the BunHttpClient handles them internally - request.reject(error); - - // Emit retry event for monitoring - if (error.retryCount) { - this.emit('retry', { - requestId: request.id, - retryCount: error.retryCount, - error - }); - } - } finally { - // Decrement active connections - this.activeRequests.set(host, Math.max(0, (this.activeRequests.get(host) || 0) - 1)); - } - } - - /** - * Extract host from URL - */ - private extractHost(url: string): string { - try { - const urlObj = new URL(url); - return urlObj.host; - } catch { - return 'default'; - } - } - - /** - * Generate unique request ID - */ - private generateRequestId(): string { - return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - /** - * Update statistics - */ - private updateStats(success: boolean, responseTime: number): void { - this.stats.requestCount++; - this.stats.totalResponseTime += responseTime; - - if (success) { - this.stats.successfulRequests++; - } else { - this.stats.failedRequests++; - } - } - - /** - * Get connection pool statistics - */ - getStats(): ConnectionStats { - const totalActive = Array.from(this.activeRequests.values()).reduce((sum, count) => sum + count, 0); - const averageResponseTime = this.stats.requestCount > 0 - ? this.stats.totalResponseTime / this.stats.requestCount - : 0; - const utilization = this.config.maxConnections > 0 - ? totalActive / this.config.maxConnections - : 0; - - // Combine our stats with the stats from all clients - const clientStats = Array.from(this.clients.values()).map(client => client.getStats()); - - let successfulRequests = this.stats.successfulRequests; - let failedRequests = this.stats.failedRequests; - - for (const stats of clientStats) { - successfulRequests += stats.successfulRequests; - failedRequests += stats.failedRequests; - } - - return { - activeConnections: totalActive, - totalConnections: this.stats.totalConnections, - successfulRequests, - failedRequests, - averageResponseTime, - connectionPoolUtilization: utilization, - requestsPerSecond: 0 // Will be calculated by the http-client - }; - } - - /** - * Start queue processor timer - */ - private startQueueProcessor(): void { - this.queueProcessor = setInterval(() => { - this.processQueue(); - }, 100); // Process queue every 100ms - } - - /** - * Close all connections and clean up - */ - async close(): Promise { - // Stop the queue processor - if (this.queueProcessor) { - clearInterval(this.queueProcessor); - } - - // Wait for pending requests to complete (with timeout) - const timeout = 30000; // 30 seconds - const startTime = Date.now(); - - while (this.requestQueue.length > 0 && Date.now() - startTime < timeout) { - await new Promise(resolve => setTimeout(resolve, 100)); - } - - // Clear remaining requests - while (this.requestQueue.length > 0) { - const request = this.requestQueue.shift()!; - request.reject(new Error('Connection pool closing')); - } - - // Close all clients - const closePromises = Array.from(this.clients.values()).map(client => client.close()); - await Promise.all(closePromises); - - // Clear clients and requests - this.clients.clear(); - this.activeRequests.clear(); - - this.emit('closed'); - } - - /** - * Health check for the connection pool - */ - async healthCheck(): Promise<{ healthy: boolean; details: any }> { - const stats = this.getStats(); - const queueSize = this.requestQueue.length; - - // Check health of all clients - const clientHealthChecks = await Promise.all( - Array.from(this.clients.entries()).map(async ([host, client]) => { - const health = await client.healthCheck(); - return { - host, - healthy: health.healthy, - details: health.details - }; - }) - ); - - const healthy = - stats.connectionPoolUtilization < 0.9 && // Less than 90% utilization - queueSize < 100 && // Queue not too large - stats.averageResponseTime < 5000 && // Average response time under 5 seconds - clientHealthChecks.every(check => check.healthy); // All clients healthy - - return { - healthy, - details: { - stats, - queueSize, - clients: clientHealthChecks, - connections: Array.from(this.clients.keys()), - }, - }; - } -} diff --git a/apps/core-services/market-data-gateway/src/services/DataNormalizer.ts b/apps/core-services/market-data-gateway/src/services/DataNormalizer.ts deleted file mode 100644 index 2d7579a..0000000 --- a/apps/core-services/market-data-gateway/src/services/DataNormalizer.ts +++ /dev/null @@ -1,396 +0,0 @@ -import { dataProviderConfigs, DataProviderConfig } from '../config/DataProviderConfig'; - -// Define local types for market data -interface MarketDataType { - symbol: string; - price: number; - bid: number; - ask: number; - volume: number; - timestamp: Date; -} - -interface OHLCVType { - symbol: string; - timestamp: Date; - open: number; - high: number; - low: number; - close: number; - volume: number; -} - -export interface DataNormalizationResult { - success: boolean; - data?: T; - error?: string; - source: string; - timestamp: Date; - processingTimeMs: number; -} - -export interface DataQualityMetrics { - completeness: number; // 0-1 - accuracy: number; // 0-1 - timeliness: number; // 0-1 - consistency: number; // 0-1 - overall: number; // 0-1 -} - -export class DataNormalizer { - private readonly providerConfigs: Record; - - constructor() { - this.providerConfigs = dataProviderConfigs; - } - - /** - * Normalize market data from different providers to our standard format - */ - normalizeMarketData(rawData: any, source: string): DataNormalizationResult { - const startTime = Date.now(); - try { - let normalizedData: MarketDataType; - - switch (source.toLowerCase()) { - case 'alpha-vantage': - normalizedData = this.normalizeAlphaVantage(rawData); - break; - case 'yahoo-finance': - normalizedData = this.normalizeYahooFinance(rawData); - break; - case 'polygon': - normalizedData = this.normalizePolygon(rawData); - break; - default: - return { - success: false, - error: `Unsupported data source: ${source}`, - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } - - // Validate the normalized data - if (!this.validateMarketData(normalizedData)) { - return { - success: false, - error: 'Data validation failed', - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } - - return { - success: true, - data: normalizedData, - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown error', - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } - } /** - * Normalize OHLCV data from different providers - */ - normalizeOHLCV(rawData: any, source: string): DataNormalizationResult { - const startTime = Date.now(); - try { - let normalizedData: OHLCVType[]; - - switch (source.toLowerCase()) { - case 'alpha-vantage': - normalizedData = this.normalizeAlphaVantageOHLCV(rawData); - break; - case 'yahoo-finance': - normalizedData = this.normalizeYahooFinanceOHLCV(rawData); - break; - case 'polygon': - normalizedData = this.normalizePolygonOHLCV(rawData); - break; - default: - return { - success: false, - error: `Unsupported data source: ${source}`, - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } - - // Validate each OHLCV entry - const validData = normalizedData.filter(item => this.validateOHLCV(item)); - - if (validData.length === 0) { - return { - success: false, - error: 'No valid OHLCV data after normalization', - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } - - return { - success: true, - data: validData, - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown error', - source, - timestamp: new Date(), - processingTimeMs: Date.now() - startTime, - }; - } - } - - private normalizeAlphaVantage(data: any): MarketDataType { - const quote = data['Global Quote']; - return { - symbol: quote['01. symbol'], - price: parseFloat(quote['05. price']), - bid: parseFloat(quote['05. price']) - 0.01, // Approximate bid/ask - ask: parseFloat(quote['05. price']) + 0.01, - volume: parseInt(quote['06. volume']), - timestamp: new Date(), - }; - } - private normalizeYahooFinance(data: any): MarketDataType { - return { - symbol: data.symbol, - price: data.regularMarketPrice, - bid: data.bid || data.regularMarketPrice - 0.01, - ask: data.ask || data.regularMarketPrice + 0.01, - volume: data.regularMarketVolume, - timestamp: new Date(data.regularMarketTime * 1000), - }; - } - - private normalizePolygon(data: any): MarketDataType { - // Polygon.io format normalization - return { - symbol: data.T || data.symbol, - price: data.c || data.price, - bid: data.b || data.bid, - ask: data.a || data.ask, - volume: data.v || data.volume, - timestamp: new Date(data.t || data.timestamp), - }; - } - - private normalizeAlphaVantageOHLCV(data: any): OHLCVType[] { - const timeSeries = data['Time Series (1min)'] || data['Time Series (5min)'] || data['Time Series (Daily)']; - const symbol = data['Meta Data']['2. Symbol']; - - return Object.entries(timeSeries).map(([timestamp, values]: [string, any]) => ({ - symbol, - timestamp: new Date(timestamp), - open: parseFloat(values['1. open']), - high: parseFloat(values['2. high']), - low: parseFloat(values['3. low']), - close: parseFloat(values['4. close']), - volume: parseInt(values['5. volume']), - })).sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); - } - private normalizeYahooFinanceOHLCV(data: any): OHLCVType[] { - const result = data.chart.result[0]; - const timestamps = result.timestamp; - const quotes = result.indicators.quote[0]; - - return timestamps.map((timestamp: number, index: number) => ({ - symbol: result.meta.symbol, - timestamp: new Date(timestamp * 1000), - open: quotes.open[index], - high: quotes.high[index], - low: quotes.low[index], - close: quotes.close[index], - volume: quotes.volume[index], - })); - } - - private normalizePolygonOHLCV(data: any): OHLCVType[] { - // Polygon.io aggregates format - if (data.results && Array.isArray(data.results)) { - return data.results.map((candle: any) => ({ - symbol: data.ticker || candle.T, - timestamp: new Date(candle.t), - open: candle.o, - high: candle.h, - low: candle.l, - close: candle.c, - volume: candle.v, - })); - } - - return []; - } /** - * Validate market data quality - */ - validateMarketData(data: MarketDataType): boolean { - return ( - data.symbol && - typeof data.symbol === 'string' && - data.symbol.length > 0 && - typeof data.price === 'number' && - data.price > 0 && - typeof data.volume === 'number' && - data.volume >= 0 && - data.timestamp instanceof Date && - !isNaN(data.timestamp.getTime()) && - typeof data.bid === 'number' && - typeof data.ask === 'number' && - data.ask >= data.bid - ) as boolean; - } - /** - * Validate OHLCV data quality - */ - validateOHLCV(data: OHLCVType): boolean { - return ( - data.symbol && - typeof data.symbol === 'string' && - data.symbol.length > 0 && - typeof data.open === 'number' && data.open > 0 && - typeof data.high === 'number' && data.high > 0 && - typeof data.low === 'number' && data.low > 0 && - typeof data.close === 'number' && data.close > 0 && - data.high >= Math.max(data.open, data.close) && - data.low <= Math.min(data.open, data.close) && - typeof data.volume === 'number' && data.volume >= 0 && - data.timestamp instanceof Date && - !isNaN(data.timestamp.getTime()) - ) as boolean; - } - /** - * Assess data quality metrics for market data - */ - assessDataQuality(data: MarketDataType[], source: string): DataQualityMetrics { - if (data.length === 0) { - return { - completeness: 0, - accuracy: 0, - timeliness: 0, - consistency: 0, - overall: 0, - }; - } - - // Completeness: percentage of valid data points - const validCount = data.filter(item => this.validateMarketData(item)).length; - const completeness = validCount / data.length; - - // Accuracy: based on price consistency and reasonable ranges - const accuracyScore = this.assessAccuracy(data); - - // Timeliness: based on data freshness - const timelinessScore = this.assessTimeliness(data); - - // Consistency: based on data patterns and outliers - const consistencyScore = this.assessConsistency(data); - - const overall = (completeness + accuracyScore + timelinessScore + consistencyScore) / 4; - - return { - completeness, - accuracy: accuracyScore, - timeliness: timelinessScore, - consistency: consistencyScore, - overall, - }; - } - - private assessAccuracy(data: MarketDataType[]): number { - let accuracySum = 0; - - for (const item of data) { - let score = 1.0; - - // Check for reasonable price ranges - if (item.price <= 0 || item.price > 100000) score -= 0.3; - - // Check bid/ask spread reasonableness - const spread = item.ask - item.bid; - const spreadPercentage = spread / item.price; - if (spreadPercentage > 0.1) score -= 0.2; // More than 10% spread is suspicious - - // Check for negative volume - if (item.volume < 0) score -= 0.5; - - accuracySum += Math.max(0, score); - } - - return data.length > 0 ? accuracySum / data.length : 0; - } - - private assessTimeliness(data: MarketDataType[]): number { - const now = new Date(); - let timelinessSum = 0; - - for (const item of data) { - const ageMs = now.getTime() - item.timestamp.getTime(); - const ageMinutes = ageMs / (1000 * 60); - - // Score based on data age (fresher is better) - let score = 1.0; - if (ageMinutes > 60) score = 0.1; // Very old data - else if (ageMinutes > 15) score = 0.5; // Moderately old - else if (ageMinutes > 5) score = 0.8; // Slightly old - - timelinessSum += score; - } - - return data.length > 0 ? timelinessSum / data.length : 0; - } - - private assessConsistency(data: MarketDataType[]): number { - if (data.length < 2) return 1.0; - - // Sort by timestamp - const sortedData = [...data].sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); - - let consistencySum = 0; - - for (let i = 1; i < sortedData.length; i++) { - const current = sortedData[i]; - const previous = sortedData[i - 1]; - - // Check for reasonable price movements - const priceChange = Math.abs(current.price - previous.price) / previous.price; - - let score = 1.0; - if (priceChange > 0.5) score -= 0.7; // More than 50% change is suspicious - else if (priceChange > 0.1) score -= 0.3; // More than 10% change is notable - - consistencySum += Math.max(0, score); - } - - return consistencySum / (sortedData.length - 1); - } - /** - * Clean and sanitize market data - */ - sanitizeMarketData(data: MarketDataType): MarketDataType { - return { - symbol: data.symbol.toUpperCase().trim(), - price: Math.max(0, Number(data.price) || 0), - bid: Math.max(0, Number(data.bid) || 0), - ask: Math.max(0, Number(data.ask) || 0), - volume: Math.max(0, Math.floor(Number(data.volume) || 0)), - timestamp: new Date(data.timestamp), - }; - } -} diff --git a/apps/core-services/market-data-gateway/src/services/DataSourceManager.ts b/apps/core-services/market-data-gateway/src/services/DataSourceManager.ts deleted file mode 100644 index 438b47c..0000000 --- a/apps/core-services/market-data-gateway/src/services/DataSourceManager.ts +++ /dev/null @@ -1,598 +0,0 @@ -import { EventEmitter } from 'eventemitter3'; -// Local logger interface to avoid pino dependency issues -interface Logger { - info(msg: string, ...args: any[]): void; - error(msg: string, ...args: any[]): void; - warn(msg: string, ...args: any[]): void; - debug(msg: string, ...args: any[]): void; - child(options: any): Logger; -} - -// Simple logger implementation -const createLogger = (name: string): Logger => ({ - info: (msg: string, ...args: any[]) => console.log(`[${name}] INFO:`, msg, ...args), - error: (msg: string, ...args: any[]) => console.error(`[${name}] ERROR:`, msg, ...args), - warn: (msg: string, ...args: any[]) => console.warn(`[${name}] WARN:`, msg, ...args), - debug: (msg: string, ...args: any[]) => console.debug(`[${name}] DEBUG:`, msg, ...args), - child: (options: any) => createLogger(`${name}.${options.component || 'child'}`) -}); - -import WebSocket from 'ws'; -// Simple HTTP client to replace axios -interface HttpClient { - get(url: string): Promise<{ data: any }>; - post(url: string, data?: any): Promise<{ data: any }>; -} - -const createHttpClient = (baseURL: string, headers?: Record): HttpClient => ({ - get: async (url: string) => { - const response = await fetch(`${baseURL}${url}`, { headers }); - return { data: await response.json() }; - }, - post: async (url: string, data?: any) => { - const response = await fetch(`${baseURL}${url}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', ...headers }, - body: data ? JSON.stringify(data) : undefined - }); - return { data: await response.json() }; - } -}); -import { - DataSourceConfig, - DataSourceMetrics, - DataSourceError, - MarketDataTick, - MarketDataCandle, - MarketDataTrade -} from '../types/MarketDataGateway'; - -interface DataSourceConnection { - config: DataSourceConfig; - connection?: WebSocket | HttpClient; - status: 'disconnected' | 'connecting' | 'connected' | 'error'; - lastConnectedAt?: Date; - lastErrorAt?: Date; - retryCount: number; - metrics: { - messagesReceived: number; - bytesReceived: number; - errors: number; - latencyMs: number[]; - }; -} - -export class DataSourceManager extends EventEmitter { - private dataSources: Map = new Map(); - private logger: Logger; - private healthCheckInterval?: NodeJS.Timeout; - private reconnectTimeouts: Map = new Map(); - - constructor(configs: DataSourceConfig[], logger: Logger) { - super(); - this.logger = logger; - this.initializeDataSources(configs); - } - - private initializeDataSources(configs: DataSourceConfig[]) { - for (const config of configs) { - this.dataSources.set(config.id, { - config, - status: 'disconnected', - retryCount: 0, - metrics: { - messagesReceived: 0, - bytesReceived: 0, - errors: 0, - latencyMs: [] - } - }); - } - } - - public async start(): Promise { - this.logger.info('Starting Data Source Manager'); - - // Connect to all enabled data sources - const connectionPromises = Array.from(this.dataSources.values()) - .filter(ds => ds.config.enabled) - .map(ds => this.connectDataSource(ds.config.id)); - - await Promise.allSettled(connectionPromises); - - // Start health check interval - this.startHealthCheck(); - - this.logger.info('Data Source Manager started'); - } - - public async stop(): Promise { - this.logger.info('Stopping Data Source Manager'); - - // Clear health check interval - if (this.healthCheckInterval) { - clearInterval(this.healthCheckInterval); - } - - // Clear all reconnect timeouts - for (const timeout of this.reconnectTimeouts.values()) { - clearTimeout(timeout); - } - this.reconnectTimeouts.clear(); - - // Disconnect all data sources - const disconnectionPromises = Array.from(this.dataSources.keys()) - .map(sourceId => this.disconnectDataSource(sourceId)); - - await Promise.allSettled(disconnectionPromises); - - this.logger.info('Data Source Manager stopped'); - } - - public async addDataSource(config: DataSourceConfig): Promise { - this.logger.info({ sourceId: config.id }, 'Adding data source'); - - this.dataSources.set(config.id, { - config, - status: 'disconnected', - retryCount: 0, - metrics: { - messagesReceived: 0, - bytesReceived: 0, - errors: 0, - latencyMs: [] - } - }); - - if (config.enabled) { - await this.connectDataSource(config.id); - } - } - public async removeDataSource(sourceId: string): Promise { - this.logger.info(`Removing data source: ${sourceId}`); - - await this.disconnectDataSource(sourceId); - this.dataSources.delete(sourceId); - - const timeout = this.reconnectTimeouts.get(sourceId); - if (timeout) { - clearTimeout(timeout); - this.reconnectTimeouts.delete(sourceId); - } - } - - public async updateDataSource(sourceId: string, updates: Partial): Promise { - const dataSource = this.dataSources.get(sourceId); - if (!dataSource) { - throw new Error(`Data source ${sourceId} not found`); - } - - this.logger.info(`Updating data source: ${sourceId}`, updates); - - // Update configuration - dataSource.config = { ...dataSource.config, ...updates }; - - // Reconnect if needed - if (dataSource.status === 'connected') { - await this.disconnectDataSource(sourceId); - if (dataSource.config.enabled) { - await this.connectDataSource(sourceId); - } - } - } - - public getDataSources(): DataSourceConfig[] { - return Array.from(this.dataSources.values()).map(ds => ds.config); - } - - public getDataSourceMetrics(sourceId?: string): DataSourceMetrics[] { - const sources = sourceId - ? [this.dataSources.get(sourceId)].filter(Boolean) - : Array.from(this.dataSources.values()); - - return sources.map(ds => ({ - sourceId: ds!.config.id, - timestamp: new Date(), - connections: { - active: ds!.status === 'connected' ? 1 : 0, - total: 1, - failed: ds!.metrics.errors - }, - messages: { - received: ds!.metrics.messagesReceived, - processed: ds!.metrics.messagesReceived, // Assuming all received are processed - errors: ds!.metrics.errors, - dropped: 0 - }, - latency: { - avgMs: this.calculateAverageLatency(ds!.metrics.latencyMs), - p50Ms: this.calculatePercentile(ds!.metrics.latencyMs, 0.5), - p95Ms: this.calculatePercentile(ds!.metrics.latencyMs, 0.95), - p99Ms: this.calculatePercentile(ds!.metrics.latencyMs, 0.99) - }, - bandwidth: { - inboundBytesPerSecond: ds!.metrics.bytesReceived / 60, // Rough estimate - outboundBytesPerSecond: 0 - } - })); - } - - private async connectDataSource(sourceId: string): Promise { - const dataSource = this.dataSources.get(sourceId); - if (!dataSource) { - throw new Error(`Data source ${sourceId} not found`); - } - - if (dataSource.status === 'connected' || dataSource.status === 'connecting') { - return; - } - - this.logger.info({ sourceId }, 'Connecting to data source'); - dataSource.status = 'connecting'; - - try { - if (dataSource.config.type === 'websocket') { - await this.connectWebSocket(dataSource); - } else if (dataSource.config.type === 'rest') { - await this.connectREST(dataSource); - } else { - throw new Error(`Unsupported data source type: ${dataSource.config.type}`); - } - - dataSource.status = 'connected'; - dataSource.lastConnectedAt = new Date(); - dataSource.retryCount = 0; - - this.logger.info({ sourceId }, 'Data source connected'); - this.emit('connected', sourceId); - - } catch (error) { - this.logger.error({ sourceId, error }, 'Failed to connect to data source'); - dataSource.status = 'error'; - dataSource.lastErrorAt = new Date(); - dataSource.metrics.errors++; - - this.emit('error', sourceId, error); - this.scheduleReconnect(sourceId); - } - } - - private async connectWebSocket(dataSource: DataSourceConnection): Promise { - const { config } = dataSource; - const ws = new WebSocket(config.connection.url, { - headers: config.connection.headers - }); - - return new Promise((resolve, reject) => { - const connectTimeout = setTimeout(() => { - ws.close(); - reject(new Error('WebSocket connection timeout')); - }, 10000); - - ws.on('open', () => { - clearTimeout(connectTimeout); - this.logger.debug({ sourceId: config.id }, 'WebSocket connected'); - - // Send subscription messages - this.sendWebSocketSubscriptions(ws, config); - - dataSource.connection = ws; - resolve(); - }); - - ws.on('message', (data: Buffer) => { - const receiveTime = Date.now(); - this.handleWebSocketMessage(config.id, data, receiveTime); - }); - - ws.on('error', (error) => { - clearTimeout(connectTimeout); - this.logger.error({ sourceId: config.id, error }, 'WebSocket error'); - reject(error); - }); - - ws.on('close', () => { - this.logger.warn({ sourceId: config.id }, 'WebSocket disconnected'); - dataSource.status = 'disconnected'; - this.emit('disconnected', config.id); - this.scheduleReconnect(config.id); - }); - }); - } - - private async connectREST(dataSource: DataSourceConnection): Promise { - const { config } = dataSource; - - const axiosInstance = axios.create({ - baseURL: config.connection.url, - headers: config.connection.headers, - timeout: 5000, - params: config.connection.queryParams - }); - - // Add authentication if configured - if (config.connection.authentication) { - this.configureAuthentication(axiosInstance, config.connection.authentication); - } - - // Test connection - try { - await axiosInstance.get('/health'); - dataSource.connection = axiosInstance; - - // Start polling for REST data sources - this.startRESTPolling(config.id); - - } catch (error) { - throw new Error(`REST API health check failed: ${error}`); - } - } - - private sendWebSocketSubscriptions(ws: WebSocket, config: DataSourceConfig): void { - const subscriptions = []; - - if (config.subscriptions.quotes) { - subscriptions.push({ - type: 'subscribe', - channel: 'quotes', - symbols: config.symbols - }); - } - - if (config.subscriptions.trades) { - subscriptions.push({ - type: 'subscribe', - channel: 'trades', - symbols: config.symbols - }); - } - - if (config.subscriptions.orderbook) { - subscriptions.push({ - type: 'subscribe', - channel: 'orderbook', - symbols: config.symbols - }); - } - - if (config.subscriptions.candles) { - subscriptions.push({ - type: 'subscribe', - channel: 'candles', - symbols: config.symbols - }); - } - - for (const subscription of subscriptions) { - ws.send(JSON.stringify(subscription)); - } - } - - private handleWebSocketMessage(sourceId: string, data: Buffer, receiveTime: number): void { - const dataSource = this.dataSources.get(sourceId); - if (!dataSource) return; - - try { - const message = JSON.parse(data.toString()); - - // Update metrics - dataSource.metrics.messagesReceived++; - dataSource.metrics.bytesReceived += data.length; - - // Calculate latency if timestamp is available - if (message.timestamp) { - const latency = receiveTime - message.timestamp; - dataSource.metrics.latencyMs.push(latency); - - // Keep only last 1000 latency measurements - if (dataSource.metrics.latencyMs.length > 1000) { - dataSource.metrics.latencyMs = dataSource.metrics.latencyMs.slice(-1000); - } - } - - // Emit normalized data - const normalizedData = this.normalizeMessage(message, sourceId); - if (normalizedData) { - this.emit('data', sourceId, normalizedData); - } - - } catch (error) { - this.logger.error({ sourceId, error }, 'Error parsing WebSocket message'); - dataSource.metrics.errors++; - } - } - - private startRESTPolling(sourceId: string): void { - const dataSource = this.dataSources.get(sourceId); - if (!dataSource || !dataSource.connection) return; - - const pollInterval = 1000; // 1 second polling - - const poll = async () => { - try { - const axiosInstance = dataSource.connection as AxiosInstance; - const response = await axiosInstance.get('/market-data'); - - dataSource.metrics.messagesReceived++; - dataSource.metrics.bytesReceived += JSON.stringify(response.data).length; - - const normalizedData = this.normalizeMessage(response.data, sourceId); - if (normalizedData) { - this.emit('data', sourceId, normalizedData); - } - - } catch (error) { - this.logger.error({ sourceId, error }, 'REST polling error'); - dataSource.metrics.errors++; - } - - // Schedule next poll - if (dataSource.status === 'connected') { - setTimeout(poll, pollInterval); - } - }; - - poll(); - } - - private normalizeMessage(message: any, sourceId: string): MarketDataTick | MarketDataCandle | MarketDataTrade | null { - // This is a simplified normalization - in reality, you'd have specific - // normalizers for each data source format - try { - if (message.type === 'quote' || message.price !== undefined) { - return { - symbol: message.symbol || message.s, - timestamp: message.timestamp || message.t || Date.now(), - price: message.price || message.p, - volume: message.volume || message.v || 0, - bid: message.bid || message.b, - ask: message.ask || message.a, - source: sourceId - } as MarketDataTick; - } - - if (message.type === 'trade') { - return { - id: message.id || `${sourceId}-${Date.now()}`, - symbol: message.symbol || message.s, - timestamp: message.timestamp || message.t || Date.now(), - price: message.price || message.p, - size: message.size || message.q, - side: message.side || 'buy', - source: sourceId - } as MarketDataTrade; - } - - if (message.type === 'candle' || message.ohlc) { - return { - symbol: message.symbol || message.s, - timestamp: message.timestamp || message.t || Date.now(), - open: message.open || message.o, - high: message.high || message.h, - low: message.low || message.l, - close: message.close || message.c, - volume: message.volume || message.v, - timeframe: message.timeframe || '1m', - source: sourceId - } as MarketDataCandle; - } - - return null; - } catch (error) { - this.logger.error({ error, message, sourceId }, 'Error normalizing message'); - return null; - } - } - - private async disconnectDataSource(sourceId: string): Promise { - const dataSource = this.dataSources.get(sourceId); - if (!dataSource || dataSource.status === 'disconnected') { - return; - } - - this.logger.info({ sourceId }, 'Disconnecting data source'); - - if (dataSource.connection) { - if (dataSource.connection instanceof WebSocket) { - dataSource.connection.close(); - } - // For REST connections, we just stop polling (handled in status check) - } - - dataSource.status = 'disconnected'; - dataSource.connection = undefined; - } - - private scheduleReconnect(sourceId: string): void { - const dataSource = this.dataSources.get(sourceId); - if (!dataSource || !dataSource.config.enabled) { - return; - } - - const { retryPolicy } = dataSource.config; - const backoffMs = Math.min( - retryPolicy.backoffMultiplier * Math.pow(2, dataSource.retryCount), - retryPolicy.maxBackoffMs - ); - - if (dataSource.retryCount < retryPolicy.maxRetries) { - this.logger.info({ - sourceId, - retryCount: dataSource.retryCount, - backoffMs - }, 'Scheduling reconnect'); - - const timeout = setTimeout(() => { - dataSource.retryCount++; - this.connectDataSource(sourceId); - this.reconnectTimeouts.delete(sourceId); - }, backoffMs); - - this.reconnectTimeouts.set(sourceId, timeout); - } else { - this.logger.error({ sourceId }, 'Max retries exceeded, giving up'); - dataSource.status = 'error'; - } - } - - private startHealthCheck(): void { - this.healthCheckInterval = setInterval(() => { - for (const [sourceId, dataSource] of this.dataSources.entries()) { - if (dataSource.config.enabled && dataSource.status === 'disconnected') { - this.logger.debug({ sourceId }, 'Health check: attempting reconnect'); - this.connectDataSource(sourceId); - } - } - }, 30000); // Check every 30 seconds - } - - private configureAuthentication(axiosInstance: AxiosInstance, auth: any): void { - switch (auth.type) { - case 'apikey': - axiosInstance.defaults.headers.common['X-API-Key'] = auth.credentials.apiKey; - break; - case 'basic': - axiosInstance.defaults.auth = { - username: auth.credentials.username, - password: auth.credentials.password - }; - break; - case 'jwt': - axiosInstance.defaults.headers.common['Authorization'] = `Bearer ${auth.credentials.token}`; - break; - } - } - - private calculateAverageLatency(latencies: number[]): number { - if (latencies.length === 0) return 0; - return latencies.reduce((sum, lat) => sum + lat, 0) / latencies.length; - } - - private calculatePercentile(values: number[], percentile: number): number { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - const index = Math.ceil(sorted.length * percentile) - 1; - return sorted[Math.max(0, index)]; - } - - public async updateConfig(configs: DataSourceConfig[]): Promise { - this.logger.info('Updating data source configurations'); - - // Remove sources that are no longer in config - const configIds = new Set(configs.map(c => c.id)); - for (const sourceId of this.dataSources.keys()) { - if (!configIds.has(sourceId)) { - await this.removeDataSource(sourceId); - } - } - - // Add or update sources - for (const config of configs) { - if (this.dataSources.has(config.id)) { - await this.updateDataSource(config.id, config); - } else { - await this.addDataSource(config); - } - } - } -} diff --git a/apps/core-services/market-data-gateway/src/services/EventPublisher.ts b/apps/core-services/market-data-gateway/src/services/EventPublisher.ts deleted file mode 100644 index 4c679a5..0000000 --- a/apps/core-services/market-data-gateway/src/services/EventPublisher.ts +++ /dev/null @@ -1,140 +0,0 @@ -import Redis from 'ioredis'; -import { databaseConfig } from '@stock-bot/config'; -import type { MarketDataEvent, SignalEvent, TradingEvent } from '@stock-bot/types'; - -export class EventPublisher { - private dragonfly: Redis; - private readonly STREAM_NAME = 'trading-events'; - - constructor() { - this.dragonfly = new Redis({ - host: databaseConfig.dragonfly.host, - port: databaseConfig.dragonfly.port, - password: databaseConfig.dragonfly.password, - maxRetriesPerRequest: databaseConfig.dragonfly.maxRetriesPerRequest, - }); - - this.dragonfly.on('connect', () => { - console.log('๐Ÿ‰ Connected to Dragonfly for event publishing'); - }); - - this.dragonfly.on('error', (error) => { - console.error('โŒ Dragonfly connection error:', error); - }); - } - - /** - * Publish a market data event to the event stream - */ async publishMarketData(event: MarketDataEvent): Promise { - try { - await this.dragonfly.xadd( - this.STREAM_NAME, - '*', - 'type', event.type, - 'data', JSON.stringify(event.data), - 'timestamp', event.timestamp.toISOString() - ); - } catch (error) { - console.error('Error publishing market data event:', error); - throw error; - } - } - /** - * Publish a trading signal event - */ - async publishSignal(event: SignalEvent): Promise { - try { - await this.dragonfly.xadd( - this.STREAM_NAME, - '*', - 'type', event.type, - 'signal', JSON.stringify(event.signal), - 'timestamp', event.timestamp.toISOString() - ); - } catch (error) { - console.error('Error publishing signal event:', error); - throw error; - } - } - - /** - * Publish any trading event - */ - async publishEvent(event: TradingEvent): Promise { - try { - const fields: string[] = ['type', event.type, 'timestamp', event.timestamp.toISOString()]; - - if ('data' in event) { - fields.push('data', JSON.stringify(event.data)); - } - if ('order' in event) { - fields.push('order', JSON.stringify(event.order)); - } - if ('signal' in event) { - fields.push('signal', JSON.stringify(event.signal)); - } - - await this.dragonfly.xadd(this.STREAM_NAME, '*', ...fields); - } catch (error) { - console.error('Error publishing event:', error); - throw error; - } - } - /** - * Cache market data in Dragonfly for quick access - */ - async cacheMarketData(symbol: string, data: any, ttl: number = 60): Promise { - try { - const key = `market-data:${symbol}`; - await this.dragonfly.setex(key, ttl, JSON.stringify(data)); - } catch (error) { - console.error('Error caching market data:', error); - } - } - /** - * Get cached market data from Dragonfly - */ - async getCachedMarketData(symbol: string): Promise { - try { - const key = `market-data:${symbol}`; - const cached = await this.dragonfly.get(key); - return cached ? JSON.parse(cached) : null; - } catch (error) { - console.error('Error getting cached market data:', error); - return null; - } - } - /** - * Publish to a specific channel for real-time subscriptions - */ - async publishToChannel(channel: string, data: any): Promise { - try { - await this.dragonfly.publish(channel, JSON.stringify(data)); - } catch (error) { - console.error(`Error publishing to channel ${channel}:`, error); - throw error; - } - } - /** - * Set up health monitoring - */ - async setServiceHealth(serviceName: string, status: 'healthy' | 'unhealthy'): Promise { - try { - const key = `health:${serviceName}`; - const healthData = { - status, - timestamp: new Date().toISOString(), - lastSeen: Date.now(), - }; - await this.dragonfly.setex(key, 300, JSON.stringify(healthData)); // 5 minutes TTL - } catch (error) { - console.error('Error setting service health:', error); - } - } - /** - * Close Dragonfly connection - */ - async disconnect(): Promise { - await this.dragonfly.quit(); - } -} diff --git a/apps/core-services/market-data-gateway/src/services/MarketDataGatewayService.ts b/apps/core-services/market-data-gateway/src/services/MarketDataGatewayService.ts deleted file mode 100644 index e69de29..0000000 diff --git a/apps/core-services/market-data-gateway/src/services/MarketDataGatewayService.ts.backup b/apps/core-services/market-data-gateway/src/services/MarketDataGatewayService.ts.backup deleted file mode 100644 index 44a548a..0000000 --- a/apps/core-services/market-data-gateway/src/services/MarketDataGatewayService.ts.backup +++ /dev/null @@ -1,404 +0,0 @@ -import { EventEmitter } from 'eventemitter3'; -// Local logger interface to avoid pino dependency issues -interface Logger { - info(msg: string | object, ...args: any[]): void; - error(msg: string | object, ...args: any[]): void; - warn(msg: string | object, ...args: any[]): void; - debug(msg: string | object, ...args: any[]): void; - child(options: any): Logger; -} - -// Simple logger implementation -const createLogger = (name: string): Logger => ({ - info: (msg: string | object, ...args: any[]) => { - if (typeof msg === 'object') { - console.log(`[${name}] INFO:`, JSON.stringify(msg), ...args); - } else { - console.log(`[${name}] INFO:`, msg, ...args); - } - }, - error: (msg: string | object, ...args: any[]) => { - if (typeof msg === 'object') { - console.error(`[${name}] ERROR:`, JSON.stringify(msg), ...args); - } else { - console.error(`[${name}] ERROR:`, msg, ...args); - } - }, - warn: (msg: string | object, ...args: any[]) => { - if (typeof msg === 'object') { - console.warn(`[${name}] WARN:`, JSON.stringify(msg), ...args); - } else { - console.warn(`[${name}] WARN:`, msg, ...args); - } - }, - debug: (msg: string | object, ...args: any[]) => { - if (typeof msg === 'object') { - console.debug(`[${name}] DEBUG:`, JSON.stringify(msg), ...args); - } else { - console.debug(`[${name}] DEBUG:`, msg, ...args); - } - }, - child: (options: any) => createLogger(`${name}.${options.component || 'child'}`) -}); -import { - GatewayConfig, - DataSourceConfig, - ProcessingPipeline, - ClientSubscription, - SubscriptionRequest, - DataSourceMetrics, - GatewayMetrics, - MarketDataTick, - MarketDataCandle, - MarketDataTrade, - MarketDataOrder, - HealthStatus -} from '../types/MarketDataGateway'; -import { DataSourceManager } from './DataSourceManager'; -import { ProcessingEngine } from './ProcessingEngine'; -import { SubscriptionManager } from './SubscriptionManager'; -import { CacheManager } from './CacheManager'; -import { MetricsCollector } from './MetricsCollector'; -import { ServiceIntegrationManager } from './ServiceIntegrationManager'; - -export class MarketDataGatewayService extends EventEmitter { - private config: GatewayConfig; - private logger: Logger; - private dataSourceManager!: DataSourceManager; - private processingEngine!: ProcessingEngine; - private subscriptionManager!: SubscriptionManager; - private cacheManager!: CacheManager; - private metricsCollector!: MetricsCollector; - private serviceIntegration!: ServiceIntegrationManager; - private _isRunning = false; - private startTime: Date = new Date(); - - constructor(config: GatewayConfig, logger: Logger) { - super(); - this.config = config; - this.logger = logger; - - this.initializeComponents(); - this.setupEventHandlers(); - } - - private initializeComponents() { - this.logger.info('Initializing Market Data Gateway components'); - - // Initialize core components - this.dataSourceManager = new DataSourceManager( - this.config.dataSources, - this.logger.child({ component: 'DataSourceManager' }) - ); - - this.processingEngine = new ProcessingEngine( - this.config.processing, - this.logger.child({ component: 'ProcessingEngine' }) - ); - - this.subscriptionManager = new SubscriptionManager( - this.logger.child({ component: 'SubscriptionManager' }) - ); - - this.cacheManager = new CacheManager( - this.config.cache, - this.logger.child({ component: 'CacheManager' }) - ); - - this.metricsCollector = new MetricsCollector( - this.logger.child({ component: 'MetricsCollector' }) - ); - - this.serviceIntegration = new ServiceIntegrationManager( - this.logger.child({ component: 'ServiceIntegration' }) - ); - } - - private setupEventHandlers() { - // Data source events - this.dataSourceManager.on('data', this.handleIncomingData.bind(this)); - this.dataSourceManager.on('error', this.handleDataSourceError.bind(this)); - this.dataSourceManager.on('connected', this.handleDataSourceConnected.bind(this)); - this.dataSourceManager.on('disconnected', this.handleDataSourceDisconnected.bind(this)); - - // Processing engine events - this.processingEngine.on('processed', this.handleProcessedData.bind(this)); - this.processingEngine.on('error', this.handleProcessingError.bind(this)); - - // Subscription events - this.subscriptionManager.on('subscribed', this.handleClientSubscribed.bind(this)); - this.subscriptionManager.on('unsubscribed', this.handleClientUnsubscribed.bind(this)); - this.subscriptionManager.on('error', this.handleSubscriptionError.bind(this)); - - // Cache events - this.cacheManager.on('cached', this.handleDataCached.bind(this)); - this.cacheManager.on('error', this.handleCacheError.bind(this)); - - // Service integration events - this.serviceIntegration.on('data-forwarded', this.handleDataForwarded.bind(this)); - this.serviceIntegration.on('integration-error', this.handleIntegrationError.bind(this)); - } - - public async start(): Promise { - if (this.isRunning) { - this.logger.warn('Gateway is already running'); - return; - } - - try { - this.logger.info('Starting Market Data Gateway'); - this.startTime = new Date(); - - // Start components in order - await this.cacheManager.start(); - await this.metricsCollector.start(); - await this.serviceIntegration.start(); - await this.processingEngine.start(); - await this.subscriptionManager.start(); - await this.dataSourceManager.start(); - - this.isRunning = true; - this.logger.info('Market Data Gateway started successfully'); - this.emit('started'); - - } catch (error) { - this.logger.error({ error }, 'Failed to start Market Data Gateway'); - await this.stop(); - throw error; - } - } - - public async stop(): Promise { - if (!this.isRunning) { - return; - } - - try { - this.logger.info('Stopping Market Data Gateway'); - - // Stop components in reverse order - await this.dataSourceManager.stop(); - await this.subscriptionManager.stop(); - await this.processingEngine.stop(); - await this.serviceIntegration.stop(); - await this.metricsCollector.stop(); - await this.cacheManager.stop(); - - this.isRunning = false; - this.logger.info('Market Data Gateway stopped'); - this.emit('stopped'); - - } catch (error) { - this.logger.error({ error }, 'Error stopping Market Data Gateway'); - throw error; - } - } - - // Data handling methods - private async handleIncomingData(sourceId: string, data: any): Promise { - try { - this.metricsCollector.recordMessage(sourceId, 'received'); - - // Process data through pipeline - const processedData = await this.processingEngine.process(data); - - // Cache processed data - await this.cacheManager.cache(processedData); - - // Forward to subscribers - await this.subscriptionManager.broadcast(processedData); - - // Forward to integrated services - await this.serviceIntegration.forwardData(processedData); - - this.emit('data-processed', { sourceId, data: processedData }); - - } catch (error) { - this.logger.error({ error, sourceId, data }, 'Error handling incoming data'); - this.metricsCollector.recordError(sourceId); - } - } - - private async handleProcessedData(data: any): Promise { - this.logger.debug({ data }, 'Data processed successfully'); - this.metricsCollector.recordMessage('processing', 'processed'); - } - - private handleDataSourceError(sourceId: string, error: Error): void { - this.logger.error({ sourceId, error }, 'Data source error'); - this.metricsCollector.recordError(sourceId); - this.emit('source-error', { sourceId, error }); - } - - private handleDataSourceConnected(sourceId: string): void { - this.logger.info({ sourceId }, 'Data source connected'); - this.metricsCollector.recordConnection(sourceId, 'connected'); - } - - private handleDataSourceDisconnected(sourceId: string): void { - this.logger.warn({ sourceId }, 'Data source disconnected'); - this.metricsCollector.recordConnection(sourceId, 'disconnected'); - } - - private handleProcessingError(error: Error, data: any): void { - this.logger.error({ error, data }, 'Processing error'); - this.emit('processing-error', { error, data }); - } - - private handleClientSubscribed(subscription: ClientSubscription): void { - this.logger.info({ - clientId: subscription.request.clientId, - symbols: subscription.request.symbols - }, 'Client subscribed'); - } - - private handleClientUnsubscribed(clientId: string): void { - this.logger.info({ clientId }, 'Client unsubscribed'); - } - - private handleSubscriptionError(error: Error, clientId: string): void { - this.logger.error({ error, clientId }, 'Subscription error'); - } - - private handleDataCached(key: string, data: any): void { - this.logger.debug({ key }, 'Data cached'); - } - - private handleCacheError(error: Error, operation: string): void { - this.logger.error({ error, operation }, 'Cache error'); - } - - private handleDataForwarded(service: string, data: any): void { - this.logger.debug({ service }, 'Data forwarded to service'); - } - - private handleIntegrationError(service: string, error: Error): void { - this.logger.error({ service, error }, 'Service integration error'); - } - - // Public API methods - public async subscribe(request: SubscriptionRequest): Promise { - return this.subscriptionManager.subscribe(request); - } - - public async unsubscribe(subscriptionId: string): Promise { - return this.subscriptionManager.unsubscribe(subscriptionId); - } - - public async getSubscriptions(clientId?: string): Promise { - return this.subscriptionManager.getSubscriptions(clientId); - } - - public async addDataSource(config: DataSourceConfig): Promise { - return this.dataSourceManager.addDataSource(config); - } - - public async removeDataSource(sourceId: string): Promise { - return this.dataSourceManager.removeDataSource(sourceId); - } - - public async updateDataSource(sourceId: string, config: Partial): Promise { - return this.dataSourceManager.updateDataSource(sourceId, config); - } - - public async getDataSources(): Promise { - return this.dataSourceManager.getDataSources(); - } - - public async addProcessingPipeline(pipeline: ProcessingPipeline): Promise { - return this.processingEngine.addPipeline(pipeline); - } - - public async removeProcessingPipeline(pipelineId: string): Promise { - return this.processingEngine.removePipeline(pipelineId); - } - - public async getProcessingPipelines(): Promise { - return this.processingEngine.getPipelines(); - } - - public async getMetrics(): Promise { - return this.metricsCollector.getMetrics(); - } - - public async getDataSourceMetrics(sourceId?: string): Promise { - return this.metricsCollector.getDataSourceMetrics(sourceId); - } - - public async getHealthStatus(): Promise { - const metrics = await this.getMetrics(); - const dataSources = await this.getDataSources(); - - // Check component health - const dependencies = [ - { - name: 'cache', - status: await this.cacheManager.isHealthy() ? 'healthy' : 'unhealthy' as const - }, - { - name: 'processing-engine', - status: this.processingEngine.isHealthy() ? 'healthy' : 'unhealthy' as const - }, - { - name: 'data-sources', - status: dataSources.every(ds => ds.enabled) ? 'healthy' : 'unhealthy' as const - } - ]; - - const hasUnhealthyDependencies = dependencies.some(dep => dep.status === 'unhealthy'); - - return { - service: 'market-data-gateway', - status: hasUnhealthyDependencies ? 'degraded' : 'healthy', - timestamp: new Date(), - uptime: Date.now() - this.startTime.getTime(), - version: process.env.SERVICE_VERSION || '1.0.0', - dependencies, - metrics: { - connectionsActive: metrics.subscriptions.active, - messagesPerSecond: metrics.processing.messagesPerSecond, - errorRate: metrics.processing.errorRate, - avgLatencyMs: metrics.dataSources.reduce((sum, ds) => sum + ds.latency.avgMs, 0) / metrics.dataSources.length || 0 - } - }; - } - - // Cache operations - public async getCachedData(key: string): Promise { - return this.cacheManager.get(key); - } - - public async setCachedData(key: string, data: any, ttl?: number): Promise { - return this.cacheManager.set(key, data, ttl); - } - - // Configuration management - public getConfig(): GatewayConfig { - return { ...this.config }; - } - - public async updateConfig(updates: Partial): Promise { - this.config = { ...this.config, ...updates }; - this.logger.info('Gateway configuration updated'); - - // Notify components of config changes - if (updates.dataSources) { - await this.dataSourceManager.updateConfig(updates.dataSources); - } - - if (updates.processing) { - await this.processingEngine.updateConfig(updates.processing); - } - - this.emit('config-updated', this.config); - } - - // Utility methods - public isRunning(): boolean { - return this.isRunning; - } - - public getUptime(): number { - return Date.now() - this.startTime.getTime(); - } -} diff --git a/apps/core-services/market-data-gateway/src/services/MarketDataService.ts b/apps/core-services/market-data-gateway/src/services/MarketDataService.ts deleted file mode 100644 index aacee20..0000000 --- a/apps/core-services/market-data-gateway/src/services/MarketDataService.ts +++ /dev/null @@ -1,278 +0,0 @@ -import type { MarketData, OHLCV, MarketDataEvent } from '@stock-bot/types'; -import { dataProviderConfigs } from '@stock-bot/config'; -import { EventPublisher } from './EventPublisher'; -import { DataNormalizer } from './DataNormalizer'; - -export class MarketDataService { - private wsClients: Set = new Set(); - private subscriptions: Map> = new Map(); - private dataUpdateInterval: Timer | null = null; - private readonly UPDATE_INTERVAL = 5000; // 5 seconds - - constructor( - private eventPublisher: EventPublisher, - private dataNormalizer: DataNormalizer - ) {} - - /** - * Initialize the market data service - */ - async initialize(): Promise { - console.log('๐Ÿ”„ Initializing Market Data Service...'); - - // Set up periodic data updates for demo purposes - this.startDataUpdates(); - - // Set service health - await this.eventPublisher.setServiceHealth('market-data-gateway', 'healthy'); - - console.log('โœ… Market Data Service initialized'); - } - - /** - * Get latest market data for a symbol - */ - async getLatestData(symbol: string): Promise { - // First check cache - const cached = await this.eventPublisher.getCachedMarketData(symbol); - if (cached) { - return cached; - } - - // Fetch fresh data (using demo data for now) - const marketData = this.generateDemoData(symbol); - - // Cache the data - await this.eventPublisher.cacheMarketData(symbol, marketData, 60); - - // Publish market data event - const event: MarketDataEvent = { - type: 'MARKET_DATA', - data: marketData, - timestamp: new Date(), - }; - await this.eventPublisher.publishMarketData(event); - - return marketData; - } - - /** - * Get OHLCV data for a symbol - */ - async getOHLCV(symbol: string, interval: string, limit: number): Promise { - // Generate demo OHLCV data - const ohlcvData = this.generateDemoOHLCV(symbol, limit); - - // Cache the data - await this.eventPublisher.cacheMarketData(`ohlcv:${symbol}:${interval}`, ohlcvData, 300); - - return ohlcvData; - } - - /** - * Add WebSocket client for real-time updates - */ - addWebSocketClient(ws: any): void { - this.wsClients.add(ws); - } - - /** - * Remove WebSocket client - */ - removeWebSocketClient(ws: any): void { - this.wsClients.delete(ws); - - // Remove from all subscriptions - for (const [symbol, clients] of this.subscriptions) { - clients.delete(ws); - if (clients.size === 0) { - this.subscriptions.delete(symbol); - } - } - } - - /** - * Handle WebSocket messages - */ - handleWebSocketMessage(ws: any, data: any): void { - try { - const message = typeof data === 'string' ? JSON.parse(data) : data; - - switch (message.type) { - case 'subscribe': - this.subscribeToSymbol(ws, message.symbol); - break; - case 'unsubscribe': - this.unsubscribeFromSymbol(ws, message.symbol); - break; - default: - console.log('Unknown WebSocket message type:', message.type); - } - } catch (error) { - console.error('Error handling WebSocket message:', error); - } - } - - /** - * Subscribe WebSocket client to symbol updates - */ - private subscribeToSymbol(ws: any, symbol: string): void { - if (!this.subscriptions.has(symbol)) { - this.subscriptions.set(symbol, new Set()); - } - this.subscriptions.get(symbol)!.add(ws); - - ws.send(JSON.stringify({ - type: 'subscribed', - symbol, - timestamp: new Date().toISOString(), - })); - } - - /** - * Unsubscribe WebSocket client from symbol updates - */ - private unsubscribeFromSymbol(ws: any, symbol: string): void { - const clients = this.subscriptions.get(symbol); - if (clients) { - clients.delete(ws); - if (clients.size === 0) { - this.subscriptions.delete(symbol); - } - } - - ws.send(JSON.stringify({ - type: 'unsubscribed', - symbol, - timestamp: new Date().toISOString(), - })); - } - - /** - * Start periodic data updates for demo - */ - private startDataUpdates(): void { - this.dataUpdateInterval = setInterval(async () => { - const symbols = ['AAPL', 'GOOGL', 'MSFT', 'TSLA', 'AMZN']; - - for (const symbol of symbols) { - if (this.subscriptions.has(symbol)) { - const marketData = this.generateDemoData(symbol); - - // Send to subscribed WebSocket clients - const clients = this.subscriptions.get(symbol)!; - const message = JSON.stringify({ - type: 'market_data', - data: marketData, - timestamp: new Date().toISOString(), - }); - - for (const client of clients) { - try { - client.send(message); - } catch (error) { - console.error('Error sending WebSocket message:', error); - clients.delete(client); - } - } - - // Publish event - const event: MarketDataEvent = { - type: 'MARKET_DATA', - data: marketData, - timestamp: new Date(), - }; - await this.eventPublisher.publishMarketData(event); - } - } - }, this.UPDATE_INTERVAL); - } - - /** - * Generate demo market data - */ - private generateDemoData(symbol: string): MarketData { - const basePrice = this.getBasePrice(symbol); - const variation = (Math.random() - 0.5) * 0.02; // ยฑ1% variation - const price = basePrice * (1 + variation); - - return { - symbol, - price: Math.round(price * 100) / 100, - bid: Math.round((price - 0.01) * 100) / 100, - ask: Math.round((price + 0.01) * 100) / 100, - volume: Math.floor(Math.random() * 1000000) + 100000, - timestamp: new Date(), - }; - } - - /** - * Generate demo OHLCV data - */ - private generateDemoOHLCV(symbol: string, limit: number): OHLCV[] { - const basePrice = this.getBasePrice(symbol); - const data: OHLCV[] = []; - let currentPrice = basePrice; - - for (let i = limit - 1; i >= 0; i--) { - const variation = (Math.random() - 0.5) * 0.05; // ยฑ2.5% variation - const open = currentPrice; - const close = open * (1 + variation); - const high = Math.max(open, close) * (1 + Math.random() * 0.02); - const low = Math.min(open, close) * (1 - Math.random() * 0.02); - - data.push({ - symbol, - timestamp: new Date(Date.now() - i * 60000), // 1 minute intervals - open: Math.round(open * 100) / 100, - high: Math.round(high * 100) / 100, - low: Math.round(low * 100) / 100, - close: Math.round(close * 100) / 100, - volume: Math.floor(Math.random() * 50000) + 10000, - }); - - currentPrice = close; - } - - return data; - } - - /** - * Get base price for demo data - */ - private getBasePrice(symbol: string): number { - const prices: Record = { - 'AAPL': 175.50, - 'GOOGL': 142.30, - 'MSFT': 378.85, - 'TSLA': 208.75, - 'AMZN': 151.20, - 'NVDA': 465.80, - 'META': 298.45, - 'NFLX': 425.60, - }; - - return prices[symbol] || 100.00; - } - - /** - * Shutdown the service - */ - async shutdown(): Promise { - console.log('๐Ÿ”„ Shutting down Market Data Service...'); - - if (this.dataUpdateInterval) { - clearInterval(this.dataUpdateInterval); - } - - // Close all WebSocket connections - for (const client of this.wsClients) { - client.close(); - } - - await this.eventPublisher.setServiceHealth('market-data-gateway', 'unhealthy'); - await this.eventPublisher.disconnect(); - - console.log('โœ… Market Data Service shutdown complete'); - } -} diff --git a/apps/core-services/market-data-gateway/src/services/MetricsCollector.ts b/apps/core-services/market-data-gateway/src/services/MetricsCollector.ts deleted file mode 100644 index 70dbbd4..0000000 --- a/apps/core-services/market-data-gateway/src/services/MetricsCollector.ts +++ /dev/null @@ -1,511 +0,0 @@ -import { EventEmitter } from 'events'; -import { - GatewayMetrics, - Logger, - HealthStatus, - ProcessingMetrics, - DataSourceMetrics, - SubscriptionMetrics -} from '../types/MarketDataGateway'; - -interface MetricPoint { - value: number; - timestamp: number; - labels?: Record; -} - -interface TimeSeriesMetric { - name: string; - points: MetricPoint[]; - maxPoints: number; -} - -interface AlertRule { - id: string; - metric: string; - condition: 'gt' | 'lt' | 'eq' | 'gte' | 'lte'; - threshold: number; - duration: number; // ms - enabled: boolean; - lastTriggered?: number; -} - -interface Alert { - id: string; - rule: AlertRule; - value: number; - timestamp: number; - message: string; - severity: 'info' | 'warning' | 'error' | 'critical'; -} - -export class MetricsCollector extends EventEmitter { - private logger: Logger; - private metrics: Map; - private aggregatedMetrics: GatewayMetrics; - private alerts: Map; - private alertRules: Map; - private collectInterval: NodeJS.Timeout | null = null; - private isRunning: boolean = false; - - constructor(logger: Logger) { - super(); - this.logger = logger; - this.metrics = new Map(); - this.alerts = new Map(); - this.alertRules = new Map(); - - this.aggregatedMetrics = { - totalMessages: 0, - messagesPerSecond: 0, - averageLatency: 0, - errorRate: 0, - activeConnections: 0, - activeSubscriptions: 0, - cacheHitRate: 0, - uptime: 0, - timestamp: new Date().toISOString(), - dataSources: new Map(), - processing: { - totalProcessed: 0, - processedPerSecond: 0, - processingLatency: 0, - errorCount: 0, - queueDepth: 0, - processorMetrics: new Map(), - }, - subscriptions: { - totalSubscriptions: 0, - activeClients: 0, - messagesSent: 0, - sendRate: 0, - subscriptionsBySymbol: new Map(), - clientMetrics: new Map(), - }, - }; - - this.setupDefaultAlertRules(); - this.startCollection(); - } - - private setupDefaultAlertRules(): void { - const defaultRules: AlertRule[] = [ - { - id: 'high-error-rate', - metric: 'errorRate', - condition: 'gt', - threshold: 0.05, // 5% - duration: 60000, // 1 minute - enabled: true, - }, - { - id: 'high-latency', - metric: 'averageLatency', - condition: 'gt', - threshold: 1000, // 1 second - duration: 30000, // 30 seconds - enabled: true, - }, - { - id: 'low-cache-hit-rate', - metric: 'cacheHitRate', - condition: 'lt', - threshold: 0.8, // 80% - duration: 300000, // 5 minutes - enabled: true, - }, - { - id: 'high-queue-depth', - metric: 'processing.queueDepth', - condition: 'gt', - threshold: 1000, - duration: 60000, // 1 minute - enabled: true, - }, - ]; - - defaultRules.forEach(rule => { - this.alertRules.set(rule.id, rule); - }); - } - - startCollection(): void { - if (this.isRunning) return; - - this.isRunning = true; - this.collectInterval = setInterval(() => { - this.collectMetrics(); - this.checkAlerts(); - this.cleanupOldMetrics(); - }, 1000); // Collect every second - - this.logger.info('Metrics collection started'); - } - - stopCollection(): void { - if (!this.isRunning) return; - - this.isRunning = false; - if (this.collectInterval) { - clearInterval(this.collectInterval); - this.collectInterval = null; - } - - this.logger.info('Metrics collection stopped'); - } - - // Metric recording methods - recordMessage(source: string, latency?: number, error?: boolean): void { - this.recordMetric('totalMessages', 1); - this.recordMetric('messagesPerSecond', 1); - - if (latency !== undefined) { - this.recordMetric('latency', latency, { source }); - } - - if (error) { - this.recordMetric('errors', 1, { source }); - } - } - - recordProcessing(processed: number, latency: number, errors: number): void { - this.recordMetric('processing.totalProcessed', processed); - this.recordMetric('processing.processedPerSecond', processed); - this.recordMetric('processing.processingLatency', latency); - this.recordMetric('processing.errorCount', errors); - } - - recordSubscription(action: 'subscribe' | 'unsubscribe', symbol: string, clientId: string): void { - this.recordMetric('subscriptions.totalSubscriptions', action === 'subscribe' ? 1 : -1); - this.recordMetric(`subscriptions.symbol.${symbol}`, action === 'subscribe' ? 1 : -1); - this.recordMetric(`subscriptions.client.${clientId}`, action === 'subscribe' ? 1 : -1); - } - - recordDataSource(sourceId: string, metrics: Partial): void { - Object.entries(metrics).forEach(([key, value]) => { - if (typeof value === 'number') { - this.recordMetric(`dataSource.${sourceId}.${key}`, value); - } - }); - } - - recordCacheMetrics(hitRate: number, size: number, memoryUsage: number): void { - this.recordMetric('cacheHitRate', hitRate); - this.recordMetric('cacheSize', size); - this.recordMetric('cacheMemoryUsage', memoryUsage); - } - - setGauge(metric: string, value: number, labels?: Record): void { - this.recordMetric(metric, value, labels, true); - } - - incrementCounter(metric: string, value: number = 1, labels?: Record): void { - this.recordMetric(metric, value, labels, false); - } - - recordHistogram(metric: string, value: number, labels?: Record): void { - this.recordMetric(`${metric}.value`, value, labels); - this.recordMetric(`${metric}.count`, 1, labels); - } - - private recordMetric( - name: string, - value: number, - labels?: Record, - isGauge: boolean = false - ): void { - const point: MetricPoint = { - value, - timestamp: Date.now(), - labels, - }; - - if (!this.metrics.has(name)) { - this.metrics.set(name, { - name, - points: [], - maxPoints: 3600, // Keep 1 hour of data at 1-second intervals - }); - } - - const metric = this.metrics.get(name)!; - - if (isGauge) { - // For gauges, replace the last value - metric.points = [point]; - } else { - // For counters and histograms, append - metric.points.push(point); - } - - // Trim old points - if (metric.points.length > metric.maxPoints) { - metric.points = metric.points.slice(-metric.maxPoints); - } - } - - // Metric retrieval methods - getMetric(name: string, duration?: number): MetricPoint[] { - const metric = this.metrics.get(name); - if (!metric) return []; - - if (!duration) return [...metric.points]; - - const cutoff = Date.now() - duration; - return metric.points.filter(point => point.timestamp >= cutoff); - } - - getAverageMetric(name: string, duration?: number): number { - const points = this.getMetric(name, duration); - if (points.length === 0) return 0; - - const sum = points.reduce((acc, point) => acc + point.value, 0); - return sum / points.length; - } - - getLatestMetric(name: string): number | null { - const metric = this.metrics.get(name); - if (!metric || metric.points.length === 0) return null; - - return metric.points[metric.points.length - 1].value; - } - - getRate(name: string, duration: number = 60000): number { - const points = this.getMetric(name, duration); - if (points.length < 2) return 0; - - const oldest = points[0]; - const newest = points[points.length - 1]; - const timeDiff = newest.timestamp - oldest.timestamp; - const valueDiff = newest.value - oldest.value; - - return timeDiff > 0 ? (valueDiff / timeDiff) * 1000 : 0; // per second - } - - getPercentile(name: string, percentile: number, duration?: number): number { - const points = this.getMetric(name, duration); - if (points.length === 0) return 0; - - const values = points.map(p => p.value).sort((a, b) => a - b); - const index = Math.ceil((percentile / 100) * values.length) - 1; - return values[Math.max(0, index)]; - } - - // Aggregated metrics - getAggregatedMetrics(): GatewayMetrics { - return { ...this.aggregatedMetrics }; - } - - private collectMetrics(): void { - const now = new Date().toISOString(); - - // Update basic metrics - this.aggregatedMetrics.totalMessages = this.getLatestMetric('totalMessages') || 0; - this.aggregatedMetrics.messagesPerSecond = this.getRate('messagesPerSecond'); - this.aggregatedMetrics.averageLatency = this.getAverageMetric('latency', 60000); - this.aggregatedMetrics.cacheHitRate = this.getLatestMetric('cacheHitRate') || 0; - this.aggregatedMetrics.timestamp = now; - - // Calculate error rate - const totalMessages = this.aggregatedMetrics.totalMessages; - const totalErrors = this.getLatestMetric('errors') || 0; - this.aggregatedMetrics.errorRate = totalMessages > 0 ? totalErrors / totalMessages : 0; - - // Update processing metrics - this.aggregatedMetrics.processing.totalProcessed = this.getLatestMetric('processing.totalProcessed') || 0; - this.aggregatedMetrics.processing.processedPerSecond = this.getRate('processing.processedPerSecond'); - this.aggregatedMetrics.processing.processingLatency = this.getAverageMetric('processing.processingLatency', 60000); - this.aggregatedMetrics.processing.errorCount = this.getLatestMetric('processing.errorCount') || 0; - this.aggregatedMetrics.processing.queueDepth = this.getLatestMetric('processing.queueDepth') || 0; - - // Update subscription metrics - this.aggregatedMetrics.subscriptions.totalSubscriptions = this.getLatestMetric('subscriptions.totalSubscriptions') || 0; - this.aggregatedMetrics.subscriptions.messagesSent = this.getLatestMetric('subscriptions.messagesSent') || 0; - this.aggregatedMetrics.subscriptions.sendRate = this.getRate('subscriptions.messagesSent'); - - this.emit('metrics-updated', this.aggregatedMetrics); - } - - // Alert management - addAlertRule(rule: AlertRule): void { - this.alertRules.set(rule.id, rule); - this.logger.info(`Alert rule added: ${rule.id}`); - } - - removeAlertRule(ruleId: string): void { - this.alertRules.delete(ruleId); - this.alerts.delete(ruleId); - this.logger.info(`Alert rule removed: ${ruleId}`); - } - - getAlertRules(): AlertRule[] { - return Array.from(this.alertRules.values()); - } - - getActiveAlerts(): Alert[] { - return Array.from(this.alerts.values()); - } - - private checkAlerts(): void { - for (const rule of this.alertRules.values()) { - if (!rule.enabled) continue; - - const value = this.getMetricValue(rule.metric); - if (value === null) continue; - - const isTriggered = this.evaluateCondition(value, rule.condition, rule.threshold); - - if (isTriggered) { - const now = Date.now(); - const existingAlert = this.alerts.get(rule.id); - - // Check if alert should be triggered based on duration - if (!existingAlert || (now - existingAlert.timestamp) >= rule.duration) { - const alert: Alert = { - id: rule.id, - rule, - value, - timestamp: now, - message: `Alert: ${rule.metric} ${rule.condition} ${rule.threshold} (current: ${value})`, - severity: this.getSeverity(rule.metric, value), - }; - - this.alerts.set(rule.id, alert); - this.emit('alert-triggered', alert); - this.logger.warn(`Alert triggered: ${alert.message}`); - } - } else { - // Clear alert if condition is no longer met - if (this.alerts.has(rule.id)) { - this.alerts.delete(rule.id); - this.emit('alert-cleared', rule.id); - this.logger.info(`Alert cleared: ${rule.id}`); - } - } - } - } - - private getMetricValue(metricPath: string): number | null { - if (metricPath.includes('.')) { - // Handle nested metric paths - const parts = metricPath.split('.'); - let value: any = this.aggregatedMetrics; - - for (const part of parts) { - if (value && typeof value === 'object' && part in value) { - value = value[part]; - } else { - return null; - } - } - - return typeof value === 'number' ? value : null; - } - - return this.getLatestMetric(metricPath); - } - - private evaluateCondition(value: number, condition: string, threshold: number): boolean { - switch (condition) { - case 'gt': return value > threshold; - case 'lt': return value < threshold; - case 'eq': return value === threshold; - case 'gte': return value >= threshold; - case 'lte': return value <= threshold; - default: return false; - } - } - - private getSeverity(metric: string, value: number): Alert['severity'] { - // Define severity based on metric type and value - if (metric.includes('error') || metric.includes('Error')) { - if (value > 0.1) return 'critical'; // > 10% error rate - if (value > 0.05) return 'error'; // > 5% error rate - if (value > 0.01) return 'warning'; // > 1% error rate - return 'info'; - } - - if (metric.includes('latency') || metric.includes('Latency')) { - if (value > 5000) return 'critical'; // > 5 seconds - if (value > 2000) return 'error'; // > 2 seconds - if (value > 1000) return 'warning'; // > 1 second - return 'info'; - } - - return 'warning'; // Default severity - } - - private cleanupOldMetrics(): void { - const cutoff = Date.now() - (24 * 60 * 60 * 1000); // 24 hours - - for (const metric of this.metrics.values()) { - metric.points = metric.points.filter(point => point.timestamp > cutoff); - } - } - - // Health and status - getHealth(): HealthStatus { - const activeAlerts = this.getActiveAlerts(); - const criticalAlerts = activeAlerts.filter(a => a.severity === 'critical'); - const errorAlerts = activeAlerts.filter(a => a.severity === 'error'); - - let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy'; - let message = 'Metrics collector is operational'; - - if (criticalAlerts.length > 0) { - status = 'unhealthy'; - message = `${criticalAlerts.length} critical alerts active`; - } else if (errorAlerts.length > 0) { - status = 'degraded'; - message = `${errorAlerts.length} error alerts active`; - } - - return { - status, - message, - timestamp: new Date().toISOString(), - details: { - isRunning: this.isRunning, - totalMetrics: this.metrics.size, - activeAlerts: activeAlerts.length, - alertRules: this.alertRules.size, - }, - }; - } - - // Export methods - exportMetrics(format: 'json' | 'prometheus' = 'json'): string { - if (format === 'prometheus') { - return this.exportPrometheusFormat(); - } - - return JSON.stringify({ - aggregated: this.aggregatedMetrics, - timeSeries: Object.fromEntries(this.metrics), - alerts: Object.fromEntries(this.alerts), - }, null, 2); - } - - private exportPrometheusFormat(): string { - const lines: string[] = []; - - // Export aggregated metrics - lines.push(`# HELP gateway_total_messages Total messages processed`); - lines.push(`# TYPE gateway_total_messages counter`); - lines.push(`gateway_total_messages ${this.aggregatedMetrics.totalMessages}`); - - lines.push(`# HELP gateway_messages_per_second Messages processed per second`); - lines.push(`# TYPE gateway_messages_per_second gauge`); - lines.push(`gateway_messages_per_second ${this.aggregatedMetrics.messagesPerSecond}`); - - lines.push(`# HELP gateway_average_latency Average processing latency in milliseconds`); - lines.push(`# TYPE gateway_average_latency gauge`); - lines.push(`gateway_average_latency ${this.aggregatedMetrics.averageLatency}`); - - lines.push(`# HELP gateway_error_rate Error rate as percentage`); - lines.push(`# TYPE gateway_error_rate gauge`); - lines.push(`gateway_error_rate ${this.aggregatedMetrics.errorRate}`); - - return lines.join('\n'); - } -} diff --git a/apps/core-services/market-data-gateway/src/services/ProcessingEngine.ts b/apps/core-services/market-data-gateway/src/services/ProcessingEngine.ts deleted file mode 100644 index aea1fc6..0000000 --- a/apps/core-services/market-data-gateway/src/services/ProcessingEngine.ts +++ /dev/null @@ -1,539 +0,0 @@ -import { EventEmitter } from 'eventemitter3'; -import { Logger } from 'pino'; -import { - ProcessingPipeline, - DataProcessor, - MarketDataTick, - MarketDataCandle, - MarketDataTrade, - ProcessingError -} from '../types/MarketDataGateway'; - -interface ProcessingJob { - id: string; - data: any; - pipeline: ProcessingPipeline; - timestamp: Date; - attempts: number; -} - -export class ProcessingEngine extends EventEmitter { - private config: any; - private logger: Logger; - private pipelines: Map = new Map(); - private processors: Map = new Map(); - private processingQueue: ProcessingJob[] = []; - private isProcessing = false; - private processingStats = { - totalProcessed: 0, - totalErrors: 0, - avgProcessingTimeMs: 0, - processingTimes: [] as number[] - }; - - constructor(config: any, logger: Logger) { - super(); - this.config = config; - this.logger = logger; - this.initializeBuiltInProcessors(); - } - - private initializeBuiltInProcessors() { - // Data validation processor - this.processors.set('data-validator', { - id: 'data-validator', - name: 'Data Validator', - type: 'validation', - enabled: true, - priority: 1, - config: {}, - process: this.validateData.bind(this) - }); - - // Data enrichment processor - this.processors.set('data-enricher', { - id: 'data-enricher', - name: 'Data Enricher', - type: 'enrichment', - enabled: true, - priority: 2, - config: {}, - process: this.enrichData.bind(this) - }); - - // Data normalization processor - this.processors.set('data-normalizer', { - id: 'data-normalizer', - name: 'Data Normalizer', - type: 'normalization', - enabled: true, - priority: 3, - config: {}, - process: this.normalizeData.bind(this) - }); - - // Outlier detection processor - this.processors.set('outlier-detector', { - id: 'outlier-detector', - name: 'Outlier Detector', - type: 'filter', - enabled: true, - priority: 4, - config: { - priceDeviationThreshold: 0.1, // 10% price deviation - volumeThreshold: 1000000 // Minimum volume threshold - }, - process: this.detectOutliers.bind(this) - }); - - // Market hours filter - this.processors.set('market-hours-filter', { - id: 'market-hours-filter', - name: 'Market Hours Filter', - type: 'filter', - enabled: true, - priority: 5, - config: { - marketOpen: '09:30', - marketClose: '16:00', - timezone: 'America/New_York' - }, - process: this.filterMarketHours.bind(this) - }); - - // OHLC aggregator - this.processors.set('ohlc-aggregator', { - id: 'ohlc-aggregator', - name: 'OHLC Aggregator', - type: 'aggregation', - enabled: true, - priority: 6, - config: { - timeframes: ['1m', '5m', '15m', '1h', '1d'] - }, - process: this.aggregateOHLC.bind(this) - }); - } - - public async start(): Promise { - this.logger.info('Starting Processing Engine'); - - // Load configured pipelines - if (this.config.pipelines) { - for (const pipeline of this.config.pipelines) { - this.addPipeline(pipeline); - } - } - - // Start processing loop - this.startProcessing(); - - this.logger.info('Processing Engine started'); - } - - public async stop(): Promise { - this.logger.info('Stopping Processing Engine'); - - this.isProcessing = false; - - // Wait for current processing to complete - while (this.processingQueue.length > 0) { - await new Promise(resolve => setTimeout(resolve, 100)); - } - - this.logger.info('Processing Engine stopped'); - } - - public async process(data: any): Promise { - const startTime = Date.now(); - - try { - // Find applicable pipelines for this data - const applicablePipelines = this.findApplicablePipelines(data); - - if (applicablePipelines.length === 0) { - // No processing needed, return original data - return data; - } - - let processedData = data; - - // Process through each applicable pipeline - for (const pipeline of applicablePipelines) { - processedData = await this.processThroughPipeline(processedData, pipeline); - } - - // Update processing stats - const processingTime = Date.now() - startTime; - this.updateProcessingStats(processingTime, false); - - this.emit('processed', processedData); - return processedData; - - } catch (error) { - this.logger.error({ error, data }, 'Processing error'); - this.updateProcessingStats(Date.now() - startTime, true); - this.emit('error', error, data); - throw error; - } - } - - public addPipeline(pipeline: ProcessingPipeline): void { - this.logger.info({ pipelineId: pipeline.id }, 'Adding processing pipeline'); - this.pipelines.set(pipeline.id, pipeline); - } - - public removePipeline(pipelineId: string): void { - this.logger.info({ pipelineId }, 'Removing processing pipeline'); - this.pipelines.delete(pipelineId); - } - - public getPipelines(): ProcessingPipeline[] { - return Array.from(this.pipelines.values()); - } - - public addProcessor(processor: DataProcessor): void { - this.logger.info({ processorId: processor.id }, 'Adding data processor'); - this.processors.set(processor.id, processor); - } - - public removeProcessor(processorId: string): void { - this.logger.info({ processorId }, 'Removing data processor'); - this.processors.delete(processorId); - } - - public getProcessors(): DataProcessor[] { - return Array.from(this.processors.values()); - } - - public getProcessingStats() { - return { - ...this.processingStats, - queueDepth: this.processingQueue.length - }; - } - - public isHealthy(): boolean { - return this.isProcessing && this.processingStats.totalErrors / Math.max(this.processingStats.totalProcessed, 1) < 0.1; - } - - private findApplicablePipelines(data: any): ProcessingPipeline[] { - const applicable: ProcessingPipeline[] = []; - - for (const pipeline of this.pipelines.values()) { - if (this.isPipelineApplicable(data, pipeline)) { - applicable.push(pipeline); - } - } - - return applicable; - } - - private isPipelineApplicable(data: any, pipeline: ProcessingPipeline): boolean { - const { inputFilter } = pipeline; - - // Check symbol filter - if (inputFilter.symbols && inputFilter.symbols.length > 0) { - if (!data.symbol || !inputFilter.symbols.includes(data.symbol)) { - return false; - } - } - - // Check source filter - if (inputFilter.sources && inputFilter.sources.length > 0) { - if (!data.source || !inputFilter.sources.includes(data.source)) { - return false; - } - } - - // Check data type filter - if (inputFilter.dataTypes && inputFilter.dataTypes.length > 0) { - const dataType = this.getDataType(data); - if (!inputFilter.dataTypes.includes(dataType)) { - return false; - } - } - - return true; - } - - private getDataType(data: any): string { - if (data.id && data.side) return 'trade'; - if (data.open !== undefined && data.high !== undefined) return 'candle'; - if (data.price !== undefined) return 'quote'; - if (data.bids || data.asks) return 'orderbook'; - return 'unknown'; - } - - private async processThroughPipeline(data: any, pipeline: ProcessingPipeline): Promise { - let processedData = data; - - // Sort processors by priority - const sortedProcessors = pipeline.processors - .filter(p => p.enabled) - .sort((a, b) => a.priority - b.priority); - - for (const processorConfig of sortedProcessors) { - const processor = this.processors.get(processorConfig.id); - if (!processor) { - this.logger.warn({ - processorId: processorConfig.id, - pipelineId: pipeline.id - }, 'Processor not found'); - continue; - } - - try { - processedData = await processor.process(processedData); - - // If processor returns null/undefined, filter out the data - if (processedData === null || processedData === undefined) { - this.logger.debug({ - processorId: processor.id, - pipelineId: pipeline.id - }, 'Data filtered out by processor'); - return null; - } - - } catch (error) { - this.logger.error({ - error, - processorId: processor.id, - pipelineId: pipeline.id, - data: processedData - }, 'Processor error'); - - // Continue processing with original data on error - // You might want to implement different error handling strategies - } - } - - return processedData; - } - - private startProcessing(): void { - this.isProcessing = true; - - const processLoop = async () => { - while (this.isProcessing) { - if (this.processingQueue.length > 0) { - const job = this.processingQueue.shift()!; - - try { - await this.processThroughPipeline(job.data, job.pipeline); - } catch (error) { - this.logger.error({ - jobId: job.id, - error - }, 'Job processing error'); - } - } else { - // Wait for new jobs - await new Promise(resolve => setTimeout(resolve, 10)); - } - } - }; - - processLoop(); - } - - private updateProcessingStats(processingTime: number, isError: boolean): void { - this.processingStats.totalProcessed++; - - if (isError) { - this.processingStats.totalErrors++; - } - - this.processingStats.processingTimes.push(processingTime); - - // Keep only last 1000 processing times - if (this.processingStats.processingTimes.length > 1000) { - this.processingStats.processingTimes = this.processingStats.processingTimes.slice(-1000); - } - - // Update average processing time - this.processingStats.avgProcessingTimeMs = - this.processingStats.processingTimes.reduce((sum, time) => sum + time, 0) / - this.processingStats.processingTimes.length; - } - - // Built-in processor implementations - private async validateData(data: any): Promise { - // Basic data validation - if (!data) { - throw new Error('Data is null or undefined'); - } - - if (!data.symbol) { - throw new Error('Missing symbol'); - } - - if (!data.timestamp) { - data.timestamp = Date.now(); - } - - // Validate price data - if (data.price !== undefined) { - if (typeof data.price !== 'number' || data.price <= 0) { - throw new Error('Invalid price'); - } - } - - // Validate volume data - if (data.volume !== undefined) { - if (typeof data.volume !== 'number' || data.volume < 0) { - throw new Error('Invalid volume'); - } - } - - return data; - } - - private async enrichData(data: any): Promise { - // Add computed fields - const enriched = { ...data }; - - // Add processing timestamp - enriched.processedAt = Date.now(); - - // Add data type - enriched.dataType = this.getDataType(data); - - // Calculate derived metrics for quotes - if (data.price && data.prevClose) { - enriched.change = data.price - data.prevClose; - enriched.changePercent = (enriched.change / data.prevClose) * 100; - } - - // Add market session info - const marketSession = this.getMarketSession(data.timestamp); - enriched.marketSession = marketSession; - - return enriched; - } - - private async normalizeData(data: any): Promise { - const normalized = { ...data }; - - // Normalize symbol format - if (normalized.symbol) { - normalized.symbol = normalized.symbol.toUpperCase().trim(); - } - - // Normalize timestamp to milliseconds - if (normalized.timestamp) { - if (typeof normalized.timestamp === 'string') { - normalized.timestamp = new Date(normalized.timestamp).getTime(); - } else if (normalized.timestamp.toString().length === 10) { - // Convert seconds to milliseconds - normalized.timestamp *= 1000; - } - } - - // Round price values to appropriate precision - if (normalized.price) { - normalized.price = Math.round(normalized.price * 10000) / 10000; - } - - return normalized; - } - - private async detectOutliers(data: any): Promise { - // Simple outlier detection - in practice, you'd use historical data - const config = this.processors.get('outlier-detector')?.config; - - if (data.price && data.prevClose) { - const priceDeviation = Math.abs(data.price - data.prevClose) / data.prevClose; - if (priceDeviation > (config?.priceDeviationThreshold || 0.1)) { - this.logger.warn({ - symbol: data.symbol, - price: data.price, - prevClose: data.prevClose, - deviation: priceDeviation - }, 'Price outlier detected'); - - // You could either filter out or flag the data - data.outlier = true; - data.outlierReason = 'price_deviation'; - } - } - - if (data.volume) { - const volumeThreshold = config?.volumeThreshold || 1000000; - if (data.volume > volumeThreshold) { - data.highVolume = true; - } - } - - return data; - } - - private async filterMarketHours(data: any): Promise { - const config = this.processors.get('market-hours-filter')?.config; - - if (!config?.marketOpen || !config?.marketClose) { - return data; - } - - // Simple market hours check - in practice, you'd use proper timezone handling - const timestamp = new Date(data.timestamp); - const timeString = timestamp.toTimeString().substring(0, 5); - - if (timeString < config.marketOpen || timeString > config.marketClose) { - // Mark as after hours - data.afterHours = true; - } - - return data; - } - - private async aggregateOHLC(data: any): Promise { - // This is a simplified version - in practice, you'd maintain - // aggregation windows and emit candles when complete - if (data.dataType === 'quote' && data.price) { - const candle = { - symbol: data.symbol, - timestamp: data.timestamp, - open: data.price, - high: data.price, - low: data.price, - close: data.price, - volume: data.volume || 0, - timeframe: '1m', - source: data.source, - dataType: 'candle' - }; - - // In practice, you'd emit this separately or include it in results - this.emit('candle-generated', candle); - } - - return data; - } - - private getMarketSession(timestamp: number): string { - const date = new Date(timestamp); - const timeString = date.toTimeString().substring(0, 5); - - if (timeString < '09:30') return 'pre-market'; - if (timeString <= '16:00') return 'regular'; - if (timeString <= '20:00') return 'after-hours'; - return 'closed'; - } - - public async updateConfig(config: any): Promise { - this.config = config; - this.logger.info('Processing engine configuration updated'); - - // Update pipelines if provided - if (config.pipelines) { - // Clear existing pipelines - this.pipelines.clear(); - - // Add new pipelines - for (const pipeline of config.pipelines) { - this.addPipeline(pipeline); - } - } - } -} diff --git a/apps/core-services/market-data-gateway/src/services/ServiceIntegrationManager.ts b/apps/core-services/market-data-gateway/src/services/ServiceIntegrationManager.ts deleted file mode 100644 index fbeca5c..0000000 --- a/apps/core-services/market-data-gateway/src/services/ServiceIntegrationManager.ts +++ /dev/null @@ -1,540 +0,0 @@ -import { EventEmitter } from 'events'; -import axios, { AxiosInstance } from 'axios'; -import { - ServiceIntegration, - Logger, - HealthStatus, - MarketDataTick, - MarketDataCandle, - ProcessedData, - DataPipelineJob, - FeatureComputationRequest, - DataAsset -} from '../types/MarketDataGateway'; - -interface ServiceEndpoint { - baseUrl: string; - timeout: number; - retries: number; - healthPath: string; -} - -interface ServiceHealth { - serviceId: string; - status: 'healthy' | 'degraded' | 'unhealthy' | 'unreachable'; - lastCheck: number; - responseTime: number; - errorCount: number; -} - -interface IntegrationMetrics { - totalRequests: number; - successfulRequests: number; - failedRequests: number; - averageResponseTime: number; - lastRequestTime: number; -} - -export class ServiceIntegrationManager extends EventEmitter { - private config: ServiceIntegration; - private logger: Logger; - private httpClients: Map; - private serviceHealth: Map; - private integrationMetrics: Map; - private healthCheckInterval: NodeJS.Timeout | null = null; - private isInitialized: boolean = false; - - constructor(config: ServiceIntegration, logger: Logger) { - super(); - this.config = config; - this.logger = logger; - this.httpClients = new Map(); - this.serviceHealth = new Map(); - this.integrationMetrics = new Map(); - } - - async initialize(): Promise { - try { - // Initialize HTTP clients for each service - const services = [ - { id: 'data-processor', config: this.config.dataProcessor }, - { id: 'feature-store', config: this.config.featureStore }, - { id: 'data-catalog', config: this.config.dataCatalog }, - ]; - - for (const service of services) { - if (service.config.enabled) { - const client = axios.create({ - baseURL: service.config.baseUrl, - timeout: service.config.timeout || 30000, - headers: { - 'Content-Type': 'application/json', - 'User-Agent': 'market-data-gateway/1.0.0', - }, - }); - - // Add request interceptor for metrics - client.interceptors.request.use((config) => { - const startTime = Date.now(); - config.metadata = { startTime }; - return config; - }); - - // Add response interceptor for metrics and error handling - client.interceptors.response.use( - (response) => { - const endTime = Date.now(); - const startTime = response.config.metadata?.startTime || endTime; - this.updateMetrics(service.id, true, endTime - startTime); - return response; - }, - (error) => { - const endTime = Date.now(); - const startTime = error.config?.metadata?.startTime || endTime; - this.updateMetrics(service.id, false, endTime - startTime); - return Promise.reject(error); - } - ); - - this.httpClients.set(service.id, client); - - // Initialize health tracking - this.serviceHealth.set(service.id, { - serviceId: service.id, - status: 'unreachable', - lastCheck: 0, - responseTime: 0, - errorCount: 0, - }); - - // Initialize metrics - this.integrationMetrics.set(service.id, { - totalRequests: 0, - successfulRequests: 0, - failedRequests: 0, - averageResponseTime: 0, - lastRequestTime: 0, - }); - } - } - - // Start health monitoring - this.startHealthMonitoring(); - this.isInitialized = true; - - this.logger.info('Service integration manager initialized successfully'); - this.emit('initialized'); - } catch (error) { - this.logger.error('Failed to initialize service integration manager:', error); - throw error; - } - } - - async shutdown(): Promise { - try { - if (this.healthCheckInterval) { - clearInterval(this.healthCheckInterval); - this.healthCheckInterval = null; - } - - this.isInitialized = false; - this.logger.info('Service integration manager shut down successfully'); - this.emit('shutdown'); - } catch (error) { - this.logger.error('Error shutting down service integration manager:', error); - } - } - - // Data Processor Integration - async sendToDataProcessor(data: ProcessedData[]): Promise { - if (!this.config.dataProcessor.enabled) { - this.logger.debug('Data processor integration disabled'); - return; - } - - try { - const client = this.httpClients.get('data-processor'); - if (!client) throw new Error('Data processor client not initialized'); - - const payload = { - source: 'market-data-gateway', - timestamp: new Date().toISOString(), - data: data, - }; - - const response = await client.post('/api/v1/data/ingest', payload); - - this.logger.debug(`Sent ${data.length} records to data processor`); - this.emit('data-sent', { service: 'data-processor', count: data.length }); - - return response.data; - } catch (error) { - this.logger.error('Failed to send data to data processor:', error); - this.emit('integration-error', { service: 'data-processor', error }); - throw error; - } - } - - async createDataPipeline(pipelineConfig: any): Promise { - if (!this.config.dataProcessor.enabled) { - throw new Error('Data processor integration disabled'); - } - - try { - const client = this.httpClients.get('data-processor'); - if (!client) throw new Error('Data processor client not initialized'); - - const response = await client.post('/api/v1/pipelines', pipelineConfig); - - this.logger.info(`Created data pipeline: ${response.data.id}`); - return response.data.id; - } catch (error) { - this.logger.error('Failed to create data pipeline:', error); - throw error; - } - } - - async triggerPipelineJob(pipelineId: string, jobConfig: Partial): Promise { - if (!this.config.dataProcessor.enabled) { - throw new Error('Data processor integration disabled'); - } - - try { - const client = this.httpClients.get('data-processor'); - if (!client) throw new Error('Data processor client not initialized'); - - const response = await client.post(`/api/v1/pipelines/${pipelineId}/jobs`, jobConfig); - - this.logger.info(`Triggered pipeline job: ${response.data.jobId}`); - return response.data.jobId; - } catch (error) { - this.logger.error('Failed to trigger pipeline job:', error); - throw error; - } - } - - // Feature Store Integration - async publishToFeatureStore(features: any[]): Promise { - if (!this.config.featureStore.enabled) { - this.logger.debug('Feature store integration disabled'); - return; - } - - try { - const client = this.httpClients.get('feature-store'); - if (!client) throw new Error('Feature store client not initialized'); - - const payload = { - source: 'market-data-gateway', - timestamp: new Date().toISOString(), - features: features, - }; - - const response = await client.post('/api/v1/features/ingest', payload); - - this.logger.debug(`Published ${features.length} features to feature store`); - this.emit('features-published', { count: features.length }); - - return response.data; - } catch (error) { - this.logger.error('Failed to publish features to feature store:', error); - this.emit('integration-error', { service: 'feature-store', error }); - throw error; - } - } - - async requestFeatureComputation(request: FeatureComputationRequest): Promise { - if (!this.config.featureStore.enabled) { - throw new Error('Feature store integration disabled'); - } - - try { - const client = this.httpClients.get('feature-store'); - if (!client) throw new Error('Feature store client not initialized'); - - const response = await client.post('/api/v1/features/compute', request); - - this.logger.info(`Requested feature computation: ${request.featureGroupId}`); - return response.data; - } catch (error) { - this.logger.error('Failed to request feature computation:', error); - throw error; - } - } - - async getFeatureGroup(featureGroupId: string): Promise { - if (!this.config.featureStore.enabled) { - throw new Error('Feature store integration disabled'); - } - - try { - const client = this.httpClients.get('feature-store'); - if (!client) throw new Error('Feature store client not initialized'); - - const response = await client.get(`/api/v1/feature-groups/${featureGroupId}`); - return response.data; - } catch (error) { - this.logger.error(`Failed to get feature group ${featureGroupId}:`, error); - throw error; - } - } - - // Data Catalog Integration - async registerDataAsset(asset: Omit): Promise { - if (!this.config.dataCatalog.enabled) { - this.logger.debug('Data catalog integration disabled'); - return ''; - } - - try { - const client = this.httpClients.get('data-catalog'); - if (!client) throw new Error('Data catalog client not initialized'); - - const response = await client.post('/api/v1/assets', asset); - - this.logger.info(`Registered data asset: ${asset.name}`); - this.emit('asset-registered', { assetId: response.data.id, name: asset.name }); - - return response.data.id; - } catch (error) { - this.logger.error('Failed to register data asset:', error); - this.emit('integration-error', { service: 'data-catalog', error }); - throw error; - } - } - - async updateDataLineage(fromAssetId: string, toAssetId: string, transformationType: string): Promise { - if (!this.config.dataCatalog.enabled) { - this.logger.debug('Data catalog integration disabled'); - return; - } - - try { - const client = this.httpClients.get('data-catalog'); - if (!client) throw new Error('Data catalog client not initialized'); - - const lineageData = { - fromAssetId, - toAssetId, - transformationType, - timestamp: new Date().toISOString(), - source: 'market-data-gateway', - }; - - await client.post('/api/v1/lineage', lineageData); - - this.logger.debug(`Updated data lineage: ${fromAssetId} -> ${toAssetId}`); - this.emit('lineage-updated', lineageData); - } catch (error) { - this.logger.error('Failed to update data lineage:', error); - this.emit('integration-error', { service: 'data-catalog', error }); - throw error; - } - } - - async reportDataQuality(assetId: string, qualityMetrics: any): Promise { - if (!this.config.dataCatalog.enabled) { - this.logger.debug('Data catalog integration disabled'); - return; - } - - try { - const client = this.httpClients.get('data-catalog'); - if (!client) throw new Error('Data catalog client not initialized'); - - const qualityReport = { - assetId, - metrics: qualityMetrics, - timestamp: new Date().toISOString(), - source: 'market-data-gateway', - }; - - await client.post('/api/v1/quality/reports', qualityReport); - - this.logger.debug(`Reported data quality for asset: ${assetId}`); - this.emit('quality-reported', { assetId, metrics: qualityMetrics }); - } catch (error) { - this.logger.error('Failed to report data quality:', error); - this.emit('integration-error', { service: 'data-catalog', error }); - throw error; - } - } - - // Health monitoring - private startHealthMonitoring(): void { - this.healthCheckInterval = setInterval(() => { - this.checkServiceHealth(); - }, 30000); // Check every 30 seconds - } - - private async checkServiceHealth(): Promise { - const healthPromises = Array.from(this.httpClients.entries()).map( - async ([serviceId, client]) => { - const startTime = Date.now(); - try { - await client.get('/health'); - const responseTime = Date.now() - startTime; - - this.updateServiceHealth(serviceId, 'healthy', responseTime, false); - } catch (error) { - const responseTime = Date.now() - startTime; - this.updateServiceHealth(serviceId, 'unhealthy', responseTime, true); - } - } - ); - - await Promise.allSettled(healthPromises); - } - - private updateServiceHealth( - serviceId: string, - status: ServiceHealth['status'], - responseTime: number, - isError: boolean - ): void { - const health = this.serviceHealth.get(serviceId); - if (!health) return; - - health.status = status; - health.lastCheck = Date.now(); - health.responseTime = responseTime; - - if (isError) { - health.errorCount++; - } else { - health.errorCount = Math.max(0, health.errorCount - 1); // Decay error count - } - - this.serviceHealth.set(serviceId, health); - this.emit('service-health-updated', { serviceId, health }); - } - - private updateMetrics(serviceId: string, success: boolean, responseTime: number): void { - const metrics = this.integrationMetrics.get(serviceId); - if (!metrics) return; - - metrics.totalRequests++; - metrics.lastRequestTime = Date.now(); - - if (success) { - metrics.successfulRequests++; - } else { - metrics.failedRequests++; - } - - // Update average response time - const totalSuccessful = metrics.successfulRequests; - if (totalSuccessful > 0) { - metrics.averageResponseTime = - (metrics.averageResponseTime * (totalSuccessful - 1) + responseTime) / totalSuccessful; - } - - this.integrationMetrics.set(serviceId, metrics); - } - - // Status and metrics - getServiceHealth(serviceId?: string): ServiceHealth | ServiceHealth[] { - if (serviceId) { - return this.serviceHealth.get(serviceId) || { - serviceId, - status: 'unreachable', - lastCheck: 0, - responseTime: 0, - errorCount: 0, - }; - } - - return Array.from(this.serviceHealth.values()); - } - - getIntegrationMetrics(serviceId?: string): IntegrationMetrics | IntegrationMetrics[] { - if (serviceId) { - return this.integrationMetrics.get(serviceId) || { - totalRequests: 0, - successfulRequests: 0, - failedRequests: 0, - averageResponseTime: 0, - lastRequestTime: 0, - }; - } - - return Array.from(this.integrationMetrics.values()); - } - - getHealth(): HealthStatus { - const allHealthy = Array.from(this.serviceHealth.values()).every( - health => health.status === 'healthy' - ); - - const degradedServices = Array.from(this.serviceHealth.values()).filter( - health => health.status === 'degraded' - ); - - const unhealthyServices = Array.from(this.serviceHealth.values()).filter( - health => health.status === 'unhealthy' || health.status === 'unreachable' - ); - - let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy'; - let message = 'All service integrations are healthy'; - - if (unhealthyServices.length > 0) { - status = 'unhealthy'; - message = `${unhealthyServices.length} services are unhealthy`; - } else if (degradedServices.length > 0) { - status = 'degraded'; - message = `${degradedServices.length} services are degraded`; - } - - return { - status, - message, - timestamp: new Date().toISOString(), - details: { - isInitialized: this.isInitialized, - totalServices: this.serviceHealth.size, - healthyServices: Array.from(this.serviceHealth.values()).filter(h => h.status === 'healthy').length, - degradedServices: degradedServices.length, - unhealthyServices: unhealthyServices.length, - serviceHealth: Object.fromEntries(this.serviceHealth), - integrationMetrics: Object.fromEntries(this.integrationMetrics), - }, - }; - } - - // Configuration management - updateServiceConfig(serviceId: string, config: Partial): void { - const currentConfig = this.getServiceConfig(serviceId); - if (!currentConfig) { - this.logger.error(`Service ${serviceId} not found for config update`); - return; - } - - // Update the configuration - Object.assign(currentConfig, config); - - // Reinitialize the HTTP client if URL changed - if (config.baseUrl) { - const client = this.httpClients.get(serviceId); - if (client) { - client.defaults.baseURL = config.baseUrl; - client.defaults.timeout = config.timeout || client.defaults.timeout; - } - } - - this.logger.info(`Updated configuration for service: ${serviceId}`); - this.emit('service-config-updated', { serviceId, config }); - } - - private getServiceConfig(serviceId: string): any { - switch (serviceId) { - case 'data-processor': - return this.config.dataProcessor; - case 'feature-store': - return this.config.featureStore; - case 'data-catalog': - return this.config.dataCatalog; - default: - return null; - } - } -} diff --git a/apps/core-services/market-data-gateway/src/services/SubscriptionManager.ts b/apps/core-services/market-data-gateway/src/services/SubscriptionManager.ts deleted file mode 100644 index 6b197b8..0000000 --- a/apps/core-services/market-data-gateway/src/services/SubscriptionManager.ts +++ /dev/null @@ -1,617 +0,0 @@ -import { EventEmitter } from 'eventemitter3'; -import { Logger } from 'pino'; -import WebSocket from 'ws'; -import { - SubscriptionRequest, - ClientSubscription, - WebSocketMessage, - WebSocketDataMessage, - MarketDataTick, - MarketDataCandle, - MarketDataTrade -} from '../types/MarketDataGateway'; - -interface WebSocketClient { - id: string; - ws: WebSocket; - subscriptions: Set; - connectedAt: Date; - lastPing: Date; - metadata: { - userAgent?: string; - ip?: string; - userId?: string; - }; -} - -export class SubscriptionManager extends EventEmitter { - private logger: Logger; - private subscriptions: Map = new Map(); - private clients: Map = new Map(); - private symbolSubscriptions: Map> = new Map(); // symbol -> subscription IDs - private heartbeatInterval?: NodeJS.Timeout; - private cleanupInterval?: NodeJS.Timeout; - - constructor(logger: Logger) { - super(); - this.logger = logger; - } - - public async start(): Promise { - this.logger.info('Starting Subscription Manager'); - - // Start heartbeat for WebSocket clients - this.startHeartbeat(); - - // Start cleanup for stale subscriptions - this.startCleanup(); - - this.logger.info('Subscription Manager started'); - } - - public async stop(): Promise { - this.logger.info('Stopping Subscription Manager'); - - // Clear intervals - if (this.heartbeatInterval) { - clearInterval(this.heartbeatInterval); - } - - if (this.cleanupInterval) { - clearInterval(this.cleanupInterval); - } - - // Close all WebSocket connections - for (const client of this.clients.values()) { - client.ws.close(); - } - - this.clients.clear(); - this.subscriptions.clear(); - this.symbolSubscriptions.clear(); - - this.logger.info('Subscription Manager stopped'); - } - - public async subscribe(request: SubscriptionRequest): Promise { - this.logger.info({ - clientId: request.clientId, - symbols: request.symbols, - dataTypes: request.dataTypes - }, 'Creating subscription'); - - // Validate subscription request - this.validateSubscriptionRequest(request); - - // Create subscription - const subscription: ClientSubscription = { - request, - status: 'active', - connectedAt: new Date(), - lastUpdate: new Date(), - metrics: { - messagesDelivered: 0, - bytesTransferred: 0, - errors: 0, - avgLatencyMs: 0 - } - }; - - this.subscriptions.set(request.id, subscription); - - // Track symbol subscriptions for efficient lookup - for (const symbol of request.symbols) { - if (!this.symbolSubscriptions.has(symbol)) { - this.symbolSubscriptions.set(symbol, new Set()); - } - this.symbolSubscriptions.get(symbol)!.add(request.id); - } - - this.emit('subscribed', subscription); - - this.logger.info({ subscriptionId: request.id }, 'Subscription created'); - return request.id; - } - - public async unsubscribe(subscriptionId: string): Promise { - const subscription = this.subscriptions.get(subscriptionId); - if (!subscription) { - throw new Error(`Subscription ${subscriptionId} not found`); - } - - this.logger.info({ subscriptionId }, 'Removing subscription'); - - // Remove from symbol tracking - for (const symbol of subscription.request.symbols) { - const symbolSubs = this.symbolSubscriptions.get(symbol); - if (symbolSubs) { - symbolSubs.delete(subscriptionId); - if (symbolSubs.size === 0) { - this.symbolSubscriptions.delete(symbol); - } - } - } - - // Remove subscription - this.subscriptions.delete(subscriptionId); - - this.emit('unsubscribed', subscription.request.clientId); - - this.logger.info({ subscriptionId }, 'Subscription removed'); - } - - public getSubscriptions(clientId?: string): ClientSubscription[] { - const subscriptions = Array.from(this.subscriptions.values()); - - if (clientId) { - return subscriptions.filter(sub => sub.request.clientId === clientId); - } - - return subscriptions; - } - - public async broadcast(data: MarketDataTick | MarketDataCandle | MarketDataTrade): Promise { - const symbol = data.symbol; - const dataType = this.getDataType(data); - - // Get subscriptions for this symbol - const subscriptionIds = this.symbolSubscriptions.get(symbol); - if (!subscriptionIds || subscriptionIds.size === 0) { - return; - } - - const deliveryPromises: Promise[] = []; - - for (const subscriptionId of subscriptionIds) { - const subscription = this.subscriptions.get(subscriptionId); - if (!subscription || subscription.status !== 'active') { - continue; - } - - // Check if subscription wants this data type - if (!subscription.request.dataTypes.includes(dataType as any)) { - continue; - } - - // Apply filters - if (!this.passesFilters(data, subscription.request.filters)) { - continue; - } - - // Apply throttling if configured - if (subscription.request.throttle && !this.passesThrottle(subscription)) { - continue; - } - - // Deliver data based on delivery method - deliveryPromises.push(this.deliverData(subscription, data)); - } - - // Wait for all deliveries - await Promise.allSettled(deliveryPromises); - } - - public addWebSocketClient(ws: WebSocket, clientId: string, metadata: any = {}): void { - this.logger.info({ clientId }, 'Adding WebSocket client'); - - const client: WebSocketClient = { - id: clientId, - ws, - subscriptions: new Set(), - connectedAt: new Date(), - lastPing: new Date(), - metadata - }; - - this.clients.set(clientId, client); - - // Setup WebSocket event handlers - ws.on('message', (message: Buffer) => { - this.handleWebSocketMessage(clientId, message); - }); - - ws.on('close', () => { - this.removeWebSocketClient(clientId); - }); - - ws.on('error', (error) => { - this.logger.error({ clientId, error }, 'WebSocket client error'); - this.removeWebSocketClient(clientId); - }); - - ws.on('pong', () => { - const client = this.clients.get(clientId); - if (client) { - client.lastPing = new Date(); - } - }); - - // Send welcome message - this.sendWebSocketMessage(ws, { - type: 'status', - id: 'welcome', - timestamp: Date.now(), - payload: { - status: 'connected', - clientId, - serverTime: new Date().toISOString() - } - }); - } - - public removeWebSocketClient(clientId: string): void { - const client = this.clients.get(clientId); - if (!client) { - return; - } - - this.logger.info({ clientId }, 'Removing WebSocket client'); - - // Unsubscribe from all subscriptions - for (const subscriptionId of client.subscriptions) { - try { - this.unsubscribe(subscriptionId); - } catch (error) { - this.logger.error({ subscriptionId, error }, 'Error unsubscribing client'); - } - } - - // Close WebSocket if still open - if (client.ws.readyState === WebSocket.OPEN) { - client.ws.close(); - } - - this.clients.delete(clientId); - } - - private validateSubscriptionRequest(request: SubscriptionRequest): void { - if (!request.id) { - throw new Error('Subscription ID is required'); - } - - if (!request.clientId) { - throw new Error('Client ID is required'); - } - - if (!request.symbols || request.symbols.length === 0) { - throw new Error('At least one symbol is required'); - } - - if (!request.dataTypes || request.dataTypes.length === 0) { - throw new Error('At least one data type is required'); - } - - if (this.subscriptions.has(request.id)) { - throw new Error(`Subscription ${request.id} already exists`); - } - - // Validate symbols format - for (const symbol of request.symbols) { - if (typeof symbol !== 'string' || symbol.length === 0) { - throw new Error(`Invalid symbol: ${symbol}`); - } - } - - // Validate data types - const validDataTypes = ['quotes', 'trades', 'orderbook', 'candles', 'news']; - for (const dataType of request.dataTypes) { - if (!validDataTypes.includes(dataType)) { - throw new Error(`Invalid data type: ${dataType}`); - } - } - } - - private getDataType(data: any): string { - if (data.id && data.side) return 'trades'; - if (data.open !== undefined && data.high !== undefined) return 'candles'; - if (data.price !== undefined) return 'quotes'; - if (data.bids || data.asks) return 'orderbook'; - return 'unknown'; - } - - private passesFilters(data: any, filters?: any): boolean { - if (!filters) { - return true; - } - - // Price range filter - if (filters.priceRange && data.price) { - if (data.price < filters.priceRange.min || data.price > filters.priceRange.max) { - return false; - } - } - - // Volume threshold filter - if (filters.volumeThreshold && data.volume) { - if (data.volume < filters.volumeThreshold) { - return false; - } - } - - // Exchange filter - if (filters.exchanges && data.exchange) { - if (!filters.exchanges.includes(data.exchange)) { - return false; - } - } - - return true; - } - - private passesThrottle(subscription: ClientSubscription): boolean { - const throttle = subscription.request.throttle; - if (!throttle) { - return true; - } - - const now = Date.now(); - const timeSinceLastUpdate = now - subscription.lastUpdate.getTime(); - const minInterval = 1000 / throttle.maxUpdatesPerSecond; - - return timeSinceLastUpdate >= minInterval; - } - - private async deliverData(subscription: ClientSubscription, data: any): Promise { - const startTime = Date.now(); - - try { - const message: WebSocketDataMessage = { - type: 'data', - id: subscription.request.id, - timestamp: Date.now(), - payload: { - dataType: this.getDataType(data), - data - } - }; - - switch (subscription.request.delivery.method) { - case 'websocket': - await this.deliverViaWebSocket(subscription, message); - break; - case 'webhook': - await this.deliverViaWebhook(subscription, message); - break; - case 'eventbus': - await this.deliverViaEventBus(subscription, message); - break; - default: - throw new Error(`Unsupported delivery method: ${subscription.request.delivery.method}`); - } - - // Update metrics - const latency = Date.now() - startTime; - subscription.metrics.messagesDelivered++; - subscription.metrics.avgLatencyMs = - (subscription.metrics.avgLatencyMs * (subscription.metrics.messagesDelivered - 1) + latency) / - subscription.metrics.messagesDelivered; - subscription.lastUpdate = new Date(); - - } catch (error) { - this.logger.error({ - subscriptionId: subscription.request.id, - error - }, 'Error delivering data'); - - subscription.metrics.errors++; - - if (subscription.metrics.errors > 10) { - subscription.status = 'error'; - this.emit('error', error, subscription.request.clientId); - } - } - } - - private async deliverViaWebSocket(subscription: ClientSubscription, message: WebSocketDataMessage): Promise { - const client = this.clients.get(subscription.request.clientId); - if (!client || client.ws.readyState !== WebSocket.OPEN) { - throw new Error('WebSocket client not available'); - } - - this.sendWebSocketMessage(client.ws, message); - - const messageSize = JSON.stringify(message).length; - subscription.metrics.bytesTransferred += messageSize; - } - - private async deliverViaWebhook(subscription: ClientSubscription, message: any): Promise { - // Webhook delivery implementation would go here - // This would use HTTP POST to deliver the data - throw new Error('Webhook delivery not implemented'); - } - - private async deliverViaEventBus(subscription: ClientSubscription, message: any): Promise { - // Event bus delivery implementation would go here - // This would publish to the event bus - this.emit('event-bus-delivery', subscription.request.clientId, message); - } - - private sendWebSocketMessage(ws: WebSocket, message: WebSocketMessage): void { - if (ws.readyState === WebSocket.OPEN) { - ws.send(JSON.stringify(message)); - } - } - - private handleWebSocketMessage(clientId: string, message: Buffer): void { - try { - const parsedMessage = JSON.parse(message.toString()) as WebSocketMessage; - - switch (parsedMessage.type) { - case 'subscribe': - this.handleWebSocketSubscribe(clientId, parsedMessage as any); - break; - case 'unsubscribe': - this.handleWebSocketUnsubscribe(clientId, parsedMessage); - break; - case 'heartbeat': - this.handleWebSocketHeartbeat(clientId); - break; - default: - this.logger.warn({ clientId, messageType: parsedMessage.type }, 'Unknown WebSocket message type'); - } - } catch (error) { - this.logger.error({ clientId, error }, 'Error parsing WebSocket message'); - } - } - - private async handleWebSocketSubscribe(clientId: string, message: any): Promise { - try { - const subscriptionRequest: SubscriptionRequest = { - id: `${clientId}-${Date.now()}`, - clientId, - symbols: message.payload.symbols, - dataTypes: message.payload.dataTypes, - filters: message.payload.filters, - throttle: message.payload.throttle, - delivery: { - method: 'websocket', - format: 'json' - } - }; - - const subscriptionId = await this.subscribe(subscriptionRequest); - - const client = this.clients.get(clientId); - if (client) { - client.subscriptions.add(subscriptionId); - } - - // Send confirmation - const confirmationMessage: WebSocketMessage = { - type: 'status', - id: message.id, - timestamp: Date.now(), - payload: { - status: 'subscribed', - subscriptionId, - symbols: subscriptionRequest.symbols, - dataTypes: subscriptionRequest.dataTypes - } - }; - - const ws = this.clients.get(clientId)?.ws; - if (ws) { - this.sendWebSocketMessage(ws, confirmationMessage); - } - - } catch (error) { - this.logger.error({ clientId, error }, 'Error handling WebSocket subscribe'); - - // Send error message - const errorMessage: WebSocketMessage = { - type: 'error', - id: message.id, - timestamp: Date.now(), - payload: { - error: error instanceof Error ? error.message : 'Unknown error' - } - }; - - const ws = this.clients.get(clientId)?.ws; - if (ws) { - this.sendWebSocketMessage(ws, errorMessage); - } - } - } - - private async handleWebSocketUnsubscribe(clientId: string, message: WebSocketMessage): Promise { - try { - const subscriptionId = message.payload?.subscriptionId; - if (!subscriptionId) { - throw new Error('Subscription ID is required'); - } - - await this.unsubscribe(subscriptionId); - - const client = this.clients.get(clientId); - if (client) { - client.subscriptions.delete(subscriptionId); - } - - // Send confirmation - const confirmationMessage: WebSocketMessage = { - type: 'status', - id: message.id, - timestamp: Date.now(), - payload: { - status: 'unsubscribed', - subscriptionId - } - }; - - const ws = this.clients.get(clientId)?.ws; - if (ws) { - this.sendWebSocketMessage(ws, confirmationMessage); - } - - } catch (error) { - this.logger.error({ clientId, error }, 'Error handling WebSocket unsubscribe'); - } - } - - private handleWebSocketHeartbeat(clientId: string): void { - const client = this.clients.get(clientId); - if (client) { - client.lastPing = new Date(); - - const heartbeatMessage: WebSocketMessage = { - type: 'heartbeat', - timestamp: Date.now(), - payload: { - serverTime: new Date().toISOString() - } - }; - - this.sendWebSocketMessage(client.ws, heartbeatMessage); - } - } - - private startHeartbeat(): void { - this.heartbeatInterval = setInterval(() => { - const now = Date.now(); - const timeout = 60000; // 60 seconds - - for (const [clientId, client] of this.clients.entries()) { - const timeSinceLastPing = now - client.lastPing.getTime(); - - if (timeSinceLastPing > timeout) { - this.logger.warn({ clientId }, 'Client heartbeat timeout'); - this.removeWebSocketClient(clientId); - } else if (client.ws.readyState === WebSocket.OPEN) { - // Send ping - client.ws.ping(); - } - } - }, 30000); // Check every 30 seconds - } - - private startCleanup(): void { - this.cleanupInterval = setInterval(() => { - const now = Date.now(); - const maxAge = 24 * 60 * 60 * 1000; // 24 hours - - for (const [subscriptionId, subscription] of this.subscriptions.entries()) { - const age = now - subscription.connectedAt.getTime(); - - if (subscription.status === 'error' || age > maxAge) { - this.logger.info({ subscriptionId }, 'Cleaning up stale subscription'); - this.unsubscribe(subscriptionId); - } - } - }, 60000); // Check every minute - } - - public getMetrics() { - return { - totalSubscriptions: this.subscriptions.size, - activeSubscriptions: Array.from(this.subscriptions.values()) - .filter(sub => sub.status === 'active').length, - connectedClients: this.clients.size, - symbolsTracked: this.symbolSubscriptions.size, - totalMessagesDelivered: Array.from(this.subscriptions.values()) - .reduce((sum, sub) => sum + sub.metrics.messagesDelivered, 0), - totalErrors: Array.from(this.subscriptions.values()) - .reduce((sum, sub) => sum + sub.metrics.errors, 0) - }; - } -} diff --git a/apps/core-services/market-data-gateway/src/shared/index.ts b/apps/core-services/market-data-gateway/src/shared/index.ts deleted file mode 100644 index 71e0850..0000000 --- a/apps/core-services/market-data-gateway/src/shared/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Shared components used by both realtime and storage modules -export { CacheManager } from '../services/CacheManager'; -export { DataNormalizer } from '../services/DataNormalizer'; -export { MetricsCollector } from '../services/MetricsCollector'; -export { ServiceIntegrationManager } from '../services/ServiceIntegrationManager'; diff --git a/apps/core-services/market-data-gateway/src/storage/ArchivalService.ts b/apps/core-services/market-data-gateway/src/storage/ArchivalService.ts deleted file mode 100644 index 37f87d5..0000000 --- a/apps/core-services/market-data-gateway/src/storage/ArchivalService.ts +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Archival service for managing data lifecycle and storage tiers - * Handles cold storage, data compression, and retention policies - */ -export class ArchivalService { - private compressionLevel: number; - private retentionPolicies: Map; - - constructor() { - this.compressionLevel = 6; // Default compression level - this.retentionPolicies = new Map(); - } - - /** - * Archive old data to cold storage - */ - async archiveData(symbol: string, cutoffDate: Date): Promise { - try { - console.log(`Archiving data for ${symbol} before ${cutoffDate}`); - // Implementation for archiving - } catch (error) { - console.error('Error archiving data:', error); - throw error; - } - } - - /** - * Compress data for storage optimization - */ - async compressData(data: any[]): Promise { - try { - // Implementation for data compression - return Buffer.from(JSON.stringify(data)); - } catch (error) { - console.error('Error compressing data:', error); - throw error; - } - } - - /** - * Apply retention policies - */ - async applyRetentionPolicies(): Promise { - try { - console.log('Applying retention policies...'); - // Implementation for applying retention policies - } catch (error) { - console.error('Error applying retention policies:', error); - throw error; - } - } -} diff --git a/apps/core-services/market-data-gateway/src/storage/QueryEngine.ts b/apps/core-services/market-data-gateway/src/storage/QueryEngine.ts deleted file mode 100644 index cc29f65..0000000 --- a/apps/core-services/market-data-gateway/src/storage/QueryEngine.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { TimeSeriesStorage } from './TimeSeriesStorage'; - -/** - * Query engine for efficient historical data retrieval - * Optimizes queries and provides various aggregation capabilities - */ -export class QueryEngine { - private storage: TimeSeriesStorage; - - constructor(storage: TimeSeriesStorage) { - this.storage = storage; - } - - /** - * Execute optimized query with caching - */ - async executeQuery(queryParams: any): Promise { - try { - // Implementation for optimized queries - console.log('Executing optimized query:', queryParams); - return []; - } catch (error) { - console.error('Error executing query:', error); - throw error; - } - } - - /** - * Aggregate data by time intervals - */ - async aggregateByInterval( - symbol: string, - interval: string, - startTime: Date, - endTime: Date - ): Promise { - try { - // Implementation for aggregation - console.log(`Aggregating ${symbol} by ${interval}`); - return []; - } catch (error) { - console.error('Error aggregating data:', error); - throw error; - } - } -} diff --git a/apps/core-services/market-data-gateway/src/storage/TimeSeriesStorage.ts b/apps/core-services/market-data-gateway/src/storage/TimeSeriesStorage.ts deleted file mode 100644 index 37e6113..0000000 --- a/apps/core-services/market-data-gateway/src/storage/TimeSeriesStorage.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { CacheManager } from '../services/CacheManager'; -import { DataNormalizer } from '../services/DataNormalizer'; -import { MetricsCollector } from '../services/MetricsCollector'; - -/** - * Historical data storage and retrieval service - * Handles time-series storage, archival, and query capabilities - */ -export class TimeSeriesStorage { - private cache: CacheManager; - private normalizer: DataNormalizer; - private metrics: MetricsCollector; - - constructor( - cache: CacheManager, - normalizer: DataNormalizer, - metrics: MetricsCollector - ) { - this.cache = cache; - this.normalizer = normalizer; - this.metrics = metrics; - } - - /** - * Store historical market data - */ - async storeHistoricalData(symbol: string, data: any[]): Promise { - try { - // Implementation for storing historical data - console.log(`Storing historical data for ${symbol}:`, data.length, 'records'); - await this.metrics.incrementCounter('historical_data_stored', { symbol }); - } catch (error) { - console.error('Error storing historical data:', error); - throw error; - } - } - - /** - * Query historical data by time range - */ - async queryTimeRange( - symbol: string, - startTime: Date, - endTime: Date, - interval?: string - ): Promise { - try { - // Implementation for querying time range data - console.log(`Querying ${symbol} from ${startTime} to ${endTime}`); - await this.metrics.incrementCounter('historical_query', { symbol }); - - // Return mock data for now - return []; - } catch (error) { - console.error('Error querying historical data:', error); - throw error; - } - } - - /** - * Get data statistics and metadata - */ - async getDataStats(symbol: string): Promise { - try { - // Implementation for getting data statistics - return { - symbol, - recordCount: 0, - firstRecord: null, - lastRecord: null, - intervals: [] - }; - } catch (error) { - console.error('Error getting data stats:', error); - throw error; - } - } -} diff --git a/apps/core-services/market-data-gateway/src/storage/index.ts b/apps/core-services/market-data-gateway/src/storage/index.ts deleted file mode 100644 index ccdc38c..0000000 --- a/apps/core-services/market-data-gateway/src/storage/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -// Storage and historical data components -export { TimeSeriesStorage } from './TimeSeriesStorage'; -export { QueryEngine } from './QueryEngine'; -export { ArchivalService } from './ArchivalService'; diff --git a/apps/core-services/market-data-gateway/src/types/MarketDataGateway.ts b/apps/core-services/market-data-gateway/src/types/MarketDataGateway.ts deleted file mode 100644 index fb00621..0000000 --- a/apps/core-services/market-data-gateway/src/types/MarketDataGateway.ts +++ /dev/null @@ -1,426 +0,0 @@ -// Market Data Gateway Types - Consolidated and organized - -// Market Data Types -export interface MarketDataTick { - symbol: string; - timestamp: number; - price: number; - volume: number; - bid?: number; - ask?: number; - bidSize?: number; - askSize?: number; - source: string; - exchange?: string; - lastTradeSize?: number; - dayHigh?: number; - dayLow?: number; - dayOpen?: number; - prevClose?: number; - change?: number; - changePercent?: number; -} - -export interface MarketDataCandle { - symbol: string; - timestamp: number; - open: number; - high: number; - low: number; - close: number; - volume: number; - timeframe: string; - source: string; - exchange?: string; - vwap?: number; - trades?: number; -} - -export interface MarketDataTrade { - id: string; - symbol: string; - timestamp: number; - price: number; - size: number; - side: 'buy' | 'sell'; - source: string; - exchange?: string; - conditions?: string[]; -} - -export interface MarketDataOrder { - id: string; - symbol: string; - timestamp: number; - side: 'buy' | 'sell'; - price: number; - size: number; - source: string; - exchange?: string; - orderType?: 'market' | 'limit' | 'stop'; - level?: number; -} - -// Data Source Configuration -export interface DataSourceConfig { - id: string; - name: string; - type: 'websocket' | 'rest' | 'fix' | 'stream'; - provider: string; - enabled: boolean; - priority: number; - rateLimit: { - requestsPerSecond: number; - burstLimit: number; - }; - connection: { - url: string; - headers?: Record; - queryParams?: Record; - authentication?: { - type: 'apikey' | 'oauth' | 'basic' | 'jwt'; - credentials: Record; - }; - }; - subscriptions: { - quotes: boolean; - trades: boolean; - orderbook: boolean; - candles: boolean; - news: boolean; - }; - symbols: string[]; - retryPolicy: { - maxRetries: number; - backoffMultiplier: number; - maxBackoffMs: number; - }; - healthCheck: { - intervalMs: number; - timeoutMs: number; - expectedLatencyMs: number; - }; -} - -// Data Processing Pipeline -export interface DataProcessor { - id: string; - name: string; - type: 'enrichment' | 'validation' | 'normalization' | 'aggregation' | 'filter'; - enabled: boolean; - priority: number; - config: Record; - process(data: MarketDataTick | MarketDataCandle | MarketDataTrade): Promise; -} - -export interface ProcessingPipeline { - id: string; - name: string; - processors: DataProcessor[]; - inputFilter: { - symbols?: string[]; - sources?: string[]; - dataTypes?: string[]; - }; - outputTargets: { - eventBus?: boolean; - database?: boolean; - cache?: boolean; - websocket?: boolean; - dataProcessor?: boolean; - featureStore?: boolean; - }; -} - -// ProcessingPipelineConfig is an alias for ProcessingPipeline -export type ProcessingPipelineConfig = ProcessingPipeline; - -// Subscription Management -export interface SubscriptionRequest { - id: string; - clientId: string; - symbols: string[]; - dataTypes: ('quotes' | 'trades' | 'orderbook' | 'candles' | 'news')[]; - filters?: { - priceRange?: { min: number; max: number }; - volumeThreshold?: number; - exchanges?: string[]; - }; - throttle?: { - maxUpdatesPerSecond: number; - aggregationWindow?: number; - }; - delivery: { - method: 'websocket' | 'webhook' | 'eventbus'; - endpoint?: string; - format: 'json' | 'protobuf' | 'avro'; - }; -} - -export interface ClientSubscription { - request: SubscriptionRequest; - status: 'active' | 'paused' | 'error' | 'stopped'; - connectedAt: Date; - lastUpdate: Date; - metrics: { - messagesDelivered: number; - bytesTransferred: number; - errors: number; - avgLatencyMs: number; - }; -} - -// Gateway Configuration -export interface GatewayConfig { - server: { - port: number; - host: string; - maxConnections: number; - cors: { - origins: string[]; - methods: string[]; - headers: string[]; - }; - }; - dataSources: DataSourceConfig[]; - processing: { - pipelines: ProcessingPipeline[]; - bufferSize: number; - batchSize: number; - flushIntervalMs: number; - }; - cache: { - redis: { - host: string; - port: number; - password?: string; - db: number; - }; - ttl: { - quotes: number; - trades: number; - candles: number; - orderbook: number; - }; - }; monitoring: { - metrics: { - enabled: boolean; - port: number; - intervalMs: number; - retention: string; - }; - alerts: { - enabled: boolean; - thresholds: { - errorRate: number; - latency: number; - latencyMs: number; - connectionLoss: number; - }; - }; - }; -} - -// Metrics and Monitoring -export interface DataSourceMetrics { - sourceId: string; - status: 'connected' | 'disconnected' | 'error'; - messagesReceived: number; - bytesReceived: number; - latencyMs: number; - errorCount: number; - lastUpdate: Date; -} - -export interface GatewayMetrics { - timestamp: Date; - uptime: number; - system: { - cpuUsage: number; - memoryUsage: number; - diskUsage: number; - networkIO: { - bytesIn: number; - bytesOut: number; - }; - }; - dataSources: DataSourceMetrics[]; - subscriptions: { - total: number; - active: number; - byDataType: Record; - }; - processing: { - messagesPerSecond: number; - avgProcessingTimeMs: number; - queueDepth: number; - errorRate: number; - }; -} - -// Health Check -export interface HealthStatus { - service: string; - status: 'healthy' | 'degraded' | 'unhealthy'; - timestamp: Date; - uptime: number; - version: string; - dependencies: { - name: string; - status: 'healthy' | 'unhealthy'; - latencyMs?: number; - error?: string; - }[]; - metrics: { - connectionsActive: number; - messagesPerSecond: number; - errorRate: number; - avgLatencyMs: number; - }; -} - -// WebSocket Types -export interface WebSocketMessage { - type: string; - payload: any; - timestamp: number; - id?: string; -} - -export interface WebSocketSubscribeMessage extends WebSocketMessage { - type: 'subscribe'; - payload: SubscriptionRequest; -} - -export interface WebSocketDataMessage extends WebSocketMessage { - type: 'data'; - payload: MarketDataTick | MarketDataTrade | MarketDataCandle | MarketDataOrder; - dataType?: string; -} - -// Error Types -export interface DataSourceError { - sourceId: string; - timestamp: Date; - type: 'connection' | 'authentication' | 'ratelimit' | 'data' | 'timeout'; - message: string; - details?: Record; - severity: 'low' | 'medium' | 'high' | 'critical'; - recoverable: boolean; -} - -// Event Types -export interface MarketDataEvent { - id: string; - type: 'market.tick' | 'market.trade' | 'market.candle' | 'market.orderbook'; - source: 'market-data-gateway'; - timestamp: Date; - data: MarketDataTick | MarketDataTrade | MarketDataCandle | MarketDataOrder; - metadata?: Record; -} - -// Processing and Integration Types -export interface ProcessingError { - code: string; - message: string; - timestamp: Date; - data?: any; - source?: string; -} - -export interface ServiceIntegration { - serviceName: string; - endpoint: string; - enabled: boolean; - config: Record; - dataProcessor: { - enabled: boolean; - endpoint: string; - timeout: number; - retries: number; - }; - featureStore: { - enabled: boolean; - endpoint: string; - timeout: number; - retries: number; - }; - dataCatalog: { - enabled: boolean; - endpoint: string; - timeout: number; - retries: number; - }; -} - -export interface Logger { - info(message: string, ...args: any[]): void; - error(message: string, ...args: any[]): void; - warn(message: string, ...args: any[]): void; - debug(message: string, ...args: any[]): void; -} - -export interface ProcessedData { - source: string; - timestamp: Date; - data: any; - processedAt: Date; - metadata?: Record; -} - -export interface DataPipelineJob { - id: string; - type: string; - status: 'pending' | 'running' | 'completed' | 'failed'; - data: any; - createdAt: Date; - startedAt?: Date; - completedAt?: Date; -} - -export interface FeatureComputationRequest { - featureGroupId: string; - features: string[]; - data: any; - timestamp: Date; - metadata?: Record; -} - -export interface DataAsset { - id: string; - name: string; - type: string; - source: string; - metadata: Record; - createdAt: Date; - updatedAt: Date; -} - -// Add missing types -export interface CacheConfig { - redis: { - host: string; - port: number; - password?: string; - db: number; - }; - ttl: { - quotes: number; - trades: number; - candles: number; - orderbook: number; - }; -} - -export interface ProcessingMetrics { - totalProcessed: number; - processedPerSecond: number; - processingLatency: number; - errorCount: number; -} - -export interface SubscriptionMetrics { - totalSubscriptions: number; - messagesSent: number; - sendRate: number; -} diff --git a/apps/core-services/market-data-gateway/tsconfig.json b/apps/core-services/market-data-gateway/tsconfig.json deleted file mode 100644 index 175764e..0000000 --- a/apps/core-services/market-data-gateway/tsconfig.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "module": "ESNext", - "moduleResolution": "bundler", - "types": ["bun-types"], - "baseUrl": "../../../", - "paths": { - "@stock-bot/*": ["libs/*/src", "libs/*/dist"] - }, - "rootDir": "../../../" - }, - "include": [ - "src/**/*", - "../../../libs/*/src/**/*" - ], - "exclude": [ - "node_modules", - "dist", - "../../../libs/*/examples/**/*", - "../../../libs/**/*.test.ts", - "../../../libs/**/*.spec.ts" - ], - "references": [ - { "path": "../../../libs/config" }, - { "path": "../../../libs/types" }, - { "path": "../../../libs/logger" }, - { "path": "../../../libs/http-client" }, - { "path": "../../../libs/event-bus" } - ] -} diff --git a/apps/core-services/risk-guardian/package.json b/apps/core-services/risk-guardian/package.json deleted file mode 100644 index 9760927..0000000 --- a/apps/core-services/risk-guardian/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "risk-guardian", - "version": "1.0.0", - "description": "Real-time risk monitoring and controls service", - "main": "src/index.ts", - "scripts": { - "dev": "bun run --watch src/index.ts", - "start": "bun run src/index.ts", - "test": "echo 'No tests yet'" - }, - "dependencies": { - "hono": "^4.6.3", - "ioredis": "^5.4.1", - "@stock-bot/config": "*", - "@stock-bot/types": "*", - "ws": "^8.18.0" - }, - "devDependencies": { - "bun-types": "^1.2.15", - "@types/ws": "^8.5.12" - } -} diff --git a/apps/core-services/risk-guardian/src/index.ts b/apps/core-services/risk-guardian/src/index.ts deleted file mode 100644 index 7f64d1c..0000000 --- a/apps/core-services/risk-guardian/src/index.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { Hono } from 'hono'; -import { WebSocketServer } from 'ws'; -import Redis from 'ioredis'; - -const app = new Hono(); -const redis = new Redis({ - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - enableReadyCheck: false, - maxRetriesPerRequest: null, -}); - -// WebSocket server for real-time risk alerts -const wss = new WebSocketServer({ port: 8081 }); - -// Risk thresholds configuration -interface RiskThresholds { - maxPositionSize: number; - maxDailyLoss: number; - maxPortfolioRisk: number; - volatilityLimit: number; -} - -const defaultThresholds: RiskThresholds = { - maxPositionSize: 100000, // $100k max position - maxDailyLoss: 10000, // $10k max daily loss - maxPortfolioRisk: 0.02, // 2% portfolio risk - volatilityLimit: 0.3 // 30% volatility limit -}; - -// Health check endpoint -app.get('/health', (c) => { - return c.json({ - service: 'risk-guardian', - status: 'healthy', - timestamp: new Date(), - version: '1.0.0', - connections: wss.clients.size - }); -}); - -// Get risk thresholds -app.get('/api/risk/thresholds', async (c) => { - try { - const thresholds = await redis.hgetall('risk:thresholds'); - const parsedThresholds = Object.keys(thresholds).length > 0 - ? Object.fromEntries( - Object.entries(thresholds).map(([k, v]) => [k, parseFloat(v as string)]) - ) - : defaultThresholds; - - return c.json({ - success: true, - data: parsedThresholds - }); - } catch (error) { - console.error('Error fetching risk thresholds:', error); - return c.json({ success: false, error: 'Failed to fetch thresholds' }, 500); - } -}); - -// Update risk thresholds -app.put('/api/risk/thresholds', async (c) => { - try { - const thresholds = await c.req.json(); - await redis.hmset('risk:thresholds', thresholds); - - // Broadcast threshold update to connected clients - const message = JSON.stringify({ - type: 'THRESHOLD_UPDATE', - data: thresholds, - timestamp: new Date() - }); - - wss.clients.forEach(client => { - if (client.readyState === 1) { // WebSocket.OPEN - client.send(message); - } - }); - - return c.json({ success: true, data: thresholds }); - } catch (error) { - console.error('Error updating risk thresholds:', error); - return c.json({ success: false, error: 'Failed to update thresholds' }, 500); - } -}); - -// Real-time risk monitoring endpoint -app.post('/api/risk/evaluate', async (c) => { - try { - const { symbol, quantity, price, portfolioValue } = await c.req.json(); - - const thresholds = await redis.hgetall('risk:thresholds'); - const activeThresholds = Object.keys(thresholds).length > 0 - ? Object.fromEntries( - Object.entries(thresholds).map(([k, v]) => [k, parseFloat(v as string)]) - ) - : defaultThresholds; - - const positionValue = quantity * price; - const positionRisk = positionValue / portfolioValue; - - const riskEvaluation = { - symbol, - positionValue, - positionRisk, - violations: [] as string[], - riskLevel: 'LOW' as 'LOW' | 'MEDIUM' | 'HIGH' - }; - - // Check risk violations - if (positionValue > activeThresholds.maxPositionSize) { - riskEvaluation.violations.push(`Position size exceeds limit: $${positionValue.toLocaleString()}`); - } - - if (positionRisk > activeThresholds.maxPortfolioRisk) { - riskEvaluation.violations.push(`Portfolio risk exceeds limit: ${(positionRisk * 100).toFixed(2)}%`); - } - - // Determine risk level - if (riskEvaluation.violations.length > 0) { - riskEvaluation.riskLevel = 'HIGH'; - } else if (positionRisk > activeThresholds.maxPortfolioRisk * 0.7) { - riskEvaluation.riskLevel = 'MEDIUM'; - } - - // Store risk evaluation - await redis.setex( - `risk:evaluation:${symbol}:${Date.now()}`, - 3600, // 1 hour TTL - JSON.stringify(riskEvaluation) - ); - - // Send real-time alert if high risk - if (riskEvaluation.riskLevel === 'HIGH') { - const alert = { - type: 'RISK_ALERT', - level: 'HIGH', - data: riskEvaluation, - timestamp: new Date() - }; - - wss.clients.forEach(client => { - if (client.readyState === 1) { - client.send(JSON.stringify(alert)); - } - }); - } - - return c.json({ success: true, data: riskEvaluation }); - } catch (error) { - console.error('Error evaluating risk:', error); - return c.json({ success: false, error: 'Failed to evaluate risk' }, 500); - } -}); - -// Get risk history -app.get('/api/risk/history', async (c) => { - try { - const keys = await redis.keys('risk:evaluation:*'); - const evaluations: any[] = []; - - for (const key of keys.slice(0, 100)) { // Limit to 100 recent evaluations - const data = await redis.get(key); - if (data) { - evaluations.push(JSON.parse(data)); - } - } - - return c.json({ - success: true, - data: evaluations.sort((a: any, b: any) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()) - }); - } catch (error) { - console.error('Error fetching risk history:', error); - return c.json({ success: false, error: 'Failed to fetch risk history' }, 500); - } -}); - -// WebSocket connection handling -wss.on('connection', (ws) => { - console.log('New risk monitoring client connected'); - - // Send welcome message - ws.send(JSON.stringify({ - type: 'CONNECTED', - message: 'Connected to Risk Guardian', - timestamp: new Date() - })); - - ws.on('close', () => { - console.log('Risk monitoring client disconnected'); - }); - - ws.on('error', (error) => { - console.error('WebSocket error:', error); - }); -}); - -// Redis event subscriptions for cross-service communication -redis.subscribe('trading:position:opened', 'trading:position:closed'); - -redis.on('message', async (channel, message) => { - try { - const data = JSON.parse(message); - - if (channel === 'trading:position:opened') { - // Auto-evaluate risk for new positions - const evaluation = await evaluatePositionRisk(data); - - // Broadcast to connected clients - wss.clients.forEach(client => { - if (client.readyState === 1) { - client.send(JSON.stringify({ - type: 'POSITION_RISK_UPDATE', - data: evaluation, - timestamp: new Date() - })); - } - }); - } - } catch (error) { - console.error('Error processing Redis message:', error); - } -}); - -async function evaluatePositionRisk(position: any) { - // Implementation would evaluate position against current thresholds - // This is a simplified version - return { - symbol: position.symbol, - riskLevel: 'LOW', - timestamp: new Date() - }; -} - -const port = parseInt(process.env.PORT || '3002'); - -console.log(`๐Ÿ›ก๏ธ Risk Guardian starting on port ${port}`); -console.log(`๐Ÿ“ก WebSocket server running on port 8081`); - -export default { - port, - fetch: app.fetch, -}; diff --git a/apps/core-services/risk-guardian/tsconfig.json b/apps/core-services/risk-guardian/tsconfig.json deleted file mode 100644 index 168d88b..0000000 --- a/apps/core-services/risk-guardian/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "module": "ESNext", - "moduleResolution": "bundler", - "types": ["bun-types"] - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/apps/data-services/data-catalog/package.json b/apps/data-services/data-catalog/package.json deleted file mode 100644 index f7fcb1a..0000000 --- a/apps/data-services/data-catalog/package.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "name": "@stock-bot/data-catalog", - "version": "1.0.0", - "private": true, - "description": "Data catalog and discovery service for stock-bot", - "type": "module", - "main": "dist/index.js", - "scripts": { - "dev": "bun run --hot src/index.ts", - "build": "tsc", - "start": "node dist/index.js", - "test": "bun test", - "type-check": "tsc --noEmit" - }, - "dependencies": { - "@stock-bot/types": "workspace:*", - "@stock-bot/utils": "workspace:*", - "@stock-bot/event-bus": "workspace:*", - "@stock-bot/api-client": "workspace:*", - "hono": "^4.0.0", - "zod": "^3.22.0", - "elasticsearch": "^16.7.3", - "neo4j-driver": "^5.15.0", - "cron": "^3.1.6", - "uuid": "^9.0.1" - }, - "devDependencies": { - "@types/uuid": "^9.0.8", - "@types/cron": "^2.4.0", - "@types/node": "^20.0.0", - "typescript": "^5.3.0" - }, - "keywords": [ - "data-catalog", - "data-discovery", - "data-lineage", - "metadata", - "stock-bot" - ] -} diff --git a/apps/data-services/data-catalog/src/controllers/DataCatalogController.ts b/apps/data-services/data-catalog/src/controllers/DataCatalogController.ts deleted file mode 100644 index 828917f..0000000 --- a/apps/data-services/data-catalog/src/controllers/DataCatalogController.ts +++ /dev/null @@ -1,360 +0,0 @@ -import { Context } from 'hono'; -import { Logger } from '@stock-bot/utils'; -import { DataCatalogService } from '../services/DataCatalogService'; -import { - CreateDataAssetRequest, - UpdateDataAssetRequest, - DataAssetType, - DataClassification -} from '../types/DataCatalog'; - -export class DataCatalogController { - constructor( - private dataCatalogService: DataCatalogService, - private logger: Logger - ) {} - - async createAsset(c: Context) { - try { - const request: CreateDataAssetRequest = await c.req.json(); - - // Validate required fields - if (!request.name || !request.type || !request.description || !request.owner) { - return c.json({ error: 'Missing required fields: name, type, description, owner' }, 400); - } - - const asset = await this.dataCatalogService.createAsset(request); - - this.logger.info('Asset created via API', { - assetId: asset.id, - name: asset.name, - type: asset.type - }); - - return c.json(asset, 201); - } catch (error) { - this.logger.error('Failed to create asset', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getAsset(c: Context) { - try { - const assetId = c.req.param('id'); - - if (!assetId) { - return c.json({ error: 'Asset ID is required' }, 400); - } - - const asset = await this.dataCatalogService.getAsset(assetId); - - if (!asset) { - return c.json({ error: 'Asset not found' }, 404); - } - - return c.json(asset); - } catch (error) { - this.logger.error('Failed to get asset', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async updateAsset(c: Context) { - try { - const assetId = c.req.param('id'); - const updates: UpdateDataAssetRequest = await c.req.json(); - - if (!assetId) { - return c.json({ error: 'Asset ID is required' }, 400); - } - - const asset = await this.dataCatalogService.updateAsset(assetId, updates); - - if (!asset) { - return c.json({ error: 'Asset not found' }, 404); - } - - this.logger.info('Asset updated via API', { - assetId, - changes: Object.keys(updates) - }); - - return c.json(asset); - } catch (error) { - this.logger.error('Failed to update asset', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async deleteAsset(c: Context) { - try { - const assetId = c.req.param('id'); - - if (!assetId) { - return c.json({ error: 'Asset ID is required' }, 400); - } - - await this.dataCatalogService.deleteAsset(assetId); - - this.logger.info('Asset deleted via API', { assetId }); - - return c.json({ message: 'Asset deleted successfully' }); - } catch (error) { - this.logger.error('Failed to delete asset', { error }); - - if (error instanceof Error && error.message.includes('not found')) { - return c.json({ error: 'Asset not found' }, 404); - } - - return c.json({ error: 'Internal server error' }, 500); - } - } - - async listAssets(c: Context) { - try { - const query = c.req.query(); - const filters: Record = {}; - - // Parse query parameters - if (query.type) filters.type = query.type; - if (query.owner) filters.owner = query.owner; - if (query.classification) filters.classification = query.classification; - if (query.tags) { - filters.tags = Array.isArray(query.tags) ? query.tags : [query.tags]; - } - - const assets = await this.dataCatalogService.listAssets(filters); - - return c.json({ - assets, - total: assets.length, - filters: filters - }); - } catch (error) { - this.logger.error('Failed to list assets', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async searchAssets(c: Context) { - try { - const query = c.req.query('q'); - const queryParams = c.req.query(); - - if (!query) { - return c.json({ error: 'Search query is required' }, 400); - } - - const filters: Record = {}; - if (queryParams.type) filters.type = queryParams.type; - if (queryParams.owner) filters.owner = queryParams.owner; - if (queryParams.classification) filters.classification = queryParams.classification; - - const assets = await this.dataCatalogService.searchAssets(query, filters); - - this.logger.info('Asset search performed', { - query, - filters, - resultCount: assets.length - }); - - return c.json({ - assets, - total: assets.length, - query, - filters - }); - } catch (error) { - this.logger.error('Failed to search assets', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getAssetsByOwner(c: Context) { - try { - const owner = c.req.param('owner'); - - if (!owner) { - return c.json({ error: 'Owner is required' }, 400); - } - - const assets = await this.dataCatalogService.getAssetsByOwner(owner); - - return c.json({ - assets, - total: assets.length, - owner - }); - } catch (error) { - this.logger.error('Failed to get assets by owner', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getAssetsByType(c: Context) { - try { - const type = c.req.param('type') as DataAssetType; - - if (!type) { - return c.json({ error: 'Asset type is required' }, 400); - } - - if (!Object.values(DataAssetType).includes(type)) { - return c.json({ error: 'Invalid asset type' }, 400); - } - - const assets = await this.dataCatalogService.getAssetsByType(type); - - return c.json({ - assets, - total: assets.length, - type - }); - } catch (error) { - this.logger.error('Failed to get assets by type', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getAssetsByClassification(c: Context) { - try { - const classification = c.req.param('classification') as DataClassification; - - if (!classification) { - return c.json({ error: 'Classification is required' }, 400); - } - - if (!Object.values(DataClassification).includes(classification)) { - return c.json({ error: 'Invalid classification' }, 400); - } - - const assets = await this.dataCatalogService.getAssetsByClassification(classification); - - return c.json({ - assets, - total: assets.length, - classification - }); - } catch (error) { - this.logger.error('Failed to get assets by classification', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getAssetsByTags(c: Context) { - try { - const tagsParam = c.req.query('tags'); - - if (!tagsParam) { - return c.json({ error: 'Tags parameter is required' }, 400); - } - - const tags = Array.isArray(tagsParam) ? tagsParam : [tagsParam]; - const assets = await this.dataCatalogService.getAssetsByTags(tags); - - return c.json({ - assets, - total: assets.length, - tags - }); - } catch (error) { - this.logger.error('Failed to get assets by tags', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getAssetMetrics(c: Context) { - try { - const assetId = c.req.param('id'); - - if (!assetId) { - return c.json({ error: 'Asset ID is required' }, 400); - } - - const asset = await this.dataCatalogService.getAsset(assetId); - - if (!asset) { - return c.json({ error: 'Asset not found' }, 404); - } - - const metrics = { - id: asset.id, - name: asset.name, - type: asset.type, - classification: asset.classification, - usage: { - accessCount: asset.usage.accessCount, - uniqueUsers: asset.usage.uniqueUsers, - lastAccessed: asset.usage.lastAccessed, - usageTrend: asset.usage.usageTrend - }, - quality: { - overallScore: asset.quality.overallScore, - lastAssessment: asset.quality.lastAssessment, - issueCount: asset.quality.issues.filter(issue => !issue.resolved).length - }, - governance: { - policiesApplied: asset.governance.policies.length, - complianceStatus: asset.governance.compliance.every(c => c.status === 'passed') ? 'compliant' : 'non-compliant', - auditEntries: asset.governance.audit.length - }, - lineage: { - upstreamCount: asset.lineage.upstreamAssets.length, - downstreamCount: asset.lineage.downstreamAssets.length - } - }; - - return c.json(metrics); - } catch (error) { - this.logger.error('Failed to get asset metrics', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getCatalogStatistics(c: Context) { - try { - const allAssets = await this.dataCatalogService.listAssets(); - - const statistics = { - totalAssets: allAssets.length, - assetsByType: this.groupByProperty(allAssets, 'type'), - assetsByClassification: this.groupByProperty(allAssets, 'classification'), - assetsByOwner: this.groupByProperty(allAssets, 'owner'), - recentAssets: allAssets - .sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()) - .slice(0, 10) - .map(asset => ({ - id: asset.id, - name: asset.name, - type: asset.type, - owner: asset.owner, - createdAt: asset.createdAt - })), - mostAccessed: allAssets - .sort((a, b) => b.usage.accessCount - a.usage.accessCount) - .slice(0, 10) - .map(asset => ({ - id: asset.id, - name: asset.name, - type: asset.type, - accessCount: asset.usage.accessCount, - lastAccessed: asset.usage.lastAccessed - })) - }; - - return c.json(statistics); - } catch (error) { - this.logger.error('Failed to get catalog statistics', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - // Helper method to group assets by property - private groupByProperty(assets: any[], property: string): Record { - return assets.reduce((acc, asset) => { - const value = asset[property]; - acc[value] = (acc[value] || 0) + 1; - return acc; - }, {}); - } -} diff --git a/apps/data-services/data-catalog/src/controllers/GovernanceController.ts b/apps/data-services/data-catalog/src/controllers/GovernanceController.ts deleted file mode 100644 index de70890..0000000 --- a/apps/data-services/data-catalog/src/controllers/GovernanceController.ts +++ /dev/null @@ -1,414 +0,0 @@ -import { Hono } from 'hono'; -import { DataGovernanceService } from '../services/DataGovernanceService'; -import { - GovernancePolicy, - ComplianceCheck, - AccessRequest, - DataSubjectRequest, - AuditLog -} from '../types/DataCatalog'; - -export class GovernanceController { - private app: Hono; - private governanceService: DataGovernanceService; - - constructor() { - this.app = new Hono(); - this.governanceService = new DataGovernanceService(); - this.setupRoutes(); - } - - private setupRoutes() { - // Create governance policy - this.app.post('/policies', async (c) => { - try { - const policy: Omit = await c.req.json(); - const createdPolicy = await this.governanceService.createPolicy(policy); - - return c.json({ - success: true, - data: createdPolicy - }); - } catch (error) { - console.error('Error creating governance policy:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get governance policies - this.app.get('/policies', async (c) => { - try { - const type = c.req.query('type'); - const category = c.req.query('category'); - const active = c.req.query('active') === 'true'; - - const filters: any = {}; - if (type) filters.type = type; - if (category) filters.category = category; - if (active !== undefined) filters.active = active; - - const policies = await this.governanceService.getPolicies(filters); - - return c.json({ - success: true, - data: policies - }); - } catch (error) { - console.error('Error getting governance policies:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Update governance policy - this.app.put('/policies/:policyId', async (c) => { - try { - const policyId = c.req.param('policyId'); - const updates: Partial = await c.req.json(); - - const updatedPolicy = await this.governanceService.updatePolicy(policyId, updates); - - return c.json({ - success: true, - data: updatedPolicy - }); - } catch (error) { - console.error('Error updating governance policy:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Delete governance policy - this.app.delete('/policies/:policyId', async (c) => { - try { - const policyId = c.req.param('policyId'); - await this.governanceService.deletePolicy(policyId); - - return c.json({ - success: true, - message: 'Governance policy deleted successfully' - }); - } catch (error) { - console.error('Error deleting governance policy:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Apply policy to asset - this.app.post('/policies/:policyId/apply/:assetId', async (c) => { - try { - const policyId = c.req.param('policyId'); - const assetId = c.req.param('assetId'); - - await this.governanceService.applyPolicy(policyId, assetId); - - return c.json({ - success: true, - message: 'Policy applied successfully' - }); - } catch (error) { - console.error('Error applying policy:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Check compliance for asset - this.app.post('/compliance/check', async (c) => { - try { - const request: { assetId: string; policyIds?: string[] } = await c.req.json(); - const complianceResult = await this.governanceService.checkCompliance( - request.assetId, - request.policyIds - ); - - return c.json({ - success: true, - data: complianceResult - }); - } catch (error) { - console.error('Error checking compliance:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get compliance violations - this.app.get('/compliance/violations', async (c) => { - try { - const assetId = c.req.query('assetId'); - const severity = c.req.query('severity'); - const status = c.req.query('status'); - const limit = c.req.query('limit') ? parseInt(c.req.query('limit')!) : 100; - const offset = c.req.query('offset') ? parseInt(c.req.query('offset')!) : 0; - - const filters: any = {}; - if (assetId) filters.assetId = assetId; - if (severity) filters.severity = severity; - if (status) filters.status = status; - - const violations = await this.governanceService.getComplianceViolations( - filters, - { limit, offset } - ); - - return c.json({ - success: true, - data: violations - }); - } catch (error) { - console.error('Error getting compliance violations:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Request access to asset - this.app.post('/access/request', async (c) => { - try { - const request: Omit = await c.req.json(); - const accessRequest = await this.governanceService.requestAccess(request); - - return c.json({ - success: true, - data: accessRequest - }); - } catch (error) { - console.error('Error requesting access:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Approve/deny access request - this.app.patch('/access/:requestId', async (c) => { - try { - const requestId = c.req.param('requestId'); - const { action, reviewedBy, reviewComments } = await c.req.json(); - - const updatedRequest = await this.governanceService.reviewAccessRequest( - requestId, - action, - reviewedBy, - reviewComments - ); - - return c.json({ - success: true, - data: updatedRequest - }); - } catch (error) { - console.error('Error reviewing access request:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Check access authorization - this.app.post('/access/check', async (c) => { - try { - const { userId, assetId, action } = await c.req.json(); - const authorized = await this.governanceService.checkAccess(userId, assetId, action); - - return c.json({ - success: true, - data: { - userId, - assetId, - action, - authorized - } - }); - } catch (error) { - console.error('Error checking access:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Handle data subject request (GDPR) - this.app.post('/privacy/subject-request', async (c) => { - try { - const request: Omit = await c.req.json(); - const subjectRequest = await this.governanceService.handleDataSubjectRequest(request); - - return c.json({ - success: true, - data: subjectRequest - }); - } catch (error) { - console.error('Error handling data subject request:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Anonymize asset data - this.app.post('/privacy/anonymize/:assetId', async (c) => { - try { - const assetId = c.req.param('assetId'); - const { fields, method, requestedBy } = await c.req.json(); - - const result = await this.governanceService.anonymizeData( - assetId, - fields, - method, - requestedBy - ); - - return c.json({ - success: true, - data: result - }); - } catch (error) { - console.error('Error anonymizing data:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get audit logs - this.app.get('/audit/logs', async (c) => { - try { - const assetId = c.req.query('assetId'); - const userId = c.req.query('userId'); - const action = c.req.query('action'); - const startDate = c.req.query('startDate'); - const endDate = c.req.query('endDate'); - const limit = c.req.query('limit') ? parseInt(c.req.query('limit')!) : 100; - const offset = c.req.query('offset') ? parseInt(c.req.query('offset')!) : 0; - - const filters: any = {}; - if (assetId) filters.assetId = assetId; - if (userId) filters.userId = userId; - if (action) filters.action = action; - if (startDate) filters.startDate = new Date(startDate); - if (endDate) filters.endDate = new Date(endDate); - - const logs = await this.governanceService.getAuditLogs(filters, { limit, offset }); - - return c.json({ - success: true, - data: logs - }); - } catch (error) { - console.error('Error getting audit logs:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Log access event - this.app.post('/audit/log', async (c) => { - try { - const logEntry: Omit = await c.req.json(); - const logged = await this.governanceService.logAccess(logEntry); - - return c.json({ - success: true, - data: logged - }); - } catch (error) { - console.error('Error logging access event:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get retention policies - this.app.get('/retention/policies', async (c) => { - try { - const assetType = c.req.query('assetType'); - const policies = await this.governanceService.getRetentionPolicies(assetType); - - return c.json({ - success: true, - data: policies - }); - } catch (error) { - console.error('Error getting retention policies:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Apply retention policy - this.app.post('/retention/apply', async (c) => { - try { - const { assetId, policyId, requestedBy } = await c.req.json(); - const result = await this.governanceService.applyRetentionPolicy( - assetId, - policyId, - requestedBy - ); - - return c.json({ - success: true, - data: result - }); - } catch (error) { - console.error('Error applying retention policy:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get governance metrics - this.app.get('/metrics', async (c) => { - try { - const timeRange = c.req.query('timeRange') || '30d'; - const metrics = await this.governanceService.getGovernanceMetrics(timeRange); - - return c.json({ - success: true, - data: metrics - }); - } catch (error) { - console.error('Error getting governance metrics:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - } - - public getApp(): Hono { - return this.app; - } -} diff --git a/apps/data-services/data-catalog/src/controllers/HealthController.ts b/apps/data-services/data-catalog/src/controllers/HealthController.ts deleted file mode 100644 index 8695106..0000000 --- a/apps/data-services/data-catalog/src/controllers/HealthController.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { Hono } from 'hono'; - -export class HealthController { - private app: Hono; - - constructor() { - this.app = new Hono(); - this.setupRoutes(); - } - - private setupRoutes() { - // Basic health check - this.app.get('/', async (c) => { - return c.json({ - service: 'data-catalog', - status: 'healthy', - timestamp: new Date().toISOString(), - version: process.env.SERVICE_VERSION || '1.0.0' - }); - }); - - // Detailed health check - this.app.get('/detailed', async (c) => { - try { - const healthStatus = { - service: 'data-catalog', - status: 'healthy', - timestamp: new Date().toISOString(), - version: process.env.SERVICE_VERSION || '1.0.0', - uptime: process.uptime(), - memory: process.memoryUsage(), - dependencies: { - database: await this.checkDatabase(), - search: await this.checkSearchService(), - eventBus: await this.checkEventBus() - } - }; - - // Determine overall status based on dependencies - const hasUnhealthyDependencies = Object.values(healthStatus.dependencies) - .some(dep => dep.status !== 'healthy'); - - if (hasUnhealthyDependencies) { - healthStatus.status = 'degraded'; - } - - const statusCode = healthStatus.status === 'healthy' ? 200 : 503; - return c.json(healthStatus, statusCode); - } catch (error) { - console.error('Health check error:', error); - return c.json({ - service: 'data-catalog', - status: 'unhealthy', - timestamp: new Date().toISOString(), - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - }); - - // Readiness check - this.app.get('/ready', async (c) => { - try { - // Check if service is ready to accept requests - const readyChecks = await Promise.all([ - this.checkDatabase(), - this.checkSearchService() - ]); - - const isReady = readyChecks.every(check => check.status === 'healthy'); - - if (isReady) { - return c.json({ - service: 'data-catalog', - ready: true, - timestamp: new Date().toISOString() - }); - } else { - return c.json({ - service: 'data-catalog', - ready: false, - timestamp: new Date().toISOString(), - checks: readyChecks - }, 503); - } - } catch (error) { - console.error('Readiness check error:', error); - return c.json({ - service: 'data-catalog', - ready: false, - timestamp: new Date().toISOString(), - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - }); - - // Liveness check - this.app.get('/live', async (c) => { - return c.json({ - service: 'data-catalog', - alive: true, - timestamp: new Date().toISOString() - }); - }); - } - - private async checkDatabase(): Promise<{ name: string; status: string; responseTime?: number }> { - const start = Date.now(); - try { - // Simulate database check - // In real implementation, this would ping the actual database - await new Promise(resolve => setTimeout(resolve, 10)); - - return { - name: 'database', - status: 'healthy', - responseTime: Date.now() - start - }; - } catch (error) { - return { - name: 'database', - status: 'unhealthy', - responseTime: Date.now() - start - }; - } - } - - private async checkSearchService(): Promise<{ name: string; status: string; responseTime?: number }> { - const start = Date.now(); - try { - // Simulate search service check - // In real implementation, this would check search index health - await new Promise(resolve => setTimeout(resolve, 5)); - - return { - name: 'search', - status: 'healthy', - responseTime: Date.now() - start - }; - } catch (error) { - return { - name: 'search', - status: 'unhealthy', - responseTime: Date.now() - start - }; - } - } - - private async checkEventBus(): Promise<{ name: string; status: string; responseTime?: number }> { - const start = Date.now(); - try { - // Simulate event bus check - // In real implementation, this would check message broker connectivity - await new Promise(resolve => setTimeout(resolve, 3)); - - return { - name: 'eventBus', - status: 'healthy', - responseTime: Date.now() - start - }; - } catch (error) { - return { - name: 'eventBus', - status: 'unhealthy', - responseTime: Date.now() - start - }; - } - } - - public getApp(): Hono { - return this.app; - } -} diff --git a/apps/data-services/data-catalog/src/controllers/LineageController.ts b/apps/data-services/data-catalog/src/controllers/LineageController.ts deleted file mode 100644 index 2e4b819..0000000 --- a/apps/data-services/data-catalog/src/controllers/LineageController.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { Hono } from 'hono'; -import { DataLineageService } from '../services/DataLineageService'; -import { CreateLineageRequest, LineageQuery, ImpactAnalysisQuery } from '../types/DataCatalog'; - -export class LineageController { - private app: Hono; - private lineageService: DataLineageService; - - constructor() { - this.app = new Hono(); - this.lineageService = new DataLineageService(); - this.setupRoutes(); - } - - private setupRoutes() { - // Create lineage relationship - this.app.post('/', async (c) => { - try { - const request: CreateLineageRequest = await c.req.json(); - const lineage = await this.lineageService.createLineage(request); - return c.json({ - success: true, - data: lineage - }); - } catch (error) { - console.error('Error creating lineage:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get lineage for asset - this.app.get('/assets/:assetId', async (c) => { - try { - const assetId = c.req.param('assetId'); - const direction = c.req.query('direction') as 'upstream' | 'downstream' | 'both'; - const depth = c.req.query('depth') ? parseInt(c.req.query('depth')!) : undefined; - - const lineage = await this.lineageService.getAssetLineage(assetId, { - direction: direction || 'both', - depth: depth || 10 - }); - - return c.json({ - success: true, - data: lineage - }); - } catch (error) { - console.error('Error getting asset lineage:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get upstream dependencies - this.app.get('/assets/:assetId/upstream', async (c) => { - try { - const assetId = c.req.param('assetId'); - const depth = c.req.query('depth') ? parseInt(c.req.query('depth')!) : 5; - - const upstream = await this.lineageService.getUpstreamDependencies(assetId, depth); - - return c.json({ - success: true, - data: upstream - }); - } catch (error) { - console.error('Error getting upstream dependencies:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get downstream dependencies - this.app.get('/assets/:assetId/downstream', async (c) => { - try { - const assetId = c.req.param('assetId'); - const depth = c.req.query('depth') ? parseInt(c.req.query('depth')!) : 5; - - const downstream = await this.lineageService.getDownstreamDependencies(assetId, depth); - - return c.json({ - success: true, - data: downstream - }); - } catch (error) { - console.error('Error getting downstream dependencies:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Perform impact analysis - this.app.post('/impact-analysis', async (c) => { - try { - const query: ImpactAnalysisQuery = await c.req.json(); - const analysis = await this.lineageService.performImpactAnalysis(query); - - return c.json({ - success: true, - data: analysis - }); - } catch (error) { - console.error('Error performing impact analysis:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get lineage graph - this.app.get('/graph', async (c) => { - try { - const assetIds = c.req.query('assetIds')?.split(',') || []; - const depth = c.req.query('depth') ? parseInt(c.req.query('depth')!) : 3; - - if (assetIds.length === 0) { - return c.json({ - success: false, - error: 'Asset IDs are required' - }, 400); - } - - const graph = await this.lineageService.getLineageGraph(assetIds, depth); - - return c.json({ - success: true, - data: graph - }); - } catch (error) { - console.error('Error getting lineage graph:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Check for circular dependencies - this.app.get('/assets/:assetId/circular-check', async (c) => { - try { - const assetId = c.req.param('assetId'); - const hasCycles = await this.lineageService.hasCircularDependencies(assetId); - - return c.json({ - success: true, - data: { - assetId, - hasCircularDependencies: hasCycles - } - }); - } catch (error) { - console.error('Error checking circular dependencies:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Delete lineage relationship - this.app.delete('/:lineageId', async (c) => { - try { - const lineageId = c.req.param('lineageId'); - await this.lineageService.deleteLineage(lineageId); - - return c.json({ - success: true, - message: 'Lineage relationship deleted successfully' - }); - } catch (error) { - console.error('Error deleting lineage:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get lineage statistics - this.app.get('/stats', async (c) => { - try { - const stats = await this.lineageService.getLineageStatistics(); - - return c.json({ - success: true, - data: stats - }); - } catch (error) { - console.error('Error getting lineage statistics:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - } - - public getApp(): Hono { - return this.app; - } -} diff --git a/apps/data-services/data-catalog/src/controllers/QualityController.ts b/apps/data-services/data-catalog/src/controllers/QualityController.ts deleted file mode 100644 index 2f3126f..0000000 --- a/apps/data-services/data-catalog/src/controllers/QualityController.ts +++ /dev/null @@ -1,321 +0,0 @@ -import { Hono } from 'hono'; -import { DataQualityService } from '../services/DataQualityService'; -import { - QualityAssessmentRequest, - QualityRule, - QualityIssue, - QualityReportRequest -} from '../types/DataCatalog'; - -export class QualityController { - private app: Hono; - private qualityService: DataQualityService; - - constructor() { - this.app = new Hono(); - this.qualityService = new DataQualityService(); - this.setupRoutes(); - } - - private setupRoutes() { - // Assess asset quality - this.app.post('/assess', async (c) => { - try { - const request: QualityAssessmentRequest = await c.req.json(); - const assessment = await this.qualityService.assessQuality(request); - - return c.json({ - success: true, - data: assessment - }); - } catch (error) { - console.error('Error assessing quality:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get quality assessment for asset - this.app.get('/assets/:assetId', async (c) => { - try { - const assetId = c.req.param('assetId'); - const assessment = await this.qualityService.getQualityAssessment(assetId); - - if (!assessment) { - return c.json({ - success: false, - error: 'Quality assessment not found' - }, 404); - } - - return c.json({ - success: true, - data: assessment - }); - } catch (error) { - console.error('Error getting quality assessment:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Create quality rule - this.app.post('/rules', async (c) => { - try { - const rule: Omit = await c.req.json(); - const createdRule = await this.qualityService.createQualityRule(rule); - - return c.json({ - success: true, - data: createdRule - }); - } catch (error) { - console.error('Error creating quality rule:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get quality rules - this.app.get('/rules', async (c) => { - try { - const assetType = c.req.query('assetType'); - const dimension = c.req.query('dimension'); - const active = c.req.query('active') === 'true'; - - const filters: any = {}; - if (assetType) filters.assetType = assetType; - if (dimension) filters.dimension = dimension; - if (active !== undefined) filters.active = active; - - const rules = await this.qualityService.getQualityRules(filters); - - return c.json({ - success: true, - data: rules - }); - } catch (error) { - console.error('Error getting quality rules:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Update quality rule - this.app.put('/rules/:ruleId', async (c) => { - try { - const ruleId = c.req.param('ruleId'); - const updates: Partial = await c.req.json(); - - const updatedRule = await this.qualityService.updateQualityRule(ruleId, updates); - - return c.json({ - success: true, - data: updatedRule - }); - } catch (error) { - console.error('Error updating quality rule:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Delete quality rule - this.app.delete('/rules/:ruleId', async (c) => { - try { - const ruleId = c.req.param('ruleId'); - await this.qualityService.deleteQualityRule(ruleId); - - return c.json({ - success: true, - message: 'Quality rule deleted successfully' - }); - } catch (error) { - console.error('Error deleting quality rule:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Validate quality rules for asset - this.app.post('/validate/:assetId', async (c) => { - try { - const assetId = c.req.param('assetId'); - const data = await c.req.json(); - - const validationResults = await this.qualityService.validateQualityRules(assetId, data); - - return c.json({ - success: true, - data: validationResults - }); - } catch (error) { - console.error('Error validating quality rules:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Report quality issue - this.app.post('/issues', async (c) => { - try { - const issue: Omit = await c.req.json(); - const reportedIssue = await this.qualityService.reportQualityIssue(issue); - - return c.json({ - success: true, - data: reportedIssue - }); - } catch (error) { - console.error('Error reporting quality issue:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get quality issues - this.app.get('/issues', async (c) => { - try { - const assetId = c.req.query('assetId'); - const severity = c.req.query('severity'); - const status = c.req.query('status'); - const dimension = c.req.query('dimension'); - const limit = c.req.query('limit') ? parseInt(c.req.query('limit')!) : 100; - const offset = c.req.query('offset') ? parseInt(c.req.query('offset')!) : 0; - - const filters: any = {}; - if (assetId) filters.assetId = assetId; - if (severity) filters.severity = severity; - if (status) filters.status = status; - if (dimension) filters.dimension = dimension; - - const issues = await this.qualityService.getQualityIssues(filters, { limit, offset }); - - return c.json({ - success: true, - data: issues - }); - } catch (error) { - console.error('Error getting quality issues:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Resolve quality issue - this.app.patch('/issues/:issueId/resolve', async (c) => { - try { - const issueId = c.req.param('issueId'); - const { resolution, resolvedBy } = await c.req.json(); - - const resolvedIssue = await this.qualityService.resolveQualityIssue( - issueId, - resolution, - resolvedBy - ); - - return c.json({ - success: true, - data: resolvedIssue - }); - } catch (error) { - console.error('Error resolving quality issue:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get quality trends - this.app.get('/trends', async (c) => { - try { - const assetId = c.req.query('assetId'); - const dimension = c.req.query('dimension'); - const timeRange = c.req.query('timeRange') || '30d'; - - const trends = await this.qualityService.getQualityTrends( - assetId, - dimension, - timeRange - ); - - return c.json({ - success: true, - data: trends - }); - } catch (error) { - console.error('Error getting quality trends:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Generate quality report - this.app.post('/reports', async (c) => { - try { - const request: QualityReportRequest = await c.req.json(); - const report = await this.qualityService.generateQualityReport(request); - - return c.json({ - success: true, - data: report - }); - } catch (error) { - console.error('Error generating quality report:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - - // Get quality metrics summary - this.app.get('/metrics/summary', async (c) => { - try { - const assetIds = c.req.query('assetIds')?.split(','); - const timeRange = c.req.query('timeRange') || '7d'; - - const summary = await this.qualityService.getQualityMetricsSummary( - assetIds, - timeRange - ); - - return c.json({ - success: true, - data: summary - }); - } catch (error) { - console.error('Error getting quality metrics summary:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - }); - } - - public getApp(): Hono { - return this.app; - } -} diff --git a/apps/data-services/data-catalog/src/controllers/SearchController.ts b/apps/data-services/data-catalog/src/controllers/SearchController.ts deleted file mode 100644 index 14e534b..0000000 --- a/apps/data-services/data-catalog/src/controllers/SearchController.ts +++ /dev/null @@ -1,334 +0,0 @@ -import { Context } from 'hono'; -import { Logger } from '@stock-bot/utils'; -import { SearchService } from '../services/SearchService'; -import { SearchQuery, SearchFilters } from '../types/DataCatalog'; - -export class SearchController { - constructor( - private searchService: SearchService, - private logger: Logger - ) {} - - async search(c: Context) { - try { - const queryParams = c.req.query(); - - const searchQuery: SearchQuery = { - text: queryParams.q || '', - offset: parseInt(queryParams.offset || '0'), - limit: parseInt(queryParams.limit || '20'), - sortBy: queryParams.sortBy, - sortOrder: queryParams.sortOrder as 'asc' | 'desc', - userId: queryParams.userId - }; - - // Parse filters - const filters: SearchFilters = {}; - if (queryParams.types) { - filters.types = Array.isArray(queryParams.types) ? queryParams.types : [queryParams.types]; - } - if (queryParams.classifications) { - filters.classifications = Array.isArray(queryParams.classifications) ? queryParams.classifications : [queryParams.classifications]; - } - if (queryParams.owners) { - filters.owners = Array.isArray(queryParams.owners) ? queryParams.owners : [queryParams.owners]; - } - if (queryParams.tags) { - filters.tags = Array.isArray(queryParams.tags) ? queryParams.tags : [queryParams.tags]; - } - if (queryParams.createdAfter) { - filters.createdAfter = new Date(queryParams.createdAfter); - } - if (queryParams.createdBefore) { - filters.createdBefore = new Date(queryParams.createdBefore); - } - - if (Object.keys(filters).length > 0) { - searchQuery.filters = filters; - } - - const result = await this.searchService.search(searchQuery); - - this.logger.info('Search API call completed', { - query: searchQuery.text, - resultCount: result.total, - searchTime: result.searchTime - }); - - return c.json(result); - } catch (error) { - this.logger.error('Search API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async suggest(c: Context) { - try { - const partial = c.req.query('q'); - - if (!partial || partial.length < 2) { - return c.json({ suggestions: [] }); - } - - const suggestions = await this.searchService.suggest(partial); - - return c.json({ suggestions }); - } catch (error) { - this.logger.error('Suggestion API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async searchByFacets(c: Context) { - try { - const facets = await c.req.json(); - - if (!facets || typeof facets !== 'object') { - return c.json({ error: 'Facets object is required' }, 400); - } - - const assets = await this.searchService.searchByFacets(facets); - - return c.json({ - assets, - total: assets.length, - facets - }); - } catch (error) { - this.logger.error('Facet search API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async searchSimilar(c: Context) { - try { - const assetId = c.req.param('id'); - const limit = parseInt(c.req.query('limit') || '10'); - - if (!assetId) { - return c.json({ error: 'Asset ID is required' }, 400); - } - - const similarAssets = await this.searchService.searchSimilar(assetId, limit); - - return c.json({ - assetId, - similarAssets, - total: similarAssets.length - }); - } catch (error) { - this.logger.error('Similar search API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getPopularSearches(c: Context) { - try { - const limit = parseInt(c.req.query('limit') || '10'); - - const popularSearches = await this.searchService.getPopularSearches(limit); - - return c.json({ - searches: popularSearches, - total: popularSearches.length - }); - } catch (error) { - this.logger.error('Popular searches API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getRecentSearches(c: Context) { - try { - const userId = c.req.param('userId'); - const limit = parseInt(c.req.query('limit') || '10'); - - if (!userId) { - return c.json({ error: 'User ID is required' }, 400); - } - - const recentSearches = await this.searchService.getRecentSearches(userId, limit); - - return c.json({ - userId, - searches: recentSearches, - total: recentSearches.length - }); - } catch (error) { - this.logger.error('Recent searches API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async reindexAssets(c: Context) { - try { - await this.searchService.reindexAll(); - - this.logger.info('Search index rebuilt via API'); - - return c.json({ message: 'Search index rebuilt successfully' }); - } catch (error) { - this.logger.error('Reindex API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getSearchAnalytics(c: Context) { - try { - const timeframe = c.req.query('timeframe') || 'week'; - - const analytics = await this.searchService.getSearchAnalytics(timeframe); - - return c.json({ - timeframe, - analytics - }); - } catch (error) { - this.logger.error('Search analytics API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async advancedSearch(c: Context) { - try { - const searchRequest = await c.req.json(); - - if (!searchRequest) { - return c.json({ error: 'Search request is required' }, 400); - } - - // Build advanced search query - const searchQuery: SearchQuery = { - text: searchRequest.query || '', - offset: searchRequest.offset || 0, - limit: searchRequest.limit || 20, - sortBy: searchRequest.sortBy, - sortOrder: searchRequest.sortOrder, - userId: searchRequest.userId, - filters: searchRequest.filters - }; - - const result = await this.searchService.search(searchQuery); - - // If no results and query is complex, try to suggest simpler alternatives - if (result.total === 0 && searchQuery.text && searchQuery.text.split(' ').length > 2) { - const simpleQuery = searchQuery.text.split(' ')[0]; - const simpleResult = await this.searchService.search({ - ...searchQuery, - text: simpleQuery - }); - - if (simpleResult.total > 0) { - result.suggestions = [`Try searching for "${simpleQuery}"`]; - } - } - - this.logger.info('Advanced search API call completed', { - query: searchQuery.text, - resultCount: result.total, - searchTime: result.searchTime - }); - - return c.json(result); - } catch (error) { - this.logger.error('Advanced search API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async exportSearchResults(c: Context) { - try { - const queryParams = c.req.query(); - const format = queryParams.format || 'json'; - - if (format !== 'json' && format !== 'csv') { - return c.json({ error: 'Unsupported export format. Use json or csv' }, 400); - } - - // Perform search with maximum results - const searchQuery: SearchQuery = { - text: queryParams.q || '', - offset: 0, - limit: 10000, // Large limit for export - sortBy: queryParams.sortBy, - sortOrder: queryParams.sortOrder as 'asc' | 'desc' - }; - - const result = await this.searchService.search(searchQuery); - - if (format === 'csv') { - const csv = this.convertToCSV(result.assets); - c.header('Content-Type', 'text/csv'); - c.header('Content-Disposition', 'attachment; filename="search-results.csv"'); - return c.text(csv); - } else { - c.header('Content-Type', 'application/json'); - c.header('Content-Disposition', 'attachment; filename="search-results.json"'); - return c.json(result); - } - } catch (error) { - this.logger.error('Export search results API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - async getSearchStatistics(c: Context) { - try { - const timeframe = c.req.query('timeframe') || 'week'; - - const analytics = await this.searchService.getSearchAnalytics(timeframe); - - const statistics = { - searchVolume: analytics.totalSearches, - uniqueQueries: analytics.uniqueQueries, - averageResultsPerSearch: Math.round(analytics.averageResults), - noResultQueriesPercent: analytics.totalSearches > 0 - ? Math.round((analytics.noResultQueries / analytics.totalSearches) * 100) - : 0, - topSearchTerms: analytics.topQueries.slice(0, 5), - searchTrend: analytics.searchTrend.trend, - facetUsage: analytics.facetUsage - }; - - return c.json({ - timeframe, - statistics - }); - } catch (error) { - this.logger.error('Search statistics API call failed', { error }); - return c.json({ error: 'Internal server error' }, 500); - } - } - - // Helper method to convert assets to CSV format - private convertToCSV(assets: any[]): string { - if (assets.length === 0) { - return 'No results found'; - } - - const headers = [ - 'ID', 'Name', 'Type', 'Description', 'Owner', 'Classification', - 'Tags', 'Created At', 'Updated At', 'Last Accessed' - ]; - - const csvRows = [headers.join(',')]; - - for (const asset of assets) { - const row = [ - asset.id, - `"${asset.name.replace(/"/g, '""')}"`, - asset.type, - `"${asset.description.replace(/"/g, '""')}"`, - asset.owner, - asset.classification, - `"${asset.tags.join('; ')}"`, - asset.createdAt.toISOString(), - asset.updatedAt.toISOString(), - asset.lastAccessed ? asset.lastAccessed.toISOString() : '' - ]; - csvRows.push(row.join(',')); - } - - return csvRows.join('\n'); - } -} diff --git a/apps/data-services/data-catalog/src/index.ts b/apps/data-services/data-catalog/src/index.ts deleted file mode 100644 index 90cdcee..0000000 --- a/apps/data-services/data-catalog/src/index.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { Hono } from 'hono'; -import { cors } from 'hono/cors'; -import { logger } from 'hono/logger'; -import { prettyJSON } from 'hono/pretty-json'; -import { serve } from '@hono/node-server'; - -// Import controllers -import { DataCatalogController } from './controllers/DataCatalogController'; -import { SearchController } from './controllers/SearchController'; -import { LineageController } from './controllers/LineageController'; -import { QualityController } from './controllers/QualityController'; -import { GovernanceController } from './controllers/GovernanceController'; -import { HealthController } from './controllers/HealthController'; - -// Create main application -const app = new Hono(); - -// Add middleware -app.use('*', cors({ - origin: ['http://localhost:3000', 'http://localhost:4000', 'http://localhost:5173'], - allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], - allowHeaders: ['Content-Type', 'Authorization', 'X-Requested-With'], - credentials: true -})); - -app.use('*', logger()); -app.use('*', prettyJSON()); - -// Initialize controllers -const dataCatalogController = new DataCatalogController(); -const searchController = new SearchController(); -const lineageController = new LineageController(); -const qualityController = new QualityController(); -const governanceController = new GovernanceController(); -const healthController = new HealthController(); - -// Setup routes -app.route('/api/v1/assets', dataCatalogController.getApp()); -app.route('/api/v1/search', searchController.getApp()); -app.route('/api/v1/lineage', lineageController.getApp()); -app.route('/api/v1/quality', qualityController.getApp()); -app.route('/api/v1/governance', governanceController.getApp()); -app.route('/health', healthController.getApp()); - -// Root endpoint -app.get('/', (c) => { - return c.json({ - service: 'Data Catalog Service', - version: '1.0.0', - description: 'Comprehensive data catalog and governance service for stock-bot data platform', - endpoints: { - assets: '/api/v1/assets', - search: '/api/v1/search', - lineage: '/api/v1/lineage', - quality: '/api/v1/quality', - governance: '/api/v1/governance', - health: '/health' - }, - documentation: '/api/v1/docs' - }); -}); - -// API documentation endpoint -app.get('/api/v1/docs', (c) => { - return c.json({ - title: 'Data Catalog Service API', - version: '1.0.0', - description: 'RESTful API for data catalog, lineage, quality, and governance operations', - endpoints: { - assets: { - description: 'Data asset management', - methods: { - 'GET /api/v1/assets': 'List assets with filtering and pagination', - 'POST /api/v1/assets': 'Create new data asset', - 'GET /api/v1/assets/:id': 'Get asset by ID', - 'PUT /api/v1/assets/:id': 'Update asset', - 'DELETE /api/v1/assets/:id': 'Delete asset', - 'GET /api/v1/assets/:id/schema': 'Get asset schema', - 'PUT /api/v1/assets/:id/schema': 'Update asset schema', - 'GET /api/v1/assets/:id/usage': 'Get asset usage analytics', - 'POST /api/v1/assets/:id/usage': 'Record asset usage' - } - }, - search: { - description: 'Data discovery and search', - methods: { - 'GET /api/v1/search': 'Search assets with full-text and faceted search', - 'GET /api/v1/search/suggest': 'Get search suggestions', - 'GET /api/v1/search/facets': 'Get available search facets', - 'GET /api/v1/search/similar/:id': 'Find similar assets', - 'GET /api/v1/search/trending': 'Get trending searches', - 'POST /api/v1/search/export': 'Export search results' - } - }, - lineage: { - description: 'Data lineage and impact analysis', - methods: { - 'POST /api/v1/lineage': 'Create lineage relationship', - 'GET /api/v1/lineage/assets/:assetId': 'Get asset lineage', - 'GET /api/v1/lineage/assets/:assetId/upstream': 'Get upstream dependencies', - 'GET /api/v1/lineage/assets/:assetId/downstream': 'Get downstream dependencies', - 'POST /api/v1/lineage/impact-analysis': 'Perform impact analysis', - 'GET /api/v1/lineage/graph': 'Get lineage graph visualization', - 'GET /api/v1/lineage/assets/:assetId/circular-check': 'Check for circular dependencies', - 'DELETE /api/v1/lineage/:lineageId': 'Delete lineage relationship', - 'GET /api/v1/lineage/stats': 'Get lineage statistics' - } - }, - quality: { - description: 'Data quality assessment and monitoring', - methods: { - 'POST /api/v1/quality/assess': 'Assess data quality', - 'GET /api/v1/quality/assets/:assetId': 'Get quality assessment', - 'POST /api/v1/quality/rules': 'Create quality rule', - 'GET /api/v1/quality/rules': 'Get quality rules', - 'PUT /api/v1/quality/rules/:ruleId': 'Update quality rule', - 'DELETE /api/v1/quality/rules/:ruleId': 'Delete quality rule', - 'POST /api/v1/quality/validate/:assetId': 'Validate quality rules', - 'POST /api/v1/quality/issues': 'Report quality issue', - 'GET /api/v1/quality/issues': 'Get quality issues', - 'PATCH /api/v1/quality/issues/:issueId/resolve': 'Resolve quality issue', - 'GET /api/v1/quality/trends': 'Get quality trends', - 'POST /api/v1/quality/reports': 'Generate quality report', - 'GET /api/v1/quality/metrics/summary': 'Get quality metrics summary' - } - }, - governance: { - description: 'Data governance and compliance', - methods: { - 'POST /api/v1/governance/policies': 'Create governance policy', - 'GET /api/v1/governance/policies': 'Get governance policies', - 'PUT /api/v1/governance/policies/:policyId': 'Update governance policy', - 'DELETE /api/v1/governance/policies/:policyId': 'Delete governance policy', - 'POST /api/v1/governance/policies/:policyId/apply/:assetId': 'Apply policy to asset', - 'POST /api/v1/governance/compliance/check': 'Check compliance', - 'GET /api/v1/governance/compliance/violations': 'Get compliance violations', - 'POST /api/v1/governance/access/request': 'Request data access', - 'PATCH /api/v1/governance/access/:requestId': 'Review access request', - 'POST /api/v1/governance/access/check': 'Check access authorization', - 'POST /api/v1/governance/privacy/subject-request': 'Handle data subject request', - 'POST /api/v1/governance/privacy/anonymize/:assetId': 'Anonymize asset data', - 'GET /api/v1/governance/audit/logs': 'Get audit logs', - 'POST /api/v1/governance/audit/log': 'Log access event', - 'GET /api/v1/governance/retention/policies': 'Get retention policies', - 'POST /api/v1/governance/retention/apply': 'Apply retention policy', - 'GET /api/v1/governance/metrics': 'Get governance metrics' - } - }, - health: { - description: 'Service health monitoring', - methods: { - 'GET /health': 'Basic health check', - 'GET /health/detailed': 'Detailed health check with dependencies', - 'GET /health/ready': 'Readiness check', - 'GET /health/live': 'Liveness check' - } - } - } - }); -}); - -// 404 handler -app.notFound((c) => { - return c.json({ - success: false, - error: 'Endpoint not found', - availableEndpoints: [ - '/api/v1/assets', - '/api/v1/search', - '/api/v1/lineage', - '/api/v1/quality', - '/api/v1/governance', - '/health' - ] - }, 404); -}); - -// Error handler -app.onError((err, c) => { - console.error('Application error:', err); - - return c.json({ - success: false, - error: 'Internal server error', - message: process.env.NODE_ENV === 'development' ? err.message : 'Something went wrong' - }, 500); -}); - -// Start server -const port = parseInt(process.env.PORT || '3003'); - -console.log(`๐Ÿš€ Data Catalog Service starting on port ${port}`); -console.log(`๐Ÿ“š API Documentation available at http://localhost:${port}/api/v1/docs`); -console.log(`โค๏ธ Health endpoint available at http://localhost:${port}/health`); - -serve({ - fetch: app.fetch, - port: port -}); - -export default app; diff --git a/apps/data-services/data-catalog/src/services/DataCatalogService.ts b/apps/data-services/data-catalog/src/services/DataCatalogService.ts deleted file mode 100644 index f85c0e0..0000000 --- a/apps/data-services/data-catalog/src/services/DataCatalogService.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { EventBus } from '@stock-bot/event-bus'; -import { Logger } from '@stock-bot/utils'; -import { - DataAsset, - CreateDataAssetRequest, - UpdateDataAssetRequest, - DataAssetType, - DataClassification -} from '../types/DataCatalog'; - -export interface DataCatalogService { - createAsset(request: CreateDataAssetRequest): Promise; - getAsset(id: string): Promise; - updateAsset(id: string, request: UpdateDataAssetRequest): Promise; - deleteAsset(id: string): Promise; - listAssets(filters?: Record): Promise; - searchAssets(query: string, filters?: Record): Promise; - getAssetsByOwner(owner: string): Promise; - getAssetsByType(type: DataAssetType): Promise; - getAssetsByClassification(classification: DataClassification): Promise; - getAssetsByTags(tags: string[]): Promise; -} - -export class DataCatalogServiceImpl implements DataCatalogService { - private assets: Map = new Map(); - - constructor( - private eventBus: EventBus, - private logger: Logger - ) {} - - async createAsset(request: CreateDataAssetRequest): Promise { - try { - const asset: DataAsset = { - id: this.generateId(), - name: request.name, - type: request.type, - description: request.description, - owner: request.owner, - steward: request.steward, - tags: request.tags || [], - classification: request.classification, - schema: request.schema, - location: request.location, - metadata: { - customProperties: {}, - ...request.metadata - }, - lineage: { - id: this.generateId(), - assetId: '', - upstreamAssets: [], - downstreamAssets: [], - transformations: [], - impact: { - downstreamAssets: [], - affectedUsers: [], - estimatedImpact: 'low', - impactDescription: '', - recommendations: [] - }, - createdAt: new Date(), - updatedAt: new Date() - }, - quality: { - id: this.generateId(), - assetId: '', - overallScore: 100, - dimensions: [], - rules: [], - issues: [], - trend: { - timeframe: 'week', - dataPoints: [], - trend: 'stable', - changeRate: 0 - }, - lastAssessment: new Date() - }, - usage: { - id: this.generateId(), - assetId: '', - accessCount: 0, - uniqueUsers: 0, - lastAccessed: new Date(), - topUsers: [], - accessPatterns: [], - popularQueries: [], - usageTrend: { - timeframe: 'week', - dataPoints: [], - trend: 'stable', - changeRate: 0 - } - }, - governance: request.governance || { - id: this.generateId(), - assetId: '', - policies: [], - compliance: [], - retention: { - retentionPeriod: 365, - retentionReason: 'Business requirement', - legalHold: false - }, - access: { - defaultAccess: 'none', - roles: [], - users: [] - }, - privacy: { - containsPII: false, - sensitiveFields: [], - anonymizationRules: [], - consentRequired: false, - dataSubjectRights: [] - }, - audit: [] - }, - createdAt: new Date(), - updatedAt: new Date() - }; - - // Set correct asset IDs in nested objects - asset.lineage.assetId = asset.id; - asset.quality.assetId = asset.id; - asset.usage.assetId = asset.id; - asset.governance.assetId = asset.id; - - this.assets.set(asset.id, asset); - - this.logger.info('Data asset created', { assetId: asset.id, name: asset.name }); - - await this.eventBus.emit('data.asset.created', { - assetId: asset.id, - asset, - timestamp: new Date() - }); - - return asset; - } catch (error) { - this.logger.error('Failed to create data asset', { request, error }); - throw error; - } - } - - async getAsset(id: string): Promise { - try { - const asset = this.assets.get(id); - - if (asset) { - // Update last accessed time - asset.lastAccessed = new Date(); - asset.usage.lastAccessed = new Date(); - asset.usage.accessCount++; - - await this.eventBus.emit('data.asset.accessed', { - assetId: id, - timestamp: new Date() - }); - } - - return asset || null; - } catch (error) { - this.logger.error('Failed to get data asset', { assetId: id, error }); - throw error; - } - } - - async updateAsset(id: string, request: UpdateDataAssetRequest): Promise { - try { - const asset = this.assets.get(id); - if (!asset) { - return null; - } - - // Update only provided fields - if (request.name !== undefined) asset.name = request.name; - if (request.description !== undefined) asset.description = request.description; - if (request.owner !== undefined) asset.owner = request.owner; - if (request.steward !== undefined) asset.steward = request.steward; - if (request.tags !== undefined) asset.tags = request.tags; - if (request.classification !== undefined) asset.classification = request.classification; - if (request.schema !== undefined) asset.schema = request.schema; - if (request.metadata !== undefined) { - asset.metadata = { ...asset.metadata, ...request.metadata }; - } - - asset.updatedAt = new Date(); - - this.assets.set(id, asset); - - this.logger.info('Data asset updated', { assetId: id, changes: request }); - - await this.eventBus.emit('data.asset.updated', { - assetId: id, - asset, - changes: request, - timestamp: new Date() - }); - - return asset; - } catch (error) { - this.logger.error('Failed to update data asset', { assetId: id, request, error }); - throw error; - } - } - - async deleteAsset(id: string): Promise { - try { - const asset = this.assets.get(id); - if (!asset) { - throw new Error(`Asset with id ${id} not found`); - } - - this.assets.delete(id); - - this.logger.info('Data asset deleted', { assetId: id }); - - await this.eventBus.emit('data.asset.deleted', { - assetId: id, - asset, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to delete data asset', { assetId: id, error }); - throw error; - } - } - - async listAssets(filters?: Record): Promise { - try { - let assets = Array.from(this.assets.values()); - - if (filters) { - assets = assets.filter(asset => { - return Object.entries(filters).every(([key, value]) => { - if (key === 'type') return asset.type === value; - if (key === 'owner') return asset.owner === value; - if (key === 'classification') return asset.classification === value; - if (key === 'tags') return Array.isArray(value) ? - value.some(tag => asset.tags.includes(tag)) : - asset.tags.includes(value); - return true; - }); - }); - } - - return assets; - } catch (error) { - this.logger.error('Failed to list data assets', { filters, error }); - throw error; - } - } - - async searchAssets(query: string, filters?: Record): Promise { - try { - let assets = Array.from(this.assets.values()); - - // Simple text search in name, description, and tags - const searchTerm = query.toLowerCase(); - assets = assets.filter(asset => - asset.name.toLowerCase().includes(searchTerm) || - asset.description.toLowerCase().includes(searchTerm) || - asset.tags.some(tag => tag.toLowerCase().includes(searchTerm)) - ); - - // Apply additional filters - if (filters) { - assets = assets.filter(asset => { - return Object.entries(filters).every(([key, value]) => { - if (key === 'type') return asset.type === value; - if (key === 'owner') return asset.owner === value; - if (key === 'classification') return asset.classification === value; - return true; - }); - }); - } - - this.logger.info('Asset search completed', { - query, - filters, - resultCount: assets.length - }); - - return assets; - } catch (error) { - this.logger.error('Failed to search data assets', { query, filters, error }); - throw error; - } - } - - async getAssetsByOwner(owner: string): Promise { - return this.listAssets({ owner }); - } - - async getAssetsByType(type: DataAssetType): Promise { - return this.listAssets({ type }); - } - - async getAssetsByClassification(classification: DataClassification): Promise { - return this.listAssets({ classification }); - } - - async getAssetsByTags(tags: string[]): Promise { - return this.listAssets({ tags }); - } - - private generateId(): string { - return `asset_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } -} diff --git a/apps/data-services/data-catalog/src/services/DataGovernanceService.ts b/apps/data-services/data-catalog/src/services/DataGovernanceService.ts deleted file mode 100644 index 5c1a526..0000000 --- a/apps/data-services/data-catalog/src/services/DataGovernanceService.ts +++ /dev/null @@ -1,764 +0,0 @@ -import { EventBus } from '@stock-bot/event-bus'; -import { Logger } from '@stock-bot/utils'; -import { - DataGovernance, - GovernancePolicy, - ComplianceCheck, - RetentionPolicy, - AccessControl, - PrivacySettings, - AuditEntry, - DataAsset, - GovernancePolicyType, - ComplianceStatus, - DataClassification -} from '../types/DataCatalog'; - -export interface DataGovernanceService { - createPolicy(policy: Omit): Promise; - updatePolicy(policyId: string, updates: Partial): Promise; - deletePolicy(policyId: string): Promise; - getPolicy(policyId: string): Promise; - listPolicies(filters?: Record): Promise; - applyPolicy(assetId: string, policyId: string): Promise; - removePolicy(assetId: string, policyId: string): Promise; - checkCompliance(assetId: string): Promise; - updateRetentionPolicy(assetId: string, retention: RetentionPolicy): Promise; - updateAccessControl(assetId: string, access: AccessControl): Promise; - updatePrivacySettings(assetId: string, privacy: PrivacySettings): Promise; - auditAccess(assetId: string, userId: string, action: string, details?: any): Promise; - getAuditTrail(assetId: string, filters?: Record): Promise; - generateComplianceReport(assetIds: string[]): Promise; - validateDataAccess(assetId: string, userId: string, action: string): Promise; - anonymizeData(assetId: string, options?: any): Promise; - handleDataSubjectRequest(assetId: string, request: any): Promise; -} - -export class DataGovernanceServiceImpl implements DataGovernanceService { - private policies: Map = new Map(); - private governance: Map = new Map(); - private assets: Map = new Map(); - - constructor( - private eventBus: EventBus, - private logger: Logger - ) { - this.initializeDefaultPolicies(); - } - - async createPolicy(policy: Omit): Promise { - try { - const fullPolicy: GovernancePolicy = { - ...policy, - id: this.generateId(), - createdAt: new Date(), - updatedAt: new Date() - }; - - this.policies.set(fullPolicy.id, fullPolicy); - - this.logger.info('Governance policy created', { - policyId: fullPolicy.id, - name: fullPolicy.name, - type: fullPolicy.type - }); - - await this.eventBus.emit('data.governance.policy.created', { - policy: fullPolicy, - timestamp: new Date() - }); - - return fullPolicy; - } catch (error) { - this.logger.error('Failed to create governance policy', { policy, error }); - throw error; - } - } - - async updatePolicy(policyId: string, updates: Partial): Promise { - try { - const policy = this.policies.get(policyId); - if (!policy) { - return null; - } - - const updatedPolicy: GovernancePolicy = { - ...policy, - ...updates, - updatedAt: new Date() - }; - - this.policies.set(policyId, updatedPolicy); - - this.logger.info('Governance policy updated', { policyId, changes: updates }); - - await this.eventBus.emit('data.governance.policy.updated', { - policy: updatedPolicy, - changes: updates, - timestamp: new Date() - }); - - return updatedPolicy; - } catch (error) { - this.logger.error('Failed to update governance policy', { policyId, updates, error }); - throw error; - } - } - - async deletePolicy(policyId: string): Promise { - try { - const policy = this.policies.get(policyId); - if (!policy) { - throw new Error(`Policy with id ${policyId} not found`); - } - - this.policies.delete(policyId); - - // Remove policy from all assets - for (const [assetId, governance] of this.governance) { - governance.policies = governance.policies.filter(p => p.id !== policyId); - this.governance.set(assetId, governance); - } - - this.logger.info('Governance policy deleted', { policyId }); - - await this.eventBus.emit('data.governance.policy.deleted', { - policyId, - policy, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to delete governance policy', { policyId, error }); - throw error; - } - } - - async getPolicy(policyId: string): Promise { - try { - return this.policies.get(policyId) || null; - } catch (error) { - this.logger.error('Failed to get governance policy', { policyId, error }); - throw error; - } - } - - async listPolicies(filters?: Record): Promise { - try { - let policies = Array.from(this.policies.values()); - - if (filters) { - policies = policies.filter(policy => { - return Object.entries(filters).every(([key, value]) => { - if (key === 'type') return policy.type === value; - if (key === 'active') return policy.active === value; - if (key === 'classification') return policy.applicableClassifications?.includes(value); - return true; - }); - }); - } - - return policies; - } catch (error) { - this.logger.error('Failed to list governance policies', { filters, error }); - throw error; - } - } - - async applyPolicy(assetId: string, policyId: string): Promise { - try { - const policy = this.policies.get(policyId); - if (!policy) { - throw new Error(`Policy with id ${policyId} not found`); - } - - let governance = this.governance.get(assetId); - if (!governance) { - governance = this.createEmptyGovernance(assetId); - } - - // Check if policy is already applied - if (!governance.policies.find(p => p.id === policyId)) { - governance.policies.push(policy); - this.governance.set(assetId, governance); - - // Perform compliance check after applying policy - await this.checkCompliance(assetId); - - this.logger.info('Policy applied to asset', { assetId, policyId }); - - await this.eventBus.emit('data.governance.policy.applied', { - assetId, - policyId, - timestamp: new Date() - }); - } - } catch (error) { - this.logger.error('Failed to apply policy to asset', { assetId, policyId, error }); - throw error; - } - } - - async removePolicy(assetId: string, policyId: string): Promise { - try { - const governance = this.governance.get(assetId); - if (!governance) { - throw new Error(`Governance not found for asset ${assetId}`); - } - - governance.policies = governance.policies.filter(p => p.id !== policyId); - this.governance.set(assetId, governance); - - this.logger.info('Policy removed from asset', { assetId, policyId }); - - await this.eventBus.emit('data.governance.policy.removed', { - assetId, - policyId, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to remove policy from asset', { assetId, policyId, error }); - throw error; - } - } - - async checkCompliance(assetId: string): Promise { - try { - const governance = this.governance.get(assetId); - const asset = this.assets.get(assetId); - - if (!governance || !asset) { - return []; - } - - const complianceChecks: ComplianceCheck[] = []; - - for (const policy of governance.policies) { - if (!policy.active) continue; - - const check = await this.performComplianceCheck(asset, policy); - complianceChecks.push(check); - } - - // Update governance with compliance results - governance.compliance = complianceChecks; - this.governance.set(assetId, governance); - - // Log compliance issues - const failedChecks = complianceChecks.filter(check => check.status === 'failed'); - if (failedChecks.length > 0) { - this.logger.warn('Compliance violations detected', { - assetId, - violationCount: failedChecks.length - }); - - await this.eventBus.emit('data.governance.compliance.violation', { - assetId, - violations: failedChecks, - timestamp: new Date() - }); - } - - return complianceChecks; - } catch (error) { - this.logger.error('Failed to check compliance', { assetId, error }); - throw error; - } - } - - async updateRetentionPolicy(assetId: string, retention: RetentionPolicy): Promise { - try { - let governance = this.governance.get(assetId); - if (!governance) { - governance = this.createEmptyGovernance(assetId); - } - - governance.retention = retention; - this.governance.set(assetId, governance); - - this.logger.info('Retention policy updated', { assetId, retentionPeriod: retention.retentionPeriod }); - - await this.eventBus.emit('data.governance.retention.updated', { - assetId, - retention, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to update retention policy', { assetId, retention, error }); - throw error; - } - } - - async updateAccessControl(assetId: string, access: AccessControl): Promise { - try { - let governance = this.governance.get(assetId); - if (!governance) { - governance = this.createEmptyGovernance(assetId); - } - - governance.access = access; - this.governance.set(assetId, governance); - - this.logger.info('Access control updated', { assetId, defaultAccess: access.defaultAccess }); - - await this.eventBus.emit('data.governance.access.updated', { - assetId, - access, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to update access control', { assetId, access, error }); - throw error; - } - } - - async updatePrivacySettings(assetId: string, privacy: PrivacySettings): Promise { - try { - let governance = this.governance.get(assetId); - if (!governance) { - governance = this.createEmptyGovernance(assetId); - } - - governance.privacy = privacy; - this.governance.set(assetId, governance); - - this.logger.info('Privacy settings updated', { - assetId, - containsPII: privacy.containsPII, - consentRequired: privacy.consentRequired - }); - - await this.eventBus.emit('data.governance.privacy.updated', { - assetId, - privacy, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to update privacy settings', { assetId, privacy, error }); - throw error; - } - } - - async auditAccess(assetId: string, userId: string, action: string, details?: any): Promise { - try { - let governance = this.governance.get(assetId); - if (!governance) { - governance = this.createEmptyGovernance(assetId); - } - - const auditEntry: AuditEntry = { - id: this.generateId(), - userId, - action, - timestamp: new Date(), - ipAddress: details?.ipAddress, - userAgent: details?.userAgent, - details - }; - - governance.audit.push(auditEntry); - this.governance.set(assetId, governance); - - this.logger.info('Access audited', { assetId, userId, action }); - - await this.eventBus.emit('data.governance.access.audited', { - assetId, - auditEntry, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to audit access', { assetId, userId, action, error }); - throw error; - } - } - - async getAuditTrail(assetId: string, filters?: Record): Promise { - try { - const governance = this.governance.get(assetId); - if (!governance) { - return []; - } - - let auditEntries = governance.audit; - - if (filters) { - auditEntries = auditEntries.filter(entry => { - return Object.entries(filters).every(([key, value]) => { - if (key === 'userId') return entry.userId === value; - if (key === 'action') return entry.action === value; - if (key === 'fromDate') return entry.timestamp >= new Date(value); - if (key === 'toDate') return entry.timestamp <= new Date(value); - return true; - }); - }); - } - - return auditEntries.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); - } catch (error) { - this.logger.error('Failed to get audit trail', { assetId, filters, error }); - throw error; - } - } - - async generateComplianceReport(assetIds: string[]): Promise { - try { - const reportData = { - summary: { - totalAssets: assetIds.length, - compliantAssets: 0, - nonCompliantAssets: 0, - violationCount: 0, - reportDate: new Date() - }, - assetCompliance: [] as any[], - policyViolations: [] as any[], - recommendations: [] as string[] - }; - - let totalViolations = 0; - - for (const assetId of assetIds) { - const governance = this.governance.get(assetId); - const asset = this.assets.get(assetId); - - if (governance && asset) { - const complianceChecks = await this.checkCompliance(assetId); - const violations = complianceChecks.filter(check => check.status === 'failed'); - const isCompliant = violations.length === 0; - - if (isCompliant) { - reportData.summary.compliantAssets++; - } else { - reportData.summary.nonCompliantAssets++; - } - - totalViolations += violations.length; - - reportData.assetCompliance.push({ - assetId, - assetName: asset.name, - classification: asset.classification, - compliant: isCompliant, - violationCount: violations.length, - policiesApplied: governance.policies.length, - lastChecked: new Date() - }); - - // Add violations to report - violations.forEach(violation => { - reportData.policyViolations.push({ - assetId, - assetName: asset.name, - policyName: violation.policyName, - violation: violation.details, - severity: violation.severity || 'medium', - checkedAt: violation.checkedAt - }); - }); - } - } - - reportData.summary.violationCount = totalViolations; - - // Generate recommendations - reportData.recommendations = this.generateComplianceRecommendations(reportData); - - this.logger.info('Compliance report generated', { - totalAssets: assetIds.length, - compliantAssets: reportData.summary.compliantAssets, - violationCount: totalViolations - }); - - return reportData; - } catch (error) { - this.logger.error('Failed to generate compliance report', { assetIds, error }); - throw error; - } - } - - async validateDataAccess(assetId: string, userId: string, action: string): Promise { - try { - const governance = this.governance.get(assetId); - const asset = this.assets.get(assetId); - - if (!governance || !asset) { - return false; - } - - // Check default access - if (governance.access.defaultAccess === 'none') { - // Must have explicit permission - const hasUserAccess = governance.access.users.some(user => - user.userId === userId && user.permissions.includes(action) - ); - - const hasRoleAccess = governance.access.roles.some(role => - role.permissions.includes(action) // Simplified - would check user roles - ); - - return hasUserAccess || hasRoleAccess; - } - - // Check if explicitly denied - const isDenied = governance.access.users.some(user => - user.userId === userId && user.permissions.includes(`deny:${action}`) - ); - - if (isDenied) { - return false; - } - - // Check classification-based access - if (asset.classification === 'restricted' || asset.classification === 'confidential') { - // Require explicit permission for sensitive data - const hasPermission = governance.access.users.some(user => - user.userId === userId && user.permissions.includes(action) - ); - return hasPermission; - } - - return true; // Default allow for non-sensitive data - } catch (error) { - this.logger.error('Failed to validate data access', { assetId, userId, action, error }); - return false; - } - } - - async anonymizeData(assetId: string, options?: any): Promise { - try { - const governance = this.governance.get(assetId); - if (!governance || !governance.privacy.containsPII) { - return; - } - - // Apply anonymization rules - for (const rule of governance.privacy.anonymizationRules) { - await this.applyAnonymizationRule(assetId, rule, options); - } - - this.logger.info('Data anonymization completed', { assetId }); - - await this.eventBus.emit('data.governance.anonymization.completed', { - assetId, - options, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to anonymize data', { assetId, options, error }); - throw error; - } - } - - async handleDataSubjectRequest(assetId: string, request: any): Promise { - try { - const governance = this.governance.get(assetId); - const asset = this.assets.get(assetId); - - if (!governance || !asset) { - throw new Error(`Asset or governance not found for ${assetId}`); - } - - let response: any = {}; - - switch (request.type) { - case 'access': - response = await this.handleAccessRequest(assetId, request); - break; - case 'rectification': - response = await this.handleRectificationRequest(assetId, request); - break; - case 'erasure': - response = await this.handleErasureRequest(assetId, request); - break; - case 'portability': - response = await this.handlePortabilityRequest(assetId, request); - break; - default: - throw new Error(`Unsupported request type: ${request.type}`); - } - - this.logger.info('Data subject request handled', { assetId, requestType: request.type }); - - await this.eventBus.emit('data.governance.subject.request.handled', { - assetId, - request, - response, - timestamp: new Date() - }); - - return response; - } catch (error) { - this.logger.error('Failed to handle data subject request', { assetId, request, error }); - throw error; - } - } - - // Private helper methods - private initializeDefaultPolicies(): void { - const defaultPolicies: GovernancePolicy[] = [ - { - id: 'policy_pii_protection', - name: 'PII Protection Policy', - description: 'Ensures proper handling of personally identifiable information', - type: 'privacy', - rules: [ - 'PII data must be encrypted at rest', - 'PII access must be logged', - 'PII retention must not exceed 7 years' - ], - applicableClassifications: ['pii'], - active: true, - severity: 'high', - createdAt: new Date(), - updatedAt: new Date() - }, - { - id: 'policy_financial_compliance', - name: 'Financial Data Compliance', - description: 'Compliance with financial regulations', - type: 'compliance', - rules: [ - 'Financial data must be retained for 7 years', - 'Access to financial data must be role-based', - 'All financial data access must be audited' - ], - applicableClassifications: ['financial'], - active: true, - severity: 'critical', - createdAt: new Date(), - updatedAt: new Date() - } - ]; - - defaultPolicies.forEach(policy => { - this.policies.set(policy.id, policy); - }); - } - - private createEmptyGovernance(assetId: string): DataGovernance { - return { - id: this.generateId(), - assetId, - policies: [], - compliance: [], - retention: { - retentionPeriod: 365, - retentionReason: 'Business requirement', - legalHold: false - }, - access: { - defaultAccess: 'none', - roles: [], - users: [] - }, - privacy: { - containsPII: false, - sensitiveFields: [], - anonymizationRules: [], - consentRequired: false, - dataSubjectRights: [] - }, - audit: [] - }; - } - - private async performComplianceCheck(asset: DataAsset, policy: GovernancePolicy): Promise { - // Mock compliance check implementation - // In real scenario, this would validate actual compliance - - const isCompliant = Math.random() > 0.1; // 90% compliance rate for demo - - const check: ComplianceCheck = { - id: this.generateId(), - policyId: policy.id, - policyName: policy.name, - status: isCompliant ? 'passed' : 'failed', - checkedAt: new Date(), - details: isCompliant ? 'All policy requirements met' : 'Policy violation detected', - severity: policy.severity - }; - - if (!isCompliant) { - check.recommendations = [ - 'Review data handling procedures', - 'Update access controls', - 'Implement additional monitoring' - ]; - } - - return check; - } - - private async applyAnonymizationRule(assetId: string, rule: any, options?: any): Promise { - // Mock anonymization implementation - this.logger.info('Applying anonymization rule', { assetId, rule: rule.type }); - } - - private async handleAccessRequest(assetId: string, request: any): Promise { - return { - status: 'completed', - data: 'Data access provided according to privacy policy', - timestamp: new Date() - }; - } - - private async handleRectificationRequest(assetId: string, request: any): Promise { - return { - status: 'completed', - changes: 'Data rectification completed', - timestamp: new Date() - }; - } - - private async handleErasureRequest(assetId: string, request: any): Promise { - return { - status: 'completed', - erasure: 'Data erasure completed', - timestamp: new Date() - }; - } - - private async handlePortabilityRequest(assetId: string, request: any): Promise { - return { - status: 'completed', - export: 'Data export provided', - timestamp: new Date() - }; - } - - private generateComplianceRecommendations(reportData: any): string[] { - const recommendations: string[] = []; - - if (reportData.summary.nonCompliantAssets > 0) { - recommendations.push(`${reportData.summary.nonCompliantAssets} assets require compliance remediation.`); - } - - if (reportData.summary.violationCount > 10) { - recommendations.push('High number of policy violations detected. Review governance policies and implementation.'); - } - - const criticalViolations = reportData.policyViolations.filter((v: any) => v.severity === 'critical'); - if (criticalViolations.length > 0) { - recommendations.push(`${criticalViolations.length} critical violations require immediate attention.`); - } - - if (recommendations.length === 0) { - recommendations.push('All assets are compliant with governance policies. Continue monitoring.'); - } - - return recommendations; - } - - private generateId(): string { - return `governance_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - // Method to inject assets (typically from DataCatalogService) - setAssets(assets: Map): void { - this.assets = assets; - } - - // Method to inject governance (typically from DataCatalogService) - setGovernance(governance: Map): void { - this.governance = governance; - } -} diff --git a/apps/data-services/data-catalog/src/services/DataLineageService.ts b/apps/data-services/data-catalog/src/services/DataLineageService.ts deleted file mode 100644 index 670846c..0000000 --- a/apps/data-services/data-catalog/src/services/DataLineageService.ts +++ /dev/null @@ -1,607 +0,0 @@ -import { EventBus } from '@stock-bot/event-bus'; -import { Logger } from '@stock-bot/utils'; -import { - DataLineage, - DataAsset, - LineageTransformation, - ImpactAnalysis, - LineageQuery, - LineageDirection -} from '../types/DataCatalog'; - -export interface DataLineageService { - addLineage(lineage: DataLineage): Promise; - getLineage(assetId: string): Promise; - updateLineage(assetId: string, lineage: Partial): Promise; - addUpstreamDependency(assetId: string, upstreamAssetId: string, transformation?: LineageTransformation): Promise; - addDownstreamDependency(assetId: string, downstreamAssetId: string, transformation?: LineageTransformation): Promise; - removeUpstreamDependency(assetId: string, upstreamAssetId: string): Promise; - removeDownstreamDependency(assetId: string, downstreamAssetId: string): Promise; - getUpstreamAssets(assetId: string, depth?: number): Promise; - getDownstreamAssets(assetId: string, depth?: number): Promise; - analyzeImpact(assetId: string): Promise; - queryLineage(query: LineageQuery): Promise; - getLineageGraph(assetId: string, direction: LineageDirection, depth?: number): Promise; - detectCircularDependencies(): Promise; -} - -export class DataLineageServiceImpl implements DataLineageService { - private lineages: Map = new Map(); - private assets: Map = new Map(); - - constructor( - private eventBus: EventBus, - private logger: Logger - ) {} - - async addLineage(lineage: DataLineage): Promise { - try { - this.lineages.set(lineage.assetId, lineage); - - this.logger.info('Data lineage added', { - assetId: lineage.assetId, - upstreamCount: lineage.upstreamAssets.length, - downstreamCount: lineage.downstreamAssets.length - }); - - await this.eventBus.emit('data.lineage.added', { - assetId: lineage.assetId, - lineage, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to add data lineage', { lineage, error }); - throw error; - } - } - - async getLineage(assetId: string): Promise { - try { - return this.lineages.get(assetId) || null; - } catch (error) { - this.logger.error('Failed to get data lineage', { assetId, error }); - throw error; - } - } - - async updateLineage(assetId: string, lineage: Partial): Promise { - try { - const existingLineage = this.lineages.get(assetId); - if (!existingLineage) { - return null; - } - - const updatedLineage: DataLineage = { - ...existingLineage, - ...lineage, - updatedAt: new Date() - }; - - this.lineages.set(assetId, updatedLineage); - - this.logger.info('Data lineage updated', { assetId, changes: lineage }); - - await this.eventBus.emit('data.lineage.updated', { - assetId, - lineage: updatedLineage, - changes: lineage, - timestamp: new Date() - }); - - return updatedLineage; - } catch (error) { - this.logger.error('Failed to update data lineage', { assetId, lineage, error }); - throw error; - } - } - - async addUpstreamDependency( - assetId: string, - upstreamAssetId: string, - transformation?: LineageTransformation - ): Promise { - try { - let lineage = this.lineages.get(assetId); - if (!lineage) { - lineage = this.createEmptyLineage(assetId); - } - - // Check if dependency already exists - if (!lineage.upstreamAssets.includes(upstreamAssetId)) { - lineage.upstreamAssets.push(upstreamAssetId); - - if (transformation) { - lineage.transformations.push(transformation); - } - - lineage.updatedAt = new Date(); - this.lineages.set(assetId, lineage); - - // Update downstream lineage of the upstream asset - await this.addDownstreamToUpstream(upstreamAssetId, assetId); - - this.logger.info('Upstream dependency added', { assetId, upstreamAssetId }); - - await this.eventBus.emit('data.lineage.dependency.added', { - assetId, - upstreamAssetId, - transformation, - timestamp: new Date() - }); - } - } catch (error) { - this.logger.error('Failed to add upstream dependency', { assetId, upstreamAssetId, error }); - throw error; - } - } - - async addDownstreamDependency( - assetId: string, - downstreamAssetId: string, - transformation?: LineageTransformation - ): Promise { - try { - let lineage = this.lineages.get(assetId); - if (!lineage) { - lineage = this.createEmptyLineage(assetId); - } - - // Check if dependency already exists - if (!lineage.downstreamAssets.includes(downstreamAssetId)) { - lineage.downstreamAssets.push(downstreamAssetId); - lineage.updatedAt = new Date(); - this.lineages.set(assetId, lineage); - - // Update upstream lineage of the downstream asset - await this.addUpstreamToDownstream(downstreamAssetId, assetId, transformation); - - this.logger.info('Downstream dependency added', { assetId, downstreamAssetId }); - - await this.eventBus.emit('data.lineage.dependency.added', { - assetId, - downstreamAssetId, - transformation, - timestamp: new Date() - }); - } - } catch (error) { - this.logger.error('Failed to add downstream dependency', { assetId, downstreamAssetId, error }); - throw error; - } - } - - async removeUpstreamDependency(assetId: string, upstreamAssetId: string): Promise { - try { - const lineage = this.lineages.get(assetId); - if (lineage) { - lineage.upstreamAssets = lineage.upstreamAssets.filter(id => id !== upstreamAssetId); - lineage.updatedAt = new Date(); - this.lineages.set(assetId, lineage); - - // Remove from downstream lineage of upstream asset - await this.removeDownstreamFromUpstream(upstreamAssetId, assetId); - - this.logger.info('Upstream dependency removed', { assetId, upstreamAssetId }); - - await this.eventBus.emit('data.lineage.dependency.removed', { - assetId, - upstreamAssetId, - timestamp: new Date() - }); - } - } catch (error) { - this.logger.error('Failed to remove upstream dependency', { assetId, upstreamAssetId, error }); - throw error; - } - } - - async removeDownstreamDependency(assetId: string, downstreamAssetId: string): Promise { - try { - const lineage = this.lineages.get(assetId); - if (lineage) { - lineage.downstreamAssets = lineage.downstreamAssets.filter(id => id !== downstreamAssetId); - lineage.updatedAt = new Date(); - this.lineages.set(assetId, lineage); - - // Remove from upstream lineage of downstream asset - await this.removeUpstreamFromDownstream(downstreamAssetId, assetId); - - this.logger.info('Downstream dependency removed', { assetId, downstreamAssetId }); - - await this.eventBus.emit('data.lineage.dependency.removed', { - assetId, - downstreamAssetId, - timestamp: new Date() - }); - } - } catch (error) { - this.logger.error('Failed to remove downstream dependency', { assetId, downstreamAssetId, error }); - throw error; - } - } - - async getUpstreamAssets(assetId: string, depth: number = 1): Promise { - try { - const visited = new Set(); - const result: DataAsset[] = []; - - await this.traverseUpstream(assetId, depth, visited, result); - - return result; - } catch (error) { - this.logger.error('Failed to get upstream assets', { assetId, depth, error }); - throw error; - } - } - - async getDownstreamAssets(assetId: string, depth: number = 1): Promise { - try { - const visited = new Set(); - const result: DataAsset[] = []; - - await this.traverseDownstream(assetId, depth, visited, result); - - return result; - } catch (error) { - this.logger.error('Failed to get downstream assets', { assetId, depth, error }); - throw error; - } - } - - async analyzeImpact(assetId: string): Promise { - try { - const downstreamAssets = await this.getDownstreamAssets(assetId, 5); // Go deep for impact analysis - const affectedUsers = new Set(); - - // Collect all users who might be affected - for (const asset of downstreamAssets) { - affectedUsers.add(asset.owner); - if (asset.steward) { - affectedUsers.add(asset.steward); - } - // Add users from usage analytics - asset.usage.topUsers.forEach(user => affectedUsers.add(user.userId)); - } - - // Calculate impact level - let estimatedImpact: 'low' | 'medium' | 'high' | 'critical' = 'low'; - if (downstreamAssets.length > 20) { - estimatedImpact = 'critical'; - } else if (downstreamAssets.length > 10) { - estimatedImpact = 'high'; - } else if (downstreamAssets.length > 5) { - estimatedImpact = 'medium'; - } - - const impact: ImpactAnalysis = { - downstreamAssets: downstreamAssets.map(asset => asset.id), - affectedUsers: Array.from(affectedUsers), - estimatedImpact, - impactDescription: this.generateImpactDescription(downstreamAssets.length, Array.from(affectedUsers).length), - recommendations: this.generateRecommendations(estimatedImpact, downstreamAssets.length) - }; - - this.logger.info('Impact analysis completed', { - assetId, - impactLevel: estimatedImpact, - affectedAssets: downstreamAssets.length, - affectedUsers: affectedUsers.size - }); - - return impact; - } catch (error) { - this.logger.error('Failed to analyze impact', { assetId, error }); - throw error; - } - } - - async queryLineage(query: LineageQuery): Promise { - try { - let results: DataAsset[] = []; - - if (query.assetIds) { - for (const assetId of query.assetIds) { - if (query.direction === 'upstream' || query.direction === 'both') { - const upstream = await this.getUpstreamAssets(assetId, query.depth); - results.push(...upstream); - } - if (query.direction === 'downstream' || query.direction === 'both') { - const downstream = await this.getDownstreamAssets(assetId, query.depth); - results.push(...downstream); - } - } - } - - // Remove duplicates - const uniqueResults = results.filter((asset, index, arr) => - arr.findIndex(a => a.id === asset.id) === index - ); - - return uniqueResults; - } catch (error) { - this.logger.error('Failed to query lineage', { query, error }); - throw error; - } - } - - async getLineageGraph(assetId: string, direction: LineageDirection, depth: number = 3): Promise { - try { - const graph = { - nodes: new Map(), - edges: [] - }; - - const visited = new Set(); - await this.buildLineageGraph(assetId, direction, depth, visited, graph); - - return { - nodes: Array.from(graph.nodes.values()), - edges: graph.edges - }; - } catch (error) { - this.logger.error('Failed to get lineage graph', { assetId, direction, depth, error }); - throw error; - } - } - - async detectCircularDependencies(): Promise { - try { - const cycles: string[][] = []; - const visited = new Set(); - const recursionStack = new Set(); - - for (const assetId of this.lineages.keys()) { - if (!visited.has(assetId)) { - const path: string[] = []; - await this.detectCycleDFS(assetId, visited, recursionStack, path, cycles); - } - } - - if (cycles.length > 0) { - this.logger.warn('Circular dependencies detected', { cycleCount: cycles.length }); - } - - return cycles; - } catch (error) { - this.logger.error('Failed to detect circular dependencies', { error }); - throw error; - } - } - - // Private helper methods - private createEmptyLineage(assetId: string): DataLineage { - return { - id: this.generateId(), - assetId, - upstreamAssets: [], - downstreamAssets: [], - transformations: [], - impact: { - downstreamAssets: [], - affectedUsers: [], - estimatedImpact: 'low', - impactDescription: '', - recommendations: [] - }, - createdAt: new Date(), - updatedAt: new Date() - }; - } - - private async addDownstreamToUpstream(upstreamAssetId: string, downstreamAssetId: string): Promise { - let upstreamLineage = this.lineages.get(upstreamAssetId); - if (!upstreamLineage) { - upstreamLineage = this.createEmptyLineage(upstreamAssetId); - } - - if (!upstreamLineage.downstreamAssets.includes(downstreamAssetId)) { - upstreamLineage.downstreamAssets.push(downstreamAssetId); - upstreamLineage.updatedAt = new Date(); - this.lineages.set(upstreamAssetId, upstreamLineage); - } - } - - private async addUpstreamToDownstream( - downstreamAssetId: string, - upstreamAssetId: string, - transformation?: LineageTransformation - ): Promise { - let downstreamLineage = this.lineages.get(downstreamAssetId); - if (!downstreamLineage) { - downstreamLineage = this.createEmptyLineage(downstreamAssetId); - } - - if (!downstreamLineage.upstreamAssets.includes(upstreamAssetId)) { - downstreamLineage.upstreamAssets.push(upstreamAssetId); - - if (transformation) { - downstreamLineage.transformations.push(transformation); - } - - downstreamLineage.updatedAt = new Date(); - this.lineages.set(downstreamAssetId, downstreamLineage); - } - } - - private async removeDownstreamFromUpstream(upstreamAssetId: string, downstreamAssetId: string): Promise { - const upstreamLineage = this.lineages.get(upstreamAssetId); - if (upstreamLineage) { - upstreamLineage.downstreamAssets = upstreamLineage.downstreamAssets.filter(id => id !== downstreamAssetId); - upstreamLineage.updatedAt = new Date(); - this.lineages.set(upstreamAssetId, upstreamLineage); - } - } - - private async removeUpstreamFromDownstream(downstreamAssetId: string, upstreamAssetId: string): Promise { - const downstreamLineage = this.lineages.get(downstreamAssetId); - if (downstreamLineage) { - downstreamLineage.upstreamAssets = downstreamLineage.upstreamAssets.filter(id => id !== upstreamAssetId); - downstreamLineage.updatedAt = new Date(); - this.lineages.set(downstreamAssetId, downstreamLineage); - } - } - - private async traverseUpstream( - assetId: string, - remainingDepth: number, - visited: Set, - result: DataAsset[] - ): Promise { - if (remainingDepth === 0 || visited.has(assetId)) { - return; - } - - visited.add(assetId); - const lineage = this.lineages.get(assetId); - - if (lineage) { - for (const upstreamId of lineage.upstreamAssets) { - const asset = this.assets.get(upstreamId); - if (asset && !result.find(a => a.id === asset.id)) { - result.push(asset); - } - await this.traverseUpstream(upstreamId, remainingDepth - 1, visited, result); - } - } - } - - private async traverseDownstream( - assetId: string, - remainingDepth: number, - visited: Set, - result: DataAsset[] - ): Promise { - if (remainingDepth === 0 || visited.has(assetId)) { - return; - } - - visited.add(assetId); - const lineage = this.lineages.get(assetId); - - if (lineage) { - for (const downstreamId of lineage.downstreamAssets) { - const asset = this.assets.get(downstreamId); - if (asset && !result.find(a => a.id === asset.id)) { - result.push(asset); - } - await this.traverseDownstream(downstreamId, remainingDepth - 1, visited, result); - } - } - } - - private async buildLineageGraph( - assetId: string, - direction: LineageDirection, - remainingDepth: number, - visited: Set, - graph: any - ): Promise { - if (remainingDepth === 0 || visited.has(assetId)) { - return; - } - - visited.add(assetId); - const asset = this.assets.get(assetId); - const lineage = this.lineages.get(assetId); - - if (asset) { - graph.nodes.set(assetId, { - id: assetId, - name: asset.name, - type: asset.type, - classification: asset.classification - }); - } - - if (lineage) { - if (direction === 'upstream' || direction === 'both') { - for (const upstreamId of lineage.upstreamAssets) { - graph.edges.push({ - source: upstreamId, - target: assetId, - type: 'upstream' - }); - await this.buildLineageGraph(upstreamId, direction, remainingDepth - 1, visited, graph); - } - } - - if (direction === 'downstream' || direction === 'both') { - for (const downstreamId of lineage.downstreamAssets) { - graph.edges.push({ - source: assetId, - target: downstreamId, - type: 'downstream' - }); - await this.buildLineageGraph(downstreamId, direction, remainingDepth - 1, visited, graph); - } - } - } - } - - private async detectCycleDFS( - assetId: string, - visited: Set, - recursionStack: Set, - path: string[], - cycles: string[][] - ): Promise { - visited.add(assetId); - recursionStack.add(assetId); - path.push(assetId); - - const lineage = this.lineages.get(assetId); - if (lineage) { - for (const downstreamId of lineage.downstreamAssets) { - if (!visited.has(downstreamId)) { - await this.detectCycleDFS(downstreamId, visited, recursionStack, path, cycles); - } else if (recursionStack.has(downstreamId)) { - // Found a cycle - const cycleStart = path.indexOf(downstreamId); - cycles.push(path.slice(cycleStart)); - } - } - } - - path.pop(); - recursionStack.delete(assetId); - } - - private generateImpactDescription(assetCount: number, userCount: number): string { - if (assetCount === 0) { - return 'No downstream dependencies identified.'; - } - - return `Changes to this asset may affect ${assetCount} downstream asset(s) and ${userCount} user(s).`; - } - - private generateRecommendations(impact: string, assetCount: number): string[] { - const recommendations: string[] = []; - - if (impact === 'critical') { - recommendations.push('Schedule maintenance window'); - recommendations.push('Notify all stakeholders in advance'); - recommendations.push('Prepare rollback plan'); - recommendations.push('Consider phased rollout'); - } else if (impact === 'high') { - recommendations.push('Notify affected users'); - recommendations.push('Test changes thoroughly'); - recommendations.push('Monitor downstream systems'); - } else if (impact === 'medium') { - recommendations.push('Test with subset of data'); - recommendations.push('Monitor for issues'); - } else { - recommendations.push('Standard testing procedures apply'); - } - - return recommendations; - } - - private generateId(): string { - return `lineage_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - // Method to inject assets (typically from DataCatalogService) - setAssets(assets: Map): void { - this.assets = assets; - } -} diff --git a/apps/data-services/data-catalog/src/services/DataQualityService.ts b/apps/data-services/data-catalog/src/services/DataQualityService.ts deleted file mode 100644 index 7617a68..0000000 --- a/apps/data-services/data-catalog/src/services/DataQualityService.ts +++ /dev/null @@ -1,734 +0,0 @@ -import { EventBus } from '@stock-bot/event-bus'; -import { Logger } from '@stock-bot/utils'; -import { - DataQuality, - QualityDimension, - QualityRule, - QualityIssue, - QualityTrend, - DataAsset, - QualityAssessmentRequest, - QualityRuleType, - QualitySeverity -} from '../types/DataCatalog'; - -export interface DataQualityService { - assessQuality(assetId: string, request: QualityAssessmentRequest): Promise; - getQuality(assetId: string): Promise; - updateQuality(assetId: string, quality: Partial): Promise; - addQualityRule(assetId: string, rule: QualityRule): Promise; - removeQualityRule(assetId: string, ruleId: string): Promise; - validateRule(assetId: string, ruleId: string): Promise; - reportIssue(assetId: string, issue: Omit): Promise; - resolveIssue(assetId: string, issueId: string): Promise; - getTrendAnalysis(assetId: string, timeframe: string): Promise; - getQualityMetrics(filters?: Record): Promise; - generateQualityReport(assetIds: string[]): Promise; -} - -export class DataQualityServiceImpl implements DataQualityService { - private qualities: Map = new Map(); - private assets: Map = new Map(); - - constructor( - private eventBus: EventBus, - private logger: Logger - ) {} - - async assessQuality(assetId: string, request: QualityAssessmentRequest): Promise { - try { - const asset = this.assets.get(assetId); - if (!asset) { - throw new Error(`Asset with id ${assetId} not found`); - } - - let quality = this.qualities.get(assetId); - if (!quality) { - quality = this.createEmptyQuality(assetId); - } - - // Perform quality assessment based on request - const assessmentResults = await this.performQualityAssessment(asset, request); - - // Update quality metrics - quality.dimensions = assessmentResults.dimensions; - quality.overallScore = this.calculateOverallScore(assessmentResults.dimensions); - quality.lastAssessment = new Date(); - - // Update trend data - this.updateQualityTrend(quality, quality.overallScore); - - this.qualities.set(assetId, quality); - - this.logger.info('Quality assessment completed', { - assetId, - overallScore: quality.overallScore, - dimensionCount: quality.dimensions.length - }); - - await this.eventBus.emit('data.quality.assessed', { - assetId, - quality, - request, - timestamp: new Date() - }); - - return quality; - } catch (error) { - this.logger.error('Failed to assess quality', { assetId, request, error }); - throw error; - } - } - - async getQuality(assetId: string): Promise { - try { - return this.qualities.get(assetId) || null; - } catch (error) { - this.logger.error('Failed to get quality', { assetId, error }); - throw error; - } - } - - async updateQuality(assetId: string, quality: Partial): Promise { - try { - const existingQuality = this.qualities.get(assetId); - if (!existingQuality) { - return null; - } - - const updatedQuality: DataQuality = { - ...existingQuality, - ...quality, - lastAssessment: new Date() - }; - - this.qualities.set(assetId, updatedQuality); - - this.logger.info('Quality updated', { assetId, changes: quality }); - - await this.eventBus.emit('data.quality.updated', { - assetId, - quality: updatedQuality, - changes: quality, - timestamp: new Date() - }); - - return updatedQuality; - } catch (error) { - this.logger.error('Failed to update quality', { assetId, quality, error }); - throw error; - } - } - - async addQualityRule(assetId: string, rule: QualityRule): Promise { - try { - let quality = this.qualities.get(assetId); - if (!quality) { - quality = this.createEmptyQuality(assetId); - } - - // Ensure rule has an ID - if (!rule.id) { - rule.id = this.generateId(); - } - - quality.rules.push(rule); - this.qualities.set(assetId, quality); - - this.logger.info('Quality rule added', { assetId, ruleId: rule.id, ruleType: rule.type }); - - await this.eventBus.emit('data.quality.rule.added', { - assetId, - rule, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to add quality rule', { assetId, rule, error }); - throw error; - } - } - - async removeQualityRule(assetId: string, ruleId: string): Promise { - try { - const quality = this.qualities.get(assetId); - if (!quality) { - throw new Error(`Quality not found for asset ${assetId}`); - } - - quality.rules = quality.rules.filter(rule => rule.id !== ruleId); - this.qualities.set(assetId, quality); - - this.logger.info('Quality rule removed', { assetId, ruleId }); - - await this.eventBus.emit('data.quality.rule.removed', { - assetId, - ruleId, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to remove quality rule', { assetId, ruleId, error }); - throw error; - } - } - - async validateRule(assetId: string, ruleId: string): Promise { - try { - const quality = this.qualities.get(assetId); - const asset = this.assets.get(assetId); - - if (!quality || !asset) { - return false; - } - - const rule = quality.rules.find(r => r.id === ruleId); - if (!rule) { - return false; - } - - const isValid = await this.executeQualityRule(asset, rule); - - if (!isValid) { - // Create quality issue - const issue: QualityIssue = { - id: this.generateId(), - ruleId: rule.id, - type: rule.type, - severity: rule.severity, - message: `Quality rule validation failed: ${rule.description}`, - detectedAt: new Date(), - resolved: false - }; - - quality.issues.push(issue); - this.qualities.set(assetId, quality); - - await this.eventBus.emit('data.quality.issue.detected', { - assetId, - issue, - rule, - timestamp: new Date() - }); - } - - return isValid; - } catch (error) { - this.logger.error('Failed to validate quality rule', { assetId, ruleId, error }); - throw error; - } - } - - async reportIssue(assetId: string, issue: Omit): Promise { - try { - let quality = this.qualities.get(assetId); - if (!quality) { - quality = this.createEmptyQuality(assetId); - } - - const fullIssue: QualityIssue = { - ...issue, - id: this.generateId(), - detectedAt: new Date() - }; - - quality.issues.push(fullIssue); - this.qualities.set(assetId, quality); - - this.logger.info('Quality issue reported', { - assetId, - issueId: fullIssue.id, - severity: fullIssue.severity - }); - - await this.eventBus.emit('data.quality.issue.reported', { - assetId, - issue: fullIssue, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to report quality issue', { assetId, issue, error }); - throw error; - } - } - - async resolveIssue(assetId: string, issueId: string): Promise { - try { - const quality = this.qualities.get(assetId); - if (!quality) { - throw new Error(`Quality not found for asset ${assetId}`); - } - - const issue = quality.issues.find(i => i.id === issueId); - if (!issue) { - throw new Error(`Issue ${issueId} not found for asset ${assetId}`); - } - - issue.resolved = true; - issue.resolvedAt = new Date(); - this.qualities.set(assetId, quality); - - this.logger.info('Quality issue resolved', { assetId, issueId }); - - await this.eventBus.emit('data.quality.issue.resolved', { - assetId, - issue, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to resolve quality issue', { assetId, issueId, error }); - throw error; - } - } - - async getTrendAnalysis(assetId: string, timeframe: string): Promise { - try { - const quality = this.qualities.get(assetId); - if (!quality) { - throw new Error(`Quality not found for asset ${assetId}`); - } - - // Filter trend data by timeframe - const filteredTrend = this.filterTrendByTimeframe(quality.trend, timeframe); - - // Calculate trend direction and change rate - const trendAnalysis = this.analyzeTrend(filteredTrend.dataPoints); - - return { - ...filteredTrend, - trend: trendAnalysis.direction, - changeRate: trendAnalysis.changeRate - }; - } catch (error) { - this.logger.error('Failed to get trend analysis', { assetId, timeframe, error }); - throw error; - } - } - - async getQualityMetrics(filters?: Record): Promise { - try { - let qualities = Array.from(this.qualities.values()); - - // Apply filters if provided - if (filters) { - const assets = Array.from(this.assets.values()); - const filteredAssets = assets.filter(asset => { - return Object.entries(filters).every(([key, value]) => { - if (key === 'type') return asset.type === value; - if (key === 'owner') return asset.owner === value; - if (key === 'classification') return asset.classification === value; - return true; - }); - }); - - qualities = qualities.filter(quality => - filteredAssets.some(asset => asset.id === quality.assetId) - ); - } - - // Calculate aggregate metrics - const metrics = { - totalAssets: qualities.length, - averageQualityScore: this.calculateAverageScore(qualities), - qualityDistribution: this.calculateQualityDistribution(qualities), - topIssues: this.getTopQualityIssues(qualities), - trendSummary: this.getTrendSummary(qualities), - ruleCompliance: this.calculateRuleCompliance(qualities) - }; - - this.logger.info('Quality metrics calculated', { - totalAssets: metrics.totalAssets, - averageScore: metrics.averageQualityScore - }); - - return metrics; - } catch (error) { - this.logger.error('Failed to get quality metrics', { filters, error }); - throw error; - } - } - - async generateQualityReport(assetIds: string[]): Promise { - try { - const reportData = { - summary: { - totalAssets: assetIds.length, - assessmentDate: new Date(), - averageScore: 0, - criticalIssues: 0, - highIssues: 0 - }, - assetDetails: [] as any[], - recommendations: [] as string[] - }; - - let totalScore = 0; - let criticalCount = 0; - let highCount = 0; - - for (const assetId of assetIds) { - const quality = this.qualities.get(assetId); - const asset = this.assets.get(assetId); - - if (quality && asset) { - totalScore += quality.overallScore; - - const criticalIssuesCount = quality.issues.filter(i => - i.severity === 'critical' && !i.resolved - ).length; - const highIssuesCount = quality.issues.filter(i => - i.severity === 'high' && !i.resolved - ).length; - - criticalCount += criticalIssuesCount; - highCount += highIssuesCount; - - reportData.assetDetails.push({ - assetId, - assetName: asset.name, - qualityScore: quality.overallScore, - dimensions: quality.dimensions, - openIssues: quality.issues.filter(i => !i.resolved).length, - criticalIssues: criticalIssuesCount, - highIssues: highIssuesCount, - lastAssessment: quality.lastAssessment - }); - } - } - - reportData.summary.averageScore = Math.round(totalScore / assetIds.length); - reportData.summary.criticalIssues = criticalCount; - reportData.summary.highIssues = highCount; - - // Generate recommendations - reportData.recommendations = this.generateQualityRecommendations(reportData); - - this.logger.info('Quality report generated', { - assetCount: assetIds.length, - averageScore: reportData.summary.averageScore, - criticalIssues: criticalCount - }); - - return reportData; - } catch (error) { - this.logger.error('Failed to generate quality report', { assetIds, error }); - throw error; - } - } - - // Private helper methods - private createEmptyQuality(assetId: string): DataQuality { - return { - id: this.generateId(), - assetId, - overallScore: 100, - dimensions: [], - rules: [], - issues: [], - trend: { - timeframe: 'week', - dataPoints: [], - trend: 'stable', - changeRate: 0 - }, - lastAssessment: new Date() - }; - } - - private async performQualityAssessment( - asset: DataAsset, - request: QualityAssessmentRequest - ): Promise<{ dimensions: QualityDimension[] }> { - const dimensions: QualityDimension[] = []; - - // Completeness assessment - if (request.checkCompleteness) { - const completeness = await this.assessCompleteness(asset); - dimensions.push(completeness); - } - - // Accuracy assessment - if (request.checkAccuracy) { - const accuracy = await this.assessAccuracy(asset); - dimensions.push(accuracy); - } - - // Consistency assessment - if (request.checkConsistency) { - const consistency = await this.assessConsistency(asset); - dimensions.push(consistency); - } - - // Validity assessment - if (request.checkValidity) { - const validity = await this.assessValidity(asset); - dimensions.push(validity); - } - - // Timeliness assessment - if (request.checkTimeliness) { - const timeliness = await this.assessTimeliness(asset); - dimensions.push(timeliness); - } - - // Uniqueness assessment - if (request.checkUniqueness) { - const uniqueness = await this.assessUniqueness(asset); - dimensions.push(uniqueness); - } - - return { dimensions }; - } - - private async assessCompleteness(asset: DataAsset): Promise { - // Mock implementation - in real scenario, this would analyze actual data - const score = Math.floor(Math.random() * 20) + 80; // 80-100 - - return { - name: 'completeness', - score, - description: 'Measures the degree to which data is complete', - rules: [`No null values in required fields`], - threshold: 95, - lastChecked: new Date() - }; - } - - private async assessAccuracy(asset: DataAsset): Promise { - const score = Math.floor(Math.random() * 15) + 85; // 85-100 - - return { - name: 'accuracy', - score, - description: 'Measures how well data represents real-world values', - rules: [`Values within expected ranges`, `Format validation`], - threshold: 90, - lastChecked: new Date() - }; - } - - private async assessConsistency(asset: DataAsset): Promise { - const score = Math.floor(Math.random() * 25) + 75; // 75-100 - - return { - name: 'consistency', - score, - description: 'Measures uniformity of data across datasets', - rules: [`Consistent data types`, `Standardized formats`], - threshold: 85, - lastChecked: new Date() - }; - } - - private async assessValidity(asset: DataAsset): Promise { - const score = Math.floor(Math.random() * 20) + 80; // 80-100 - - return { - name: 'validity', - score, - description: 'Measures conformity to defined business rules', - rules: [`Business rule compliance`, `Schema validation`], - threshold: 90, - lastChecked: new Date() - }; - } - - private async assessTimeliness(asset: DataAsset): Promise { - const score = Math.floor(Math.random() * 30) + 70; // 70-100 - - return { - name: 'timeliness', - score, - description: 'Measures how up-to-date the data is', - rules: [`Data refreshed within SLA`, `Timestamp validation`], - threshold: 85, - lastChecked: new Date() - }; - } - - private async assessUniqueness(asset: DataAsset): Promise { - const score = Math.floor(Math.random() * 25) + 75; // 75-100 - - return { - name: 'uniqueness', - score, - description: 'Measures absence of duplicate records', - rules: [`No duplicate primary keys`, `Unique constraints enforced`], - threshold: 95, - lastChecked: new Date() - }; - } - - private async executeQualityRule(asset: DataAsset, rule: QualityRule): Promise { - // Mock implementation - in real scenario, this would execute the actual rule - // For demo purposes, randomly pass/fail rules - const passRate = rule.severity === 'critical' ? 0.9 : 0.95; - return Math.random() < passRate; - } - - private calculateOverallScore(dimensions: QualityDimension[]): number { - if (dimensions.length === 0) return 100; - - const totalScore = dimensions.reduce((sum, dim) => sum + dim.score, 0); - return Math.round(totalScore / dimensions.length); - } - - private updateQualityTrend(quality: DataQuality, newScore: number): void { - quality.trend.dataPoints.push({ - timestamp: new Date(), - value: newScore - }); - - // Keep only last 30 data points - if (quality.trend.dataPoints.length > 30) { - quality.trend.dataPoints = quality.trend.dataPoints.slice(-30); - } - - // Update trend analysis - const trendAnalysis = this.analyzeTrend(quality.trend.dataPoints); - quality.trend.trend = trendAnalysis.direction; - quality.trend.changeRate = trendAnalysis.changeRate; - } - - private filterTrendByTimeframe(trend: QualityTrend, timeframe: string): QualityTrend { - const now = new Date(); - let cutoffDate: Date; - - switch (timeframe) { - case 'day': - cutoffDate = new Date(now.getTime() - 24 * 60 * 60 * 1000); - break; - case 'week': - cutoffDate = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); - break; - case 'month': - cutoffDate = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); - break; - default: - cutoffDate = new Date(0); // All time - } - - const filteredDataPoints = trend.dataPoints.filter(dp => dp.timestamp >= cutoffDate); - - return { - ...trend, - timeframe, - dataPoints: filteredDataPoints - }; - } - - private analyzeTrend(dataPoints: { timestamp: Date; value: number }[]): { direction: 'improving' | 'declining' | 'stable'; changeRate: number } { - if (dataPoints.length < 2) { - return { direction: 'stable', changeRate: 0 }; - } - - const values = dataPoints.map(dp => dp.value); - const firstValue = values[0]; - const lastValue = values[values.length - 1]; - const changeRate = ((lastValue - firstValue) / firstValue) * 100; - - let direction: 'improving' | 'declining' | 'stable'; - if (Math.abs(changeRate) < 2) { - direction = 'stable'; - } else if (changeRate > 0) { - direction = 'improving'; - } else { - direction = 'declining'; - } - - return { direction, changeRate: Math.round(changeRate * 100) / 100 }; - } - - private calculateAverageScore(qualities: DataQuality[]): number { - if (qualities.length === 0) return 0; - - const totalScore = qualities.reduce((sum, quality) => sum + quality.overallScore, 0); - return Math.round(totalScore / qualities.length); - } - - private calculateQualityDistribution(qualities: DataQuality[]): Record { - const distribution = { excellent: 0, good: 0, fair: 0, poor: 0 }; - - qualities.forEach(quality => { - if (quality.overallScore >= 90) distribution.excellent++; - else if (quality.overallScore >= 80) distribution.good++; - else if (quality.overallScore >= 70) distribution.fair++; - else distribution.poor++; - }); - - return distribution; - } - - private getTopQualityIssues(qualities: DataQuality[]): Array<{ type: string; count: number }> { - const issueTypes = new Map(); - - qualities.forEach(quality => { - quality.issues.filter(issue => !issue.resolved).forEach(issue => { - issueTypes.set(issue.type, (issueTypes.get(issue.type) || 0) + 1); - }); - }); - - return Array.from(issueTypes.entries()) - .map(([type, count]) => ({ type, count })) - .sort((a, b) => b.count - a.count) - .slice(0, 5); - } - - private getTrendSummary(qualities: DataQuality[]): Record { - const trends = { improving: 0, declining: 0, stable: 0 }; - - qualities.forEach(quality => { - trends[quality.trend.trend]++; - }); - - return trends; - } - - private calculateRuleCompliance(qualities: DataQuality[]): number { - let totalRules = 0; - let passedRules = 0; - - qualities.forEach(quality => { - totalRules += quality.rules.length; - // Mock compliance calculation - passedRules += Math.floor(quality.rules.length * (quality.overallScore / 100)); - }); - - return totalRules > 0 ? Math.round((passedRules / totalRules) * 100) : 100; - } - - private generateQualityRecommendations(reportData: any): string[] { - const recommendations: string[] = []; - - if (reportData.summary.averageScore < 80) { - recommendations.push('Overall data quality is below acceptable threshold. Consider implementing comprehensive data quality monitoring.'); - } - - if (reportData.summary.criticalIssues > 0) { - recommendations.push(`${reportData.summary.criticalIssues} critical quality issues require immediate attention.`); - } - - if (reportData.summary.highIssues > 5) { - recommendations.push('High number of quality issues detected. Review data validation processes.'); - } - - // Asset-specific recommendations - const lowScoreAssets = reportData.assetDetails.filter((asset: any) => asset.qualityScore < 70); - if (lowScoreAssets.length > 0) { - recommendations.push(`${lowScoreAssets.length} assets have quality scores below 70% and need immediate remediation.`); - } - - if (recommendations.length === 0) { - recommendations.push('Data quality is within acceptable ranges. Continue monitoring and maintain current practices.'); - } - - return recommendations; - } - - private generateId(): string { - return `quality_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - // Method to inject assets (typically from DataCatalogService) - setAssets(assets: Map): void { - this.assets = assets; - } -} diff --git a/apps/data-services/data-catalog/src/services/SearchService.ts b/apps/data-services/data-catalog/src/services/SearchService.ts deleted file mode 100644 index 35afcde..0000000 --- a/apps/data-services/data-catalog/src/services/SearchService.ts +++ /dev/null @@ -1,801 +0,0 @@ -import { EventBus } from '@stock-bot/event-bus'; -import { Logger } from '@stock-bot/utils'; -import { - DataAsset, - SearchQuery, - SearchResult, - SearchFilters, - SearchSuggestion, - DataAssetType, - DataClassification -} from '../types/DataCatalog'; - -export interface SearchService { - search(query: SearchQuery): Promise; - suggest(partial: string): Promise; - searchByFacets(facets: Record): Promise; - searchSimilar(assetId: string, limit?: number): Promise; - getPopularSearches(limit?: number): Promise; - getRecentSearches(userId: string, limit?: number): Promise; - indexAsset(asset: DataAsset): Promise; - removeFromIndex(assetId: string): Promise; - reindexAll(): Promise; - getSearchAnalytics(timeframe?: string): Promise; -} - -export class SearchServiceImpl implements SearchService { - private searchIndex: Map = new Map(); - private searchHistory: Array<{ query: string; userId?: string; timestamp: Date; resultCount: number }> = []; - private assets: Map = new Map(); - - // In-memory inverted index for search - private wordToAssets: Map> = new Map(); - private tagToAssets: Map> = new Map(); - private typeToAssets: Map> = new Map(); - private classificationToAssets: Map> = new Map(); - private ownerToAssets: Map> = new Map(); - - constructor( - private eventBus: EventBus, - private logger: Logger - ) {} - - async search(query: SearchQuery): Promise { - try { - const startTime = Date.now(); - let results: DataAsset[] = []; - - if (query.text) { - results = await this.performTextSearch(query.text); - } else { - results = Array.from(this.assets.values()); - } - - // Apply filters - if (query.filters) { - results = this.applyFilters(results, query.filters); - } - - // Sort results - results = this.sortResults(results, query.sortBy, query.sortOrder); - - // Apply pagination - const total = results.length; - const offset = query.offset || 0; - const limit = query.limit || 20; - const paginatedResults = results.slice(offset, offset + limit); - - // Calculate facets - const facets = this.calculateFacets(results); - - const searchTime = Date.now() - startTime; - - const searchResult: SearchResult = { - assets: paginatedResults, - total, - offset, - limit, - searchTime, - facets, - suggestions: await this.generateSearchSuggestions(query.text || '', results) - }; - - // Record search in history - this.recordSearch(query.text || '', query.userId, total); - - this.logger.info('Search completed', { - query: query.text, - resultCount: total, - searchTime - }); - - await this.eventBus.emit('data.catalog.search.performed', { - query, - resultCount: total, - searchTime, - timestamp: new Date() - }); - - return searchResult; - } catch (error) { - this.logger.error('Search failed', { query, error }); - throw error; - } - } - - async suggest(partial: string): Promise { - try { - const suggestions: SearchSuggestion[] = []; - const normalizedPartial = partial.toLowerCase().trim(); - - if (normalizedPartial.length < 2) { - return suggestions; - } - - // Asset name suggestions - for (const asset of this.assets.values()) { - if (asset.name.toLowerCase().includes(normalizedPartial)) { - suggestions.push({ - text: asset.name, - type: 'asset_name', - count: 1, - highlight: this.highlightMatch(asset.name, partial) - }); - } - } - - // Tag suggestions - const tagCounts = new Map(); - for (const asset of this.assets.values()) { - for (const tag of asset.tags) { - if (tag.toLowerCase().includes(normalizedPartial)) { - tagCounts.set(tag, (tagCounts.get(tag) || 0) + 1); - } - } - } - - for (const [tag, count] of tagCounts) { - suggestions.push({ - text: tag, - type: 'tag', - count, - highlight: this.highlightMatch(tag, partial) - }); - } - - // Owner suggestions - const ownerCounts = new Map(); - for (const asset of this.assets.values()) { - if (asset.owner.toLowerCase().includes(normalizedPartial)) { - ownerCounts.set(asset.owner, (ownerCounts.get(asset.owner) || 0) + 1); - } - } - - for (const [owner, count] of ownerCounts) { - suggestions.push({ - text: owner, - type: 'owner', - count, - highlight: this.highlightMatch(owner, partial) - }); - } - - // Popular search suggestions - const popularSearches = this.getPopularSearchTerms().filter(term => - term.toLowerCase().includes(normalizedPartial) - ); - - for (const search of popularSearches.slice(0, 5)) { - suggestions.push({ - text: search, - type: 'popular_search', - count: this.getSearchCount(search), - highlight: this.highlightMatch(search, partial) - }); - } - - // Sort by relevance and count - return suggestions - .sort((a, b) => { - // Prefer exact matches - const aExact = a.text.toLowerCase().startsWith(normalizedPartial) ? 1 : 0; - const bExact = b.text.toLowerCase().startsWith(normalizedPartial) ? 1 : 0; - - if (aExact !== bExact) return bExact - aExact; - - // Then by count - return b.count - a.count; - }) - .slice(0, 10); - } catch (error) { - this.logger.error('Suggestion generation failed', { partial, error }); - throw error; - } - } - - async searchByFacets(facets: Record): Promise { - try { - let results: Set = new Set(); - let isFirstFacet = true; - - for (const [facetType, values] of Object.entries(facets)) { - const facetResults = new Set(); - - for (const value of values) { - let assetIds: Set | undefined; - - switch (facetType) { - case 'type': - assetIds = this.typeToAssets.get(value); - break; - case 'classification': - assetIds = this.classificationToAssets.get(value); - break; - case 'owner': - assetIds = this.ownerToAssets.get(value); - break; - case 'tags': - assetIds = this.tagToAssets.get(value); - break; - } - - if (assetIds) { - for (const assetId of assetIds) { - facetResults.add(assetId); - } - } - } - - if (isFirstFacet) { - results = facetResults; - isFirstFacet = false; - } else { - // Intersection of results - results = new Set([...results].filter(id => facetResults.has(id))); - } - } - - const assets = Array.from(results) - .map(id => this.assets.get(id)) - .filter((asset): asset is DataAsset => asset !== undefined); - - this.logger.info('Facet search completed', { - facets, - resultCount: assets.length - }); - - return assets; - } catch (error) { - this.logger.error('Facet search failed', { facets, error }); - throw error; - } - } - - async searchSimilar(assetId: string, limit: number = 10): Promise { - try { - const targetAsset = this.assets.get(assetId); - if (!targetAsset) { - return []; - } - - const similarities: Array<{ asset: DataAsset; score: number }> = []; - - for (const asset of this.assets.values()) { - if (asset.id === assetId) continue; - - const score = this.calculateSimilarity(targetAsset, asset); - if (score > 0.1) { // Minimum similarity threshold - similarities.push({ asset, score }); - } - } - - // Sort by similarity score and return top results - const results = similarities - .sort((a, b) => b.score - a.score) - .slice(0, limit) - .map(item => item.asset); - - this.logger.info('Similar assets found', { - assetId, - similarCount: results.length - }); - - return results; - } catch (error) { - this.logger.error('Similar asset search failed', { assetId, error }); - throw error; - } - } - - async getPopularSearches(limit: number = 10): Promise { - try { - const searchCounts = new Map(); - - // Count search frequency - for (const search of this.searchHistory) { - if (search.query) { - searchCounts.set(search.query, (searchCounts.get(search.query) || 0) + 1); - } - } - - // Sort by frequency and return top searches - return Array.from(searchCounts.entries()) - .sort((a, b) => b[1] - a[1]) - .slice(0, limit) - .map(([query]) => query); - } catch (error) { - this.logger.error('Failed to get popular searches', { error }); - throw error; - } - } - - async getRecentSearches(userId: string, limit: number = 10): Promise { - try { - return this.searchHistory - .filter(search => search.userId === userId && search.query) - .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()) - .slice(0, limit) - .map(search => search.query); - } catch (error) { - this.logger.error('Failed to get recent searches', { userId, error }); - throw error; - } - } - - async indexAsset(asset: DataAsset): Promise { - try { - // Add to main index - this.searchIndex.set(asset.id, asset); - this.assets.set(asset.id, asset); - - // Update inverted indices - this.updateInvertedIndices(asset); - - this.logger.debug('Asset indexed', { assetId: asset.id, name: asset.name }); - - await this.eventBus.emit('data.catalog.asset.indexed', { - assetId: asset.id, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to index asset', { asset, error }); - throw error; - } - } - - async removeFromIndex(assetId: string): Promise { - try { - const asset = this.searchIndex.get(assetId); - if (!asset) { - return; - } - - // Remove from main index - this.searchIndex.delete(assetId); - this.assets.delete(assetId); - - // Remove from inverted indices - this.removeFromInvertedIndices(asset); - - this.logger.debug('Asset removed from index', { assetId }); - - await this.eventBus.emit('data.catalog.asset.unindexed', { - assetId, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to remove asset from index', { assetId, error }); - throw error; - } - } - - async reindexAll(): Promise { - try { - // Clear all indices - this.searchIndex.clear(); - this.wordToAssets.clear(); - this.tagToAssets.clear(); - this.typeToAssets.clear(); - this.classificationToAssets.clear(); - this.ownerToAssets.clear(); - - // Reindex all assets - for (const asset of this.assets.values()) { - await this.indexAsset(asset); - } - - this.logger.info('Search index rebuilt', { assetCount: this.assets.size }); - - await this.eventBus.emit('data.catalog.index.rebuilt', { - assetCount: this.assets.size, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to rebuild search index', { error }); - throw error; - } - } - - async getSearchAnalytics(timeframe: string = 'week'): Promise { - try { - const now = new Date(); - let cutoffDate: Date; - - switch (timeframe) { - case 'day': - cutoffDate = new Date(now.getTime() - 24 * 60 * 60 * 1000); - break; - case 'week': - cutoffDate = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); - break; - case 'month': - cutoffDate = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); - break; - default: - cutoffDate = new Date(0); - } - - const recentSearches = this.searchHistory.filter(search => search.timestamp >= cutoffDate); - - const analytics = { - totalSearches: recentSearches.length, - uniqueQueries: new Set(recentSearches.map(s => s.query)).size, - averageResults: recentSearches.length > 0 ? - recentSearches.reduce((sum, s) => sum + s.resultCount, 0) / recentSearches.length : 0, - noResultQueries: recentSearches.filter(s => s.resultCount === 0).length, - topQueries: this.getTopQueries(recentSearches, 10), - searchTrend: this.calculateSearchTrend(recentSearches, timeframe), - facetUsage: this.getFacetUsage(recentSearches) - }; - - return analytics; - } catch (error) { - this.logger.error('Failed to get search analytics', { timeframe, error }); - throw error; - } - } - - // Private helper methods - private async performTextSearch(text: string): Promise { - const words = this.tokenize(text); - const assetScores = new Map(); - - for (const word of words) { - const assetIds = this.wordToAssets.get(word) || new Set(); - - for (const assetId of assetIds) { - assetScores.set(assetId, (assetScores.get(assetId) || 0) + 1); - } - } - - // Sort by relevance score - const sortedAssetIds = Array.from(assetScores.entries()) - .sort((a, b) => b[1] - a[1]) - .map(([assetId]) => assetId); - - return sortedAssetIds - .map(id => this.assets.get(id)) - .filter((asset): asset is DataAsset => asset !== undefined); - } - - private applyFilters(assets: DataAsset[], filters: SearchFilters): DataAsset[] { - return assets.filter(asset => { - if (filters.types && filters.types.length > 0) { - if (!filters.types.includes(asset.type)) return false; - } - - if (filters.classifications && filters.classifications.length > 0) { - if (!filters.classifications.includes(asset.classification)) return false; - } - - if (filters.owners && filters.owners.length > 0) { - if (!filters.owners.includes(asset.owner)) return false; - } - - if (filters.tags && filters.tags.length > 0) { - if (!filters.tags.some(tag => asset.tags.includes(tag))) return false; - } - - if (filters.createdAfter) { - if (asset.createdAt < filters.createdAfter) return false; - } - - if (filters.createdBefore) { - if (asset.createdAt > filters.createdBefore) return false; - } - - return true; - }); - } - - private sortResults(assets: DataAsset[], sortBy?: string, sortOrder?: 'asc' | 'desc'): DataAsset[] { - if (!sortBy) { - return assets; // Return as-is (relevance order) - } - - const order = sortOrder === 'desc' ? -1 : 1; - - return assets.sort((a, b) => { - let comparison = 0; - - switch (sortBy) { - case 'name': - comparison = a.name.localeCompare(b.name); - break; - case 'createdAt': - comparison = a.createdAt.getTime() - b.createdAt.getTime(); - break; - case 'updatedAt': - comparison = a.updatedAt.getTime() - b.updatedAt.getTime(); - break; - case 'lastAccessed': - const aAccessed = a.lastAccessed?.getTime() || 0; - const bAccessed = b.lastAccessed?.getTime() || 0; - comparison = aAccessed - bAccessed; - break; - case 'usage': - comparison = a.usage.accessCount - b.usage.accessCount; - break; - default: - comparison = 0; - } - - return comparison * order; - }); - } - - private calculateFacets(assets: DataAsset[]): Record> { - const facets: Record> = { - types: new Map(), - classifications: new Map(), - owners: new Map(), - tags: new Map() - }; - - for (const asset of assets) { - // Type facet - facets.types.set(asset.type, (facets.types.get(asset.type) || 0) + 1); - - // Classification facet - facets.classifications.set(asset.classification, (facets.classifications.get(asset.classification) || 0) + 1); - - // Owner facet - facets.owners.set(asset.owner, (facets.owners.get(asset.owner) || 0) + 1); - - // Tags facet - for (const tag of asset.tags) { - facets.tags.set(tag, (facets.tags.get(tag) || 0) + 1); - } - } - - // Convert to required format - const result: Record> = {}; - - for (const [facetName, facetMap] of Object.entries(facets)) { - result[facetName] = Array.from(facetMap.entries()) - .map(([value, count]) => ({ value, count })) - .sort((a, b) => b.count - a.count); - } - - return result; - } - - private async generateSearchSuggestions(query: string, results: DataAsset[]): Promise { - if (!query || results.length === 0) { - return []; - } - - const suggestions: string[] = []; - - // Extract common tags from results - const tagCounts = new Map(); - for (const asset of results.slice(0, 10)) { // Top 10 results - for (const tag of asset.tags) { - tagCounts.set(tag, (tagCounts.get(tag) || 0) + 1); - } - } - - // Add top tags as suggestions - const topTags = Array.from(tagCounts.entries()) - .sort((a, b) => b[1] - a[1]) - .slice(0, 3) - .map(([tag]) => `${query} ${tag}`); - - suggestions.push(...topTags); - - return suggestions; - } - - private updateInvertedIndices(asset: DataAsset): void { - // Index words from name and description - const words = [ - ...this.tokenize(asset.name), - ...this.tokenize(asset.description) - ]; - - for (const word of words) { - if (!this.wordToAssets.has(word)) { - this.wordToAssets.set(word, new Set()); - } - this.wordToAssets.get(word)!.add(asset.id); - } - - // Index tags - for (const tag of asset.tags) { - if (!this.tagToAssets.has(tag)) { - this.tagToAssets.set(tag, new Set()); - } - this.tagToAssets.get(tag)!.add(asset.id); - } - - // Index type - if (!this.typeToAssets.has(asset.type)) { - this.typeToAssets.set(asset.type, new Set()); - } - this.typeToAssets.get(asset.type)!.add(asset.id); - - // Index classification - if (!this.classificationToAssets.has(asset.classification)) { - this.classificationToAssets.set(asset.classification, new Set()); - } - this.classificationToAssets.get(asset.classification)!.add(asset.id); - - // Index owner - if (!this.ownerToAssets.has(asset.owner)) { - this.ownerToAssets.set(asset.owner, new Set()); - } - this.ownerToAssets.get(asset.owner)!.add(asset.id); - } - - private removeFromInvertedIndices(asset: DataAsset): void { - // Remove from word index - const words = [ - ...this.tokenize(asset.name), - ...this.tokenize(asset.description) - ]; - - for (const word of words) { - const assetSet = this.wordToAssets.get(word); - if (assetSet) { - assetSet.delete(asset.id); - if (assetSet.size === 0) { - this.wordToAssets.delete(word); - } - } - } - - // Remove from other indices - this.removeFromIndex(this.tagToAssets, asset.tags, asset.id); - this.removeFromIndex(this.typeToAssets, [asset.type], asset.id); - this.removeFromIndex(this.classificationToAssets, [asset.classification], asset.id); - this.removeFromIndex(this.ownerToAssets, [asset.owner], asset.id); - } - - private removeFromIndex(index: Map>, values: string[], assetId: string): void { - for (const value of values) { - const assetSet = index.get(value); - if (assetSet) { - assetSet.delete(assetId); - if (assetSet.size === 0) { - index.delete(value); - } - } - } - } - - private tokenize(text: string): string[] { - return text - .toLowerCase() - .replace(/[^\w\s]/g, ' ') - .split(/\s+/) - .filter(word => word.length > 2); - } - - private calculateSimilarity(asset1: DataAsset, asset2: DataAsset): number { - let score = 0; - - // Type similarity - if (asset1.type === asset2.type) score += 0.3; - - // Classification similarity - if (asset1.classification === asset2.classification) score += 0.2; - - // Owner similarity - if (asset1.owner === asset2.owner) score += 0.1; - - // Tag similarity (Jaccard similarity) - const tags1 = new Set(asset1.tags); - const tags2 = new Set(asset2.tags); - const intersection = new Set([...tags1].filter(tag => tags2.has(tag))); - const union = new Set([...tags1, ...tags2]); - - if (union.size > 0) { - score += (intersection.size / union.size) * 0.4; - } - - return score; - } - - private highlightMatch(text: string, query: string): string { - const regex = new RegExp(`(${query})`, 'gi'); - return text.replace(regex, '$1'); - } - - private recordSearch(query: string, userId?: string, resultCount: number = 0): void { - this.searchHistory.push({ - query, - userId, - timestamp: new Date(), - resultCount - }); - - // Keep only last 1000 searches - if (this.searchHistory.length > 1000) { - this.searchHistory = this.searchHistory.slice(-1000); - } - } - - private getPopularSearchTerms(): string[] { - const searchCounts = new Map(); - - for (const search of this.searchHistory) { - if (search.query) { - searchCounts.set(search.query, (searchCounts.get(search.query) || 0) + 1); - } - } - - return Array.from(searchCounts.entries()) - .sort((a, b) => b[1] - a[1]) - .map(([query]) => query); - } - - private getSearchCount(query: string): number { - return this.searchHistory.filter(search => search.query === query).length; - } - - private getTopQueries(searches: any[], limit: number): Array<{ query: string; count: number }> { - const queryCounts = new Map(); - - for (const search of searches) { - if (search.query) { - queryCounts.set(search.query, (queryCounts.get(search.query) || 0) + 1); - } - } - - return Array.from(queryCounts.entries()) - .map(([query, count]) => ({ query, count })) - .sort((a, b) => b.count - a.count) - .slice(0, limit); - } - - private calculateSearchTrend(searches: any[], timeframe: string): any { - // Group searches by day - const dailyCounts = new Map(); - - for (const search of searches) { - const day = search.timestamp.toISOString().split('T')[0]; - dailyCounts.set(day, (dailyCounts.get(day) || 0) + 1); - } - - const dataPoints = Array.from(dailyCounts.entries()) - .map(([date, count]) => ({ date, count })) - .sort((a, b) => a.date.localeCompare(b.date)); - - return { - dataPoints, - trend: this.analyzeTrend(dataPoints.map(p => p.count)) - }; - } - - private analyzeTrend(values: number[]): string { - if (values.length < 2) return 'stable'; - - const firstHalf = values.slice(0, Math.floor(values.length / 2)); - const secondHalf = values.slice(Math.floor(values.length / 2)); - - const firstAvg = firstHalf.reduce((sum, val) => sum + val, 0) / firstHalf.length; - const secondAvg = secondHalf.reduce((sum, val) => sum + val, 0) / secondHalf.length; - - const changePercent = ((secondAvg - firstAvg) / firstAvg) * 100; - - if (Math.abs(changePercent) < 10) return 'stable'; - return changePercent > 0 ? 'increasing' : 'decreasing'; - } - - private getFacetUsage(searches: any[]): Record { - // Mock facet usage tracking - return { - types: Math.floor(searches.length * 0.3), - classifications: Math.floor(searches.length * 0.2), - owners: Math.floor(searches.length * 0.1), - tags: Math.floor(searches.length * 0.4) - }; - } - - // Method to inject assets (typically from DataCatalogService) - setAssets(assets: Map): void { - this.assets = assets; - // Reindex all assets when assets are updated - this.reindexAll(); - } -} diff --git a/apps/data-services/data-catalog/src/types/DataCatalog.ts b/apps/data-services/data-catalog/src/types/DataCatalog.ts deleted file mode 100644 index aac30fc..0000000 --- a/apps/data-services/data-catalog/src/types/DataCatalog.ts +++ /dev/null @@ -1,524 +0,0 @@ -// Data Asset Types -export interface DataAsset { - id: string; - name: string; - type: DataAssetType; - description: string; - owner: string; - steward?: string; - tags: string[]; - classification: DataClassification; - schema?: DataSchema; - location: DataLocation; - metadata: DataAssetMetadata; - lineage: DataLineage; - quality: DataQuality; - usage: DataUsage; - governance: DataGovernance; - createdAt: Date; - updatedAt: Date; - lastAccessed?: Date; -} - -export enum DataAssetType { - TABLE = 'table', - VIEW = 'view', - DATASET = 'dataset', - API = 'api', - FILE = 'file', - STREAM = 'stream', - MODEL = 'model', - FEATURE_GROUP = 'feature_group', - PIPELINE = 'pipeline', - REPORT = 'report' -} - -export enum DataClassification { - PUBLIC = 'public', - INTERNAL = 'internal', - CONFIDENTIAL = 'confidential', - RESTRICTED = 'restricted', - PII = 'pii', - FINANCIAL = 'financial' -} - -export interface DataSchema { - version: string; - fields: DataField[]; - primaryKeys?: string[]; - foreignKeys?: ForeignKey[]; - indexes?: Index[]; -} - -export interface DataField { - name: string; - type: string; - nullable: boolean; - description?: string; - constraints?: FieldConstraint[]; - tags?: string[]; - classification?: DataClassification; -} - -export interface ForeignKey { - fields: string[]; - referencedAsset: string; - referencedFields: string[]; -} - -export interface Index { - name: string; - fields: string[]; - unique: boolean; - type: 'btree' | 'hash' | 'gin' | 'gist'; -} - -export interface FieldConstraint { - type: 'not_null' | 'unique' | 'check' | 'range' | 'pattern'; - value?: any; - description?: string; -} - -export interface DataLocation { - type: 'database' | 'file_system' | 'cloud_storage' | 'api' | 'stream'; - connection: string; - path: string; - format?: string; - compression?: string; - partitioning?: PartitionInfo; -} - -export interface PartitionInfo { - fields: string[]; - strategy: 'range' | 'hash' | 'list'; - count?: number; -} - -export interface DataAssetMetadata { - size?: number; - rowCount?: number; - columnCount?: number; - fileFormat?: string; - encoding?: string; - delimiter?: string; - compression?: string; - checksums?: Record; - customProperties?: Record; -} - -// Data Lineage Types -export interface DataLineage { - id: string; - assetId: string; - upstreamAssets: LineageEdge[]; - downstreamAssets: LineageEdge[]; - transformations: DataTransformation[]; - impact: ImpactAnalysis; - createdAt: Date; - updatedAt: Date; -} - -export interface LineageEdge { - sourceAssetId: string; - targetAssetId: string; - relationship: LineageRelationship; - transformations: string[]; - confidence: number; - metadata?: Record; -} - -export enum LineageRelationship { - DERIVED_FROM = 'derived_from', - AGGREGATED_FROM = 'aggregated_from', - JOINED_WITH = 'joined_with', - FILTERED_FROM = 'filtered_from', - TRANSFORMED_FROM = 'transformed_from', - COPIED_FROM = 'copied_from', - ENRICHED_WITH = 'enriched_with' -} - -export interface DataTransformation { - id: string; - name: string; - type: TransformationType; - description?: string; - code?: string; - inputFields: string[]; - outputFields: string[]; - logic: string; - parameters?: Record; -} - -export enum TransformationType { - FILTER = 'filter', - AGGREGATE = 'aggregate', - JOIN = 'join', - UNION = 'union', - PIVOT = 'pivot', - UNPIVOT = 'unpivot', - SORT = 'sort', - DEDUPLICATE = 'deduplicate', - CALCULATE = 'calculate', - CAST = 'cast', - RENAME = 'rename' -} - -export interface ImpactAnalysis { - downstreamAssets: string[]; - affectedUsers: string[]; - estimatedImpact: 'low' | 'medium' | 'high' | 'critical'; - impactDescription: string; - recommendations: string[]; -} - -// Data Quality Types -export interface DataQuality { - id: string; - assetId: string; - overallScore: number; - dimensions: QualityDimension[]; - rules: QualityRule[]; - issues: QualityIssue[]; - trend: QualityTrend; - lastAssessment: Date; - nextAssessment?: Date; -} - -export interface QualityDimension { - name: QualityDimensionType; - score: number; - weight: number; - description: string; - metrics: QualityMetric[]; -} - -export enum QualityDimensionType { - COMPLETENESS = 'completeness', - ACCURACY = 'accuracy', - CONSISTENCY = 'consistency', - VALIDITY = 'validity', - UNIQUENESS = 'uniqueness', - TIMELINESS = 'timeliness', - INTEGRITY = 'integrity' -} - -export interface QualityRule { - id: string; - name: string; - description: string; - dimension: QualityDimensionType; - type: QualityRuleType; - field?: string; - condition: string; - threshold: number; - severity: 'low' | 'medium' | 'high' | 'critical'; - enabled: boolean; -} - -export enum QualityRuleType { - NULL_CHECK = 'null_check', - RANGE_CHECK = 'range_check', - PATTERN_CHECK = 'pattern_check', - REFERENCE_CHECK = 'reference_check', - DUPLICATE_CHECK = 'duplicate_check', - FRESHNESS_CHECK = 'freshness_check', - CUSTOM = 'custom' -} - -export interface QualityMetric { - name: string; - value: number; - unit?: string; - threshold?: number; - status: 'pass' | 'warn' | 'fail'; -} - -export interface QualityIssue { - id: string; - ruleId: string; - severity: 'low' | 'medium' | 'high' | 'critical'; - description: string; - field?: string; - affectedRows?: number; - detectedAt: Date; - status: 'open' | 'acknowledged' | 'resolved' | 'false_positive'; - assignee?: string; - resolution?: string; - resolvedAt?: Date; -} - -export interface QualityTrend { - timeframe: 'day' | 'week' | 'month'; - dataPoints: QualityDataPoint[]; - trend: 'improving' | 'stable' | 'degrading'; - changeRate: number; -} - -export interface QualityDataPoint { - timestamp: Date; - score: number; - dimensionScores: Record; -} - -// Data Usage Types -export interface DataUsage { - id: string; - assetId: string; - accessCount: number; - uniqueUsers: number; - lastAccessed: Date; - topUsers: UserUsage[]; - accessPatterns: AccessPattern[]; - popularQueries: PopularQuery[]; - usageTrend: UsageTrend; -} - -export interface UserUsage { - userId: string; - userName: string; - accessCount: number; - lastAccessed: Date; - accessType: 'read' | 'write' | 'query' | 'download'; -} - -export interface AccessPattern { - timeOfDay: number; // Hour 0-23 - dayOfWeek: number; // 0-6 - frequency: number; - accessType: 'read' | 'write' | 'query' | 'download'; -} - -export interface PopularQuery { - query: string; - count: number; - avgExecutionTime: number; - lastExecuted: Date; - users: string[]; -} - -export interface UsageTrend { - timeframe: 'day' | 'week' | 'month'; - dataPoints: UsageDataPoint[]; - trend: 'increasing' | 'stable' | 'decreasing'; - changeRate: number; -} - -export interface UsageDataPoint { - timestamp: Date; - accessCount: number; - uniqueUsers: number; - avgResponseTime?: number; -} - -// Data Governance Types -export interface DataGovernance { - id: string; - assetId: string; - policies: GovernancePolicy[]; - compliance: ComplianceStatus[]; - retention: RetentionPolicy; - access: AccessPolicy; - privacy: PrivacySettings; - audit: AuditTrail[]; -} - -export interface GovernancePolicy { - id: string; - name: string; - type: PolicyType; - description: string; - rules: PolicyRule[]; - enforcement: 'advisory' | 'preventive' | 'detective'; - status: 'active' | 'inactive' | 'draft'; -} - -export enum PolicyType { - ACCESS_CONTROL = 'access_control', - DATA_RETENTION = 'data_retention', - DATA_PRIVACY = 'data_privacy', - DATA_QUALITY = 'data_quality', - USAGE_MONITORING = 'usage_monitoring', - COMPLIANCE = 'compliance' -} - -export interface PolicyRule { - id: string; - condition: string; - action: string; - parameters?: Record; -} - -export interface ComplianceStatus { - regulation: 'GDPR' | 'CCPA' | 'SOX' | 'HIPAA' | 'PCI_DSS' | 'CUSTOM'; - status: 'compliant' | 'non_compliant' | 'unknown'; - lastAssessment: Date; - issues: ComplianceIssue[]; -} - -export interface ComplianceIssue { - id: string; - description: string; - severity: 'low' | 'medium' | 'high' | 'critical'; - requirement: string; - remediation: string; - dueDate?: Date; -} - -export interface RetentionPolicy { - retentionPeriod: number; // in days - archiveAfter?: number; // in days - deleteAfter?: number; // in days - retentionReason: string; - legalHold: boolean; -} - -export interface AccessPolicy { - defaultAccess: 'none' | 'read' | 'write' | 'admin'; - roles: RolePermission[]; - users: UserPermission[]; - conditions?: AccessCondition[]; -} - -export interface RolePermission { - role: string; - permissions: Permission[]; - conditions?: AccessCondition[]; -} - -export interface UserPermission { - userId: string; - permissions: Permission[]; - conditions?: AccessCondition[]; - expiresAt?: Date; -} - -export enum Permission { - READ = 'read', - WRITE = 'write', - DELETE = 'delete', - ADMIN = 'admin', - QUERY = 'query', - EXPORT = 'export' -} - -export interface AccessCondition { - type: 'time_based' | 'location_based' | 'purpose_based' | 'data_sensitivity'; - condition: string; - value: any; -} - -export interface PrivacySettings { - containsPII: boolean; - sensitiveFields: string[]; - anonymizationRules: AnonymizationRule[]; - consentRequired: boolean; - dataSubjectRights: DataSubjectRight[]; -} - -export interface AnonymizationRule { - field: string; - method: 'mask' | 'hash' | 'encrypt' | 'tokenize' | 'generalize' | 'suppress'; - parameters?: Record; -} - -export interface DataSubjectRight { - type: 'access' | 'rectification' | 'erasure' | 'portability' | 'restriction'; - enabled: boolean; - automatedResponse: boolean; -} - -export interface AuditTrail { - id: string; - timestamp: Date; - userId: string; - action: string; - resource: string; - details: Record; - outcome: 'success' | 'failure'; - ipAddress?: string; - userAgent?: string; -} - -// Search and Discovery Types -export interface SearchRequest { - query: string; - filters?: SearchFilter[]; - facets?: string[]; - sortBy?: string; - sortOrder?: 'asc' | 'desc'; - limit?: number; - offset?: number; -} - -export interface SearchFilter { - field: string; - operator: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte' | 'in' | 'contains' | 'startswith' | 'endswith'; - value: any; -} - -export interface SearchResponse { - total: number; - assets: DataAsset[]; - facets: SearchFacet[]; - suggestions: string[]; -} - -export interface SearchFacet { - field: string; - values: FacetValue[]; -} - -export interface FacetValue { - value: string; - count: number; -} - -// API Request/Response Types -export interface CreateDataAssetRequest { - name: string; - type: DataAssetType; - description: string; - owner: string; - steward?: string; - tags?: string[]; - classification: DataClassification; - schema?: DataSchema; - location: DataLocation; - metadata?: Partial; - governance?: Partial; -} - -export interface UpdateDataAssetRequest { - name?: string; - description?: string; - owner?: string; - steward?: string; - tags?: string[]; - classification?: DataClassification; - schema?: DataSchema; - metadata?: Partial; -} - -export interface LineageRequest { - assetId: string; - direction: 'upstream' | 'downstream' | 'both'; - depth?: number; - includeTransformations?: boolean; -} - -export interface QualityAssessmentRequest { - assetId: string; - rules?: string[]; - immediate?: boolean; -} - -export interface CreateQualityRuleRequest { - name: string; - description: string; - dimension: QualityDimensionType; - type: QualityRuleType; - field?: string; - condition: string; - threshold: number; - severity: 'low' | 'medium' | 'high' | 'critical'; -} diff --git a/apps/data-services/data-catalog/tsconfig.json b/apps/data-services/data-catalog/tsconfig.json deleted file mode 100644 index e1af3ae..0000000 --- a/apps/data-services/data-catalog/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "baseUrl": "./src", - "paths": { - "@/*": ["*"] - } - }, - "include": [ - "src/**/*" - ], - "exclude": [ - "node_modules", - "dist", - "**/*.test.ts", - "**/*.spec.ts" - ] -} diff --git a/apps/data-services/data-processor/package.json b/apps/data-services/data-processor/package.json deleted file mode 100644 index 412d408..0000000 --- a/apps/data-services/data-processor/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "data-processor", - "version": "1.0.0", - "description": "Data processing and pipeline orchestration service", - "main": "src/index.ts", "scripts": { - "dev": "bun run --watch src/index.ts", - "start": "bun run src/index.ts", - "build": "bun build src/index.ts --outdir=dist --target=bun", - "test": "bun test", - "lint": "eslint src/**/*.ts", - "type-check": "tsc --noEmit" - },"dependencies": { - "@stock-bot/types": "*", - "@stock-bot/event-bus": "*", - "@stock-bot/utils": "*", - "@stock-bot/logger": "*", - "@stock-bot/api-client": "*", - "hono": "^4.6.3", - "ioredis": "^5.4.1", - "cron": "^3.1.6", - "bull": "^4.12.2", - "axios": "^1.6.2", - "node-fetch": "^3.3.2", - "csv-parser": "^3.0.0", - "joi": "^17.11.0" - }, - "devDependencies": { - "bun-types": "^1.2.15", - "@types/node": "^20.10.5", - "@types/bull": "^4.10.0", - "typescript": "^5.3.3", - "eslint": "^8.56.0" - } -} diff --git a/apps/data-services/data-processor/src/controllers/HealthController.ts b/apps/data-services/data-processor/src/controllers/HealthController.ts deleted file mode 100644 index 123943e..0000000 --- a/apps/data-services/data-processor/src/controllers/HealthController.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { Context } from 'hono'; -import { getLogger } from '@stock-bot/logger'; - -const logger = getLogger('HealthController'); - -export class HealthController { - async getHealth(c: Context): Promise { - try { - const health = { - status: 'healthy', - timestamp: new Date().toISOString(), - service: 'data-processor', - version: process.env.npm_package_version || '1.0.0', - uptime: process.uptime(), - environment: process.env.NODE_ENV || 'development', - dependencies: { - redis: await this.checkRedisHealth(), - eventBus: await this.checkEventBusHealth(), - } - }; - - return c.json(health); - } catch (error) { - logger.error('Health check failed:', error); - - return c.json({ - status: 'unhealthy', - timestamp: new Date().toISOString(), - service: 'data-processor', - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - } - - async getDetailedHealth(c: Context): Promise { - try { - const health = { - status: 'healthy', - timestamp: new Date().toISOString(), - service: 'data-processor', - version: process.env.npm_package_version || '1.0.0', - uptime: process.uptime(), - environment: process.env.NODE_ENV || 'development', - system: { - platform: process.platform, - architecture: process.arch, - nodeVersion: process.version, - memory: process.memoryUsage(), - pid: process.pid - }, - dependencies: { - redis: await this.checkRedisHealth(), - eventBus: await this.checkEventBusHealth(), - }, - metrics: { - activePipelines: 0, // Will be populated by orchestrator - runningJobs: 0, // Will be populated by orchestrator - totalProcessedRecords: 0 // Will be populated by orchestrator - } - }; - - return c.json(health); - } catch (error) { - logger.error('Detailed health check failed:', error); - - return c.json({ - status: 'unhealthy', - timestamp: new Date().toISOString(), - service: 'data-processor', - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - } - - private async checkRedisHealth(): Promise<{ status: string; latency?: number; error?: string }> { - try { - const startTime = Date.now(); - // In a real implementation, ping Redis here - const latency = Date.now() - startTime; - - return { - status: 'healthy', - latency - }; - } catch (error) { - return { - status: 'unhealthy', - error: error instanceof Error ? error.message : 'Redis connection failed' - }; - } - } - - private async checkEventBusHealth(): Promise<{ status: string; error?: string }> { - try { - // In a real implementation, check event bus connection here - return { - status: 'healthy' - }; - } catch (error) { - return { - status: 'unhealthy', - error: error instanceof Error ? error.message : 'Event bus connection failed' - }; - } - } -} diff --git a/apps/data-services/data-processor/src/controllers/JobController.ts b/apps/data-services/data-processor/src/controllers/JobController.ts deleted file mode 100644 index 3a365e9..0000000 --- a/apps/data-services/data-processor/src/controllers/JobController.ts +++ /dev/null @@ -1,299 +0,0 @@ -import { Context } from 'hono'; -import { getLogger } from '@stock-bot/logger'; - -const logger = getLogger('JobController'); -import { DataPipelineOrchestrator } from '../core/DataPipelineOrchestrator'; -import { JobStatus } from '../types/DataPipeline'; - -export class JobController { - constructor(private orchestrator: DataPipelineOrchestrator) {} - - async listJobs(c: Context): Promise { - try { - const pipelineId = c.req.query('pipelineId'); - const status = c.req.query('status') as JobStatus; - const limit = parseInt(c.req.query('limit') || '50'); - const offset = parseInt(c.req.query('offset') || '0'); - - let jobs = this.orchestrator.listJobs(pipelineId); - - // Filter by status if provided - if (status) { - jobs = jobs.filter(job => job.status === status); - } - - // Sort by creation time (newest first) - jobs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()); - - // Apply pagination - const totalJobs = jobs.length; - const paginatedJobs = jobs.slice(offset, offset + limit); - - return c.json({ - success: true, - data: paginatedJobs, - pagination: { - total: totalJobs, - limit, - offset, - hasMore: offset + limit < totalJobs - } - }); - } catch (error) { - logger.error('Failed to list jobs:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to list jobs' - }, 500); - } - } - - async getJob(c: Context): Promise { - try { - const jobId = c.req.param('id'); - const job = this.orchestrator.getJob(jobId); - - if (!job) { - return c.json({ - success: false, - error: 'Job not found' - }, 404); - } - - return c.json({ - success: true, - data: job - }); - } catch (error) { - logger.error('Failed to get job:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to get job' - }, 500); - } - } - - async cancelJob(c: Context): Promise { - try { - const jobId = c.req.param('id'); - const job = this.orchestrator.getJob(jobId); - - if (!job) { - return c.json({ - success: false, - error: 'Job not found' - }, 404); - } - - if (job.status !== JobStatus.RUNNING && job.status !== JobStatus.PENDING) { - return c.json({ - success: false, - error: 'Job cannot be cancelled in current status' - }, 400); - } - - // Update job status to cancelled - job.status = JobStatus.CANCELLED; - job.completedAt = new Date(); - job.error = 'Job cancelled by user'; - - logger.info(`Cancelled job: ${jobId}`); - - return c.json({ - success: true, - message: 'Job cancelled successfully', - data: job - }); - } catch (error) { - logger.error('Failed to cancel job:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to cancel job' - }, 500); - } - } - - async retryJob(c: Context): Promise { - try { - const jobId = c.req.param('id'); - const job = this.orchestrator.getJob(jobId); - - if (!job) { - return c.json({ - success: false, - error: 'Job not found' - }, 404); - } - - if (job.status !== JobStatus.FAILED) { - return c.json({ - success: false, - error: 'Only failed jobs can be retried' - }, 400); - } - - // Create a new job with the same parameters - const newJob = await this.orchestrator.runPipeline(job.pipelineId, job.parameters); - - logger.info(`Retried job: ${jobId} as new job: ${newJob.id}`); - - return c.json({ - success: true, - message: 'Job retried successfully', - data: { - originalJob: job, - newJob: newJob - } - }); - } catch (error) { - logger.error('Failed to retry job:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to retry job' - }, 500); - } - } - - async getJobLogs(c: Context): Promise { - try { - const jobId = c.req.param('id'); - const job = this.orchestrator.getJob(jobId); - - if (!job) { - return c.json({ - success: false, - error: 'Job not found' - }, 404); - } - - // In a real implementation, fetch logs from a log store - const logs = [ - { - timestamp: job.createdAt, - level: 'info', - message: `Job ${jobId} created` - }, - ...(job.startedAt ? [{ - timestamp: job.startedAt, - level: 'info', - message: `Job ${jobId} started` - }] : []), - ...(job.completedAt ? [{ - timestamp: job.completedAt, - level: job.status === JobStatus.COMPLETED ? 'info' : 'error', - message: job.status === JobStatus.COMPLETED ? - `Job ${jobId} completed successfully` : - `Job ${jobId} failed: ${job.error}` - }] : []) - ]; - - return c.json({ - success: true, - data: { - jobId, - logs, - totalLogs: logs.length - } - }); - } catch (error) { - logger.error('Failed to get job logs:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to get job logs' - }, 500); - } - } - - async getJobMetrics(c: Context): Promise { - try { - const jobId = c.req.param('id'); - const job = this.orchestrator.getJob(jobId); - - if (!job) { - return c.json({ - success: false, - error: 'Job not found' - }, 404); - } - - const metrics = { - ...job.metrics, - duration: job.completedAt && job.startedAt ? - job.completedAt.getTime() - job.startedAt.getTime() : null, - successRate: job.metrics.recordsProcessed > 0 ? - (job.metrics.recordsSuccessful / job.metrics.recordsProcessed) * 100 : 0, - errorRate: job.metrics.recordsProcessed > 0 ? - (job.metrics.recordsFailed / job.metrics.recordsProcessed) * 100 : 0, - status: job.status, - startedAt: job.startedAt, - completedAt: job.completedAt - }; - - return c.json({ - success: true, - data: metrics - }); - } catch (error) { - logger.error('Failed to get job metrics:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to get job metrics' - }, 500); - } - } - - async getJobStats(c: Context): Promise { - try { - const jobs = this.orchestrator.listJobs(); - - const stats = { - total: jobs.length, - byStatus: { - pending: jobs.filter(j => j.status === JobStatus.PENDING).length, - running: jobs.filter(j => j.status === JobStatus.RUNNING).length, - completed: jobs.filter(j => j.status === JobStatus.COMPLETED).length, - failed: jobs.filter(j => j.status === JobStatus.FAILED).length, - cancelled: jobs.filter(j => j.status === JobStatus.CANCELLED).length, - }, - metrics: { - totalRecordsProcessed: jobs.reduce((sum, j) => sum + j.metrics.recordsProcessed, 0), - totalRecordsSuccessful: jobs.reduce((sum, j) => sum + j.metrics.recordsSuccessful, 0), - totalRecordsFailed: jobs.reduce((sum, j) => sum + j.metrics.recordsFailed, 0), - averageProcessingTime: jobs.length > 0 ? - jobs.reduce((sum, j) => sum + j.metrics.processingTimeMs, 0) / jobs.length : 0, - successRate: jobs.length > 0 ? - (jobs.filter(j => j.status === JobStatus.COMPLETED).length / jobs.length) * 100 : 0 - }, - recentJobs: jobs - .sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()) - .slice(0, 10) - .map(job => ({ - id: job.id, - pipelineId: job.pipelineId, - status: job.status, - createdAt: job.createdAt, - processingTime: job.metrics.processingTimeMs, - recordsProcessed: job.metrics.recordsProcessed - })) - }; - - return c.json({ - success: true, - data: stats - }); - } catch (error) { - logger.error('Failed to get job stats:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to get job stats' - }, 500); - } - } -} diff --git a/apps/data-services/data-processor/src/controllers/PipelineController.ts b/apps/data-services/data-processor/src/controllers/PipelineController.ts deleted file mode 100644 index fce48f4..0000000 --- a/apps/data-services/data-processor/src/controllers/PipelineController.ts +++ /dev/null @@ -1,348 +0,0 @@ -import { Context } from 'hono'; -import { getLogger } from '@stock-bot/logger'; -import { DataPipelineOrchestrator } from '../core/DataPipelineOrchestrator'; -import { DataPipeline, PipelineStatus } from '../types/DataPipeline'; - -const logger = getLogger('pipeline-controller'); - -export class PipelineController { - constructor(private orchestrator: DataPipelineOrchestrator) {} - - async listPipelines(c: Context): Promise { - try { - const pipelines = this.orchestrator.listPipelines(); - - return c.json({ - success: true, - data: pipelines, - total: pipelines.length - }); - } catch (error) { - logger.error('Failed to list pipelines:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to list pipelines' - }, 500); - } - } - - async createPipeline(c: Context): Promise { - try { - const pipelineData = await c.req.json(); - - // Validate required fields - if (!pipelineData.name) { - return c.json({ - success: false, - error: 'Pipeline name is required' - }, 400); - } - - const pipeline = await this.orchestrator.createPipeline(pipelineData); - - logger.info(`Created pipeline: ${pipeline.name} (${pipeline.id})`); - - return c.json({ - success: true, - data: pipeline - }, 201); - } catch (error) { - logger.error('Failed to create pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to create pipeline' - }, 500); - } - } - - async getPipeline(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - const pipeline = this.orchestrator.getPipeline(pipelineId); - - if (!pipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - return c.json({ - success: true, - data: pipeline - }); - } catch (error) { - logger.error('Failed to get pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to get pipeline' - }, 500); - } - } - - async updatePipeline(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - const updateData = await c.req.json(); - - const existingPipeline = this.orchestrator.getPipeline(pipelineId); - if (!existingPipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - // Update pipeline (in a real implementation, this would use a proper update method) - const updatedPipeline: DataPipeline = { - ...existingPipeline, - ...updateData, - id: pipelineId, // Ensure ID doesn't change - updatedAt: new Date() - }; - - // In a real implementation, save to persistent storage - logger.info(`Updated pipeline: ${updatedPipeline.name} (${pipelineId})`); - - return c.json({ - success: true, - data: updatedPipeline - }); - } catch (error) { - logger.error('Failed to update pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to update pipeline' - }, 500); - } - } - - async deletePipeline(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - - const pipeline = this.orchestrator.getPipeline(pipelineId); - if (!pipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - // Check if pipeline is running - const runningJobs = this.orchestrator.listJobs(pipelineId); - if (runningJobs.length > 0) { - return c.json({ - success: false, - error: 'Cannot delete pipeline with running jobs' - }, 400); - } - - // In a real implementation, delete from persistent storage - logger.info(`Deleted pipeline: ${pipeline.name} (${pipelineId})`); - - return c.json({ - success: true, - message: 'Pipeline deleted successfully' - }); - } catch (error) { - logger.error('Failed to delete pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to delete pipeline' - }, 500); - } - } - - async runPipeline(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - const parameters = await c.req.json().catch(() => ({})); - - const pipeline = this.orchestrator.getPipeline(pipelineId); - if (!pipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - if (pipeline.status !== PipelineStatus.ACTIVE) { - return c.json({ - success: false, - error: 'Pipeline is not active' - }, 400); - } - - const job = await this.orchestrator.runPipeline(pipelineId, parameters); - - logger.info(`Started pipeline job: ${job.id} for pipeline: ${pipelineId}`); - - return c.json({ - success: true, - data: job - }, 202); - } catch (error) { - logger.error('Failed to run pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to run pipeline' - }, 500); - } - } - - async schedulePipeline(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - const { cronExpression } = await c.req.json(); - - if (!cronExpression) { - return c.json({ - success: false, - error: 'Cron expression is required' - }, 400); - } - - const pipeline = this.orchestrator.getPipeline(pipelineId); - if (!pipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - await this.orchestrator.schedulePipeline(pipelineId, cronExpression); - - logger.info(`Scheduled pipeline: ${pipelineId} with cron: ${cronExpression}`); - - return c.json({ - success: true, - message: 'Pipeline scheduled successfully', - data: { - pipelineId, - cronExpression - } - }); - } catch (error) { - logger.error('Failed to schedule pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to schedule pipeline' - }, 500); - } - } - - async pausePipeline(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - - const pipeline = this.orchestrator.getPipeline(pipelineId); - if (!pipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - // Update pipeline status to paused - pipeline.status = PipelineStatus.PAUSED; - pipeline.updatedAt = new Date(); - - logger.info(`Paused pipeline: ${pipelineId}`); - - return c.json({ - success: true, - message: 'Pipeline paused successfully', - data: pipeline - }); - } catch (error) { - logger.error('Failed to pause pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to pause pipeline' - }, 500); - } - } - - async resumePipeline(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - - const pipeline = this.orchestrator.getPipeline(pipelineId); - if (!pipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - // Update pipeline status to active - pipeline.status = PipelineStatus.ACTIVE; - pipeline.updatedAt = new Date(); - - logger.info(`Resumed pipeline: ${pipelineId}`); - - return c.json({ - success: true, - message: 'Pipeline resumed successfully', - data: pipeline - }); - } catch (error) { - logger.error('Failed to resume pipeline:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to resume pipeline' - }, 500); - } - } - - async getPipelineMetrics(c: Context): Promise { - try { - const pipelineId = c.req.param('id'); - - const pipeline = this.orchestrator.getPipeline(pipelineId); - if (!pipeline) { - return c.json({ - success: false, - error: 'Pipeline not found' - }, 404); - } - - const jobs = this.orchestrator.listJobs(pipelineId); - - const metrics = { - totalJobs: jobs.length, - completedJobs: jobs.filter(j => j.status === 'completed').length, - failedJobs: jobs.filter(j => j.status === 'failed').length, - runningJobs: jobs.filter(j => j.status === 'running').length, - totalRecordsProcessed: jobs.reduce((sum, j) => sum + j.metrics.recordsProcessed, 0), - totalProcessingTime: jobs.reduce((sum, j) => sum + j.metrics.processingTimeMs, 0), - averageProcessingTime: jobs.length > 0 ? - jobs.reduce((sum, j) => sum + j.metrics.processingTimeMs, 0) / jobs.length : 0, - successRate: jobs.length > 0 ? - (jobs.filter(j => j.status === 'completed').length / jobs.length) * 100 : 0 - }; - - return c.json({ - success: true, - data: metrics - }); - } catch (error) { - logger.error('Failed to get pipeline metrics:', error); - - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Failed to get pipeline metrics' - }, 500); - } - } -} diff --git a/apps/data-services/data-processor/src/core/DataPipelineOrchestrator.ts b/apps/data-services/data-processor/src/core/DataPipelineOrchestrator.ts deleted file mode 100644 index 03921c7..0000000 --- a/apps/data-services/data-processor/src/core/DataPipelineOrchestrator.ts +++ /dev/null @@ -1,299 +0,0 @@ -import { EventBus, EventBusConfig } from '@stock-bot/event-bus'; -import { DataPipelineEvent, DataJobEvent } from '@stock-bot/types'; -import { getLogger } from '@stock-bot/logger'; -import { DataPipeline, PipelineStatus, PipelineJob, JobStatus } from '../types/DataPipeline'; -import { DataIngestionService } from '../services/DataIngestionService'; -import { DataTransformationService } from '../services/DataTransformationService'; -import { DataValidationService } from '../services/DataValidationService'; -import { DataQualityService } from '../services/DataQualityService'; -import { PipelineScheduler } from './PipelineScheduler'; -import { JobQueue } from './JobQueue'; - -const logger = getLogger('data-pipeline-orchestrator'); - -export class DataPipelineOrchestrator { - private eventBus: EventBus; - private scheduler: PipelineScheduler; - private jobQueue: JobQueue; - private pipelines: Map = new Map(); - private runningJobs: Map = new Map(); - constructor( - private ingestionService: DataIngestionService, - private transformationService: DataTransformationService, - private validationService: DataValidationService, - private qualityService: DataQualityService - ) { - const eventBusConfig: EventBusConfig = { - redisHost: process.env.REDIS_HOST || 'localhost', - redisPort: parseInt(process.env.REDIS_PORT || '6379'), - redisPassword: process.env.REDIS_PASSWORD - }; - this.eventBus = new EventBus(eventBusConfig); - this.scheduler = new PipelineScheduler(this); - this.jobQueue = new JobQueue(this); - } - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Data Pipeline Orchestrator...'); - - // EventBus doesn't have initialize method, it connects automatically - await this.scheduler.initialize(); - await this.jobQueue.initialize(); - - // Subscribe to pipeline events - this.eventBus.subscribe('data.pipeline.*', this.handlePipelineEvent.bind(this)); - this.eventBus.subscribe('data.job.*', this.handleJobEvent.bind(this)); - - // Load existing pipelines - await this.loadPipelines(); - - logger.info('โœ… Data Pipeline Orchestrator initialized'); - } - - async createPipeline(pipeline: Omit): Promise { - const pipelineWithId: DataPipeline = { - ...pipeline, - id: this.generatePipelineId(), - status: PipelineStatus.DRAFT, - createdAt: new Date(), - updatedAt: new Date(), - }; this.pipelines.set(pipelineWithId.id, pipelineWithId); - - await this.eventBus.publish('data.pipeline.created', { - type: 'PIPELINE_CREATED', - pipelineId: pipelineWithId.id, - pipelineName: pipelineWithId.name, - timestamp: new Date() - } as DataPipelineEvent); - - logger.info(`๐Ÿ“‹ Created pipeline: ${pipelineWithId.name} (${pipelineWithId.id})`); - return pipelineWithId; - } - - async runPipeline(pipelineId: string, parameters?: Record): Promise { - const pipeline = this.pipelines.get(pipelineId); - if (!pipeline) { - throw new Error(`Pipeline not found: ${pipelineId}`); - } - - if (pipeline.status !== PipelineStatus.ACTIVE) { - throw new Error(`Pipeline is not active: ${pipeline.status}`); - } - - const job: PipelineJob = { - id: this.generateJobId(), - pipelineId, - status: JobStatus.PENDING, - parameters: parameters || {}, - createdAt: new Date(), - startedAt: null, - completedAt: null, - error: null, - metrics: { - recordsProcessed: 0, - recordsSuccessful: 0, - recordsFailed: 0, - processingTimeMs: 0, - }, - }; - - this.runningJobs.set(job.id, job); - // Queue the job for execution - await this.jobQueue.enqueueJob(job); - - await this.eventBus.publish('data.job.queued', { - type: 'JOB_STARTED', - jobId: job.id, - pipelineId, - timestamp: new Date() - } as DataJobEvent); - - logger.info(`๐Ÿš€ Queued pipeline job: ${job.id} for pipeline: ${pipeline.name}`); - return job; - } - - async executePipelineJob(job: PipelineJob): Promise { - const pipeline = this.pipelines.get(job.pipelineId); - if (!pipeline) { - throw new Error(`Pipeline not found: ${job.pipelineId}`); - } - - const startTime = Date.now(); job.status = JobStatus.RUNNING; - job.startedAt = new Date(); - - await this.eventBus.publish('data.job.started', { - type: 'JOB_STARTED', - jobId: job.id, - pipelineId: job.pipelineId, - timestamp: new Date() - } as DataJobEvent); - - try { - logger.info(`โš™๏ธ Executing pipeline job: ${job.id}`); - - // Execute pipeline steps - await this.executeIngestionStep(pipeline, job); - await this.executeTransformationStep(pipeline, job); - await this.executeValidationStep(pipeline, job); - await this.executeQualityChecks(pipeline, job); - - // Complete the job - job.status = JobStatus.COMPLETED; job.completedAt = new Date(); - job.metrics.processingTimeMs = Date.now() - startTime; - - await this.eventBus.publish('data.job.completed', { - type: 'JOB_COMPLETED', - jobId: job.id, - pipelineId: job.pipelineId, - timestamp: new Date() - } as DataJobEvent); - - logger.info(`โœ… Pipeline job completed: ${job.id} in ${job.metrics.processingTimeMs}ms`); - - } catch (error) { - job.status = JobStatus.FAILED; - job.completedAt = new Date(); job.error = error instanceof Error ? error.message : 'Unknown error'; - job.metrics.processingTimeMs = Date.now() - startTime; - - await this.eventBus.publish('data.job.failed', { - type: 'JOB_FAILED', - jobId: job.id, - pipelineId: job.pipelineId, - error: job.error, - timestamp: new Date() - } as DataJobEvent); - - logger.error(`โŒ Pipeline job failed: ${job.id}`, error); - throw error; - } - } - - private async executeIngestionStep(pipeline: DataPipeline, job: PipelineJob): Promise { - if (!pipeline.steps.ingestion) return; - - logger.info(`๐Ÿ“ฅ Executing ingestion step for job: ${job.id}`); - - const result = await this.ingestionService.ingestData( - pipeline.steps.ingestion, - job.parameters - ); - - job.metrics.recordsProcessed += result.recordsProcessed; - job.metrics.recordsSuccessful += result.recordsSuccessful; - job.metrics.recordsFailed += result.recordsFailed; - } - - private async executeTransformationStep(pipeline: DataPipeline, job: PipelineJob): Promise { - if (!pipeline.steps.transformation) return; - - logger.info(`๐Ÿ”„ Executing transformation step for job: ${job.id}`); - - const result = await this.transformationService.transformData( - pipeline.steps.transformation, - job.parameters - ); - - job.metrics.recordsProcessed += result.recordsProcessed; - job.metrics.recordsSuccessful += result.recordsSuccessful; - job.metrics.recordsFailed += result.recordsFailed; - } - - private async executeValidationStep(pipeline: DataPipeline, job: PipelineJob): Promise { - if (!pipeline.steps.validation) return; - - logger.info(`โœ… Executing validation step for job: ${job.id}`); - - const result = await this.validationService.validateData( - pipeline.steps.validation, - job.parameters - ); - - job.metrics.recordsProcessed += result.recordsProcessed; - job.metrics.recordsSuccessful += result.recordsSuccessful; - job.metrics.recordsFailed += result.recordsFailed; - } - - private async executeQualityChecks(pipeline: DataPipeline, job: PipelineJob): Promise { - if (!pipeline.steps.qualityChecks) return; - - logger.info(`๐Ÿ” Executing quality checks for job: ${job.id}`); - - await this.qualityService.runQualityChecks( - pipeline.steps.qualityChecks, - job.parameters - ); - } - - async schedulePipeline(pipelineId: string, cronExpression: string): Promise { - const pipeline = this.pipelines.get(pipelineId); - if (!pipeline) { - throw new Error(`Pipeline not found: ${pipelineId}`); - } - - await this.scheduler.schedulePipeline(pipelineId, cronExpression); - - pipeline.schedule = { - cronExpression, - enabled: true, - lastRun: null, nextRun: this.scheduler.getNextRunTime(cronExpression), - }; - - await this.eventBus.publish('data.pipeline.scheduled', { - type: 'PIPELINE_STARTED', - pipelineId, - pipelineName: pipeline.name, - timestamp: new Date() - } as DataPipelineEvent); - - logger.info(`๐Ÿ“… Scheduled pipeline: ${pipeline.name} with cron: ${cronExpression}`); - } - - // Pipeline CRUD operations - getPipeline(pipelineId: string): DataPipeline | undefined { - return this.pipelines.get(pipelineId); - } - - listPipelines(): DataPipeline[] { - return Array.from(this.pipelines.values()); - } - - getJob(jobId: string): PipelineJob | undefined { - return this.runningJobs.get(jobId); - } - - listJobs(pipelineId?: string): PipelineJob[] { - const jobs = Array.from(this.runningJobs.values()); - return pipelineId ? jobs.filter(job => job.pipelineId === pipelineId) : jobs; - } - - private async handlePipelineEvent(event: any): Promise { - logger.debug('๐Ÿ“จ Received pipeline event:', event); - // Handle pipeline-level events - } - - private async handleJobEvent(event: any): Promise { - logger.debug('๐Ÿ“จ Received job event:', event); - // Handle job-level events - } - - private async loadPipelines(): Promise { - // In a real implementation, load pipelines from persistent storage - logger.info('๐Ÿ“‚ Loading existing pipelines...'); - } - - private generatePipelineId(): string { - return `pipeline_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - private generateJobId(): string { - return `job_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Data Pipeline Orchestrator...'); - - await this.scheduler.shutdown(); - await this.jobQueue.shutdown(); - await this.eventBus.close(); - - logger.info('โœ… Data Pipeline Orchestrator shutdown complete'); - } -} diff --git a/apps/data-services/data-processor/src/core/JobQueue.ts b/apps/data-services/data-processor/src/core/JobQueue.ts deleted file mode 100644 index 672da25..0000000 --- a/apps/data-services/data-processor/src/core/JobQueue.ts +++ /dev/null @@ -1,79 +0,0 @@ -import Queue from 'bull'; -import { getLogger } from '@stock-bot/logger'; -import { PipelineJob } from '../types/DataPipeline'; -import { DataPipelineOrchestrator } from './DataPipelineOrchestrator'; - -const logger = getLogger('job-queue'); - -export class JobQueue { - private queue: Queue.Queue; - - constructor(private orchestrator: DataPipelineOrchestrator) { - this.queue = new Queue('data-pipeline-jobs', { - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - }, - }); - } - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Job Queue...'); - - // Process jobs with a maximum of 5 concurrent jobs - this.queue.process('pipeline-job', 5, async (job) => { - const pipelineJob: PipelineJob = job.data; - await this.orchestrator.executePipelineJob(pipelineJob); - }); - - // Handle job events - this.queue.on('completed', (job) => { - logger.info(`โœ… Job completed: ${job.id}`); - }); - - this.queue.on('failed', (job, error) => { - logger.error(`โŒ Job failed: ${job.id}`, error); - }); - - this.queue.on('stalled', (job) => { - logger.warn(`โš ๏ธ Job stalled: ${job.id}`); - }); - - logger.info('โœ… Job Queue initialized'); - } - - async enqueueJob(job: PipelineJob): Promise { - await this.queue.add('pipeline-job', job, { - jobId: job.id, - removeOnComplete: 100, // Keep last 100 completed jobs - removeOnFail: 50, // Keep last 50 failed jobs - attempts: 3, // Retry failed jobs up to 3 times - backoff: { - type: 'exponential', - delay: 2000, - }, - }); - - logger.info(`๐Ÿ“ค Enqueued job: ${job.id}`); - } - - async getJobStats(): Promise { - const waiting = await this.queue.getWaiting(); - const active = await this.queue.getActive(); - const completed = await this.queue.getCompleted(); - const failed = await this.queue.getFailed(); - - return { - waiting: waiting.length, - active: active.length, - completed: completed.length, - failed: failed.length, - }; - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Job Queue...'); - await this.queue.close(); - logger.info('โœ… Job Queue shutdown complete'); - } -} diff --git a/apps/data-services/data-processor/src/core/PipelineScheduler.ts b/apps/data-services/data-processor/src/core/PipelineScheduler.ts deleted file mode 100644 index 99c373c..0000000 --- a/apps/data-services/data-processor/src/core/PipelineScheduler.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { CronJob } from 'cron'; -import { getLogger } from '@stock-bot/logger'; -import { DataPipelineOrchestrator } from './DataPipelineOrchestrator'; - -const logger = getLogger('pipeline-scheduler'); - -export class PipelineScheduler { - private scheduledJobs: Map = new Map(); - - constructor(private orchestrator: DataPipelineOrchestrator) {} - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Pipeline Scheduler...'); - logger.info('โœ… Pipeline Scheduler initialized'); - } - - async schedulePipeline(pipelineId: string, cronExpression: string): Promise { - // Cancel existing schedule if it exists - if (this.scheduledJobs.has(pipelineId)) { - this.cancelSchedule(pipelineId); - } - - const cronJob = new CronJob( - cronExpression, - async () => { - try { - logger.info(`โฐ Scheduled execution triggered for pipeline: ${pipelineId}`); - await this.orchestrator.runPipeline(pipelineId); - } catch (error) { - logger.error(`โŒ Scheduled pipeline execution failed: ${pipelineId}`, error); - } - }, - null, - true, // Start immediately - 'UTC' - ); - - this.scheduledJobs.set(pipelineId, cronJob); - logger.info(`๐Ÿ“… Scheduled pipeline ${pipelineId} with cron: ${cronExpression}`); - } - - cancelSchedule(pipelineId: string): void { - const job = this.scheduledJobs.get(pipelineId); - if (job) { - job.stop(); - this.scheduledJobs.delete(pipelineId); - logger.info(`๐Ÿšซ Cancelled schedule for pipeline: ${pipelineId}`); - } - } - getNextRunTime(cronExpression: string): Date { - const job = new CronJob(cronExpression, () => {}, null, false); - return job.nextDate().toJSDate(); - } - - getScheduledPipelines(): string[] { - return Array.from(this.scheduledJobs.keys()); - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Pipeline Scheduler...'); - - for (const [pipelineId, job] of this.scheduledJobs) { - job.stop(); - logger.info(`๐Ÿšซ Stopped scheduled job for pipeline: ${pipelineId}`); - } - - this.scheduledJobs.clear(); - logger.info('โœ… Pipeline Scheduler shutdown complete'); - } -} diff --git a/apps/data-services/data-processor/src/index.ts b/apps/data-services/data-processor/src/index.ts deleted file mode 100644 index a05ed8c..0000000 --- a/apps/data-services/data-processor/src/index.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { Hono } from 'hono'; -import { serve } from 'bun'; -import { getLogger, loggingMiddleware, errorLoggingMiddleware } from '@stock-bot/logger'; -import { DataPipelineOrchestrator } from './core/DataPipelineOrchestrator'; -import { DataQualityService } from './services/DataQualityService'; -import { DataIngestionService } from './services/DataIngestionService'; -import { DataTransformationService } from './services/DataTransformationService'; -import { DataValidationService } from './services/DataValidationService'; -import { HealthController } from './controllers/HealthController'; -import { PipelineController } from './controllers/PipelineController'; -import { JobController } from './controllers/JobController'; - -const app = new Hono(); - -// Initialize logger -const logger = getLogger('data-processor'); - -// Add logging middleware -app.use('*', loggingMiddleware({ - serviceName: 'data-processor', - skipPaths: ['/health'] -})); - -// Add error logging middleware -app.use('*', errorLoggingMiddleware(logger)); - -// Services -const dataQualityService = new DataQualityService(); -const dataIngestionService = new DataIngestionService(); -const dataTransformationService = new DataTransformationService(); -const dataValidationService = new DataValidationService(); - -// Core orchestrator -const pipelineOrchestrator = new DataPipelineOrchestrator( - dataIngestionService, - dataTransformationService, - dataValidationService, - dataQualityService -); - -// Controllers -const healthController = new HealthController(); -const pipelineController = new PipelineController(pipelineOrchestrator); -const jobController = new JobController(pipelineOrchestrator); - -// Health endpoints -app.get('/health', healthController.getHealth.bind(healthController)); -app.get('/health/detailed', healthController.getDetailedHealth.bind(healthController)); - -// Pipeline management -app.get('/api/pipelines', pipelineController.listPipelines.bind(pipelineController)); -app.post('/api/pipelines', pipelineController.createPipeline.bind(pipelineController)); -app.get('/api/pipelines/:id', pipelineController.getPipeline.bind(pipelineController)); -app.put('/api/pipelines/:id', pipelineController.updatePipeline.bind(pipelineController)); -app.delete('/api/pipelines/:id', pipelineController.deletePipeline.bind(pipelineController)); -app.post('/api/pipelines/:id/run', pipelineController.runPipeline.bind(pipelineController)); -app.post('/api/pipelines/:id/schedule', pipelineController.schedulePipeline.bind(pipelineController)); -app.post('/api/pipelines/:id/pause', pipelineController.pausePipeline.bind(pipelineController)); -app.post('/api/pipelines/:id/resume', pipelineController.resumePipeline.bind(pipelineController)); -app.get('/api/pipelines/:id/metrics', pipelineController.getPipelineMetrics.bind(pipelineController)); - -// Job management -app.get('/api/jobs', jobController.listJobs.bind(jobController)); -app.get('/api/jobs/stats', jobController.getJobStats.bind(jobController)); -app.get('/api/jobs/:id', jobController.getJob.bind(jobController)); -app.get('/api/jobs/:id/logs', jobController.getJobLogs.bind(jobController)); -app.get('/api/jobs/:id/metrics', jobController.getJobMetrics.bind(jobController)); -app.post('/api/jobs/:id/cancel', jobController.cancelJob.bind(jobController)); -app.post('/api/jobs/:id/retry', jobController.retryJob.bind(jobController)); - -// Data quality endpoints -app.get('/api/data-quality/metrics', async (c) => { - const metrics = await dataQualityService.getQualityMetrics(); - return c.json({ success: true, data: metrics }); -}); - -app.get('/api/data-quality/report/:dataset', async (c) => { - const dataset = c.req.param('dataset'); - const report = await dataQualityService.generateReport(dataset); - return c.json({ success: true, data: report }); -}); - -const PORT = parseInt(process.env.DATA_PROCESSOR_PORT || '5001'); - -// Initialize services -async function initializeServices() { - try { - logger.info('๐Ÿ”„ Initializing Data Processor services...'); - - await dataQualityService.initialize(); - await dataIngestionService.initialize(); - await dataTransformationService.initialize(); - await dataValidationService.initialize(); - await pipelineOrchestrator.initialize(); - - logger.info('โœ… Data Processor services initialized successfully'); - } catch (error) { - logger.error('โŒ Failed to initialize Data Processor services:', error); - process.exit(1); - } -} - -// Graceful shutdown -process.on('SIGINT', async () => { - logger.info('๐Ÿ”„ Gracefully shutting down Data Processor...'); - await pipelineOrchestrator.shutdown(); - process.exit(0); -}); - -initializeServices().then(() => { - serve({ - port: PORT, - fetch: app.fetch, - }); - - logger.info(`๐Ÿš€ Data Processor running on port ${PORT}`); - logger.info(`๐Ÿ” Health check: http://localhost:${PORT}/health`); - logger.info(`๐Ÿ“Š API documentation: http://localhost:${PORT}/api`); -}); diff --git a/apps/data-services/data-processor/src/services/DataIngestionService.ts b/apps/data-services/data-processor/src/services/DataIngestionService.ts deleted file mode 100644 index c924142..0000000 --- a/apps/data-services/data-processor/src/services/DataIngestionService.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { getLogger } from '@stock-bot/logger'; - -const logger = getLogger('DataIngestionService'); -import { IngestionStep, ProcessingResult, DataSource } from '../types/DataPipeline'; -import axios from 'axios'; -import csv from 'csv-parser'; -import * as fs from 'fs'; - -export class DataIngestionService { - private activeConnections: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Data Ingestion Service...'); - logger.info('โœ… Data Ingestion Service initialized'); - } - - async ingestData(step: IngestionStep, parameters: Record): Promise { - const startTime = Date.now(); - logger.info(`๐Ÿ“ฅ Starting data ingestion from ${step.source.type}: ${step.source.connection.url || step.source.connection.host}`); - - try { - switch (step.source.type) { - case 'api': - return await this.ingestFromApi(step.source, parameters); - case 'file': - return await this.ingestFromFile(step.source, parameters); - case 'database': - return await this.ingestFromDatabase(step.source, parameters); - case 'stream': - return await this.ingestFromStream(step.source, parameters); - default: - throw new Error(`Unsupported ingestion type: ${step.source.type}`); - } - } catch (error) { - const processingTime = Date.now() - startTime; - logger.error(`โŒ Data ingestion failed after ${processingTime}ms:`, error); - - return { - recordsProcessed: 0, - recordsSuccessful: 0, - recordsFailed: 0, - errors: [{ - record: 0, - message: error instanceof Error ? error.message : 'Unknown error', - code: 'INGESTION_ERROR' - }], - metadata: { processingTimeMs: processingTime } - }; - } - } - - private async ingestFromApi(source: DataSource, parameters: Record): Promise { - const config = { - method: 'GET', - url: source.connection.url, - headers: source.connection.headers || {}, - params: { ...source.connection.params, ...parameters }, - }; - - if (source.connection.apiKey) { - config.headers['Authorization'] = `Bearer ${source.connection.apiKey}`; - } - - const response = await axios(config); - const data = response.data; - - // Process the data based on format - let records: any[] = []; - - if (Array.isArray(data)) { - records = data; - } else if (data.data && Array.isArray(data.data)) { - records = data.data; - } else if (data.results && Array.isArray(data.results)) { - records = data.results; - } else { - records = [data]; - } - - logger.info(`๐Ÿ“Š Ingested ${records.length} records from API: ${source.connection.url}`); - - return { - recordsProcessed: records.length, - recordsSuccessful: records.length, - recordsFailed: 0, - errors: [], - metadata: { - source: 'api', - url: source.connection.url, - statusCode: response.status, - responseSize: JSON.stringify(data).length - } - }; - } - - private async ingestFromFile(source: DataSource, parameters: Record): Promise { - const filePath = source.connection.url || parameters.filePath; - - if (!filePath) { - throw new Error('File path is required for file ingestion'); - } - - switch (source.format) { - case 'csv': - return await this.ingestCsvFile(filePath); - case 'json': - return await this.ingestJsonFile(filePath); - default: - throw new Error(`Unsupported file format: ${source.format}`); - } - } - - private async ingestCsvFile(filePath: string): Promise { - return new Promise((resolve, reject) => { - const records: any[] = []; - const errors: any[] = []; - let recordCount = 0; fs.createReadStream(filePath) - .pipe(csv()) - .on('data', (data: any) => { - recordCount++; - try { - records.push(data); - } catch (error) { - errors.push({ - record: recordCount, - message: error instanceof Error ? error.message : 'Parse error', - code: 'CSV_PARSE_ERROR' - }); - } - }) - .on('end', () => { - logger.info(`๐Ÿ“Š Ingested ${records.length} records from CSV: ${filePath}`); - resolve({ - recordsProcessed: recordCount, - recordsSuccessful: records.length, - recordsFailed: errors.length, - errors, - metadata: { - source: 'file', - format: 'csv', - filePath - } - }); - }) - .on('error', reject); - }); - } - - private async ingestJsonFile(filePath: string): Promise { - const fileContent = await fs.promises.readFile(filePath, 'utf8'); - const data = JSON.parse(fileContent); - - let records: any[] = []; - - if (Array.isArray(data)) { - records = data; - } else { - records = [data]; - } - - logger.info(`๐Ÿ“Š Ingested ${records.length} records from JSON: ${filePath}`); - - return { - recordsProcessed: records.length, - recordsSuccessful: records.length, - recordsFailed: 0, - errors: [], - metadata: { - source: 'file', - format: 'json', - filePath, - fileSize: fileContent.length - } - }; - } - - private async ingestFromDatabase(source: DataSource, parameters: Record): Promise { - // Placeholder for database ingestion - // In a real implementation, this would connect to various databases - // (PostgreSQL, MySQL, MongoDB, etc.) and execute queries - - throw new Error('Database ingestion not yet implemented'); - } - - private async ingestFromStream(source: DataSource, parameters: Record): Promise { - // Placeholder for stream ingestion - // In a real implementation, this would connect to streaming sources - // (Kafka, Kinesis, WebSocket, etc.) - - throw new Error('Stream ingestion not yet implemented'); - } - - async getIngestionMetrics(): Promise { - return { - activeConnections: this.activeConnections.size, - supportedSources: ['api', 'file', 'database', 'stream'], - supportedFormats: ['json', 'csv', 'xml', 'parquet', 'avro'] - }; - } -} diff --git a/apps/data-services/data-processor/src/services/DataQualityService.ts b/apps/data-services/data-processor/src/services/DataQualityService.ts deleted file mode 100644 index 1d35a14..0000000 --- a/apps/data-services/data-processor/src/services/DataQualityService.ts +++ /dev/null @@ -1,376 +0,0 @@ -import { getLogger } from '@stock-bot/logger'; - -const logger = getLogger('DataQualityService'); -import { QualityCheckStep, ProcessingResult, QualityCheck, QualityThresholds } from '../types/DataPipeline'; - -export class DataQualityService { - private qualityMetrics: Map = new Map(); - private qualityReports: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Data Quality Service...'); - - // Initialize quality metrics storage - this.qualityMetrics.clear(); - this.qualityReports.clear(); - - logger.info('โœ… Data Quality Service initialized'); - } - - async runQualityChecks(step: QualityCheckStep, parameters: Record): Promise { - const startTime = Date.now(); - logger.info(`๐Ÿ” Running ${step.checks.length} quality checks`); - - const inputData = parameters.inputData || []; - const results: any[] = []; - const errors: any[] = []; - let totalScore = 0; - - try { - for (const check of step.checks) { - const checkResult = await this.executeQualityCheck(check, inputData); - results.push(checkResult); - totalScore += checkResult.score; - - // Check if the quality score meets thresholds - if (checkResult.score < step.thresholds.error) { - errors.push({ - record: 0, - field: check.field, - message: `Quality check failed: ${check.name} scored ${checkResult.score}%, below error threshold ${step.thresholds.error}%`, - code: 'QUALITY_CHECK_ERROR' - }); - } else if (checkResult.score < step.thresholds.warning) { - logger.warn(`โš ๏ธ Quality warning: ${check.name} scored ${checkResult.score}%, below warning threshold ${step.thresholds.warning}%`); - } - } - - const averageScore = totalScore / step.checks.length; - const processingTime = Date.now() - startTime; - - // Store quality metrics - this.storeQualityMetrics({ - timestamp: new Date(), - averageScore, - checksRun: step.checks.length, - results, - processingTimeMs: processingTime - }); - - logger.info(`๐Ÿ” Quality checks completed: ${averageScore.toFixed(2)}% average score in ${processingTime}ms`); - - return { - recordsProcessed: inputData.length, - recordsSuccessful: errors.length === 0 ? inputData.length : 0, - recordsFailed: errors.length > 0 ? inputData.length : 0, - errors, - metadata: { - qualityScore: averageScore, - checksRun: step.checks.length, - results, - processingTimeMs: processingTime - } - }; - - } catch (error) { - const processingTime = Date.now() - startTime; - logger.error(`โŒ Quality checks failed after ${processingTime}ms:`, error); - - return { - recordsProcessed: inputData.length, - recordsSuccessful: 0, - recordsFailed: inputData.length, - errors: [{ - record: 0, - message: error instanceof Error ? error.message : 'Unknown quality check error', - code: 'QUALITY_SERVICE_ERROR' - }], - metadata: { processingTimeMs: processingTime } - }; - } - } - - private async executeQualityCheck(check: QualityCheck, data: any[]): Promise { - switch (check.type) { - case 'completeness': - return this.checkCompleteness(check, data); - case 'accuracy': - return this.checkAccuracy(check, data); - case 'consistency': - return this.checkConsistency(check, data); - case 'validity': - return this.checkValidity(check, data); - case 'uniqueness': - return this.checkUniqueness(check, data); - default: - throw new Error(`Unsupported quality check type: ${check.type}`); - } - } - - private checkCompleteness(check: QualityCheck, data: any[]): any { - if (!check.field) { - throw new Error('Completeness check requires a field'); - } - - const totalRecords = data.length; - const completeRecords = data.filter(record => { - const value = this.getFieldValue(record, check.field!); - return value !== null && value !== undefined && value !== ''; - }).length; - - const score = totalRecords > 0 ? (completeRecords / totalRecords) * 100 : 100; - - return { - checkName: check.name, - type: 'completeness', - field: check.field, - score, - passed: score >= check.threshold, - details: { - totalRecords, - completeRecords, - missingRecords: totalRecords - completeRecords - } - }; - } - - private checkAccuracy(check: QualityCheck, data: any[]): any { - // Placeholder for accuracy checks - // In a real implementation, this would validate data against known references - // or business rules specific to stock market data - - const score = 95; // Mock score - - return { - checkName: check.name, - type: 'accuracy', - field: check.field, - score, - passed: score >= check.threshold, - details: { - validatedRecords: data.length, - accurateRecords: Math.floor(data.length * 0.95) - } - }; - } - - private checkConsistency(check: QualityCheck, data: any[]): any { - if (!check.field) { - throw new Error('Consistency check requires a field'); - } - - // Check for consistent data types and formats - const fieldValues = data.map(record => this.getFieldValue(record, check.field!)); - const types = [...new Set(fieldValues.map(val => typeof val))]; - - // For stock symbols, check consistent format - if (check.field === 'symbol') { - const validSymbols = fieldValues.filter(symbol => - typeof symbol === 'string' && /^[A-Z]{1,5}$/.test(symbol) - ).length; - - const score = fieldValues.length > 0 ? (validSymbols / fieldValues.length) * 100 : 100; - - return { - checkName: check.name, - type: 'consistency', - field: check.field, - score, - passed: score >= check.threshold, - details: { - totalValues: fieldValues.length, - consistentValues: validSymbols, - inconsistentValues: fieldValues.length - validSymbols - } - }; - } - - // Generic consistency check - const score = types.length === 1 ? 100 : 0; - - return { - checkName: check.name, - type: 'consistency', - field: check.field, - score, - passed: score >= check.threshold, - details: { - dataTypes: types, - isConsistent: types.length === 1 - } - }; - } - - private checkValidity(check: QualityCheck, data: any[]): any { - if (!check.field) { - throw new Error('Validity check requires a field'); - } - - let validRecords = 0; - const totalRecords = data.length; - - for (const record of data) { - const value = this.getFieldValue(record, check.field); - - if (this.isValidValue(check.field, value)) { - validRecords++; - } - } - - const score = totalRecords > 0 ? (validRecords / totalRecords) * 100 : 100; - - return { - checkName: check.name, - type: 'validity', - field: check.field, - score, - passed: score >= check.threshold, - details: { - totalRecords, - validRecords, - invalidRecords: totalRecords - validRecords - } - }; - } - - private checkUniqueness(check: QualityCheck, data: any[]): any { - if (!check.field) { - throw new Error('Uniqueness check requires a field'); - } - - const fieldValues = data.map(record => this.getFieldValue(record, check.field!)); - const uniqueValues = new Set(fieldValues); - - const score = fieldValues.length > 0 ? (uniqueValues.size / fieldValues.length) * 100 : 100; - - return { - checkName: check.name, - type: 'uniqueness', - field: check.field, - score, - passed: score >= check.threshold, - details: { - totalValues: fieldValues.length, - uniqueValues: uniqueValues.size, - duplicateValues: fieldValues.length - uniqueValues.size - } - }; - } - - private getFieldValue(record: any, fieldPath: string): any { - return fieldPath.split('.').reduce((obj, field) => obj?.[field], record); - } - - private isValidValue(field: string, value: any): boolean { - switch (field) { - case 'symbol': - return typeof value === 'string' && /^[A-Z]{1,5}$/.test(value); - case 'price': - return typeof value === 'number' && value > 0 && value < 1000000; - case 'volume': - return typeof value === 'number' && value >= 0 && Number.isInteger(value); - case 'timestamp': - return value instanceof Date || !isNaN(new Date(value).getTime()); - default: - return value !== null && value !== undefined; - } - } - - private storeQualityMetrics(metrics: any): void { - const key = `metrics_${Date.now()}`; - this.qualityMetrics.set(key, metrics); - // Keep only last 100 metrics - if (this.qualityMetrics.size > 100) { - const oldestKey = this.qualityMetrics.keys().next().value; - if (oldestKey) { - this.qualityMetrics.delete(oldestKey); - } - } - } - - async getQualityMetrics(dataset?: string): Promise { - const allMetrics = Array.from(this.qualityMetrics.values()); - - if (allMetrics.length === 0) { - return { - totalChecks: 0, - averageScore: 0, - recentResults: [] - }; - } - - const totalChecks = allMetrics.reduce((sum, m) => sum + m.checksRun, 0); - const averageScore = allMetrics.reduce((sum, m) => sum + m.averageScore, 0) / allMetrics.length; - const recentResults = allMetrics.slice(-10); - - return { - totalChecks, - averageScore: Math.round(averageScore * 100) / 100, - recentResults, - summary: { - totalRuns: allMetrics.length, - averageProcessingTime: allMetrics.reduce((sum, m) => sum + m.processingTimeMs, 0) / allMetrics.length - } - }; - } - - async generateReport(dataset: string): Promise { - const metrics = await this.getQualityMetrics(dataset); - - const report = { - dataset, - generatedAt: new Date(), - summary: metrics, - recommendations: this.generateRecommendations(metrics), - trends: this.analyzeTrends(metrics.recentResults) - }; - - this.qualityReports.set(dataset, report); - - return report; - } - - private generateRecommendations(metrics: any): string[] { - const recommendations: string[] = []; - - if (metrics.averageScore < 80) { - recommendations.push('Overall data quality is below acceptable threshold. Review data ingestion processes.'); - } - - if (metrics.averageScore < 95 && metrics.averageScore >= 80) { - recommendations.push('Data quality is acceptable but could be improved. Consider implementing additional validation rules.'); - } - - if (metrics.totalChecks === 0) { - recommendations.push('No quality checks have been run. Implement quality monitoring for your data pipelines.'); - } - - return recommendations; - } - - private analyzeTrends(recentResults: any[]): any { - if (recentResults.length < 2) { - return { trend: 'insufficient_data', message: 'Not enough data to analyze trends' }; - } - - const scores = recentResults.map(r => r.averageScore); - const latestScore = scores[scores.length - 1]; - const previousScore = scores[scores.length - 2]; - - if (latestScore > previousScore) { - return { trend: 'improving', message: 'Data quality is improving' }; - } else if (latestScore < previousScore) { - return { trend: 'declining', message: 'Data quality is declining' }; - } else { - return { trend: 'stable', message: 'Data quality is stable' }; - } - } - - async getAvailableReports(): Promise { - return Array.from(this.qualityReports.keys()); - } - - async getReport(dataset: string): Promise { - return this.qualityReports.get(dataset) || null; - } -} diff --git a/apps/data-services/data-processor/src/services/DataTransformationService.ts b/apps/data-services/data-processor/src/services/DataTransformationService.ts deleted file mode 100644 index 59eb97d..0000000 --- a/apps/data-services/data-processor/src/services/DataTransformationService.ts +++ /dev/null @@ -1,293 +0,0 @@ -import { getLogger } from '@stock-bot/logger'; - -const logger = getLogger('DataTransformationService'); -import { TransformationStep, ProcessingResult } from '../types/DataPipeline'; - -export class DataTransformationService { - private transformationFunctions: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Data Transformation Service...'); - - // Register built-in transformation functions - this.registerBuiltInTransformations(); - - logger.info('โœ… Data Transformation Service initialized'); - } - - async transformData(step: TransformationStep, parameters: Record): Promise { - const startTime = Date.now(); - logger.info(`๐Ÿ”„ Starting data transformation: ${step.type}`); - - try { - switch (step.type) { - case 'javascript': - return await this.executeJavaScriptTransformation(step, parameters); - case 'sql': - return await this.executeSqlTransformation(step, parameters); - case 'custom': - return await this.executeCustomTransformation(step, parameters); - default: - throw new Error(`Unsupported transformation type: ${step.type}`); - } - } catch (error) { - const processingTime = Date.now() - startTime; - logger.error(`โŒ Data transformation failed after ${processingTime}ms:`, error); - - return { - recordsProcessed: 0, - recordsSuccessful: 0, - recordsFailed: 0, - errors: [{ - record: 0, - message: error instanceof Error ? error.message : 'Unknown error', - code: 'TRANSFORMATION_ERROR' - }], - metadata: { processingTimeMs: processingTime } - }; - } - } - - private async executeJavaScriptTransformation(step: TransformationStep, parameters: Record): Promise { - const { code, inputData } = step.configuration; - - if (!code || !inputData) { - throw new Error('JavaScript transformation requires code and inputData configuration'); - } - - const transformedRecords: any[] = []; - const errors: any[] = []; - let recordCount = 0; - - // Execute transformation for each record - for (const record of inputData) { - recordCount++; - - try { - // Create a safe execution context - const context = { - record, - parameters, - utils: this.getTransformationUtils(), - }; - - // Execute the transformation code - const transformFunction = new Function('context', ` - const { record, parameters, utils } = context; - ${code} - `); - - const result = transformFunction(context); - - if (result !== undefined) { - transformedRecords.push(result); - } else { - transformedRecords.push(record); // Keep original if no transformation result - } - } catch (error) { - errors.push({ - record: recordCount, - message: error instanceof Error ? error.message : 'Transformation error', - code: 'JS_TRANSFORM_ERROR' - }); - } - } - - logger.info(`๐Ÿ”„ Transformed ${transformedRecords.length} records using JavaScript`); - - return { - recordsProcessed: recordCount, - recordsSuccessful: transformedRecords.length, - recordsFailed: errors.length, - errors, - metadata: { - transformationType: 'javascript', - outputData: transformedRecords - } - }; - } - - private async executeSqlTransformation(step: TransformationStep, parameters: Record): Promise { - // Placeholder for SQL transformation - // In a real implementation, this would execute SQL queries against a data warehouse - // or in-memory SQL engine like DuckDB - - throw new Error('SQL transformation not yet implemented'); - } - - private async executeCustomTransformation(step: TransformationStep, parameters: Record): Promise { - const { functionName, inputData } = step.configuration; - - if (!functionName) { - throw new Error('Custom transformation requires functionName configuration'); - } - - const transformFunction = this.transformationFunctions.get(functionName); - if (!transformFunction) { - throw new Error(`Custom transformation function not found: ${functionName}`); - } - - const result = await transformFunction(inputData, parameters); - - logger.info(`๐Ÿ”„ Executed custom transformation: ${functionName}`); - - return result; - } - - private registerBuiltInTransformations(): void { - // Market data normalization - this.transformationFunctions.set('normalizeMarketData', (data: any[], parameters: any) => { - const normalized = data.map(record => ({ - symbol: record.symbol?.toUpperCase(), - price: parseFloat(record.price) || 0, - volume: parseInt(record.volume) || 0, - timestamp: new Date(record.timestamp || Date.now()), - source: parameters.source || 'unknown' - })); - - return { - recordsProcessed: data.length, - recordsSuccessful: normalized.length, - recordsFailed: 0, - errors: [], - metadata: { outputData: normalized } - }; - }); - - // Financial data aggregation - this.transformationFunctions.set('aggregateFinancialData', (data: any[], parameters: any) => { - const { groupBy = 'symbol', aggregations = ['avg', 'sum'] } = parameters; - - const grouped = data.reduce((acc, record) => { - const key = record[groupBy]; - if (!acc[key]) { - acc[key] = []; - } - acc[key].push(record); - return acc; - }, {} as Record); - - const aggregated = Object.entries(grouped).map(([key, records]) => { - const recordsArray = records as any[]; - const result: any = { [groupBy]: key }; - - if (aggregations.includes('avg')) { - result.avgPrice = recordsArray.reduce((sum: number, r: any) => sum + (r.price || 0), 0) / recordsArray.length; - } - - if (aggregations.includes('sum')) { - result.totalVolume = recordsArray.reduce((sum: number, r: any) => sum + (r.volume || 0), 0); - } - - if (aggregations.includes('count')) { - result.count = recordsArray.length; - } - - return result; - }); - - return { - recordsProcessed: data.length, - recordsSuccessful: aggregated.length, - recordsFailed: 0, - errors: [], - metadata: { outputData: aggregated } - }; - }); - - // Data cleaning - this.transformationFunctions.set('cleanData', (data: any[], parameters: any) => { - const { removeNulls = true, trimStrings = true, validateNumbers = true } = parameters; - const cleaned: any[] = []; - const errors: any[] = []; - - data.forEach((record, index) => { - try { - let cleanRecord = { ...record }; - - if (removeNulls) { - Object.keys(cleanRecord).forEach(key => { - if (cleanRecord[key] === null || cleanRecord[key] === undefined) { - delete cleanRecord[key]; - } - }); - } - - if (trimStrings) { - Object.keys(cleanRecord).forEach(key => { - if (typeof cleanRecord[key] === 'string') { - cleanRecord[key] = cleanRecord[key].trim(); - } - }); - } - - if (validateNumbers) { - Object.keys(cleanRecord).forEach(key => { - if (typeof cleanRecord[key] === 'string' && !isNaN(Number(cleanRecord[key]))) { - cleanRecord[key] = Number(cleanRecord[key]); - } - }); - } - - cleaned.push(cleanRecord); - } catch (error) { - errors.push({ - record: index + 1, - message: error instanceof Error ? error.message : 'Cleaning error', - code: 'DATA_CLEANING_ERROR' - }); - } - }); - - return { - recordsProcessed: data.length, - recordsSuccessful: cleaned.length, - recordsFailed: errors.length, - errors, - metadata: { outputData: cleaned } - }; - }); - } - - private getTransformationUtils() { - return { - // Date utilities - formatDate: (date: Date | string, format: string = 'ISO') => { - const d = new Date(date); - switch (format) { - case 'ISO': - return d.toISOString(); - case 'YYYY-MM-DD': - return d.toISOString().split('T')[0]; - default: - return d.toString(); - } - }, - - // Number utilities - round: (num: number, decimals: number = 2) => { - return Math.round(num * Math.pow(10, decimals)) / Math.pow(10, decimals); - }, - - // String utilities - slugify: (str: string) => { - return str.toLowerCase().replace(/[^a-z0-9]/g, '-').replace(/-+/g, '-'); - }, - - // Market data utilities - calculatePercentageChange: (current: number, previous: number) => { - if (previous === 0) return 0; - return ((current - previous) / previous) * 100; - } - }; - } - - registerCustomTransformation(name: string, func: Function): void { - this.transformationFunctions.set(name, func); - logger.info(`โœ… Registered custom transformation: ${name}`); - } - - getAvailableTransformations(): string[] { - return Array.from(this.transformationFunctions.keys()); - } -} diff --git a/apps/data-services/data-processor/src/services/DataValidationService.ts b/apps/data-services/data-processor/src/services/DataValidationService.ts deleted file mode 100644 index 29c009e..0000000 --- a/apps/data-services/data-processor/src/services/DataValidationService.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { getLogger } from '@stock-bot/logger'; - -const logger = getLogger('DataValidationService'); -import { ValidationStep, ProcessingResult, ValidationRule } from '../types/DataPipeline'; -import Joi from 'joi'; - -export class DataValidationService { - private validators: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Data Validation Service...'); - - // Register built-in validators - this.registerBuiltInValidators(); - - logger.info('โœ… Data Validation Service initialized'); - } - - async validateData(step: ValidationStep, parameters: Record): Promise { - const startTime = Date.now(); - logger.info(`โœ… Starting data validation with ${step.rules.length} rules`); - - const inputData = parameters.inputData || []; - const validRecords: any[] = []; - const errors: any[] = []; - let recordCount = 0; - - try { - for (const record of inputData) { - recordCount++; - const recordErrors: any[] = []; - - // Apply all validation rules to this record - for (const rule of step.rules) { - try { - const isValid = await this.applyValidationRule(record, rule); - if (!isValid) { - recordErrors.push({ - record: recordCount, - field: rule.field, - message: rule.message || `Validation failed for rule: ${rule.type}`, - code: `VALIDATION_${rule.type.toUpperCase()}_FAILED` - }); - } - } catch (error) { - recordErrors.push({ - record: recordCount, - field: rule.field, - message: error instanceof Error ? error.message : 'Validation error', - code: 'VALIDATION_ERROR' - }); - } - } - - if (recordErrors.length === 0) { - validRecords.push(record); - } else { - errors.push(...recordErrors); - - // Handle validation failure based on strategy - if (step.onFailure === 'stop') { - break; - } - } - } - - const processingTime = Date.now() - startTime; - logger.info(`โœ… Validation completed: ${validRecords.length}/${recordCount} records valid in ${processingTime}ms`); - - return { - recordsProcessed: recordCount, - recordsSuccessful: validRecords.length, - recordsFailed: recordCount - validRecords.length, - errors, - metadata: { - validationRules: step.rules.length, - onFailure: step.onFailure, - processingTimeMs: processingTime, - outputData: validRecords - } - }; - - } catch (error) { - const processingTime = Date.now() - startTime; - logger.error(`โŒ Data validation failed after ${processingTime}ms:`, error); - - return { - recordsProcessed: recordCount, - recordsSuccessful: 0, - recordsFailed: recordCount, - errors: [{ - record: 0, - message: error instanceof Error ? error.message : 'Unknown validation error', - code: 'VALIDATION_SERVICE_ERROR' - }], - metadata: { processingTimeMs: processingTime } - }; - } - } - - private async applyValidationRule(record: any, rule: ValidationRule): Promise { - const fieldValue = this.getFieldValue(record, rule.field); - - switch (rule.type) { - case 'required': - return this.validateRequired(fieldValue); - - case 'type': - return this.validateType(fieldValue, rule.value); - - case 'range': - return this.validateRange(fieldValue, rule.value); - - case 'pattern': - return this.validatePattern(fieldValue, rule.value); - - case 'custom': - return await this.validateCustom(record, rule); - - default: - throw new Error(`Unknown validation rule type: ${rule.type}`); - } - } - - private getFieldValue(record: any, fieldPath: string): any { - return fieldPath.split('.').reduce((obj, key) => obj?.[key], record); - } - - private validateRequired(value: any): boolean { - return value !== null && value !== undefined && value !== ''; - } - - private validateType(value: any, expectedType: string): boolean { - if (value === null || value === undefined) { - return false; - } - - switch (expectedType) { - case 'string': - return typeof value === 'string'; - case 'number': - return typeof value === 'number' && !isNaN(value); - case 'boolean': - return typeof value === 'boolean'; - case 'date': - return value instanceof Date || !isNaN(Date.parse(value)); - case 'array': - return Array.isArray(value); - case 'object': - return typeof value === 'object' && !Array.isArray(value); - default: - return false; - } - } - - private validateRange(value: any, range: { min?: number; max?: number }): boolean { - if (typeof value !== 'number') { - return false; - } - - if (range.min !== undefined && value < range.min) { - return false; - } - - if (range.max !== undefined && value > range.max) { - return false; - } - - return true; - } - - private validatePattern(value: any, pattern: string): boolean { - if (typeof value !== 'string') { - return false; - } - - const regex = new RegExp(pattern); - return regex.test(value); - } - - private async validateCustom(record: any, rule: ValidationRule): Promise { - const validatorName = rule.value as string; - const validator = this.validators.get(validatorName); - - if (!validator) { - throw new Error(`Custom validator not found: ${validatorName}`); - } - - return await validator(record, rule.field); - } - - private registerBuiltInValidators(): void { - // Stock symbol validator - this.validators.set('stockSymbol', (record: any, field: string) => { - const symbol = this.getFieldValue(record, field); - if (typeof symbol !== 'string') return false; - - // Basic stock symbol validation: 1-5 uppercase letters - return /^[A-Z]{1,5}$/.test(symbol); - }); - - // Price validator - this.validators.set('stockPrice', (record: any, field: string) => { - const price = this.getFieldValue(record, field); - - // Must be a positive number - return typeof price === 'number' && price > 0 && price < 1000000; - }); - - // Volume validator - this.validators.set('stockVolume', (record: any, field: string) => { - const volume = this.getFieldValue(record, field); - - // Must be a non-negative integer - return Number.isInteger(volume) && volume >= 0; - }); - - // Market data timestamp validator - this.validators.set('marketTimestamp', (record: any, field: string) => { - const timestamp = this.getFieldValue(record, field); - - if (!timestamp) return false; - - const date = new Date(timestamp); - if (isNaN(date.getTime())) return false; - - // Check if timestamp is within reasonable bounds (not too old or in future) - const now = new Date(); - const oneYearAgo = new Date(now.getTime() - 365 * 24 * 60 * 60 * 1000); - const oneHourInFuture = new Date(now.getTime() + 60 * 60 * 1000); - - return date >= oneYearAgo && date <= oneHourInFuture; - }); - - // Email validator - this.validators.set('email', (record: any, field: string) => { - const email = this.getFieldValue(record, field); - if (typeof email !== 'string') return false; - - const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - return emailRegex.test(email); - }); - - // JSON schema validator - this.validators.set('jsonSchema', (record: any, field: string, schema?: any) => { - if (!schema) return false; - - try { - const joiSchema = Joi.object(schema); - const { error } = joiSchema.validate(record); - return !error; - } catch { - return false; - } - }); - - // Data completeness validator - this.validators.set('completeness', (record: any, field: string) => { - const requiredFields = ['symbol', 'price', 'timestamp']; - return requiredFields.every(f => { - const value = this.getFieldValue(record, f); - return value !== null && value !== undefined && value !== ''; - }); - }); - } - - registerCustomValidator(name: string, validator: Function): void { - this.validators.set(name, validator); - logger.info(`โœ… Registered custom validator: ${name}`); - } - - getAvailableValidators(): string[] { - return Array.from(this.validators.keys()); - } - - async validateSchema(data: any[], schema: any): Promise { - const joiSchema = Joi.array().items(Joi.object(schema)); - const { error, value } = joiSchema.validate(data); - - if (error) { - return { - recordsProcessed: data.length, - recordsSuccessful: 0, - recordsFailed: data.length, - errors: [{ - record: 0, - message: error.message, - code: 'SCHEMA_VALIDATION_FAILED' - }], - metadata: { schemaValidation: true } - }; - } - - return { - recordsProcessed: data.length, - recordsSuccessful: data.length, - recordsFailed: 0, - errors: [], - metadata: { - schemaValidation: true, - outputData: value - } - }; - } -} diff --git a/apps/data-services/data-processor/src/types/DataPipeline.ts b/apps/data-services/data-processor/src/types/DataPipeline.ts deleted file mode 100644 index 9a95377..0000000 --- a/apps/data-services/data-processor/src/types/DataPipeline.ts +++ /dev/null @@ -1,178 +0,0 @@ -// Data Pipeline Types - -export interface DataPipeline { - id: string; - name: string; - description?: string; - status: PipelineStatus; - steps: PipelineSteps; - schedule?: PipelineSchedule; - metadata: Record; - createdAt: Date; - updatedAt: Date; -} - -export enum PipelineStatus { - DRAFT = 'draft', - ACTIVE = 'active', - PAUSED = 'paused', - DISABLED = 'disabled', -} - -export interface PipelineSteps { - ingestion?: IngestionStep; - transformation?: TransformationStep; - validation?: ValidationStep; - qualityChecks?: QualityCheckStep; -} - -export interface IngestionStep { - type: 'api' | 'file' | 'database' | 'stream'; - source: DataSource; - configuration: Record; - retryPolicy?: RetryPolicy; -} - -export interface TransformationStep { - type: 'sql' | 'javascript' | 'python' | 'custom'; - configuration: Record; - schema?: DataSchema; -} - -export interface ValidationStep { - rules: ValidationRule[]; - onFailure: 'stop' | 'continue' | 'alert'; -} - -export interface QualityCheckStep { - checks: QualityCheck[]; - thresholds: QualityThresholds; -} - -export interface PipelineSchedule { - cronExpression: string; - enabled: boolean; - lastRun: Date | null; - nextRun: Date | null; -} - -// Job Types - -export interface PipelineJob { - id: string; - pipelineId: string; - status: JobStatus; - parameters: Record; - createdAt: Date; - startedAt: Date | null; - completedAt: Date | null; - error: string | null; - metrics: JobMetrics; -} - -export enum JobStatus { - PENDING = 'pending', - RUNNING = 'running', - COMPLETED = 'completed', - FAILED = 'failed', - CANCELLED = 'cancelled', -} - -export interface JobMetrics { - recordsProcessed: number; - recordsSuccessful: number; - recordsFailed: number; - processingTimeMs: number; -} - -// Data Source Types - -export interface DataSource { - type: 'api' | 'file' | 'database' | 'stream'; - connection: ConnectionConfig; - format?: 'json' | 'csv' | 'xml' | 'parquet' | 'avro'; -} - -export interface ConnectionConfig { - url?: string; - host?: string; - port?: number; - database?: string; - username?: string; - password?: string; - apiKey?: string; - headers?: Record; - params?: Record; -} - -// Schema Types - -export interface DataSchema { - fields: SchemaField[]; - constraints?: SchemaConstraint[]; -} - -export interface SchemaField { - name: string; - type: 'string' | 'number' | 'boolean' | 'date' | 'object' | 'array'; - required: boolean; - nullable: boolean; - format?: string; - description?: string; -} - -export interface SchemaConstraint { - type: 'unique' | 'reference' | 'range' | 'pattern'; - field: string; - value: any; -} - -// Validation Types - -export interface ValidationRule { - field: string; - type: 'required' | 'type' | 'range' | 'pattern' | 'custom'; - value: any; - message?: string; -} - -// Quality Check Types - -export interface QualityCheck { - name: string; - type: 'completeness' | 'accuracy' | 'consistency' | 'validity' | 'uniqueness'; - field?: string; - condition: string; - threshold: number; -} - -export interface QualityThresholds { - error: number; // 0-100 percentage - warning: number; // 0-100 percentage -} - -// Processing Result Types - -export interface ProcessingResult { - recordsProcessed: number; - recordsSuccessful: number; - recordsFailed: number; - errors: ProcessingError[]; - metadata: Record; -} - -export interface ProcessingError { - record: number; - field?: string; - message: string; - code?: string; -} - -// Retry Policy Types - -export interface RetryPolicy { - maxAttempts: number; - backoffStrategy: 'fixed' | 'exponential' | 'linear'; - initialDelay: number; - maxDelay: number; -} diff --git a/apps/data-services/data-processor/tsconfig.json b/apps/data-services/data-processor/tsconfig.json deleted file mode 100644 index db42c0d..0000000 --- a/apps/data-services/data-processor/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "module": "ESNext", - "moduleResolution": "bundler", - "noEmit": true, - "allowImportingTsExtensions": true, - "declarationMap": true, - "types": ["bun-types"] - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"], "references": [ - { "path": "../../../libs/api-client" }, - { "path": "../../../libs/config" }, - { "path": "../../../libs/event-bus" }, - { "path": "../../../libs/http-client" }, - { "path": "../../../libs/logger" }, - { "path": "../../../libs/types" }, - { "path": "../../../libs/utils" }, - ] -} diff --git a/apps/data-services/feature-store/package.json b/apps/data-services/feature-store/package.json deleted file mode 100644 index 53f541a..0000000 --- a/apps/data-services/feature-store/package.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "feature-store", - "version": "1.0.0", - "description": "ML feature management and serving service", - "main": "src/index.ts", - "scripts": { - "dev": "bun run --watch src/index.ts", - "start": "bun run src/index.ts", - "build": "bun build src/index.ts --outdir=dist", - "test": "bun test", - "lint": "eslint src/**/*.ts", - "type-check": "tsc --noEmit" - }, - "dependencies": { - "@stock-bot/types": "*", - "@stock-bot/event-bus": "*", - "@stock-bot/utils": "*", - "@stock-bot/api-client": "*", - "hono": "^4.6.3", - "ioredis": "^5.4.1", - "node-fetch": "^3.3.2", - "date-fns": "^2.30.0", - "lodash": "^4.17.21", - "compression": "^1.7.4", - "cors": "^2.8.5", - "helmet": "^7.1.0" - }, - "devDependencies": { - "@types/bun": "latest", - "@types/lodash": "^4.14.200", - "@types/compression": "^1.7.5", - "@types/cors": "^2.8.17", - "typescript": "^5.3.0", - "eslint": "^8.55.0", - "@typescript-eslint/eslint-plugin": "^6.13.1", - "@typescript-eslint/parser": "^6.13.1" - }, - "peerDependencies": { - "typescript": "^5.0.0" - } -} diff --git a/apps/data-services/feature-store/src/controllers/ComputationController.ts b/apps/data-services/feature-store/src/controllers/ComputationController.ts deleted file mode 100644 index c712682..0000000 --- a/apps/data-services/feature-store/src/controllers/ComputationController.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { Context } from 'hono'; -import { FeatureComputationService } from '../services/FeatureComputationService'; -import { Logger } from '@stock-bot/utils'; -import { - ComputationJob, - CreateComputationJobRequest, - UpdateComputationJobRequest -} from '../types/FeatureStore'; - -export class ComputationController { - constructor( - private computationService: FeatureComputationService, - private logger: Logger - ) {} - - async createComputationJob(c: Context) { - try { - const request: CreateComputationJobRequest = await c.req.json(); - - const job = await this.computationService.createComputationJob(request); - - this.logger.info('Computation job created', { jobId: job.id }); - - return c.json({ - success: true, - data: job - }, 201); - } catch (error) { - this.logger.error('Failed to create computation job', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getComputationJob(c: Context) { - try { - const jobId = c.req.param('id'); - - const job = await this.computationService.getComputationJob(jobId); - - if (!job) { - return c.json({ - success: false, - error: 'Computation job not found' - }, 404); - } - - return c.json({ - success: true, - data: job - }); - } catch (error) { - this.logger.error('Failed to get computation job', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async updateComputationJob(c: Context) { - try { - const jobId = c.req.param('id'); - const request: UpdateComputationJobRequest = await c.req.json(); - - const job = await this.computationService.updateComputationJob(jobId, request); - - if (!job) { - return c.json({ - success: false, - error: 'Computation job not found' - }, 404); - } - - this.logger.info('Computation job updated', { jobId }); - - return c.json({ - success: true, - data: job - }); - } catch (error) { - this.logger.error('Failed to update computation job', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async deleteComputationJob(c: Context) { - try { - const jobId = c.req.param('id'); - - await this.computationService.deleteComputationJob(jobId); - - this.logger.info('Computation job deleted', { jobId }); - - return c.json({ - success: true, - message: 'Computation job deleted successfully' - }); - } catch (error) { - this.logger.error('Failed to delete computation job', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async listComputationJobs(c: Context) { - try { - const featureGroupId = c.req.query('featureGroupId'); - const status = c.req.query('status'); - - const jobs = await this.computationService.listComputationJobs({ - featureGroupId, - status: status as any - }); - - return c.json({ - success: true, - data: jobs - }); - } catch (error) { - this.logger.error('Failed to list computation jobs', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async executeComputationJob(c: Context) { - try { - const jobId = c.req.param('id'); - - const result = await this.computationService.executeComputationJob(jobId); - - this.logger.info('Computation job executed', { jobId, result }); - - return c.json({ - success: true, - data: result - }); - } catch (error) { - this.logger.error('Failed to execute computation job', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async scheduleComputationJob(c: Context) { - try { - const jobId = c.req.param('id'); - const { schedule } = await c.req.json(); - - await this.computationService.scheduleComputationJob(jobId, schedule); - - this.logger.info('Computation job scheduled', { jobId, schedule }); - - return c.json({ - success: true, - message: 'Computation job scheduled successfully' - }); - } catch (error) { - this.logger.error('Failed to schedule computation job', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async unscheduleComputationJob(c: Context) { - try { - const jobId = c.req.param('id'); - - await this.computationService.unscheduleComputationJob(jobId); - - this.logger.info('Computation job unscheduled', { jobId }); - - return c.json({ - success: true, - message: 'Computation job unscheduled successfully' - }); - } catch (error) { - this.logger.error('Failed to unschedule computation job', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getComputationJobHistory(c: Context) { - try { - const jobId = c.req.param('id'); - const limit = parseInt(c.req.query('limit') || '10'); - const offset = parseInt(c.req.query('offset') || '0'); - - const history = await this.computationService.getComputationJobHistory(jobId, limit, offset); - - return c.json({ - success: true, - data: history - }); - } catch (error) { - this.logger.error('Failed to get computation job history', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } -} diff --git a/apps/data-services/feature-store/src/controllers/FeatureController.ts b/apps/data-services/feature-store/src/controllers/FeatureController.ts deleted file mode 100644 index e3fe17e..0000000 --- a/apps/data-services/feature-store/src/controllers/FeatureController.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { Context } from 'hono'; -import { FeatureStoreService } from '../services/FeatureStoreService'; -import { Logger } from '@stock-bot/utils'; -import { - FeatureGroup, - CreateFeatureGroupRequest, - UpdateFeatureGroupRequest, - FeatureValue, - GetFeaturesRequest -} from '../types/FeatureStore'; - -export class FeatureController { - constructor( - private featureStoreService: FeatureStoreService, - private logger: Logger - ) {} - - async createFeatureGroup(c: Context) { - try { - const request: CreateFeatureGroupRequest = await c.req.json(); - - const featureGroup = await this.featureStoreService.createFeatureGroup(request); - - this.logger.info('Feature group created', { featureGroupId: featureGroup.id }); - - return c.json({ - success: true, - data: featureGroup - }, 201); - } catch (error) { - this.logger.error('Failed to create feature group', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getFeatureGroup(c: Context) { - try { - const featureGroupId = c.req.param('id'); - - const featureGroup = await this.featureStoreService.getFeatureGroup(featureGroupId); - - if (!featureGroup) { - return c.json({ - success: false, - error: 'Feature group not found' - }, 404); - } - - return c.json({ - success: true, - data: featureGroup - }); - } catch (error) { - this.logger.error('Failed to get feature group', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async updateFeatureGroup(c: Context) { - try { - const featureGroupId = c.req.param('id'); - const request: UpdateFeatureGroupRequest = await c.req.json(); - - const featureGroup = await this.featureStoreService.updateFeatureGroup(featureGroupId, request); - - if (!featureGroup) { - return c.json({ - success: false, - error: 'Feature group not found' - }, 404); - } - - this.logger.info('Feature group updated', { featureGroupId }); - - return c.json({ - success: true, - data: featureGroup - }); - } catch (error) { - this.logger.error('Failed to update feature group', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async deleteFeatureGroup(c: Context) { - try { - const featureGroupId = c.req.param('id'); - - await this.featureStoreService.deleteFeatureGroup(featureGroupId); - - this.logger.info('Feature group deleted', { featureGroupId }); - - return c.json({ - success: true, - message: 'Feature group deleted successfully' - }); - } catch (error) { - this.logger.error('Failed to delete feature group', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async listFeatureGroups(c: Context) { - try { - const featureGroups = await this.featureStoreService.listFeatureGroups(); - - return c.json({ - success: true, - data: featureGroups - }); - } catch (error) { - this.logger.error('Failed to list feature groups', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getFeatures(c: Context) { - try { - const featureGroupId = c.req.param('id'); - const entityId = c.req.query('entityId'); - const timestamp = c.req.query('timestamp'); - - if (!entityId) { - return c.json({ - success: false, - error: 'entityId query parameter is required' - }, 400); - } - - const request: GetFeaturesRequest = { - featureGroupId, - entityId, - timestamp: timestamp ? new Date(timestamp) : undefined - }; - - const features = await this.featureStoreService.getFeatures(request); - - return c.json({ - success: true, - data: features - }); - } catch (error) { - this.logger.error('Failed to get features', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async storeFeatures(c: Context) { - try { - const featureGroupId = c.req.param('id'); - const features: FeatureValue[] = await c.req.json(); - - await this.featureStoreService.storeFeatures(featureGroupId, features); - - this.logger.info('Features stored', { - featureGroupId, - featureCount: features.length - }); - - return c.json({ - success: true, - message: 'Features stored successfully' - }); - } catch (error) { - this.logger.error('Failed to store features', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getFeatureHistory(c: Context) { - try { - const featureGroupId = c.req.param('id'); - const featureName = c.req.param('featureName'); - const entityId = c.req.query('entityId'); - const startTime = c.req.query('startTime'); - const endTime = c.req.query('endTime'); - - if (!entityId) { - return c.json({ - success: false, - error: 'entityId query parameter is required' - }, 400); - } - - const history = await this.featureStoreService.getFeatureHistory( - featureGroupId, - featureName, - entityId, - startTime ? new Date(startTime) : undefined, - endTime ? new Date(endTime) : undefined - ); - - return c.json({ - success: true, - data: history - }); - } catch (error) { - this.logger.error('Failed to get feature history', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } -} diff --git a/apps/data-services/feature-store/src/controllers/HealthController.ts b/apps/data-services/feature-store/src/controllers/HealthController.ts deleted file mode 100644 index 26f50a7..0000000 --- a/apps/data-services/feature-store/src/controllers/HealthController.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { Context } from 'hono'; -import { Logger } from '@stock-bot/utils'; - -export class HealthController { - constructor(private logger: Logger) {} - - async getHealth(c: Context) { - try { - const health = { - status: 'healthy', - timestamp: new Date().toISOString(), - service: 'feature-store', - version: '1.0.0', - uptime: process.uptime(), - memory: { - used: Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100, - total: Math.round((process.memoryUsage().heapTotal / 1024 / 1024) * 100) / 100 - }, - dependencies: { - redis: await this.checkRedisHealth(), - database: await this.checkDatabaseHealth(), - eventBus: await this.checkEventBusHealth() - } - }; - - return c.json(health); - } catch (error) { - this.logger.error('Health check failed', { error }); - return c.json({ - status: 'unhealthy', - timestamp: new Date().toISOString(), - service: 'feature-store', - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getReadiness(c: Context) { - try { - const readiness = { - status: 'ready', - timestamp: new Date().toISOString(), - service: 'feature-store', - checks: { - onlineStore: await this.checkOnlineStoreReadiness(), - offlineStore: await this.checkOfflineStoreReadiness(), - metadataStore: await this.checkMetadataStoreReadiness(), - computationEngine: await this.checkComputationEngineReadiness() - } - }; - - const isReady = Object.values(readiness.checks).every(check => check.status === 'ready'); - - return c.json(readiness, isReady ? 200 : 503); - } catch (error) { - this.logger.error('Readiness check failed', { error }); - return c.json({ - status: 'not_ready', - timestamp: new Date().toISOString(), - service: 'feature-store', - error: error instanceof Error ? error.message : 'Unknown error' - }, 503); - } - } - - async getLiveness(c: Context) { - try { - const liveness = { - status: 'alive', - timestamp: new Date().toISOString(), - service: 'feature-store', - pid: process.pid, - uptime: process.uptime() - }; - - return c.json(liveness); - } catch (error) { - this.logger.error('Liveness check failed', { error }); - return c.json({ - status: 'dead', - timestamp: new Date().toISOString(), - service: 'feature-store', - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - private async checkRedisHealth(): Promise<{ status: string; latency?: number }> { - try { - const start = Date.now(); - // TODO: Implement actual Redis health check - const latency = Date.now() - start; - return { status: 'healthy', latency }; - } catch (error) { - return { status: 'unhealthy' }; - } - } - - private async checkDatabaseHealth(): Promise<{ status: string; latency?: number }> { - try { - const start = Date.now(); - // TODO: Implement actual database health check - const latency = Date.now() - start; - return { status: 'healthy', latency }; - } catch (error) { - return { status: 'unhealthy' }; - } - } - - private async checkEventBusHealth(): Promise<{ status: string }> { - try { - // TODO: Implement actual event bus health check - return { status: 'healthy' }; - } catch (error) { - return { status: 'unhealthy' }; - } - } - - private async checkOnlineStoreReadiness(): Promise<{ status: string; message?: string }> { - try { - // TODO: Implement actual online store readiness check - return { status: 'ready' }; - } catch (error) { - return { - status: 'not_ready', - message: error instanceof Error ? error.message : 'Unknown error' - }; - } - } - - private async checkOfflineStoreReadiness(): Promise<{ status: string; message?: string }> { - try { - // TODO: Implement actual offline store readiness check - return { status: 'ready' }; - } catch (error) { - return { - status: 'not_ready', - message: error instanceof Error ? error.message : 'Unknown error' - }; - } - } - - private async checkMetadataStoreReadiness(): Promise<{ status: string; message?: string }> { - try { - // TODO: Implement actual metadata store readiness check - return { status: 'ready' }; - } catch (error) { - return { - status: 'not_ready', - message: error instanceof Error ? error.message : 'Unknown error' - }; - } - } - - private async checkComputationEngineReadiness(): Promise<{ status: string; message?: string }> { - try { - // TODO: Implement actual computation engine readiness check - return { status: 'ready' }; - } catch (error) { - return { - status: 'not_ready', - message: error instanceof Error ? error.message : 'Unknown error' - }; - } - } -} diff --git a/apps/data-services/feature-store/src/controllers/MonitoringController.ts b/apps/data-services/feature-store/src/controllers/MonitoringController.ts deleted file mode 100644 index 3b74380..0000000 --- a/apps/data-services/feature-store/src/controllers/MonitoringController.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Context } from 'hono'; -import { FeatureMonitoringService } from '../services/FeatureMonitoringService'; -import { Logger } from '@stock-bot/utils'; -import { - FeatureMonitoringConfig, - FeatureValue -} from '../types/FeatureStore'; - -export class MonitoringController { - constructor( - private monitoringService: FeatureMonitoringService, - private logger: Logger - ) {} - - async startMonitoring(c: Context) { - try { - const featureGroupId = c.req.param('id'); - const config: FeatureMonitoringConfig = await c.req.json(); - - await this.monitoringService.startMonitoring(featureGroupId, config); - - this.logger.info('Monitoring started', { featureGroupId }); - - return c.json({ - success: true, - message: 'Monitoring started successfully' - }); - } catch (error) { - this.logger.error('Failed to start monitoring', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async stopMonitoring(c: Context) { - try { - const featureGroupId = c.req.param('id'); - - await this.monitoringService.stopMonitoring(featureGroupId); - - this.logger.info('Monitoring stopped', { featureGroupId }); - - return c.json({ - success: true, - message: 'Monitoring stopped successfully' - }); - } catch (error) { - this.logger.error('Failed to stop monitoring', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async detectDrift(c: Context) { - try { - const featureGroupId = c.req.param('id'); - const recentData: FeatureValue[] = await c.req.json(); - - const alerts = await this.monitoringService.detectDrift(featureGroupId, recentData); - - this.logger.info('Drift detection completed', { - featureGroupId, - alertsCount: alerts.length - }); - - return c.json({ - success: true, - data: alerts - }); - } catch (error) { - this.logger.error('Failed to detect drift', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async getMonitoringMetrics(c: Context) { - try { - const featureGroupId = c.req.param('id'); - - const metrics = await this.monitoringService.getMonitoringMetrics(featureGroupId); - - return c.json({ - success: true, - data: metrics - }); - } catch (error) { - this.logger.error('Failed to get monitoring metrics', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } - - async updateMonitoringConfig(c: Context) { - try { - const featureGroupId = c.req.param('id'); - const config: FeatureMonitoringConfig = await c.req.json(); - - await this.monitoringService.updateMonitoringConfig(featureGroupId, config); - - this.logger.info('Monitoring config updated', { featureGroupId }); - - return c.json({ - success: true, - message: 'Monitoring configuration updated successfully' - }); - } catch (error) { - this.logger.error('Failed to update monitoring config', { error }); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } - } -} diff --git a/apps/data-services/feature-store/src/index.ts b/apps/data-services/feature-store/src/index.ts deleted file mode 100644 index aa124bc..0000000 --- a/apps/data-services/feature-store/src/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Hono } from 'hono'; -import { cors } from 'hono/cors'; -import { logger } from 'hono/logger'; - -// Controllers -import { HealthController } from './controllers/HealthController'; - -const app = new Hono(); - -// Middleware -app.use('*', cors()); -app.use('*', logger()); - -// Initialize logger for services -const appLogger = { info: console.log, error: console.error, warn: console.warn, debug: console.log }; - -// Controllers -const healthController = new HealthController(appLogger); - -// Health endpoints -app.get('/health', healthController.getHealth.bind(healthController)); -app.get('/health/readiness', healthController.getReadiness.bind(healthController)); -app.get('/health/liveness', healthController.getLiveness.bind(healthController)); - -// API endpoints will be implemented as services are completed -app.get('/api/v1/feature-groups', async (c) => { - return c.json({ message: 'Feature groups endpoint - not implemented yet' }); -}); - -app.post('/api/v1/feature-groups', async (c) => { - return c.json({ message: 'Create feature group endpoint - not implemented yet' }); -}); - -const port = process.env.PORT || 3003; - -console.log(`Feature Store service running on port ${port}`); - -export default { - port, - fetch: app.fetch, -}; diff --git a/apps/data-services/feature-store/src/services/FeatureComputationService.ts b/apps/data-services/feature-store/src/services/FeatureComputationService.ts deleted file mode 100644 index 70c7bf8..0000000 --- a/apps/data-services/feature-store/src/services/FeatureComputationService.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { logger } from '@stock-bot/utils'; -import { - FeatureComputation, - ComputationStatus, - ComputationError -} from '../types/FeatureStore'; - -export class FeatureComputationService { - private computations: Map = new Map(); - private runningComputations: Set = new Set(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Feature Computation Service...'); - - this.computations.clear(); - this.runningComputations.clear(); - - logger.info('โœ… Feature Computation Service initialized'); - } - - async startComputation( - featureGroupId: string, - parameters: Record - ): Promise { - const computation: FeatureComputation = { - id: this.generateComputationId(), - featureGroupId, - status: ComputationStatus.PENDING, - startTime: new Date(), - recordsProcessed: 0, - recordsGenerated: 0, - errors: [], - metadata: parameters, - }; - - this.computations.set(computation.id, computation); - - // Start computation asynchronously - this.executeComputation(computation); - - logger.info(`โš™๏ธ Started feature computation: ${computation.id} for group: ${featureGroupId}`); - return computation; - } - - async getComputation(id: string): Promise { - return this.computations.get(id) || null; - } - - async listComputations(featureGroupId?: string): Promise { - const computations = Array.from(this.computations.values()); - return featureGroupId ? - computations.filter(c => c.featureGroupId === featureGroupId) : - computations; - } - - async cancelComputation(id: string): Promise { - const computation = this.computations.get(id); - if (!computation) { - return false; - } - - if (computation.status === ComputationStatus.RUNNING) { - computation.status = ComputationStatus.CANCELLED; - computation.endTime = new Date(); - this.runningComputations.delete(id); - - logger.info(`โŒ Cancelled computation: ${id}`); - return true; - } - - return false; - } - - private async executeComputation(computation: FeatureComputation): Promise { - try { - computation.status = ComputationStatus.RUNNING; - this.runningComputations.add(computation.id); - - logger.info(`โš™๏ธ Executing computation: ${computation.id}`); - - // Simulate computation work - const totalRecords = 1000; // Mock value - const batchSize = 100; - - for (let processed = 0; processed < totalRecords; processed += batchSize) { - // Check if computation was cancelled - if (computation.status === ComputationStatus.CANCELLED) { - return; - } - - // Simulate processing time - await new Promise(resolve => setTimeout(resolve, 100)); - - const currentBatch = Math.min(batchSize, totalRecords - processed); - computation.recordsProcessed += currentBatch; - computation.recordsGenerated += currentBatch; // Assume 1:1 for simplicity - - // Simulate some errors - if (Math.random() < 0.05) { // 5% error rate - const error: ComputationError = { - entityId: `entity_${processed}`, - error: 'Simulated processing error', - timestamp: new Date(), - }; - computation.errors.push(error); - } - } - - computation.status = ComputationStatus.COMPLETED; - computation.endTime = new Date(); - this.runningComputations.delete(computation.id); - - logger.info(`โœ… Completed computation: ${computation.id}`); - - } catch (error) { - computation.status = ComputationStatus.FAILED; - computation.endTime = new Date(); - this.runningComputations.delete(computation.id); - - const computationError: ComputationError = { - entityId: 'unknown', - error: error instanceof Error ? error.message : 'Unknown error', - timestamp: new Date(), - }; - computation.errors.push(computationError); - - logger.error(`โŒ Computation failed: ${computation.id}`, error); - } - } - - private generateComputationId(): string { - return `comp_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - async getComputationStats(): Promise { - const computations = Array.from(this.computations.values()); - - return { - total: computations.length, - running: this.runningComputations.size, - byStatus: { - pending: computations.filter(c => c.status === ComputationStatus.PENDING).length, - running: computations.filter(c => c.status === ComputationStatus.RUNNING).length, - completed: computations.filter(c => c.status === ComputationStatus.COMPLETED).length, - failed: computations.filter(c => c.status === ComputationStatus.FAILED).length, - cancelled: computations.filter(c => c.status === ComputationStatus.CANCELLED).length, - }, - totalRecordsProcessed: computations.reduce((sum, c) => sum + c.recordsProcessed, 0), - totalRecordsGenerated: computations.reduce((sum, c) => sum + c.recordsGenerated, 0), - totalErrors: computations.reduce((sum, c) => sum + c.errors.length, 0), - }; - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Feature Computation Service...'); - - // Cancel all running computations - for (const computationId of this.runningComputations) { - await this.cancelComputation(computationId); - } - - this.computations.clear(); - this.runningComputations.clear(); - - logger.info('โœ… Feature Computation Service shutdown complete'); - } -} diff --git a/apps/data-services/feature-store/src/services/FeatureMonitoringService.ts b/apps/data-services/feature-store/src/services/FeatureMonitoringService.ts deleted file mode 100644 index 9669845..0000000 --- a/apps/data-services/feature-store/src/services/FeatureMonitoringService.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { EventBus } from '@stock-bot/event-bus'; -import { Logger } from '@stock-bot/utils'; -import { - FeatureGroup, - FeatureDriftAlert, - FeatureMonitoringConfig, - FeatureMonitoringMetrics, - FeatureValue, - DriftDetectionMethod -} from '../types/FeatureStore'; - -export interface FeatureMonitoringService { - startMonitoring(featureGroupId: string, config: FeatureMonitoringConfig): Promise; - stopMonitoring(featureGroupId: string): Promise; - detectDrift(featureGroupId: string, recentData: FeatureValue[]): Promise; - getMonitoringMetrics(featureGroupId: string): Promise; - updateMonitoringConfig(featureGroupId: string, config: FeatureMonitoringConfig): Promise; -} - -export class FeatureMonitoringServiceImpl implements FeatureMonitoringService { - private monitoringJobs: Map = new Map(); - private baselineStats: Map = new Map(); - - constructor( - private eventBus: EventBus, - private logger: Logger - ) {} - - async startMonitoring(featureGroupId: string, config: FeatureMonitoringConfig): Promise { - try { - // Stop existing monitoring if running - await this.stopMonitoring(featureGroupId); - - // Start new monitoring job - const interval = setInterval(async () => { - await this.runMonitoringCheck(featureGroupId, config); - }, config.checkInterval * 1000); - - this.monitoringJobs.set(featureGroupId, interval); - - this.logger.info(`Started monitoring for feature group: ${featureGroupId}`); - - await this.eventBus.emit('feature.monitoring.started', { - featureGroupId, - config, - timestamp: new Date() - }); - } catch (error) { - this.logger.error('Failed to start feature monitoring', { featureGroupId, error }); - throw error; - } - } - - async stopMonitoring(featureGroupId: string): Promise { - try { - const job = this.monitoringJobs.get(featureGroupId); - if (job) { - clearInterval(job); - this.monitoringJobs.delete(featureGroupId); - - this.logger.info(`Stopped monitoring for feature group: ${featureGroupId}`); - - await this.eventBus.emit('feature.monitoring.stopped', { - featureGroupId, - timestamp: new Date() - }); - } - } catch (error) { - this.logger.error('Failed to stop feature monitoring', { featureGroupId, error }); - throw error; - } - } - - async detectDrift(featureGroupId: string, recentData: FeatureValue[]): Promise { - try { - const alerts: FeatureDriftAlert[] = []; - const baseline = this.baselineStats.get(featureGroupId); - - if (!baseline) { - // No baseline available, collect current data as baseline - await this.updateBaseline(featureGroupId, recentData); - return alerts; - } - - // Group data by feature name - const featureData = this.groupByFeature(recentData); - - for (const [featureName, values] of featureData) { - const currentStats = this.calculateStatistics(values); - const baselineFeatureStats = baseline[featureName]; - - if (!baselineFeatureStats) continue; - - // Detect drift using various methods - const driftScore = await this.calculateDriftScore( - baselineFeatureStats, - currentStats, - DriftDetectionMethod.KOLMOGOROV_SMIRNOV - ); - - if (driftScore > 0.1) { // Threshold for drift detection - alerts.push({ - id: `drift_${featureGroupId}_${featureName}_${Date.now()}`, - featureGroupId, - featureName, - driftScore, - severity: driftScore > 0.3 ? 'high' : driftScore > 0.2 ? 'medium' : 'low', - detectionMethod: DriftDetectionMethod.KOLMOGOROV_SMIRNOV, - baselineStats: baselineFeatureStats, - currentStats, - detectedAt: new Date(), - message: `Feature drift detected for ${featureName} with score ${driftScore.toFixed(3)}` - }); - } - } - - if (alerts.length > 0) { - await this.eventBus.emit('feature.drift.detected', { - featureGroupId, - alerts, - timestamp: new Date() - }); - } - - return alerts; - } catch (error) { - this.logger.error('Failed to detect drift', { featureGroupId, error }); - throw error; - } - } - - async getMonitoringMetrics(featureGroupId: string): Promise { - try { - const isActive = this.monitoringJobs.has(featureGroupId); - const baseline = this.baselineStats.get(featureGroupId); - - return { - featureGroupId, - isActive, - lastCheckTime: new Date(), - totalChecks: 0, // Would be stored in persistent storage - driftAlertsCount: 0, // Would be queried from alert storage - averageDriftScore: 0, - featuresMonitored: baseline ? Object.keys(baseline).length : 0, - uptime: isActive ? Date.now() : 0 // Would calculate actual uptime - }; - } catch (error) { - this.logger.error('Failed to get monitoring metrics', { featureGroupId, error }); - throw error; - } - } - - async updateMonitoringConfig(featureGroupId: string, config: FeatureMonitoringConfig): Promise { - try { - // Restart monitoring with new config - if (this.monitoringJobs.has(featureGroupId)) { - await this.stopMonitoring(featureGroupId); - await this.startMonitoring(featureGroupId, config); - } - - this.logger.info(`Updated monitoring config for feature group: ${featureGroupId}`); - } catch (error) { - this.logger.error('Failed to update monitoring config', { featureGroupId, error }); - throw error; - } - } - - private async runMonitoringCheck(featureGroupId: string, config: FeatureMonitoringConfig): Promise { - try { - // In a real implementation, this would fetch recent data from the feature store - const recentData: FeatureValue[] = []; // Placeholder - - await this.detectDrift(featureGroupId, recentData); - } catch (error) { - this.logger.error('Monitoring check failed', { featureGroupId, error }); - } - } - - private async updateBaseline(featureGroupId: string, data: FeatureValue[]): Promise { - const featureData = this.groupByFeature(data); - const baseline: Record = {}; - - for (const [featureName, values] of featureData) { - baseline[featureName] = this.calculateStatistics(values); - } - - this.baselineStats.set(featureGroupId, baseline); - } - - private groupByFeature(data: FeatureValue[]): Map { - const grouped = new Map(); - - for (const item of data) { - if (!grouped.has(item.featureName)) { - grouped.set(item.featureName, []); - } - grouped.get(item.featureName)!.push(item.value as number); - } - - return grouped; - } - - private calculateStatistics(values: number[]): any { - const sorted = values.sort((a, b) => a - b); - const n = values.length; - const mean = values.reduce((sum, val) => sum + val, 0) / n; - const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / n; - const stdDev = Math.sqrt(variance); - - return { - count: n, - mean, - stdDev, - min: sorted[0], - max: sorted[n - 1], - median: n % 2 === 0 ? (sorted[n/2 - 1] + sorted[n/2]) / 2 : sorted[Math.floor(n/2)], - q25: sorted[Math.floor(n * 0.25)], - q75: sorted[Math.floor(n * 0.75)] - }; - } - - private async calculateDriftScore( - baseline: any, - current: any, - method: DriftDetectionMethod - ): Promise { - switch (method) { - case DriftDetectionMethod.KOLMOGOROV_SMIRNOV: - // Simplified KS test approximation - return Math.abs(baseline.mean - current.mean) / (baseline.stdDev + current.stdDev + 1e-8); - - case DriftDetectionMethod.POPULATION_STABILITY_INDEX: - // Simplified PSI calculation - const expectedRatio = baseline.mean / (baseline.mean + current.mean + 1e-8); - const actualRatio = current.mean / (baseline.mean + current.mean + 1e-8); - return Math.abs(expectedRatio - actualRatio); - - case DriftDetectionMethod.JENSEN_SHANNON_DIVERGENCE: - // Simplified JS divergence approximation - return Math.min(1.0, Math.abs(baseline.mean - current.mean) / Math.max(baseline.stdDev, current.stdDev, 1e-8)); - - default: - return 0; - } - } -} diff --git a/apps/data-services/feature-store/src/services/FeatureStatisticsService.ts b/apps/data-services/feature-store/src/services/FeatureStatisticsService.ts deleted file mode 100644 index b5202a5..0000000 --- a/apps/data-services/feature-store/src/services/FeatureStatisticsService.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { logger } from '@stock-bot/utils'; -import { FeatureStatistics, HistogramBucket, ValueCount } from '../types/FeatureStore'; - -export class FeatureStatisticsService { - private statistics: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Feature Statistics Service...'); - - this.statistics.clear(); - - logger.info('โœ… Feature Statistics Service initialized'); - } - - async computeStatistics( - featureGroupId: string, - featureName: string, - data: any[] - ): Promise { - const values = data.map(item => item[featureName]).filter(v => v !== null && v !== undefined); - - const statistics: FeatureStatistics = { - featureGroupId, - featureName, - statistics: { - count: data.length, - nullCount: data.length - values.length, - distinctCount: new Set(values).size, - }, - computedAt: new Date(), - }; - - // Compute numerical statistics if applicable - const numericalValues = values.filter(v => typeof v === 'number'); - if (numericalValues.length > 0) { - const sorted = numericalValues.sort((a, b) => a - b); - const sum = numericalValues.reduce((acc, val) => acc + val, 0); - const mean = sum / numericalValues.length; - - statistics.statistics.min = sorted[0]; - statistics.statistics.max = sorted[sorted.length - 1]; - statistics.statistics.mean = mean; - statistics.statistics.median = this.calculateMedian(sorted); - statistics.statistics.stdDev = this.calculateStandardDeviation(numericalValues, mean); - statistics.statistics.percentiles = this.calculatePercentiles(sorted); - statistics.statistics.histogram = this.calculateHistogram(numericalValues); - } - - // Compute top values for categorical data - statistics.statistics.topValues = this.calculateTopValues(values); - - const key = `${featureGroupId}.${featureName}`; - this.statistics.set(key, statistics); - - logger.info(`๐Ÿ“Š Computed statistics for feature: ${featureGroupId}.${featureName}`); - return statistics; - } - - async getStatistics(featureGroupId: string, featureName: string): Promise { - const key = `${featureGroupId}.${featureName}`; - return this.statistics.get(key) || null; - } - - async getFeatureGroupStatistics(featureGroupId: string): Promise { - const groupStats: FeatureStatistics[] = []; - - for (const [key, stats] of this.statistics.entries()) { - if (stats.featureGroupId === featureGroupId) { - groupStats.push(stats); - } - } - - return groupStats; - } - - async getAllStatistics(): Promise { - return Array.from(this.statistics.values()); - } - - async deleteStatistics(featureGroupId: string, featureName?: string): Promise { - if (featureName) { - const key = `${featureGroupId}.${featureName}`; - this.statistics.delete(key); - } else { - // Delete all statistics for the feature group - const keysToDelete: string[] = []; - for (const [key, stats] of this.statistics.entries()) { - if (stats.featureGroupId === featureGroupId) { - keysToDelete.push(key); - } - } - - for (const key of keysToDelete) { - this.statistics.delete(key); - } - } - } - - private calculateMedian(sortedValues: number[]): number { - const length = sortedValues.length; - if (length % 2 === 0) { - return (sortedValues[length / 2 - 1] + sortedValues[length / 2]) / 2; - } else { - return sortedValues[Math.floor(length / 2)]; - } - } - - private calculateStandardDeviation(values: number[], mean: number): number { - const squaredDifferences = values.map(value => Math.pow(value - mean, 2)); - const avgSquaredDiff = squaredDifferences.reduce((acc, val) => acc + val, 0) / values.length; - return Math.sqrt(avgSquaredDiff); - } - - private calculatePercentiles(sortedValues: number[]): Record { - const percentiles = [5, 10, 25, 50, 75, 90, 95]; - const result: Record = {}; - - for (const p of percentiles) { - const index = (p / 100) * (sortedValues.length - 1); - if (Number.isInteger(index)) { - result[`p${p}`] = sortedValues[index]; - } else { - const lower = Math.floor(index); - const upper = Math.ceil(index); - const weight = index - lower; - result[`p${p}`] = sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight; - } - } - - return result; - } - - private calculateHistogram(values: number[], buckets: number = 10): HistogramBucket[] { - const min = Math.min(...values); - const max = Math.max(...values); - const bucketSize = (max - min) / buckets; - - const histogram: HistogramBucket[] = []; - - for (let i = 0; i < buckets; i++) { - const bucketMin = min + i * bucketSize; - const bucketMax = i === buckets - 1 ? max : min + (i + 1) * bucketSize; - - const count = values.filter(v => v >= bucketMin && v < bucketMax).length; - - histogram.push({ - min: bucketMin, - max: bucketMax, - count, - }); - } - - return histogram; - } - - private calculateTopValues(values: any[], limit: number = 10): ValueCount[] { - const valueCounts = new Map(); - - for (const value of values) { - valueCounts.set(value, (valueCounts.get(value) || 0) + 1); - } - - const sortedCounts = Array.from(valueCounts.entries()) - .map(([value, count]) => ({ - value, - count, - percentage: (count / values.length) * 100, - })) - .sort((a, b) => b.count - a.count) - .slice(0, limit); - - return sortedCounts; - } - - async getStatisticsSummary(): Promise { - const allStats = Array.from(this.statistics.values()); - - return { - totalFeatures: allStats.length, - totalRecords: allStats.reduce((sum, s) => sum + s.statistics.count, 0), - totalNullValues: allStats.reduce((sum, s) => sum + s.statistics.nullCount, 0), - featureGroups: new Set(allStats.map(s => s.featureGroupId)).size, - lastComputed: allStats.length > 0 ? - Math.max(...allStats.map(s => s.computedAt.getTime())) : null, - }; - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Feature Statistics Service...'); - - this.statistics.clear(); - - logger.info('โœ… Feature Statistics Service shutdown complete'); - } -} diff --git a/apps/data-services/feature-store/src/services/FeatureStoreService.ts b/apps/data-services/feature-store/src/services/FeatureStoreService.ts deleted file mode 100644 index 842aca7..0000000 --- a/apps/data-services/feature-store/src/services/FeatureStoreService.ts +++ /dev/null @@ -1,313 +0,0 @@ -import { EventBus } from '@stock-bot/event-bus'; -import { logger } from '@stock-bot/utils'; -import { - FeatureGroup, - FeatureGroupStatus, - FeatureVector, - FeatureRequest, - FeatureResponse, - FeatureStorageConfig, - FeatureRegistry -} from '../types/FeatureStore'; -import { OnlineStore } from './storage/OnlineStore'; -import { OfflineStore } from './storage/OfflineStore'; -import { MetadataStore } from './storage/MetadataStore'; - -export class FeatureStoreService { - private eventBus: EventBus; - private onlineStore: OnlineStore; - private offlineStore: OfflineStore; - private metadataStore: MetadataStore; - private registry: FeatureRegistry; - - constructor() { - this.eventBus = new EventBus(); - this.onlineStore = new OnlineStore(); - this.offlineStore = new OfflineStore(); - this.metadataStore = new MetadataStore(); - this.registry = { - featureGroups: new Map(), - features: new Map(), - dependencies: new Map(), - lineage: new Map() - }; - } - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Feature Store Service...'); - - await this.eventBus.initialize(); - await this.onlineStore.initialize(); - await this.offlineStore.initialize(); - await this.metadataStore.initialize(); - - // Load existing feature groups from metadata store - await this.loadFeatureGroups(); - - // Subscribe to feature events - await this.eventBus.subscribe('feature.*', this.handleFeatureEvent.bind(this)); - - logger.info('โœ… Feature Store Service initialized'); - } - - async createFeatureGroup(featureGroup: Omit): Promise { - const featureGroupWithId: FeatureGroup = { - ...featureGroup, - id: this.generateFeatureGroupId(), - status: FeatureGroupStatus.DRAFT, - createdAt: new Date(), - updatedAt: new Date(), - }; - - // Store in metadata store - await this.metadataStore.saveFeatureGroup(featureGroupWithId); - - // Update registry - this.registry.featureGroups.set(featureGroupWithId.id, featureGroupWithId); - - // Register individual features - for (const feature of featureGroupWithId.features) { - const featureKey = `${featureGroupWithId.id}.${feature.name}`; - this.registry.features.set(featureKey, feature); - } - - await this.eventBus.publish('feature.group.created', { - featureGroupId: featureGroupWithId.id, - featureGroup: featureGroupWithId, - }); - - logger.info(`๐Ÿ“‹ Created feature group: ${featureGroupWithId.name} (${featureGroupWithId.id})`); - return featureGroupWithId; - } - - async updateFeatureGroup(id: string, updates: Partial): Promise { - const existingGroup = this.registry.featureGroups.get(id); - if (!existingGroup) { - throw new Error(`Feature group not found: ${id}`); - } - - const updatedGroup: FeatureGroup = { - ...existingGroup, - ...updates, - id, // Ensure ID doesn't change - updatedAt: new Date(), - }; - - // Store in metadata store - await this.metadataStore.saveFeatureGroup(updatedGroup); - - // Update registry - this.registry.featureGroups.set(id, updatedGroup); - - await this.eventBus.publish('feature.group.updated', { - featureGroupId: id, - featureGroup: updatedGroup, - }); - - logger.info(`๐Ÿ“ Updated feature group: ${updatedGroup.name} (${id})`); - return updatedGroup; - } - - async deleteFeatureGroup(id: string): Promise { - const featureGroup = this.registry.featureGroups.get(id); - if (!featureGroup) { - throw new Error(`Feature group not found: ${id}`); - } - - // Remove from stores - await this.metadataStore.deleteFeatureGroup(id); - await this.onlineStore.deleteFeatureGroup(id); - await this.offlineStore.deleteFeatureGroup(id); - - // Update registry - this.registry.featureGroups.delete(id); - - // Remove features from registry - for (const feature of featureGroup.features) { - const featureKey = `${id}.${feature.name}`; - this.registry.features.delete(featureKey); - } - - await this.eventBus.publish('feature.group.deleted', { - featureGroupId: id, - featureGroup, - }); - - logger.info(`๐Ÿ—‘๏ธ Deleted feature group: ${featureGroup.name} (${id})`); - } - - async getFeatureGroup(id: string): Promise { - return this.registry.featureGroups.get(id) || null; - } - - async listFeatureGroups(status?: FeatureGroupStatus): Promise { - const groups = Array.from(this.registry.featureGroups.values()); - return status ? groups.filter(group => group.status === status) : groups; - } - - async getOnlineFeatures(request: FeatureRequest): Promise { - logger.info(`๐Ÿ” Getting online features for ${request.entityIds.length} entities`); - - const responses: FeatureResponse[] = []; - - for (const entityId of request.entityIds) { - const features: Record = {}; - const metadata: Record = {}; - - for (const featureGroupId of request.featureGroups) { - const featureGroup = this.registry.featureGroups.get(featureGroupId); - if (!featureGroup) { - logger.warn(`Feature group not found: ${featureGroupId}`); - continue; - } - - const featureVector = await this.onlineStore.getFeatures( - entityId, - request.entityType, - featureGroupId, - request.asOfTime - ); - - if (featureVector) { - Object.assign(features, featureVector.values); - if (request.includeMetadata) { - metadata[featureGroupId] = featureVector.metadata; - } - } - } - - responses.push({ - entityId, - entityType: request.entityType, - features, - metadata, - timestamp: request.asOfTime || new Date(), - }); - } - - return responses; - } - - async getHistoricalFeatures(request: FeatureRequest): Promise { - logger.info(`๐Ÿ“Š Getting historical features for ${request.entityIds.length} entities`); - - return await this.offlineStore.getHistoricalFeatures(request); - } - - async getBatchFeatures(request: FeatureRequest): Promise { - logger.info(`๐Ÿ“ฆ Getting batch features for ${request.entityIds.length} entities`); - - // For batch requests, use offline store for efficiency - return await this.offlineStore.getBatchFeatures(request); - } - - async ingestFeatures(featureVectors: FeatureVector[]): Promise { - logger.info(`๐Ÿ“ฅ Ingesting ${featureVectors.length} feature vectors`); - - // Store in both online and offline stores - await Promise.all([ - this.onlineStore.writeFeatures(featureVectors), - this.offlineStore.writeFeatures(featureVectors) - ]); - - await this.eventBus.publish('feature.ingested', { - vectorCount: featureVectors.length, - timestamp: new Date(), - }); - } - - async searchFeatures(query: string, filters?: Record): Promise { - const results: any[] = []; - - for (const [groupId, group] of this.registry.featureGroups) { - for (const feature of group.features) { - const featureInfo = { - featureGroupId: groupId, - featureGroupName: group.name, - featureName: feature.name, - description: feature.description, - type: feature.type, - valueType: feature.valueType, - tags: feature.tags, - }; - - // Simple text search - const searchText = `${group.name} ${feature.name} ${feature.description || ''} ${feature.tags.join(' ')}`.toLowerCase(); - if (searchText.includes(query.toLowerCase())) { - // Apply filters if provided - if (filters) { - let matches = true; - for (const [key, value] of Object.entries(filters)) { - if (featureInfo[key as keyof typeof featureInfo] !== value) { - matches = false; - break; - } - } - if (matches) { - results.push(featureInfo); - } - } else { - results.push(featureInfo); - } - } - } - } - - return results; - } - - async getFeatureLineage(featureGroupId: string, featureName: string): Promise { - const lineageKey = `${featureGroupId}.${featureName}`; - return this.registry.lineage.get(lineageKey) || null; - } - - async getFeatureUsage(featureGroupId: string, featureName: string): Promise { - // In a real implementation, this would track feature usage across models and applications - return { - featureGroupId, - featureName, - usageCount: 0, - lastUsed: null, - consumers: [], - models: [] - }; - } - - private async loadFeatureGroups(): Promise { - logger.info('๐Ÿ“‚ Loading existing feature groups...'); - - const featureGroups = await this.metadataStore.getAllFeatureGroups(); - - for (const group of featureGroups) { - this.registry.featureGroups.set(group.id, group); - - // Register individual features - for (const feature of group.features) { - const featureKey = `${group.id}.${feature.name}`; - this.registry.features.set(featureKey, feature); - } - } - - logger.info(`๐Ÿ“‚ Loaded ${featureGroups.length} feature groups`); - } - - private async handleFeatureEvent(event: any): Promise { - logger.debug('๐Ÿ“จ Received feature event:', event); - // Handle feature-level events - } - - private generateFeatureGroupId(): string { - return `fg_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Feature Store Service...'); - - await this.onlineStore.shutdown(); - await this.offlineStore.shutdown(); - await this.metadataStore.shutdown(); - await this.eventBus.disconnect(); - - logger.info('โœ… Feature Store Service shutdown complete'); - } -} diff --git a/apps/data-services/feature-store/src/services/storage/MetadataStore.ts b/apps/data-services/feature-store/src/services/storage/MetadataStore.ts deleted file mode 100644 index 72a0b79..0000000 --- a/apps/data-services/feature-store/src/services/storage/MetadataStore.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { logger } from '@stock-bot/utils'; -import { FeatureGroup } from '../../types/FeatureStore'; - -export class MetadataStore { - private featureGroups: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Metadata Store...'); - - // In a real implementation, connect to PostgreSQL or other metadata store - this.featureGroups.clear(); - - logger.info('โœ… Metadata Store initialized'); - } - - async saveFeatureGroup(featureGroup: FeatureGroup): Promise { - this.featureGroups.set(featureGroup.id, { ...featureGroup }); - logger.debug(`๐Ÿ’พ Saved feature group metadata: ${featureGroup.id}`); - } - - async getFeatureGroup(id: string): Promise { - return this.featureGroups.get(id) || null; - } - - async getAllFeatureGroups(): Promise { - return Array.from(this.featureGroups.values()); - } - - async deleteFeatureGroup(id: string): Promise { - this.featureGroups.delete(id); - logger.debug(`๐Ÿ—‘๏ธ Deleted feature group metadata: ${id}`); - } - - async findFeatureGroups(criteria: Partial): Promise { - const groups = Array.from(this.featureGroups.values()); - - return groups.filter(group => { - for (const [key, value] of Object.entries(criteria)) { - if (group[key as keyof FeatureGroup] !== value) { - return false; - } - } - return true; - }); - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Metadata Store...'); - this.featureGroups.clear(); - logger.info('โœ… Metadata Store shutdown complete'); - } -} diff --git a/apps/data-services/feature-store/src/services/storage/OfflineStore.ts b/apps/data-services/feature-store/src/services/storage/OfflineStore.ts deleted file mode 100644 index 3966400..0000000 --- a/apps/data-services/feature-store/src/services/storage/OfflineStore.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { logger } from '@stock-bot/utils'; -import { FeatureVector, FeatureRequest, FeatureResponse } from '../../types/FeatureStore'; - -export class OfflineStore { - private store: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Offline Store...'); - - // In a real implementation, connect to data warehouse, S3, etc. - this.store.clear(); - - logger.info('โœ… Offline Store initialized'); - } - - async writeFeatures(featureVectors: FeatureVector[]): Promise { - for (const vector of featureVectors) { - const partitionKey = this.buildPartitionKey(vector.entityType, vector.featureGroupId); - - if (!this.store.has(partitionKey)) { - this.store.set(partitionKey, []); - } - - this.store.get(partitionKey)!.push(vector); - } - - logger.debug(`๐Ÿ’พ Stored ${featureVectors.length} feature vectors in offline store`); - } - - async getHistoricalFeatures(request: FeatureRequest): Promise { - const responses: FeatureResponse[] = []; - - for (const entityId of request.entityIds) { - const features: Record = {}; - const metadata: Record = {}; - - for (const featureGroupId of request.featureGroups) { - const partitionKey = this.buildPartitionKey(request.entityType, featureGroupId); - const vectors = this.store.get(partitionKey) || []; - - // Find the most recent vector for this entity before asOfTime - const relevantVectors = vectors - .filter(v => v.entityId === entityId) - .filter(v => !request.asOfTime || v.timestamp <= request.asOfTime) - .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); - - if (relevantVectors.length > 0) { - const latestVector = relevantVectors[0]; - Object.assign(features, latestVector.values); - - if (request.includeMetadata) { - metadata[featureGroupId] = latestVector.metadata; - } - } - } - - responses.push({ - entityId, - entityType: request.entityType, - features, - metadata, - timestamp: request.asOfTime || new Date(), - }); - } - - return responses; - } - - async getBatchFeatures(request: FeatureRequest): Promise { - // For simplicity, use the same logic as historical features - // In a real implementation, this would use optimized batch processing - return await this.getHistoricalFeatures(request); - } - - async getFeatureData( - featureGroupId: string, - entityType: string, - startTime?: Date, - endTime?: Date - ): Promise { - const partitionKey = this.buildPartitionKey(entityType, featureGroupId); - let vectors = this.store.get(partitionKey) || []; - - // Apply time filters - if (startTime) { - vectors = vectors.filter(v => v.timestamp >= startTime); - } - - if (endTime) { - vectors = vectors.filter(v => v.timestamp <= endTime); - } - - return vectors; - } - - async deleteFeatureGroup(featureGroupId: string): Promise { - const keysToDelete: string[] = []; - - for (const key of this.store.keys()) { - if (key.includes(`:${featureGroupId}`)) { - keysToDelete.push(key); - } - } - - for (const key of keysToDelete) { - this.store.delete(key); - } - - logger.debug(`๐Ÿ—‘๏ธ Deleted ${keysToDelete.length} partitions for feature group: ${featureGroupId}`); - } - - private buildPartitionKey(entityType: string, featureGroupId: string): string { - return `${entityType}:${featureGroupId}`; - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Offline Store...'); - this.store.clear(); - logger.info('โœ… Offline Store shutdown complete'); - } -} diff --git a/apps/data-services/feature-store/src/services/storage/OnlineStore.ts b/apps/data-services/feature-store/src/services/storage/OnlineStore.ts deleted file mode 100644 index 599a8d5..0000000 --- a/apps/data-services/feature-store/src/services/storage/OnlineStore.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { logger } from '@stock-bot/utils'; -import { FeatureVector, FeatureRequest, FeatureResponse } from '../../types/FeatureStore'; - -export class OnlineStore { - private store: Map = new Map(); - - async initialize(): Promise { - logger.info('๐Ÿ”„ Initializing Online Store...'); - - // In a real implementation, connect to Redis or other online store - this.store.clear(); - - logger.info('โœ… Online Store initialized'); - } - - async writeFeatures(featureVectors: FeatureVector[]): Promise { - for (const vector of featureVectors) { - const key = this.buildKey(vector.entityId, vector.entityType, vector.featureGroupId); - - this.store.set(key, { - ...vector, - timestamp: vector.timestamp, - }); - } - - logger.debug(`๐Ÿ’พ Stored ${featureVectors.length} feature vectors in online store`); - } - - async getFeatures( - entityId: string, - entityType: string, - featureGroupId: string, - asOfTime?: Date - ): Promise { - const key = this.buildKey(entityId, entityType, featureGroupId); - const storedVector = this.store.get(key); - - if (!storedVector) { - return null; - } - - // If asOfTime is specified, check if the stored vector is valid at that time - if (asOfTime && storedVector.timestamp > asOfTime) { - return null; - } - - return storedVector; - } - - async deleteFeatureGroup(featureGroupId: string): Promise { - const keysToDelete: string[] = []; - - for (const key of this.store.keys()) { - if (key.includes(`:${featureGroupId}`)) { - keysToDelete.push(key); - } - } - - for (const key of keysToDelete) { - this.store.delete(key); - } - - logger.debug(`๐Ÿ—‘๏ธ Deleted ${keysToDelete.length} records for feature group: ${featureGroupId}`); - } - - private buildKey(entityId: string, entityType: string, featureGroupId: string): string { - return `${entityType}:${entityId}:${featureGroupId}`; - } - - async shutdown(): Promise { - logger.info('๐Ÿ”„ Shutting down Online Store...'); - this.store.clear(); - logger.info('โœ… Online Store shutdown complete'); - } -} diff --git a/apps/data-services/feature-store/src/types/FeatureStore.ts b/apps/data-services/feature-store/src/types/FeatureStore.ts deleted file mode 100644 index decaf51..0000000 --- a/apps/data-services/feature-store/src/types/FeatureStore.ts +++ /dev/null @@ -1,243 +0,0 @@ -// Feature Store Types - -export interface FeatureGroup { - id: string; - name: string; - description?: string; - version: string; - features: Feature[]; - source: FeatureSource; - schedule?: FeatureSchedule; - metadata: Record; - createdAt: Date; - updatedAt: Date; - status: FeatureGroupStatus; -} - -export enum FeatureGroupStatus { - DRAFT = 'draft', - ACTIVE = 'active', - DEPRECATED = 'deprecated', - ARCHIVED = 'archived', -} - -export interface Feature { - name: string; - type: FeatureType; - description?: string; - valueType: 'number' | 'string' | 'boolean' | 'array' | 'object'; - nullable: boolean; - defaultValue?: any; - validation?: FeatureValidation; - transformation?: FeatureTransformation; - tags: string[]; -} - -export enum FeatureType { - NUMERICAL = 'numerical', - CATEGORICAL = 'categorical', - BOOLEAN = 'boolean', - TEXT = 'text', - TIMESTAMP = 'timestamp', - DERIVED = 'derived', -} - -export interface FeatureSource { - type: 'batch' | 'stream' | 'sql' | 'api' | 'file'; - connection: Record; - query?: string; - transformation?: string; - refreshInterval?: number; -} - -export interface FeatureSchedule { - cronExpression: string; - enabled: boolean; - lastRun: Date | null; - nextRun: Date | null; -} - -export interface FeatureValidation { - required: boolean; - minValue?: number; - maxValue?: number; - allowedValues?: any[]; - pattern?: string; - customValidator?: string; -} - -export interface FeatureTransformation { - type: 'normalize' | 'standardize' | 'encode' | 'custom'; - parameters: Record; -} - -// Feature Value Types - -export interface FeatureVector { - entityId: string; - entityType: string; - featureGroupId: string; - timestamp: Date; - values: Record; - metadata?: Record; -} - -export interface FeatureRequest { - entityIds: string[]; - entityType: string; - featureGroups: string[]; - asOfTime?: Date; - pointInTime?: boolean; - includeMetadata?: boolean; -} - -export interface FeatureResponse { - entityId: string; - entityType: string; - features: Record; - metadata: Record; - timestamp: Date; -} - -// Feature Store Operations - -export interface FeatureComputation { - id: string; - featureGroupId: string; - status: ComputationStatus; - startTime: Date; - endTime?: Date; - recordsProcessed: number; - recordsGenerated: number; - errors: ComputationError[]; - metadata: Record; -} - -export enum ComputationStatus { - PENDING = 'pending', - RUNNING = 'running', - COMPLETED = 'completed', - FAILED = 'failed', - CANCELLED = 'cancelled', -} - -export interface ComputationError { - entityId: string; - error: string; - timestamp: Date; -} - -// Feature Statistics - -export interface FeatureStatistics { - featureGroupId: string; - featureName: string; - statistics: { - count: number; - nullCount: number; - distinctCount: number; - min?: number; - max?: number; - mean?: number; - median?: number; - stdDev?: number; - percentiles?: Record; - histogram?: HistogramBucket[]; - topValues?: ValueCount[]; - }; - computedAt: Date; -} - -export interface HistogramBucket { - min: number; - max: number; - count: number; -} - -export interface ValueCount { - value: any; - count: number; - percentage: number; -} - -// Feature Registry - -export interface FeatureRegistry { - featureGroups: Map; - features: Map; - dependencies: Map; - lineage: Map; -} - -export interface FeatureLineage { - featureGroupId: string; - featureName: string; - upstream: FeatureDependency[]; - downstream: FeatureDependency[]; - transformations: string[]; -} - -export interface FeatureDependency { - featureGroupId: string; - featureName: string; - dependencyType: 'direct' | 'derived' | 'aggregated'; -} - -// Storage Types - -export interface FeatureStorageConfig { - online: OnlineStoreConfig; - offline: OfflineStoreConfig; - metadata: MetadataStoreConfig; -} - -export interface OnlineStoreConfig { - type: 'redis' | 'dynamodb' | 'cassandra'; - connection: Record; - ttl?: number; - keyFormat?: string; -} - -export interface OfflineStoreConfig { - type: 'parquet' | 'delta' | 'postgresql' | 's3'; - connection: Record; - partitioning?: PartitioningConfig; -} - -export interface MetadataStoreConfig { - type: 'postgresql' | 'mysql' | 'sqlite'; - connection: Record; -} - -export interface PartitioningConfig { - columns: string[]; - strategy: 'time' | 'hash' | 'range'; - granularity?: 'hour' | 'day' | 'month'; -} - -// Monitoring and Alerting - -export interface FeatureMonitoring { - featureGroupId: string; - featureName: string; - monitors: FeatureMonitor[]; - alerts: FeatureAlert[]; -} - -export interface FeatureMonitor { - name: string; - type: 'drift' | 'freshness' | 'availability' | 'quality'; - threshold: number; - enabled: boolean; - configuration: Record; -} - -export interface FeatureAlert { - id: string; - monitorName: string; - level: 'warning' | 'error' | 'critical'; - message: string; - timestamp: Date; - resolved: boolean; - resolvedAt?: Date; -} diff --git a/apps/data-services/feature-store/tsconfig.json b/apps/data-services/feature-store/tsconfig.json deleted file mode 100644 index c324c66..0000000 --- a/apps/data-services/feature-store/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "module": "ESNext", - "moduleResolution": "bundler", - "declaration": true, - "declarationMap": true, - "allowImportingTsExtensions": true, - "noEmit": true, - "paths": { - "@/*": ["./src/*"] - } - }, - "include": [ - "src/**/*" - ], - "exclude": [ - "node_modules", - "dist" - ] -} diff --git a/apps/intelligence-services/backtest-engine/README.md b/apps/intelligence-services/backtest-engine/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/apps/intelligence-services/backtest-engine/package.json b/apps/intelligence-services/backtest-engine/package.json deleted file mode 100644 index c0bb127..0000000 --- a/apps/intelligence-services/backtest-engine/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "backtest-engine", - "version": "1.0.0", - "description": "Dedicated backtesting engine for trading strategies", - "main": "src/index.ts", - "scripts": { - "dev": "bun run --watch src/index.ts", - "start": "bun run src/index.ts", - "test": "bun test --timeout 10000 src/tests/**/*.test.ts", - "test:watch": "bun test --watch src/tests/**/*.test.ts" - }, "dependencies": { - "hono": "^4.6.3", - "@stock-bot/types": "workspace:*", - "@stock-bot/utils": "workspace:*", - "@stock-bot/event-bus": "workspace:*", - "@stock-bot/api-client": "workspace:*", - "@stock-bot/config": "*", - "ws": "^8.18.0", - "axios": "^1.6.2" - }, - "devDependencies": { - "bun-types": "^1.2.15", - "@types/ws": "^8.5.12" - } -} \ No newline at end of file diff --git a/apps/intelligence-services/backtest-engine/src/core/BacktestEngine.ts b/apps/intelligence-services/backtest-engine/src/core/BacktestEngine.ts deleted file mode 100644 index f808235..0000000 --- a/apps/intelligence-services/backtest-engine/src/core/BacktestEngine.ts +++ /dev/null @@ -1,650 +0,0 @@ -import { EventEmitter } from 'events'; -import { OHLCV } from '@stock-bot/types'; -import { Order, Position } from '@stock-bot/types'; -import { createLogger } from '@stock-bot/utils'; -import { financialUtils } from '@stock-bot/utils'; - -const logger = createLogger('backtest-engine'); - -// Use OHLCV from types as BarData equivalent -export type BarData = OHLCV; - -// Strategy interface to match existing pattern -export interface StrategyInterface { - id: string; - onBar(bar: BarData): Promise | Order[]; - stop(): Promise; -} - -export interface BacktestConfig { - initialCapital: number; - startDate: Date; - endDate: Date; - commission: number; - slippage: number; -} - -export interface BacktestResult { - startDate: Date; - endDate: Date; - initialCapital: number; - finalCapital: number; - totalReturn: number; - totalTrades: number; - winningTrades: number; - losingTrades: number; - winRate: number; - avgWin: number; - avgLoss: number; - profitFactor: number; - sharpeRatio: number; - maxDrawdown: number; - trades: Array<{ - id: string; - symbol: string; - side: 'BUY' | 'SELL'; - quantity: number; - entryTime: Date; - exitTime: Date; - entryPrice: number; - exitPrice: number; - pnl: number; - pnlPercent: number; - }>; - dailyReturns: Array<{ - date: Date; - portfolioValue: number; - dayReturn: number; - }>; -} - -export interface BacktestProgress { - currentDate: Date; - progress: number; // 0-100 - portfolioValue: number; - totalTrades: number; -} - -export interface DataFeed { - getHistoricalData(symbol: string, startDate: Date, endDate: Date): Promise; -} - -// Extended Position interface that includes additional fields needed for backtesting -export interface BacktestPosition { - symbol: string; - quantity: number; - averagePrice: number; - marketValue: number; - unrealizedPnL: number; - timestamp: Date; - // Additional fields for backtesting - avgPrice: number; // Alias for averagePrice - entryTime: Date; -} - -// Extended Order interface that includes additional fields needed for backtesting -export interface BacktestOrder extends Order { - fillPrice?: number; - fillTime?: Date; -} - trades: Array<{ - symbol: string; - entryTime: Date; - entryPrice: number; - exitTime: Date; - exitPrice: number; - quantity: number; - pnl: number; - pnlPercent: number; - }>; -} - -export interface BacktestProgress { - progress: number; // 0-100 - currentDate: Date; - processingSpeed: number; // Bars per second - estimatedTimeRemaining: number; // milliseconds - currentCapital: number; - currentReturn: number; - currentDrawdown: number; -} - -export interface DataFeed { - getHistoricalData(symbol: string, resolution: string, start: Date, end: Date): Promise; - hasDataFor(symbol: string, resolution: string, start: Date, end: Date): Promise; -} - -export class BacktestEngine extends EventEmitter { - private config: BacktestConfig; - private strategy: StrategyInterface; - private dataFeed: DataFeed; - private isRunning: boolean = false; - private barBuffer: Map = new Map(); - private pendingOrders: BacktestOrder[] = []; - private filledOrders: BacktestOrder[] = []; - private currentTime: Date; - private startTime: number = 0; // For performance tracking - private processedBars: number = 0; - private marketData: Map = new Map(); - - // Results tracking - private initialCapital: number; - private currentCapital: number; - private positions = new Map(); - private trades: BacktestResult['trades'] = []; - private dailyReturns: BacktestResult['dailyReturns'] = []; - private previousPortfolioValue: number; - private highWaterMark: number; - private maxDrawdown: number = 0; - private drawdownStartTime: Date | null = null; - private maxDrawdownDuration: number = 0; - private winningTrades: number = 0; - private losingTrades: number = 0; - private breakEvenTrades: number = 0; - private totalProfits: number = 0; - private totalLosses: number = 0; - constructor(strategy: StrategyInterface, config: BacktestConfig, dataFeed: DataFeed) { - super(); - this.strategy = strategy; - this.config = config; - this.dataFeed = dataFeed; - this.currentTime = new Date(config.startDate); - this.initialCapital = config.initialCapital; - this.currentCapital = config.initialCapital; - this.previousPortfolioValue = config.initialCapital; - this.highWaterMark = config.initialCapital; - } - async run(): Promise { - if (this.isRunning) { - throw new Error('Backtest is already running'); - } - - this.isRunning = true; - this.startTime = Date.now(); - this.emit('started', { strategyId: this.strategy.id, config: this.config }); - - try { - await this.runEventBased(); - const result = this.generateResults(); - this.emit('completed', { strategyId: this.strategy.id, result }); - this.isRunning = false; - return result; - } catch (error) { - this.isRunning = false; - this.emit('error', { strategyId: this.strategy.id, error }); - throw error; - } - } - - private async runEventBased(): Promise { - // Load market data for all symbols - await this.loadMarketData(); - - // Initialize the strategy - await this.strategy.start(); - - // Create a merged timeline of all bars across all symbols, sorted by timestamp - const timeline = this.createMergedTimeline(); - - // Process each event in chronological order - let lastProgressUpdate = Date.now(); - let prevDate = new Date(0); - - for (let i = 0; i < timeline.length; i++) { - const bar = timeline[i]; - this.currentTime = bar.timestamp; - - // Process any pending orders - await this.processOrders(bar); - - // Update positions with current prices - this.updatePositions(bar); - - // If we've crossed to a new day, calculate daily return - if (this.currentTime.toDateString() !== prevDate.toDateString()) { - this.calculateDailyReturn(); - prevDate = this.currentTime; - } - - // Send the new bar to the strategy - const orders = await this.strategy.onBar(bar); - - // Add any new orders to the pending orders queue - if (orders && orders.length > 0) { - this.pendingOrders.push(...orders); - } - - // Update progress periodically - if (Date.now() - lastProgressUpdate > 1000) { // Update every second - this.updateProgress(i / timeline.length); - lastProgressUpdate = Date.now(); - } - } - - // Process any remaining orders - for (const order of this.pendingOrders) { - await this.processOrder(order); - } - - // Close any remaining positions at the last known price - await this.closeAllPositions(); - - // Clean up strategy - await this.strategy.stop(); - } - - private async runVectorized(): Promise { - // Load market data for all symbols - await this.loadMarketData(); - - // To implement a vectorized approach, we need to: - // 1. Pre-compute technical indicators - // 2. Generate buy/sell signals for the entire dataset - // 3. Calculate portfolio values based on signals - - // This is a simplified implementation since specific vectorized strategies - // will need to be implemented separately based on the strategy type - - const timeline = this.createMergedTimeline(); - const startTime = Date.now(); - - // Initialize variables for tracking performance - let currentPositions = new Map(); - let currentCash = this.initialCapital; - let prevPortfolioValue = this.initialCapital; - let highWaterMark = this.initialCapital; - let maxDrawdown = 0; - let maxDrawdownStartDate = new Date(); - let maxDrawdownEndDate = new Date(); - let currentDrawdownStart = new Date(); - - // Pre-process data (this would be implemented by the specific strategy) - const allBars = new Map(); - for (const symbol of this.config.symbols) { - allBars.set(symbol, this.marketData.get(symbol) || []); - } - - // Apply strategy logic (vectorized implementation would be here) - // For now, we'll just simulate the processing - - this.emit('completed', { message: 'Vectorized backtest completed in fast mode' }); - } - - private async loadMarketData(): Promise { - for (const symbol of this.config.symbols) { - this.emit('loading', { symbol, resolution: this.config.dataResolution }); - - // Check if data is available - const hasData = await this.dataFeed.hasDataFor( - symbol, - this.config.dataResolution, - this.config.startDate, - this.config.endDate - ); - - if (!hasData) { - throw new Error(`No data available for ${symbol} at resolution ${this.config.dataResolution}`); - } - - // Load data - const data = await this.dataFeed.getHistoricalData( - symbol, - this.config.dataResolution, - this.config.startDate, - this.config.endDate - ); - - this.marketData.set(symbol, data); - this.emit('loaded', { symbol, count: data.length }); - } - } - - private createMergedTimeline(): BarData[] { - const allBars: BarData[] = []; - - for (const [symbol, bars] of this.marketData.entries()) { - allBars.push(...bars); - } - - // Sort by timestamp - return allBars.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); - } - - private async processOrders(currentBar: BarData): Promise { - // Find orders for the current symbol - const ordersToProcess = this.pendingOrders.filter(order => order.symbol === currentBar.symbol); - - if (ordersToProcess.length === 0) return; - - // Remove these orders from pendingOrders - this.pendingOrders = this.pendingOrders.filter(order => order.symbol !== currentBar.symbol); - - // Process each order - for (const order of ordersToProcess) { - await this.processOrder(order); - } - } - - private async processOrder(order: Order): Promise { - // Get the latest price for the symbol - const latestBars = this.marketData.get(order.symbol); - if (!latestBars || latestBars.length === 0) { - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'No market data available' }); - return; - } - - // Find the bar closest to the order time - const bar = latestBars.find(b => - b.timestamp.getTime() >= order.timestamp.getTime() - ) || latestBars[latestBars.length - 1]; - - // Calculate fill price with slippage - let fillPrice: number; - if (order.type === 'MARKET') { - // Apply slippage model - const slippageFactor = 1 + (order.side === 'BUY' ? this.config.slippage : -this.config.slippage); - fillPrice = bar.close * slippageFactor; - } else if (order.type === 'LIMIT' && order.price !== undefined) { - // For limit orders, check if the price was reached - if ((order.side === 'BUY' && bar.low <= order.price) || - (order.side === 'SELL' && bar.high >= order.price)) { - fillPrice = order.price; - } else { - // Limit price not reached - return; - } - } else { - // Other order types not implemented - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'Order type not supported' }); - return; - } - - // Calculate commission - const orderValue = order.quantity * fillPrice; - const commission = orderValue * this.config.commission; - - // Check if we have enough cash for BUY orders - if (order.side === 'BUY') { - const totalCost = orderValue + commission; - if (totalCost > this.currentCapital) { - // Not enough cash - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'Insufficient funds' }); - return; - } - - // Update cash - this.currentCapital -= totalCost; - - // Update or create position - const existingPosition = this.positions.get(order.symbol); - if (existingPosition) { - // Update existing position (average down) - const totalShares = existingPosition.quantity + order.quantity; - const totalCost = (existingPosition.quantity * existingPosition.avgPrice) + (order.quantity * fillPrice); - existingPosition.avgPrice = totalCost / totalShares; - existingPosition.quantity = totalShares; - } else { - // Create new position - this.positions.set(order.symbol, { - symbol: order.symbol, - quantity: order.quantity, - avgPrice: fillPrice, - side: 'LONG', - entryTime: this.currentTime - }); - } - } else if (order.side === 'SELL') { - const position = this.positions.get(order.symbol); - - if (!position || position.quantity < order.quantity) { - // Not enough shares to sell - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'Insufficient position' }); - return; - } - - // Calculate P&L - const pnl = (fillPrice - position.avgPrice) * order.quantity; - - // Update cash - this.currentCapital += orderValue - commission; - - // Update position - position.quantity -= order.quantity; - - if (position.quantity === 0) { - // Position closed, record the trade - this.positions.delete(order.symbol); - - this.trades.push({ - symbol: order.symbol, - entryTime: position.entryTime, - entryPrice: position.avgPrice, - exitTime: this.currentTime, - exitPrice: fillPrice, - quantity: order.quantity, - pnl: pnl, - pnlPercent: (pnl / (position.avgPrice * order.quantity)) * 100 - }); - - // Update statistics - if (pnl > 0) { - this.winningTrades++; - this.totalProfits += pnl; - } else if (pnl < 0) { - this.losingTrades++; - this.totalLosses -= pnl; // Make positive for easier calculations - } else { - this.breakEvenTrades++; - } - } - } - - // Mark order as filled - order.status = 'FILLED'; - order.fillPrice = fillPrice; - order.fillTime = this.currentTime; - this.filledOrders.push(order); - - // Notify strategy - await this.strategy.onOrderFilled(order); - - this.emit('orderFilled', { order }); - } - - private updatePositions(currentBar: BarData): void { - // Update the unrealized P&L for positions in this symbol - const position = this.positions.get(currentBar.symbol); - if (position) { - const currentPrice = currentBar.close; - const unrealizedPnL = (currentPrice - position.avgPrice) * position.quantity; - position.unrealizedPnL = unrealizedPnL; - } - - // Calculate total portfolio value - const portfolioValue = this.calculatePortfolioValue(); - - // Check for new high water mark - if (portfolioValue > this.highWaterMark) { - this.highWaterMark = portfolioValue; - this.drawdownStartTime = null; - } - - // Check for drawdown - if (this.drawdownStartTime === null && portfolioValue < this.highWaterMark) { - this.drawdownStartTime = this.currentTime; - } - - // Update max drawdown - if (this.highWaterMark > 0) { - const currentDrawdown = (this.highWaterMark - portfolioValue) / this.highWaterMark; - if (currentDrawdown > this.maxDrawdown) { - this.maxDrawdown = currentDrawdown; - - // Calculate drawdown duration - if (this.drawdownStartTime !== null) { - const drawdownDuration = (this.currentTime.getTime() - this.drawdownStartTime.getTime()) / (1000 * 60 * 60 * 24); // In days - if (drawdownDuration > this.maxDrawdownDuration) { - this.maxDrawdownDuration = drawdownDuration; - } - } - } - } - - this.previousPortfolioValue = portfolioValue; - } - - private calculatePortfolioValue(): number { - let totalValue = this.currentCapital; - - // Add the current value of all positions - for (const [symbol, position] of this.positions.entries()) { - // Find the latest price for this symbol - const bars = this.marketData.get(symbol); - if (bars && bars.length > 0) { - const latestBar = bars[bars.length - 1]; - totalValue += position.quantity * latestBar.close; - } else { - // If no price data, use the average price (not ideal but better than nothing) - totalValue += position.quantity * position.avgPrice; - } - } - - return totalValue; - } - - private calculateDailyReturn(): void { - const portfolioValue = this.calculatePortfolioValue(); - const dailyReturn = (portfolioValue - this.previousPortfolioValue) / this.previousPortfolioValue; - - this.dailyReturns.push({ - date: new Date(this.currentTime), - return: dailyReturn - }); - - this.previousPortfolioValue = portfolioValue; - } - - private async closeAllPositions(): Promise { - for (const [symbol, position] of this.positions.entries()) { - // Find the latest price - const bars = this.marketData.get(symbol); - if (!bars || bars.length === 0) continue; - - const lastBar = bars[bars.length - 1]; - const closePrice = lastBar.close; - - // Calculate P&L - const pnl = (closePrice - position.avgPrice) * position.quantity; - - // Update cash - this.currentCapital += position.quantity * closePrice; - - // Record the trade - this.trades.push({ - symbol, - entryTime: position.entryTime, - entryPrice: position.avgPrice, - exitTime: this.currentTime, - exitPrice: closePrice, - quantity: position.quantity, - pnl, - pnlPercent: (pnl / (position.avgPrice * position.quantity)) * 100 - }); - - // Update statistics - if (pnl > 0) { - this.winningTrades++; - this.totalProfits += pnl; - } else if (pnl < 0) { - this.losingTrades++; - this.totalLosses -= pnl; // Make positive for easier calculations - } else { - this.breakEvenTrades++; - } - } - - // Clear positions - this.positions.clear(); - } - - private updateProgress(progress: number): void { - const currentPortfolioValue = this.calculatePortfolioValue(); - const currentDrawdown = this.highWaterMark > 0 - ? (this.highWaterMark - currentPortfolioValue) / this.highWaterMark - : 0; - - const elapsedMs = Date.now() - this.startTime; - const totalEstimatedMs = elapsedMs / progress; - const remainingMs = totalEstimatedMs - elapsedMs; - - this.emit('progress', { - progress: progress * 100, - currentDate: this.currentTime, - processingSpeed: this.processedBars / (elapsedMs / 1000), - estimatedTimeRemaining: remainingMs, - currentCapital: this.currentCapital, - currentReturn: (currentPortfolioValue - this.initialCapital) / this.initialCapital, - currentDrawdown - } as BacktestProgress); - } - - private generateResults(): BacktestResult { - const currentPortfolioValue = this.calculatePortfolioValue(); - const totalReturn = (currentPortfolioValue - this.initialCapital) / this.initialCapital; - - // Calculate annualized return - const days = (this.config.endDate.getTime() - this.config.startDate.getTime()) / (1000 * 60 * 60 * 24); - const annualizedReturn = Math.pow(1 + totalReturn, 365 / days) - 1; - - // Calculate Sharpe Ratio - let sharpeRatio = 0; - if (this.dailyReturns.length > 1) { - const dailyReturnValues = this.dailyReturns.map(dr => dr.return); - const avgDailyReturn = dailyReturnValues.reduce((sum, ret) => sum + ret, 0) / dailyReturnValues.length; - const stdDev = Math.sqrt( - dailyReturnValues.reduce((sum, ret) => sum + Math.pow(ret - avgDailyReturn, 2), 0) / dailyReturnValues.length - ); - - // Annualize - sharpeRatio = stdDev > 0 - ? (avgDailyReturn * 252) / (stdDev * Math.sqrt(252)) - : 0; - } - - // Calculate win rate and profit factor - const totalTrades = this.winningTrades + this.losingTrades + this.breakEvenTrades; - const winRate = totalTrades > 0 ? this.winningTrades / totalTrades : 0; - const profitFactor = this.totalLosses > 0 ? this.totalProfits / this.totalLosses : (this.totalProfits > 0 ? Infinity : 0); - - // Calculate average winning and losing trade - const avgWinningTrade = this.winningTrades > 0 ? this.totalProfits / this.winningTrades : 0; - const avgLosingTrade = this.losingTrades > 0 ? this.totalLosses / this.losingTrades : 0; - - return { - strategyId: this.strategy.id, - startDate: this.config.startDate, - endDate: this.config.endDate, - duration: Date.now() - this.startTime, - initialCapital: this.initialCapital, - finalCapital: currentPortfolioValue, - totalReturn, - annualizedReturn, - sharpeRatio, - maxDrawdown: this.maxDrawdown, - maxDrawdownDuration: this.maxDrawdownDuration, - winRate, - totalTrades, - winningTrades: this.winningTrades, - losingTrades: this.losingTrades, - averageWinningTrade: avgWinningTrade, - averageLosingTrade: avgLosingTrade, - profitFactor, - dailyReturns: this.dailyReturns, - trades: this.trades - }; - } -} diff --git a/apps/intelligence-services/backtest-engine/src/core/BacktestService.ts b/apps/intelligence-services/backtest-engine/src/core/BacktestService.ts deleted file mode 100644 index 7dbb639..0000000 --- a/apps/intelligence-services/backtest-engine/src/core/BacktestService.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { BaseStrategy } from '../Strategy'; -import { BacktestConfig, BacktestEngine, BacktestResult } from './BacktestEngine'; -import { MarketDataFeed } from './MarketDataFeed'; -import { StrategyRegistry, StrategyType } from '../strategies/StrategyRegistry'; - -export interface BacktestRequest { - strategyType: StrategyType; - strategyParams: Record; - symbols: string[]; - startDate: Date | string; - endDate: Date | string; - initialCapital: number; - dataResolution: '1m' | '5m' | '15m' | '30m' | '1h' | '4h' | '1d'; - commission: number; - slippage: number; - mode: 'event' | 'vector'; -} - -/** - * Backtesting Service - * - * A service that handles backtesting requests and manages backtesting sessions. - */ -export class BacktestService { - private readonly strategyRegistry: StrategyRegistry; - private readonly dataFeed: MarketDataFeed; - private readonly activeBacktests: Map = new Map(); - - constructor(apiBaseUrl: string = 'http://localhost:3001/api') { - this.strategyRegistry = StrategyRegistry.getInstance(); - this.dataFeed = new MarketDataFeed(apiBaseUrl); - } - - /** - * Run a backtest based on a request - */ - async runBacktest(request: BacktestRequest): Promise { - // Create a strategy instance - const strategyId = `backtest_${Date.now()}`; - const strategy = this.strategyRegistry.createStrategy( - request.strategyType, - strategyId, - `Backtest ${request.strategyType}`, - `Generated backtest for ${request.symbols.join(', ')}`, - request.symbols, - request.strategyParams - ); - - // Parse dates if they are strings - const startDate = typeof request.startDate === 'string' - ? new Date(request.startDate) - : request.startDate; - - const endDate = typeof request.endDate === 'string' - ? new Date(request.endDate) - : request.endDate; - - // Create backtest configuration - const config: BacktestConfig = { - startDate, - endDate, - symbols: request.symbols, - initialCapital: request.initialCapital, - commission: request.commission, - slippage: request.slippage, - dataResolution: request.dataResolution, - mode: request.mode - }; - - // Create and run the backtest engine - const engine = new BacktestEngine(strategy, config, this.dataFeed); - this.activeBacktests.set(strategyId, engine); - - try { - // Set up event forwarding - const forwardEvents = (eventName: string) => { - engine.on(eventName, (data) => { - console.log(`[Backtest ${strategyId}] ${eventName}:`, data); - }); - }; - - forwardEvents('started'); - forwardEvents('loading'); - forwardEvents('loaded'); - forwardEvents('progress'); - forwardEvents('orderFilled'); - forwardEvents('orderRejected'); - forwardEvents('completed'); - forwardEvents('error'); - - // Run the backtest - const result = await engine.run(); - - // Clean up - this.activeBacktests.delete(strategyId); - - return result; - } catch (error) { - this.activeBacktests.delete(strategyId); - throw error; - } - } - - /** - * Optimize a strategy by running multiple backtests with different parameters - */ - async optimizeStrategy( - baseRequest: BacktestRequest, - parameterGrid: Record - ): Promise }>> { - const results: Array }> = []; - - // Generate parameter combinations - const paramKeys = Object.keys(parameterGrid); - const combinations = this.generateParameterCombinations(parameterGrid, paramKeys); - - // Run backtest for each combination - for (const paramSet of combinations) { - const request = { - ...baseRequest, - strategyParams: { - ...baseRequest.strategyParams, - ...paramSet - } - }; - - try { - const result = await this.runBacktest(request); - results.push({ - ...result, - parameters: paramSet - }); - } catch (error) { - console.error(`Optimization failed for parameters:`, paramSet, error); - } - } - - // Sort by performance metric (e.g., Sharpe ratio) - return results.sort((a, b) => b.sharpeRatio - a.sharpeRatio); - } - - /** - * Generate all combinations of parameters for grid search - */ - private generateParameterCombinations( - grid: Record, - keys: string[], - current: Record = {}, - index: number = 0, - result: Record[] = [] - ): Record[] { - if (index === keys.length) { - result.push({ ...current }); - return result; - } - - const key = keys[index]; - const values = grid[key]; - - for (const value of values) { - current[key] = value; - this.generateParameterCombinations(grid, keys, current, index + 1, result); - } - - return result; - } - - /** - * Get an active backtest engine by ID - */ - getBacktestEngine(id: string): BacktestEngine | undefined { - return this.activeBacktests.get(id); - } - - /** - * Cancel a running backtest - */ - cancelBacktest(id: string): boolean { - const engine = this.activeBacktests.get(id); - if (!engine) return false; - - // No explicit cancel method on engine, but we can clean up - this.activeBacktests.delete(id); - return true; - } -} diff --git a/apps/intelligence-services/backtest-engine/src/core/MarketDataFeed.ts b/apps/intelligence-services/backtest-engine/src/core/MarketDataFeed.ts deleted file mode 100644 index 7d15369..0000000 --- a/apps/intelligence-services/backtest-engine/src/core/MarketDataFeed.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { BarData } from '../Strategy'; -import { DataFeed } from './BacktestEngine'; -import axios from 'axios'; - -export class MarketDataFeed implements DataFeed { - private readonly apiBaseUrl: string; - private cache: Map = new Map(); - - constructor(apiBaseUrl: string = 'http://localhost:3001/api') { - this.apiBaseUrl = apiBaseUrl; - } - - async getHistoricalData(symbol: string, resolution: string, start: Date, end: Date): Promise { - const cacheKey = this.getCacheKey(symbol, resolution, start, end); - - // Check cache first - if (this.cache.has(cacheKey)) { - return this.cache.get(cacheKey)!; - } - - try { - // Format dates for API request - const startStr = start.toISOString(); - const endStr = end.toISOString(); - - const response = await axios.get(`${this.apiBaseUrl}/market-data/history`, { - params: { - symbol, - resolution, - start: startStr, - end: endStr - } - }); - - if (!response.data.success || !response.data.data) { - throw new Error(`Failed to fetch historical data for ${symbol}`); - } - - // Transform API response to BarData objects - const bars: BarData[] = response.data.data.map((bar: any) => ({ - symbol, - timestamp: new Date(bar.timestamp), - open: bar.open, - high: bar.high, - low: bar.low, - close: bar.close, - volume: bar.volume - })); - - // Cache the result - this.cache.set(cacheKey, bars); - - return bars; - } catch (error) { - console.error(`Error fetching historical data for ${symbol}:`, error); - // Return fallback test data if API call fails - return this.generateFallbackTestData(symbol, resolution, start, end); - } - } - - async hasDataFor(symbol: string, resolution: string, start: Date, end: Date): Promise { - try { - const startStr = start.toISOString(); - const endStr = end.toISOString(); - - const response = await axios.get(`${this.apiBaseUrl}/market-data/available`, { - params: { - symbol, - resolution, - start: startStr, - end: endStr - } - }); - - return response.data.success && response.data.data.available; - } catch (error) { - console.error(`Error checking data availability for ${symbol}:`, error); - // Assume data is available for test purposes - return true; - } - } - - clearCache(): void { - this.cache.clear(); - } - - private getCacheKey(symbol: string, resolution: string, start: Date, end: Date): string { - return `${symbol}_${resolution}_${start.getTime()}_${end.getTime()}`; - } - - private generateFallbackTestData(symbol: string, resolution: string, start: Date, end: Date): BarData[] { - console.warn(`Generating fallback test data for ${symbol} from ${start} to ${end}`); - - const bars: BarData[] = []; - let current = new Date(start); - let basePrice = this.getBasePrice(symbol); - - // Generate daily bars by default - const interval = this.getIntervalFromResolution(resolution); - - while (current.getTime() <= end.getTime()) { - // Only generate bars for trading days (skip weekends) - if (current.getDay() !== 0 && current.getDay() !== 6) { - // Generate a random daily price movement (-1% to +1%) - const dailyChange = (Math.random() * 2 - 1) / 100; - - // Add some randomness to the volatility - const volatility = 0.005 + Math.random() * 0.01; // 0.5% to 1.5% - - const open = basePrice * (1 + (Math.random() * 0.002 - 0.001)); - const close = open * (1 + dailyChange); - const high = Math.max(open, close) * (1 + Math.random() * volatility); - const low = Math.min(open, close) * (1 - Math.random() * volatility); - const volume = Math.floor(100000 + Math.random() * 900000); - - bars.push({ - symbol, - timestamp: new Date(current), - open, - high, - low, - close, - volume - }); - - // Update base price for next bar - basePrice = close; - } - - // Move to next interval - current = new Date(current.getTime() + interval); - } - - return bars; - } - - private getBasePrice(symbol: string): number { - // Return a realistic base price for common symbols - switch (symbol.toUpperCase()) { - case 'AAPL': return 170 + Math.random() * 30; - case 'MSFT': return 370 + Math.random() * 50; - case 'AMZN': return 140 + Math.random() * 20; - case 'GOOGL': return 130 + Math.random() * 20; - case 'META': return 300 + Math.random() * 50; - case 'TSLA': return 180 + Math.random() * 70; - case 'NVDA': return 700 + Math.random() * 200; - case 'SPY': return 450 + Math.random() * 30; - case 'QQQ': return 370 + Math.random() * 40; - default: return 100 + Math.random() * 50; - } - } - - private getIntervalFromResolution(resolution: string): number { - // Return milliseconds for each resolution - switch (resolution) { - case '1m': return 60 * 1000; - case '5m': return 5 * 60 * 1000; - case '15m': return 15 * 60 * 1000; - case '30m': return 30 * 60 * 1000; - case '1h': return 60 * 60 * 1000; - case '4h': return 4 * 60 * 60 * 1000; - case '1d': return 24 * 60 * 60 * 1000; - default: return 24 * 60 * 60 * 1000; // Default to daily - } - } -} diff --git a/apps/intelligence-services/backtest-engine/src/core/PerformanceAnalytics.ts b/apps/intelligence-services/backtest-engine/src/core/PerformanceAnalytics.ts deleted file mode 100644 index 9238718..0000000 --- a/apps/intelligence-services/backtest-engine/src/core/PerformanceAnalytics.ts +++ /dev/null @@ -1,325 +0,0 @@ -import { BacktestResult } from './BacktestEngine'; - -/** - * Performance Analysis Utilities - * - * Provides additional metrics and analysis tools for backtesting results. - */ -export class PerformanceAnalytics { - /** - * Calculate additional metrics from backtest results - */ - static enhanceResults(result: BacktestResult): BacktestResult { - // Calculate additional metrics - const enhancedResult = { - ...result, - ...this.calculateAdvancedMetrics(result) - }; - - return enhancedResult; - } - - /** - * Calculate advanced performance metrics - */ - private static calculateAdvancedMetrics(result: BacktestResult): Partial { - // Extract daily returns - const dailyReturns = result.dailyReturns.map(dr => dr.return); - - // Calculate Sortino ratio - const sortinoRatio = this.calculateSortinoRatio(dailyReturns); - - // Calculate Calmar ratio - const calmarRatio = result.maxDrawdown > 0 - ? result.annualizedReturn / result.maxDrawdown - : Infinity; - - // Calculate Omega ratio - const omegaRatio = this.calculateOmegaRatio(dailyReturns); - - // Calculate CAGR - const startTimestamp = result.startDate.getTime(); - const endTimestamp = result.endDate.getTime(); - const yearsElapsed = (endTimestamp - startTimestamp) / (365 * 24 * 60 * 60 * 1000); - const cagr = Math.pow(result.finalCapital / result.initialCapital, 1 / yearsElapsed) - 1; - - // Calculate additional volatility and return metrics - const volatility = this.calculateVolatility(dailyReturns); - const ulcerIndex = this.calculateUlcerIndex(result.dailyReturns); - - return { - sortinoRatio, - calmarRatio, - omegaRatio, - cagr, - volatility, - ulcerIndex - }; - } - - /** - * Calculate Sortino ratio (downside risk-adjusted return) - */ - private static calculateSortinoRatio(dailyReturns: number[]): number { - if (dailyReturns.length === 0) return 0; - - const avgReturn = dailyReturns.reduce((sum, ret) => sum + ret, 0) / dailyReturns.length; - - // Filter only negative returns (downside) - const negativeReturns = dailyReturns.filter(ret => ret < 0); - - if (negativeReturns.length === 0) return Infinity; - - // Calculate downside deviation - const downsideDeviation = Math.sqrt( - negativeReturns.reduce((sum, ret) => sum + Math.pow(ret, 2), 0) / negativeReturns.length - ); - - // Annualize - const annualizedReturn = avgReturn * 252; - const annualizedDownsideDeviation = downsideDeviation * Math.sqrt(252); - - return annualizedDownsideDeviation > 0 - ? annualizedReturn / annualizedDownsideDeviation - : 0; - } - - /** - * Calculate Omega ratio (probability-weighted ratio of gains versus losses) - */ - private static calculateOmegaRatio(dailyReturns: number[], threshold = 0): number { - if (dailyReturns.length === 0) return 0; - - let sumGains = 0; - let sumLosses = 0; - - for (const ret of dailyReturns) { - if (ret > threshold) { - sumGains += (ret - threshold); - } else { - sumLosses += (threshold - ret); - } - } - - return sumLosses > 0 ? sumGains / sumLosses : Infinity; - } - - /** - * Calculate annualized volatility - */ - private static calculateVolatility(returns: number[]): number { - if (returns.length < 2) return 0; - - const mean = returns.reduce((sum, ret) => sum + ret, 0) / returns.length; - const variance = returns.reduce((sum, ret) => sum + Math.pow(ret - mean, 2), 0) / returns.length; - - // Annualize - return Math.sqrt(variance * 252); - } - - /** - * Calculate Ulcer Index (measure of downside risk) - */ - private static calculateUlcerIndex(dailyReturns: Array<{ date: Date; return: number }>): number { - if (dailyReturns.length === 0) return 0; - - // Calculate running equity curve - let equity = 1; - const equityCurve = dailyReturns.map(dr => { - equity *= (1 + dr.return); - return equity; - }); - - // Find running maximum - const runningMax: number[] = []; - let currentMax = equityCurve[0]; - - for (const value of equityCurve) { - currentMax = Math.max(currentMax, value); - runningMax.push(currentMax); - } - - // Calculate percentage drawdowns - const percentDrawdowns = equityCurve.map((value, i) => - (runningMax[i] - value) / runningMax[i] - ); - - // Calculate Ulcer Index - const sumSquaredDrawdowns = percentDrawdowns.reduce( - (sum, dd) => sum + dd * dd, 0 - ); - - return Math.sqrt(sumSquaredDrawdowns / percentDrawdowns.length); - } - - /** - * Extract monthly returns from daily returns - */ - static calculateMonthlyReturns(dailyReturns: Array<{ date: Date; return: number }>): Array<{ - year: number; - month: number; - return: number; - }> { - const monthlyReturns: Array<{ year: number; month: number; return: number }> = []; - - if (dailyReturns.length === 0) return monthlyReturns; - - // Group returns by year and month - const groupedReturns: Record = {}; - - for (const dr of dailyReturns) { - const year = dr.date.getFullYear(); - const month = dr.date.getMonth(); - const key = `${year}-${month}`; - - if (!groupedReturns[key]) { - groupedReturns[key] = []; - } - - groupedReturns[key].push(dr.return); - } - - // Calculate compound return for each month - for (const key in groupedReturns) { - const [yearStr, monthStr] = key.split('-'); - const year = parseInt(yearStr); - const month = parseInt(monthStr); - - // Compound the daily returns for the month - const monthReturn = groupedReturns[key].reduce( - (product, ret) => product * (1 + ret), 1 - ) - 1; - - monthlyReturns.push({ year, month, return: monthReturn }); - } - - // Sort by date - return monthlyReturns.sort((a, b) => { - if (a.year !== b.year) return a.year - b.year; - return a.month - b.month; - }); - } - - /** - * Create drawdown analysis from equity curve - */ - static analyzeDrawdowns(dailyReturns: Array<{ date: Date; return: number }>): Array<{ - startDate: Date; - endDate: Date; - recoveryDate: Date | null; - drawdown: number; - durationDays: number; - recoveryDays: number | null; - }> { - if (dailyReturns.length === 0) return []; - - // Calculate equity curve - let equity = 1; - const equityCurve = dailyReturns.map(dr => { - equity *= (1 + dr.return); - return { date: dr.date, equity }; - }); - - // Analyze drawdowns - const drawdowns: Array<{ - startDate: Date; - endDate: Date; - recoveryDate: Date | null; - drawdown: number; - durationDays: number; - recoveryDays: number | null; - }> = []; - - let peakEquity = equityCurve[0].equity; - let peakDate = equityCurve[0].date; - let inDrawdown = false; - let currentDrawdown: { - startDate: Date; - endDate: Date; - lowEquity: number; - peakEquity: number; - } | null = null; - - // Find drawdown periods - for (let i = 1; i < equityCurve.length; i++) { - const { date, equity } = equityCurve[i]; - - // New peak - if (equity > peakEquity) { - peakEquity = equity; - peakDate = date; - - // If recovering from drawdown, record recovery - if (inDrawdown && currentDrawdown) { - const recoveryDate = date; - const drawdownPct = (currentDrawdown.peakEquity - currentDrawdown.lowEquity) / - currentDrawdown.peakEquity; - - const durationDays = Math.floor( - (currentDrawdown.endDate.getTime() - currentDrawdown.startDate.getTime()) / - (1000 * 60 * 60 * 24) - ); - - const recoveryDays = Math.floor( - (recoveryDate.getTime() - currentDrawdown.endDate.getTime()) / - (1000 * 60 * 60 * 24) - ); - - drawdowns.push({ - startDate: currentDrawdown.startDate, - endDate: currentDrawdown.endDate, - recoveryDate, - drawdown: drawdownPct, - durationDays, - recoveryDays - }); - - inDrawdown = false; - currentDrawdown = null; - } - } - // In drawdown - else { - const drawdownPct = (peakEquity - equity) / peakEquity; - - if (!inDrawdown) { - // Start of new drawdown - inDrawdown = true; - currentDrawdown = { - startDate: peakDate, - endDate: date, - lowEquity: equity, - peakEquity - }; - } else if (currentDrawdown && equity < currentDrawdown.lowEquity) { - // New low in current drawdown - currentDrawdown.lowEquity = equity; - currentDrawdown.endDate = date; - } - } - } - - // Handle any ongoing drawdown at the end - if (inDrawdown && currentDrawdown) { - const drawdownPct = (currentDrawdown.peakEquity - currentDrawdown.lowEquity) / - currentDrawdown.peakEquity; - - const durationDays = Math.floor( - (currentDrawdown.endDate.getTime() - currentDrawdown.startDate.getTime()) / - (1000 * 60 * 60 * 24) - ); - - drawdowns.push({ - startDate: currentDrawdown.startDate, - endDate: currentDrawdown.endDate, - recoveryDate: null, - drawdown: drawdownPct, - durationDays, - recoveryDays: null - }); - } - - // Sort by drawdown magnitude - return drawdowns.sort((a, b) => b.drawdown - a.drawdown); - } -} diff --git a/apps/intelligence-services/backtest-engine/src/index.ts b/apps/intelligence-services/backtest-engine/src/index.ts deleted file mode 100644 index e69de29..0000000 diff --git a/apps/intelligence-services/backtest-engine/tsconfig.json b/apps/intelligence-services/backtest-engine/tsconfig.json deleted file mode 100644 index 0c12276..0000000 --- a/apps/intelligence-services/backtest-engine/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "module": "ESNext", - "moduleResolution": "bundler", - "types": ["bun-types"] - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} \ No newline at end of file diff --git a/apps/intelligence-services/signal-engine/README.md b/apps/intelligence-services/signal-engine/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/apps/intelligence-services/signal-engine/package.json b/apps/intelligence-services/signal-engine/package.json deleted file mode 100644 index d8db7f9..0000000 --- a/apps/intelligence-services/signal-engine/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "signal-engine", - "version": "1.0.0", - "description": "Real-time signal generation and processing engine", - "main": "src/index.ts", - "scripts": { - "dev": "bun run --watch src/index.ts", - "start": "bun run src/index.ts", - "test": "bun test --timeout 10000 src/tests/**/*.test.ts", - "test:watch": "bun test --watch src/tests/**/*.test.ts" - }, - "dependencies": { - "hono": "^4.6.3", - "@stock-bot/config": "*", - "@stock-bot/types": "*", - "ws": "^8.18.0" - }, - "devDependencies": { - "bun-types": "^1.2.15", - "@types/ws": "^8.5.12" - } -} \ No newline at end of file diff --git a/apps/intelligence-services/signal-engine/src/index.ts b/apps/intelligence-services/signal-engine/src/index.ts deleted file mode 100644 index e69de29..0000000 diff --git a/apps/intelligence-services/signal-engine/tsconfig.json b/apps/intelligence-services/signal-engine/tsconfig.json deleted file mode 100644 index 0c12276..0000000 --- a/apps/intelligence-services/signal-engine/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "module": "ESNext", - "moduleResolution": "bundler", - "types": ["bun-types"] - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} \ No newline at end of file diff --git a/apps/intelligence-services/strategy-orchestrator/package.json b/apps/intelligence-services/strategy-orchestrator/package.json deleted file mode 100644 index 0bf586f..0000000 --- a/apps/intelligence-services/strategy-orchestrator/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "strategy-orchestrator", - "version": "1.0.0", - "description": "Trading strategy lifecycle management service", - "main": "src/index.ts", "scripts": { - "dev": "bun run --watch src/index.ts", - "start": "bun run src/index.ts", - "test": "bun test --timeout 10000 src/tests/**/*.test.ts", - "test:watch": "bun test --watch src/tests/**/*.test.ts" - }, "dependencies": { - "hono": "^4.6.3", - "ioredis": "^5.4.1", - "@stock-bot/config": "*", - "@stock-bot/types": "*", - "ws": "^8.18.0", - "node-cron": "^3.0.3", - "axios": "^1.6.2" - }, - "devDependencies": { - "bun-types": "^1.2.15", - "@types/ws": "^8.5.12", - "@types/node-cron": "^3.0.11" - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/controllers/StrategyController.ts b/apps/intelligence-services/strategy-orchestrator/src/controllers/StrategyController.ts deleted file mode 100644 index a840158..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/controllers/StrategyController.ts +++ /dev/null @@ -1,266 +0,0 @@ -import { Request, Response } from 'express'; -import { StrategyRegistry, StrategyType } from '../core/strategies/StrategyRegistry'; -import { BacktestRequest, BacktestService } from '../core/backtesting/BacktestService'; -import { BaseStrategy } from '../core/Strategy'; -import { PerformanceAnalytics } from '../core/backtesting/PerformanceAnalytics'; - -/** - * Strategy Controller - * - * Handles HTTP requests related to strategy management, backtesting, and execution. - */ -export class StrategyController { - private readonly strategyRegistry: StrategyRegistry; - private readonly backtestService: BacktestService; - - constructor(apiBaseUrl: string = 'http://localhost:3001/api') { - this.strategyRegistry = StrategyRegistry.getInstance(); - this.backtestService = new BacktestService(apiBaseUrl); - } - - /** - * Get all available strategy types - */ - public getStrategyTypes(req: Request, res: Response): void { - const types = Object.values(StrategyType); - res.json({ - success: true, - data: types - }); - } - - /** - * Get all strategies - */ - public getStrategies(req: Request, res: Response): void { - const strategies = this.strategyRegistry.getAllStrategies(); - - // Convert to array of plain objects for serialization - const serializedStrategies = strategies.map(strategy => ({ - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type: this.strategyRegistry.getStrategyType(strategy) - })); - - res.json({ - success: true, - data: serializedStrategies - }); - } - - /** - * Get a specific strategy by ID - */ - public getStrategy(req: Request, res: Response): void { - const { id } = req.params; - const strategy = this.strategyRegistry.getStrategyById(id); - - if (!strategy) { - res.status(404).json({ - success: false, - error: `Strategy with ID ${id} not found` - }); - return; - } - - const type = this.strategyRegistry.getStrategyType(strategy); - - res.json({ - success: true, - data: { - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type - } - }); - } - - /** - * Create a new strategy - */ - public createStrategy(req: Request, res: Response): void { - try { - const { name, description, symbols, parameters, type } = req.body; - - if (!type || !Object.values(StrategyType).includes(type)) { - res.status(400).json({ - success: false, - error: 'Invalid strategy type' - }); - return; - } - - const strategy = this.strategyRegistry.createStrategy( - type as StrategyType, - `strategy_${Date.now()}`, // Generate an ID - name || `New ${type} Strategy`, - description || `Generated ${type} strategy`, - symbols || [], - parameters || {} - ); - - res.status(201).json({ - success: true, - data: { - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type - } - }); - } catch (error) { - res.status(500).json({ - success: false, - error: (error as Error).message - }); - } - } - - /** - * Update an existing strategy - */ - public updateStrategy(req: Request, res: Response): void { - const { id } = req.params; - const { name, description, symbols, parameters } = req.body; - - const strategy = this.strategyRegistry.getStrategyById(id); - - if (!strategy) { - res.status(404).json({ - success: false, - error: `Strategy with ID ${id} not found` - }); - return; - } - // Update properties - if (name !== undefined) strategy.name = name; - if (description !== undefined) strategy.description = description; - if (symbols !== undefined) strategy.symbols = symbols; - if (parameters !== undefined) strategy.parameters = parameters; - - res.json({ - success: true, - data: { - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type: this.strategyRegistry.getStrategyType(strategy) - } - }); - } - - /** - * Delete a strategy - */ - public deleteStrategy(req: Request, res: Response): void { - const { id } = req.params; - const success = this.strategyRegistry.deleteStrategy(id); - - if (!success) { - res.status(404).json({ - success: false, - error: `Strategy with ID ${id} not found` - }); - return; - } - - res.json({ - success: true, - data: { id } - }); - } - - /** - * Run a backtest - */ - public async runBacktest(req: Request, res: Response): Promise { - try { - const backtestRequest: BacktestRequest = req.body; - - // Validate request - if (!backtestRequest.strategyType) { - res.status(400).json({ - success: false, - error: 'Strategy type is required' - }); - return; - } - - if (!backtestRequest.symbols || backtestRequest.symbols.length === 0) { - res.status(400).json({ - success: false, - error: 'At least one symbol is required' - }); - return; - } - - // Run the backtest - const result = await this.backtestService.runBacktest(backtestRequest); - - // Enhance results with additional metrics - const enhancedResult = PerformanceAnalytics.enhanceResults(result); - - // Calculate additional analytics - const monthlyReturns = PerformanceAnalytics.calculateMonthlyReturns(result.dailyReturns); - const drawdowns = PerformanceAnalytics.analyzeDrawdowns(result.dailyReturns); - - res.json({ - success: true, - data: { - ...enhancedResult, - monthlyReturns, - drawdowns - } - }); - } catch (error) { - console.error('Backtest error:', error); - res.status(500).json({ - success: false, - error: (error as Error).message - }); - } - } - - /** - * Optimize a strategy with grid search - */ - public async optimizeStrategy(req: Request, res: Response): Promise { - try { - const { baseRequest, parameterGrid } = req.body; - - // Validate request - if (!baseRequest || !parameterGrid) { - res.status(400).json({ - success: false, - error: 'Base request and parameter grid are required' - }); - return; - } - - // Run optimization - const results = await this.backtestService.optimizeStrategy(baseRequest, parameterGrid); - - res.json({ - success: true, - data: results - }); - } catch (error) { - res.status(500).json({ - success: false, - error: (error as Error).message - }); - } - } -} - -export default StrategyController; diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/Strategy.ts b/apps/intelligence-services/strategy-orchestrator/src/core/Strategy.ts deleted file mode 100644 index 51ec0f3..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/Strategy.ts +++ /dev/null @@ -1,287 +0,0 @@ -import { EventEmitter } from 'events'; - -export interface BarData { - symbol: string; - timestamp: Date; - open: number; - high: number; - low: number; - close: number; - volume: number; -} - -export interface Position { - symbol: string; - quantity: number; - avgPrice: number; - side: 'LONG' | 'SHORT'; - entryTime: Date; - unrealizedPnL?: number; - realizedPnL?: number; -} - -export interface Order { - id: string; - symbol: string; - side: 'BUY' | 'SELL'; - quantity: number; - price?: number; // Market order if undefined - type: 'MARKET' | 'LIMIT' | 'STOP' | 'STOP_LIMIT'; - status: 'PENDING' | 'FILLED' | 'CANCELLED' | 'REJECTED'; - timestamp: Date; - fillPrice?: number; - fillTime?: Date; -} - -export interface StrategyContext { - currentTime: Date; - portfolio: { - cash: number; - positions: Map; - totalValue: number; - }; - marketData: Map; // Historical data for each symbol - indicators: Map; // Cached indicator values -} - -export interface StrategyParameters { - [key: string]: number | string | boolean | any[]; -} - -export interface StrategyMetrics { - totalReturn: number; - totalTrades: number; - winningTrades: number; - losingTrades: number; - winRate: number; - avgWin: number; - avgLoss: number; - profitFactor: number; - sharpeRatio: number; - maxDrawdown: number; - maxDrawdownDuration: number; - calmarRatio: number; - sortinoRatio: number; - beta: number; - alpha: number; - volatility: number; -} - -export abstract class BaseStrategy extends EventEmitter { - public readonly id: string; - public name: string; - public description: string; - public symbols: string[]; - public parameters: StrategyParameters; - - protected context: StrategyContext; - protected isInitialized: boolean = false; - - constructor( - id: string, - name: string, - description: string, - symbols: string[], - parameters: StrategyParameters = {} - ) { - super(); - this.id = id; - this.name = name; - this.description = description; - this.symbols = symbols; - this.parameters = parameters; - - this.context = { - currentTime: new Date(), - portfolio: { - cash: 100000, // Default starting capital - positions: new Map(), - totalValue: 100000 - }, - marketData: new Map(), - indicators: new Map() - }; - } - - // Abstract methods that must be implemented by strategy subclasses - abstract initialize(): Promise; - abstract onBar(bar: BarData): Promise; - abstract onOrderFilled(order: Order): Promise; - abstract cleanup(): Promise; - - // Lifecycle methods - async start(): Promise { - if (!this.isInitialized) { - await this.initialize(); - this.isInitialized = true; - } - this.emit('started', { strategyId: this.id }); - } - - async stop(): Promise { - await this.cleanup(); - this.emit('stopped', { strategyId: this.id }); - } - - // Market data management - addBar(bar: BarData): void { - this.context.currentTime = bar.timestamp; - - if (!this.context.marketData.has(bar.symbol)) { - this.context.marketData.set(bar.symbol, []); - } - - const bars = this.context.marketData.get(bar.symbol)!; - bars.push(bar); - - // Keep only last 1000 bars to manage memory - if (bars.length > 1000) { - bars.shift(); - } - } - - // Portfolio management helpers - protected getCurrentPrice(symbol: string): number | null { - const bars = this.context.marketData.get(symbol); - return bars && bars.length > 0 ? bars[bars.length - 1].close : null; - } - - protected getPosition(symbol: string): Position | null { - return this.context.portfolio.positions.get(symbol) || null; - } - - protected hasPosition(symbol: string): boolean { - return this.context.portfolio.positions.has(symbol); - } - - protected getAvailableCash(): number { - return this.context.portfolio.cash; - } - - protected calculatePositionValue(symbol: string): number { - const position = this.getPosition(symbol); - const currentPrice = this.getCurrentPrice(symbol); - - if (!position || !currentPrice) return 0; - - return position.quantity * currentPrice; - } - - protected updatePortfolioValue(): void { - let totalValue = this.context.portfolio.cash; - - for (const [symbol, position] of this.context.portfolio.positions) { - const currentPrice = this.getCurrentPrice(symbol); - if (currentPrice) { - totalValue += position.quantity * currentPrice; - } - } - - this.context.portfolio.totalValue = totalValue; - } - - // Order creation helpers - protected createMarketOrder(symbol: string, side: 'BUY' | 'SELL', quantity: number): Order { - return { - id: this.generateOrderId(), - symbol, - side, - quantity: Math.abs(quantity), - type: 'MARKET', - status: 'PENDING', - timestamp: this.context.currentTime - }; - } - - protected createLimitOrder( - symbol: string, - side: 'BUY' | 'SELL', - quantity: number, - price: number - ): Order { - return { - id: this.generateOrderId(), - symbol, - side, - quantity: Math.abs(quantity), - price, - type: 'LIMIT', - status: 'PENDING', - timestamp: this.context.currentTime - }; - } - - protected createStopOrder( - symbol: string, - side: 'BUY' | 'SELL', - quantity: number, - stopPrice: number - ): Order { - return { - id: this.generateOrderId(), - symbol, - side, - quantity: Math.abs(quantity), - price: stopPrice, - type: 'STOP', - status: 'PENDING', - timestamp: this.context.currentTime - }; - } - - private generateOrderId(): string { - return `${this.id}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - // Utility methods for common strategy patterns - protected getBarsSince(symbol: string, periods: number): BarData[] { - const bars = this.context.marketData.get(symbol) || []; - return bars.slice(-periods); - } - - protected getReturns(symbol: string, periods: number): number[] { - const bars = this.getBarsSince(symbol, periods + 1); - const returns: number[] = []; - - for (let i = 1; i < bars.length; i++) { - const returnPct = (bars[i].close - bars[i - 1].close) / bars[i - 1].close; - returns.push(returnPct); - } - - return returns; - } - - protected getVolatility(symbol: string, periods: number): number { - const returns = this.getReturns(symbol, periods); - if (returns.length === 0) return 0; - - const mean = returns.reduce((sum, ret) => sum + ret, 0) / returns.length; - const variance = returns.reduce((sum, ret) => sum + Math.pow(ret - mean, 2), 0) / returns.length; - - return Math.sqrt(variance * 252); // Annualized volatility - } - - // Parameter validation - protected validateParameters(): boolean { - // Override in subclasses for parameter validation - return true; - } - - // Get strategy state for serialization - getState() { - return { - id: this.id, - name: this.name, - description: this.description, - symbols: this.symbols, - parameters: this.parameters, - isInitialized: this.isInitialized, - currentTime: this.context.currentTime, - portfolio: { - cash: this.context.portfolio.cash, - totalValue: this.context.portfolio.totalValue, - positions: Array.from(this.context.portfolio.positions.entries()) - } - }; - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/analysis/TechnicalIndicators.ts b/apps/intelligence-services/strategy-orchestrator/src/core/analysis/TechnicalIndicators.ts deleted file mode 100644 index fdb8553..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/analysis/TechnicalIndicators.ts +++ /dev/null @@ -1,362 +0,0 @@ -import { BarData } from '../Strategy'; - -export class TechnicalIndicators { - /** - * Calculate Simple Moving Average (SMA) - * @param prices Array of price values - * @param period Number of periods for calculation - * @returns Array of SMA values - */ - static sma(prices: number[], period: number): number[] { - if (period <= 0 || prices.length === 0) return []; - - const result: number[] = []; - - // Not enough data for calculation - if (prices.length < period) { - return Array(prices.length).fill(NaN); - } - - // Calculate first SMA - let sum = 0; - for (let i = 0; i < period; i++) { - sum += prices[i]; - } - - result.push(sum / period); - - // Calculate subsequent SMAs using previous sum - for (let i = period; i < prices.length; i++) { - sum = sum - prices[i - period] + prices[i]; - result.push(sum / period); - } - - // Fill beginning with NaN - const nanValues = Array(period - 1).fill(NaN); - - return [...nanValues, ...result]; - } - - /** - * Calculate Exponential Moving Average (EMA) - * @param prices Array of price values - * @param period Number of periods for calculation - * @returns Array of EMA values - */ - static ema(prices: number[], period: number): number[] { - if (period <= 0 || prices.length === 0) return []; - - const result: number[] = []; - const multiplier = 2 / (period + 1); - - // Not enough data for calculation - if (prices.length < period) { - return Array(prices.length).fill(NaN); - } - - // Calculate SMA for first EMA value - let sum = 0; - for (let i = 0; i < period; i++) { - sum += prices[i]; - } - - // First EMA is SMA - let ema = sum / period; - result.push(ema); - - // Calculate subsequent EMAs - for (let i = period; i < prices.length; i++) { - ema = (prices[i] - ema) * multiplier + ema; - result.push(ema); - } - - // Fill beginning with NaN - const nanValues = Array(period - 1).fill(NaN); - - return [...nanValues, ...result]; - } - - /** - * Calculate Relative Strength Index (RSI) - * @param prices Array of price values - * @param period Number of periods for calculation - * @returns Array of RSI values - */ - static rsi(prices: number[], period: number): number[] { - if (period <= 0 || prices.length < period + 1) { - return Array(prices.length).fill(NaN); - } - - const result: number[] = []; - const gains: number[] = []; - const losses: number[] = []; - - // Calculate price changes - for (let i = 1; i < prices.length; i++) { - const change = prices[i] - prices[i - 1]; - gains.push(change > 0 ? change : 0); - losses.push(change < 0 ? Math.abs(change) : 0); - } - - // Not enough data - if (gains.length < period) { - return Array(prices.length).fill(NaN); - } - - // Calculate first average gain and loss - let avgGain = 0; - let avgLoss = 0; - - for (let i = 0; i < period; i++) { - avgGain += gains[i]; - avgLoss += losses[i]; - } - - avgGain /= period; - avgLoss /= period; - - // Calculate first RSI - let rs = avgGain / (avgLoss === 0 ? 0.001 : avgLoss); // Avoid division by zero - let rsi = 100 - (100 / (1 + rs)); - result.push(rsi); - - // Calculate subsequent RSIs - for (let i = period; i < gains.length; i++) { - // Smooth averages - avgGain = ((avgGain * (period - 1)) + gains[i]) / period; - avgLoss = ((avgLoss * (period - 1)) + losses[i]) / period; - - // Calculate RS and RSI - rs = avgGain / (avgLoss === 0 ? 0.001 : avgLoss); - rsi = 100 - (100 / (1 + rs)); - result.push(rsi); - } - - // Fill beginning with NaN - const nanValues = Array(period).fill(NaN); - - return [...nanValues, ...result]; - } - - /** - * Calculate Moving Average Convergence Divergence (MACD) - * @param prices Array of price values - * @param fastPeriod Fast EMA period (default: 12) - * @param slowPeriod Slow EMA period (default: 26) - * @param signalPeriod Signal line period (default: 9) - * @returns Object containing MACD line, signal line, and histogram - */ - static macd( - prices: number[], - fastPeriod: number = 12, - slowPeriod: number = 26, - signalPeriod: number = 9 - ): { macdLine: number[], signalLine: number[], histogram: number[] } { - // Calculate EMAs - const fastEMA = this.ema(prices, fastPeriod); - const slowEMA = this.ema(prices, slowPeriod); - - // Calculate MACD line (fast EMA - slow EMA) - const macdLine: number[] = []; - for (let i = 0; i < prices.length; i++) { - macdLine.push(isNaN(fastEMA[i]) || isNaN(slowEMA[i]) - ? NaN - : fastEMA[i] - slowEMA[i]); - } - - // Calculate signal line (EMA of MACD line) - const signalLine = this.ema(macdLine.filter(val => !isNaN(val)), signalPeriod); - - // Pad signal line with NaNs to match original length - const paddedSignalLine = Array(prices.length - signalLine.length).fill(NaN).concat(signalLine); - - // Calculate histogram (MACD line - signal line) - const histogram: number[] = []; - for (let i = 0; i < prices.length; i++) { - histogram.push(isNaN(macdLine[i]) || isNaN(paddedSignalLine[i]) - ? NaN - : macdLine[i] - paddedSignalLine[i]); - } - - return { - macdLine, - signalLine: paddedSignalLine, - histogram - }; - } - - /** - * Calculate Bollinger Bands - * @param prices Array of price values - * @param period SMA period (default: 20) - * @param stdDevMultiplier Standard deviation multiplier (default: 2) - * @returns Object containing upper band, middle band, and lower band - */ - static bollingerBands( - prices: number[], - period: number = 20, - stdDevMultiplier: number = 2 - ): { upper: number[], middle: number[], lower: number[] } { - // Calculate middle band (SMA) - const middle = this.sma(prices, period); - - // Calculate standard deviation for each point - const upper: number[] = []; - const lower: number[] = []; - - for (let i = 0; i < prices.length; i++) { - if (isNaN(middle[i])) { - upper.push(NaN); - lower.push(NaN); - continue; - } - - // Calculate standard deviation using values in the period window - let stdDev = 0; - let count = 0; - - // Start index for the window - const startIdx = Math.max(0, i - period + 1); - - for (let j = startIdx; j <= i; j++) { - stdDev += Math.pow(prices[j] - middle[i], 2); - count++; - } - - stdDev = Math.sqrt(stdDev / count); - - // Calculate bands - upper.push(middle[i] + (stdDevMultiplier * stdDev)); - lower.push(middle[i] - (stdDevMultiplier * stdDev)); - } - - return { upper, middle, lower }; - } - - /** - * Calculate Average True Range (ATR) - * @param bars Array of BarData objects - * @param period Number of periods for calculation - * @returns Array of ATR values - */ - static atr(bars: BarData[], period: number): number[] { - if (period <= 0 || bars.length < 2) { - return Array(bars.length).fill(NaN); - } - - // Calculate True Range for each bar - const trueRanges: number[] = []; - - // First TR is high - low - trueRanges.push(bars[0].high - bars[0].low); - - // Calculate remaining TRs - for (let i = 1; i < bars.length; i++) { - const currentHigh = bars[i].high; - const currentLow = bars[i].low; - const previousClose = bars[i - 1].close; - - const tr1 = currentHigh - currentLow; - const tr2 = Math.abs(currentHigh - previousClose); - const tr3 = Math.abs(currentLow - previousClose); - - const tr = Math.max(tr1, tr2, tr3); - trueRanges.push(tr); - } - - // Calculate ATR (first value is simple average) - const result: number[] = []; - - // Not enough data - if (trueRanges.length < period) { - return Array(bars.length).fill(NaN); - } - - // First ATR is simple average of true ranges - let atr = 0; - for (let i = 0; i < period; i++) { - atr += trueRanges[i]; - } - atr /= period; - result.push(atr); - - // Calculate subsequent ATRs using smoothing - for (let i = period; i < trueRanges.length; i++) { - atr = ((atr * (period - 1)) + trueRanges[i]) / period; - result.push(atr); - } - - // Fill beginning with NaN - const nanValues = Array(period).fill(NaN); - - return [...nanValues, ...result]; - } - - /** - * Calculate Stochastic Oscillator - * @param bars Array of BarData objects - * @param period %K period (default: 14) - * @param smoothK %K smoothing (default: 3) - * @param smoothD %D period (default: 3) - * @returns Object containing %K and %D values - */ - static stochastic( - bars: BarData[], - period: number = 14, - smoothK: number = 3, - smoothD: number = 3 - ): { k: number[], d: number[] } { - if (period <= 0 || bars.length < period) { - return { k: Array(bars.length).fill(NaN), d: Array(bars.length).fill(NaN) }; - } - - const rawK: number[] = []; - - // Calculate raw %K values - for (let i = period - 1; i < bars.length; i++) { - let highest = -Infinity; - let lowest = Infinity; - - // Find highest high and lowest low in the period - for (let j = i - (period - 1); j <= i; j++) { - highest = Math.max(highest, bars[j].high); - lowest = Math.min(lowest, bars[j].low); - } - - // Calculate raw %K - const currentClose = bars[i].close; - const rawKValue = 100 * ((currentClose - lowest) / (highest - lowest)); - rawK.push(rawKValue); - } - - // Fill beginning with NaN - const nanValues = Array(period - 1).fill(NaN); - const fullRawK = [...nanValues, ...rawK]; - - // Apply smoothing to %K (SMA of raw %K) - const filteredK = fullRawK.filter(val => !isNaN(val)); - let k = this.sma(filteredK, smoothK); - - // Pad with NaNs - k = [...Array(fullRawK.length - k.length).fill(NaN), ...k]; - - // Calculate %D (SMA of %K) - const filteredSmoothedK = k.filter(val => !isNaN(val)); - let d = this.sma(filteredSmoothedK, smoothD); - - // Pad with NaNs - d = [...Array(k.length - d.length).fill(NaN), ...d]; - - return { k, d }; - } - - /** - * Extract specific price from bars (e.g., close, open, high, low) - * @param bars Array of BarData objects - * @param field Price field to extract - * @returns Array of extracted price values - */ - static extractPrice(bars: BarData[], field: 'open' | 'high' | 'low' | 'close' = 'close'): number[] { - return bars.map(bar => bar[field]); - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/BacktestEngine.ts b/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/BacktestEngine.ts deleted file mode 100644 index 90d1bcf..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/BacktestEngine.ts +++ /dev/null @@ -1,604 +0,0 @@ -import { EventEmitter } from 'events'; -import { BaseStrategy } from '../Strategy'; -import { BarData, Order, Position } from '../Strategy'; - -export interface BacktestConfig { - startDate: Date; - endDate: Date; - symbols: string[]; - initialCapital: number; - commission: number; // Per trade commission (percentage) - slippage: number; // Slippage model (percentage) - dataResolution: '1m' | '5m' | '15m' | '30m' | '1h' | '4h' | '1d'; - mode: 'event' | 'vector'; -} - -export interface BacktestResult { - strategyId: string; - startDate: Date; - endDate: Date; - duration: number; // In milliseconds - initialCapital: number; - finalCapital: number; - totalReturn: number; - annualizedReturn: number; - sharpeRatio: number; - maxDrawdown: number; - maxDrawdownDuration: number; // In days - winRate: number; - totalTrades: number; - winningTrades: number; - losingTrades: number; - averageWinningTrade: number; - averageLosingTrade: number; - profitFactor: number; - dailyReturns: Array<{ date: Date; return: number }>; - trades: Array<{ - symbol: string; - entryTime: Date; - entryPrice: number; - exitTime: Date; - exitPrice: number; - quantity: number; - pnl: number; - pnlPercent: number; - }>; -} - -export interface BacktestProgress { - progress: number; // 0-100 - currentDate: Date; - processingSpeed: number; // Bars per second - estimatedTimeRemaining: number; // milliseconds - currentCapital: number; - currentReturn: number; - currentDrawdown: number; -} - -export interface DataFeed { - getHistoricalData(symbol: string, resolution: string, start: Date, end: Date): Promise; - hasDataFor(symbol: string, resolution: string, start: Date, end: Date): Promise; -} - -export class BacktestEngine extends EventEmitter { - private config: BacktestConfig; - private strategy: BaseStrategy; - private dataFeed: DataFeed; - private isRunning: boolean = false; - private barBuffer: Map = new Map(); - private pendingOrders: Order[] = []; - private filledOrders: Order[] = []; - private currentTime: Date; - private startTime: number = 0; // For performance tracking - private processedBars: number = 0; - private marketData: Map = new Map(); - - // Results tracking - private initialCapital: number; - private currentCapital: number; - private positions = new Map(); - private trades: BacktestResult['trades'] = []; - private dailyReturns: BacktestResult['dailyReturns'] = []; - private previousPortfolioValue: number; - private highWaterMark: number; - private maxDrawdown: number = 0; - private drawdownStartTime: Date | null = null; - private maxDrawdownDuration: number = 0; - private winningTrades: number = 0; - private losingTrades: number = 0; - private breakEvenTrades: number = 0; - private totalProfits: number = 0; - private totalLosses: number = 0; - - constructor(strategy: BaseStrategy, config: BacktestConfig, dataFeed: DataFeed) { - super(); - this.strategy = strategy; - this.config = config; - this.dataFeed = dataFeed; - this.currentTime = new Date(config.startDate); - this.initialCapital = config.initialCapital; - this.currentCapital = config.initialCapital; - this.previousPortfolioValue = config.initialCapital; - this.highWaterMark = config.initialCapital; - } - - async run(): Promise { - if (this.isRunning) { - throw new Error('Backtest is already running'); - } - - this.isRunning = true; - this.startTime = Date.now(); - this.emit('started', { strategyId: this.strategy.id, config: this.config }); - - try { - // Load data based on configured mode - if (this.config.mode === 'event') { - await this.runEventBased(); - } else { - await this.runVectorized(); - } - - const result = this.generateResults(); - this.emit('completed', { strategyId: this.strategy.id, result }); - this.isRunning = false; - return result; - } catch (error) { - this.isRunning = false; - this.emit('error', { strategyId: this.strategy.id, error }); - throw error; - } - } - - private async runEventBased(): Promise { - // Load market data for all symbols - await this.loadMarketData(); - - // Initialize the strategy - await this.strategy.start(); - - // Create a merged timeline of all bars across all symbols, sorted by timestamp - const timeline = this.createMergedTimeline(); - - // Process each event in chronological order - let lastProgressUpdate = Date.now(); - let prevDate = new Date(0); - - for (let i = 0; i < timeline.length; i++) { - const bar = timeline[i]; - this.currentTime = bar.timestamp; - - // Process any pending orders - await this.processOrders(bar); - - // Update positions with current prices - this.updatePositions(bar); - - // If we've crossed to a new day, calculate daily return - if (this.currentTime.toDateString() !== prevDate.toDateString()) { - this.calculateDailyReturn(); - prevDate = this.currentTime; - } - - // Send the new bar to the strategy - const orders = await this.strategy.onBar(bar); - - // Add any new orders to the pending orders queue - if (orders && orders.length > 0) { - this.pendingOrders.push(...orders); - } - - // Update progress periodically - if (Date.now() - lastProgressUpdate > 1000) { // Update every second - this.updateProgress(i / timeline.length); - lastProgressUpdate = Date.now(); - } - } - - // Process any remaining orders - for (const order of this.pendingOrders) { - await this.processOrder(order); - } - - // Close any remaining positions at the last known price - await this.closeAllPositions(); - - // Clean up strategy - await this.strategy.stop(); - } - - private async runVectorized(): Promise { - // Load market data for all symbols - await this.loadMarketData(); - - // To implement a vectorized approach, we need to: - // 1. Pre-compute technical indicators - // 2. Generate buy/sell signals for the entire dataset - // 3. Calculate portfolio values based on signals - - // This is a simplified implementation since specific vectorized strategies - // will need to be implemented separately based on the strategy type - - const timeline = this.createMergedTimeline(); - const startTime = Date.now(); - - // Initialize variables for tracking performance - let currentPositions = new Map(); - let currentCash = this.initialCapital; - let prevPortfolioValue = this.initialCapital; - let highWaterMark = this.initialCapital; - let maxDrawdown = 0; - let maxDrawdownStartDate = new Date(); - let maxDrawdownEndDate = new Date(); - let currentDrawdownStart = new Date(); - - // Pre-process data (this would be implemented by the specific strategy) - const allBars = new Map(); - for (const symbol of this.config.symbols) { - allBars.set(symbol, this.marketData.get(symbol) || []); - } - - // Apply strategy logic (vectorized implementation would be here) - // For now, we'll just simulate the processing - - this.emit('completed', { message: 'Vectorized backtest completed in fast mode' }); - } - - private async loadMarketData(): Promise { - for (const symbol of this.config.symbols) { - this.emit('loading', { symbol, resolution: this.config.dataResolution }); - - // Check if data is available - const hasData = await this.dataFeed.hasDataFor( - symbol, - this.config.dataResolution, - this.config.startDate, - this.config.endDate - ); - - if (!hasData) { - throw new Error(`No data available for ${symbol} at resolution ${this.config.dataResolution}`); - } - - // Load data - const data = await this.dataFeed.getHistoricalData( - symbol, - this.config.dataResolution, - this.config.startDate, - this.config.endDate - ); - - this.marketData.set(symbol, data); - this.emit('loaded', { symbol, count: data.length }); - } - } - - private createMergedTimeline(): BarData[] { - const allBars: BarData[] = []; - - for (const [symbol, bars] of this.marketData.entries()) { - allBars.push(...bars); - } - - // Sort by timestamp - return allBars.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); - } - - private async processOrders(currentBar: BarData): Promise { - // Find orders for the current symbol - const ordersToProcess = this.pendingOrders.filter(order => order.symbol === currentBar.symbol); - - if (ordersToProcess.length === 0) return; - - // Remove these orders from pendingOrders - this.pendingOrders = this.pendingOrders.filter(order => order.symbol !== currentBar.symbol); - - // Process each order - for (const order of ordersToProcess) { - await this.processOrder(order); - } - } - - private async processOrder(order: Order): Promise { - // Get the latest price for the symbol - const latestBars = this.marketData.get(order.symbol); - if (!latestBars || latestBars.length === 0) { - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'No market data available' }); - return; - } - - // Find the bar closest to the order time - const bar = latestBars.find(b => - b.timestamp.getTime() >= order.timestamp.getTime() - ) || latestBars[latestBars.length - 1]; - - // Calculate fill price with slippage - let fillPrice: number; - if (order.type === 'MARKET') { - // Apply slippage model - const slippageFactor = 1 + (order.side === 'BUY' ? this.config.slippage : -this.config.slippage); - fillPrice = bar.close * slippageFactor; - } else if (order.type === 'LIMIT' && order.price !== undefined) { - // For limit orders, check if the price was reached - if ((order.side === 'BUY' && bar.low <= order.price) || - (order.side === 'SELL' && bar.high >= order.price)) { - fillPrice = order.price; - } else { - // Limit price not reached - return; - } - } else { - // Other order types not implemented - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'Order type not supported' }); - return; - } - - // Calculate commission - const orderValue = order.quantity * fillPrice; - const commission = orderValue * this.config.commission; - - // Check if we have enough cash for BUY orders - if (order.side === 'BUY') { - const totalCost = orderValue + commission; - if (totalCost > this.currentCapital) { - // Not enough cash - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'Insufficient funds' }); - return; - } - - // Update cash - this.currentCapital -= totalCost; - - // Update or create position - const existingPosition = this.positions.get(order.symbol); - if (existingPosition) { - // Update existing position (average down) - const totalShares = existingPosition.quantity + order.quantity; - const totalCost = (existingPosition.quantity * existingPosition.avgPrice) + (order.quantity * fillPrice); - existingPosition.avgPrice = totalCost / totalShares; - existingPosition.quantity = totalShares; - } else { - // Create new position - this.positions.set(order.symbol, { - symbol: order.symbol, - quantity: order.quantity, - avgPrice: fillPrice, - side: 'LONG', - entryTime: this.currentTime - }); - } - } else if (order.side === 'SELL') { - const position = this.positions.get(order.symbol); - - if (!position || position.quantity < order.quantity) { - // Not enough shares to sell - order.status = 'REJECTED'; - this.emit('orderRejected', { order, reason: 'Insufficient position' }); - return; - } - - // Calculate P&L - const pnl = (fillPrice - position.avgPrice) * order.quantity; - - // Update cash - this.currentCapital += orderValue - commission; - - // Update position - position.quantity -= order.quantity; - - if (position.quantity === 0) { - // Position closed, record the trade - this.positions.delete(order.symbol); - - this.trades.push({ - symbol: order.symbol, - entryTime: position.entryTime, - entryPrice: position.avgPrice, - exitTime: this.currentTime, - exitPrice: fillPrice, - quantity: order.quantity, - pnl: pnl, - pnlPercent: (pnl / (position.avgPrice * order.quantity)) * 100 - }); - - // Update statistics - if (pnl > 0) { - this.winningTrades++; - this.totalProfits += pnl; - } else if (pnl < 0) { - this.losingTrades++; - this.totalLosses -= pnl; // Make positive for easier calculations - } else { - this.breakEvenTrades++; - } - } - } - - // Mark order as filled - order.status = 'FILLED'; - order.fillPrice = fillPrice; - order.fillTime = this.currentTime; - this.filledOrders.push(order); - - // Notify strategy - await this.strategy.onOrderFilled(order); - - this.emit('orderFilled', { order }); - } - - private updatePositions(currentBar: BarData): void { - // Update the unrealized P&L for positions in this symbol - const position = this.positions.get(currentBar.symbol); - if (position) { - const currentPrice = currentBar.close; - const unrealizedPnL = (currentPrice - position.avgPrice) * position.quantity; - position.unrealizedPnL = unrealizedPnL; - } - - // Calculate total portfolio value - const portfolioValue = this.calculatePortfolioValue(); - - // Check for new high water mark - if (portfolioValue > this.highWaterMark) { - this.highWaterMark = portfolioValue; - this.drawdownStartTime = null; - } - - // Check for drawdown - if (this.drawdownStartTime === null && portfolioValue < this.highWaterMark) { - this.drawdownStartTime = this.currentTime; - } - - // Update max drawdown - if (this.highWaterMark > 0) { - const currentDrawdown = (this.highWaterMark - portfolioValue) / this.highWaterMark; - if (currentDrawdown > this.maxDrawdown) { - this.maxDrawdown = currentDrawdown; - - // Calculate drawdown duration - if (this.drawdownStartTime !== null) { - const drawdownDuration = (this.currentTime.getTime() - this.drawdownStartTime.getTime()) / (1000 * 60 * 60 * 24); // In days - if (drawdownDuration > this.maxDrawdownDuration) { - this.maxDrawdownDuration = drawdownDuration; - } - } - } - } - - this.previousPortfolioValue = portfolioValue; - } - - private calculatePortfolioValue(): number { - let totalValue = this.currentCapital; - - // Add the current value of all positions - for (const [symbol, position] of this.positions.entries()) { - // Find the latest price for this symbol - const bars = this.marketData.get(symbol); - if (bars && bars.length > 0) { - const latestBar = bars[bars.length - 1]; - totalValue += position.quantity * latestBar.close; - } else { - // If no price data, use the average price (not ideal but better than nothing) - totalValue += position.quantity * position.avgPrice; - } - } - - return totalValue; - } - - private calculateDailyReturn(): void { - const portfolioValue = this.calculatePortfolioValue(); - const dailyReturn = (portfolioValue - this.previousPortfolioValue) / this.previousPortfolioValue; - - this.dailyReturns.push({ - date: new Date(this.currentTime), - return: dailyReturn - }); - - this.previousPortfolioValue = portfolioValue; - } - - private async closeAllPositions(): Promise { - for (const [symbol, position] of this.positions.entries()) { - // Find the latest price - const bars = this.marketData.get(symbol); - if (!bars || bars.length === 0) continue; - - const lastBar = bars[bars.length - 1]; - const closePrice = lastBar.close; - - // Calculate P&L - const pnl = (closePrice - position.avgPrice) * position.quantity; - - // Update cash - this.currentCapital += position.quantity * closePrice; - - // Record the trade - this.trades.push({ - symbol, - entryTime: position.entryTime, - entryPrice: position.avgPrice, - exitTime: this.currentTime, - exitPrice: closePrice, - quantity: position.quantity, - pnl, - pnlPercent: (pnl / (position.avgPrice * position.quantity)) * 100 - }); - - // Update statistics - if (pnl > 0) { - this.winningTrades++; - this.totalProfits += pnl; - } else if (pnl < 0) { - this.losingTrades++; - this.totalLosses -= pnl; // Make positive for easier calculations - } else { - this.breakEvenTrades++; - } - } - - // Clear positions - this.positions.clear(); - } - - private updateProgress(progress: number): void { - const currentPortfolioValue = this.calculatePortfolioValue(); - const currentDrawdown = this.highWaterMark > 0 - ? (this.highWaterMark - currentPortfolioValue) / this.highWaterMark - : 0; - - const elapsedMs = Date.now() - this.startTime; - const totalEstimatedMs = elapsedMs / progress; - const remainingMs = totalEstimatedMs - elapsedMs; - - this.emit('progress', { - progress: progress * 100, - currentDate: this.currentTime, - processingSpeed: this.processedBars / (elapsedMs / 1000), - estimatedTimeRemaining: remainingMs, - currentCapital: this.currentCapital, - currentReturn: (currentPortfolioValue - this.initialCapital) / this.initialCapital, - currentDrawdown - } as BacktestProgress); - } - - private generateResults(): BacktestResult { - const currentPortfolioValue = this.calculatePortfolioValue(); - const totalReturn = (currentPortfolioValue - this.initialCapital) / this.initialCapital; - - // Calculate annualized return - const days = (this.config.endDate.getTime() - this.config.startDate.getTime()) / (1000 * 60 * 60 * 24); - const annualizedReturn = Math.pow(1 + totalReturn, 365 / days) - 1; - - // Calculate Sharpe Ratio - let sharpeRatio = 0; - if (this.dailyReturns.length > 1) { - const dailyReturnValues = this.dailyReturns.map(dr => dr.return); - const avgDailyReturn = dailyReturnValues.reduce((sum, ret) => sum + ret, 0) / dailyReturnValues.length; - const stdDev = Math.sqrt( - dailyReturnValues.reduce((sum, ret) => sum + Math.pow(ret - avgDailyReturn, 2), 0) / dailyReturnValues.length - ); - - // Annualize - sharpeRatio = stdDev > 0 - ? (avgDailyReturn * 252) / (stdDev * Math.sqrt(252)) - : 0; - } - - // Calculate win rate and profit factor - const totalTrades = this.winningTrades + this.losingTrades + this.breakEvenTrades; - const winRate = totalTrades > 0 ? this.winningTrades / totalTrades : 0; - const profitFactor = this.totalLosses > 0 ? this.totalProfits / this.totalLosses : (this.totalProfits > 0 ? Infinity : 0); - - // Calculate average winning and losing trade - const avgWinningTrade = this.winningTrades > 0 ? this.totalProfits / this.winningTrades : 0; - const avgLosingTrade = this.losingTrades > 0 ? this.totalLosses / this.losingTrades : 0; - - return { - strategyId: this.strategy.id, - startDate: this.config.startDate, - endDate: this.config.endDate, - duration: Date.now() - this.startTime, - initialCapital: this.initialCapital, - finalCapital: currentPortfolioValue, - totalReturn, - annualizedReturn, - sharpeRatio, - maxDrawdown: this.maxDrawdown, - maxDrawdownDuration: this.maxDrawdownDuration, - winRate, - totalTrades, - winningTrades: this.winningTrades, - losingTrades: this.losingTrades, - averageWinningTrade: avgWinningTrade, - averageLosingTrade: avgLosingTrade, - profitFactor, - dailyReturns: this.dailyReturns, - trades: this.trades - }; - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/BacktestService.ts b/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/BacktestService.ts deleted file mode 100644 index 7dbb639..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/BacktestService.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { BaseStrategy } from '../Strategy'; -import { BacktestConfig, BacktestEngine, BacktestResult } from './BacktestEngine'; -import { MarketDataFeed } from './MarketDataFeed'; -import { StrategyRegistry, StrategyType } from '../strategies/StrategyRegistry'; - -export interface BacktestRequest { - strategyType: StrategyType; - strategyParams: Record; - symbols: string[]; - startDate: Date | string; - endDate: Date | string; - initialCapital: number; - dataResolution: '1m' | '5m' | '15m' | '30m' | '1h' | '4h' | '1d'; - commission: number; - slippage: number; - mode: 'event' | 'vector'; -} - -/** - * Backtesting Service - * - * A service that handles backtesting requests and manages backtesting sessions. - */ -export class BacktestService { - private readonly strategyRegistry: StrategyRegistry; - private readonly dataFeed: MarketDataFeed; - private readonly activeBacktests: Map = new Map(); - - constructor(apiBaseUrl: string = 'http://localhost:3001/api') { - this.strategyRegistry = StrategyRegistry.getInstance(); - this.dataFeed = new MarketDataFeed(apiBaseUrl); - } - - /** - * Run a backtest based on a request - */ - async runBacktest(request: BacktestRequest): Promise { - // Create a strategy instance - const strategyId = `backtest_${Date.now()}`; - const strategy = this.strategyRegistry.createStrategy( - request.strategyType, - strategyId, - `Backtest ${request.strategyType}`, - `Generated backtest for ${request.symbols.join(', ')}`, - request.symbols, - request.strategyParams - ); - - // Parse dates if they are strings - const startDate = typeof request.startDate === 'string' - ? new Date(request.startDate) - : request.startDate; - - const endDate = typeof request.endDate === 'string' - ? new Date(request.endDate) - : request.endDate; - - // Create backtest configuration - const config: BacktestConfig = { - startDate, - endDate, - symbols: request.symbols, - initialCapital: request.initialCapital, - commission: request.commission, - slippage: request.slippage, - dataResolution: request.dataResolution, - mode: request.mode - }; - - // Create and run the backtest engine - const engine = new BacktestEngine(strategy, config, this.dataFeed); - this.activeBacktests.set(strategyId, engine); - - try { - // Set up event forwarding - const forwardEvents = (eventName: string) => { - engine.on(eventName, (data) => { - console.log(`[Backtest ${strategyId}] ${eventName}:`, data); - }); - }; - - forwardEvents('started'); - forwardEvents('loading'); - forwardEvents('loaded'); - forwardEvents('progress'); - forwardEvents('orderFilled'); - forwardEvents('orderRejected'); - forwardEvents('completed'); - forwardEvents('error'); - - // Run the backtest - const result = await engine.run(); - - // Clean up - this.activeBacktests.delete(strategyId); - - return result; - } catch (error) { - this.activeBacktests.delete(strategyId); - throw error; - } - } - - /** - * Optimize a strategy by running multiple backtests with different parameters - */ - async optimizeStrategy( - baseRequest: BacktestRequest, - parameterGrid: Record - ): Promise }>> { - const results: Array }> = []; - - // Generate parameter combinations - const paramKeys = Object.keys(parameterGrid); - const combinations = this.generateParameterCombinations(parameterGrid, paramKeys); - - // Run backtest for each combination - for (const paramSet of combinations) { - const request = { - ...baseRequest, - strategyParams: { - ...baseRequest.strategyParams, - ...paramSet - } - }; - - try { - const result = await this.runBacktest(request); - results.push({ - ...result, - parameters: paramSet - }); - } catch (error) { - console.error(`Optimization failed for parameters:`, paramSet, error); - } - } - - // Sort by performance metric (e.g., Sharpe ratio) - return results.sort((a, b) => b.sharpeRatio - a.sharpeRatio); - } - - /** - * Generate all combinations of parameters for grid search - */ - private generateParameterCombinations( - grid: Record, - keys: string[], - current: Record = {}, - index: number = 0, - result: Record[] = [] - ): Record[] { - if (index === keys.length) { - result.push({ ...current }); - return result; - } - - const key = keys[index]; - const values = grid[key]; - - for (const value of values) { - current[key] = value; - this.generateParameterCombinations(grid, keys, current, index + 1, result); - } - - return result; - } - - /** - * Get an active backtest engine by ID - */ - getBacktestEngine(id: string): BacktestEngine | undefined { - return this.activeBacktests.get(id); - } - - /** - * Cancel a running backtest - */ - cancelBacktest(id: string): boolean { - const engine = this.activeBacktests.get(id); - if (!engine) return false; - - // No explicit cancel method on engine, but we can clean up - this.activeBacktests.delete(id); - return true; - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/MarketDataFeed.ts b/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/MarketDataFeed.ts deleted file mode 100644 index 7d15369..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/MarketDataFeed.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { BarData } from '../Strategy'; -import { DataFeed } from './BacktestEngine'; -import axios from 'axios'; - -export class MarketDataFeed implements DataFeed { - private readonly apiBaseUrl: string; - private cache: Map = new Map(); - - constructor(apiBaseUrl: string = 'http://localhost:3001/api') { - this.apiBaseUrl = apiBaseUrl; - } - - async getHistoricalData(symbol: string, resolution: string, start: Date, end: Date): Promise { - const cacheKey = this.getCacheKey(symbol, resolution, start, end); - - // Check cache first - if (this.cache.has(cacheKey)) { - return this.cache.get(cacheKey)!; - } - - try { - // Format dates for API request - const startStr = start.toISOString(); - const endStr = end.toISOString(); - - const response = await axios.get(`${this.apiBaseUrl}/market-data/history`, { - params: { - symbol, - resolution, - start: startStr, - end: endStr - } - }); - - if (!response.data.success || !response.data.data) { - throw new Error(`Failed to fetch historical data for ${symbol}`); - } - - // Transform API response to BarData objects - const bars: BarData[] = response.data.data.map((bar: any) => ({ - symbol, - timestamp: new Date(bar.timestamp), - open: bar.open, - high: bar.high, - low: bar.low, - close: bar.close, - volume: bar.volume - })); - - // Cache the result - this.cache.set(cacheKey, bars); - - return bars; - } catch (error) { - console.error(`Error fetching historical data for ${symbol}:`, error); - // Return fallback test data if API call fails - return this.generateFallbackTestData(symbol, resolution, start, end); - } - } - - async hasDataFor(symbol: string, resolution: string, start: Date, end: Date): Promise { - try { - const startStr = start.toISOString(); - const endStr = end.toISOString(); - - const response = await axios.get(`${this.apiBaseUrl}/market-data/available`, { - params: { - symbol, - resolution, - start: startStr, - end: endStr - } - }); - - return response.data.success && response.data.data.available; - } catch (error) { - console.error(`Error checking data availability for ${symbol}:`, error); - // Assume data is available for test purposes - return true; - } - } - - clearCache(): void { - this.cache.clear(); - } - - private getCacheKey(symbol: string, resolution: string, start: Date, end: Date): string { - return `${symbol}_${resolution}_${start.getTime()}_${end.getTime()}`; - } - - private generateFallbackTestData(symbol: string, resolution: string, start: Date, end: Date): BarData[] { - console.warn(`Generating fallback test data for ${symbol} from ${start} to ${end}`); - - const bars: BarData[] = []; - let current = new Date(start); - let basePrice = this.getBasePrice(symbol); - - // Generate daily bars by default - const interval = this.getIntervalFromResolution(resolution); - - while (current.getTime() <= end.getTime()) { - // Only generate bars for trading days (skip weekends) - if (current.getDay() !== 0 && current.getDay() !== 6) { - // Generate a random daily price movement (-1% to +1%) - const dailyChange = (Math.random() * 2 - 1) / 100; - - // Add some randomness to the volatility - const volatility = 0.005 + Math.random() * 0.01; // 0.5% to 1.5% - - const open = basePrice * (1 + (Math.random() * 0.002 - 0.001)); - const close = open * (1 + dailyChange); - const high = Math.max(open, close) * (1 + Math.random() * volatility); - const low = Math.min(open, close) * (1 - Math.random() * volatility); - const volume = Math.floor(100000 + Math.random() * 900000); - - bars.push({ - symbol, - timestamp: new Date(current), - open, - high, - low, - close, - volume - }); - - // Update base price for next bar - basePrice = close; - } - - // Move to next interval - current = new Date(current.getTime() + interval); - } - - return bars; - } - - private getBasePrice(symbol: string): number { - // Return a realistic base price for common symbols - switch (symbol.toUpperCase()) { - case 'AAPL': return 170 + Math.random() * 30; - case 'MSFT': return 370 + Math.random() * 50; - case 'AMZN': return 140 + Math.random() * 20; - case 'GOOGL': return 130 + Math.random() * 20; - case 'META': return 300 + Math.random() * 50; - case 'TSLA': return 180 + Math.random() * 70; - case 'NVDA': return 700 + Math.random() * 200; - case 'SPY': return 450 + Math.random() * 30; - case 'QQQ': return 370 + Math.random() * 40; - default: return 100 + Math.random() * 50; - } - } - - private getIntervalFromResolution(resolution: string): number { - // Return milliseconds for each resolution - switch (resolution) { - case '1m': return 60 * 1000; - case '5m': return 5 * 60 * 1000; - case '15m': return 15 * 60 * 1000; - case '30m': return 30 * 60 * 1000; - case '1h': return 60 * 60 * 1000; - case '4h': return 4 * 60 * 60 * 1000; - case '1d': return 24 * 60 * 60 * 1000; - default: return 24 * 60 * 60 * 1000; // Default to daily - } - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/PerformanceAnalytics.ts b/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/PerformanceAnalytics.ts deleted file mode 100644 index 9238718..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/backtesting/PerformanceAnalytics.ts +++ /dev/null @@ -1,325 +0,0 @@ -import { BacktestResult } from './BacktestEngine'; - -/** - * Performance Analysis Utilities - * - * Provides additional metrics and analysis tools for backtesting results. - */ -export class PerformanceAnalytics { - /** - * Calculate additional metrics from backtest results - */ - static enhanceResults(result: BacktestResult): BacktestResult { - // Calculate additional metrics - const enhancedResult = { - ...result, - ...this.calculateAdvancedMetrics(result) - }; - - return enhancedResult; - } - - /** - * Calculate advanced performance metrics - */ - private static calculateAdvancedMetrics(result: BacktestResult): Partial { - // Extract daily returns - const dailyReturns = result.dailyReturns.map(dr => dr.return); - - // Calculate Sortino ratio - const sortinoRatio = this.calculateSortinoRatio(dailyReturns); - - // Calculate Calmar ratio - const calmarRatio = result.maxDrawdown > 0 - ? result.annualizedReturn / result.maxDrawdown - : Infinity; - - // Calculate Omega ratio - const omegaRatio = this.calculateOmegaRatio(dailyReturns); - - // Calculate CAGR - const startTimestamp = result.startDate.getTime(); - const endTimestamp = result.endDate.getTime(); - const yearsElapsed = (endTimestamp - startTimestamp) / (365 * 24 * 60 * 60 * 1000); - const cagr = Math.pow(result.finalCapital / result.initialCapital, 1 / yearsElapsed) - 1; - - // Calculate additional volatility and return metrics - const volatility = this.calculateVolatility(dailyReturns); - const ulcerIndex = this.calculateUlcerIndex(result.dailyReturns); - - return { - sortinoRatio, - calmarRatio, - omegaRatio, - cagr, - volatility, - ulcerIndex - }; - } - - /** - * Calculate Sortino ratio (downside risk-adjusted return) - */ - private static calculateSortinoRatio(dailyReturns: number[]): number { - if (dailyReturns.length === 0) return 0; - - const avgReturn = dailyReturns.reduce((sum, ret) => sum + ret, 0) / dailyReturns.length; - - // Filter only negative returns (downside) - const negativeReturns = dailyReturns.filter(ret => ret < 0); - - if (negativeReturns.length === 0) return Infinity; - - // Calculate downside deviation - const downsideDeviation = Math.sqrt( - negativeReturns.reduce((sum, ret) => sum + Math.pow(ret, 2), 0) / negativeReturns.length - ); - - // Annualize - const annualizedReturn = avgReturn * 252; - const annualizedDownsideDeviation = downsideDeviation * Math.sqrt(252); - - return annualizedDownsideDeviation > 0 - ? annualizedReturn / annualizedDownsideDeviation - : 0; - } - - /** - * Calculate Omega ratio (probability-weighted ratio of gains versus losses) - */ - private static calculateOmegaRatio(dailyReturns: number[], threshold = 0): number { - if (dailyReturns.length === 0) return 0; - - let sumGains = 0; - let sumLosses = 0; - - for (const ret of dailyReturns) { - if (ret > threshold) { - sumGains += (ret - threshold); - } else { - sumLosses += (threshold - ret); - } - } - - return sumLosses > 0 ? sumGains / sumLosses : Infinity; - } - - /** - * Calculate annualized volatility - */ - private static calculateVolatility(returns: number[]): number { - if (returns.length < 2) return 0; - - const mean = returns.reduce((sum, ret) => sum + ret, 0) / returns.length; - const variance = returns.reduce((sum, ret) => sum + Math.pow(ret - mean, 2), 0) / returns.length; - - // Annualize - return Math.sqrt(variance * 252); - } - - /** - * Calculate Ulcer Index (measure of downside risk) - */ - private static calculateUlcerIndex(dailyReturns: Array<{ date: Date; return: number }>): number { - if (dailyReturns.length === 0) return 0; - - // Calculate running equity curve - let equity = 1; - const equityCurve = dailyReturns.map(dr => { - equity *= (1 + dr.return); - return equity; - }); - - // Find running maximum - const runningMax: number[] = []; - let currentMax = equityCurve[0]; - - for (const value of equityCurve) { - currentMax = Math.max(currentMax, value); - runningMax.push(currentMax); - } - - // Calculate percentage drawdowns - const percentDrawdowns = equityCurve.map((value, i) => - (runningMax[i] - value) / runningMax[i] - ); - - // Calculate Ulcer Index - const sumSquaredDrawdowns = percentDrawdowns.reduce( - (sum, dd) => sum + dd * dd, 0 - ); - - return Math.sqrt(sumSquaredDrawdowns / percentDrawdowns.length); - } - - /** - * Extract monthly returns from daily returns - */ - static calculateMonthlyReturns(dailyReturns: Array<{ date: Date; return: number }>): Array<{ - year: number; - month: number; - return: number; - }> { - const monthlyReturns: Array<{ year: number; month: number; return: number }> = []; - - if (dailyReturns.length === 0) return monthlyReturns; - - // Group returns by year and month - const groupedReturns: Record = {}; - - for (const dr of dailyReturns) { - const year = dr.date.getFullYear(); - const month = dr.date.getMonth(); - const key = `${year}-${month}`; - - if (!groupedReturns[key]) { - groupedReturns[key] = []; - } - - groupedReturns[key].push(dr.return); - } - - // Calculate compound return for each month - for (const key in groupedReturns) { - const [yearStr, monthStr] = key.split('-'); - const year = parseInt(yearStr); - const month = parseInt(monthStr); - - // Compound the daily returns for the month - const monthReturn = groupedReturns[key].reduce( - (product, ret) => product * (1 + ret), 1 - ) - 1; - - monthlyReturns.push({ year, month, return: monthReturn }); - } - - // Sort by date - return monthlyReturns.sort((a, b) => { - if (a.year !== b.year) return a.year - b.year; - return a.month - b.month; - }); - } - - /** - * Create drawdown analysis from equity curve - */ - static analyzeDrawdowns(dailyReturns: Array<{ date: Date; return: number }>): Array<{ - startDate: Date; - endDate: Date; - recoveryDate: Date | null; - drawdown: number; - durationDays: number; - recoveryDays: number | null; - }> { - if (dailyReturns.length === 0) return []; - - // Calculate equity curve - let equity = 1; - const equityCurve = dailyReturns.map(dr => { - equity *= (1 + dr.return); - return { date: dr.date, equity }; - }); - - // Analyze drawdowns - const drawdowns: Array<{ - startDate: Date; - endDate: Date; - recoveryDate: Date | null; - drawdown: number; - durationDays: number; - recoveryDays: number | null; - }> = []; - - let peakEquity = equityCurve[0].equity; - let peakDate = equityCurve[0].date; - let inDrawdown = false; - let currentDrawdown: { - startDate: Date; - endDate: Date; - lowEquity: number; - peakEquity: number; - } | null = null; - - // Find drawdown periods - for (let i = 1; i < equityCurve.length; i++) { - const { date, equity } = equityCurve[i]; - - // New peak - if (equity > peakEquity) { - peakEquity = equity; - peakDate = date; - - // If recovering from drawdown, record recovery - if (inDrawdown && currentDrawdown) { - const recoveryDate = date; - const drawdownPct = (currentDrawdown.peakEquity - currentDrawdown.lowEquity) / - currentDrawdown.peakEquity; - - const durationDays = Math.floor( - (currentDrawdown.endDate.getTime() - currentDrawdown.startDate.getTime()) / - (1000 * 60 * 60 * 24) - ); - - const recoveryDays = Math.floor( - (recoveryDate.getTime() - currentDrawdown.endDate.getTime()) / - (1000 * 60 * 60 * 24) - ); - - drawdowns.push({ - startDate: currentDrawdown.startDate, - endDate: currentDrawdown.endDate, - recoveryDate, - drawdown: drawdownPct, - durationDays, - recoveryDays - }); - - inDrawdown = false; - currentDrawdown = null; - } - } - // In drawdown - else { - const drawdownPct = (peakEquity - equity) / peakEquity; - - if (!inDrawdown) { - // Start of new drawdown - inDrawdown = true; - currentDrawdown = { - startDate: peakDate, - endDate: date, - lowEquity: equity, - peakEquity - }; - } else if (currentDrawdown && equity < currentDrawdown.lowEquity) { - // New low in current drawdown - currentDrawdown.lowEquity = equity; - currentDrawdown.endDate = date; - } - } - } - - // Handle any ongoing drawdown at the end - if (inDrawdown && currentDrawdown) { - const drawdownPct = (currentDrawdown.peakEquity - currentDrawdown.lowEquity) / - currentDrawdown.peakEquity; - - const durationDays = Math.floor( - (currentDrawdown.endDate.getTime() - currentDrawdown.startDate.getTime()) / - (1000 * 60 * 60 * 24) - ); - - drawdowns.push({ - startDate: currentDrawdown.startDate, - endDate: currentDrawdown.endDate, - recoveryDate: null, - drawdown: drawdownPct, - durationDays, - recoveryDays: null - }); - } - - // Sort by drawdown magnitude - return drawdowns.sort((a, b) => b.drawdown - a.drawdown); - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/execution/StrategyExecutionService.ts b/apps/intelligence-services/strategy-orchestrator/src/core/execution/StrategyExecutionService.ts deleted file mode 100644 index 2dfe600..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/execution/StrategyExecutionService.ts +++ /dev/null @@ -1,512 +0,0 @@ -import { EventEmitter } from 'events'; -import { BaseStrategy } from '../Strategy'; -import { BarData, Order, Position } from '../Strategy'; -import { MarketDataFeed } from '../backtesting/MarketDataFeed'; -import { StrategyRegistry } from '../strategies/StrategyRegistry'; -import { WebSocket } from 'ws'; - -export interface ExecutionConfig { - symbols: string[]; - dataResolution: '1m' | '5m' | '15m' | '30m' | '1h' | '4h' | '1d'; - realTrading: boolean; - maxPositionValue: number; - maxOrdersPerMinute: number; - stopLossPercentage: number; -} - -/** - * Strategy Execution Service - * - * Handles the real-time execution of trading strategies. - * Manages live data feeds, order execution, and position tracking. - */ -export class StrategyExecutionService extends EventEmitter { - private strategyRegistry: StrategyRegistry; - private marketDataFeed: MarketDataFeed; - private activeStrategies: Map; - lastBar: Map; - }> = new Map(); - - private isRunning: boolean = false; - private dataPollingIntervals: Map = new Map(); - private webSocketServer: WebSocket.Server | null = null; - private wsClients: Set = new Set(); - private marketDataCache: Map = new Map(); - - constructor(apiBaseUrl: string = 'http://localhost:3001/api', wsPort: number = 8082) { - super(); - this.strategyRegistry = StrategyRegistry.getInstance(); - this.marketDataFeed = new MarketDataFeed(apiBaseUrl); - this.initializeWebSocketServer(wsPort); - } - - /** - * Initialize WebSocket server for real-time updates - */ - private initializeWebSocketServer(port: number): void { - try { - this.webSocketServer = new WebSocket.Server({ port }); - - this.webSocketServer.on('connection', (ws) => { - console.log('New client connected to strategy execution service'); - this.wsClients.add(ws); - - ws.on('message', (message) => { - try { - const data = JSON.parse(message.toString()); - this.handleWebSocketMessage(ws, data); - } catch (error) { - console.error('Error handling WebSocket message:', error); - } - }); - - ws.on('close', () => { - console.log('Client disconnected from strategy execution service'); - this.wsClients.delete(ws); - }); - - // Send initial state - this.sendAllStrategyStatus(ws); - }); - - console.log(`WebSocket server started on port ${port}`); - } catch (error) { - console.error('Failed to initialize WebSocket server:', error); - } - } - - /** - * Handle incoming WebSocket messages - */ - private handleWebSocketMessage(ws: WebSocket, message: any): void { - switch (message.type) { - case 'get_active_strategies': - this.sendAllStrategyStatus(ws); - break; - case 'start_strategy': - this.startStrategy(message.id, message.config); - break; - case 'stop_strategy': - this.stopStrategy(message.id); - break; - case 'pause_strategy': - this.pauseStrategy(message.id); - break; - default: - console.warn(`Unknown WebSocket message type: ${message.type}`); - } - } - - /** - * Send a message to all connected WebSocket clients - */ - private broadcastMessage(message: any): void { - const messageStr = JSON.stringify(message); - for (const client of this.wsClients) { - if (client.readyState === WebSocket.OPEN) { - client.send(messageStr); - } - } - } - - /** - * Send current status of all active strategies to a specific client - */ - private sendAllStrategyStatus(ws: WebSocket): void { - const statusList = Array.from(this.activeStrategies.entries()).map(([id, data]) => ({ - id, - name: data.strategy.name, - status: this.isRunning ? 'ACTIVE' : 'PAUSED', - symbols: data.config.symbols, - positions: Array.from(data.positions.entries()).map(([symbol, pos]) => ({ - symbol, - quantity: pos.quantity, - entryPrice: pos.entryPrice, - currentValue: pos.currentValue - })) - })); - - ws.send(JSON.stringify({ - type: 'strategy_status_list', - timestamp: new Date().toISOString(), - data: statusList - })); - } - - /** - * Start the execution service (global) - */ - start(): void { - if (this.isRunning) return; - - this.isRunning = true; - console.log('Strategy execution service started'); - - // Start data polling for all active strategies - for (const [strategyId, data] of this.activeStrategies.entries()) { - this.startDataPollingForStrategy(strategyId, data); - } - - this.broadcastMessage({ - type: 'execution_service_status', - timestamp: new Date().toISOString(), - data: { status: 'RUNNING' } - }); - } - - /** - * Stop the execution service (global) - */ - stop(): void { - if (!this.isRunning) return; - - this.isRunning = false; - console.log('Strategy execution service stopped'); - - // Clear all data polling intervals - for (const interval of this.dataPollingIntervals.values()) { - clearInterval(interval); - } - this.dataPollingIntervals.clear(); - - this.broadcastMessage({ - type: 'execution_service_status', - timestamp: new Date().toISOString(), - data: { status: 'STOPPED' } - }); - } - - /** - * Start a specific strategy - */ - startStrategy(strategyId: string, config?: ExecutionConfig): boolean { - const strategy = this.strategyRegistry.getStrategyById(strategyId); - - if (!strategy) { - console.error(`Strategy with ID ${strategyId} not found`); - return false; - } - - // If strategy is already active, return false - if (this.activeStrategies.has(strategyId)) { - console.warn(`Strategy ${strategyId} is already active`); - return false; - } - - // Use provided config or create default - const executionConfig: ExecutionConfig = config || { - symbols: strategy.symbols, - dataResolution: '1m', - realTrading: false, - maxPositionValue: 10000, - maxOrdersPerMinute: 5, - stopLossPercentage: 0.02 - }; - - // Initialize strategy data - const strategyData = { - strategy, - config: executionConfig, - positions: new Map(), - lastBar: new Map() - }; - - this.activeStrategies.set(strategyId, strategyData); - - // If execution service is running, start data polling for this strategy - if (this.isRunning) { - this.startDataPollingForStrategy(strategyId, strategyData); - } - - console.log(`Strategy ${strategyId} started with ${executionConfig.symbols.length} symbols`); - - this.broadcastMessage({ - type: 'strategy_started', - timestamp: new Date().toISOString(), - data: { - strategyId, - name: strategy.name, - symbols: executionConfig.symbols - } - }); - - return true; - } - - /** - * Stop a specific strategy - */ - stopStrategy(strategyId: string): boolean { - if (!this.activeStrategies.has(strategyId)) { - console.warn(`Strategy ${strategyId} is not active`); - return false; - } - - // Clear data polling interval for this strategy - const intervalId = this.dataPollingIntervals.get(strategyId); - if (intervalId) { - clearInterval(intervalId); - this.dataPollingIntervals.delete(strategyId); - } - - // Get strategy data before removing - const strategyData = this.activeStrategies.get(strategyId)!; - const { strategy } = strategyData; - - // Close any open positions (in real implementation) - // ... - - this.activeStrategies.delete(strategyId); - - console.log(`Strategy ${strategyId} stopped`); - - this.broadcastMessage({ - type: 'strategy_stopped', - timestamp: new Date().toISOString(), - data: { - strategyId, - name: strategy.name - } - }); - - return true; - } - - /** - * Pause a specific strategy - */ - pauseStrategy(strategyId: string): boolean { - if (!this.activeStrategies.has(strategyId)) { - console.warn(`Strategy ${strategyId} is not active`); - return false; - } - - // Clear data polling interval for this strategy - const intervalId = this.dataPollingIntervals.get(strategyId); - if (intervalId) { - clearInterval(intervalId); - this.dataPollingIntervals.delete(strategyId); - } - - const { strategy } = this.activeStrategies.get(strategyId)!; - - console.log(`Strategy ${strategyId} paused`); - - this.broadcastMessage({ - type: 'strategy_paused', - timestamp: new Date().toISOString(), - data: { - strategyId, - name: strategy.name - } - }); - - return true; - } - - /** - * Start data polling for a specific strategy - */ - private startDataPollingForStrategy( - strategyId: string, - data: { - strategy: BaseStrategy; - config: ExecutionConfig; - positions: Map; - lastBar: Map; - } - ): void { - const { strategy, config } = data; - - // Determine polling interval based on data resolution - const pollingIntervalMs = this.getPollingIntervalFromResolution(config.dataResolution); - - // Set up interval for data polling - const intervalId = setInterval(async () => { - await this.pollMarketData(strategyId); - }, pollingIntervalMs); - - this.dataPollingIntervals.set(strategyId, intervalId); - - console.log(`Started data polling for strategy ${strategyId} with interval ${pollingIntervalMs}ms`); - } - - /** - * Convert data resolution to polling interval - */ - private getPollingIntervalFromResolution(resolution: string): number { - switch (resolution) { - case '1m': return 60 * 1000; // 60 seconds - case '5m': return 5 * 60 * 1000; // 5 minutes - case '15m': return 15 * 60 * 1000; // 15 minutes - case '30m': return 30 * 60 * 1000; // 30 minutes - case '1h': return 60 * 60 * 1000; // 1 hour - case '4h': return 4 * 60 * 60 * 1000; // 4 hours - case '1d': return 24 * 60 * 60 * 1000; // 1 day - default: return 60 * 1000; // Default to 1 minute - } - } - - /** - * Poll market data for a specific strategy - */ - private async pollMarketData(strategyId: string): Promise { - const strategyData = this.activeStrategies.get(strategyId); - - if (!strategyData) { - console.warn(`Strategy ${strategyId} not found in active strategies`); - return; - } - - const { strategy, config, lastBar } = strategyData; - - try { - // Get latest market data for all symbols - for (const symbol of config.symbols) { - // Get latest bar - const now = new Date(); - const startTime = new Date(now.getTime() - 24 * 60 * 60 * 1000); // 24 hours ago - - // Fetch latest bar - const bars = await this.marketDataFeed.getHistoricalData( - symbol, - config.dataResolution, - startTime, - now - ); - - if (bars.length > 0) { - const latestBar = bars[bars.length - 1]; - - // Only process if this is a new bar - const currentLastBar = lastBar.get(symbol); - if (!currentLastBar || latestBar.timestamp.getTime() > currentLastBar.timestamp.getTime()) { - // Update last bar - lastBar.set(symbol, latestBar); - - // Process the bar with the strategy - this.processBar(strategyId, symbol, latestBar); - } - } - } - } catch (error) { - console.error(`Error polling market data for strategy ${strategyId}:`, error); - } - } - - /** - * Process a new bar with the strategy - */ private processBar(strategyId: string, symbol: string, bar: BarData): void { - const strategyData = this.activeStrategies.get(strategyId); - - if (!strategyData) { - console.warn(`Strategy ${strategyId} not found in active strategies`); - return; - } - - const { strategy } = strategyData; - - // Call strategy's onBar method to get trading signals - const signal = strategy.onBar(bar); - - if (signal) { - // Create a signal object with timestamp and ID - const signalObject = { - id: `sig_${Date.now()}_${Math.floor(Math.random() * 10000)}`, - strategyId, - name: strategy.name, - symbol: bar.symbol, - price: bar.close, - action: signal.action, - quantity: signal.quantity, - metadata: signal.metadata, - timestamp: new Date().toISOString(), - confidence: signal.confidence || 0.8 // Default confidence if not provided - }; - - // Store the signal in Redis (this would be in a production environment) - try { - // This would normally be injected as a dependency, but for simplicity: - const redis = require('ioredis') ? - new (require('ioredis'))({ - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379') - }) : null; - - if (redis) { - // Store signal with TTL of 7 days - redis.setex( - `signal:${strategyId}:${signalObject.id}`, - 60 * 60 * 24 * 7, // 7 days TTL - JSON.stringify(signalObject) - ); - } - } catch (err) { - console.error('Error storing signal in Redis:', err); - } - - // Broadcast the signal - this.broadcastMessage({ - type: 'strategy_signal', - timestamp: new Date().toISOString(), - data: signalObject - }); - - // Execute the signal if real trading is enabled - if (strategyData.config.realTrading) { - this.executeSignal(strategyId, signal); - } - } - } - - /** - * Execute a trading signal - */ - private executeSignal(strategyId: string, signal: any): void { - // In a real implementation, this would connect to the order execution service - // For now, just log the signal and broadcast a mock trade - console.log(`Executing signal for strategy ${strategyId}:`, signal); - - const strategyData = this.activeStrategies.get(strategyId); - if (!strategyData) return; - - const { strategy } = strategyData; - - // Broadcast mock trade execution - this.broadcastMessage({ - type: 'strategy_trade', - timestamp: new Date().toISOString(), - data: { - strategyId, - name: strategy.name, - symbol: signal.symbol, - price: signal.price, - action: signal.action, - quantity: signal.quantity, - executionTime: new Date().toISOString(), - status: 'FILLED' - } - }); - } - - /** - * Close all connections when service is shut down - */ - shutdown(): void { - this.stop(); - - if (this.webSocketServer) { - for (const client of this.wsClients) { - client.close(); - } - this.wsClients.clear(); - - this.webSocketServer.close(() => { - console.log('WebSocket server closed'); - }); - } - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/MeanReversionStrategy.ts b/apps/intelligence-services/strategy-orchestrator/src/core/strategies/MeanReversionStrategy.ts deleted file mode 100644 index be19bcb..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/MeanReversionStrategy.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { BarData, Order } from '../Strategy'; -import { TechnicalIndicators } from '../analysis/TechnicalIndicators'; -import { VectorizedStrategy, VectorizedStrategyParams } from './VectorizedStrategy'; - -export interface MeanReversionParams extends VectorizedStrategyParams { - lookback: number; // Period for mean calculation - entryDeviation: number; // Standard deviations for entry - exitDeviation: number; // Standard deviations for exit - rsiPeriod: number; // RSI period - rsiOverbought: number; // RSI overbought level - rsiOversold: number; // RSI oversold level - useRsi: boolean; // Whether to use RSI filter -} - -/** - * Mean Reversion Strategy - * - * A vectorized strategy that trades mean reversion by detecting - * overbought/oversold conditions and entering when price deviates - * significantly from its moving average. - */ -export class MeanReversionStrategy extends VectorizedStrategy { - // Default parameters - protected static readonly DEFAULT_PARAMS: MeanReversionParams = { - ...VectorizedStrategy.DEFAULT_PARAMS, - lookback: 20, - entryDeviation: 1.5, - exitDeviation: 0.5, - lookbackPeriod: 100, - rsiPeriod: 14, - rsiOverbought: 70, - rsiOversold: 30, - useRsi: true - }; - - constructor( - id: string, - name: string, - description: string, - symbols: string[], - params: Partial = {} - ) { - super(id, name, description, symbols, { - ...MeanReversionStrategy.DEFAULT_PARAMS, - ...params - }); - } - - /** - * Computes buy/sell signals based on mean reversion logic - */ - protected computeSignals( - symbol: string, - prices: number[], - volumes: number[] - ): Array<1 | 0 | -1> { - const params = this.parameters as MeanReversionParams; - const result: Array<1 | 0 | -1> = Array(prices.length).fill(0); - - // Not enough data - if (prices.length < params.lookback) { - return result; - } - - // Calculate moving average - const sma = TechnicalIndicators.sma(prices, params.lookback); - - // Calculate standard deviation - const stdDevs: number[] = []; - for (let i = params.lookback - 1; i < prices.length; i++) { - let sum = 0; - for (let j = i - params.lookback + 1; j <= i; j++) { - sum += Math.pow(prices[j] - sma[i], 2); - } - stdDevs.push(Math.sqrt(sum / params.lookback)); - } - - // Pad standard deviations with NaN - const paddedStdDevs = [...Array(params.lookback - 1).fill(NaN), ...stdDevs]; - - // Calculate upper and lower bands - const upperBand: number[] = []; - const lowerBand: number[] = []; - - for (let i = 0; i < prices.length; i++) { - if (isNaN(sma[i]) || isNaN(paddedStdDevs[i])) { - upperBand.push(NaN); - lowerBand.push(NaN); - } else { - upperBand.push(sma[i] + paddedStdDevs[i] * params.entryDeviation); - lowerBand.push(sma[i] - paddedStdDevs[i] * params.entryDeviation); - } - } - - // Calculate RSI if used - let rsi: number[] = []; - if (params.useRsi) { - rsi = TechnicalIndicators.rsi(prices, params.rsiPeriod); - } - - // Generate signals - for (let i = params.lookback; i < prices.length; i++) { - // Check if price is outside bands - const outsideUpperBand = prices[i] > upperBand[i]; - const outsideLowerBand = prices[i] < lowerBand[i]; - - // Check RSI conditions - const rsiOverbought = params.useRsi && rsi[i] > params.rsiOverbought; - const rsiOversold = params.useRsi && rsi[i] < params.rsiOversold; - - // Check if price is returning to mean - const returningFromUpper = - outsideUpperBand && prices[i] < prices[i-1] && prices[i-1] > prices[i-2]; - - const returningFromLower = - outsideLowerBand && prices[i] > prices[i-1] && prices[i-1] < prices[i-2]; - - // Generate signals - if ((returningFromUpper && (!params.useRsi || rsiOverbought))) { - result[i] = -1; // SELL signal - } else if ((returningFromLower && (!params.useRsi || rsiOversold))) { - result[i] = 1; // BUY signal - } else if (Math.abs(prices[i] - sma[i]) < paddedStdDevs[i] * params.exitDeviation) { - // Price returned to mean - exit position - if (i > 0 && result[i-1] !== 0) { - result[i] = result[i-1] * -1; // Opposite of previous signal - } - } - } - - return result; - } - - async onBar(bar: BarData): Promise { - // Use the vectorized implementation from the base class - return super.onBar(bar); - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/MovingAverageCrossover.ts b/apps/intelligence-services/strategy-orchestrator/src/core/strategies/MovingAverageCrossover.ts deleted file mode 100644 index 23a49e7..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/MovingAverageCrossover.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { BaseStrategy, BarData, Order, StrategyParameters } from '../Strategy'; -import { TechnicalIndicators } from '../analysis/TechnicalIndicators'; - -export interface MovingAverageCrossoverParams extends StrategyParameters { - fastPeriod: number; // Fast moving average period - slowPeriod: number; // Slow moving average period - positionSize: number; // Position size as percentage (0-1) - stopLoss: number; // Stop loss percentage (0-1) - takeProfit: number; // Take profit percentage (0-1) -} - -/** - * Moving Average Crossover Strategy - * - * A simple strategy that buys when fast MA crosses above slow MA, - * and sells when fast MA crosses below slow MA. - */ -export class MovingAverageCrossover extends BaseStrategy { - // Default parameters - protected static readonly DEFAULT_PARAMS: MovingAverageCrossoverParams = { - fastPeriod: 10, - slowPeriod: 30, - positionSize: 0.2, // 20% of portfolio - stopLoss: 0.02, // 2% stop loss - takeProfit: 0.05 // 5% take profit - }; - - private fastMA: Map = new Map(); - private slowMA: Map = new Map(); - private lastSignal: Map = new Map(); - private stopLossLevels: Map = new Map(); - private takeProfitLevels: Map = new Map(); - - constructor( - id: string, - name: string, - description: string, - symbols: string[], - params: Partial = {} - ) { - super(id, name, description, symbols, { - ...MovingAverageCrossover.DEFAULT_PARAMS, - ...params - }); - } - - async initialize(): Promise { - // Initialize state for each symbol - for (const symbol of this.symbols) { - this.fastMA.set(symbol, []); - this.slowMA.set(symbol, []); - this.lastSignal.set(symbol, null); - } - - // Validate parameters - const params = this.parameters as MovingAverageCrossoverParams; - - if (params.fastPeriod >= params.slowPeriod) { - console.warn('Fast period should be smaller than slow period'); - } - - if (params.positionSize <= 0 || params.positionSize > 1) { - console.warn('Position size should be between 0 and 1'); - params.positionSize = 0.2; // Reset to default - } - } - - async onBar(bar: BarData): Promise { - const symbol = bar.symbol; - const params = this.parameters as MovingAverageCrossoverParams; - - // Update market data - this.addBar(bar); - - // Get historical data - const bars = this.context.marketData.get(symbol) || []; - - // Wait until we have enough data - if (bars.length < params.slowPeriod + 1) { - return []; - } - - // Extract close prices - const prices = TechnicalIndicators.extractPrice(bars, 'close'); - - // Calculate moving averages - const fastMA = TechnicalIndicators.sma(prices, params.fastPeriod); - const slowMA = TechnicalIndicators.sma(prices, params.slowPeriod); - - // Update MA values - this.fastMA.set(symbol, fastMA); - this.slowMA.set(symbol, slowMA); - - // Get current and previous values - const currentFast = fastMA[fastMA.length - 1]; - const currentSlow = slowMA[slowMA.length - 1]; - const previousFast = fastMA[fastMA.length - 2]; - const previousSlow = slowMA[slowMA.length - 2]; - - // Check for crossovers - const orders: Order[] = []; - - // Check for stop loss and take profit first - if (this.hasPosition(symbol)) { - const position = this.getPosition(symbol)!; - const currentPrice = bar.close; - - const stopLossLevel = this.stopLossLevels.get(symbol); - const takeProfitLevel = this.takeProfitLevels.get(symbol); - - // Check stop loss - if (stopLossLevel && currentPrice <= stopLossLevel) { - orders.push(this.createMarketOrder(symbol, 'SELL', position.quantity)); - this.stopLossLevels.delete(symbol); - this.takeProfitLevels.delete(symbol); - console.log(`${symbol} Stop loss triggered at ${currentPrice}`); - return orders; - } - - // Check take profit - if (takeProfitLevel && currentPrice >= takeProfitLevel) { - orders.push(this.createMarketOrder(symbol, 'SELL', position.quantity)); - this.stopLossLevels.delete(symbol); - this.takeProfitLevels.delete(symbol); - console.log(`${symbol} Take profit triggered at ${currentPrice}`); - return orders; - } - } - - // Check for moving average crossover signals - const crossedAbove = previousFast <= previousSlow && currentFast > currentSlow; - const crossedBelow = previousFast >= previousSlow && currentFast < currentSlow; - - if (crossedAbove && !this.hasPosition(symbol)) { - // Buy signal - calculate position size - const cash = this.getAvailableCash(); - const positionValue = cash * params.positionSize; - const quantity = Math.floor(positionValue / bar.close); - - if (quantity > 0) { - orders.push(this.createMarketOrder(symbol, 'BUY', quantity)); - this.lastSignal.set(symbol, 'BUY'); - - // Set stop loss and take profit levels - this.stopLossLevels.set(symbol, bar.close * (1 - params.stopLoss)); - this.takeProfitLevels.set(symbol, bar.close * (1 + params.takeProfit)); - console.log(`${symbol} Buy signal at ${bar.close}, quantity: ${quantity}`); - } - } else if (crossedBelow && this.hasPosition(symbol)) { - // Sell signal - const position = this.getPosition(symbol)!; - orders.push(this.createMarketOrder(symbol, 'SELL', position.quantity)); - this.lastSignal.set(symbol, 'SELL'); - this.stopLossLevels.delete(symbol); - this.takeProfitLevels.delete(symbol); - console.log(`${symbol} Sell signal at ${bar.close}`); - } - - return orders; - } - - async onOrderFilled(order: Order): Promise { - console.log(`Order filled: ${order.symbol} ${order.side} ${order.quantity} @ ${order.fillPrice}`); - - // Additional post-order logic can be added here - if (order.side === 'BUY') { - // Update stop loss and take profit levels based on fill price - const params = this.parameters as MovingAverageCrossoverParams; - this.stopLossLevels.set(order.symbol, order.fillPrice! * (1 - params.stopLoss)); - this.takeProfitLevels.set(order.symbol, order.fillPrice! * (1 + params.takeProfit)); - } - } - - async cleanup(): Promise { - // Clean up any resources or state - this.fastMA.clear(); - this.slowMA.clear(); - this.lastSignal.clear(); - this.stopLossLevels.clear(); - this.takeProfitLevels.clear(); - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/StrategyRegistry.ts b/apps/intelligence-services/strategy-orchestrator/src/core/strategies/StrategyRegistry.ts deleted file mode 100644 index 5f75420..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/StrategyRegistry.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { BaseStrategy, StrategyParameters } from '../Strategy'; -import { MovingAverageCrossover, MovingAverageCrossoverParams } from './MovingAverageCrossover'; -import { MeanReversionStrategy, MeanReversionParams } from './MeanReversionStrategy'; - -// Define the strategy types -export type StrategyType = 'MOVING_AVERAGE_CROSSOVER' | 'MEAN_REVERSION' | 'CUSTOM'; - -// Strategy factory function type -export type StrategyFactory = ( - id: string, - name: string, - description: string, - symbols: string[], - parameters: StrategyParameters -) => BaseStrategy; - -/** - * Strategy Registry - * - * Manages and creates strategy instances based on strategy type. - * Allows dynamic registration of new strategy types. - */ -export class StrategyRegistry { - private static instance: StrategyRegistry; - private factories: Map = new Map(); - - /** - * Get the singleton instance of StrategyRegistry - */ - public static getInstance(): StrategyRegistry { - if (!StrategyRegistry.instance) { - StrategyRegistry.instance = new StrategyRegistry(); - } - return StrategyRegistry.instance; - } - - /** - * Private constructor to enforce singleton pattern - */ - private constructor() { - this.registerBuiltInStrategies(); - } - - /** - * Register built-in strategies - */ - private registerBuiltInStrategies(): void { - // Register Moving Average Crossover - this.registerStrategy('MOVING_AVERAGE_CROSSOVER', (id, name, description, symbols, parameters) => { - return new MovingAverageCrossover( - id, - name, - description, - symbols, - parameters as MovingAverageCrossoverParams - ); - }); - - // Register Mean Reversion - this.registerStrategy('MEAN_REVERSION', (id, name, description, symbols, parameters) => { - return new MeanReversionStrategy( - id, - name, - description, - symbols, - parameters as MeanReversionParams - ); - }); - } - - /** - * Register a new strategy type with factory function - */ - public registerStrategy(type: StrategyType, factory: StrategyFactory): void { - this.factories.set(type, factory); - } - /** - * Create a strategy instance based on type and parameters - */ - public createStrategy( - type: StrategyType, - id: string, - name: string, - description: string, - symbols: string[], - parameters: StrategyParameters = {} - ): BaseStrategy { - const factory = this.factories.get(type); - - if (!factory) { - throw new Error(`Strategy type '${type}' is not registered`); - } - - const strategy = factory(id, name, description, symbols, parameters); - - // Store the strategy for management - this.storeStrategy(strategy); - - return strategy; - } - - /** - * Get default parameters for a strategy type - */ - public getDefaultParameters(type: StrategyType): StrategyParameters { - switch(type) { - case 'MOVING_AVERAGE_CROSSOVER': - return { - fastPeriod: 10, - slowPeriod: 30, - positionSize: 0.2, - stopLoss: 0.02, - takeProfit: 0.05 - }; - case 'MEAN_REVERSION': - return { - lookback: 20, - entryDeviation: 1.5, - exitDeviation: 0.5, - lookbackPeriod: 100, - positionSize: 0.2, - stopLoss: 0.02, - takeProfit: 0.05, - useBollingerBands: true, - bollingerPeriod: 20, - bollingerDeviation: 2, - rsiPeriod: 14, - rsiOverbought: 70, - rsiOversold: 30, - useRsi: true - }; - default: - return {}; - } - } - - /** - * Get all registered strategy types - */ - public getStrategyTypes(): StrategyType[] { - return Array.from(this.factories.keys()); - } - - /** - * Check if a strategy type is registered - */ - public hasStrategyType(type: StrategyType): boolean { - return this.factories.has(type); - } - - // Store created strategies for management - private strategies: Map = new Map(); - - /** - * Store strategy instance for later retrieval - * Used when creating strategies through registry - */ - private storeStrategy(strategy: BaseStrategy): void { - this.strategies.set(strategy.id, strategy); - } - - /** - * Get all registered strategies - */ - public getAllStrategies(): BaseStrategy[] { - return Array.from(this.strategies.values()); - } - - /** - * Get a strategy by ID - */ - public getStrategyById(id: string): BaseStrategy | undefined { - return this.strategies.get(id); - } - - /** - * Delete a strategy by ID - */ - public deleteStrategy(id: string): boolean { - return this.strategies.delete(id); - } - - /** - * Get the type of a strategy instance - */ - public getStrategyType(strategy: BaseStrategy): StrategyType { - // Determine the type based on constructor - if (strategy instanceof MovingAverageCrossover) { - return 'MOVING_AVERAGE_CROSSOVER'; - } else if (strategy instanceof MeanReversionStrategy) { - return 'MEAN_REVERSION'; - } - - // Default to CUSTOM if we can't determine - return 'CUSTOM'; - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/VectorizedStrategy.ts b/apps/intelligence-services/strategy-orchestrator/src/core/strategies/VectorizedStrategy.ts deleted file mode 100644 index a3fe1bd..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/core/strategies/VectorizedStrategy.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { BaseStrategy, BarData, Order, StrategyParameters } from '../Strategy'; -import { TechnicalIndicators } from '../analysis/TechnicalIndicators'; - -export interface VectorizedStrategyParams extends StrategyParameters { - lookbackPeriod: number; // Data history to use for calculations - positionSize: number; // Position size as percentage (0-1) - stopLoss: number; // Stop loss percentage (0-1) - takeProfit: number; // Take profit percentage (0-1) - useBollingerBands: boolean; // Use Bollinger Bands for volatility - bollingerPeriod: number; // Bollinger Band period - bollingerDeviation: number; // Number of standard deviations -} - -/** - * Vectorized Strategy Base Class - * - * A performant strategy implementation designed for fast backtesting. - * Instead of processing bars one at a time, it pre-calculates signals - * for large chunks of data and executes them in batches. - */ -export abstract class VectorizedStrategy extends BaseStrategy { - // Default parameters - protected static readonly DEFAULT_PARAMS: VectorizedStrategyParams = { - lookbackPeriod: 100, - positionSize: 0.2, // 20% of portfolio - stopLoss: 0.02, // 2% stop loss - takeProfit: 0.05, // 5% take profit - useBollingerBands: true, - bollingerPeriod: 20, - bollingerDeviation: 2 - }; - - protected indicators: Map; // 1 = BUY, 0 = NEUTRAL, -1 = SELL - stopLevels: number[]; - profitLevels: number[]; - bollingerUpper?: number[]; - bollingerLower?: number[]; - }> = new Map(); - - protected stopLossLevels: Map = new Map(); - protected takeProfitLevels: Map = new Map(); - - constructor( - id: string, - name: string, - description: string, - symbols: string[], - params: Partial = {} - ) { - super(id, name, description, symbols, { - ...VectorizedStrategy.DEFAULT_PARAMS, - ...params - }); - } - - async initialize(): Promise { - // Initialize state for each symbol - for (const symbol of this.symbols) { - this.indicators.set(symbol, { - prices: [], - signals: [], - stopLevels: [], - profitLevels: [] - }); - } - - // Validate parameters - const params = this.parameters as VectorizedStrategyParams; - - if (params.positionSize <= 0 || params.positionSize > 1) { - console.warn('Position size should be between 0 and 1'); - params.positionSize = 0.2; // Reset to default - } - } - - /** - * The main method that must be implemented by vectorized strategy subclasses. - * It should compute signals based on price data in a vectorized manner. - */ - protected abstract computeSignals( - symbol: string, - prices: number[], - volumes: number[] - ): Array<1 | 0 | -1>; - - async onBar(bar: BarData): Promise { - const symbol = bar.symbol; - const params = this.parameters as VectorizedStrategyParams; - - // Update market data - this.addBar(bar); - - // Get historical data - const bars = this.context.marketData.get(symbol) || []; - - // If we don't have enough bars yet, wait - if (bars.length < params.lookbackPeriod) { - return []; - } - - // Extract price and volume data - const prices = TechnicalIndicators.extractPrice(bars, 'close'); - const volumes = bars.map(b => b.volume); - - // Update indicators cache - const indicators = this.indicators.get(symbol)!; - indicators.prices = prices; - - // Check for stop loss and take profit first - const orders: Order[] = []; - - if (this.hasPosition(symbol)) { - const position = this.getPosition(symbol)!; - const currentPrice = bar.close; - - const stopLossLevel = this.stopLossLevels.get(symbol); - const takeProfitLevel = this.takeProfitLevels.get(symbol); - - // Check stop loss - if (stopLossLevel && currentPrice <= stopLossLevel) { - orders.push(this.createMarketOrder(symbol, 'SELL', position.quantity)); - this.stopLossLevels.delete(symbol); - this.takeProfitLevels.delete(symbol); - return orders; - } - - // Check take profit - if (takeProfitLevel && currentPrice >= takeProfitLevel) { - orders.push(this.createMarketOrder(symbol, 'SELL', position.quantity)); - this.stopLossLevels.delete(symbol); - this.takeProfitLevels.delete(symbol); - return orders; - } - } - - // Compute vectorized signals only when we have enough new data - // (optimization to avoid recomputing on every bar) - if (bars.length % 10 === 0 || !indicators.signals.length) { - indicators.signals = this.computeSignals(symbol, prices, volumes); - - // Update Bollinger Bands if configured - if (params.useBollingerBands) { - const bands = TechnicalIndicators.bollingerBands( - prices, - params.bollingerPeriod, - params.bollingerDeviation - ); - indicators.bollingerUpper = bands.upper; - indicators.bollingerLower = bands.lower; - } - } - - // Get the latest signal - const latestSignal = indicators.signals[indicators.signals.length - 1]; - - // Generate orders based on signals - if (latestSignal === 1 && !this.hasPosition(symbol)) { - // Buy signal - calculate position size - const cash = this.getAvailableCash(); - const positionValue = cash * params.positionSize; - const quantity = Math.floor(positionValue / bar.close); - - if (quantity > 0) { - orders.push(this.createMarketOrder(symbol, 'BUY', quantity)); - - // Set stop loss and take profit levels - this.stopLossLevels.set(symbol, bar.close * (1 - params.stopLoss)); - this.takeProfitLevels.set(symbol, bar.close * (1 + params.takeProfit)); - } - } else if (latestSignal === -1 && this.hasPosition(symbol)) { - // Sell signal - const position = this.getPosition(symbol)!; - orders.push(this.createMarketOrder(symbol, 'SELL', position.quantity)); - this.stopLossLevels.delete(symbol); - this.takeProfitLevels.delete(symbol); - } - - return orders; - } - - async onOrderFilled(order: Order): Promise { - // Update stop loss and take profit levels based on fill price - if (order.side === 'BUY') { - const params = this.parameters as VectorizedStrategyParams; - this.stopLossLevels.set(order.symbol, order.fillPrice! * (1 - params.stopLoss)); - this.takeProfitLevels.set(order.symbol, order.fillPrice! * (1 + params.takeProfit)); - } - } - - async cleanup(): Promise { - // Clean up any resources or state - this.indicators.clear(); - this.stopLossLevels.clear(); - this.takeProfitLevels.clear(); - } -} diff --git a/apps/intelligence-services/strategy-orchestrator/src/index.ts b/apps/intelligence-services/strategy-orchestrator/src/index.ts deleted file mode 100644 index 76b3a95..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/index.ts +++ /dev/null @@ -1,701 +0,0 @@ -import { Hono } from 'hono'; -import { WebSocketServer } from 'ws'; -import Redis from 'ioredis'; -import * as cron from 'node-cron'; -import { BaseStrategy } from './core/Strategy'; -import { StrategyRegistry } from './core/strategies/StrategyRegistry'; -import { BacktestService, BacktestRequest } from './core/backtesting/BacktestService'; -import { BacktestResult } from './core/backtesting/BacktestEngine'; -import { PerformanceAnalytics } from './core/backtesting/PerformanceAnalytics'; - -const app = new Hono(); -const redis = new Redis({ - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - enableReadyCheck: false, - maxRetriesPerRequest: null, -}); - -// WebSocket server for real-time strategy updates -const wss = new WebSocketServer({ port: 8082 }); - -// Initialize strategy registry and backtest service -const strategyRegistry = StrategyRegistry.getInstance(); -const backtestService = new BacktestService(); - -// Strategy interfaces -interface TradingStrategy { - id: string; - name: string; - description: string; - status: 'ACTIVE' | 'INACTIVE' | 'PAUSED' | 'ERROR'; - type: 'MOMENTUM' | 'MEAN_REVERSION' | 'ARBITRAGE' | 'CUSTOM'; - symbols: string[]; - parameters: Record; - performance: { - totalTrades: number; - winRate: number; - totalReturn: number; - sharpeRatio: number; - maxDrawdown: number; - }; - createdAt: Date; - updatedAt: Date; -} - -interface StrategySignal { - strategyId: string; - symbol: string; - action: 'BUY' | 'SELL' | 'HOLD'; - confidence: number; - price: number; - quantity: number; - timestamp: Date; - metadata: Record; -} - -// In-memory strategy registry (in production, this would be persisted) -const strategies = new Map(); - -// Health check endpoint -app.get('/health', (c) => { - return c.json({ - service: 'strategy-orchestrator', - status: 'healthy', - timestamp: new Date(), - version: '1.0.0', - activeStrategies: Array.from(strategies.values()).filter(s => s.status === 'ACTIVE').length, - registeredStrategies: strategyRegistry.getAllStrategies().length, - connections: wss.clients.size - }); -}); - -// API Routes -// Strategy management endpoints -app.get('/api/strategy-types', async (c) => { - try { - const types = Object.values(strategyRegistry.getStrategyTypes()); - return c.json({ success: true, data: types }); - } catch (error) { - console.error('Error getting strategy types:', error); - return c.json({ success: false, error: 'Failed to get strategy types' }, 500); - } -}); - -app.get('/api/strategies', async (c) => { - try { - const strategies = strategyRegistry.getAllStrategies(); - const serializedStrategies = strategies.map(strategy => ({ - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type: strategyRegistry.getStrategyType(strategy) - })); - - return c.json({ success: true, data: serializedStrategies }); - } catch (error) { - console.error('Error fetching strategies:', error); - return c.json({ success: false, error: 'Failed to fetch strategies' }, 500); - } -}); - -app.get('/api/strategies/:id', async (c) => { - try { - const id = c.req.param('id'); - const strategy = strategyRegistry.getStrategyById(id); - - if (!strategy) { - return c.json({ success: false, error: `Strategy with ID ${id} not found` }, 404); - } - - const type = strategyRegistry.getStrategyType(strategy); - - return c.json({ - success: true, - data: { - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type - } - }); - } catch (error) { - console.error('Error fetching strategy:', error); - return c.json({ success: false, error: 'Failed to fetch strategy' }, 500); - } -}); - -app.post('/api/strategies', async (c) => { - try { - const { name, description, symbols, parameters, type } = await c.req.json(); - - if (!type) { - return c.json({ - success: false, - error: 'Invalid strategy type' - }, 400); - } - - const strategy = strategyRegistry.createStrategy( - type, - `strategy_${Date.now()}`, // Generate an ID - name || `New ${type} Strategy`, - description || `Generated ${type} strategy`, - symbols || [], - parameters || {} - ); - - return c.json({ - success: true, - data: { - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type - } - }, 201); - } catch (error) { - console.error('Error creating strategy:', error); - return c.json({ success: false, error: (error as Error).message }, 500); - } -}); - -app.put('/api/strategies/:id', async (c) => { - try { - const id = c.req.param('id'); - const { name, description, symbols, parameters } = await c.req.json(); - - const strategy = strategyRegistry.getStrategyById(id); - - if (!strategy) { - return c.json({ success: false, error: `Strategy with ID ${id} not found` }, 404); - } - // Update properties - if (name !== undefined) strategy.name = name; - if (description !== undefined) strategy.description = description; - if (symbols !== undefined) strategy.symbols = symbols; - if (parameters !== undefined) strategy.parameters = parameters; - - return c.json({ - success: true, - data: { - id: strategy.id, - name: strategy.name, - description: strategy.description, - symbols: strategy.symbols, - parameters: strategy.parameters, - type: strategyRegistry.getStrategyType(strategy) - } - }); - } catch (error) { - console.error('Error updating strategy:', error); - return c.json({ success: false, error: (error as Error).message }, 500); - } -}); - -app.delete('/api/strategies/:id', async (c) => { - try { - const id = c.req.param('id'); - const success = strategyRegistry.deleteStrategy(id); - - if (!success) { - return c.json({ success: false, error: `Strategy with ID ${id} not found` }, 404); - } - - return c.json({ success: true, data: { id } }); - } catch (error) { - console.error('Error deleting strategy:', error); - return c.json({ success: false, error: (error as Error).message }, 500); - } -}); - -// Backtesting endpoints -app.post('/api/backtest', async (c) => { - try { - const backtestRequest = await c.req.json() as BacktestRequest; - - // Validate request - if (!backtestRequest.strategyType) { - return c.json({ success: false, error: 'Strategy type is required' }, 400); - } - - if (!backtestRequest.symbols || backtestRequest.symbols.length === 0) { - return c.json({ success: false, error: 'At least one symbol is required' }, 400); - } - - // Run the backtest - const result = await backtestService.runBacktest(backtestRequest); - - // Enhance results with additional metrics - const enhancedResult = PerformanceAnalytics.enhanceResults(result); - - // Calculate additional analytics - const monthlyReturns = PerformanceAnalytics.calculateMonthlyReturns(result.dailyReturns); - const drawdowns = PerformanceAnalytics.analyzeDrawdowns(result.dailyReturns); - - return c.json({ - success: true, - data: { - ...enhancedResult, - monthlyReturns, - drawdowns - } - }); - } catch (error) { - console.error('Backtest error:', error); - return c.json({ success: false, error: (error as Error).message }, 500); - } -}); - -app.post('/api/optimize', async (c) => { - try { - const { baseRequest, parameterGrid } = await c.req.json(); - - // Validate request - if (!baseRequest || !parameterGrid) { - return c.json({ success: false, error: 'Base request and parameter grid are required' }, 400); - } - - // Run optimization - const results = await backtestService.optimizeStrategy(baseRequest, parameterGrid); - - return c.json({ success: true, data: results }); - } catch (error) { - console.error('Strategy optimization error:', error); - return c.json({ success: false, error: (error as Error).message }, 500); - } -}); - -// Create new strategy -app.post('/api/strategies', async (c) => { - try { - const strategyData = await c.req.json(); - - const strategy: TradingStrategy = { - id: `strategy_${Date.now()}`, - name: strategyData.name, - description: strategyData.description || '', - status: 'INACTIVE', - type: strategyData.type || 'CUSTOM', - symbols: strategyData.symbols || [], - parameters: strategyData.parameters || {}, - performance: { - totalTrades: 0, - winRate: 0, - totalReturn: 0, - sharpeRatio: 0, - maxDrawdown: 0 - }, - createdAt: new Date(), - updatedAt: new Date() - }; - - strategies.set(strategy.id, strategy); - - // Store in Redis for persistence - await redis.setex( - `strategy:${strategy.id}`, - 86400, // 24 hours TTL - JSON.stringify(strategy) - ); - - // Broadcast to connected clients - broadcastToClients({ - type: 'STRATEGY_CREATED', - data: strategy, - timestamp: new Date() - }); - - return c.json({ success: true, data: strategy }); - } catch (error) { - console.error('Error creating strategy:', error); - return c.json({ success: false, error: 'Failed to create strategy' }, 500); - } -}); - -// Update strategy -app.put('/api/strategies/:id', async (c) => { - try { - const id = c.req.param('id'); - const updateData = await c.req.json(); - const strategy = strategies.get(id); - - if (!strategy) { - return c.json({ success: false, error: 'Strategy not found' }, 404); - } - - const updatedStrategy = { - ...strategy, - ...updateData, - id, // Ensure ID doesn't change - updatedAt: new Date() - }; - - strategies.set(id, updatedStrategy); - - // Update in Redis - await redis.setex( - `strategy:${id}`, - 86400, - JSON.stringify(updatedStrategy) - ); - - // Broadcast update - broadcastToClients({ - type: 'STRATEGY_UPDATED', - data: updatedStrategy, - timestamp: new Date() - }); - - return c.json({ success: true, data: updatedStrategy }); - } catch (error) { - console.error('Error updating strategy:', error); - return c.json({ success: false, error: 'Failed to update strategy' }, 500); - } -}); - -// Start/Stop strategy -app.post('/api/strategies/:id/:action', async (c) => { - try { - const id = c.req.param('id'); - const action = c.req.param('action'); - const strategy = strategies.get(id); - - if (!strategy) { - return c.json({ success: false, error: 'Strategy not found' }, 404); - } - - if (!['start', 'stop', 'pause'].includes(action)) { - return c.json({ success: false, error: 'Invalid action' }, 400); - } - - const statusMap = { - start: 'ACTIVE' as const, - stop: 'INACTIVE' as const, - pause: 'PAUSED' as const - }; - - strategy.status = statusMap[action as keyof typeof statusMap]; - strategy.updatedAt = new Date(); - - strategies.set(id, strategy); - - // Update in Redis - await redis.setex( - `strategy:${id}`, - 86400, - JSON.stringify(strategy) - ); - - // Broadcast status change - broadcastToClients({ - type: 'STRATEGY_STATUS_CHANGED', - data: { id, status: strategy.status, action }, - timestamp: new Date() - }); - - return c.json({ success: true, data: strategy }); - } catch (error) { - console.error('Error changing strategy status:', error); - return c.json({ success: false, error: 'Failed to change strategy status' }, 500); - } -}); - -// Get strategy signals -app.get('/api/strategies/:id/signals', async (c) => { - try { - const id = c.req.param('id'); - const limit = parseInt(c.req.query('limit') || '50'); - - const signalKeys = await redis.keys(`signal:${id}:*`); - const signals: any[] = []; - - for (const key of signalKeys.slice(0, limit)) { - const data = await redis.get(key); - if (data) { - signals.push(JSON.parse(data)); - } - } - - return c.json({ - success: true, - data: signals.sort((a: any, b: any) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime()) - }); - } catch (error) { - console.error('Error fetching strategy signals:', error); - return c.json({ success: false, error: 'Failed to fetch signals' }, 500); - } -}); - -// Get strategy trades -app.get('/api/strategies/:id/trades', async (c) => { - try { - const id = c.req.param('id'); - const limit = parseInt(c.req.query('limit') || '50'); - - const tradeKeys = await redis.keys(`trade:${id}:*`); - const trades: any[] = []; - - for (const key of tradeKeys.slice(0, limit)) { - const data = await redis.get(key); - if (data) { - trades.push(JSON.parse(data)); - } - } - - return c.json({ - success: true, - data: trades.sort((a: any, b: any) => new Date(b.exitTime || b.timestamp).getTime() - new Date(a.exitTime || a.timestamp).getTime()) - }); - } catch (error) { - console.error('Error fetching strategy trades:', error); - return c.json({ success: false, error: 'Failed to fetch trades' }, 500); - } -}); - -// Generate demo signal (for testing) -app.post('/api/strategies/:id/generate-signal', async (c) => { - try { - const id = c.req.param('id'); - const strategy = strategies.get(id); - - if (!strategy) { - return c.json({ success: false, error: 'Strategy not found' }, 404); - } - - if (strategy.status !== 'ACTIVE') { - return c.json({ success: false, error: 'Strategy is not active' }, 400); - } - - // Generate demo signal - const symbol = strategy.symbols[Math.floor(Math.random() * strategy.symbols.length)] || 'AAPL'; - const signal: StrategySignal = { - strategyId: id, - symbol, - action: ['BUY', 'SELL', 'HOLD'][Math.floor(Math.random() * 3)] as any, - confidence: Math.random() * 0.4 + 0.6, // 60-100% confidence - price: 150 + Math.random() * 50, - quantity: Math.floor(Math.random() * 100) + 1, - timestamp: new Date(), - metadata: { - indicator1: Math.random(), - indicator2: Math.random(), - rsi: Math.random() * 100 - } - }; - - // Store signal - await redis.setex( - `signal:${id}:${Date.now()}`, - 3600, // 1 hour TTL - JSON.stringify(signal) - ); - - // Broadcast signal - broadcastToClients({ - type: 'STRATEGY_SIGNAL', - data: signal, - timestamp: new Date() - }); - - // Publish to trading system - await redis.publish('trading:signals', JSON.stringify(signal)); - - return c.json({ success: true, data: signal }); - } catch (error) { - console.error('Error generating signal:', error); - return c.json({ success: false, error: 'Failed to generate signal' }, 500); - } -}); - -// WebSocket connection handling -wss.on('connection', (ws) => { - console.log('New strategy monitoring client connected'); - - ws.send(JSON.stringify({ - type: 'CONNECTED', - message: 'Connected to Strategy Orchestrator', - timestamp: new Date() - })); - - ws.on('close', () => { - console.log('Strategy monitoring client disconnected'); - }); - - ws.on('error', (error) => { - console.error('WebSocket error:', error); - }); -}); - -function broadcastToClients(message: any) { - const messageStr = JSON.stringify(message); - wss.clients.forEach(client => { - if (client.readyState === 1) { // WebSocket.OPEN - client.send(messageStr); - } - }); -} - -// Scheduled tasks for strategy management -cron.schedule('*/5 * * * *', async () => { - // Every 5 minutes: Check strategy health and generate signals for active strategies - console.log('Running strategy health check...'); - - for (const [id, strategy] of strategies.entries()) { - if (strategy.status === 'ACTIVE') { - try { - // Generate signals for active strategies (demo mode) - if (Math.random() > 0.7) { // 30% chance to generate a signal - const symbol = strategy.symbols[Math.floor(Math.random() * strategy.symbols.length)] || 'AAPL'; - const signal: StrategySignal = { - strategyId: id, - symbol, - action: ['BUY', 'SELL'][Math.floor(Math.random() * 2)] as any, - confidence: Math.random() * 0.3 + 0.7, - price: 150 + Math.random() * 50, - quantity: Math.floor(Math.random() * 100) + 1, - timestamp: new Date(), - metadata: { scheduled: true } - }; - - await redis.setex( - `signal:${id}:${Date.now()}`, - 3600, - JSON.stringify(signal) - ); - - broadcastToClients({ - type: 'STRATEGY_SIGNAL', - data: signal, - timestamp: new Date() - }); - - await redis.publish('trading:signals', JSON.stringify(signal)); - } - } catch (error) { - console.error(`Error in scheduled task for strategy ${id}:`, error); - } - } - } -}); - -// Backtesting API endpoints -app.post('/api/backtest', async (c) => { - try { - const request = await c.req.json() as BacktestRequest; - console.log('Received backtest request:', request); - - const result = await backtestService.runBacktest(request); - const enhancedResult = PerformanceAnalytics.enhanceResults(result); - - // Store backtest result in Redis for persistence - await redis.setex( - `backtest:${result.strategyId}`, - 86400 * 7, // 7 days TTL - JSON.stringify(enhancedResult) - ); - - return c.json({ success: true, data: enhancedResult }); - } catch (error) { - console.error('Backtest error:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } -}); - -app.post('/api/backtest/optimize', async (c) => { - try { - const { baseRequest, parameterGrid } = await c.req.json() as { - baseRequest: BacktestRequest, - parameterGrid: Record - }; - - console.log('Received optimization request:', baseRequest, parameterGrid); - - const results = await backtestService.optimizeStrategy(baseRequest, parameterGrid); - - return c.json({ success: true, data: results }); - } catch (error) { - console.error('Optimization error:', error); - return c.json({ - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }, 500); - } -}); - -app.get('/api/backtest/:id', async (c) => { - try { - const id = c.req.param('id'); - const data = await redis.get(`backtest:${id}`); - - if (!data) { - return c.json({ success: false, error: 'Backtest not found' }, 404); - } - - const result = JSON.parse(data) as BacktestResult; - return c.json({ success: true, data: result }); - } catch (error) { - console.error('Error fetching backtest:', error); - return c.json({ success: false, error: 'Failed to fetch backtest' }, 500); - } -}); - -app.get('/api/strategy-types', (c) => { - const types = strategyRegistry.getStrategyTypes(); - return c.json({ success: true, data: types }); -}); - -app.get('/api/strategy-parameters/:type', (c) => { - try { - const type = c.req.param('type') as any; - - if (!strategyRegistry.hasStrategyType(type)) { - return c.json({ success: false, error: 'Strategy type not found' }, 404); - } - - const params = strategyRegistry.getDefaultParameters(type); - return c.json({ success: true, data: params }); - } catch (error) { - console.error('Error fetching strategy parameters:', error); - return c.json({ success: false, error: 'Failed to fetch parameters' }, 500); - } -}); - -// Load existing strategies from Redis on startup -async function loadStrategiesFromRedis() { - try { - const strategyKeys = await redis.keys('strategy:*'); - for (const key of strategyKeys) { - const data = await redis.get(key); - if (data) { - const strategy = JSON.parse(data); - strategies.set(strategy.id, strategy); - } - } - console.log(`Loaded ${strategies.size} strategies from Redis`); - } catch (error) { - console.error('Error loading strategies from Redis:', error); - } -} - -const port = parseInt(process.env.PORT || '4001'); - -console.log(`๐ŸŽฏ Strategy Orchestrator starting on port ${port}`); -console.log(`๐Ÿ“ก WebSocket server running on port 8082`); - -// Load existing strategies -loadStrategiesFromRedis(); - -export default { - port, - fetch: app.fetch, -}; diff --git a/apps/intelligence-services/strategy-orchestrator/src/tests/backtesting/BacktestService.test.ts b/apps/intelligence-services/strategy-orchestrator/src/tests/backtesting/BacktestService.test.ts deleted file mode 100644 index fdc2405..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/tests/backtesting/BacktestService.test.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { describe, test, expect, beforeEach } from 'bun:test'; -import { BacktestService, BacktestRequest } from '../../core/backtesting/BacktestService'; -import { StrategyRegistry, StrategyType } from '../../core/strategies/StrategyRegistry'; -import { MarketDataFeed } from '../../core/backtesting/MarketDataFeed'; - -// Mock dependencies -jest.mock('../../core/backtesting/MarketDataFeed'); - -describe('BacktestService', () => { - let backtestService: BacktestService; - let mockRequest: BacktestRequest; - - beforeEach(() => { - // Reset mocks and create fresh service instance - jest.clearAllMocks(); - backtestService = new BacktestService('http://test.api'); - - // Create a standard backtest request for tests - mockRequest = { - strategyType: 'MEAN_REVERSION' as StrategyType, - strategyParams: { - lookback: 20, - entryDeviation: 1.5, - exitDeviation: 0.5, - lookbackPeriod: 100 - }, - symbols: ['AAPL'], - startDate: new Date('2023-01-01'), - endDate: new Date('2023-02-01'), - initialCapital: 100000, - dataResolution: '1d', - commission: 0.001, - slippage: 0.001, - mode: 'vector' - }; - - // Mock the MarketDataFeed implementation - (MarketDataFeed.prototype.getHistoricalData as jest.Mock).mockResolvedValue([ - // Generate some sample data - ...Array(30).fill(0).map((_, i) => ({ - symbol: 'AAPL', - timestamp: new Date(`2023-01-${(i + 1).toString().padStart(2, '0')}`), - open: 150 + Math.random() * 10, - high: 155 + Math.random() * 10, - low: 145 + Math.random() * 10, - close: 150 + Math.random() * 10, - volume: 1000000 + Math.random() * 500000 - })) - ]); - }); - - test('should run a backtest successfully', async () => { - // Act - const result = await backtestService.runBacktest(mockRequest); - - // Assert - expect(result).toBeDefined(); - expect(result.strategyId).toBeDefined(); - expect(result.initialCapital).toBe(100000); - expect(result.trades).toBeDefined(); - expect(result.dailyReturns).toBeDefined(); - - // Verify market data was requested - expect(MarketDataFeed.prototype.getHistoricalData).toHaveBeenCalledTimes(mockRequest.symbols.length); - }); - - test('should optimize strategy parameters', async () => { - // Arrange - const parameterGrid = { - lookback: [10, 20], - entryDeviation: [1.0, 1.5, 2.0] - }; - - // We should get 2ร—3 = 6 combinations - - // Act - const results = await backtestService.optimizeStrategy(mockRequest, parameterGrid); - - // Assert - expect(results).toHaveLength(6); - expect(results[0].parameters).toBeDefined(); - - // Check that results are sorted by performance (sharpe ratio) - for (let i = 0; i < results.length - 1; i++) { - expect(results[i].sharpeRatio).toBeGreaterThanOrEqual(results[i + 1].sharpeRatio); - } - }); - - test('should handle errors during backtest', async () => { - // Arrange - (MarketDataFeed.prototype.getHistoricalData as jest.Mock).mockRejectedValue( - new Error('Data source error') - ); - - // Act & Assert - await expect(backtestService.runBacktest(mockRequest)) - .rejects - .toThrow(); - }); - - test('should generate correct parameter combinations', () => { - // Arrange - const grid = { - param1: [1, 2], - param2: ['a', 'b'], - param3: [true, false] - }; - - // Act - const combinations = (backtestService as any).generateParameterCombinations(grid, Object.keys(grid)); - - // Assert - should get 2ร—2ร—2 = 8 combinations - expect(combinations).toHaveLength(8); - - // Check that all combinations are generated - expect(combinations).toContainEqual({ param1: 1, param2: 'a', param3: true }); - expect(combinations).toContainEqual({ param1: 1, param2: 'a', param3: false }); - expect(combinations).toContainEqual({ param1: 1, param2: 'b', param3: true }); - expect(combinations).toContainEqual({ param1: 1, param2: 'b', param3: false }); - expect(combinations).toContainEqual({ param1: 2, param2: 'a', param3: true }); - expect(combinations).toContainEqual({ param1: 2, param2: 'a', param3: false }); - expect(combinations).toContainEqual({ param1: 2, param2: 'b', param3: true }); - expect(combinations).toContainEqual({ param1: 2, param2: 'b', param3: false }); - }); - - test('should track active backtests', () => { - // Arrange - const activeBacktests = (backtestService as any).activeBacktests; - - // Act - let promise = backtestService.runBacktest(mockRequest); - - // Assert - expect(activeBacktests.size).toBe(1); - - // Clean up - return promise; - }); -}); diff --git a/apps/intelligence-services/strategy-orchestrator/src/tests/backtesting/PerformanceAnalytics.test.ts b/apps/intelligence-services/strategy-orchestrator/src/tests/backtesting/PerformanceAnalytics.test.ts deleted file mode 100644 index 3a2e218..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/tests/backtesting/PerformanceAnalytics.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { describe, test, expect } from 'bun:test'; -import { PerformanceAnalytics } from '../../core/backtesting/PerformanceAnalytics'; -import { BacktestResult } from '../../core/backtesting/BacktestEngine'; - -describe('PerformanceAnalytics', () => { - // Sample backtest result for testing - const sampleResult: BacktestResult = { - strategyId: 'test-strategy', - startDate: new Date('2023-01-01'), - endDate: new Date('2023-12-31'), - duration: 31536000000, // 1 year in ms - initialCapital: 100000, - finalCapital: 125000, - totalReturn: 0.25, // 25% return - annualizedReturn: 0.25, - sharpeRatio: 1.5, - maxDrawdown: 0.10, // 10% drawdown - maxDrawdownDuration: 30, // 30 days - winRate: 0.6, // 60% win rate - totalTrades: 50, - winningTrades: 30, - losingTrades: 20, - averageWinningTrade: 0.05, // 5% average win - averageLosingTrade: -0.03, // 3% average loss - profitFactor: 2.5, - dailyReturns: [ - // Generate 365 days of sample daily returns with some randomness - ...Array(365).fill(0).map((_, i) => ({ - date: new Date(new Date('2023-01-01').getTime() + i * 24 * 3600 * 1000), - return: 0.001 + (Math.random() - 0.45) * 0.01 // Mean positive return with noise - })) - ], - trades: [ - // Generate sample trades - ...Array(50).fill(0).map((_, i) => ({ - symbol: 'AAPL', - entryTime: new Date(`2023-${Math.floor(i / 4) + 1}-${(i % 28) + 1}`), - entryPrice: 150 + Math.random() * 10, - exitTime: new Date(`2023-${Math.floor(i / 4) + 1}-${(i % 28) + 5}`), - exitPrice: 155 + Math.random() * 10, - quantity: 10, - pnl: 500 * (Math.random() - 0.3), // Some wins, some losses - pnlPercent: 0.05 * (Math.random() - 0.3) - })) - ] - }; - - test('should calculate advanced metrics', () => { - // Act - const enhancedResult = PerformanceAnalytics.enhanceResults(sampleResult); - - // Assert - expect(enhancedResult.sortinoRatio).toBeDefined(); - expect(enhancedResult.calmarRatio).toBeDefined(); - expect(enhancedResult.omegaRatio).toBeDefined(); - expect(enhancedResult.cagr).toBeDefined(); - expect(enhancedResult.volatility).toBeDefined(); - expect(enhancedResult.ulcerIndex).toBeDefined(); - - // Check that the original result properties are preserved - expect(enhancedResult.strategyId).toBe(sampleResult.strategyId); - expect(enhancedResult.totalReturn).toBe(sampleResult.totalReturn); - - // Validate some calculations - expect(enhancedResult.calmarRatio).toBeCloseTo(sampleResult.annualizedReturn / sampleResult.maxDrawdown); - expect(typeof enhancedResult.sortinoRatio).toBe('number'); - }); - - test('should calculate monthly returns', () => { - // Act - const monthlyReturns = PerformanceAnalytics.calculateMonthlyReturns(sampleResult.dailyReturns); - - // Assert - expect(monthlyReturns).toBeDefined(); - expect(monthlyReturns.length).toBe(12); // 12 months in a year - expect(monthlyReturns[0].year).toBe(2023); - expect(monthlyReturns[0].month).toBe(0); // January is 0 - - // Verify sorting - let lastDate = { year: 0, month: 0 }; - for (const mr of monthlyReturns) { - expect(mr.year >= lastDate.year).toBeTruthy(); - if (mr.year === lastDate.year) { - expect(mr.month >= lastDate.month).toBeTruthy(); - } - lastDate = { year: mr.year, month: mr.month }; - } - }); - - test('should analyze drawdowns', () => { - // Act - const drawdowns = PerformanceAnalytics.analyzeDrawdowns(sampleResult.dailyReturns); - - // Assert - expect(drawdowns).toBeDefined(); - expect(drawdowns.length).toBeGreaterThan(0); - - // Check drawdown properties - for (const dd of drawdowns) { - expect(dd.startDate).toBeInstanceOf(Date); - expect(dd.endDate).toBeInstanceOf(Date); - expect(dd.drawdown).toBeGreaterThan(0); - expect(dd.durationDays).toBeGreaterThanOrEqual(0); - - // Recovery date and days might be null for ongoing drawdowns - if (dd.recoveryDate) { - expect(dd.recoveryDate).toBeInstanceOf(Date); - expect(dd.recoveryDays).toBeGreaterThanOrEqual(0); - } - } - - // Check sorting by drawdown magnitude - for (let i = 0; i < drawdowns.length - 1; i++) { - expect(drawdowns[i].drawdown).toBeGreaterThanOrEqual(drawdowns[i + 1].drawdown); - } - }); - - test('should handle empty inputs', () => { - // Act & Assert - expect(() => PerformanceAnalytics.calculateMonthlyReturns([])).not.toThrow(); - expect(() => PerformanceAnalytics.analyzeDrawdowns([])).not.toThrow(); - - const emptyMonthlyReturns = PerformanceAnalytics.calculateMonthlyReturns([]); - const emptyDrawdowns = PerformanceAnalytics.analyzeDrawdowns([]); - - expect(emptyMonthlyReturns).toEqual([]); - expect(emptyDrawdowns).toEqual([]); - }); - - test('should calculate special cases correctly', () => { - // Case with no negative returns - const allPositiveReturns = { - dailyReturns: Array(30).fill(0).map((_, i) => ({ - date: new Date(`2023-01-${i + 1}`), - return: 0.01 // Always positive - })) - }; - - // Case with no recovery from drawdown - const noRecoveryReturns = { - dailyReturns: [ - ...Array(30).fill(0).map((_, i) => ({ - date: new Date(`2023-01-${i + 1}`), - return: 0.01 // Positive returns - })), - ...Array(30).fill(0).map((_, i) => ({ - date: new Date(`2023-02-${i + 1}`), - return: -0.005 // Negative returns with no recovery - })) - ] - }; - - // Act - const positiveMetrics = PerformanceAnalytics.enhanceResults({ - ...sampleResult, - dailyReturns: allPositiveReturns.dailyReturns - }); - - const noRecoveryDrawdowns = PerformanceAnalytics.analyzeDrawdowns(noRecoveryReturns.dailyReturns); - - // Assert - expect(positiveMetrics.sortinoRatio).toBe(Infinity); // No downside risk - - // Last drawdown should have no recovery - const lastDrawdown = noRecoveryDrawdowns[noRecoveryDrawdowns.length - 1]; - expect(lastDrawdown.recoveryDate).toBeNull(); - expect(lastDrawdown.recoveryDays).toBeNull(); - }); -}); diff --git a/apps/intelligence-services/strategy-orchestrator/src/tests/execution/StrategyExecutionService.test.ts b/apps/intelligence-services/strategy-orchestrator/src/tests/execution/StrategyExecutionService.test.ts deleted file mode 100644 index 0c9824d..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/tests/execution/StrategyExecutionService.test.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, jest } from 'bun:test'; -import { EventEmitter } from 'events'; -import { WebSocket } from 'ws'; -import { StrategyExecutionService } from '../../core/execution/StrategyExecutionService'; -import { StrategyRegistry } from '../../core/strategies/StrategyRegistry'; -import { MarketDataFeed } from '../../core/backtesting/MarketDataFeed'; -import { BaseStrategy, BarData, Order } from '../../core/Strategy'; - -// Mock WebSocket to avoid actual network connections during tests -jest.mock('ws', () => { - const EventEmitter = require('events'); - - class MockWebSocket extends EventEmitter { - static OPEN = 1; - readyState = 1; - close = jest.fn(); - send = jest.fn(); - } - - class MockServer extends EventEmitter { - clients = new Set(); - - constructor() { - super(); - // Add a mock client to the set - const mockClient = new MockWebSocket(); - this.clients.add(mockClient); - } - - close(callback) { - callback(); - } - } - - return { - WebSocket: MockWebSocket, - Server: MockServer - }; -}); - -// Mock MarketDataFeed to avoid actual API calls -jest.mock('../../core/backtesting/MarketDataFeed', () => { - return { - MarketDataFeed: class { - async getHistoricalData(symbol, resolution, startDate, endDate) { - // Return mock data - return [ - { - symbol, - timestamp: new Date(), - open: 100, - high: 105, - low: 95, - close: 102, - volume: 1000 - } - ]; - } - } - }; -}); - -// Mock strategy for testing -class MockStrategy extends BaseStrategy { - name = 'MockStrategy'; - description = 'A mock strategy for testing'; - symbols = ['AAPL', 'MSFT']; - parameters = { param1: 1, param2: 2 }; - - constructor(id: string) { - super(id); - } - - async start(): Promise {} - - async stop(): Promise {} - - onBar(bar: BarData) { - // Return a mock signal - return { - action: 'BUY', - symbol: bar.symbol, - price: bar.close, - quantity: 10, - metadata: { reason: 'Test signal' } - }; - } - - async onOrderFilled(order: Order): Promise {} -} - -// Mock StrategyRegistry -jest.mock('../../core/strategies/StrategyRegistry', () => { - const mockInstance = { - getStrategyById: jest.fn(), - getStrategyTypes: () => [{ id: 'mock-strategy', name: 'Mock Strategy' }], - getAllStrategies: () => [new MockStrategy('mock-1')] - }; - - return { - StrategyRegistry: { - getInstance: () => mockInstance - } - }; -}); - -describe('StrategyExecutionService', () => { - let executionService: StrategyExecutionService; - let strategyRegistry: typeof StrategyRegistry.getInstance; - - beforeEach(() => { - // Reset mocks - jest.clearAllMocks(); - - // Create a new execution service for each test - executionService = new StrategyExecutionService('http://localhost:3001/api', 8082); - strategyRegistry = StrategyRegistry.getInstance(); - - // Setup mock strategy - const mockStrategy = new MockStrategy('test-strategy'); - (strategyRegistry.getStrategyById as jest.Mock).mockReturnValue(mockStrategy); - }); - - afterEach(() => { - executionService.shutdown(); - }); - - it('should initialize correctly', () => { - expect(executionService).toBeDefined(); - }); - - it('should start a strategy correctly', () => { - // Arrange & Act - const result = executionService.startStrategy('test-strategy'); - - // Assert - expect(result).toBe(true); - expect(strategyRegistry.getStrategyById).toHaveBeenCalledWith('test-strategy'); - - // Check if WebSocket broadcast happened - const ws = executionService['webSocketServer'].clients.values().next().value; - expect(ws.send).toHaveBeenCalled(); - - // Check the broadcast message contains the correct type - const lastCall = ws.send.mock.calls[0][0]; - const message = JSON.parse(lastCall); - expect(message.type).toBe('strategy_started'); - expect(message.data.strategyId).toBe('test-strategy'); - }); - - it('should stop a strategy correctly', () => { - // Arrange - executionService.startStrategy('test-strategy'); - - // Act - const result = executionService.stopStrategy('test-strategy'); - - // Assert - expect(result).toBe(true); - - // Check if WebSocket broadcast happened - const ws = executionService['webSocketServer'].clients.values().next().value; - const lastCallIndex = ws.send.mock.calls.length - 1; - const lastCall = ws.send.mock.calls[lastCallIndex][0]; - const message = JSON.parse(lastCall); - - expect(message.type).toBe('strategy_stopped'); - expect(message.data.strategyId).toBe('test-strategy'); - }); - - it('should pause a strategy correctly', () => { - // Arrange - executionService.startStrategy('test-strategy'); - - // Act - const result = executionService.pauseStrategy('test-strategy'); - - // Assert - expect(result).toBe(true); - - // Check if WebSocket broadcast happened - const ws = executionService['webSocketServer'].clients.values().next().value; - const lastCallIndex = ws.send.mock.calls.length - 1; - const lastCall = ws.send.mock.calls[lastCallIndex][0]; - const message = JSON.parse(lastCall); - - expect(message.type).toBe('strategy_paused'); - expect(message.data.strategyId).toBe('test-strategy'); - }); - - it('should process market data and generate signals', async () => { - // Arrange - executionService.startStrategy('test-strategy'); - - // Act - Trigger market data polling manually - await executionService['pollMarketData']('test-strategy'); - - // Assert - Check if signal was generated and broadcast - const ws = executionService['webSocketServer'].clients.values().next().value; - - // Find the strategy_signal message - const signalMessages = ws.send.mock.calls - .map(call => JSON.parse(call[0])) - .filter(msg => msg.type === 'strategy_signal'); - - expect(signalMessages.length).toBeGreaterThan(0); - expect(signalMessages[0].data.action).toBe('BUY'); - expect(signalMessages[0].data.strategyId).toBe('test-strategy'); - }); - - it('should handle WebSocket client connections', () => { - // Arrange - const mockWs = new WebSocket(); - const mockMessage = JSON.stringify({ type: 'get_active_strategies' }); - - // Act - Simulate connection and message - executionService['webSocketServer'].emit('connection', mockWs); - mockWs.emit('message', mockMessage); - - // Assert - expect(mockWs.send).toHaveBeenCalled(); - - // Check that the response is a strategy_status_list message - const lastCall = mockWs.send.mock.calls[0][0]; - const message = JSON.parse(lastCall); - expect(message.type).toBe('strategy_status_list'); - }); - - it('should shut down correctly', () => { - // Act - executionService.shutdown(); - - // Assert - WebSocket server should be closed - const ws = executionService['webSocketServer'].clients.values().next().value; - expect(ws.close).toHaveBeenCalled(); - }); -}); diff --git a/apps/intelligence-services/strategy-orchestrator/src/tests/strategies/MeanReversionStrategy.test.ts b/apps/intelligence-services/strategy-orchestrator/src/tests/strategies/MeanReversionStrategy.test.ts deleted file mode 100644 index 1fdb8d2..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/tests/strategies/MeanReversionStrategy.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { describe, test, expect, beforeEach, mock } from 'bun:test'; -import { MeanReversionStrategy } from '../../core/strategies/MeanReversionStrategy'; -import { BarData } from '../../core/Strategy'; - -describe('MeanReversionStrategy', () => { - let strategy: MeanReversionStrategy; - let mockData: BarData[]; - - beforeEach(() => { - // Create a strategy instance with test parameters - strategy = new MeanReversionStrategy( - 'test_id', - 'Test Mean Reversion', - 'A test strategy', - ['AAPL'], - { - lookback: 20, - entryDeviation: 1.5, - exitDeviation: 0.5, - lookbackPeriod: 100, - positionSize: 0.2, - stopLoss: 0.02, - takeProfit: 0.05, - useBollingerBands: true, - bollingerPeriod: 20, - bollingerDeviation: 2, - rsiPeriod: 14, - rsiOverbought: 70, - rsiOversold: 30, - useRsi: true - } - ); - - // Create mock price data - const now = new Date(); - mockData = []; - - // Create 100 bars of data with a mean-reverting pattern - let price = 100; - for (let i = 0; i < 100; i++) { - // Add some mean reversion pattern (oscillating around 100) - price = price + Math.sin(i / 10) * 5 + (Math.random() - 0.5) * 2; - - mockData.push({ - symbol: 'AAPL', - timestamp: new Date(now.getTime() - (100 - i) * 60000), // 1-minute bars - open: price - 0.5, - high: price + 1, - low: price - 1, - close: price, - volume: 1000 + Math.random() * 1000 - }); - } - }); - - test('should initialize with correct parameters', () => { - expect(strategy.id).toBe('test_id'); - expect(strategy.name).toBe('Test Mean Reversion'); - expect(strategy.description).toBe('A test strategy'); - expect(strategy.symbols).toEqual(['AAPL']); - expect(strategy.parameters.lookback).toBe(20); - expect(strategy.parameters.entryDeviation).toBe(1.5); - }); - - test('should generate signals with vectorized calculation', async () => { - // Arrange a price series with fake mean reversion - const results = await strategy.runVectorized({ - symbols: ['AAPL'], - data: { 'AAPL': mockData }, - initialCapital: 10000, - startIndex: 20, // Skip the first 20 bars for indicator warmup - endIndex: mockData.length - 1 - }); - - // Assert - expect(results).toBeDefined(); - expect(results.positions).toBeDefined(); - // Should generate at least one trade in this artificial data - expect(results.trades.length).toBeGreaterThan(0); - expect(results.equityCurve.length).toBeGreaterThan(0); - }); - - test('should calculate correct entry and exit signals', () => { - // Mock the indicator calculations to test logic directly - // We'll create a simple scenario where price is 2 standard deviations away - const mockBar: BarData = { - symbol: 'AAPL', - timestamp: new Date(), - open: 100, - high: 102, - low: 98, - close: 100, - volume: 1000 - }; - - // Mock the calculation context - const context = { - mean: 100, - stdDev: 5, - upperBand: 110, - lowerBand: 90, - rsi: 25, // Oversold - shouldEnterLong: true, - shouldExitLong: false, - shouldEnterShort: false, - shouldExitShort: false - }; - - // Call the internal signal generation logic via a protected method - // (For testing purposes, we're accessing a protected method) - const result = (strategy as any).calculateSignals('AAPL', mockBar, context); - - // Assert the signals based on our scenario - expect(result).toBeDefined(); - expect(result.action).toBe('BUY'); // Should buy in oversold condition - }); - - test('should handle empty data correctly', async () => { - // Act & Assert - await expect(async () => { - await strategy.runVectorized({ - symbols: ['AAPL'], - data: { 'AAPL': [] }, - initialCapital: 10000, - startIndex: 0, - endIndex: 0 - }); - }).not.toThrow(); - }); -}); diff --git a/apps/intelligence-services/strategy-orchestrator/src/tests/strategies/StrategyRegistry.test.ts b/apps/intelligence-services/strategy-orchestrator/src/tests/strategies/StrategyRegistry.test.ts deleted file mode 100644 index 2af3c31..0000000 --- a/apps/intelligence-services/strategy-orchestrator/src/tests/strategies/StrategyRegistry.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { describe, test, expect, beforeEach } from 'bun:test'; -import { BaseStrategy } from '../../core/Strategy'; -import { StrategyRegistry, StrategyType } from '../../core/strategies/StrategyRegistry'; -import { MovingAverageCrossover } from '../../core/strategies/MovingAverageCrossover'; -import { MeanReversionStrategy } from '../../core/strategies/MeanReversionStrategy'; -import { VectorizedStrategy } from '../../core/strategies/VectorizedStrategy'; - -describe('Strategy Registry', () => { - let registry: StrategyRegistry; - - beforeEach(() => { - // Reset the singleton for testing - (StrategyRegistry as any).instance = null; - registry = StrategyRegistry.getInstance(); - }); - - test('should create a MovingAverageCrossover strategy', () => { - // Arrange - const id = 'test_id'; - const name = 'Test Strategy'; - const description = 'A test strategy'; - const symbols = ['AAPL', 'MSFT']; - const parameters = { fastPeriod: 10, slowPeriod: 30 }; - - // Act - const strategy = registry.createStrategy( - 'MOVING_AVERAGE_CROSSOVER', - id, - name, - description, - symbols, - parameters - ); - - // Assert - expect(strategy).toBeInstanceOf(MovingAverageCrossover); - expect(strategy.id).toEqual(id); - expect(strategy.name).toEqual(name); - expect(strategy.description).toEqual(description); - expect(strategy.symbols).toEqual(symbols); - expect(strategy.parameters).toMatchObject(parameters); - }); - - test('should create a MeanReversion strategy', () => { - // Arrange - const id = 'test_id'; - const name = 'Test Strategy'; - const description = 'A test strategy'; - const symbols = ['AAPL', 'MSFT']; - const parameters = { lookback: 20, entryDeviation: 1.5 }; - - // Act - const strategy = registry.createStrategy( - 'MEAN_REVERSION', - id, - name, - description, - symbols, - parameters - ); - - // Assert - expect(strategy).toBeInstanceOf(MeanReversionStrategy); - expect(strategy.id).toEqual(id); - expect(strategy.name).toEqual(name); - expect(strategy.description).toEqual(description); - expect(strategy.symbols).toEqual(symbols); - expect(strategy.parameters).toMatchObject(parameters); - }); - - test('should throw error for invalid strategy type', () => { - // Arrange - const id = 'test_id'; - const name = 'Test Strategy'; - const description = 'A test strategy'; - const symbols = ['AAPL', 'MSFT']; - const parameters = {}; - - // Act & Assert - expect(() => { - registry.createStrategy( - 'INVALID_TYPE' as StrategyType, - id, - name, - description, - symbols, - parameters - ); - }).toThrow("Strategy type 'INVALID_TYPE' is not registered"); - }); - - test('should register a custom strategy', () => { - // Arrange - const mockStrategyFactory = ( - id: string, - name: string, - description: string, - symbols: string[], - parameters: any - ) => { - return new MovingAverageCrossover(id, name, description, symbols, parameters); - }; - - // Act - registry.registerStrategy('CUSTOM' as StrategyType, mockStrategyFactory); - - // Assert - expect(registry.hasStrategyType('CUSTOM')).toBe(true); - - const strategy = registry.createStrategy( - 'CUSTOM', - 'custom_id', - 'Custom Strategy', - 'A custom strategy', - ['BTC/USD'], - {} - ); - - expect(strategy).toBeInstanceOf(MovingAverageCrossover); - }); - - test('should get default parameters for a strategy type', () => { - // Act - const macParams = registry.getDefaultParameters('MOVING_AVERAGE_CROSSOVER'); - const mrParams = registry.getDefaultParameters('MEAN_REVERSION'); - - // Assert - expect(macParams).toHaveProperty('fastPeriod'); - expect(macParams).toHaveProperty('slowPeriod'); - expect(mrParams).toHaveProperty('lookback'); - expect(mrParams).toHaveProperty('entryDeviation'); - }); - - test('should return empty object for unknown strategy default parameters', () => { - // Act - const params = registry.getDefaultParameters('CUSTOM' as StrategyType); - - // Assert - expect(params).toEqual({}); - }); - - test('should get all registered strategy types', () => { - // Act - const types = registry.getStrategyTypes(); - - // Assert - expect(types).toContain('MOVING_AVERAGE_CROSSOVER'); - expect(types).toContain('MEAN_REVERSION'); - }); - - test('should check if strategy type is registered', () => { - // Act & Assert - expect(registry.hasStrategyType('MOVING_AVERAGE_CROSSOVER')).toBe(true); - expect(registry.hasStrategyType('INVALID_TYPE' as StrategyType)).toBe(false); - }); - - test('should get all registered strategies', () => { - // Arrange - registry.createStrategy( - 'MOVING_AVERAGE_CROSSOVER', - 'mac_id', - 'MAC Strategy', - 'MAC strategy', - ['AAPL'], - {} - ); - - registry.createStrategy( - 'MEAN_REVERSION', - 'mr_id', - 'MR Strategy', - 'MR strategy', - ['MSFT'], - {} - ); - - // Act - const strategies = registry.getAllStrategies(); - - // Assert - expect(strategies).toHaveLength(2); - expect(strategies[0].id).toEqual('mac_id'); - expect(strategies[1].id).toEqual('mr_id'); - }); - - test('should get strategy by ID', () => { - // Arrange - registry.createStrategy( - 'MOVING_AVERAGE_CROSSOVER', - 'mac_id', - 'MAC Strategy', - 'MAC strategy', - ['AAPL'], - {} - ); - - // Act - const strategy = registry.getStrategyById('mac_id'); - const nonExistent = registry.getStrategyById('non_existent'); - - // Assert - expect(strategy).not.toBeNull(); - expect(strategy?.id).toEqual('mac_id'); - expect(nonExistent).toBeUndefined(); - }); - - test('should delete strategy by ID', () => { - // Arrange - registry.createStrategy( - 'MOVING_AVERAGE_CROSSOVER', - 'mac_id', - 'MAC Strategy', - 'MAC strategy', - ['AAPL'], - {} - ); - - // Act - const result1 = registry.deleteStrategy('mac_id'); - const result2 = registry.deleteStrategy('non_existent'); - - // Assert - expect(result1).toBe(true); - expect(result2).toBe(false); - expect(registry.getStrategyById('mac_id')).toBeUndefined(); - }); - - test('should identify strategy type from instance', () => { - // Arrange - const macStrategy = registry.createStrategy( - 'MOVING_AVERAGE_CROSSOVER', - 'mac_id', - 'MAC Strategy', - 'MAC strategy', - ['AAPL'], - {} - ); - - const mrStrategy = registry.createStrategy( - 'MEAN_REVERSION', - 'mr_id', - 'MR Strategy', - 'MR strategy', - ['MSFT'], - {} - ); - - // Act - const macType = registry.getStrategyType(macStrategy); - const mrType = registry.getStrategyType(mrStrategy); - - // Assert - expect(macType).toEqual('MOVING_AVERAGE_CROSSOVER'); - expect(mrType).toEqual('MEAN_REVERSION'); - }); -}); diff --git a/apps/intelligence-services/strategy-orchestrator/tsconfig.json b/apps/intelligence-services/strategy-orchestrator/tsconfig.json deleted file mode 100644 index 168d88b..0000000 --- a/apps/intelligence-services/strategy-orchestrator/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "../../../tsconfig.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "module": "ESNext", - "moduleResolution": "bundler", - "types": ["bun-types"] - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/bun.lock b/bun.lock index 9538290..5cd895c 100644 --- a/bun.lock +++ b/bun.lock @@ -293,9 +293,9 @@ "dependencies": { "@stock-bot/config": "workspace:*", "@stock-bot/types": "workspace:*", - "winston": "^3.11.0", - "winston-daily-rotate-file": "^4.7.1", - "winston-loki": "^6.0.8", + "pino": "^9.7.0", + "pino-loki": "^2.6.0", + "pino-pretty": "^13.0.0", }, "devDependencies": { "@types/jest": "^29.5.2", @@ -435,9 +435,7 @@ "@bcoe/v8-coverage": ["@bcoe/v8-coverage@0.2.3", "", {}, "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw=="], - "@colors/colors": ["@colors/colors@1.6.0", "", {}, "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA=="], - - "@dabh/diagnostics": ["@dabh/diagnostics@2.0.3", "", { "dependencies": { "colorspace": "1.1.x", "enabled": "2.0.x", "kuler": "^2.0.0" } }, "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA=="], + "@colors/colors": ["@colors/colors@1.5.0", "", {}, "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="], "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-9o3TMmpmftaCMepOdA5k/yDw8SfInyzWWTjYTFCX3kPSDJMROQTb8jg+h9Cnwnmm1vOzvxN7gIfB5V2ewpjtGA=="], @@ -649,32 +647,6 @@ "@napi-rs/nice-win32-x64-msvc": ["@napi-rs/nice-win32-x64-msvc@1.0.1", "", { "os": "win32", "cpu": "x64" }, "sha512-JlF+uDcatt3St2ntBG8H02F1mM45i5SF9W+bIKiReVE6wiy3o16oBP/yxt+RZ+N6LbCImJXJ6bXNO2kn9AXicg=="], - "@napi-rs/snappy-android-arm-eabi": ["@napi-rs/snappy-android-arm-eabi@7.2.2", "", { "os": "android", "cpu": "arm" }, "sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw=="], - - "@napi-rs/snappy-android-arm64": ["@napi-rs/snappy-android-arm64@7.2.2", "", { "os": "android", "cpu": "arm64" }, "sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw=="], - - "@napi-rs/snappy-darwin-arm64": ["@napi-rs/snappy-darwin-arm64@7.2.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g=="], - - "@napi-rs/snappy-darwin-x64": ["@napi-rs/snappy-darwin-x64@7.2.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ=="], - - "@napi-rs/snappy-freebsd-x64": ["@napi-rs/snappy-freebsd-x64@7.2.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA=="], - - "@napi-rs/snappy-linux-arm-gnueabihf": ["@napi-rs/snappy-linux-arm-gnueabihf@7.2.2", "", { "os": "linux", "cpu": "arm" }, "sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg=="], - - "@napi-rs/snappy-linux-arm64-gnu": ["@napi-rs/snappy-linux-arm64-gnu@7.2.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ=="], - - "@napi-rs/snappy-linux-arm64-musl": ["@napi-rs/snappy-linux-arm64-musl@7.2.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw=="], - - "@napi-rs/snappy-linux-x64-gnu": ["@napi-rs/snappy-linux-x64-gnu@7.2.2", "", { "os": "linux", "cpu": "x64" }, "sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw=="], - - "@napi-rs/snappy-linux-x64-musl": ["@napi-rs/snappy-linux-x64-musl@7.2.2", "", { "os": "linux", "cpu": "x64" }, "sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA=="], - - "@napi-rs/snappy-win32-arm64-msvc": ["@napi-rs/snappy-win32-arm64-msvc@7.2.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA=="], - - "@napi-rs/snappy-win32-ia32-msvc": ["@napi-rs/snappy-win32-ia32-msvc@7.2.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ=="], - - "@napi-rs/snappy-win32-x64-msvc": ["@napi-rs/snappy-win32-x64-msvc@7.2.2", "", { "os": "win32", "cpu": "x64" }, "sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w=="], - "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], @@ -729,26 +701,6 @@ "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], - "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], - - "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], - - "@protobufjs/codegen": ["@protobufjs/codegen@2.0.4", "", {}, "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="], - - "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="], - - "@protobufjs/fetch": ["@protobufjs/fetch@1.1.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ=="], - - "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], - - "@protobufjs/inquire": ["@protobufjs/inquire@1.1.0", "", {}, "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="], - - "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], - - "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], - - "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], - "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.40.2", "", { "os": "android", "cpu": "arm" }, "sha512-JkdNEq+DFxZfUwxvB58tHMHBHVgX23ew41g1OQinthJ+ryhdRk67O31S7sYw8u2lTjHUPFxwar07BBt1KHp/hg=="], "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.40.2", "", { "os": "android", "cpu": "arm64" }, "sha512-13unNoZ8NzUmnndhPTkWPWbX3vtHodYmy+I9kuLxN+F+l+x3LdVF7UCu8TWVMt1POHLh6oDHhnOA04n8oJZhBw=="], @@ -935,8 +887,6 @@ "@types/stack-utils": ["@types/stack-utils@2.0.3", "", {}, "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw=="], - "@types/triple-beam": ["@types/triple-beam@1.3.5", "", {}, "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw=="], - "@types/uuid": ["@types/uuid@9.0.8", "", {}, "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA=="], "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], @@ -997,10 +947,6 @@ "array-union": ["array-union@2.1.0", "", {}, "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw=="], - "async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="], - - "async-exit-hook": ["async-exit-hook@2.0.1", "", {}, "sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw=="], - "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="], @@ -1043,8 +989,6 @@ "bser": ["bser@2.1.1", "", { "dependencies": { "node-int64": "^0.4.0" } }, "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ=="], - "btoa": ["btoa@1.2.1", "", { "bin": { "btoa": "bin/btoa.js" } }, "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g=="], - "buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], @@ -1097,18 +1041,12 @@ "collect-v8-coverage": ["collect-v8-coverage@1.0.2", "", {}, "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q=="], - "color": ["color@3.2.1", "", { "dependencies": { "color-convert": "^1.9.3", "color-string": "^1.6.0" } }, "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA=="], - "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], - "color-string": ["color-string@1.9.1", "", { "dependencies": { "color-name": "^1.0.0", "simple-swizzle": "^0.2.2" } }, "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg=="], - "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], - "colorspace": ["colorspace@1.1.4", "", { "dependencies": { "color": "^3.1.3", "text-hex": "1.0.x" } }, "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w=="], - "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], "compressible": ["compressible@2.0.18", "", { "dependencies": { "mime-db": ">= 1.43.0 < 2" } }, "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg=="], @@ -1151,6 +1089,8 @@ "date-format": ["date-format@4.0.14", "", {}, "sha512-39BOQLs9ZjKh0/patS9nrT8wc3ioX3/eA/zgbKNopnF2wCqJEoxywwwElATYvRsXdnOxA/OQeQoFZ3rFjVajhg=="], + "dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="], + "debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], "dedent": ["dedent@1.6.0", "", { "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, "optionalPeers": ["babel-plugin-macros"] }, "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA=="], @@ -1205,12 +1145,12 @@ "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], - "enabled": ["enabled@2.0.0", "", {}, "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ=="], - "encodeurl": ["encodeurl@1.0.2", "", {}, "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="], "encoding": ["encoding@0.1.13", "", { "dependencies": { "iconv-lite": "^0.6.2" } }, "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A=="], + "end-of-stream": ["end-of-stream@1.4.4", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q=="], + "engine.io": ["engine.io@6.6.4", "", { "dependencies": { "@types/cors": "^2.8.12", "@types/node": ">=10.0.0", "accepts": "~1.3.4", "base64id": "2.0.0", "cookie": "~0.7.2", "cors": "~2.8.5", "debug": "~4.3.1", "engine.io-parser": "~5.2.1", "ws": "~8.17.1" } }, "sha512-ZCkIjSYNDyGn0R6ewHDtXgns/Zre/NT6Agvq1/WobF7JXgFff4SeDroKiCO3fNJreU9YG429Sc81o4w5ok/W5g=="], "engine.io-parser": ["engine.io-parser@5.2.3", "", {}, "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q=="], @@ -1283,6 +1223,8 @@ "external-editor": ["external-editor@3.1.0", "", { "dependencies": { "chardet": "^0.7.0", "iconv-lite": "^0.4.24", "tmp": "^0.0.33" } }, "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew=="], + "fast-copy": ["fast-copy@3.0.2", "", {}, "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="], + "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], @@ -1295,6 +1237,8 @@ "fast-redact": ["fast-redact@3.5.0", "", {}, "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A=="], + "fast-safe-stringify": ["fast-safe-stringify@2.1.1", "", {}, "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="], + "fast-uri": ["fast-uri@2.4.0", "", {}, "sha512-ypuAmmMKInk5q7XcepxlnUWDLWv4GFtaJqAzWKqn62IpQ3pejtr5dTVbt3vwqVaMKmkNR55sTT+CqUKIaT21BA=="], "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], @@ -1305,14 +1249,10 @@ "feature-store": ["feature-store@workspace:apps/data-services/feature-store"], - "fecha": ["fecha@4.2.3", "", {}, "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw=="], - "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], "file-entry-cache": ["file-entry-cache@6.0.1", "", { "dependencies": { "flat-cache": "^3.0.4" } }, "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg=="], - "file-stream-rotator": ["file-stream-rotator@0.6.1", "", { "dependencies": { "moment": "^2.29.1" } }, "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ=="], - "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], "finalhandler": ["finalhandler@1.1.2", "", { "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "on-finished": "~2.3.0", "parseurl": "~1.3.3", "statuses": "~1.5.0", "unpipe": "~1.0.0" } }, "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA=="], @@ -1323,8 +1263,6 @@ "flatted": ["flatted@3.3.3", "", {}, "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg=="], - "fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="], - "follow-redirects": ["follow-redirects@1.15.9", "", {}, "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ=="], "foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="], @@ -1389,6 +1327,8 @@ "helmet": ["helmet@7.2.0", "", {}, "sha512-ZRiwvN089JfMXokizgqEPXsl2Guk094yExfoDXR0cBYWxtBbaSww/w+vT4WEJsBW2iTUi1GgZ6swmoug3Oy4Xw=="], + "help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="], + "hono": ["hono@4.7.11", "", {}, "sha512-rv0JMwC0KALbbmwJDEnxvQCeJh+xbS3KEWW5PC9cMJ08Ur9xgatI0HmtgYZfOdOSOeYsp5LO2cOhdI8cLEbDEQ=="], "hosted-git-info": ["hosted-git-info@8.1.0", "", { "dependencies": { "lru-cache": "^10.0.1" } }, "sha512-Rw/B2DNQaPBICNXEm8balFz9a6WpZrkCGpcWFpy7nCj+NyhSdqXipmfvtmWt9xGfp0wZnBxB+iVpLmQMYt47Tw=="], @@ -1537,6 +1477,8 @@ "joi": ["joi@17.13.3", "", { "dependencies": { "@hapi/hoek": "^9.3.0", "@hapi/topo": "^5.1.0", "@sideway/address": "^4.1.5", "@sideway/formula": "^3.0.1", "@sideway/pinpoint": "^2.0.0" } }, "sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA=="], + "joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="], + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], "js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="], @@ -1577,8 +1519,6 @@ "kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="], - "kuler": ["kuler@2.0.0", "", {}, "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A=="], - "leven": ["leven@3.1.0", "", {}, "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A=="], "levn": ["levn@0.4.1", "", { "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" } }, "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ=="], @@ -1627,10 +1567,6 @@ "log4js": ["log4js@6.9.1", "", { "dependencies": { "date-format": "^4.0.14", "debug": "^4.3.4", "flatted": "^3.2.7", "rfdc": "^1.3.0", "streamroller": "^3.1.5" } }, "sha512-1somDdy9sChrr9/f4UlzhdaGfDR2c/SaD2a4T7qEkG4jTS57/B3qmnjLYePwQ8cqWnUHZI0iAKxMBpCZICiZ2g=="], - "logform": ["logform@2.7.0", "", { "dependencies": { "@colors/colors": "1.6.0", "@types/triple-beam": "^1.3.2", "fecha": "^4.2.0", "ms": "^2.1.1", "safe-stable-stringify": "^2.3.1", "triple-beam": "^1.3.0" } }, "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ=="], - - "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], - "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], "luxon": ["luxon@3.5.0", "", {}, "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="], @@ -1683,8 +1619,6 @@ "mkdirp": ["mkdirp@0.5.6", "", { "dependencies": { "minimist": "^1.2.6" }, "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw=="], - "moment": ["moment@2.30.1", "", {}, "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how=="], - "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -1749,8 +1683,6 @@ "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], - "object-hash": ["object-hash@2.2.0", "", {}, "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw=="], - "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], "on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="], @@ -1761,8 +1693,6 @@ "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], - "one-time": ["one-time@1.0.0", "", { "dependencies": { "fn.name": "1.x.x" } }, "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g=="], - "onetime": ["onetime@5.1.2", "", { "dependencies": { "mimic-fn": "^2.1.0" } }, "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg=="], "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], @@ -1813,11 +1743,15 @@ "picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="], - "pino": ["pino@8.21.0", "", { "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^1.2.0", "pino-std-serializers": "^6.0.0", "process-warning": "^3.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^3.7.0", "thread-stream": "^2.6.0" }, "bin": { "pino": "bin.js" } }, "sha512-ip4qdzjkAyDDZklUaZkcRFb2iA118H9SgRh8yzTkSQK8HilsOJF7rSY8HoW5+I0M46AZgX/pxbprf2vvzQCE0Q=="], + "pino": ["pino@9.7.0", "", { "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-vnMCM6xZTb1WDmLvtG2lE/2p+t9hDEIvTWJsu6FejkE62vB7gDhvzrpFR4Cw2to+9JNQxVnkAKVPA1KPB98vWg=="], - "pino-abstract-transport": ["pino-abstract-transport@1.2.0", "", { "dependencies": { "readable-stream": "^4.0.0", "split2": "^4.0.0" } }, "sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q=="], + "pino-abstract-transport": ["pino-abstract-transport@2.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw=="], - "pino-std-serializers": ["pino-std-serializers@6.2.2", "", {}, "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="], + "pino-loki": ["pino-loki@2.6.0", "", { "dependencies": { "pino-abstract-transport": "^2.0.0", "pump": "^3.0.2" }, "bin": { "pino-loki": "dist/cli.js" } }, "sha512-Qy+NeIdb0YmZe/M5mgnO5aGaAyVaeqgwn45T6VajhRXZlZVfGe1YNYhFa9UZyCeNFAPGaUkD2e9yPGjx+2BBYA=="], + + "pino-pretty": ["pino-pretty@13.0.0", "", { "dependencies": { "colorette": "^2.0.7", "dateformat": "^4.6.3", "fast-copy": "^3.0.2", "fast-safe-stringify": "^2.1.1", "help-me": "^5.0.0", "joycon": "^3.1.1", "minimist": "^1.2.6", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pump": "^3.0.0", "secure-json-parse": "^2.4.0", "sonic-boom": "^4.0.1", "strip-json-comments": "^3.1.1" }, "bin": { "pino-pretty": "bin.js" } }, "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA=="], + + "pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="], "pirates": ["pirates@4.0.7", "", {}, "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA=="], @@ -1839,16 +1773,16 @@ "process": ["process@0.11.10", "", {}, "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A=="], - "process-warning": ["process-warning@3.0.0", "", {}, "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="], + "process-warning": ["process-warning@5.0.0", "", {}, "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA=="], "promise-retry": ["promise-retry@2.0.1", "", { "dependencies": { "err-code": "^2.0.2", "retry": "^0.12.0" } }, "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g=="], "prompts": ["prompts@2.4.2", "", { "dependencies": { "kleur": "^3.0.3", "sisteransi": "^1.0.5" } }, "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q=="], - "protobufjs": ["protobufjs@7.5.3", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw=="], - "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + "pump": ["pump@3.0.2", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw=="], + "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], @@ -1869,7 +1803,7 @@ "react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], - "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], "readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], @@ -1923,6 +1857,8 @@ "sass": ["sass@1.88.0", "", { "dependencies": { "chokidar": "^4.0.0", "immutable": "^5.0.2", "source-map-js": ">=0.6.2 <2.0.0" }, "optionalDependencies": { "@parcel/watcher": "^2.4.1" }, "bin": { "sass": "sass.js" } }, "sha512-sF6TWQqjFvr4JILXzG4ucGOLELkESHL+I5QJhh7CNaE+Yge0SI+ehCatsXhJ7ymU1hAFcIS3/PBpjdIbXoyVbg=="], + "secure-json-parse": ["secure-json-parse@2.7.0", "", {}, "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw=="], + "semver": ["semver@7.7.2", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA=="], "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], @@ -1945,8 +1881,6 @@ "sigstore": ["sigstore@3.1.0", "", { "dependencies": { "@sigstore/bundle": "^3.1.0", "@sigstore/core": "^2.0.0", "@sigstore/protobuf-specs": "^0.4.0", "@sigstore/sign": "^3.1.0", "@sigstore/tuf": "^3.1.0", "@sigstore/verify": "^2.1.0" } }, "sha512-ZpzWAFHIFqyFE56dXqgX/DkDRZdz+rRcjoIk/RQU4IX0wiCv1l8S7ZrXDHcCc+uaf+6o7w3h2l3g6GYG5TKN9Q=="], - "simple-swizzle": ["simple-swizzle@0.2.2", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg=="], - "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], "slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], @@ -1955,8 +1889,6 @@ "smart-buffer": ["smart-buffer@4.2.0", "", {}, "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="], - "snappy": ["snappy@7.2.2", "", { "optionalDependencies": { "@napi-rs/snappy-android-arm-eabi": "7.2.2", "@napi-rs/snappy-android-arm64": "7.2.2", "@napi-rs/snappy-darwin-arm64": "7.2.2", "@napi-rs/snappy-darwin-x64": "7.2.2", "@napi-rs/snappy-freebsd-x64": "7.2.2", "@napi-rs/snappy-linux-arm-gnueabihf": "7.2.2", "@napi-rs/snappy-linux-arm64-gnu": "7.2.2", "@napi-rs/snappy-linux-arm64-musl": "7.2.2", "@napi-rs/snappy-linux-x64-gnu": "7.2.2", "@napi-rs/snappy-linux-x64-musl": "7.2.2", "@napi-rs/snappy-win32-arm64-msvc": "7.2.2", "@napi-rs/snappy-win32-ia32-msvc": "7.2.2", "@napi-rs/snappy-win32-x64-msvc": "7.2.2" } }, "sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA=="], - "socket.io": ["socket.io@4.8.1", "", { "dependencies": { "accepts": "~1.3.4", "base64id": "~2.0.0", "cors": "~2.8.5", "debug": "~4.3.2", "engine.io": "~6.6.0", "socket.io-adapter": "~2.5.2", "socket.io-parser": "~4.2.4" } }, "sha512-oZ7iUCxph8WYRHHcjBEc9unw3adt5CmSNlppj/5Q4k2RIrhl8Z5yY2Xr4j9zj0+wzVZ0bxmYoGSzKJnRl6A4yg=="], "socket.io-adapter": ["socket.io-adapter@2.5.5", "", { "dependencies": { "debug": "~4.3.4", "ws": "~8.17.1" } }, "sha512-eLDQas5dzPgOWCk9GuuJC2lBqItuhKI4uxGgo9aIV7MYbk2h9Q6uULEh8WBzThoI7l+qU9Ast9fVUmkqPP9wYg=="], @@ -1967,7 +1899,7 @@ "socks-proxy-agent": ["socks-proxy-agent@8.0.5", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "^4.3.4", "socks": "^2.8.3" } }, "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw=="], - "sonic-boom": ["sonic-boom@3.8.1", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg=="], + "sonic-boom": ["sonic-boom@4.2.0", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww=="], "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], @@ -1989,8 +1921,6 @@ "ssri": ["ssri@12.0.0", "", { "dependencies": { "minipass": "^7.0.3" } }, "sha512-S7iGNosepx9RadX82oimUkvr0Ct7IjJbEbs4mJcTxst8um95J3sDYU1RBEOvdu6oL1Wek2ODI5i4MAw+dZ6cAQ=="], - "stack-trace": ["stack-trace@0.0.10", "", {}, "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg=="], - "stack-utils": ["stack-utils@2.0.6", "", { "dependencies": { "escape-string-regexp": "^2.0.0" } }, "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ=="], "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], @@ -2033,11 +1963,9 @@ "test-exclude": ["test-exclude@6.0.0", "", { "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", "minimatch": "^3.0.4" } }, "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w=="], - "text-hex": ["text-hex@1.0.0", "", {}, "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="], - "text-table": ["text-table@0.2.0", "", {}, "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw=="], - "thread-stream": ["thread-stream@2.7.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw=="], + "thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="], "tinyglobby": ["tinyglobby@0.2.13", "", { "dependencies": { "fdir": "^6.4.4", "picomatch": "^4.0.2" } }, "sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw=="], @@ -2051,8 +1979,6 @@ "trading-dashboard": ["trading-dashboard@workspace:apps/interface-services/trading-dashboard"], - "triple-beam": ["triple-beam@1.4.1", "", {}, "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg=="], - "ts-api-utils": ["ts-api-utils@1.4.3", "", { "peerDependencies": { "typescript": ">=4.2.0" } }, "sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw=="], "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], @@ -2099,10 +2025,6 @@ "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], - "url-polyfill": ["url-polyfill@1.1.13", "", {}, "sha512-tXzkojrv2SujumYthZ/WjF7jaSfNhSXlYMpE5AYdL2I3D7DCeo+mch8KtW2rUuKjDg+3VXODXHVgipt8yGY/eQ=="], - - "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], - "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], "uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], @@ -2131,14 +2053,6 @@ "which": ["which@1.3.1", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "which": "./bin/which" } }, "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ=="], - "winston": ["winston@3.17.0", "", { "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.2", "async": "^3.2.3", "is-stream": "^2.0.0", "logform": "^2.7.0", "one-time": "^1.0.0", "readable-stream": "^3.4.0", "safe-stable-stringify": "^2.3.1", "stack-trace": "0.0.x", "triple-beam": "^1.3.0", "winston-transport": "^4.9.0" } }, "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw=="], - - "winston-daily-rotate-file": ["winston-daily-rotate-file@4.7.1", "", { "dependencies": { "file-stream-rotator": "^0.6.1", "object-hash": "^2.0.1", "triple-beam": "^1.3.0", "winston-transport": "^4.4.0" }, "peerDependencies": { "winston": "^3" } }, "sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA=="], - - "winston-loki": ["winston-loki@6.1.3", "", { "dependencies": { "async-exit-hook": "2.0.1", "btoa": "^1.2.1", "protobufjs": "^7.2.4", "url-polyfill": "^1.1.12", "winston-transport": "^4.3.0" }, "optionalDependencies": { "snappy": "^7.2.2" } }, "sha512-DjWtJ230xHyYQWr9mZJa93yhwHttn3JEtSYWP8vXZWJOahiQheUhf+88dSIidbGXB3u0oLweV6G1vkL/ouT62Q=="], - - "winston-transport": ["winston-transport@4.9.0", "", { "dependencies": { "logform": "^2.7.0", "readable-stream": "^3.6.2", "triple-beam": "^1.3.0" } }, "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A=="], - "word-wrap": ["word-wrap@1.2.5", "", {}, "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA=="], "wrap-ansi": ["wrap-ansi@9.0.0", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q=="], @@ -2213,6 +2127,8 @@ "@parcel/watcher/node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], + "@stock-bot/market-data-gateway/pino": ["pino@8.21.0", "", { "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^1.2.0", "pino-std-serializers": "^6.0.0", "process-warning": "^3.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^3.7.0", "thread-stream": "^2.6.0" }, "bin": { "pino": "bin.js" } }, "sha512-ip4qdzjkAyDDZklUaZkcRFb2iA118H9SgRh8yzTkSQK8HilsOJF7rSY8HoW5+I0M46AZgX/pxbprf2vvzQCE0Q=="], + "@tailwindcss/oxide/tar": ["tar@7.4.3", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" } }, "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.4.3", "", { "dependencies": { "@emnapi/wasi-threads": "1.0.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-4m62DuCE07lw01soJwPiBGC0nAww0Q+RY70VZ+n49yDIO13yyinhbWCeNnaob0lakDtWQzSdtNWzJeOJt2ma+g=="], @@ -2257,8 +2173,6 @@ "cliui/wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], - "color/color-convert": ["color-convert@1.9.3", "", { "dependencies": { "color-name": "1.1.3" } }, "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg=="], - "compression/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], "connect/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], @@ -2311,8 +2225,6 @@ "jest-worker/supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], - "karma/@colors/colors": ["@colors/colors@1.5.0", "", {}, "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="], - "karma/chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], "karma/yargs": ["yargs@16.2.0", "", { "dependencies": { "cliui": "^7.0.2", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.0", "y18n": "^5.0.5", "yargs-parser": "^20.2.2" } }, "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw=="], @@ -2359,14 +2271,10 @@ "path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], - "pino-abstract-transport/readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], - "pkg-dir/find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="], "restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], - "simple-swizzle/is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="], - "slice-ansi/ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="], "slice-ansi/is-fullwidth-code-point": ["is-fullwidth-code-point@4.0.0", "", {}, "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ=="], @@ -2427,6 +2335,16 @@ "@npmcli/run-script/which/isexe": ["isexe@3.1.1", "", {}, "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ=="], + "@stock-bot/market-data-gateway/pino/pino-abstract-transport": ["pino-abstract-transport@1.2.0", "", { "dependencies": { "readable-stream": "^4.0.0", "split2": "^4.0.0" } }, "sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q=="], + + "@stock-bot/market-data-gateway/pino/pino-std-serializers": ["pino-std-serializers@6.2.2", "", {}, "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="], + + "@stock-bot/market-data-gateway/pino/process-warning": ["process-warning@3.0.0", "", {}, "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="], + + "@stock-bot/market-data-gateway/pino/sonic-boom": ["sonic-boom@3.8.1", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg=="], + + "@stock-bot/market-data-gateway/pino/thread-stream": ["thread-stream@2.7.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw=="], + "@tailwindcss/oxide/tar/chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], "@tailwindcss/oxide/tar/mkdirp": ["mkdirp@3.0.1", "", { "bin": { "mkdirp": "dist/cjs/src/bin.js" } }, "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="], @@ -2459,8 +2377,6 @@ "cliui/wrap-ansi/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], - "color/color-convert/color-name": ["color-name@1.1.3", "", {}, "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="], - "compression/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], "connect/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], diff --git a/docs/PROGRESS_TRACKER.md b/docs/PROGRESS_TRACKER.md deleted file mode 100644 index 5bc8048..0000000 --- a/docs/PROGRESS_TRACKER.md +++ /dev/null @@ -1,248 +0,0 @@ -# Stock Bot Platform - Service Progress Tracker - -*Last Updated: June 3, 2025* - -## Overall Platform Progress: 68% - ---- - -## ๐Ÿ—„๏ธ **DATABASE SERVICES** - -### PostgreSQL (Primary Database) - 85% -- โœ… Configuration module complete -- โœ… Environment variables standardized (POSTGRES_*) -- โœ… Connection pooling configured -- โœ… SSL/TLS support -- โš ๏ธ Migration system needs setup -- โŒ Backup/restore automation pending -- โŒ Performance monitoring integration - -### QuestDB (Time-Series) - 75% -- โœ… Configuration module complete -- โœ… HTTP and PostgreSQL wire protocol ports -- โœ… InfluxDB line protocol support -- โœ… Docker integration -- โš ๏ธ Schema design for OHLCV data pending -- โŒ Data retention policies not configured -- โŒ Monitoring dashboards missing - -### MongoDB (Document Store) - 70% -- โœ… Configuration module complete -- โœ… Connection with authentication -- โœ… Database and collection setup -- โš ๏ธ Indexes for performance optimization needed -- โŒ Aggregation pipelines for analytics -- โŒ Full-text search configuration - -### Dragonfly (Cache/Redis) - 90% -- โœ… Configuration module complete -- โœ… Connection pooling -- โœ… TLS support -- โœ… Cluster mode support -- โœ… Memory management -- โš ๏ธ Cache strategies need implementation -- โŒ Pub/sub for real-time events - ---- - -## ๐Ÿ“Š **MONITORING & OBSERVABILITY** - -### Prometheus (Metrics) - 60% -- โœ… Configuration module complete -- โœ… Docker service setup -- โš ๏ธ Custom metrics collection pending -- โŒ Alerting rules not configured -- โŒ Service discovery setup -- โŒ Retention policies - -### Grafana (Dashboards) - 55% -- โœ… Configuration module complete -- โœ… Docker service setup -- โœ… Prometheus data source -- โš ๏ธ Trading-specific dashboards needed -- โŒ Alert notifications -- โŒ User management - -### Loki (Logs) - 40% -- โœ… Configuration module complete -- โš ๏ธ Log aggregation setup pending -- โŒ Log parsing rules -- โŒ Integration with application logs -- โŒ Log retention policies - ---- - -## ๐Ÿ”ง **CONFIGURATION MANAGEMENT** - -### Config Library (@stock-bot/config) - 95% -- โœ… Migrated from Zod to Envalid -- โœ… All service configurations complete -- โœ… Environment variable validation -- โœ… TypeScript type safety -- โœ… Example documentation -- โš ๏ธ Runtime configuration reloading - -### Environment Management - 80% -- โœ… Development environment (.env) -- โœ… Docker environment (.env.docker) -- โœ… Production templates -- โš ๏ธ Secrets management (HashiCorp Vault?) -- โŒ Environment-specific overrides - ---- - -## ๐Ÿ“ˆ **TRADING SERVICES** - -### Risk Management - 30% -- โœ… Risk configuration module -- โœ… Position sizing parameters -- โœ… Stop-loss/take-profit settings -- โŒ Real-time risk calculation engine -- โŒ Portfolio exposure monitoring -- โŒ Circuit breaker implementation - -### Data Providers - 45% -- โœ… Configuration for multiple providers -- โœ… Alpaca integration setup -- โœ… Polygon.io configuration -- โš ๏ธ Rate limiting implementation -- โŒ Data normalization layer -- โŒ Failover mechanisms - -### Order Management - 0% -- โŒ Order placement system -- โŒ Order status tracking -- โŒ Fill reporting -- โŒ Position management -- โŒ Trade execution logic - -### Strategy Engine - 0% -- โŒ Strategy framework -- โŒ Backtesting engine -- โŒ Live trading execution -- โŒ Performance analytics -- โŒ Strategy configuration - ---- - -## ๐Ÿ—๏ธ **INFRASTRUCTURE** - -### Docker Services - 85% -- โœ… All database containers configured -- โœ… Monitoring stack setup -- โœ… Network configuration -- โœ… Volume management -- โš ๏ธ Health checks need refinement -- โŒ Production orchestration (K8s?) - -### Build System - 70% -- โœ… Nx monorepo setup -- โœ… TypeScript configuration -- โœ… Library build processes -- โš ๏ธ Testing framework setup -- โŒ CI/CD pipeline -- โŒ Deployment automation - ---- - -## ๐Ÿงช **TESTING & QUALITY** - -### Unit Testing - 10% -- โš ๏ธ Test framework selection needed -- โŒ Config library tests -- โŒ Service layer tests -- โŒ Integration tests -- โŒ End-to-end tests - -### Code Quality - 60% -- โœ… TypeScript strict mode -- โœ… ESLint configuration -- โœ… Prettier formatting -- โŒ Code coverage reporting -- โŒ Security scanning - ---- - -## ๐Ÿ” **SECURITY** - -### Authentication & Authorization - 0% -- โŒ User authentication system -- โŒ API key management -- โŒ Role-based access control -- โŒ Session management -- โŒ OAuth integration - -### Data Security - 20% -- โœ… Database connection encryption -- โœ… Environment variable protection -- โŒ Data encryption at rest -- โŒ API rate limiting -- โŒ Audit logging - ---- - -## ๐Ÿ“‹ **IMMEDIATE NEXT STEPS** - -### High Priority (Next 2 weeks) -1. **Complete PostgreSQL setup** - Migrations, schemas -2. **Implement basic logging integration** - Connect services to Loki -3. **Create Grafana dashboards** - System and business metrics -4. **Setup testing framework** - Jest/Vitest configuration -5. **Risk management engine** - Core calculation logic - -### Medium Priority (Next month) -1. **Data provider integration** - Real market data ingestion -2. **QuestDB schema design** - Time-series data structure -3. **MongoDB indexing** - Performance optimization -4. **CI/CD pipeline** - Automated testing and deployment -5. **Basic order management** - Place and track orders - -### Low Priority (Next quarter) -1. **Strategy engine framework** - Backtesting and live trading -2. **Security implementation** - Authentication and authorization -3. **Production deployment** - Kubernetes or cloud setup -4. **Advanced monitoring** - Custom metrics and alerting -5. **Performance optimization** - System tuning and scaling - ---- - -## ๐Ÿ“Š **SERVICE COMPLETION SUMMARY** - -| Service Category | Progress | Status | -|------------------|----------|---------| -| Configuration | 95% | โœ… Nearly Complete | -| Databases | 77% | ๐ŸŸก Good Progress | -| Monitoring | 52% | ๐ŸŸก Moderate Progress | -| Infrastructure | 78% | ๐ŸŸก Good Progress | -| Trading Services | 19% | ๐Ÿ”ด Early Stage | -| Testing | 35% | ๐Ÿ”ด Needs Attention | -| Security | 10% | ๐Ÿ”ด Critical Gap | - -**Legend:** -- โœ… Complete (90-100%) -- ๐ŸŸก In Progress (50-89%) -- ๐Ÿ”ด Early Stage (0-49%) -- โš ๏ธ Partially Complete -- โŒ Not Started - ---- - -## ๐ŸŽฏ **SUCCESS METRICS** - -### Technical Metrics -- [ ] All services start without errors -- [ ] Database connections stable -- [ ] Monitoring dashboards operational -- [ ] Tests achieve >90% coverage -- [ ] Build time < 2 minutes - -### Business Metrics -- [ ] Can place live trades -- [ ] Risk management active -- [ ] Real-time data ingestion -- [ ] Performance tracking -- [ ] Error rate < 0.1% - ---- - -*This document is automatically updated as services reach completion milestones.* \ No newline at end of file diff --git a/docs/core-services/.gitkeep b/docs/core-services/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/core-services/README.md b/docs/core-services/README.md deleted file mode 100644 index 21d1c16..0000000 --- a/docs/core-services/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Core Services - -Core services provide fundamental infrastructure and foundational capabilities for the stock trading platform. - -## Services - -### Market Data Gateway -- **Purpose**: Real-time market data processing and distribution -- **Key Functions**: - - WebSocket streaming for live market data - - Multi-source data aggregation (Alpaca, Yahoo Finance, etc.) - - Data caching and normalization - - Rate limiting and connection management - - Error handling and reconnection logic - -### Risk Guardian -- **Purpose**: Real-time risk monitoring and controls -- **Key Functions**: - - Position monitoring and risk threshold enforcement - - Portfolio risk assessment and alerting - - Real-time risk metric calculations - - Automated risk controls and circuit breakers - - Risk reporting and compliance monitoring - -## Architecture - -Core services form the backbone of the trading platform, providing essential data flow and risk management capabilities that all other services depend upon. They handle the most critical and time-sensitive operations requiring high reliability and performance. diff --git a/docs/core-services/market-data-gateway/.gitkeep b/docs/core-services/market-data-gateway/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/core-services/market-data-gateway/README.md b/docs/core-services/market-data-gateway/README.md deleted file mode 100644 index 83a54e1..0000000 --- a/docs/core-services/market-data-gateway/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Market Data Gateway - -## Overview -The Market Data Gateway (MDG) service serves as the central hub for real-time market data processing and distribution within the stock-bot platform. It acts as the intermediary between external market data providers and internal platform services, ensuring consistent, normalized, and reliable market data delivery. - -## Key Features - -### Real-time Data Processing -- **WebSocket Streaming**: Provides low-latency data streams for market updates -- **Multi-source Aggregation**: Integrates data from multiple providers (Alpaca, Yahoo Finance, etc.) -- **Normalized Data Model**: Transforms varied provider formats into a unified platform data model -- **Subscription Management**: Allows services to subscribe to specific data streams - -### Data Quality Management -- **Validation & Sanitization**: Ensures data integrity through validation rules -- **Anomaly Detection**: Identifies unusual price movements or data issues -- **Gap Filling**: Interpolation strategies for missing data points -- **Data Reconciliation**: Cross-validates data from multiple sources - -### Performance Optimization -- **Caching Layer**: In-memory cache for frequently accessed data -- **Rate Limiting**: Protects against API quota exhaustion -- **Connection Pooling**: Efficiently manages provider connections -- **Compression**: Minimizes data transfer size for bandwidth efficiency - -### Operational Resilience -- **Automatic Reconnection**: Handles provider disconnections gracefully -- **Circuit Breaking**: Prevents cascade failures during outages -- **Failover Mechanisms**: Switches to alternative data sources when primary sources fail -- **Health Monitoring**: Self-reports service health metrics - -## Integration Points - -### Upstream Connections -- Alpaca Markets API (primary data source) -- Yahoo Finance API (secondary data source) -- Potential future integrations with IEX, Polygon, etc. - -### Downstream Consumers -- Strategy Orchestrator -- Risk Guardian -- Trading Dashboard -- Data Persistence Layer - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Messaging**: WebSockets for real-time streaming -- **Caching**: Redis for shared cache -- **Metrics**: Prometheus metrics for monitoring -- **Configuration**: Environment-based with runtime updates - -### Architecture Pattern -- Event-driven microservice with publisher-subscriber model -- Horizontally scalable to handle increased data volumes -- Stateless design with external state management - -## Development Guidelines - -### Error Handling -- Detailed error classification and handling strategy -- Graceful degradation during partial outages -- Comprehensive error logging with context - -### Testing Strategy -- Unit tests for data transformation logic -- Integration tests with mock data providers -- Performance tests for throughput capacity -- Chaos testing for resilience verification - -### Observability -- Detailed logs for troubleshooting -- Performance metrics for optimization -- Health checks for system monitoring -- Tracing for request flow analysis - -## Future Enhancements -- Support for options and derivatives data -- Real-time news and sentiment integration -- Machine learning-based data quality improvements -- Enhanced historical data query capabilities \ No newline at end of file diff --git a/docs/core-services/risk-guardian/.gitkeep b/docs/core-services/risk-guardian/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/core-services/risk-guardian/README.md b/docs/core-services/risk-guardian/README.md deleted file mode 100644 index e115286..0000000 --- a/docs/core-services/risk-guardian/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Risk Guardian - -## Overview -The Risk Guardian service provides real-time risk monitoring and control mechanisms for the stock-bot platform. It serves as the protective layer that ensures trading activities remain within defined risk parameters, safeguarding the platform and its users from excessive market exposure and potential losses. - -## Key Features - -### Real-time Risk Monitoring -- **Position Tracking**: Continuously monitors all open positions -- **Risk Metric Calculation**: Calculates key risk metrics (VaR, volatility, exposure) -- **Threshold Management**: Configurable risk thresholds with multiple severity levels -- **Aggregated Risk Views**: Risk assessment at portfolio, strategy, and position levels - -### Automated Risk Controls -- **Pre-trade Validation**: Validates orders against risk limits before execution -- **Circuit Breakers**: Automatically halts trading when thresholds are breached -- **Position Liquidation**: Controlled unwinding of positions when necessary -- **Trading Restrictions**: Enforces instrument, size, and frequency restrictions - -### Risk Alerting -- **Real-time Notifications**: Immediate alerts for threshold breaches -- **Escalation Paths**: Multi-level alerting based on severity -- **Alert History**: Maintains historical record of all risk events -- **Custom Alert Rules**: Configurable alerting conditions and criteria - -### Compliance Management -- **Regulatory Reporting**: Assists with required regulatory reporting -- **Audit Trails**: Comprehensive logging of risk-related decisions -- **Rule-based Controls**: Implements compliance-driven trading restrictions -- **Documentation**: Maintains evidence of risk control effectiveness - -## Integration Points - -### Upstream Connections -- Market Data Gateway (for price data) -- Strategy Orchestrator (for active strategies) -- Order Management System (for position tracking) - -### Downstream Consumers -- Trading Dashboard (for risk visualization) -- Strategy Orchestrator (for trading restrictions) -- Notification Service (for alerting) - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Database**: Time-series database for risk metrics -- **Messaging**: Event-driven architecture with message bus -- **Math Libraries**: Specialized libraries for risk calculations -- **Caching**: In-memory risk state management - -### Architecture Pattern -- Reactive microservice with event sourcing -- Command Query Responsibility Segregation (CQRS) -- Rule engine for risk evaluation -- Stateful service with persistence - -## Development Guidelines - -### Risk Calculation Approach -- Clear documentation of all risk formulas -- Validation against industry standard calculations -- Performance optimization for real-time processing -- Regular backtesting of risk models - -### Testing Strategy -- Unit tests for risk calculation logic -- Scenario-based testing for specific market conditions -- Stress testing with extreme market movements -- Performance testing for high-frequency updates - -### Calibration Process -- Documented process for risk model calibration -- Historical data validation -- Parameter sensitivity analysis -- Regular recalibration schedule - -## Future Enhancements -- Machine learning for anomaly detection -- Scenario analysis and stress testing -- Custom risk models per strategy type -- Enhanced visualization of risk exposures -- Factor-based risk decomposition \ No newline at end of file diff --git a/docs/data-services/.gitkeep b/docs/data-services/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/data-services/README.md b/docs/data-services/README.md deleted file mode 100644 index 7cdcebe..0000000 --- a/docs/data-services/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Data Services - -Data services manage data storage, processing, and discovery across the trading platform, providing structured access to market data, features, and metadata. - -## Services - -### Data Catalog -- **Purpose**: Data asset management and discovery -- **Key Functions**: - - Data asset discovery and search capabilities - - Metadata management and governance - - Data lineage tracking - - Schema registry and versioning - - Data quality monitoring - -### Data Processor -- **Purpose**: Data transformation and processing pipelines -- **Key Functions**: - - ETL/ELT pipeline orchestration - - Data cleaning and normalization - - Batch and stream processing - - Data validation and quality checks - -### Feature Store -- **Purpose**: ML feature management and serving -- **Key Functions**: - - Online and offline feature storage - - Feature computation and serving - - Feature statistics and monitoring - - Feature lineage and versioning - - Real-time feature retrieval for ML models - -### Market Data Gateway -- **Purpose**: Market data storage and historical access -- **Key Functions**: - - Historical market data storage - - Data archival and retention policies - - Query optimization for time-series data - - Data backup and recovery - -## Architecture - -Data services create a unified data layer that enables efficient data discovery, processing, and consumption across the platform. They ensure data quality, consistency, and accessibility for both operational and analytical workloads. diff --git a/docs/data-services/data-catalog/.gitkeep b/docs/data-services/data-catalog/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/data-services/data-catalog/README.md b/docs/data-services/data-catalog/README.md deleted file mode 100644 index b2ecc7d..0000000 --- a/docs/data-services/data-catalog/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Data Catalog - -## Overview -The Data Catalog service provides a centralized system for data asset discovery, management, and governance within the stock-bot platform. It serves as the single source of truth for all data assets, their metadata, and relationships, enabling efficient data discovery and utilization across the platform. - -## Key Features - -### Data Asset Management -- **Asset Registration**: Automated and manual registration of data assets -- **Metadata Management**: Comprehensive metadata for all data assets -- **Versioning**: Tracks changes to data assets over time -- **Schema Registry**: Central repository of data schemas and formats - -### Data Discovery -- **Search Capabilities**: Advanced search across all data assets -- **Categorization**: Hierarchical categorization of data assets -- **Tagging**: Flexible tagging system for improved findability -- **Popularity Tracking**: Identifies most-used data assets - -### Data Governance -- **Access Control**: Fine-grained access control for data assets -- **Lineage Tracking**: Visualizes data origins and transformations -- **Quality Metrics**: Monitors and reports on data quality -- **Compliance Tracking**: Ensures regulatory compliance for sensitive data - -### Integration Framework -- **API-first Design**: Comprehensive API for programmatic access -- **Event Notifications**: Real-time notifications for data changes -- **Bulk Operations**: Efficient handling of batch operations -- **Extensibility**: Plugin architecture for custom extensions - -## Integration Points - -### Upstream Connections -- Data Processor (for processed data assets) -- Feature Store (for feature metadata) -- Market Data Gateway (for market data assets) - -### Downstream Consumers -- Strategy Development Environment -- Data Analysis Tools -- Machine Learning Pipeline -- Reporting Systems - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Database**: Document database for flexible metadata storage -- **Search**: Elasticsearch for advanced search capabilities -- **API**: GraphQL for flexible querying -- **UI**: React-based web interface - -### Architecture Pattern -- Domain-driven design for complex metadata management -- Microservice architecture for scalability -- Event sourcing for change tracking -- CQRS for optimized read/write operations - -## Development Guidelines - -### Metadata Standards -- Adherence to common metadata standards -- Required vs. optional metadata fields -- Validation rules for metadata quality -- Consistent naming conventions - -### Extension Development -- Plugin architecture documentation -- Custom metadata field guidelines -- Integration hook documentation -- Testing requirements for extensions - -### Performance Considerations -- Indexing strategies for efficient search -- Caching recommendations -- Bulk operation best practices -- Query optimization techniques - -## Future Enhancements -- Automated metadata extraction -- Machine learning for data classification -- Advanced lineage visualization -- Enhanced data quality scoring -- Collaborative annotations and discussions -- Integration with external data marketplaces diff --git a/docs/data-services/data-processor/.gitkeep b/docs/data-services/data-processor/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/data-services/data-processor/README.md b/docs/data-services/data-processor/README.md deleted file mode 100644 index f2af27b..0000000 --- a/docs/data-services/data-processor/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Data Processor - -## Overview -The Data Processor service provides robust data transformation, cleaning, and enrichment capabilities for the stock-bot platform. It serves as the ETL (Extract, Transform, Load) backbone, handling both batch and streaming data processing needs to prepare raw data for consumption by downstream services. - -## Key Features - -### Data Transformation -- **Format Conversion**: Transforms data between different formats (JSON, CSV, Parquet, etc.) -- **Schema Mapping**: Maps between different data schemas -- **Normalization**: Standardizes data values and formats -- **Aggregation**: Creates summary data at different time intervals - -### Data Quality Management -- **Validation Rules**: Enforces data quality rules and constraints -- **Cleansing**: Removes or corrects invalid data -- **Missing Data Handling**: Strategies for handling incomplete data -- **Anomaly Detection**: Identifies and flags unusual data patterns - -### Pipeline Orchestration -- **Workflow Definition**: Configurable data processing workflows -- **Scheduling**: Time-based and event-based pipeline execution -- **Dependency Management**: Handles dependencies between processing steps -- **Error Handling**: Graceful error recovery and retry mechanisms - -### Data Enrichment -- **Reference Data Integration**: Enhances data with reference sources -- **Feature Engineering**: Creates derived features for analysis -- **Cross-source Joins**: Combines data from multiple sources -- **Temporal Enrichment**: Adds time-based context and features - -## Integration Points - -### Upstream Connections -- Market Data Gateway (for raw market data) -- External Data Connectors (for alternative data) -- Data Lake/Storage (for historical data) - -### Downstream Consumers -- Feature Store (for processed features) -- Data Catalog (for processed datasets) -- Intelligence Services (for analysis input) -- Data Warehouse (for reporting data) - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Processing Frameworks**: Apache Spark for batch, Kafka Streams for streaming -- **Storage**: Object storage for intermediate data -- **Orchestration**: Airflow for pipeline management -- **Configuration**: YAML-based pipeline definitions - -### Architecture Pattern -- Data pipeline architecture -- Pluggable transformation components -- Separation of pipeline definition from execution -- Idempotent processing for reliability - -## Development Guidelines - -### Pipeline Development -- Modular transformation development -- Testing requirements for transformations -- Performance optimization techniques -- Documentation requirements - -### Data Quality Controls -- Quality rule definition standards -- Error handling and reporting -- Data quality metric collection -- Threshold-based alerting - -### Operational Considerations -- Monitoring requirements -- Resource utilization guidelines -- Scaling recommendations -- Failure recovery procedures - -## Future Enhancements -- Machine learning-based data cleaning -- Advanced schema evolution handling -- Visual pipeline builder -- Enhanced pipeline monitoring dashboard -- Automated data quality remediation -- Real-time processing optimizations diff --git a/docs/data-services/feature-store/.gitkeep b/docs/data-services/feature-store/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/data-services/feature-store/README.md b/docs/data-services/feature-store/README.md deleted file mode 100644 index a4c9af8..0000000 --- a/docs/data-services/feature-store/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Feature Store - -## Overview -The Feature Store service provides a centralized repository for managing, serving, and monitoring machine learning features within the stock-bot platform. It bridges the gap between data engineering and machine learning, ensuring consistent feature computation and reliable feature access for both training and inference. - -## Key Features - -### Feature Management -- **Feature Registry**: Central catalog of all ML features -- **Feature Definitions**: Standardized declarations of feature computation logic -- **Feature Versioning**: Tracks changes to feature definitions over time -- **Feature Groups**: Logical grouping of related features - -### Serving Capabilities -- **Online Serving**: Low-latency access for real-time predictions -- **Offline Serving**: Batch access for model training -- **Point-in-time Correctness**: Historical feature values for specific timestamps -- **Feature Vectors**: Grouped feature retrieval for models - -### Data Quality & Monitoring -- **Statistics Tracking**: Monitors feature distributions and statistics -- **Drift Detection**: Identifies shifts in feature patterns -- **Validation Rules**: Enforces constraints on feature values -- **Alerting**: Notifies of anomalies or quality issues - -### Operational Features -- **Caching**: Performance optimization for frequently-used features -- **Backfilling**: Recomputation of historical feature values -- **Feature Lineage**: Tracks data sources and transformations -- **Access Controls**: Security controls for feature access - -## Integration Points - -### Upstream Connections -- Data Processor (for feature computation) -- Market Data Gateway (for real-time input data) -- Data Catalog (for feature metadata) - -### Downstream Consumers -- Signal Engine (for feature consumption) -- Strategy Orchestrator (for real-time feature access) -- Backtest Engine (for historical feature access) -- Model Training Pipeline - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Online Storage**: Redis for low-latency access -- **Offline Storage**: Parquet files in object storage -- **Metadata Store**: Document database for feature registry -- **API**: RESTful and gRPC interfaces - -### Architecture Pattern -- Dual-storage architecture (online/offline) -- Event-driven feature computation -- Schema-on-read with strong validation -- Separation of storage from compute - -## Development Guidelines - -### Feature Definition -- Feature specification format -- Transformation function requirements -- Testing requirements for features -- Documentation standards - -### Performance Considerations -- Caching strategies -- Batch vs. streaming computation -- Storage optimization techniques -- Query patterns and optimization - -### Quality Controls -- Feature validation requirements -- Monitoring configuration -- Alerting thresholds -- Remediation procedures - -## Future Enhancements -- Feature discovery and recommendations -- Automated feature generation -- Enhanced visualization of feature relationships -- Feature importance tracking -- Integrated A/B testing for features -- On-demand feature computation diff --git a/docs/data-services/market-data-gateway/.gitkeep b/docs/data-services/market-data-gateway/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/data-services/market-data-gateway/README.md b/docs/data-services/market-data-gateway/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/execution-services/.gitkeep b/docs/execution-services/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/execution-services/README.md b/docs/execution-services/README.md deleted file mode 100644 index 7da7b77..0000000 --- a/docs/execution-services/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Execution Services - -Execution services handle trade execution, order management, and broker integrations for the trading platform. - -## Services - -*Currently in planning phase - no active services deployed* - -## Planned Capabilities - -### Order Management System (OMS) -- **Purpose**: Centralized order lifecycle management -- **Planned Functions**: - - Order routing and execution - - Order validation and risk checks - - Execution quality monitoring - - Fill reporting and trade confirmations - -### Broker Gateway -- **Purpose**: Multi-broker connectivity and abstraction -- **Planned Functions**: - - Broker API integration and management - - Order routing optimization - - Execution venue selection - - Trade settlement and clearing - -### Portfolio Manager -- **Purpose**: Position tracking and portfolio management -- **Planned Functions**: - - Real-time position tracking - - Portfolio rebalancing - - Corporate actions processing - - P&L calculation and reporting - -## Architecture - -Execution services will form the operational core of trade execution, ensuring reliable and efficient order processing while maintaining proper risk controls and compliance requirements. diff --git a/docs/execution-services/broker-gateway/README.md b/docs/execution-services/broker-gateway/README.md deleted file mode 100644 index b8727b5..0000000 --- a/docs/execution-services/broker-gateway/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Broker Gateway - -## Overview -The Broker Gateway service will provide a unified interface for connecting to multiple broker APIs and trading venues within the stock-bot platform. It will abstract the complexities of different broker systems, providing a standardized way to route orders, receive executions, and manage account information across multiple execution venues. - -## Planned Features - -### Broker Integration -- **Multi-broker Support**: Connectivity to multiple brokerage platforms -- **Unified API**: Standardized interface across different brokers -- **Credential Management**: Secure handling of broker authentication -- **Connection Management**: Monitoring and maintenance of broker connections -- **Error Handling**: Standardized error processing across providers - -### Order Routing -- **Smart Routing**: Intelligent selection of optimal execution venues -- **Route Optimization**: Cost and execution quality-based routing decisions -- **Failover Routing**: Automatic rerouting in case of broker issues -- **Split Orders**: Distribution of large orders across multiple venues -- **Order Translation**: Mapping platform orders to broker-specific formats - -### Account Management -- **Balance Tracking**: Real-time account balance monitoring -- **Position Reconciliation**: Verification of position data with brokers -- **Margin Calculation**: Standardized margin requirement calculation -- **Account Limits**: Enforcement of account-level trading restrictions -- **Multi-account Support**: Management of multiple trading accounts - -### Market Access -- **Market Data Proxying**: Standardized access to broker market data -- **Instrument Coverage**: Management of tradable instrument universe -- **Trading Hours**: Handling of exchange trading calendars -- **Fee Structure**: Tracking of broker-specific fee models -- **Corporate Actions**: Processing of splits, dividends, and other events - -## Planned Integration Points - -### Upstream Connections -- Order Management System (for order routing) -- Risk Guardian (for account risk monitoring) -- Authentication Service (for user permissions) - -### Downstream Connections -- External Broker APIs (e.g., Alpaca, Interactive Brokers) -- Market Data Providers -- Clearing Systems - -## Planned Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Database**: Fast key-value store for state management -- **Messaging**: Message bus for order events -- **Authentication**: Secure credential vault -- **Monitoring**: Real-time connection monitoring - -### Architecture Pattern -- Adapter pattern for broker integrations -- Circuit breaker for fault tolerance -- Rate limiting for API compliance -- Idempotent operations for reliability - -## Development Guidelines - -### Broker Integration -- API implementation requirements -- Authentication methods -- Error mapping standards -- Testing requirements - -### Performance Considerations -- Latency expectations -- Throughput requirements -- Resource utilization guidelines -- Connection pooling recommendations - -### Reliability Measures -- Retry strategies -- Circuit breaker configurations -- Monitoring requirements -- Failover procedures - -## Implementation Roadmap -1. Core integration with primary broker (Alpaca) -2. Order routing and execution tracking -3. Account management and position reconciliation -4. Additional broker integrations -5. Smart routing and optimization features diff --git a/docs/execution-services/order-management-system/README.md b/docs/execution-services/order-management-system/README.md deleted file mode 100644 index aa8bcbf..0000000 --- a/docs/execution-services/order-management-system/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# Order Management System - -## Overview -The Order Management System (OMS) will provide centralized order lifecycle management for the stock-bot platform. It will handle the entire order process from creation through routing, execution, and settlement, ensuring reliable and efficient trade processing while maintaining proper audit trails. - -## Planned Features - -### Order Lifecycle Management -- **Order Creation**: Clean API for creating various order types -- **Order Validation**: Pre-execution validation and risk checks -- **Order Routing**: Intelligent routing to appropriate brokers/venues -- **Execution Tracking**: Real-time tracking of order status -- **Fill Management**: Processing of full and partial fills -- **Cancellation & Modification**: Handling order changes and cancellations - -### Order Types & Algorithms -- **Market & Limit Orders**: Basic order type handling -- **Stop & Stop-Limit Orders**: Risk-controlling conditional orders -- **Time-in-Force Options**: Day, GTC, IOC, FOK implementations -- **Algorithmic Orders**: TWAP, VWAP, Iceberg, and custom algorithms -- **Bracket Orders**: OCO (One-Cancels-Other) and other complex orders - -### Execution Quality -- **Best Execution**: Strategies for achieving optimal execution prices -- **Transaction Cost Analysis**: Measurement and optimization of execution costs -- **Slippage Monitoring**: Tracking of execution vs. expected prices -- **Fill Reporting**: Comprehensive reporting on execution quality - -### Operational Features -- **Audit Trail**: Complete history of all order events -- **Reconciliation**: Matching of orders with executions -- **Exception Handling**: Management of rejected or failed orders -- **Compliance Rules**: Implementation of regulatory requirements - -## Planned Integration Points - -### Upstream Connections -- Strategy Orchestrator (order requests) -- Risk Guardian (risk validation) -- Authentication Services (permission validation) - -### Downstream Consumers -- Broker Gateway (order routing) -- Portfolio Manager (position impact) -- Trading Dashboard (order visualization) -- Data Warehouse (execution analytics) - -## Planned Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Database**: High-performance transactional database -- **Messaging**: Low-latency message bus for order events -- **API**: RESTful and WebSocket interfaces -- **Persistence**: Event sourcing for order history - -### Architecture Pattern -- Event-driven architecture for real-time processing -- CQRS for optimized read/write operations -- Microservice decomposition by functionality -- High availability and fault tolerance design - -## Development Guidelines - -### Order Management -- Order state machine definitions -- Order type specifications -- Validation rule implementation -- Exception handling procedures - -### Performance Requirements -- Order throughput expectations -- Latency budgets by component -- Scaling approaches -- Resource utilization guidelines - -### Testing Strategy -- Unit testing requirements -- Integration testing approach -- Performance testing methodology -- Compliance verification procedures - -## Implementation Roadmap -1. Core order types and lifecycle management -2. Basic routing and execution tracking -3. Advanced order types and algorithms -4. Execution quality analytics and optimization -5. Compliance and regulatory reporting features diff --git a/docs/execution-services/portfolio-manager/README.md b/docs/execution-services/portfolio-manager/README.md deleted file mode 100644 index d718fef..0000000 --- a/docs/execution-services/portfolio-manager/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Portfolio Manager - -## Overview -The Portfolio Manager service will provide comprehensive position tracking, portfolio analysis, and management capabilities for the stock-bot platform. It will maintain the current state of all trading portfolios, calculate performance metrics, and support portfolio-level decision making. - -## Planned Features - -### Position Management -- **Real-time Position Tracking**: Accurate tracking of all open positions -- **Position Reconciliation**: Validation against broker records -- **Average Price Calculation**: Tracking of position entry prices -- **Lot Management**: FIFO/LIFO/average cost basis tracking -- **Multi-currency Support**: Handling positions across different currencies - -### Portfolio Analytics -- **Performance Metrics**: Return calculation (absolute, relative, time-weighted) -- **Risk Metrics**: Volatility, beta, correlation, VaR calculations -- **Attribution Analysis**: Performance attribution by sector, strategy, asset class -- **Scenario Analysis**: What-if analysis for portfolio changes -- **Benchmark Comparison**: Performance vs. standard benchmarks - -### Corporate Action Processing -- **Dividend Processing**: Impact of cash and stock dividends -- **Split Adjustments**: Handling of stock splits and reverse splits -- **Merger & Acquisition Handling**: Position adjustments for M&A events -- **Rights & Warrants**: Processing of corporate rights events -- **Spin-offs**: Management of position changes from spin-off events - -### Portfolio Construction -- **Rebalancing Tools**: Portfolio rebalancing against targets -- **Optimization**: Portfolio optimization for various objectives -- **Constraint Management**: Enforcement of portfolio constraints -- **Tax-aware Trading**: Consideration of tax implications -- **Cash Management**: Handling of cash positions and forecasting - -## Planned Integration Points - -### Upstream Connections -- Order Management System (for executed trades) -- Broker Gateway (for position verification) -- Market Data Gateway (for pricing) -- Strategy Orchestrator (for allocation decisions) - -### Downstream Consumers -- Risk Guardian (for portfolio risk assessment) -- Trading Dashboard (for portfolio visualization) -- Reporting System (for performance reporting) -- Tax Reporting (for tax calculations) - -## Planned Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Database**: Fast database with transaction support -- **Calculation Engine**: Optimized financial calculation libraries -- **API**: RESTful interface with WebSocket updates -- **Caching**: In-memory position cache for performance - -### Architecture Pattern -- Event sourcing for position history -- CQRS for optimized read/write operations -- Eventual consistency for distributed state -- Snapshotting for performance optimization - -## Development Guidelines - -### Position Calculations -- Cost basis methodologies -- Corporate action handling -- FX conversion approaches -- Performance calculation standards - -### Data Consistency -- Reconciliation procedures -- Error detection and correction -- Data validation requirements -- Audit trail requirements - -### Performance Optimization -- Caching strategies -- Calculation optimization -- Query patterns -- Batch processing approaches - -## Implementation Roadmap -1. Basic position tracking and P&L calculation -2. Portfolio analytics and performance metrics -3. Corporate action processing -4. Advanced portfolio construction tools -5. Tax and regulatory reporting features diff --git a/docs/integration-services/.gitkeep b/docs/integration-services/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/integration-services/README.md b/docs/integration-services/README.md deleted file mode 100644 index 7eb2770..0000000 --- a/docs/integration-services/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Integration Services - -Integration services provide connectivity, messaging, and interoperability between internal services and external systems. - -## Services - -*Currently in planning phase - no active services deployed* - -## Planned Capabilities - -### Message Bus -- **Purpose**: Event-driven communication between services -- **Planned Functions**: - - Event publishing and subscription - - Message routing and transformation - - Dead letter queue handling - - Event sourcing and replay capabilities - -### API Gateway -- **Purpose**: Unified API management and routing -- **Planned Functions**: - - API endpoint consolidation - - Authentication and authorization - - Rate limiting and throttling - - Request/response transformation - -### External Data Connectors -- **Purpose**: Third-party data source integration -- **Planned Functions**: - - Alternative data provider connections - - News and sentiment data feeds - - Economic indicator integrations - - Social media sentiment tracking - -### Notification Service -- **Purpose**: Multi-channel alerting and notifications -- **Planned Functions**: - - Email, SMS, and push notifications - - Alert routing and escalation - - Notification templates and personalization - - Delivery tracking and analytics - -## Architecture - -Integration services will provide the connectivity fabric that enables seamless communication between all platform components and external systems, ensuring loose coupling and high scalability. diff --git a/docs/integration-services/api-gateway/README.md b/docs/integration-services/api-gateway/README.md deleted file mode 100644 index 587e567..0000000 --- a/docs/integration-services/api-gateway/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# API Gateway - -## Overview -The API Gateway service will provide a unified entry point for all external API requests to the stock-bot platform. It will handle request routing, composition, protocol translation, authentication, and other cross-cutting concerns, providing a simplified interface for clients while abstracting the internal microservice architecture. - -## Planned Features - -### Request Management -- **Routing**: Direct requests to appropriate backend services -- **Aggregation**: Combine results from multiple microservices -- **Transformation**: Convert between different data formats and protocols -- **Parameter Validation**: Validate request parameters before forwarding -- **Service Discovery**: Dynamically locate service instances - -### Security Features -- **Authentication**: Centralized authentication for all API requests -- **Authorization**: Role-based access control for API endpoints -- **API Keys**: Management of client API keys and quotas -- **JWT Validation**: Token-based authentication handling -- **OAuth Integration**: Support for OAuth 2.0 flows - -### Traffic Management -- **Rate Limiting**: Protect services from excessive requests -- **Throttling**: Client-specific request throttling -- **Circuit Breaking**: Prevent cascading failures -- **Load Balancing**: Distribute requests among service instances -- **Retries**: Automatic retry of failed requests - -### Operational Features -- **Request Logging**: Comprehensive logging of API activity -- **Metrics Collection**: Performance and usage metrics -- **Caching**: Response caching for improved performance -- **Documentation**: Auto-generated API documentation -- **Versioning**: Support for multiple API versions - -## Planned Integration Points - -### Frontend Connections -- Trading Dashboard (web client) -- Mobile applications -- Third-party integrations -- Partner systems - -### Backend Services -- All platform microservices -- Authentication services -- Monitoring and logging systems - -## Planned Technical Implementation - -### Technology Stack -- **API Gateway**: Kong, AWS API Gateway, or custom solution -- **Runtime**: Node.js with TypeScript -- **Documentation**: OpenAPI/Swagger -- **Cache**: Redis for response caching -- **Storage**: Database for API configurations - -### Architecture Pattern -- Backend for Frontend (BFF) pattern -- API Gateway pattern -- Circuit breaker pattern -- Bulkhead pattern for isolation - -## Development Guidelines - -### API Design -- RESTful API design standards -- Error response format -- Versioning strategy -- Resource naming conventions - -### Security Implementation -- Authentication requirements -- Authorization approach -- API key management -- Rate limit configuration - -### Performance Optimization -- Caching strategies -- Request batching techniques -- Response compression -- Timeout configurations - -## Implementation Roadmap -1. Core routing and basic security features -2. Traffic management and monitoring -3. Request aggregation and transformation -4. Advanced security features -5. Developer portal and documentation diff --git a/docs/integration-services/message-bus/README.md b/docs/integration-services/message-bus/README.md deleted file mode 100644 index cc0cc63..0000000 --- a/docs/integration-services/message-bus/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Message Bus - -## Overview -The Message Bus service will provide the central event-driven communication infrastructure for the stock-bot platform. It will enable reliable, scalable, and decoupled interaction between services through asynchronous messaging, event streaming, and publish-subscribe patterns. - -## Planned Features - -### Messaging Infrastructure -- **Topic-based Messaging**: Publish-subscribe communication model -- **Message Queuing**: Reliable message delivery with persistence -- **Event Streaming**: Real-time event processing with replay capabilities -- **Message Routing**: Dynamic routing based on content and metadata -- **Quality of Service**: Various delivery guarantee levels (at-least-once, exactly-once) - -### Message Processing -- **Message Transformation**: Content transformation and enrichment -- **Message Filtering**: Rules-based filtering of messages -- **Schema Validation**: Enforcement of message format standards -- **Serialization Formats**: Support for JSON, Protocol Buffers, Avro -- **Compression**: Message compression for efficiency - -### Operational Features -- **Dead Letter Handling**: Management of unprocessable messages -- **Message Tracing**: End-to-end tracing of message flow -- **Event Sourcing**: Event storage and replay capability -- **Rate Limiting**: Protection against message floods -- **Back-pressure Handling**: Flow control mechanisms - -### Integration Capabilities -- **Service Discovery**: Dynamic discovery of publishers/subscribers -- **Protocol Bridging**: Support for multiple messaging protocols -- **External System Integration**: Connectors for external message systems -- **Legacy System Adapters**: Integration with non-event-driven systems -- **Web Integration**: WebSocket and SSE support for web clients - -## Planned Integration Points - -### Service Connections -- All platform microservices as publishers and subscribers -- Trading Dashboard (for real-time updates) -- External Data Sources (for ingestion) -- Monitoring Systems (for operational events) - -## Planned Technical Implementation - -### Technology Stack -- **Messaging Platform**: Kafka or RabbitMQ -- **Client Libraries**: Native TypeScript SDK -- **Monitoring**: Prometheus integration for metrics -- **Management**: Admin interface for topic/queue management -- **Storage**: Optimized storage for event persistence - -### Architecture Pattern -- Event-driven architecture -- Publish-subscribe pattern -- Command pattern for request-response -- Circuit breaker for resilience - -## Development Guidelines - -### Message Design -- Event schema standards -- Versioning approach -- Backward compatibility requirements -- Message size considerations - -### Integration Patterns -- Event notification pattern -- Event-carried state transfer -- Command messaging pattern -- Request-reply pattern implementations - -### Operational Considerations -- Monitoring requirements -- Scaling guidelines -- Disaster recovery approach -- Message retention policies - -## Implementation Roadmap -1. Core messaging infrastructure -2. Service integration patterns -3. Operational tooling and monitoring -4. Advanced features (replay, transformation) -5. External system connectors and adapters diff --git a/docs/intelligence-services/.gitkeep b/docs/intelligence-services/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/intelligence-services/README.md b/docs/intelligence-services/README.md deleted file mode 100644 index d1d6359..0000000 --- a/docs/intelligence-services/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Intelligence Services - -Intelligence services provide AI/ML capabilities, strategy execution, and algorithmic trading intelligence for the platform. - -## Services - -### Backtest Engine -- **Purpose**: Historical strategy testing and performance analysis -- **Key Functions**: - - Strategy backtesting with historical data - - Performance analytics and metrics calculation - - Vectorized and event-based processing modes - - Risk-adjusted return analysis - - Strategy comparison and optimization - -### Signal Engine -- **Purpose**: Trading signal generation and processing -- **Key Functions**: - - Technical indicator calculations - - Signal generation from multiple sources - - Signal aggregation and filtering - - Real-time signal processing - - Signal quality assessment - -### Strategy Orchestrator -- **Purpose**: Trading strategy execution and management -- **Key Functions**: - - Strategy lifecycle management - - Event-driven strategy execution - - Multi-strategy coordination - - Strategy performance monitoring - - Risk integration and position management - -## Architecture - -Intelligence services form the "brain" of the trading platform, combining market analysis, machine learning, and algorithmic decision-making to generate actionable trading insights. They work together to create a comprehensive trading intelligence pipeline from signal generation to strategy execution. diff --git a/docs/intelligence-services/backtest-engine/.gitkeep b/docs/intelligence-services/backtest-engine/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/intelligence-services/backtest-engine/README.md b/docs/intelligence-services/backtest-engine/README.md deleted file mode 100644 index 8df1959..0000000 --- a/docs/intelligence-services/backtest-engine/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Backtest Engine - -## Overview -The Backtest Engine service provides comprehensive historical simulation capabilities for trading strategies within the stock-bot platform. It enables strategy developers to evaluate performance, risk, and robustness of trading algorithms using historical market data before deploying them to production. - -## Key Features - -### Simulation Framework -- **Event-based Processing**: True event-driven simulation of market activities -- **Vectorized Processing**: High-performance batch processing for speed -- **Multi-asset Support**: Simultaneous testing across multiple instruments -- **Historical Market Data**: Access to comprehensive price and volume history - -### Performance Analytics -- **Return Metrics**: CAGR, absolute return, risk-adjusted metrics -- **Risk Metrics**: Drawdown, volatility, VaR, expected shortfall -- **Transaction Analysis**: Slippage modeling, fee impact, market impact -- **Statistical Analysis**: Win rate, profit factor, Sharpe/Sortino ratios - -### Realistic Simulation -- **Order Book Simulation**: Realistic market depth modeling -- **Latency Modeling**: Simulates execution and market data delays -- **Fill Probability Models**: Realistic order execution simulation -- **Market Impact Models**: Adjusts prices based on order sizes - -### Development Tools -- **Parameter Optimization**: Grid search and genetic algorithm optimization -- **Walk-forward Testing**: Time-based validation with parameter stability -- **Monte Carlo Analysis**: Probability distribution of outcomes -- **Sensitivity Analysis**: Impact of parameter changes on performance - -## Integration Points - -### Upstream Connections -- Market Data Gateway (for historical data) -- Feature Store (for historical feature values) -- Strategy Repository (for strategy definitions) - -### Downstream Consumers -- Strategy Orchestrator (for optimized parameters) -- Risk Guardian (for risk model validation) -- Trading Dashboard (for backtest visualization) -- Strategy Development Environment - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Computation Engine**: Optimized numerical libraries -- **Storage**: Time-series database for results -- **Visualization**: Interactive performance charts -- **Distribution**: Parallel processing for large backtests - -### Architecture Pattern -- Pipeline architecture for data flow -- Plugin system for custom components -- Separation of strategy logic from simulation engine -- Reproducible random state management - -## Development Guidelines - -### Strategy Development -- Strategy interface definition -- Testing harness documentation -- Performance optimization guidelines -- Validation requirements - -### Simulation Configuration -- Parameter specification format -- Simulation control options -- Market assumption configuration -- Execution model settings - -### Results Analysis -- Standard metrics calculation -- Custom metric development -- Visualization best practices -- Comparative analysis techniques - -## Future Enhancements -- Agent-based simulation for market microstructure -- Cloud-based distributed backtesting -- Real market data replay with tick data -- Machine learning for parameter optimization -- Strategy combination and portfolio optimization -- Enhanced visualization and reporting capabilities diff --git a/docs/intelligence-services/signal-engine/.gitkeep b/docs/intelligence-services/signal-engine/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/intelligence-services/signal-engine/README.md b/docs/intelligence-services/signal-engine/README.md deleted file mode 100644 index 1d00256..0000000 --- a/docs/intelligence-services/signal-engine/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Signal Engine - -## Overview -The Signal Engine service generates, processes, and manages trading signals within the stock-bot platform. It transforms raw market data and feature inputs into actionable trading signals that inform strategy execution decisions, serving as the analytical brain of the trading system. - -## Key Features - -### Signal Generation -- **Technical Indicators**: Comprehensive library of technical analysis indicators -- **Statistical Models**: Mean-reversion, momentum, and other statistical signals -- **Pattern Recognition**: Identification of chart patterns and formations -- **Custom Signal Definition**: Framework for creating proprietary signals - -### Signal Processing -- **Filtering**: Noise reduction and signal cleaning -- **Aggregation**: Combining multiple signals into composite indicators -- **Normalization**: Standardizing signals across different instruments -- **Ranking**: Relative strength measurement across instruments - -### Quality Management -- **Signal Strength Metrics**: Quantitative assessment of signal reliability -- **Historical Performance**: Tracking of signal predictive power -- **Decay Modeling**: Time-based degradation of signal relevance -- **Correlation Analysis**: Identifying redundant or correlated signals - -### Operational Features -- **Real-time Processing**: Low-latency signal generation -- **Batch Processing**: Overnight/weekend comprehensive signal computation -- **Signal Repository**: Historical storage of generated signals -- **Signal Subscription**: Event-based notification of new signals - -## Integration Points - -### Upstream Connections -- Market Data Gateway (for price and volume data) -- Feature Store (for derived trading features) -- Alternative Data Services (for sentiment, news factors) -- Data Processor (for preprocessed data) - -### Downstream Consumers -- Strategy Orchestrator (for signal consumption) -- Backtest Engine (for signal effectiveness analysis) -- Trading Dashboard (for signal visualization) -- Risk Guardian (for risk factor identification) - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **Calculation Engine**: Optimized numerical libraries -- **Storage**: Time-series database for signal storage -- **Messaging**: Event-driven notification system -- **Parallel Processing**: Multi-threaded computation for intensive signals - -### Architecture Pattern -- Pipeline architecture for signal flow -- Pluggable signal component design -- Separation of data preparation from signal generation -- Event sourcing for signal versioning - -## Development Guidelines - -### Signal Development -- Signal specification format -- Performance optimization techniques -- Testing requirements and methodology -- Documentation standards - -### Quality Controls -- Validation methodology -- Backtesting requirements -- Correlation thresholds -- Signal deprecation process - -### Operational Considerations -- Computation scheduling -- Resource utilization guidelines -- Monitoring requirements -- Failover procedures - -## Future Enhancements -- Machine learning-based signal generation -- Adaptive signal weighting -- Real-time signal quality feedback -- Advanced signal visualization -- Cross-asset class signals -- Alternative data integration diff --git a/docs/intelligence-services/strategy-orchestrator/.gitkeep b/docs/intelligence-services/strategy-orchestrator/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/intelligence-services/strategy-orchestrator/README.md b/docs/intelligence-services/strategy-orchestrator/README.md deleted file mode 100644 index 2c8787e..0000000 --- a/docs/intelligence-services/strategy-orchestrator/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Strategy Orchestrator - -## Overview -The Strategy Orchestrator service coordinates the execution and lifecycle management of trading strategies within the stock-bot platform. It serves as the central orchestration engine that translates trading signals into executable orders while managing strategy state, performance monitoring, and risk integration. - -## Key Features - -### Strategy Lifecycle Management -- **Strategy Registration**: Onboarding and configuration of trading strategies -- **Version Control**: Management of strategy versions and deployments -- **State Management**: Tracking of strategy execution state -- **Activation/Deactivation**: Controlled enabling and disabling of strategies - -### Execution Coordination -- **Signal Processing**: Consumes and processes signals from Signal Engine -- **Order Generation**: Translates signals into executable trading orders -- **Execution Timing**: Optimizes order timing based on market conditions -- **Multi-strategy Coordination**: Manages interactions between strategies - -### Performance Monitoring -- **Real-time Metrics**: Tracks strategy performance metrics in real-time -- **Alerting**: Notifies on strategy performance anomalies -- **Execution Quality**: Measures and reports on execution quality -- **Strategy Attribution**: Attributes P&L to specific strategies - -### Risk Integration -- **Pre-trade Risk Checks**: Validates orders against risk parameters -- **Position Tracking**: Monitors strategy position and exposure -- **Risk Limit Enforcement**: Ensures compliance with risk thresholds -- **Circuit Breakers**: Implements strategy-specific circuit breakers - -## Integration Points - -### Upstream Connections -- Signal Engine (for trading signals) -- Feature Store (for real-time feature access) -- Market Data Gateway (for market data) -- Backtest Engine (for optimized parameters) - -### Downstream Consumers -- Order Management System (for order execution) -- Risk Guardian (for risk monitoring) -- Trading Dashboard (for strategy visualization) -- Data Catalog (for strategy performance data) - -## Technical Implementation - -### Technology Stack -- **Runtime**: Node.js with TypeScript -- **State Management**: Redis for distributed state -- **Messaging**: Event-driven architecture with message bus -- **Database**: Time-series database for performance metrics -- **API**: RESTful API for management functions - -### Architecture Pattern -- Event-driven architecture for reactive processing -- Command pattern for strategy operations -- State machine for strategy lifecycle -- Circuit breaker pattern for fault tolerance - -## Development Guidelines - -### Strategy Integration -- Strategy interface specification -- Required callback implementations -- Configuration schema definition -- Testing and validation requirements - -### Performance Optimization -- Event processing efficiency -- State management best practices -- Resource utilization guidelines -- Latency minimization techniques - -### Operational Procedures -- Strategy deployment process -- Monitoring requirements -- Troubleshooting guidelines -- Failover procedures - -## Future Enhancements -- Advanced multi-strategy optimization -- Machine learning for execution optimization -- Enhanced strategy analytics dashboard -- Dynamic parameter adjustment -- Auto-scaling based on market conditions -- Strategy recommendation engine diff --git a/docs/migration-guide.md b/docs/migration-guide.md deleted file mode 100644 index 8826436..0000000 --- a/docs/migration-guide.md +++ /dev/null @@ -1,98 +0,0 @@ -# Migration Guide: From packages to libs - -This guide will help you migrate your service to use the new library structure for better separation of concerns. - -## Steps for each service - -1. Update your `package.json` dependencies to use the new libraries: - -```diff - "dependencies": { -- "@stock-bot/types": "workspace:*", -+ "@stock-bot/types": "workspace:*", -+ "@stock-bot/utils": "workspace:*", -+ "@stock-bot/event-bus": "workspace:*", -+ "@stock-bot/api-client": "workspace:*", - ... - } -``` - -2. Update your imports to use the domain-specific modules: - -```diff -- import { OHLCV, Strategy, Order } from '@stock-bot/types'; -+ import { OHLCV } from '@stock-bot/types'; -+ import { Strategy } from '@stock-bot/types'; -+ import { Order } from '@stock-bot/types'; -``` - -For logging: -```diff -- // Custom logging or console.log usage -+ import { createLogger, LogLevel } from '@stock-bot/utils'; -+ -+ const logger = createLogger('your-service-name'); -+ logger.info('Message'); -``` - -For API client usage: -```diff -- // Manual axios calls -+ import { createBacktestClient, createStrategyClient } from '@stock-bot/api-client'; -+ -+ const backtestClient = createBacktestClient(); -+ const result = await backtestClient.runBacktest(config); -``` - -For event-based communication: -```diff -- // Manual Redis/Dragonfly usage -+ import { createEventBus } from '@stock-bot/event-bus'; -+ import { MarketDataEvent } from '@stock-bot/types'; -+ -+ const eventBus = createEventBus({ -+ redisHost: process.env.REDIS_HOST || 'localhost', -+ redisPort: parseInt(process.env.REDIS_PORT || '6379') -+ }); -+ -+ eventBus.subscribe('market.data', async (event) => { -+ // Handle event -+ }); -``` - -## Example: Updating BacktestEngine - -```typescript -// Before -import { Strategy, BacktestConfig } from '@stock-bot/types'; -import Redis from 'ioredis'; - -// After -import { Strategy } from '@stock-bot/types'; -import { BacktestConfig } from '@stock-bot/types'; -import { createLogger } from '@stock-bot/utils'; -import { createEventBus } from '@stock-bot/event-bus'; - -const logger = createLogger('backtest-engine'); -const eventBus = createEventBus({ - redisHost: process.env.REDIS_HOST || 'localhost', - redisPort: parseInt(process.env.REDIS_PORT || '6379') -}); -``` - -## Updating build scripts - -If your turbo.json configuration references specific packages, update the dependencies: - -```diff - "backtest": { - "dependsOn": [ - "^build", -- "packages/types#build" -+ "libs/types#build", -+ "libs/utils#build", -+ "libs/event-bus#build", -+ "libs/api-client#build" - ], - } -``` diff --git a/docs/platform-services/.gitkeep b/docs/platform-services/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/platform-services/README.md b/docs/platform-services/README.md deleted file mode 100644 index 7ed6214..0000000 --- a/docs/platform-services/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Platform Services - -Platform services provide foundational infrastructure, monitoring, and operational capabilities that support all other services. - -## Services - -*Currently in planning phase - no active services deployed* - -## Planned Capabilities - -### Service Discovery -- **Purpose**: Dynamic service registration and discovery -- **Planned Functions**: - - Service health monitoring - - Load balancing and routing - - Service mesh coordination - - Configuration management - -### Logging & Monitoring -- **Purpose**: Observability and operational insights -- **Planned Functions**: - - Centralized logging aggregation - - Metrics collection and analysis - - Distributed tracing - - Performance monitoring and alerting - -### Configuration Management -- **Purpose**: Centralized configuration and secrets management -- **Planned Functions**: - - Environment-specific configurations - - Secrets encryption and rotation - - Dynamic configuration updates - - Configuration versioning and rollback - -### Authentication & Authorization -- **Purpose**: Security and access control -- **Planned Functions**: - - User authentication and session management - - Role-based access control (RBAC) - - API security and token management - - Audit logging and compliance - -### Backup & Recovery -- **Purpose**: Data protection and disaster recovery -- **Planned Functions**: - - Automated backup scheduling - - Point-in-time recovery - - Cross-region replication - - Disaster recovery orchestration - -## Architecture - -Platform services provide the operational foundation that enables reliable, secure, and observable operation of the entire trading platform. They implement cross-cutting concerns and best practices for production deployments. diff --git a/docs/platform-services/authentication-authorization/README.md b/docs/platform-services/authentication-authorization/README.md deleted file mode 100644 index e1c0e92..0000000 --- a/docs/platform-services/authentication-authorization/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Authentication & Authorization - -## Overview -The Authentication & Authorization service will provide comprehensive security controls for the stock-bot platform. It will manage user identity, authentication, access control, and security policy enforcement across all platform components, ensuring proper security governance and compliance with regulatory requirements. - -## Planned Features - -### User Management -- **User Provisioning**: Account creation and management -- **Identity Sources**: Local and external identity providers -- **User Profiles**: Customizable user attributes -- **Group Management**: User grouping and organization -- **Account Lifecycle**: Comprehensive user lifecycle management - -### Authentication -- **Multiple Factors**: Support for MFA/2FA -- **Single Sign-On**: Integration with enterprise SSO solutions -- **Social Login**: Support for third-party identity providers -- **Session Management**: Secure session handling and expiration -- **Password Policies**: Configurable password requirements - -### Authorization -- **Role-Based Access Control**: Fine-grained permission management -- **Attribute-Based Access**: Context-aware access decisions -- **Permission Management**: Centralized permission administration -- **Dynamic Policies**: Rule-based access policies -- **Delegated Administration**: Hierarchical permission management - -### Security Features -- **Token Management**: JWT and OAuth token handling -- **API Security**: Protection of API endpoints -- **Rate Limiting**: Prevention of brute force attacks -- **Audit Logging**: Comprehensive security event logging -- **Compliance Reporting**: Reports for regulatory requirements - -## Planned Integration Points - -### Service Integration -- All platform microservices -- API Gateway -- Frontend applications -- External systems and partners - -### Identity Providers -- Internal identity store -- Enterprise directory services -- Social identity providers -- OAuth/OIDC providers - -## Planned Technical Implementation - -### Technology Stack -- **Identity Server**: Keycloak or Auth0 -- **API Protection**: OAuth 2.0 and OpenID Connect -- **Token Format**: JWT with appropriate claims -- **Storage**: Secure credential and policy storage -- **Encryption**: Industry-standard encryption for sensitive data - -### Architecture Pattern -- Identity as a service -- Policy-based access control -- Token-based authentication -- Layered security model - -## Development Guidelines - -### Authentication Integration -- Authentication flow implementation -- Token handling best practices -- Session management requirements -- Credential security standards - -### Authorization Implementation -- Permission modeling approach -- Policy definition format -- Access decision points -- Contextual authorization techniques - -### Security Considerations -- Token security requirements -- Key rotation procedures -- Security event monitoring -- Penetration testing requirements - -## Implementation Roadmap -1. Core user management and authentication -2. Basic role-based authorization -3. API security and token management -4. Advanced access control policies -5. Compliance reporting and auditing diff --git a/docs/platform-services/backup-recovery/README.md b/docs/platform-services/backup-recovery/README.md deleted file mode 100644 index e14fcd3..0000000 --- a/docs/platform-services/backup-recovery/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Backup & Recovery - -## Overview -The Backup & Recovery service will provide comprehensive data protection, disaster recovery, and business continuity capabilities for the stock-bot platform. It will ensure that critical data and system configurations are preserved, with reliable recovery options in case of system failures, data corruption, or catastrophic events. - -## Planned Features - -### Backup Management -- **Automated Backups**: Scheduled backup of all critical data -- **Incremental Backups**: Efficient storage of incremental changes -- **Multi-tier Backup**: Different retention policies by data importance -- **Backup Verification**: Automated testing of backup integrity -- **Backup Catalog**: Searchable index of available backups - -### Recovery Capabilities -- **Point-in-time Recovery**: Restore to specific moments in time -- **Granular Recovery**: Restore specific objects or datasets -- **Self-service Recovery**: User portal for simple recovery operations -- **Recovery Testing**: Regular validation of recovery procedures -- **Recovery Performance**: Optimized for minimal downtime - -### Disaster Recovery -- **Cross-region Replication**: Geographic data redundancy -- **Recovery Site**: Standby environment for critical services -- **Failover Automation**: Scripted failover procedures -- **Recovery Orchestration**: Coordinated multi-system recovery -- **DR Testing**: Regular disaster scenario testing - -### Data Protection -- **Encryption**: At-rest and in-transit data encryption -- **Access Controls**: Restricted access to backup data -- **Retention Policies**: Compliance with data retention requirements -- **Immutable Backups**: Protection against ransomware -- **Air-gapped Storage**: Ultimate protection for critical backups - -## Planned Integration Points - -### Data Sources -- Platform databases (MongoDB, PostgreSQL) -- Object storage and file systems -- Service configurations -- Message queues and event streams -- User data and preferences - -### System Integration -- Infrastructure as Code systems -- Monitoring and alerting -- Compliance reporting -- Operations management tools - -## Planned Technical Implementation - -### Technology Stack -- **Backup Tools**: Cloud-native backup solutions -- **Storage**: Object storage with versioning -- **Orchestration**: Infrastructure as Code for recovery -- **Monitoring**: Backup health and status monitoring -- **Automation**: Scripted recovery procedures - -### Architecture Pattern -- Centralized backup management -- Distributed backup agents -- Immutable backup storage -- Recovery validation automation - -## Development Guidelines - -### Backup Strategy -- Backup frequency guidelines -- Retention period standards -- Versioning requirements -- Validation procedures - -### Recovery Procedures -- Recovery time objectives -- Recovery point objectives -- Testing frequency requirements -- Documentation standards - -### Security Requirements -- Encryption standards -- Access control implementation -- Audit requirements -- Secure deletion procedures - -## Implementation Roadmap -1. Core database backup capabilities -2. Basic recovery procedures -3. Cross-region replication -4. Automated recovery testing -5. Advanced protection features diff --git a/docs/platform-services/configuration-management/README.md b/docs/platform-services/configuration-management/README.md deleted file mode 100644 index 4ab465a..0000000 --- a/docs/platform-services/configuration-management/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Configuration Management - -## Overview -The Configuration Management service will provide centralized management of application and service configurations across the stock-bot platform. It will handle environment-specific settings, dynamic configuration updates, secrets management, and configuration versioning to ensure consistent and secure system configuration. - -## Planned Features - -### Configuration Storage -- **Hierarchical Configuration**: Nested configuration structure -- **Environment Separation**: Environment-specific configurations -- **Schema Validation**: Configuration format validation -- **Default Values**: Fallback configuration defaults -- **Configuration as Code**: Version-controlled configuration - -### Dynamic Configuration -- **Runtime Updates**: Changes without service restart -- **Configuration Propagation**: Real-time distribution of changes -- **Subscription Model**: Configuration change notifications -- **Batch Updates**: Atomic multi-value changes -- **Feature Flags**: Dynamic feature enablement - -### Secrets Management -- **Secure Storage**: Encrypted storage of sensitive values -- **Access Control**: Fine-grained access to secrets -- **Secret Versioning**: Historical versions of secrets -- **Automatic Rotation**: Scheduled credential rotation -- **Key Management**: Management of encryption keys - -### Operational Features -- **Configuration History**: Tracking of configuration changes -- **Rollbacks**: Revert to previous configurations -- **Audit Trail**: Comprehensive change logging -- **Configuration Comparison**: Diff between configurations -- **Import/Export**: Bulk configuration operations - -## Planned Integration Points - -### Service Integration -- All platform microservices -- CI/CD pipelines -- Infrastructure components -- Development environments - -### External Systems -- Secret management services -- Source control systems -- Operational monitoring -- Compliance systems - -## Planned Technical Implementation - -### Technology Stack -- **Configuration Server**: Spring Cloud Config or custom solution -- **Secret Store**: HashiCorp Vault or AWS Secrets Manager -- **Storage**: Git-backed or database storage -- **API**: RESTful interface with versioning -- **SDK**: Client libraries for service integration - -### Architecture Pattern -- Configuration as a service -- Event-driven configuration updates -- Layered access control model -- High-availability design - -## Development Guidelines - -### Configuration Structure -- Naming conventions -- Hierarchy organization -- Type validation -- Documentation requirements - -### Secret Management -- Secret classification -- Rotation requirements -- Access request process -- Emergency access procedures - -### Integration Approach -- Client library usage -- Caching recommendations -- Failure handling -- Update processing - -## Implementation Roadmap -1. Static configuration management -2. Basic secrets storage -3. Dynamic configuration updates -4. Advanced secret management features -5. Operational tooling and integration diff --git a/docs/platform-services/logging-monitoring/README.md b/docs/platform-services/logging-monitoring/README.md deleted file mode 100644 index b9913b1..0000000 --- a/docs/platform-services/logging-monitoring/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# Logging & Monitoring - -## Overview -The Logging & Monitoring service will provide comprehensive observability capabilities for the stock-bot platform. It will collect, process, store, and visualize logs, metrics, and traces from all platform components, enabling effective operational monitoring, troubleshooting, and performance optimization. - -## Planned Features - -### Centralized Logging -- **Log Aggregation**: Collection of logs from all services -- **Structured Logging**: Standardized log format across services -- **Log Processing**: Parsing, enrichment, and transformation -- **Log Storage**: Efficient storage with retention policies -- **Log Search**: Advanced search capabilities with indexing - -### Metrics Collection -- **System Metrics**: CPU, memory, disk, network usage -- **Application Metrics**: Custom application-specific metrics -- **Business Metrics**: Trading and performance indicators -- **SLI/SLO Tracking**: Service level indicators and objectives -- **Alerting Thresholds**: Metric-based alert configuration - -### Distributed Tracing -- **Request Tracing**: End-to-end tracing of requests -- **Span Collection**: Detailed operation timing -- **Trace Correlation**: Connect logs, metrics, and traces -- **Latency Analysis**: Performance bottleneck identification -- **Dependency Mapping**: Service dependency visualization - -### Alerting & Notification -- **Alert Rules**: Multi-condition alert definitions -- **Notification Channels**: Email, SMS, chat integrations -- **Alert Grouping**: Intelligent alert correlation -- **Escalation Policies**: Tiered notification escalation -- **On-call Management**: Rotation and scheduling - -## Planned Integration Points - -### Data Sources -- All platform microservices -- Infrastructure components -- Databases and storage systems -- Message bus and event streams -- External dependencies - -### Consumers -- Operations team dashboards -- Incident management systems -- Capacity planning tools -- Automated remediation systems - -## Planned Technical Implementation - -### Technology Stack -- **Logging**: ELK Stack (Elasticsearch, Logstash, Kibana) or similar -- **Metrics**: Prometheus and Grafana -- **Tracing**: Jaeger or Zipkin -- **Alerting**: AlertManager or PagerDuty -- **Collection**: Vector, Fluentd, or similar collectors - -### Architecture Pattern -- Centralized collection with distributed agents -- Push and pull metric collection models -- Sampling for high-volume telemetry -- Buffering for resilient data collection - -## Development Guidelines - -### Instrumentation Standards -- Logging best practices -- Metric naming conventions -- Trace instrumentation approach -- Cardinality management - -### Performance Impact -- Sampling strategies -- Buffer configurations -- Resource utilization limits -- Batching recommendations - -### Data Management -- Retention policies -- Aggregation strategies -- Storage optimization -- Query efficiency guidelines - -## Implementation Roadmap -1. Core logging infrastructure -2. Basic metrics collection -3. Critical alerting capability -4. Distributed tracing -5. Advanced analytics and visualization diff --git a/docs/platform-services/service-discovery/README.md b/docs/platform-services/service-discovery/README.md deleted file mode 100644 index fc616c8..0000000 --- a/docs/platform-services/service-discovery/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Service Discovery - -## Overview -The Service Discovery component will provide dynamic registration, discovery, and health monitoring of services within the stock-bot platform. It will enable services to locate and communicate with each other without hardcoded endpoints, supporting a flexible and resilient microservices architecture. - -## Planned Features - -### Service Registration -- **Automatic Registration**: Self-registration of services on startup -- **Metadata Management**: Service capabilities and endpoint information -- **Instance Tracking**: Multiple instances of the same service -- **Version Information**: Service version and compatibility data -- **Registration Expiry**: TTL-based registration with renewal - -### Service Discovery -- **Name-based Lookup**: Find services by logical names -- **Filtering**: Discovery based on metadata and attributes -- **Load Balancing**: Client or server-side load balancing -- **Caching**: Client-side caching of service information -- **DNS Integration**: Optional DNS-based discovery - -### Health Monitoring -- **Health Checks**: Customizable health check protocols -- **Automatic Deregistration**: Removal of unhealthy instances -- **Status Propagation**: Health status notifications -- **Dependency Health**: Cascading health status for dependencies -- **Self-healing**: Automatic recovery procedures - -### Configuration Management -- **Dynamic Configuration**: Runtime configuration updates -- **Environment-specific Settings**: Configuration by environment -- **Configuration Versioning**: History and rollback capabilities -- **Secret Management**: Secure handling of sensitive configuration -- **Configuration Change Events**: Notifications of config changes - -## Planned Integration Points - -### Service Integration -- All platform microservices -- External service dependencies -- Infrastructure components -- Monitoring systems - -## Planned Technical Implementation - -### Technology Stack -- **Service Registry**: Consul, etcd, or ZooKeeper -- **Client Libraries**: TypeScript SDK for services -- **Health Check**: HTTP, TCP, and custom health checks -- **Configuration Store**: Distributed key-value store -- **Load Balancer**: Client-side or service mesh integration - -### Architecture Pattern -- Service registry pattern -- Client-side discovery pattern -- Health check pattern -- Circuit breaker integration - -## Development Guidelines - -### Service Integration -- Registration process -- Discovery implementation -- Health check implementation -- Configuration consumption - -### Resilience Practices -- Caching strategy -- Fallback mechanisms -- Retry configuration -- Circuit breaker settings - -### Operational Considerations -- High availability setup -- Disaster recovery approach -- Scaling guidelines -- Monitoring requirements - -## Implementation Roadmap -1. Core service registry implementation -2. Basic health checking -3. Service discovery integration -4. Configuration management -5. Advanced health monitoring with dependency tracking diff --git a/docs/system-architecture.md b/docs/system-architecture.md deleted file mode 100644 index 85d8e7f..0000000 --- a/docs/system-architecture.md +++ /dev/null @@ -1,358 +0,0 @@ -# ๐Ÿ—๏ธ Stock Bot System Architecture - -## System Communication Flow Diagram - -```mermaid -graph TB - %% External Systems - subgraph "External APIs" - AV[Alpha Vantage API] - YF[Yahoo Finance API] - AL[Alpaca Broker API] - IB[Interactive Brokers] - NEWS[News APIs] - end - - %% Core Services Layer - subgraph "Core Services" - MDG[Market Data Gateway
:3001] - RG[Risk Guardian
:3002] - EE[Execution Engine
:3003] - PM[Portfolio Manager
:3004] - end - - %% Intelligence Services Layer - subgraph "Intelligence Services" - SO[Strategy Orchestrator
:4001] - SG[Signal Generator
:4002] - BA[Backtesting Engine
:4003] - ML[ML Pipeline
:4004] - end - - %% Data Services Layer - subgraph "Data Services" - HDS[Historical Data Service
:5001] - AS[Analytics Service
:5002] - DQS[Data Quality Service
:5003] - ETLS[ETL Service
:5004] - end - - %% Platform Services Layer - subgraph "Platform Services" - LM[Log Manager
:6001] - CM[Config Manager
:6002] - AM[Alert Manager
:6003] - SM[Service Monitor
:6004] - end - - %% Integration Services Layer - subgraph "Integration Services" - BAS[Broker Adapter
:7001] - DAS[Data Adapter
:7002] - NS[Notification Service
:7003] - WHS[Webhook Service
:7004] - end - - %% Interface Services Layer - subgraph "Interface Services" - TD[Trading Dashboard
:5173] - API[REST API Gateway
:8001] - WS[WebSocket Server
Embedded] - end - - %% Storage Layer - subgraph "Storage Layer" - DRAGONFLY[(Dragonfly
Events & Cache)] - QDB[(QuestDB
Time Series)] - PGDB[(PostgreSQL
Relational)] - FS[(File System
Logs & Config)] - end - - %% Communication Flows - - %% External to Core - AV --> MDG - YF --> MDG - AL --> BAS - IB --> BAS - NEWS --> DAS - - %% Core Service Communications - MDG -->|Market Data Events| DRAGONFLY - MDG -->|Real-time Stream| WS - MDG -->|Cache| DRAGONFLY - - RG -->|Risk Events| DRAGONFLY - RG -->|Risk Alerts| AM - RG -->|Position Limits| PM - - EE -->|Order Events| DRAGONFLY - EE -->|Trade Execution| BAS - EE -->|Order Status| PM - - PM -->|Portfolio Events| DRAGONFLY - PM -->|P&L Updates| TD - PM -->|Position Data| RG - - %% Intelligence Communications - SO -->|Strategy Events| DRAGONFLY - SO -->|Signal Requests| SG - SO -->|Execution Orders| EE - SO -->|Risk Check| RG - - SG -->|Trading Signals| SO - SG -->|ML Requests| ML - SG -->|Market Data| DRAGONFLY - - BA -->|Backtest Results| SO - BA -->|Historical Data| HDS - - ML -->|Predictions| SG - ML -->|Training Data| HDS - - %% Data Service Communications - HDS -->|Store Data| QDB - HDS -->|Query Data| QDB - HDS -->|Data Events| DRAGONFLY - - AS -->|Analytics| QDB - AS -->|Metrics| SM - AS -->|Reports| TD - - DQS -->|Data Quality| DRAGONFLY - DQS -->|Alerts| AM - - ETLS -->|Raw Data| DAS - ETLS -->|Processed Data| HDS - - %% Platform Communications - LM -->|Logs| FS - LM -->|Log Events| DRAGONFLY - - CM -->|Config| FS - CM -->|Config Updates| DRAGONFLY - - AM -->|Alerts| NS - AM -->|Alert Events| DRAGONFLY - - SM -->|Health Checks| DRAGONFLY - SM -->|Metrics| QDB - - %% Integration Communications - BAS -->|Orders| AL - BAS -->|Orders| IB - BAS -->|Order Updates| EE - - DAS -->|Data Feed| MDG - DAS -->|External Data| HDS - - NS -->|Notifications| WHS - NS -->|Alerts| TD - - WHS -->|Webhooks| External - - %% Interface Communications - TD -->|API Calls| API - TD -->|WebSocket| WS - TD -->|Dashboard Data| PM - - API -->|Service Calls| SO - API -->|Data Queries| HDS - API -->|System Status| SM - - WS -->|Real-time Data| TD - WS -->|Events| DRAGONFLY - - %% Storage Access DRAGONFLY -.->|Events| SO - DRAGONFLY -.->|Events| RG - DRAGONFLY -.->|Events| PM - DRAGONFLY -.->|Cache| MDG - - QDB -.->|Time Series| HDS - QDB -.->|Analytics| AS - QDB -.->|Metrics| SM - - PGDB -.->|Relational| PM - PGDB -.->|Config| CM - PGDB -.->|Users| API - - %% Styling - classDef external fill:#ff9999 - classDef core fill:#99ccff - classDef intelligence fill:#99ff99 - classDef data fill:#ffcc99 - classDef platform fill:#cc99ff - classDef integration fill:#ffff99 - classDef interface fill:#ff99cc - classDef storage fill:#cccccc - - class AV,YF,AL,IB,NEWS external - class MDG,RG,EE,PM core - class SO,SG,BA,ML intelligence - class HDS,AS,DQS,ETLS data - class LM,CM,AM,SM platform - class BAS,DAS,NS,WHS integration - class TD,API,WS interface - class DRAGONFLY,QDB,PGDB,FS storage -``` - -## Communication Patterns - -### 1. **Event-Driven Architecture (Dragonfly Streams)** -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” Dragonfly Events โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Service โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ†’ โ”‚ Service โ”‚ -โ”‚ A โ”‚ โ”‚ B โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -**Event Types:** -- `MARKET_DATA` - Real-time price updates -- `ORDER_CREATED/FILLED/CANCELLED` - Order lifecycle -- `SIGNAL_GENERATED` - Trading signals -- `RISK_ALERT` - Risk threshold violations -- `PORTFOLIO_UPDATE` - Position changes -- `STRATEGY_START/STOP` - Strategy lifecycle - -### 2. **Request-Response (HTTP/REST)** -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” HTTP Request โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Client โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ†’ โ”‚ Service โ”‚ -โ”‚ โ”‚ โ†โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”‚ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ HTTP Response โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -**API Endpoints:** -- `/api/market-data/:symbol` - Current market data -- `/api/portfolio/positions` - Portfolio positions -- `/api/strategies` - Strategy management -- `/api/orders` - Order management -- `/api/health` - Service health checks - -### 3. **Real-time Streaming (WebSocket)** -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” WebSocket โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Client โ”‚ โ†โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ†’ โ”‚ Server โ”‚ -โ”‚ โ”‚ Bidirectional โ”‚ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -**WebSocket Messages:** -- Market data subscriptions -- Portfolio updates -- Trading signals -- Risk alerts -- System notifications - -### 4. **Data Persistence** -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” Store/Query โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Service โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ†’ โ”‚ Database โ”‚ -โ”‚ โ”‚ โ†โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”‚ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -**Storage Types:** -- **Dragonfly**: Events, cache, sessions -- **QuestDB**: Time-series data, metrics -- **PostgreSQL**: Configuration, users, metadata -- **File System**: Logs, configurations - -## Service Communication Matrix - -| Service | Publishes Events | Subscribes to Events | HTTP APIs | WebSocket | Storage | -|---------|-----------------|---------------------|-----------|-----------|---------| -| Market Data Gateway | โœ… Market Data | - | โœ… REST | โœ… Server | Dragonfly Cache | -| Risk Guardian | โœ… Risk Alerts | โœ… All Events | โœ… REST | - | PostgreSQL | -| Strategy Orchestrator | โœ… Strategy Events | โœ… Market Data, Signals | โœ… REST | - | PostgreSQL | -| Execution Engine | โœ… Order Events | โœ… Strategy Events | โœ… REST | - | PostgreSQL | -| Portfolio Manager | โœ… Portfolio Events | โœ… Order Events | โœ… REST | - | PostgreSQL | -| Trading Dashboard | - | โœ… All Events | โœ… Client | โœ… Client | - | - -## Data Flow Example: Trade Execution - -```mermaid -sequenceDiagram - participant TD as Trading Dashboard - participant SO as Strategy Orchestrator - participant SG as Signal Generator - participant RG as Risk Guardian - participant EE as Execution Engine - participant BAS as Broker Adapter - participant PM as Portfolio Manager - participant DRAGONFLY as Dragonfly Events - - Note over TD,DRAGONFLY: User starts a trading strategy - - TD->>SO: POST /api/strategies/start - SO->>DRAGONFLY: Publish STRATEGY_START event - - Note over SO,SG: Strategy generates signals - - SO->>SG: Request signals for AAPL - SG->>SO: Return BUY signal - SO->>DRAGONFLY: Publish SIGNAL_GENERATED event - - Note over SO,RG: Risk check before execution - - SO->>RG: Check risk limits - RG->>SO: Risk approved - - Note over SO,EE: Execute the trade - - SO->>EE: Submit order - EE->>DRAGONFLY: Publish ORDER_CREATED event - EE->>BAS: Send order to broker - BAS->>EE: Order filled - EE->>DRAGONFLY: Publish ORDER_FILLED event - - Note over PM,TD: Update portfolio and notify user - - PM->>DRAGONFLY: Subscribe to ORDER_FILLED - PM->>PM: Update positions PM->>DRAGONFLY: Publish PORTFOLIO_UPDATE - TD->>DRAGONFLY: Subscribe to PORTFOLIO_UPDATE - TD->>TD: Update dashboard -``` - -## Port Allocation - -| Service Category | Port Range | Services | -|-----------------|------------|----------| -| Core Services | 3001-3099 | Market Data Gateway (3001), Risk Guardian (3002) | -| Intelligence Services | 4001-4099 | Strategy Orchestrator (4001), Signal Generator (4002) | -| Data Services | 5001-5099 | Historical Data (5001), Analytics (5002) | -| Platform Services | 6001-6099 | Log Manager (6001), Config Manager (6002) | -| Integration Services | 7001-7099 | Broker Adapter (7001), Data Adapter (7002) | -| Interface Services | 8001-8099 | API Gateway (8001), Dashboard (5173-Vite) | - -## Security & Authentication - -```mermaid -graph LR - subgraph "Security Layer" - JWT[JWT Tokens] - API_KEY[API Keys] - TLS[TLS/HTTPS] - RBAC[Role-Based Access] - end - - subgraph "External Security" - BROKER_AUTH[Broker Authentication] - DATA_AUTH[Data Provider Auth] - WEBHOOK_SIG[Webhook Signatures] - end - - JWT --> API_KEY - API_KEY --> TLS - TLS --> RBAC - RBAC --> BROKER_AUTH - BROKER_AUTH --> DATA_AUTH - DATA_AUTH --> WEBHOOK_SIG -``` - -This architecture provides: -- **Scalability**: Services can be scaled independently -- **Reliability**: Event-driven communication with retry mechanisms -- **Maintainability**: Clear separation of concerns -- **Observability**: Centralized logging and monitoring -- **Security**: Multiple layers of authentication and authorization diff --git a/docs/system-communication-chart.md b/docs/system-communication-chart.md deleted file mode 100644 index c3f3c50..0000000 --- a/docs/system-communication-chart.md +++ /dev/null @@ -1,140 +0,0 @@ -# ๐Ÿ“Š Stock Bot System Communication - Quick Reference - -## Current System (Active) -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ TRADING BOT SYSTEM โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - -External APIs Core Services Interface Services -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Demo Data โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ–ถโ”‚ Market Data โ”‚โ—€โ”€โ”€โ”€โ”€โ”€โ”€โ–ถโ”‚ Trading โ”‚ -โ”‚ Alpha Vant. โ”‚ โ”‚ Gateway โ”‚ โ”‚ Dashboard โ”‚ -โ”‚ Yahoo Fin. โ”‚ โ”‚ Port: 3001 โ”‚ โ”‚ Port: 5173 โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ Status: โœ… LIVE โ”‚ โ”‚ Status: โœ… LIVE โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ–ฒ - โ–ผ โ”‚ - โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ - โ”‚ Dragonfly Events โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ Cache & Streams โ”‚ - โ”‚ Status: โœ… READYโ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -## Next Phase (Ready to Implement) -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ EXPANDED TRADING SYSTEM โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - -Intelligence Services Core Services Interface Services -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Strategy โ”‚โ—€โ”€โ”€โ”€โ”€โ–ถโ”‚ Market Data โ”‚โ—€โ”€โ”€โ”€โ–ถโ”‚ Trading โ”‚ -โ”‚ Orchestrator โ”‚ โ”‚ Gateway โ”‚ โ”‚ Dashboard โ”‚ -โ”‚ Port: 4001 โ”‚ โ”‚ Port: 3001 โ”‚ โ”‚ Port: 5173 โ”‚ -โ”‚ Status: ๐Ÿ“‹ PLAN โ”‚ โ”‚ Status: โœ… LIVE โ”‚ โ”‚ Status: โœ… LIVE โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ–ฒ โ”‚ โ–ฒ - โ”‚ โ–ผ โ”‚ - โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ถโ”‚ Dragonfly Event โ”‚โ—€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ Stream Hub โ”‚ - โ”‚ Status: โœ… READYโ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ–ฒ - โ”‚ - โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” - โ”‚ Risk Guardian โ”‚ - โ”‚ Port: 3002 โ”‚ - โ”‚ Status: ๐Ÿ“‹ PLAN โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -## Communication Protocols - -### HTTP REST API -``` -Client โ”€โ”€โ”€โ”€ GET/POST โ”€โ”€โ”€โ–ถ Server - โ—€โ”€โ”€โ”€ JSON โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -``` - -### WebSocket Real-time -``` -Client โ—€โ•โ•โ• Stream โ•โ•โ•โ–ถ Server - โ—€โ•โ•โ• Events โ•โ•โ–ถ -``` - -### Dragonfly Event Bus -``` -Service A โ”€โ”€โ”€โ”€ Publish โ”€โ”€โ”€โ–ถ Dragonfly โ”€โ”€โ”€โ”€ Subscribe โ”€โ”€โ”€โ–ถ Service B - โ—€โ”€โ”€โ”€ Confirm โ”€โ”€โ”€โ”€ โ—€โ”€โ”€โ”€ Events โ”€โ”€โ”€โ”€ -``` - -## Event Types - -| Event Type | Publisher | Subscribers | Frequency | -|------------|-----------|-------------|-----------| -| `MARKET_DATA` | Market Data Gateway | Dashboard, Strategy Orchestrator | Every 5s | -| `SIGNAL_GENERATED` | Strategy Orchestrator | Risk Guardian, Execution Engine | As needed | -| `RISK_ALERT` | Risk Guardian | Dashboard, Alert Manager | As needed | -| `PORTFOLIO_UPDATE` | Portfolio Manager | Dashboard, Risk Guardian | On trades | - -## Service Status Matrix - -| Service | Port | Status | Dependencies | Ready to Implement | -|---------|------|--------|--------------|-------------------| -| Market Data Gateway | 3001 | โœ… Running | Dragonfly, Config | โœ… Complete | -| Trading Dashboard | 5173 | โœ… Running | MDG API | โœ… Complete | -| Strategy Orchestrator | 4001 | ๐Ÿ“‹ Planned | Dragonfly, MDG | โœ… Package Ready | -| Risk Guardian | 3002 | ๐Ÿ“‹ Planned | Dragonfly, Config | โœ… Package Ready | -| Portfolio Manager | 3004 | โณ Future | Database, Orders | โŒ Not Started | -| Execution Engine | 3003 | โณ Future | Brokers, Portfolio | โŒ Not Started | - -## Data Flow Summary - -1. **Market Data Flow** - ``` - External APIs โ†’ Market Data Gateway โ†’ Dragonfly Events โ†’ Dashboard - โ†’ Strategy Orchestrator - ``` - -2. **Trading Signal Flow** - ``` - Market Data โ†’ Strategy Orchestrator โ†’ Trading Signals โ†’ Risk Guardian - โ†’ Execution Engine - ``` - -3. **Risk Management Flow** - ``` - All Events โ†’ Risk Guardian โ†’ Risk Alerts โ†’ Alert Manager - โ†’ Risk Blocks โ†’ Strategy Orchestrator - ``` - -4. **User Interface Flow** - ``` - WebSocket โ† Dashboard โ†’ REST API โ†’ Services - Events โ† โ†’ Commands โ†’ - ``` - -## Implementation Priority - -### Phase 1 (Current) โœ… -- [x] Market Data Gateway -- [x] Trading Dashboard -- [x] Dragonfly Infrastructure -- [x] WebSocket Communication - -### Phase 2 (Next) ๐Ÿ“‹ -- [ ] Strategy Orchestrator -- [ ] Risk Guardian -- [ ] Event-driven Strategy Execution -- [ ] Risk Monitoring & Alerts - -### Phase 3 (Future) โณ -- [ ] Portfolio Manager -- [ ] Execution Engine -- [ ] Broker Integration -- [ ] Database Persistence - -The system is designed for incremental development where each service can be implemented and tested independently while maintaining full system functionality. diff --git a/docs/websocket-api.md b/docs/websocket-api.md deleted file mode 100644 index fbebaee..0000000 --- a/docs/websocket-api.md +++ /dev/null @@ -1,247 +0,0 @@ -# WebSocket API Documentation - Strategy Orchestrator - -## Overview - -This document outlines the WebSocket API provided by the Strategy Orchestrator service for real-time communication between the backend and frontend. - -## Connection Endpoints - -- **Strategy Orchestrator WebSocket:** `ws://localhost:8082` -- **Market Data Gateway WebSocket:** `ws://localhost:3001/ws` -- **Risk Guardian WebSocket:** `ws://localhost:3002/ws` - -## Message Format - -All WebSocket messages follow this standard format: - -```json -{ - "type": "message_type", - "timestamp": "ISO-8601 timestamp", - "data": { - // Message-specific data - } -} -``` - -## Client to Server Messages - -### Get Active Strategies - -Request information about all active strategies. - -```json -{ - "type": "get_active_strategies" -} -``` - -### Start Strategy - -Start a specific strategy. - -```json -{ - "type": "start_strategy", - "id": "strategy-id", - "config": { - "symbols": ["AAPL", "MSFT"], - "dataResolution": "1m", - "realTrading": false, - "maxPositionValue": 10000, - "maxOrdersPerMinute": 5, - "stopLossPercentage": 0.02 - } -} -``` - -### Stop Strategy - -Stop a running strategy. - -```json -{ - "type": "stop_strategy", - "id": "strategy-id" -} -``` - -### Pause Strategy - -Pause a running strategy. - -```json -{ - "type": "pause_strategy", - "id": "strategy-id" -} -``` - -## Server to Client Messages - -### Strategy Status List - -Response to `get_active_strategies` request. - -```json -{ - "type": "strategy_status_list", - "timestamp": "2023-07-10T12:34:56Z", - "data": [ - { - "id": "strategy-123", - "name": "Moving Average Crossover", - "status": "ACTIVE", - "symbols": ["AAPL", "MSFT"], - "positions": [ - { - "symbol": "AAPL", - "quantity": 10, - "entryPrice": 150.25, - "currentValue": 1550.50 - } - ] - } - ] -} -``` - -### Strategy Started - -Notification that a strategy has been started. - -```json -{ - "type": "strategy_started", - "timestamp": "2023-07-10T12:34:56Z", - "data": { - "strategyId": "strategy-123", - "name": "Moving Average Crossover", - "symbols": ["AAPL", "MSFT"] - } -} -``` - -### Strategy Stopped - -Notification that a strategy has been stopped. - -```json -{ - "type": "strategy_stopped", - "timestamp": "2023-07-10T12:34:56Z", - "data": { - "strategyId": "strategy-123", - "name": "Moving Average Crossover" - } -} -``` - -### Strategy Paused - -Notification that a strategy has been paused. - -```json -{ - "type": "strategy_paused", - "timestamp": "2023-07-10T12:34:56Z", - "data": { - "strategyId": "strategy-123", - "name": "Moving Average Crossover" - } -} -``` - -### Strategy Signal - -Trading signal generated by a strategy. - -```json -{ - "type": "strategy_signal", - "timestamp": "2023-07-10T12:34:56Z", - "data": { - "id": "sig_123456789", - "strategyId": "strategy-123", - "name": "Moving Average Crossover", - "symbol": "AAPL", - "price": 152.75, - "action": "BUY", - "quantity": 10, - "metadata": { "orderType": "MARKET" }, - "timestamp": "2023-07-10T12:34:56Z", - "confidence": 0.9 - } -} -``` - -### Strategy Trade - -Notification of a trade executed by a strategy. - -```json -{ - "type": "strategy_trade", - "timestamp": "2023-07-10T12:34:56Z", - "data": { - "id": "trade_123456789", - "strategyId": "strategy-123", - "orderId": "order-123", - "symbol": "AAPL", - "side": "BUY", - "quantity": 10, - "entryPrice": 152.75, - "entryTime": "2023-07-10T12:34:56Z", - "status": "FILLED", - "timestamp": "2023-07-10T12:34:56Z" - } -} -``` - -### Execution Service Status - -Notification about the overall execution service status. - -```json -{ - "type": "execution_service_status", - "timestamp": "2023-07-10T12:34:56Z", - "data": { - "status": "RUNNING" // or "STOPPED" - } -} -``` - -## Frontend Integration - -The frontend can use the `WebSocketService` to interact with the WebSocket API: - -```typescript -// Subscribe to strategy signals -webSocketService.getStrategySignals(strategyId) - .subscribe(signal => { - // Handle the signal - }); - -// Subscribe to strategy trades -webSocketService.getStrategyTrades(strategyId) - .subscribe(trade => { - // Handle the trade - }); - -// Subscribe to strategy status updates -webSocketService.getStrategyUpdates() - .subscribe(update => { - // Handle the update - }); -``` - -## Error Handling - -WebSocket connections will automatically attempt to reconnect if disconnected. The frontend can monitor connection status using: - -```typescript -webSocketService.isConnected() // Returns a boolean signal indicating connection status -``` - -If a WebSocket message fails to process, the error will be logged, but the connection will be maintained. diff --git a/libs/config/src/data-providers.ts b/libs/config/src/data-providers.ts index 417b0be..cfd7876 100644 --- a/libs/config/src/data-providers.ts +++ b/libs/config/src/data-providers.ts @@ -3,6 +3,19 @@ */ import { cleanEnv, str, num, bool } from 'envalid'; +export interface ProviderConfig { + name: string; + type: 'rest' | 'websocket'; + enabled: boolean; + baseUrl?: string; + apiKey?: string; + apiSecret?: string; + rateLimits?: { + maxRequestsPerMinute?: number; + maxRequestsPerSecond?: number; + maxRequestsPerHour?: number; + }; +} /** * Data providers configuration with validation and defaults */ @@ -53,6 +66,8 @@ export const dataProvidersConfig = cleanEnv(process.env, { * Helper function to get provider-specific configuration */ export function getProviderConfig(providerName: string) { + // make a interface for the provider config + const name = providerName.toUpperCase(); switch (name) { @@ -128,6 +143,20 @@ export function getDefaultProvider() { // Export typed configuration object export type DataProvidersConfig = typeof dataProvidersConfig; +export class DataProviders { + static getProviderConfig(providerName: string): ProviderConfig { + return getProviderConfig(providerName); + } + + static getEnabledProviders(): ProviderConfig[] { + return getEnabledProviders(); + } + + static getDefaultProvider(): ProviderConfig { + return getDefaultProvider(); + } +} + // Export individual config values for convenience export const { diff --git a/libs/logger/package.json b/libs/logger/package.json index eb1906e..e67ddc7 100644 --- a/libs/logger/package.json +++ b/libs/logger/package.json @@ -11,11 +11,11 @@ "test": "jest" }, "dependencies": { - "@stock-bot/types": "workspace:*", "@stock-bot/config": "workspace:*", - "winston": "^3.11.0", - "winston-loki": "^6.0.8", - "winston-daily-rotate-file": "^4.7.1" + "@stock-bot/types": "workspace:*", + "pino": "^9.7.0", + "pino-loki": "^2.6.0", + "pino-pretty": "^13.0.0" }, "devDependencies": { "@types/jest": "^29.5.2", diff --git a/libs/logger/src/index.ts b/libs/logger/src/index.ts index 32f95fc..8a5eb8a 100644 --- a/libs/logger/src/index.ts +++ b/libs/logger/src/index.ts @@ -21,11 +21,15 @@ export { LogThrottle } from './utils'; -// Express middleware +// Hono middleware export { loggingMiddleware, errorLoggingMiddleware, - createRequestLogger + createRequestLogger, + performanceMiddleware, + securityMiddleware, + businessEventMiddleware, + comprehensiveLoggingMiddleware } from './middleware'; // Type exports diff --git a/libs/logger/src/logger.ts b/libs/logger/src/logger.ts index 3b4a451..11b687e 100644 --- a/libs/logger/src/logger.ts +++ b/libs/logger/src/logger.ts @@ -1,23 +1,123 @@ /** - * Enhanced logger with Loki integration for Stock Bot platform + * Enhanced Pino-based logger with Loki integration for Stock Bot platform * * Features: - * - Multiple log levels (debug, info, warn, error) - * - Console and file logging + * - High performance JSON logging with Pino + * - Multiple log levels (debug, info, warn, error, http, verbose, silly) + * - Console and file logging with pino-pretty formatting * - Loki integration for centralized logging * - Structured logging with metadata - * - Performance optimized with batching + * - Flexible message handling (string or object) * - Service-specific context + * - TypeScript-friendly interfaces */ -import winston from 'winston'; -import LokiTransport from 'winston-loki'; -import DailyRotateFile from 'winston-daily-rotate-file'; +import pino from 'pino'; import { loggingConfig, lokiConfig } from '@stock-bot/config'; import type { LogLevel, LogContext, LogMetadata } from './types'; // Global logger instances cache -const loggerInstances = new Map(); +const loggerInstances = new Map(); + +// Pino log level mapping from string to number +const PINO_LEVELS: Record = { + silly: 10, + debug: 20, + verbose: 25, + http: 30, + info: 30, + warn: 40, + error: 50 +}; + +/** + * Create transport configuration for Pino based on options + */ +function createTransports(serviceName: string, options?: { + enableConsole?: boolean; + enableFile?: boolean; + enableLoki?: boolean; +}): any { + const { + enableConsole = loggingConfig.LOG_CONSOLE, + enableFile = loggingConfig.LOG_FILE, + enableLoki = true + } = options || {}; + + const targets: any[] = []; + + // Console transport with pretty formatting + if (enableConsole) { + targets.push({ + target: 'pino-pretty', + level: loggingConfig.LOG_LEVEL, + options: { + colorize: true, + translateTime: 'yyyy-mm-dd HH:MM:ss.l', + ignore: 'pid,hostname', + messageFormat: '[{service}] {msg}', + customPrettifiers: { + service: (service: string) => `${service}`, + level: (level: string) => `[${level.toUpperCase()}]` + } + } + }); + } + + // File transport for general logs + if (enableFile) { + targets.push({ + target: 'pino/file', + level: loggingConfig.LOG_LEVEL, + options: { + destination: `${loggingConfig.LOG_FILE_PATH}/${serviceName}.log`, + mkdir: true + } + }); + + // Separate error file if enabled + if (loggingConfig.LOG_ERROR_FILE) { + targets.push({ + target: 'pino/file', + level: 'error', + options: { + destination: `${loggingConfig.LOG_FILE_PATH}/${serviceName}-error.log`, + mkdir: true + } + }); + } + } + + // Loki transport for centralized logging + if (enableLoki && lokiConfig.LOKI_HOST) { + targets.push({ + target: 'pino-loki', + level: loggingConfig.LOG_LEVEL, + options: { + batching: true, + interval: lokiConfig.LOKI_FLUSH_INTERVAL_MS, + host: lokiConfig.LOKI_URL || `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`, + basicAuth: lokiConfig.LOKI_USERNAME && lokiConfig.LOKI_PASSWORD + ? { + username: lokiConfig.LOKI_USERNAME, + password: lokiConfig.LOKI_PASSWORD + } + : undefined, + labels: { + service: serviceName, + environment: lokiConfig.LOKI_ENVIRONMENT_LABEL, + ...(lokiConfig.LOKI_DEFAULT_LABELS ? JSON.parse(lokiConfig.LOKI_DEFAULT_LABELS) : {}) + }, + timeout: lokiConfig.LOKI_PUSH_TIMEOUT || 10000, + silenceErrors: false + } + }); + } + + return { + targets + }; +} /** * Create or retrieve a logger instance for a specific service @@ -27,7 +127,7 @@ export function createLogger(serviceName: string, options?: { enableLoki?: boolean; enableFile?: boolean; enableConsole?: boolean; -}): winston.Logger { +}): pino.Logger { const key = `${serviceName}-${JSON.stringify(options || {})}`; if (loggerInstances.has(key)) { @@ -41,124 +141,52 @@ export function createLogger(serviceName: string, options?: { } /** - * Build a winston logger with all configured transports + * Build a Pino logger with all configured transports */ function buildLogger(serviceName: string, options?: { level?: LogLevel; enableLoki?: boolean; enableFile?: boolean; enableConsole?: boolean; -}): winston.Logger { +}): pino.Logger { const { level = loggingConfig.LOG_LEVEL as LogLevel, enableLoki = true, enableFile = loggingConfig.LOG_FILE, enableConsole = loggingConfig.LOG_CONSOLE - } = options || {}; // Base logger configuration - const transports: winston.transport[] = []; + } = options || {}; - // Console transport - if (enableConsole) { - transports.push(new winston.transports.Console({ - format: winston.format.combine( - winston.format.colorize(), - winston.format.simple(), - winston.format.printf(({ timestamp, level, service, message, metadata }) => { - const meta = metadata && Object.keys(metadata).length > 0 - ? `\n${JSON.stringify(metadata, null, 2)}` - : ''; - return `${timestamp} [${level}] [${service}] ${message}${meta}`; - }) - ) - })); - } + const transport = createTransports(serviceName, { + enableConsole, + enableFile, + enableLoki + }); - // File transport with daily rotation - if (enableFile) { - // General log file - transports.push(new DailyRotateFile({ - filename: `${loggingConfig.LOG_FILE_PATH}/${serviceName}-%DATE%.log`, - datePattern: loggingConfig.LOG_FILE_DATE_PATTERN, - zippedArchive: true, - maxSize: loggingConfig.LOG_FILE_MAX_SIZE, - maxFiles: loggingConfig.LOG_FILE_MAX_FILES, - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json() - ) - })); - - // Separate error log file - if (loggingConfig.LOG_ERROR_FILE) { - transports.push(new DailyRotateFile({ - level: 'error', - filename: `${loggingConfig.LOG_FILE_PATH}/${serviceName}-error-%DATE%.log`, - datePattern: loggingConfig.LOG_FILE_DATE_PATTERN, - zippedArchive: true, - maxSize: loggingConfig.LOG_FILE_MAX_SIZE, - maxFiles: loggingConfig.LOG_FILE_MAX_FILES, - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json() - ) - })); - } - } - - // Loki transport for centralized logging - if (enableLoki && lokiConfig.LOKI_HOST) { - try { - const lokiTransport = new LokiTransport({ - host: lokiConfig.LOKI_URL || `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`, - labels: { - service: serviceName, - environment: lokiConfig.LOKI_ENVIRONMENT_LABEL, - ...(lokiConfig.LOKI_DEFAULT_LABELS ? JSON.parse(lokiConfig.LOKI_DEFAULT_LABELS) : {}) - }, - json: true, - batching: true, - interval: lokiConfig.LOKI_FLUSH_INTERVAL_MS, - timeout: lokiConfig.LOKI_PUSH_TIMEOUT, - basicAuth: lokiConfig.LOKI_USERNAME && lokiConfig.LOKI_PASSWORD - ? `${lokiConfig.LOKI_USERNAME}:${lokiConfig.LOKI_PASSWORD}` - : undefined, - onConnectionError: (err) => { - console.error('Loki connection error:', err); - } - }); - - transports.push(lokiTransport); - } catch (error) { - console.warn('Failed to initialize Loki transport:', error); - } - } - - const loggerConfig: winston.LoggerOptions = { - level, - format: winston.format.combine( - winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss.SSS' }), - winston.format.errors({ stack: true }), - winston.format.metadata({ - fillExcept: ['message', 'level', 'timestamp', 'service'] - }), - winston.format.json() - ), - defaultMeta: { + const loggerConfig: pino.LoggerOptions = { + level: PINO_LEVELS[level] ? level : 'info', + customLevels: PINO_LEVELS, + useOnlyCustomLevels: false, + timestamp: () => `,"timestamp":"${new Date().toISOString()}"`, + formatters: { + level: (label: string) => ({ level: label }), + bindings: () => ({}) + }, + base: { service: serviceName, environment: loggingConfig.LOG_ENVIRONMENT, version: loggingConfig.LOG_SERVICE_VERSION }, - transports + transport }; - return winston.createLogger(loggerConfig); + return pino(loggerConfig); } /** - * Enhanced Logger class with convenience methods + * Enhanced Logger class with convenience methods and flexible message handling */ export class Logger { - private winston: winston.Logger; + private pino: pino.Logger; private serviceName: string; private context: LogContext; @@ -170,34 +198,90 @@ export class Logger { }) { this.serviceName = serviceName; this.context = context; - this.winston = createLogger(serviceName, options); + this.pino = createLogger(serviceName, options); + } + /** + * Flexible log method that accepts string or object messages + */ + log(level: LogLevel, message: string | object, metadata?: LogMetadata): void { + const logData = { + ...this.context, + ...metadata, + timestamp: new Date().toISOString() + }; + + // Map custom log levels to Pino levels + const pinoLevel = this.mapToPinoLevel(level); + + if (typeof message === 'string') { + (this.pino as any)[pinoLevel](logData, message); + } else { + (this.pino as any)[pinoLevel]({ ...logData, ...message }); + } + } + + /** + * Map custom log levels to Pino levels + */ + private mapToPinoLevel(level: LogLevel): string { + switch (level) { + case 'silly': + case 'verbose': + case 'debug': + return 'debug'; + case 'http': + case 'info': + return 'info'; + case 'warn': + return 'warn'; + case 'error': + return 'error'; + default: + return 'info'; + } } /** * Debug level logging */ - debug(message: string, metadata?: LogMetadata): void { + debug(message: string | object, metadata?: LogMetadata): void { this.log('debug', message, metadata); } + /** + * Verbose level logging + */ + verbose(message: string | object, metadata?: LogMetadata): void { + // Map verbose to debug level since Pino doesn't have verbose by default + this.log('debug', message, { ...metadata, originalLevel: 'verbose' }); + } + + /** + * Silly level logging + */ + silly(message: string | object, metadata?: LogMetadata): void { + // Map silly to debug level since Pino doesn't have silly by default + this.log('debug', message, { ...metadata, originalLevel: 'silly' }); + } + /** * Info level logging */ - info(message: string, metadata?: LogMetadata): void { + info(message: string | object, metadata?: LogMetadata): void { this.log('info', message, metadata); } /** * Warning level logging */ - warn(message: string, metadata?: LogMetadata): void { + warn(message: string | object, metadata?: LogMetadata): void { this.log('warn', message, metadata); } /** * Error level logging */ - error(message: string, error?: Error | any, metadata?: LogMetadata): void { + error(message: string | object, error?: Error | any, metadata?: LogMetadata): void { const logData: LogMetadata = { ...metadata }; if (error) { @@ -218,7 +302,7 @@ export class Logger { /** * HTTP request logging */ - http(message: string, requestData?: { + http(message: string | object, requestData?: { method?: string; url?: string; statusCode?: number; @@ -228,7 +312,7 @@ export class Logger { }): void { if (!loggingConfig.LOG_HTTP_REQUESTS) return; - this.log('http', message, { + this.log('info', message, { request: requestData, type: 'http_request' }); @@ -237,7 +321,7 @@ export class Logger { /** * Performance/timing logging */ - performance(message: string, timing: { + performance(message: string | object, timing: { operation: string; duration: number; startTime?: number; @@ -254,7 +338,7 @@ export class Logger { /** * Business event logging */ - business(message: string, event: { + business(message: string | object, event: { type: string; entity?: string; action?: string; @@ -271,7 +355,7 @@ export class Logger { /** * Security event logging */ - security(message: string, event: { + security(message: string | object, event: { type: 'authentication' | 'authorization' | 'access' | 'vulnerability'; user?: string; resource?: string; @@ -301,33 +385,27 @@ export class Logger { } /** - * Internal logging method + * Get the underlying Pino logger */ - private log(level: LogLevel, message: string, metadata?: LogMetadata): void { - const logData = { - ...this.context, - ...metadata, - timestamp: new Date().toISOString() - }; - - this.winston.log(level, message, logData); + getPinoLogger(): pino.Logger { + return this.pino; } /** - * Get the underlying winston logger + * Compatibility method for existing code that expects Winston logger */ - getWinstonLogger(): winston.Logger { - return this.winston; + getWinstonLogger(): any { + console.warn('getWinstonLogger() is deprecated. Use getPinoLogger() instead.'); + return this.pino; } + /** * Gracefully close all transports */ async close(): Promise { - return new Promise((resolve) => { - this.winston.end(() => { - resolve(); - }); - }); + // Pino doesn't require explicit closing like Winston + // But we can flush any pending logs + this.pino.flush(); } } @@ -342,15 +420,13 @@ export function getLogger(serviceName: string, context?: LogContext): Logger { * Shutdown all logger instances gracefully */ export async function shutdownLoggers(): Promise { - const closePromises = Array.from(loggerInstances.values()).map(logger => - new Promise((resolve) => { - logger.end(() => { - resolve(); - }); - }) - ); + // Flush all logger instances + const flushPromises = Array.from(loggerInstances.values()).map(logger => { + logger.flush(); + return Promise.resolve(); + }); - await Promise.all(closePromises); + await Promise.all(flushPromises); loggerInstances.clear(); } diff --git a/libs/logger/src/middleware.ts b/libs/logger/src/middleware.ts index 35631e5..ce10e73 100644 --- a/libs/logger/src/middleware.ts +++ b/libs/logger/src/middleware.ts @@ -1,10 +1,10 @@ /** - * Hono middleware for request logging + * Comprehensive Hono middleware for request logging with Pino integration */ import type { Context, Next } from 'hono'; import { Logger } from './logger'; -import { generateCorrelationId, createTimer } from './utils'; +import { generateCorrelationId, createTimer, sanitizeMetadata, LogThrottle } from './utils'; export interface LoggingMiddlewareOptions { logger?: Logger; @@ -14,10 +14,18 @@ export interface LoggingMiddlewareOptions { logRequestBody?: boolean; logResponseBody?: boolean; maxBodySize?: number; + enablePerformanceMetrics?: boolean; + enableThrottling?: boolean; + throttleConfig?: { + maxLogs?: number; + windowMs?: number; + }; + sensitiveHeaders?: string[]; + redactSensitiveData?: boolean; } /** - * Hono middleware for HTTP request/response logging + * Hono middleware for HTTP request/response logging with enhanced features */ export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { const { @@ -26,11 +34,19 @@ export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { skipSuccessfulRequests = false, logRequestBody = false, logResponseBody = false, - maxBodySize = 1024 + maxBodySize = 1024, + enablePerformanceMetrics = true, + enableThrottling = false, + throttleConfig = { maxLogs: 100, windowMs: 60000 }, + sensitiveHeaders = ['authorization', 'cookie', 'x-api-key', 'x-auth-token'], + redactSensitiveData = true } = options; // Create logger if not provided const logger = options.logger || new Logger(serviceName); + + // Create throttle instance if enabled + const throttle = enableThrottling ? new LogThrottle(throttleConfig.maxLogs, throttleConfig.windowMs) : null; return async (c: Context, next: Next) => { const url = new URL(c.req.url); @@ -39,7 +55,15 @@ export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { // Skip certain paths if (skipPaths.some(skipPath => path.startsWith(skipPath))) { return next(); - } // Generate correlation ID + } + + // Check throttling + const throttleKey = `${c.req.method}-${path}`; + if (throttle && !throttle.shouldLog(throttleKey)) { + return next(); + } + + // Generate correlation ID const correlationId = generateCorrelationId(); // Store correlation ID in context for later use c.set('correlationId', correlationId); @@ -47,7 +71,23 @@ export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { c.header('x-correlation-id', correlationId); // Start timer - const timer = createTimer(`${c.req.method} ${path}`); // Extract request metadata + const timer = createTimer(`${c.req.method} ${path}`); + + // Extract request headers (sanitized) + const rawHeaders = Object.fromEntries( + Array.from(c.req.raw.headers.entries()) + ); + + const headers = redactSensitiveData + ? Object.fromEntries( + Object.entries(rawHeaders).map(([key, value]) => [ + key, + sensitiveHeaders.some(sensitive => key.toLowerCase().includes(sensitive.toLowerCase())) + ? '[REDACTED]' + : value + ]) + ) + : rawHeaders; // Extract request metadata following LogMetadata.request structure const requestMetadata: any = { method: c.req.method, url: c.req.url, @@ -55,9 +95,9 @@ export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { userAgent: c.req.header('user-agent'), contentType: c.req.header('content-type'), contentLength: c.req.header('content-length'), - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' + ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown', + headers: headers }; - // Add request body if enabled if (logRequestBody) { try { @@ -69,8 +109,9 @@ export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { } } catch (error) { // Body might not be available or already consumed + requestMetadata.bodyError = 'Unable to read request body'; } - } // Log request start + } // Log request start with structured data logger.http('HTTP Request started', { method: c.req.method, url: c.req.url, @@ -78,71 +119,103 @@ export function loggingMiddleware(options: LoggingMiddlewareOptions = {}) { ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' }); - // Process request - await next(); + let error: Error | null = null; + let responseTime: number = 0; - // Calculate response time - const timing = timer.end(); - - // Get response information - const response = c.res; - const status = response.status; - - // Determine log level based on status code - const isError = status >= 400; - const isSuccess = status >= 200 && status < 300; - - // Skip successful requests if configured - if (skipSuccessfulRequests && isSuccess) { - return; - } + try { + // Process request + await next(); + } catch (err) { + error = err instanceof Error ? err : new Error(String(err)); + throw error; // Re-throw to maintain error handling flow + } finally { + // Calculate response time + const timing = timer.end(); + responseTime = timing.duration; + + // Get response information + const response = c.res; + const status = response.status; + + // Determine log level based on status code + const isError = status >= 400 || error !== null; + const isSuccess = status >= 200 && status < 300; + const isRedirect = status >= 300 && status < 400; + + // Skip successful requests if configured + if (skipSuccessfulRequests && isSuccess && !error) { + return; + } - const responseMetadata: any = { - correlationId, - request: requestMetadata, - response: { + // Extract response headers (sanitized) + const rawResponseHeaders = Object.fromEntries( + Array.from(response.headers.entries()) + ); + + const responseHeaders = redactSensitiveData + ? Object.fromEntries( + Object.entries(rawResponseHeaders).map(([key, value]) => [ + key, + sensitiveHeaders.some(sensitive => key.toLowerCase().includes(sensitive.toLowerCase())) + ? '[REDACTED]' + : value + ]) + ) + : rawResponseHeaders; + + const responseMetadata: any = { statusCode: status, statusText: response.statusText, + headers: responseHeaders, contentLength: response.headers.get('content-length'), contentType: response.headers.get('content-type') - }, - performance: timing - }; + }; - // Add response body if enabled - if (logResponseBody) { - try { - const responseBody = await response.clone().text(); - if (responseBody) { - responseMetadata.response.body = responseBody.length > maxBodySize - ? responseBody.substring(0, maxBodySize) + '...[truncated]' - : responseBody; + // Add response body if enabled and not an error + if (logResponseBody && !isError) { + try { + const responseBody = await response.clone().text(); + if (responseBody) { + responseMetadata.body = responseBody.length > maxBodySize + ? responseBody.substring(0, maxBodySize) + '...[truncated]' + : responseBody; + } + } catch (bodyError) { + responseMetadata.bodyError = 'Unable to read response body'; } - } catch (error) { - // Response body might not be available - } - } // Log based on status code - if (isError) { - logger.error('HTTP Request failed', undefined, { + } // Performance metrics matching LogMetadata.performance structure + const performanceMetrics = { + operation: timing.operation, + duration: timing.duration, + startTime: enablePerformanceMetrics ? timing.startTime : undefined, + endTime: enablePerformanceMetrics ? timing.endTime : undefined + }; // Create comprehensive log entry + const logEntry: any = { correlationId, request: requestMetadata, - response: { + response: responseMetadata, + performance: performanceMetrics, + type: error ? 'custom' : isError ? 'custom' : 'http_request' + }; + + // Sanitize if needed + const finalLogEntry = redactSensitiveData ? sanitizeMetadata(logEntry) : logEntry; // Log based on status code and errors + if (error) { + logger.error('HTTP Request error', error, finalLogEntry); + } else if (isError) { + logger.warn('HTTP Request failed', finalLogEntry); + } else if (isRedirect) { + logger.info('HTTP Request redirected', finalLogEntry); + } else { + logger.http('HTTP Request completed', { + method: c.req.method, + url: c.req.url, statusCode: status, - statusText: response.statusText, - contentLength: response.headers.get('content-length'), - contentType: response.headers.get('content-type') - }, - performance: timing - }); - } else { - logger.http('HTTP Request completed', { - method: c.req.method, - url: c.req.url, - statusCode: status, - responseTime: timing.duration, - userAgent: c.req.header('user-agent'), - ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' - }); + responseTime: timing.duration, + userAgent: c.req.header('user-agent'), + ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' + }); + } } }; } @@ -198,3 +271,192 @@ export function createRequestLogger(c: Context, baseLogger: Logger): Logger { ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown' }); } + +/** + * Performance monitoring middleware for specific operations + */ +export function performanceMiddleware(operationName?: string, logger?: Logger) { + return async (c: Context, next: Next) => { + const perfLogger = logger || new Logger('performance'); + const operation = operationName || `${c.req.method} ${new URL(c.req.url).pathname}`; + const timer = createTimer(operation); + + try { + await next(); + + const timing = timer.end(); + perfLogger.info('Operation completed', { + type: 'performance', + performance: timing, + correlationId: c.get('correlationId') + }); + } catch (error) { + const timing = timer.end(); + perfLogger.warn('Operation failed', { + type: 'performance', + performance: timing, + correlationId: c.get('correlationId'), + error: error instanceof Error ? { + name: error.name, + message: error.message, + stack: error.stack + } : { message: String(error) } + }); + throw error; + } + }; +} + +/** + * Security event logging middleware + */ +export function securityMiddleware(logger?: Logger) { + return async (c: Context, next: Next) => { + const secLogger = logger || new Logger('security'); + const url = new URL(c.req.url); + + // Log authentication attempts + const authHeader = c.req.header('authorization'); + if (authHeader) { + secLogger.info('Authentication attempt', { + type: 'security_event', + security: { + type: 'authentication', + ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown', + resource: url.pathname, + action: 'access_attempt' + }, + correlationId: c.get('correlationId') + }); + } + + await next(); + + // Log access control + const status = c.res.status; + if (status === 401 || status === 403) { + secLogger.warn('Access denied', { + type: 'security_event', + security: { + type: 'authorization', + result: 'failure', + ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || 'unknown', + resource: url.pathname, + action: c.req.method, + severity: status === 401 ? 'medium' : 'high' + }, + correlationId: c.get('correlationId') + }); + } + }; +} + +/** + * Business event logging middleware for trading operations + */ +export function businessEventMiddleware(logger?: Logger) { + return async (c: Context, next: Next) => { + const bizLogger = logger || new Logger('business'); + const url = new URL(c.req.url); + const path = url.pathname; + + // Check if this is a business-critical endpoint + const businessEndpoints = [ + '/api/orders', + '/api/trades', + '/api/portfolio', + '/api/strategies', + '/api/signals' + ]; + + const isBusinessEndpoint = businessEndpoints.some(endpoint => path.startsWith(endpoint)); + + if (isBusinessEndpoint) { + const timer = createTimer(`business_${c.req.method}_${path}`); + + try { + await next(); + + const timing = timer.end(); + const status = c.res.status; + + bizLogger.info('Business operation completed', { + type: 'business_event', + business: { + type: 'trading_operation', + action: c.req.method, + result: status >= 200 && status < 300 ? 'success' : 'failure' + }, + performance: timing, + correlationId: c.get('correlationId') + }); + } catch (error) { + const timing = timer.end(); + + bizLogger.error('Business operation failed', error, { + type: 'business_event', + business: { + type: 'trading_operation', + action: c.req.method, + result: 'failure' + }, + performance: timing, + correlationId: c.get('correlationId') + }); + throw error; + } + } else { + await next(); + } + }; +} + +/** + * Comprehensive logging middleware that combines all logging features + */ +export function comprehensiveLoggingMiddleware(options: LoggingMiddlewareOptions & { + enableSecurity?: boolean; + enableBusiness?: boolean; + enablePerformance?: boolean; +} = {}) { + const { + enableSecurity = true, + enableBusiness = true, + enablePerformance = true, + ...loggingOptions + } = options; + return async (c: Context, next: Next) => { + const middlewares: Array<(c: Context, next: Next) => Promise> = []; + + // Add security middleware + if (enableSecurity) { + middlewares.push(securityMiddleware(options.logger)); + } + + // Add performance middleware + if (enablePerformance) { + middlewares.push(performanceMiddleware(undefined, options.logger)); + } + + // Add business event middleware + if (enableBusiness) { + middlewares.push(businessEventMiddleware(options.logger)); + } + + // Add main logging middleware + middlewares.push(loggingMiddleware(loggingOptions)); + + // Execute middleware chain + let index = 0; + async function dispatch(i: number): Promise { + if (i >= middlewares.length) { + return next(); + } + + const middleware = middlewares[i]; + return middleware(c, () => dispatch(i + 1)); + } + + return dispatch(0); + }; +} diff --git a/libs/types/src/data/index.ts b/libs/types/src/data/index.ts new file mode 100644 index 0000000..40c837e --- /dev/null +++ b/libs/types/src/data/index.ts @@ -0,0 +1,6 @@ +// Data Processing and Pipeline Types +export * from './sources'; +export * from './processors'; +export * from './pipelines'; +export * from './transformations'; +export * from './quality'; diff --git a/libs/types/src/data/pipelines.ts b/libs/types/src/data/pipelines.ts new file mode 100644 index 0000000..522c900 --- /dev/null +++ b/libs/types/src/data/pipelines.ts @@ -0,0 +1,181 @@ +import { AuthenticationConfig } from '../config/security'; +import { ProcessingError } from './processors'; + +// Data Pipeline Types +export interface ProcessingPipeline { + id: string; + name: string; + description?: string; + version: string; + processors: PipelineProcessor[]; + inputFilter: InputFilter; + outputTargets: OutputTargets; + schedule?: PipelineSchedule; + status: PipelineStatus; + metadata: PipelineMetadata; +} + +export interface PipelineProcessor { + processorId: string; + order: number; + config?: Record; + conditions?: ProcessingCondition[]; + parallelBranches?: PipelineProcessor[][]; +} + +export interface ProcessingCondition { + field: string; + operator: 'equals' | 'not_equals' | 'contains' | 'regex' | 'greater_than' | 'less_than'; + value: any; + logicalOperator?: 'AND' | 'OR'; +} + +export interface InputFilter { + symbols?: string[]; + sources?: string[]; + dataTypes?: string[]; + timeRange?: TimeRange; + conditions?: ProcessingCondition[]; +} + +export interface TimeRange { + start?: Date; + end?: Date; + duration?: string; + timezone?: string; +} + +export interface OutputTargets { + eventBus?: boolean; + database?: DatabaseTarget; + cache?: CacheTarget; + websocket?: WebSocketTarget; + dataProcessor?: ServiceTarget; + featureStore?: ServiceTarget; + file?: FileTarget; + api?: ApiTarget; +} + +export interface DatabaseTarget { + enabled: boolean; + connectionId: string; + table: string; + batchSize?: number; + flushInterval?: number; +} + +export interface CacheTarget { + enabled: boolean; + connectionId: string; + keyPattern: string; + ttl?: number; + compression?: boolean; +} + +export interface WebSocketTarget { + enabled: boolean; + rooms?: string[]; + filters?: Record; +} + +export interface ServiceTarget { + enabled: boolean; + serviceId: string; + endpoint: string; + batchSize?: number; + timeout?: number; +} + +export interface FileTarget { + enabled: boolean; + path: string; + format: 'json' | 'csv' | 'parquet' | 'avro'; + compression?: 'gzip' | 'bzip2' | 'lz4'; + rotation?: FileRotation; +} + +export interface FileRotation { + size?: string; + time?: string; + count?: number; +} + +export interface ApiTarget { + enabled: boolean; + url: string; + method: string; + headers?: Record; + authentication?: AuthenticationConfig; +} + +export interface PipelineSchedule { + enabled: boolean; + cronExpression?: string; + interval?: string; + timezone?: string; + startDate?: Date; + endDate?: Date; +} + +export enum PipelineStatus { + DRAFT = 'draft', + ACTIVE = 'active', + PAUSED = 'paused', + ERROR = 'error', + COMPLETED = 'completed', + ARCHIVED = 'archived' +} + +export interface PipelineMetadata { + createdBy: string; + createdAt: Date; + updatedBy: string; + updatedAt: Date; + tags: string[]; + documentation?: string; + changeLog?: ChangeLogEntry[]; +} + +export interface ChangeLogEntry { + version: string; + timestamp: Date; + author: string; + changes: string[]; + breaking: boolean; +} + +export interface PipelineExecution { + id: string; + pipelineId: string; + status: ExecutionStatus; + startTime: Date; + endTime?: Date; + recordsProcessed: number; + recordsSuccess: number; + recordsError: number; + errors: ProcessingError[]; + metadata: ExecutionMetadata; +} + +export enum ExecutionStatus { + PENDING = 'pending', + RUNNING = 'running', + COMPLETED = 'completed', + FAILED = 'failed', + CANCELLED = 'cancelled', + TIMEOUT = 'timeout' +} + +export interface ExecutionMetadata { + triggeredBy: 'schedule' | 'manual' | 'event'; + trigger?: string; + parameters?: Record; + resources?: ResourceUsage; +} + +export interface ResourceUsage { + cpuTimeMs: number; + memoryPeakMB: number; + diskIOBytes: number; + networkIOBytes: number; +} diff --git a/libs/types/src/data/processors.ts b/libs/types/src/data/processors.ts new file mode 100644 index 0000000..164e070 --- /dev/null +++ b/libs/types/src/data/processors.ts @@ -0,0 +1,103 @@ +// Data Processing Types +export interface DataProcessor { + id: string; + name: string; + type: ProcessorType; + enabled: boolean; + priority: number; + config: ProcessorConfig; + inputSchema?: DataSchema; + outputSchema?: DataSchema; + metrics: ProcessorMetrics; +} + +export enum ProcessorType { + ENRICHMENT = 'enrichment', + VALIDATION = 'validation', + NORMALIZATION = 'normalization', + AGGREGATION = 'aggregation', + FILTER = 'filter', + TRANSFORMATION = 'transformation', + QUALITY_CHECK = 'quality_check', + DEDUPLICATION = 'deduplication' +} + +export interface ProcessorConfig { + batchSize?: number; + timeoutMs?: number; + retryAttempts?: number; + parallelism?: number; + parameters?: Record; + errorHandling?: ErrorHandlingConfig; +} + +export interface ErrorHandlingConfig { + strategy: 'fail_fast' | 'skip_invalid' | 'retry' | 'quarantine'; + maxErrors?: number; + quarantineLocation?: string; + notificationChannels?: string[]; +} + +export interface ProcessorMetrics { + recordsProcessed: number; + recordsSuccess: number; + recordsError: number; + avgProcessingTimeMs: number; + lastProcessed: Date; + errorRate: number; + throughputPerSecond: number; +} + +export interface ProcessingError { + processorId: string; + pipelineId: string; + timestamp: Date; + inputData: any; + error: string; + stackTrace?: string; + severity: 'low' | 'medium' | 'high' | 'critical'; + recoverable: boolean; +} + +export interface DataSchema { + version: string; + fields: SchemaField[]; + primaryKeys?: string[]; + foreignKeys?: ForeignKey[]; + indexes?: Index[]; + constraints?: SchemaConstraint[]; +} + +export interface SchemaField { + name: string; + type: string; + nullable: boolean; + description?: string; + defaultValue?: any; + format?: string; + pattern?: string; + enum?: any[]; +} + +export interface ForeignKey { + fields: string[]; + referencedTable: string; + referencedFields: string[]; + onDelete?: 'cascade' | 'restrict' | 'set_null'; + onUpdate?: 'cascade' | 'restrict' | 'set_null'; +} + +export interface Index { + name: string; + fields: string[]; + unique: boolean; + type: 'btree' | 'hash' | 'gin' | 'gist'; +} + +export interface SchemaConstraint { + name: string; + type: 'check' | 'unique' | 'not_null' | 'range'; + fields: string[]; + condition?: string; + parameters?: Record; +} diff --git a/libs/types/src/data/quality.ts b/libs/types/src/data/quality.ts new file mode 100644 index 0000000..aeffa4d --- /dev/null +++ b/libs/types/src/data/quality.ts @@ -0,0 +1,193 @@ +// Data Quality Types +export interface DataQuality { + id: string; + assetId: string; + overallScore: number; + dimensions: QualityDimension[]; + rules: QualityRule[]; + issues: QualityIssue[]; + trend: QualityTrend; + lastAssessment: Date; + nextAssessment?: Date; +} + +export interface QualityDimension { + name: QualityDimensionType; + score: number; + weight: number; + description: string; + metrics: QualityMetric[]; + status: 'pass' | 'warn' | 'fail'; +} + +export enum QualityDimensionType { + COMPLETENESS = 'completeness', + ACCURACY = 'accuracy', + CONSISTENCY = 'consistency', + VALIDITY = 'validity', + UNIQUENESS = 'uniqueness', + TIMELINESS = 'timeliness', + INTEGRITY = 'integrity', + CONFORMITY = 'conformity', + RELEVANCE = 'relevance' +} + +export interface QualityRule { + id: string; + name: string; + description: string; + dimension: QualityDimensionType; + type: QualityRuleType; + field?: string; + condition: string; + threshold: number; + severity: 'low' | 'medium' | 'high' | 'critical'; + enabled: boolean; + metadata?: Record; +} + +export enum QualityRuleType { + NULL_CHECK = 'null_check', + RANGE_CHECK = 'range_check', + PATTERN_CHECK = 'pattern_check', + REFERENCE_CHECK = 'reference_check', + DUPLICATE_CHECK = 'duplicate_check', + FRESHNESS_CHECK = 'freshness_check', + OUTLIER_CHECK = 'outlier_check', + CUSTOM = 'custom' +} + +export interface QualityMetric { + name: string; + value: number; + unit?: string; + threshold?: number; + status: 'pass' | 'warn' | 'fail'; + timestamp: Date; + trend?: 'improving' | 'stable' | 'degrading'; +} + +export interface QualityIssue { + id: string; + ruleId: string; + severity: 'low' | 'medium' | 'high' | 'critical'; + description: string; + field?: string; + affectedRows?: number; + sampleValues?: any[]; + detectedAt: Date; + status: 'open' | 'acknowledged' | 'resolved' | 'false_positive'; + assignee?: string; + resolution?: string; + resolvedAt?: Date; + impact?: QualityImpact; +} + +export interface QualityImpact { + scope: 'field' | 'table' | 'dataset' | 'system'; + affectedRecords: number; + downstreamAssets: string[]; + businessImpact: 'low' | 'medium' | 'high' | 'critical'; + estimatedCost?: number; +} + +export interface QualityTrend { + timeframe: 'day' | 'week' | 'month' | 'quarter'; + dataPoints: QualityDataPoint[]; + trend: 'improving' | 'stable' | 'degrading'; + changeRate: number; + projectedScore?: number; +} + +export interface QualityDataPoint { + timestamp: Date; + score: number; + dimensionScores: Record; + issueCount: number; + rulesPassed: number; + rulesTotal: number; +} + +export interface QualityProfile { + assetId: string; + fieldProfiles: FieldProfile[]; + rowCount: number; + columnCount: number; + dataTypes: Record; + nullCounts: Record; + uniqueCounts: Record; + profiledAt: Date; +} + +export interface FieldProfile { + fieldName: string; + dataType: string; + nullCount: number; + uniqueCount: number; + minValue?: any; + maxValue?: any; + avgValue?: number; + medianValue?: number; + standardDeviation?: number; + topValues?: ValueFrequency[]; + histogram?: HistogramBucket[]; +} + +export interface ValueFrequency { + value: any; + count: number; + percentage: number; +} + +export interface HistogramBucket { + min: number; + max: number; + count: number; + percentage: number; +} + +export interface QualityReport { + id: string; + assetId: string; + reportType: 'summary' | 'detailed' | 'trend' | 'comparison'; + generatedAt: Date; + period: string; + summary: QualitySummary; + details?: QualityDetails; + recommendations?: QualityRecommendation[]; +} + +export interface QualitySummary { + overallScore: number; + scoreTrend: 'improving' | 'stable' | 'degrading'; + totalIssues: number; + criticalIssues: number; + resolvedIssues: number; + rulesPassed: number; + rulesTotal: number; +} + +export interface QualityDetails { + dimensionBreakdown: Record; + topIssues: QualityIssue[]; + ruleResults: QualityRuleResult[]; + historicalTrend: QualityDataPoint[]; +} + +export interface QualityRuleResult { + ruleId: string; + ruleName: string; + status: 'pass' | 'fail'; + score: number; + violations: number; + executionTime: number; +} + +export interface QualityRecommendation { + type: 'rule_adjustment' | 'data_cleanup' | 'process_improvement' | 'monitoring'; + priority: 'low' | 'medium' | 'high'; + description: string; + impact: string; + effort: string; + steps: string[]; +} diff --git a/libs/types/src/data/sources.ts b/libs/types/src/data/sources.ts new file mode 100644 index 0000000..5c815a1 --- /dev/null +++ b/libs/types/src/data/sources.ts @@ -0,0 +1,84 @@ +// Data Source Configuration Types +export interface DataSourceConfig { + id: string; + name: string; + type: 'websocket' | 'rest' | 'fix' | 'stream' | 'file' | 'database'; + enabled: boolean; + priority: number; + rateLimit: DataRateLimitConfig; + connection: DataConnectionConfig; + subscriptions: SubscriptionConfig; + symbols: string[]; + retryPolicy: RetryPolicyConfig; + healthCheck: HealthCheckConfig; + metadata?: Record; +} + +export interface DataRateLimitConfig { + requestsPerSecond: number; + burstLimit: number; + windowMs?: number; +} + +export interface DataConnectionConfig { + url: string; + headers?: Record; + queryParams?: Record; + authentication?: DataAuthenticationConfig; + timeout?: number; + keepAlive?: boolean; + compression?: boolean; +} + +export interface DataAuthenticationConfig { + type: 'apikey' | 'oauth' | 'basic' | 'jwt' | 'cert'; + credentials: Record; + refreshInterval?: number; +} + +export interface SubscriptionConfig { + quotes: boolean; + trades: boolean; + orderbook: boolean; + candles: boolean; + news: boolean; + fundamentals?: boolean; + options?: boolean; + futures?: boolean; +} + +export interface RetryPolicyConfig { + maxRetries: number; + backoffMultiplier: number; + maxBackoffMs: number; + enableJitter?: boolean; +} + +export interface HealthCheckConfig { + intervalMs: number; + timeoutMs: number; + expectedLatencyMs: number; + failureThreshold?: number; +} + +export interface DataSourceStatus { + id: string; + status: 'connected' | 'disconnected' | 'connecting' | 'error'; + lastUpdate: Date; + connectionTime?: Date; + disconnectionTime?: Date; + errorCount: number; + messagesReceived: number; + latencyMs: number; + throughputPerSecond: number; +} + +export interface DataSourceError { + sourceId: string; + timestamp: Date; + type: 'connection' | 'authentication' | 'ratelimit' | 'data' | 'timeout' | 'parsing'; + message: string; + details?: Record; + severity: 'low' | 'medium' | 'high' | 'critical'; + recoverable: boolean; +} diff --git a/libs/types/src/data/transformations.ts b/libs/types/src/data/transformations.ts new file mode 100644 index 0000000..8dcf20f --- /dev/null +++ b/libs/types/src/data/transformations.ts @@ -0,0 +1,119 @@ +// Data Transformation Types +export interface DataTransformation { + id: string; + name: string; + type: TransformationType; + description?: string; + code?: string; + inputFields: string[]; + outputFields: string[]; + logic: string; + parameters?: Record; + validationRules?: ValidationRule[]; +} + +export enum TransformationType { + FILTER = 'filter', + AGGREGATE = 'aggregate', + JOIN = 'join', + UNION = 'union', + PIVOT = 'pivot', + UNPIVOT = 'unpivot', + SORT = 'sort', + DEDUPLICATE = 'deduplicate', + CALCULATE = 'calculate', + CAST = 'cast', + RENAME = 'rename', + SPLIT = 'split', + MERGE = 'merge', + NORMALIZE = 'normalize', + ENRICH = 'enrich' +} + +export interface ValidationRule { + field: string; + type: ValidationType; + parameters?: Record; + errorMessage?: string; + severity: 'warning' | 'error' | 'critical'; +} + +export enum ValidationType { + REQUIRED = 'required', + TYPE_CHECK = 'type_check', + RANGE = 'range', + PATTERN = 'pattern', + LENGTH = 'length', + UNIQUE = 'unique', + REFERENCE = 'reference', + CUSTOM = 'custom' +} + +export interface TransformationRule { + id: string; + sourceField: string; + targetField: string; + transformation: string; + condition?: string; + parameters?: Record; +} + +export interface MappingRule { + sourceField: string; + targetField: string; + transformation?: string; + defaultValue?: any; + required: boolean; +} + +export interface AggregationRule { + groupByFields: string[]; + aggregations: AggregationFunction[]; + having?: string; + windowFunction?: WindowFunction; +} + +export interface AggregationFunction { + function: 'sum' | 'avg' | 'count' | 'min' | 'max' | 'first' | 'last' | 'stddev' | 'variance'; + field: string; + alias?: string; + distinct?: boolean; +} + +export interface WindowFunction { + type: 'row_number' | 'rank' | 'dense_rank' | 'lag' | 'lead' | 'first_value' | 'last_value'; + partitionBy?: string[]; + orderBy?: OrderByClause[]; + frame?: WindowFrame; +} + +export interface OrderByClause { + field: string; + direction: 'asc' | 'desc'; + nulls?: 'first' | 'last'; +} + +export interface WindowFrame { + type: 'rows' | 'range'; + start: FrameBoundary; + end: FrameBoundary; +} + +export interface FrameBoundary { + type: 'unbounded_preceding' | 'current_row' | 'unbounded_following' | 'preceding' | 'following'; + offset?: number; +} + +export interface JoinRule { + type: 'inner' | 'left' | 'right' | 'full' | 'cross'; + leftTable: string; + rightTable: string; + joinConditions: JoinCondition[]; + selectFields?: string[]; +} + +export interface JoinCondition { + leftField: string; + rightField: string; + operator: '=' | '<>' | '<' | '>' | '<=' | '>='; +} diff --git a/scripts/create-package-aliases.ps1 b/scripts/create-package-aliases.ps1 deleted file mode 100644 index 7b6059a..0000000 --- a/scripts/create-package-aliases.ps1 +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env pwsh - -# Migration script for moving from packages/ to libs/ -# This script helps maintain backward compatibility during the transition - -$ErrorActionPreference = "Stop" -$PackagesDir = "g:\repos\stock-bot\packages" -$LibsDir = "g:\repos\stock-bot\libs" - -function Write-ColorOutput($ForegroundColor) { - $fc = $host.UI.RawUI.ForegroundColor - $host.UI.RawUI.ForegroundColor = $ForegroundColor - if ($args) { - Write-Output $args - } - $host.UI.RawUI.ForegroundColor = $fc -} - -function Create-PackageAlias { - param ( - [string]$PackageName - ) - - $PackageDir = Join-Path $PackagesDir $PackageName - $LibDir = Join-Path $LibsDir $PackageName - - # Check if the lib already exists - if (-not (Test-Path $LibDir)) { - Write-ColorOutput "Red" "Error: Library $PackageName not found in $LibsDir" - return - } - - # Create the package directory if it doesn't exist - if (-not (Test-Path $PackageDir)) { - New-Item -ItemType Directory -Force -Path $PackageDir | Out-Null - Write-ColorOutput "Green" "Created directory: $PackageDir" - } - - # Create a package.json that points to the lib - $PackageJsonPath = Join-Path $PackageDir "package.json" - $PackageJson = @{ - name = "@stock-bot/$PackageName" - version = "1.0.0" - description = "Alias package that points to the new library location" - main = "../../libs/$PackageName/dist/index.js" - types = "../../libs/$PackageName/dist/index.d.ts" - files = @("README.md") - } - - $JsonContent = $PackageJson | ConvertTo-Json -Depth 10 - Set-Content -Path $PackageJsonPath -Value $JsonContent - Write-ColorOutput "Green" "Created alias package.json: $PackageJsonPath" - - # Create a README explaining what this is - $ReadmePath = Join-Path $PackageDir "README.md" - $ReadmeContent = @" -# $PackageName (Deprecated Path) - -This package is now maintained in the \`libs/$PackageName\` directory. - -This is a compatibility alias that points to the new location. - -## Migration - -Please update your imports to use the new path: - -\`\`\`diff -- import { ... } from '@stock-bot/$PackageName' -+ import { ... } from '@stock-bot/$PackageName' -\`\`\` - -You don't need to change your imports yet, but this path will be removed in a future version. -"@ - - Set-Content -Path $ReadmePath -Value $ReadmeContent - Write-ColorOutput "Green" "Created README: $ReadmePath" -} - -# Create aliases for each library -Write-ColorOutput "Yellow" "Creating package aliases for backward compatibility..." -Create-PackageAlias "types" - -# Create additional aliases as needed for other libraries - -Write-ColorOutput "Blue" "โœ“ Aliases created successfully. Importing from either packages/ or libs/ will work." -Write-ColorOutput "Blue" " Please prefer importing from libs/ in new code." diff --git a/tools/check-loki-status.bat b/tools/check-loki-status.bat deleted file mode 100644 index aead9dc..0000000 --- a/tools/check-loki-status.bat +++ /dev/null @@ -1,27 +0,0 @@ -@echo off -REM Check Loki status and accessibility - -echo Checking Loki container status... -docker ps | findstr "loki" - -echo. -echo Testing Loki API... -curl -s http://localhost:3100/ready -echo. - -echo. -echo Testing Loki labels... -curl -s "http://localhost:3100/loki/api/v1/labels" | findstr /C:"service" -echo. - -echo. -echo Checking Grafana... -curl -s http://localhost:3000/api/health | findstr /C:"database" -echo. - -echo. -echo To view logs in Grafana: -echo 1. Open http://localhost:3000 in your browser -echo 2. Login with admin/admin -echo 3. Go to Explore and select Loki as the datasource -echo 4. Try the query: {service=~".+"} diff --git a/tools/test-loki-logging.ts b/tools/test-loki-logging.ts deleted file mode 100644 index 9120851..0000000 --- a/tools/test-loki-logging.ts +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bun - -/** - * Test script to verify the Loki logging integration - */ -import { Logger, LogLevel } from '@stock-bot/utils'; - -// Create a logger for testing -const logger = new Logger('test-service', LogLevel.DEBUG); - -// Log test messages -logger.info('Starting test log messages...'); -logger.debug('This is a DEBUG level message'); -logger.info('This is an INFO level message'); -logger.warn('This is a WARNING level message'); -logger.error('This is an ERROR level message'); - -// Add some structured data -logger.info('Processing trade', { symbol: 'AAPL', price: 190.50, quantity: 100 }); -logger.info('Processing trade', { symbol: 'MSFT', price: 410.75, quantity: 50 }); - -// Simulate an error -try { - throw new Error('This is a simulated error'); -} catch (error) { - logger.error('An error occurred', error); -} - -logger.info('Test log messages complete. Check Grafana at http://localhost:3000 to view logs.'); - -// Wait to ensure all logs are sent -setTimeout(() => { - process.exit(0); -}, 1000);