added initial integration tests with bun

libs/mongodb-client/README.md

@@ -0,0 +1,72 @@
+# MongoDB Client Library
+
+A comprehensive MongoDB client library for the Stock Bot trading platform, designed for handling document storage, raw data, and unstructured content.
+
+## Features
+
+- **Connection Management**: Robust connection pooling and failover
+- **Schema Validation**: Built-in validation using Zod schemas
+- **Type Safety**: Full TypeScript support with typed collections
+- **Error Handling**: Comprehensive error handling and retry logic
+- **Health Monitoring**: Connection health monitoring and metrics
+- **Transactions**: Support for multi-document transactions
+- **Aggregation**: Helper methods for complex aggregation pipelines
+
+## Usage
+
+```typescript
+import { MongoDBClient } from '@stock-bot/mongodb-client';
+
+// Initialize client
+const mongoClient = new MongoDBClient();
+await mongoClient.connect();
+
+// Get a typed collection
+const collection = mongoClient.getCollection('sentiment_data');
+
+// Insert document
+await collection.insertOne({
+  symbol: 'AAPL',
+  sentiment: 'positive',
+  source: 'reddit',
+  timestamp: new Date()
+});
+
+// Query with aggregation
+const results = await collection.aggregate([
+  { $match: { symbol: 'AAPL' } },
+  { $group: { _id: '$sentiment', count: { $sum: 1 } } }
+]);
+```
+
+## Collections
+
+The client provides typed access to the following collections:
+
+- **sentiment_data**: Social media sentiment analysis
+- **raw_documents**: Unprocessed documents and content
+- **news_articles**: Financial news and articles
+- **sec_filings**: SEC filing documents
+- **earnings_transcripts**: Earnings call transcripts
+- **analyst_reports**: Research reports and analysis
+
+## Configuration
+
+Configure using environment variables:
+
+```env
+MONGODB_HOST=localhost
+MONGODB_PORT=27017
+MONGODB_DATABASE=trading_documents
+MONGODB_USERNAME=trading_admin
+MONGODB_PASSWORD=your_password
+```
+
+## Health Monitoring
+
+The client includes built-in health monitoring:
+
+```typescript
+const health = await mongoClient.getHealth();
+console.log(health.status); // 'healthy' | 'degraded' | 'unhealthy'
+```

libs/mongodb-client/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@stock-bot/mongodb-client",
+  "version": "1.0.0",
+  "description": "MongoDB client library for Stock Bot platform",
+  "main": "src/index.ts",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "test": "bun test",
+    "lint": "eslint src/**/*.ts",
+    "type-check": "tsc --noEmit",
+    "dev": "tsc --watch"
+  },
+  "dependencies": {
+    "@stock-bot/config": "*",
+    "@stock-bot/logger": "*",
+    "@stock-bot/types": "*",
+    "mongodb": "^6.3.0",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.0",
+    "typescript": "^5.3.0",
+    "eslint": "^8.56.0",
+    "@typescript-eslint/eslint-plugin": "^6.19.0",
+    "@typescript-eslint/parser": "^6.19.0",
+    "bun-types": "^1.2.15"
+  },
+  "keywords": [
+    "mongodb",
+    "database",
+    "client",
+    "stock-bot"
+  ],
+  "exports": {
+    ".": {
+      "import": "./src/index.ts",
+      "require": "./dist/index.js"
+    }
+  }
+}

libs/mongodb-client/src/aggregation.ts

@@ -0,0 +1,247 @@
+import type { MongoDBClient } from './client';
+import type { CollectionNames } from './types';
+
+/**
+ * MongoDB Aggregation Builder
+ * 
+ * Provides a fluent interface for building MongoDB aggregation pipelines
+ */
+export class MongoDBAggregationBuilder {
+  private pipeline: any[] = [];
+  private readonly client: MongoDBClient;
+  private collection: CollectionNames | null = null;
+
+  constructor(client: MongoDBClient) {
+    this.client = client;
+  }
+
+  /**
+   * Set the collection to aggregate on
+   */
+  from(collection: CollectionNames): this {
+    this.collection = collection;
+    return this;
+  }
+
+  /**
+   * Add a match stage
+   */
+  match(filter: any): this {
+    this.pipeline.push({ $match: filter });
+    return this;
+  }
+
+  /**
+   * Add a group stage
+   */
+  group(groupBy: any): this {
+    this.pipeline.push({ $group: groupBy });
+    return this;
+  }
+
+  /**
+   * Add a sort stage
+   */
+  sort(sortBy: any): this {
+    this.pipeline.push({ $sort: sortBy });
+    return this;
+  }
+
+  /**
+   * Add a limit stage
+   */
+  limit(count: number): this {
+    this.pipeline.push({ $limit: count });
+    return this;
+  }
+
+  /**
+   * Add a skip stage
+   */
+  skip(count: number): this {
+    this.pipeline.push({ $skip: count });
+    return this;
+  }
+
+  /**
+   * Add a project stage
+   */
+  project(projection: any): this {
+    this.pipeline.push({ $project: projection });
+    return this;
+  }
+
+  /**
+   * Add an unwind stage
+   */
+  unwind(field: string, options?: any): this {
+    this.pipeline.push({ 
+      $unwind: options ? { path: field, ...options } : field 
+    });
+    return this;
+  }
+
+  /**
+   * Add a lookup stage (join)
+   */
+  lookup(from: string, localField: string, foreignField: string, as: string): this {
+    this.pipeline.push({
+      $lookup: {
+        from,
+        localField,
+        foreignField,
+        as
+      }
+    });
+    return this;
+  }
+
+  /**
+   * Add a custom stage
+   */
+  addStage(stage: any): this {
+    this.pipeline.push(stage);
+    return this;
+  }
+
+  /**
+   * Execute the aggregation pipeline
+   */
+  async execute<T = any>(): Promise<T[]> {
+    if (!this.collection) {
+      throw new Error('Collection not specified. Use .from() to set the collection.');
+    }
+
+    const collection = this.client.getCollection(this.collection);
+    return await collection.aggregate<T>(this.pipeline).toArray();
+  }
+
+  /**
+   * Get the pipeline array
+   */
+  getPipeline(): any[] {
+    return [...this.pipeline];
+  }
+
+  /**
+   * Reset the pipeline
+   */
+  reset(): this {
+    this.pipeline = [];
+    this.collection = null;
+    return this;
+  }
+
+  // Convenience methods for common aggregations
+
+  /**
+   * Sentiment analysis aggregation
+   */
+  sentimentAnalysis(symbol?: string, timeframe?: { start: Date; end: Date }): this {
+    this.from('sentiment_data');
+    
+    const matchConditions: any = {};
+    if (symbol) matchConditions.symbol = symbol;
+    if (timeframe) {
+      matchConditions.timestamp = {
+        $gte: timeframe.start,
+        $lte: timeframe.end
+      };
+    }
+    
+    if (Object.keys(matchConditions).length > 0) {
+      this.match(matchConditions);
+    }
+
+    return this.group({
+      _id: {
+        symbol: '$symbol',
+        sentiment: '$sentiment_label'
+      },
+      count: { $sum: 1 },
+      avgScore: { $avg: '$sentiment_score' },
+      avgConfidence: { $avg: '$confidence' }
+    });
+  }
+
+  /**
+   * News article aggregation by publication
+   */
+  newsByPublication(symbols?: string[]): this {
+    this.from('news_articles');
+    
+    if (symbols && symbols.length > 0) {
+      this.match({ symbols: { $in: symbols } });
+    }
+
+    return this.group({
+      _id: '$publication',
+      articleCount: { $sum: 1 },
+      symbols: { $addToSet: '$symbols' },
+      avgSentiment: { $avg: '$sentiment_score' },
+      latestArticle: { $max: '$published_date' }
+    });
+  }
+
+  /**
+   * SEC filings by company
+   */
+  secFilingsByCompany(filingTypes?: string[]): this {
+    this.from('sec_filings');
+    
+    if (filingTypes && filingTypes.length > 0) {
+      this.match({ filing_type: { $in: filingTypes } });
+    }
+
+    return this.group({
+      _id: {
+        cik: '$cik',
+        company: '$company_name'
+      },
+      filingCount: { $sum: 1 },
+      filingTypes: { $addToSet: '$filing_type' },
+      latestFiling: { $max: '$filing_date' },
+      symbols: { $addToSet: '$symbols' }
+    });
+  }
+
+  /**
+   * Document processing status summary
+   */
+  processingStatusSummary(collection: CollectionNames): this {
+    this.from(collection);
+    
+    return this.group({
+      _id: '$processing_status',
+      count: { $sum: 1 },
+      avgSizeBytes: { $avg: '$size_bytes' },
+      oldestDocument: { $min: '$created_at' },
+      newestDocument: { $max: '$created_at' }
+    });
+  }
+
+  /**
+   * Time-based aggregation (daily/hourly counts)
+   */
+  timeBasedCounts(
+    collection: CollectionNames,
+    dateField: string = 'created_at',
+    interval: 'hour' | 'day' | 'week' | 'month' = 'day'
+  ): this {
+    this.from(collection);
+
+    const dateFormat = {
+      hour: { $dateToString: { format: '%Y-%m-%d %H:00:00', date: `$${dateField}` } },
+      day: { $dateToString: { format: '%Y-%m-%d', date: `$${dateField}` } },
+      week: { $dateToString: { format: '%Y-W%V', date: `$${dateField}` } },
+      month: { $dateToString: { format: '%Y-%m', date: `$${dateField}` } }
+    };
+
+    return this.group({
+      _id: dateFormat[interval],
+      count: { $sum: 1 },
+      firstDocument: { $min: `$${dateField}` },
+      lastDocument: { $max: `$${dateField}` }
+    }).sort({ _id: 1 });
+  }
+}

libs/mongodb-client/src/client.ts

@@ -0,0 +1,380 @@
+import { MongoClient, Db, Collection, MongoClientOptions } from 'mongodb';
+import { mongodbConfig } from '@stock-bot/config';
+import { Logger } from '@stock-bot/logger';
+import type {
+  MongoDBClientConfig,
+  MongoDBConnectionOptions,
+  CollectionNames,
+  DocumentBase,
+  SentimentData,
+  RawDocument,
+  NewsArticle,
+  SecFiling,
+  EarningsTranscript,
+  AnalystReport
+} from './types';
+import { MongoDBHealthMonitor } from './health';
+import { schemaMap } from './schemas';
+import { z } from 'zod';
+
+/**
+ * MongoDB Client for Stock Bot
+ * 
+ * Provides type-safe access to MongoDB collections with built-in
+ * health monitoring, connection pooling, and schema validation.
+ */
+export class MongoDBClient {
+  private client: MongoClient | null = null;
+  private db: Db | null = null;
+  private readonly config: MongoDBClientConfig;
+  private readonly options: MongoDBConnectionOptions;
+  private readonly logger: Logger;
+  private readonly healthMonitor: MongoDBHealthMonitor;
+  private isConnected = false;
+
+  constructor(
+    config?: Partial<MongoDBClientConfig>,
+    options?: MongoDBConnectionOptions
+  ) {
+    this.config = this.buildConfig(config);
+    this.options = {
+      retryAttempts: 3,
+      retryDelay: 1000,
+      healthCheckInterval: 30000,
+      ...options
+    };
+    
+    this.logger = new Logger('MongoDBClient');
+    this.healthMonitor = new MongoDBHealthMonitor(this);
+  }
+
+  /**
+   * Connect to MongoDB
+   */
+  async connect(): Promise<void> {
+    if (this.isConnected && this.client) {
+      return;
+    }
+
+    const uri = this.buildConnectionUri();
+    const clientOptions = this.buildClientOptions();
+
+    let lastError: Error | null = null;
+    
+    for (let attempt = 1; attempt <= this.options.retryAttempts!; attempt++) {
+      try {
+        this.logger.info(`Connecting to MongoDB (attempt ${attempt}/${this.options.retryAttempts})...`);
+        
+        this.client = new MongoClient(uri, clientOptions);
+        await this.client.connect();
+        
+        // Test the connection
+        await this.client.db(this.config.database).admin().ping();
+        
+        this.db = this.client.db(this.config.database);
+        this.isConnected = true;
+        
+        this.logger.info('Successfully connected to MongoDB');
+        
+        // Start health monitoring
+        this.healthMonitor.start();
+        
+        return;
+      } catch (error) {
+        lastError = error as Error;
+        this.logger.error(`MongoDB connection attempt ${attempt} failed:`, error);
+        
+        if (this.client) {
+          await this.client.close();
+          this.client = null;
+        }
+        
+        if (attempt < this.options.retryAttempts!) {
+          await this.delay(this.options.retryDelay! * attempt);
+        }
+      }
+    }
+
+    throw new Error(`Failed to connect to MongoDB after ${this.options.retryAttempts} attempts: ${lastError?.message}`);
+  }
+
+  /**
+   * Disconnect from MongoDB
+   */
+  async disconnect(): Promise<void> {
+    if (!this.client) {
+      return;
+    }
+
+    try {
+      this.healthMonitor.stop();
+      await this.client.close();
+      this.isConnected = false;
+      this.client = null;
+      this.db = null;
+      this.logger.info('Disconnected from MongoDB');
+    } catch (error) {
+      this.logger.error('Error disconnecting from MongoDB:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Get a typed collection
+   */
+  getCollection<T extends DocumentBase>(name: CollectionNames): Collection<T> {
+    if (!this.db) {
+      throw new Error('MongoDB client not connected');
+    }
+    return this.db.collection<T>(name);
+  }
+
+  /**
+   * Insert a document with validation
+   */
+  async insertOne<T extends DocumentBase>(
+    collectionName: CollectionNames,
+    document: Omit<T, '_id' | 'created_at' | 'updated_at'> & Partial<Pick<T, 'created_at' | 'updated_at'>>
+  ): Promise<T> {
+    const collection = this.getCollection<T>(collectionName);
+    
+    // Add timestamps
+    const now = new Date();
+    const docWithTimestamps = {
+      ...document,
+      created_at: document.created_at || now,
+      updated_at: now
+    } as T;
+
+    // Validate document if schema exists
+    if (schemaMap[collectionName]) {
+      try {
+        schemaMap[collectionName].parse(docWithTimestamps);
+      } catch (error) {
+        if (error instanceof z.ZodError) {
+          this.logger.error(`Document validation failed for ${collectionName}:`, error.errors);
+          throw new Error(`Document validation failed: ${error.errors.map(e => e.message).join(', ')}`);
+        }
+        throw error;
+      }
+    }
+
+    const result = await collection.insertOne(docWithTimestamps);
+    return { ...docWithTimestamps, _id: result.insertedId } as T;
+  }
+
+  /**
+   * Update a document with validation
+   */
+  async updateOne<T extends DocumentBase>(
+    collectionName: CollectionNames,
+    filter: any,
+    update: Partial<T>
+  ): Promise<boolean> {
+    const collection = this.getCollection<T>(collectionName);
+    
+    // Add updated timestamp
+    const updateWithTimestamp = {
+      ...update,
+      updated_at: new Date()
+    };
+
+    const result = await collection.updateOne(filter, { $set: updateWithTimestamp });
+    return result.modifiedCount > 0;
+  }
+
+  /**
+   * Find documents with optional validation
+   */
+  async find<T extends DocumentBase>(
+    collectionName: CollectionNames,
+    filter: any = {},
+    options: any = {}
+  ): Promise<T[]> {
+    const collection = this.getCollection<T>(collectionName);
+    return await collection.find(filter, options).toArray();
+  }
+
+  /**
+   * Find one document
+   */
+  async findOne<T extends DocumentBase>(
+    collectionName: CollectionNames,
+    filter: any
+  ): Promise<T | null> {
+    const collection = this.getCollection<T>(collectionName);
+    return await collection.findOne(filter);
+  }
+
+  /**
+   * Aggregate with type safety
+   */
+  async aggregate<T extends DocumentBase>(
+    collectionName: CollectionNames,
+    pipeline: any[]
+  ): Promise<T[]> {
+    const collection = this.getCollection<T>(collectionName);
+    return await collection.aggregate<T>(pipeline).toArray();
+  }
+
+  /**
+   * Count documents
+   */
+  async countDocuments(
+    collectionName: CollectionNames,
+    filter: any = {}
+  ): Promise<number> {
+    const collection = this.getCollection(collectionName);
+    return await collection.countDocuments(filter);
+  }
+
+  /**
+   * Create indexes for better performance
+   */
+  async createIndexes(): Promise<void> {
+    if (!this.db) {
+      throw new Error('MongoDB client not connected');
+    }
+
+    try {
+      // Sentiment data indexes
+      await this.db.collection('sentiment_data').createIndexes([
+        { key: { symbol: 1, timestamp: -1 } },
+        { key: { sentiment_label: 1 } },
+        { key: { source_type: 1 } },
+        { key: { created_at: -1 } }
+      ]);
+
+      // News articles indexes
+      await this.db.collection('news_articles').createIndexes([
+        { key: { symbols: 1, published_date: -1 } },
+        { key: { publication: 1 } },
+        { key: { categories: 1 } },
+        { key: { created_at: -1 } }
+      ]);
+
+      // SEC filings indexes
+      await this.db.collection('sec_filings').createIndexes([
+        { key: { symbols: 1, filing_date: -1 } },
+        { key: { filing_type: 1 } },
+        { key: { cik: 1 } },
+        { key: { created_at: -1 } }
+      ]);
+
+      // Raw documents indexes
+      await this.db.collection('raw_documents').createIndexes([
+        { key: { content_hash: 1 }, options: { unique: true } },
+        { key: { processing_status: 1 } },
+        { key: { document_type: 1 } },
+        { key: { created_at: -1 } }
+      ]);
+
+      this.logger.info('MongoDB indexes created successfully');
+    } catch (error) {
+      this.logger.error('Error creating MongoDB indexes:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Get database statistics
+   */
+  async getStats(): Promise<any> {
+    if (!this.db) {
+      throw new Error('MongoDB client not connected');
+    }
+    return await this.db.stats();
+  }
+
+  /**
+   * Check if client is connected
+   */
+  get connected(): boolean {
+    return this.isConnected && !!this.client;
+  }
+
+  /**
+   * Get the underlying MongoDB client
+   */
+  get mongoClient(): MongoClient | null {
+    return this.client;
+  }
+
+  /**
+   * Get the database instance
+   */
+  get database(): Db | null {
+    return this.db;
+  }
+
+  private buildConfig(config?: Partial<MongoDBClientConfig>): MongoDBClientConfig {
+    return {
+      host: config?.host || mongodbConfig.MONGODB_HOST,
+      port: config?.port || mongodbConfig.MONGODB_PORT,
+      database: config?.database || mongodbConfig.MONGODB_DATABASE,
+      username: config?.username || mongodbConfig.MONGODB_USERNAME,
+      password: config?.password || mongodbConfig.MONGODB_PASSWORD,
+      authSource: config?.authSource || mongodbConfig.MONGODB_AUTH_SOURCE,
+      uri: config?.uri || mongodbConfig.MONGODB_URI,
+      poolSettings: {
+        maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
+        minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE,
+        maxIdleTime: mongodbConfig.MONGODB_MAX_IDLE_TIME,
+        ...config?.poolSettings
+      },
+      timeouts: {
+        connectTimeout: mongodbConfig.MONGODB_CONNECT_TIMEOUT,
+        socketTimeout: mongodbConfig.MONGODB_SOCKET_TIMEOUT,
+        serverSelectionTimeout: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT,
+        ...config?.timeouts
+      },
+      tls: {
+        enabled: mongodbConfig.MONGODB_TLS,
+        insecure: mongodbConfig.MONGODB_TLS_INSECURE,
+        caFile: mongodbConfig.MONGODB_TLS_CA_FILE,
+        ...config?.tls
+      },
+      options: {
+        retryWrites: mongodbConfig.MONGODB_RETRY_WRITES,
+        journal: mongodbConfig.MONGODB_JOURNAL,
+        readPreference: mongodbConfig.MONGODB_READ_PREFERENCE as any,
+        writeConcern: mongodbConfig.MONGODB_WRITE_CONCERN,
+        ...config?.options
+      }
+    };
+  }
+
+  private buildConnectionUri(): string {
+    if (this.config.uri) {
+      return this.config.uri;
+    }
+
+    const { host, port, username, password, database, authSource } = this.config;
+    const auth = username && password ? `${username}:${password}@` : '';
+    const authDb = authSource ? `?authSource=${authSource}` : '';
+    
+    return `mongodb://${auth}${host}:${port}/${database}${authDb}`;
+  }
+
+  private buildClientOptions(): MongoClientOptions {
+    return {
+      maxPoolSize: this.config.poolSettings?.maxPoolSize,
+      minPoolSize: this.config.poolSettings?.minPoolSize,
+      maxIdleTimeMS: this.config.poolSettings?.maxIdleTime,
+      connectTimeoutMS: this.config.timeouts?.connectTimeout,
+      socketTimeoutMS: this.config.timeouts?.socketTimeout,
+      serverSelectionTimeoutMS: this.config.timeouts?.serverSelectionTimeout,
+      retryWrites: this.config.options?.retryWrites,
+      journal: this.config.options?.journal,
+      readPreference: this.config.options?.readPreference,
+      writeConcern: { w: this.config.options?.writeConcern },
+      tls: this.config.tls?.enabled,
+      tlsInsecure: this.config.tls?.insecure,
+      tlsCAFile: this.config.tls?.caFile
+    };
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}

libs/mongodb-client/src/factory.ts

@@ -0,0 +1,66 @@
+import { MongoDBClient } from './client';
+import { mongodbConfig } from '@stock-bot/config';
+import type { MongoDBClientConfig, MongoDBConnectionOptions } from './types';
+
+/**
+ * Factory function to create a MongoDB client instance
+ */
+export function createMongoDBClient(
+  config?: Partial<MongoDBClientConfig>,
+  options?: MongoDBConnectionOptions
+): MongoDBClient {
+  return new MongoDBClient(config, options);
+}
+
+/**
+ * Create a MongoDB client with default configuration
+ */
+export function createDefaultMongoDBClient(): MongoDBClient {
+  const config: Partial<MongoDBClientConfig> = {
+    host: mongodbConfig.MONGODB_HOST,
+    port: mongodbConfig.MONGODB_PORT,
+    database: mongodbConfig.MONGODB_DATABASE,
+    username: mongodbConfig.MONGODB_USERNAME,
+    password: mongodbConfig.MONGODB_PASSWORD,
+    uri: mongodbConfig.MONGODB_URI
+  };
+
+  return new MongoDBClient(config);
+}
+
+/**
+ * Singleton MongoDB client instance
+ */
+let defaultClient: MongoDBClient | null = null;
+
+/**
+ * Get or create the default MongoDB client instance
+ */
+export function getMongoDBClient(): MongoDBClient {
+  if (!defaultClient) {
+    defaultClient = createDefaultMongoDBClient();
+  }
+  return defaultClient;
+}
+
+/**
+ * Connect to MongoDB using the default client
+ */
+export async function connectMongoDB(): Promise<MongoDBClient> {
+  const client = getMongoDBClient();
+  if (!client.connected) {
+    await client.connect();
+    await client.createIndexes();
+  }
+  return client;
+}
+
+/**
+ * Disconnect from MongoDB
+ */
+export async function disconnectMongoDB(): Promise<void> {
+  if (defaultClient) {
+    await defaultClient.disconnect();
+    defaultClient = null;
+  }
+}

libs/mongodb-client/src/health.ts

@@ -0,0 +1,228 @@
+import { Logger } from '@stock-bot/logger';
+import type { MongoDBClient } from './client';
+import type { MongoDBHealthCheck, MongoDBHealthStatus, MongoDBMetrics } from './types';
+
+/**
+ * MongoDB Health Monitor
+ * 
+ * Monitors MongoDB connection health and provides metrics
+ */
+export class MongoDBHealthMonitor {
+  private readonly client: MongoDBClient;
+  private readonly logger: Logger;
+  private healthCheckInterval: NodeJS.Timeout | null = null;
+  private metrics: MongoDBMetrics;
+  private lastHealthCheck: MongoDBHealthCheck | null = null;
+
+  constructor(client: MongoDBClient) {
+    this.client = client;
+    this.logger = new Logger('MongoDBHealthMonitor');
+    this.metrics = {
+      operationsPerSecond: 0,
+      averageLatency: 0,
+      errorRate: 0,
+      connectionPoolUtilization: 0,
+      documentsProcessed: 0
+    };
+  }
+
+  /**
+   * Start health monitoring
+   */
+  start(intervalMs: number = 30000): void {
+    if (this.healthCheckInterval) {
+      this.stop();
+    }
+
+    this.logger.info(`Starting MongoDB health monitoring (interval: ${intervalMs}ms)`);
+    
+    this.healthCheckInterval = setInterval(async () => {
+      try {
+        await this.performHealthCheck();
+      } catch (error) {
+        this.logger.error('Health check failed:', error);
+      }
+    }, intervalMs);
+
+    // Perform initial health check
+    this.performHealthCheck().catch(error => {
+      this.logger.error('Initial health check failed:', error);
+    });
+  }
+
+  /**
+   * Stop health monitoring
+   */
+  stop(): void {
+    if (this.healthCheckInterval) {
+      clearInterval(this.healthCheckInterval);
+      this.healthCheckInterval = null;
+      this.logger.info('Stopped MongoDB health monitoring');
+    }
+  }
+
+  /**
+   * Get current health status
+   */
+  async getHealth(): Promise<MongoDBHealthCheck> {
+    if (!this.lastHealthCheck) {
+      await this.performHealthCheck();
+    }
+    return this.lastHealthCheck!;
+  }
+
+  /**
+   * Get current metrics
+   */
+  getMetrics(): MongoDBMetrics {
+    return { ...this.metrics };
+  }
+
+  /**
+   * Perform a health check
+   */
+  private async performHealthCheck(): Promise<void> {
+    const startTime = Date.now();
+    const errors: string[] = [];
+    let status: MongoDBHealthStatus = 'healthy';
+
+    try {
+      if (!this.client.connected) {
+        errors.push('MongoDB client not connected');
+        status = 'unhealthy';
+      } else {
+        // Test basic connectivity
+        const mongoClient = this.client.mongoClient;
+        const db = this.client.database;
+
+        if (!mongoClient || !db) {
+          errors.push('MongoDB client or database not available');
+          status = 'unhealthy';
+        } else {
+          // Ping the database
+          await db.admin().ping();
+
+          // Get server status for metrics
+          try {
+            const serverStatus = await db.admin().serverStatus();
+            this.updateMetricsFromServerStatus(serverStatus);
+
+            // Check connection pool status
+            const poolStats = this.getConnectionPoolStats(serverStatus);
+            
+            if (poolStats.utilization > 0.9) {
+              errors.push('High connection pool utilization');
+              status = status === 'healthy' ? 'degraded' : status;
+            }
+
+            // Check for high latency
+            const latency = Date.now() - startTime;
+            if (latency > 1000) {
+              errors.push(`High latency: ${latency}ms`);
+              status = status === 'healthy' ? 'degraded' : status;
+            }
+
+          } catch (statusError) {
+            errors.push(`Failed to get server status: ${(statusError as Error).message}`);
+            status = 'degraded';
+          }
+        }
+      }
+    } catch (error) {
+      errors.push(`Health check failed: ${(error as Error).message}`);
+      status = 'unhealthy';
+    }
+
+    const latency = Date.now() - startTime;
+    
+    // Get connection stats
+    const connectionStats = this.getConnectionStats();
+
+    this.lastHealthCheck = {
+      status,
+      timestamp: new Date(),
+      latency,
+      connections: connectionStats,
+      errors: errors.length > 0 ? errors : undefined
+    };
+
+    // Log health status changes
+    if (status !== 'healthy') {
+      this.logger.warn(`MongoDB health status: ${status}`, { errors, latency });
+    } else {
+      this.logger.debug(`MongoDB health check passed (${latency}ms)`);
+    }
+  }
+
+  /**
+   * Update metrics from MongoDB server status
+   */
+  private updateMetricsFromServerStatus(serverStatus: any): void {
+    try {
+      const opcounters = serverStatus.opcounters || {};
+      const connections = serverStatus.connections || {};
+      const dur = serverStatus.dur || {};
+
+      // Calculate operations per second (approximate)
+      const totalOps = Object.values(opcounters).reduce((sum: number, count: any) => sum + (count || 0), 0);
+      this.metrics.operationsPerSecond = totalOps;
+
+      // Connection pool utilization
+      if (connections.current && connections.available) {
+        const total = connections.current + connections.available;
+        this.metrics.connectionPoolUtilization = connections.current / total;
+      }
+
+      // Average latency (from durability stats if available)
+      if (dur.timeMS) {
+        this.metrics.averageLatency = dur.timeMS.dt || 0;
+      }
+
+    } catch (error) {
+      this.logger.debug('Error parsing server status for metrics:', error);
+    }
+  }
+
+  /**
+   * Get connection pool statistics
+   */
+  private getConnectionPoolStats(serverStatus: any): { utilization: number; active: number; available: number } {
+    const connections = serverStatus.connections || {};
+    const active = connections.current || 0;
+    const available = connections.available || 0;
+    const total = active + available;
+    
+    return {
+      utilization: total > 0 ? active / total : 0,
+      active,
+      available
+    };
+  }
+
+  /**
+   * Get connection statistics
+   */
+  private getConnectionStats(): { active: number; available: number; total: number } {
+    // This would ideally come from the MongoDB driver's connection pool
+    // For now, we'll return estimated values
+    return {
+      active: 1,
+      available: 9,
+      total: 10
+    };
+  }
+
+  /**
+   * Update error rate metric
+   */
+  updateErrorRate(errorCount: number, totalOperations: number): void {
+    this.metrics.errorRate = totalOperations > 0 ? errorCount / totalOperations : 0;
+  }
+
+  /**
+   * Update documents processed metric
+   */
+  updateDocumentsProcessed(count: number): void {
+    this.metrics.documentsProcessed += count;
+  }
+}

libs/mongodb-client/src/index.ts

@@ -0,0 +1,40 @@
+/**
+ * MongoDB Client Library for Stock Bot
+ * 
+ * Provides type-safe MongoDB access for document storage, sentiment data,
+ * and raw content processing.
+ */
+
+export { MongoDBClient } from './client';
+export { MongoDBHealthMonitor } from './health';
+export { MongoDBTransactionManager } from './transactions';
+export { MongoDBAggregationBuilder } from './aggregation';
+
+// Types
+export type {
+  MongoDBClientConfig,
+  MongoDBConnectionOptions,
+  MongoDBHealthStatus,
+  MongoDBMetrics,
+  CollectionNames,
+  DocumentBase,
+  SentimentData,
+  RawDocument,
+  NewsArticle,
+  SecFiling,
+  EarningsTranscript,
+  AnalystReport
+} from './types';
+
+// Schemas
+export {
+  sentimentDataSchema,
+  rawDocumentSchema,
+  newsArticleSchema,
+  secFilingSchema,
+  earningsTranscriptSchema,
+  analystReportSchema
+} from './schemas';
+
+// Utils
+export { createMongoDBClient } from './factory';

libs/mongodb-client/src/schemas.ts

@@ -0,0 +1,132 @@
+import { z } from 'zod';
+
+/**
+ * Zod Schemas for MongoDB Document Validation
+ */
+
+// Base schema for all documents
+export const documentBaseSchema = z.object({
+  _id: z.any().optional(),
+  created_at: z.date(),
+  updated_at: z.date(),
+  source: z.string(),
+  metadata: z.record(z.any()).optional(),
+});
+
+// Sentiment Data Schema
+export const sentimentDataSchema = documentBaseSchema.extend({
+  symbol: z.string().min(1).max(10),
+  sentiment_score: z.number().min(-1).max(1),
+  sentiment_label: z.enum(['positive', 'negative', 'neutral']),
+  confidence: z.number().min(0).max(1),
+  text: z.string().min(1),
+  source_type: z.enum(['reddit', 'twitter', 'news', 'forums']),
+  source_id: z.string(),
+  timestamp: z.date(),
+  processed_at: z.date(),
+  language: z.string().default('en'),
+  keywords: z.array(z.string()),
+  entities: z.array(z.object({
+    name: z.string(),
+    type: z.string(),
+    confidence: z.number().min(0).max(1),
+  })),
+});
+
+// Raw Document Schema
+export const rawDocumentSchema = documentBaseSchema.extend({
+  document_type: z.enum(['html', 'pdf', 'text', 'json', 'xml']),
+  content: z.string(),
+  content_hash: z.string(),
+  url: z.string().url().optional(),
+  title: z.string().optional(),
+  author: z.string().optional(),
+  published_date: z.date().optional(),
+  extracted_text: z.string().optional(),
+  processing_status: z.enum(['pending', 'processed', 'failed']),
+  size_bytes: z.number().positive(),
+  language: z.string().optional(),
+});
+
+// News Article Schema
+export const newsArticleSchema = documentBaseSchema.extend({
+  headline: z.string().min(1),
+  content: z.string().min(1),
+  summary: z.string().optional(),
+  author: z.string(),
+  publication: z.string(),
+  published_date: z.date(),
+  url: z.string().url(),
+  symbols: z.array(z.string()),
+  categories: z.array(z.string()),
+  sentiment_score: z.number().min(-1).max(1).optional(),
+  relevance_score: z.number().min(0).max(1).optional(),
+  image_url: z.string().url().optional(),
+  tags: z.array(z.string()),
+});
+
+// SEC Filing Schema
+export const secFilingSchema = documentBaseSchema.extend({
+  cik: z.string(),
+  accession_number: z.string(),
+  filing_type: z.string(),
+  company_name: z.string(),
+  symbols: z.array(z.string()),
+  filing_date: z.date(),
+  period_end_date: z.date(),
+  url: z.string().url(),
+  content: z.string(),
+  extracted_data: z.record(z.any()).optional(),
+  financial_statements: z.array(z.object({
+    statement_type: z.string(),
+    data: z.record(z.number()),
+  })).optional(),
+  processing_status: z.enum(['pending', 'processed', 'failed']),
+});
+
+// Earnings Transcript Schema
+export const earningsTranscriptSchema = documentBaseSchema.extend({
+  symbol: z.string().min(1).max(10),
+  company_name: z.string(),
+  quarter: z.string(),
+  year: z.number().min(2000).max(3000),
+  call_date: z.date(),
+  transcript: z.string(),
+  participants: z.array(z.object({
+    name: z.string(),
+    title: z.string(),
+    type: z.enum(['executive', 'analyst']),
+  })),
+  key_topics: z.array(z.string()),
+  sentiment_analysis: z.object({
+    overall_sentiment: z.number().min(-1).max(1),
+    topic_sentiments: z.record(z.number()),
+  }).optional(),
+  financial_highlights: z.record(z.number()).optional(),
+});
+
+// Analyst Report Schema
+export const analystReportSchema = documentBaseSchema.extend({
+  symbol: z.string().min(1).max(10),
+  analyst_firm: z.string(),
+  analyst_name: z.string(),
+  report_title: z.string(),
+  report_date: z.date(),
+  rating: z.enum(['buy', 'hold', 'sell', 'strong_buy', 'strong_sell']),
+  price_target: z.number().positive().optional(),
+  previous_rating: z.string().optional(),
+  content: z.string(),
+  summary: z.string(),
+  key_points: z.array(z.string()),
+  financial_projections: z.record(z.number()).optional(),
+});
+
+// Schema mapping for collections
+export const schemaMap = {
+  sentiment_data: sentimentDataSchema,
+  raw_documents: rawDocumentSchema,
+  news_articles: newsArticleSchema,
+  sec_filings: secFilingSchema,
+  earnings_transcripts: earningsTranscriptSchema,
+  analyst_reports: analystReportSchema,
+} as const;

libs/mongodb-client/src/transactions.ts

@@ -0,0 +1,242 @@
+import { Logger } from '@stock-bot/logger';
+import type { MongoDBClient } from './client';
+import type { CollectionNames, DocumentBase } from './types';
+
+/**
+ * MongoDB Transaction Manager
+ * 
+ * Provides transaction support for multi-document operations
+ */
+export class MongoDBTransactionManager {
+  private readonly client: MongoDBClient;
+  private readonly logger: Logger;
+
+  constructor(client: MongoDBClient) {
+    this.client = client;
+    this.logger = new Logger('MongoDBTransactionManager');
+  }
+
+  /**
+   * Execute operations within a transaction
+   */
+  async withTransaction<T>(
+    operations: (session: any) => Promise<T>,
+    options?: {
+      readPreference?: string;
+      readConcern?: string;
+      writeConcern?: any;
+      maxCommitTimeMS?: number;
+    }
+  ): Promise<T> {
+    const mongoClient = this.client.mongoClient;
+    if (!mongoClient) {
+      throw new Error('MongoDB client not connected');
+    }
+
+    const session = mongoClient.startSession();
+    
+    try {
+      this.logger.debug('Starting MongoDB transaction');
+      
+      const result = await session.withTransaction(
+        async () => {
+          return await operations(session);
+        },
+        {
+          readPreference: options?.readPreference,
+          readConcern: { level: options?.readConcern || 'majority' },
+          writeConcern: options?.writeConcern || { w: 'majority' },
+          maxCommitTimeMS: options?.maxCommitTimeMS || 10000
+        }
+      );
+
+      this.logger.debug('MongoDB transaction completed successfully');
+      return result;
+      
+    } catch (error) {
+      this.logger.error('MongoDB transaction failed:', error);
+      throw error;
+    } finally {
+      await session.endSession();
+    }
+  }
+
+  /**
+   * Batch insert documents across collections within a transaction
+   */
+  async batchInsert(
+    operations: Array<{
+      collection: CollectionNames;
+      documents: DocumentBase[];
+    }>,
+    options?: { ordered?: boolean; bypassDocumentValidation?: boolean }
+  ): Promise<void> {
+    await this.withTransaction(async (session) => {
+      for (const operation of operations) {
+        const collection = this.client.getCollection(operation.collection);
+        
+        // Add timestamps to all documents
+        const now = new Date();
+        const documentsWithTimestamps = operation.documents.map(doc => ({
+          ...doc,
+          created_at: doc.created_at || now,
+          updated_at: now
+        }));
+
+        await collection.insertMany(documentsWithTimestamps, {
+          session,
+          ordered: options?.ordered ?? true,
+          bypassDocumentValidation: options?.bypassDocumentValidation ?? false
+        });
+        
+        this.logger.debug(`Inserted ${documentsWithTimestamps.length} documents into ${operation.collection}`);
+      }
+    });
+  }
+
+  /**
+   * Batch update documents across collections within a transaction
+   */
+  async batchUpdate(
+    operations: Array<{
+      collection: CollectionNames;
+      filter: any;
+      update: any;
+      options?: any;
+    }>
+  ): Promise<void> {
+    await this.withTransaction(async (session) => {
+      const results = [];
+      
+      for (const operation of operations) {
+        const collection = this.client.getCollection(operation.collection);
+        
+        // Add updated timestamp
+        const updateWithTimestamp = {
+          ...operation.update,
+          $set: {
+            ...operation.update.$set,
+            updated_at: new Date()
+          }
+        };
+
+        const result = await collection.updateMany(
+          operation.filter,
+          updateWithTimestamp,
+          {
+            session,
+            ...operation.options
+          }
+        );
+        
+        results.push(result);
+        this.logger.debug(`Updated ${result.modifiedCount} documents in ${operation.collection}`);
+      }
+      
+      return results;
+    });
+  }
+
+  /**
+   * Move documents between collections within a transaction
+   */
+  async moveDocuments<T extends DocumentBase>(
+    fromCollection: CollectionNames,
+    toCollection: CollectionNames,
+    filter: any,
+    transform?: (doc: T) => T
+  ): Promise<number> {
+    return await this.withTransaction(async (session) => {
+      const sourceCollection = this.client.getCollection<T>(fromCollection);
+      const targetCollection = this.client.getCollection<T>(toCollection);
+
+      // Find documents to move
+      const documents = await sourceCollection.find(filter, { session }).toArray();
+      
+      if (documents.length === 0) {
+        return 0;
+      }
+
+      // Transform documents if needed
+      const documentsToInsert = transform 
+        ? documents.map(transform)
+        : documents;
+
+      // Add updated timestamp
+      const now = new Date();
+      documentsToInsert.forEach(doc => {
+        doc.updated_at = now;
+      });
+
+      // Insert into target collection
+      await targetCollection.insertMany(documentsToInsert, { session });
+
+      // Remove from source collection
+      const deleteResult = await sourceCollection.deleteMany(filter, { session });
+
+      this.logger.info(`Moved ${documents.length} documents from ${fromCollection} to ${toCollection}`);
+      
+      return deleteResult.deletedCount || 0;
+    });
+  }
+
+  /**
+   * Archive old documents within a transaction
+   */
+  async archiveDocuments(
+    sourceCollection: CollectionNames,
+    archiveCollection: CollectionNames,
+    cutoffDate: Date,
+    batchSize: number = 1000
+  ): Promise<number> {
+    let totalArchived = 0;
+    
+    while (true) {
+      const batchArchived = await this.withTransaction(async (session) => {
+        const collection = this.client.getCollection(sourceCollection);
+        const archiveCol = this.client.getCollection(archiveCollection);
+
+        // Find old documents
+        const documents = await collection.find(
+          { created_at: { $lt: cutoffDate } },
+          { limit: batchSize, session }
+        ).toArray();
+
+        if (documents.length === 0) {
+          return 0;
+        }
+
+        // Add archive metadata
+        const now = new Date();
+        const documentsToArchive = documents.map(doc => ({
+          ...doc,
+          archived_at: now,
+          archived_from: sourceCollection
+        }));
+
+        // Insert into archive collection
+        await archiveCol.insertMany(documentsToArchive, { session });
+
+        // Remove from source collection
+        const ids = documents.map(doc => doc._id);
+        const deleteResult = await collection.deleteMany(
+          { _id: { $in: ids } },
+          { session }
+        );
+
+        return deleteResult.deletedCount || 0;
+      });
+
+      totalArchived += batchArchived;
+      
+      if (batchArchived === 0) {
+        break;
+      }
+
+      this.logger.debug(`Archived batch of ${batchArchived} documents`);
+    }
+
+    this.logger.info(`Archived ${totalArchived} documents from ${sourceCollection} to ${archiveCollection}`);
+    return totalArchived;
+  }
+}

libs/mongodb-client/src/types.ts

@@ -0,0 +1,215 @@
+import { z } from 'zod';
+import type { ObjectId } from 'mongodb';
+
+/**
+ * MongoDB Client Configuration
+ */
+export interface MongoDBClientConfig {
+  host: string;
+  port: number;
+  database: string;
+  username?: string;
+  password?: string;
+  authSource?: string;
+  uri?: string;
+  poolSettings?: {
+    maxPoolSize: number;
+    minPoolSize: number;
+    maxIdleTime: number;
+  };
+  timeouts?: {
+    connectTimeout: number;
+    socketTimeout: number;
+    serverSelectionTimeout: number;
+  };
+  tls?: {
+    enabled: boolean;
+    insecure: boolean;
+    caFile?: string;
+  };
+  options?: {
+    retryWrites: boolean;
+    journal: boolean;
+    readPreference: 'primary' | 'primaryPreferred' | 'secondary' | 'secondaryPreferred' | 'nearest';
+    writeConcern: string;
+  };
+}
+
+/**
+ * MongoDB Connection Options
+ */
+export interface MongoDBConnectionOptions {
+  retryAttempts?: number;
+  retryDelay?: number;
+  healthCheckInterval?: number;
+}
+
+/**
+ * Health Status Types
+ */
+export type MongoDBHealthStatus = 'healthy' | 'degraded' | 'unhealthy';
+
+export interface MongoDBHealthCheck {
+  status: MongoDBHealthStatus;
+  timestamp: Date;
+  latency: number;
+  connections: {
+    active: number;
+    available: number;
+    total: number;
+  };
+  errors?: string[];
+}
+
+export interface MongoDBMetrics {
+  operationsPerSecond: number;
+  averageLatency: number;
+  errorRate: number;
+  connectionPoolUtilization: number;
+  documentsProcessed: number;
+}
+
+/**
+ * Collection Names
+ */
+export type CollectionNames = 
+  | 'sentiment_data'
+  | 'raw_documents'
+  | 'news_articles'
+  | 'sec_filings'
+  | 'earnings_transcripts'
+  | 'analyst_reports'
+  | 'social_media_posts'
+  | 'market_events'
+  | 'economic_indicators';
+
+/**
+ * Base Document Interface
+ */
+export interface DocumentBase {
+  _id?: ObjectId;
+  created_at: Date;
+  updated_at: Date;
+  source: string;
+  metadata?: Record<string, any>;
+}
+
+/**
+ * Sentiment Data Document
+ */
+export interface SentimentData extends DocumentBase {
+  symbol: string;
+  sentiment_score: number;
+  sentiment_label: 'positive' | 'negative' | 'neutral';
+  confidence: number;
+  text: string;
+  source_type: 'reddit' | 'twitter' | 'news' | 'forums';
+  source_id: string;
+  timestamp: Date;
+  processed_at: Date;
+  language: string;
+  keywords: string[];
+  entities: Array<{
+    name: string;
+    type: string;
+    confidence: number;
+  }>;
+}
+
+/**
+ * Raw Document
+ */
+export interface RawDocument extends DocumentBase {
+  document_type: 'html' | 'pdf' | 'text' | 'json' | 'xml';
+  content: string;
+  content_hash: string;
+  url?: string;
+  title?: string;
+  author?: string;
+  published_date?: Date;
+  extracted_text?: string;
+  processing_status: 'pending' | 'processed' | 'failed';
+  size_bytes: number;
+  language?: string;
+}
+
+/**
+ * News Article
+ */
+export interface NewsArticle extends DocumentBase {
+  headline: string;
+  content: string;
+  summary?: string;
+  author: string;
+  publication: string;
+  published_date: Date;
+  url: string;
+  symbols: string[];
+  categories: string[];
+  sentiment_score?: number;
+  relevance_score?: number;
+  image_url?: string;
+  tags: string[];
+}
+
+/**
+ * SEC Filing
+ */
+export interface SecFiling extends DocumentBase {
+  cik: string;
+  accession_number: string;
+  filing_type: string;
+  company_name: string;
+  symbols: string[];
+  filing_date: Date;
+  period_end_date: Date;
+  url: string;
+  content: string;
+  extracted_data?: Record<string, any>;
+  financial_statements?: Array<{
+    statement_type: string;
+    data: Record<string, number>;
+  }>;
+  processing_status: 'pending' | 'processed' | 'failed';
+}
+
+/**
+ * Earnings Transcript
+ */
+export interface EarningsTranscript extends DocumentBase {
+  symbol: string;
+  company_name: string;
+  quarter: string;
+  year: number;
+  call_date: Date;
+  transcript: string;
+  participants: Array<{
+    name: string;
+    title: string;
+    type: 'executive' | 'analyst';
+  }>;
+  key_topics: string[];
+  sentiment_analysis?: {
+    overall_sentiment: number;
+    topic_sentiments: Record<string, number>;
+  };
+  financial_highlights?: Record<string, number>;
+}
+
+/**
+ * Analyst Report
+ */
+export interface AnalystReport extends DocumentBase {
+  symbol: string;
+  analyst_firm: string;
+  analyst_name: string;
+  report_title: string;
+  report_date: Date;
+  rating: 'buy' | 'hold' | 'sell' | 'strong_buy' | 'strong_sell';
+  price_target?: number;
+  previous_rating?: string;
+  content: string;
+  summary: string;
+  key_points: string[];
+  financial_projections?: Record<string, number>;
+}

libs/mongodb-client/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "dist",
+    "node_modules",
+    "**/*.test.ts",
+    "**/*.spec.ts"
+  ]
+}