added new di with connection pool and rebuild of the database/cache services

libs/connection-factory/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@stock-bot/connection-factory",
+  "version": "1.0.0",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "dependencies": {
+    "@stock-bot/config": "workspace:*",
+    "@stock-bot/logger": "workspace:*",
+    "@stock-bot/mongodb-client": "workspace:*",
+    "@stock-bot/postgres-client": "workspace:*",
+    "@stock-bot/cache": "workspace:*",
+    "mongodb": "^6.3.0",
+    "pg": "^8.11.3"
+  },
+  "devDependencies": {
+    "@types/pg": "^8.10.7"
+  }
+}

libs/connection-factory/src/connection-factory.ts

@@ -0,0 +1,234 @@
+import { getLogger, type Logger } from '@stock-bot/logger';
+import { MongoDBClient, createMongoDBClient, type ConnectionEvents } from '@stock-bot/mongodb-client';
+import { PostgreSQLClient, createPostgreSQLClient } from '@stock-bot/postgres-client';
+import { createCache, type CacheProvider } from '@stock-bot/cache';
+import type {
+  ConnectionFactory as IConnectionFactory,
+  ConnectionPool,
+  ConnectionFactoryConfig,
+  MongoDBPoolConfig,
+  PostgreSQLPoolConfig,
+  CachePoolConfig,
+  PoolMetrics,
+} from './types';
+
+export class ConnectionFactory implements IConnectionFactory {
+  private readonly logger: Logger;
+  private readonly pools: Map<string, ConnectionPool<any>> = new Map();
+  private readonly config: ConnectionFactoryConfig;
+
+  constructor(config: ConnectionFactoryConfig) {
+    this.config = config;
+    this.logger = getLogger(`connection-factory:${config.service}`);
+  }
+
+  async createMongoDB(poolConfig: MongoDBPoolConfig): Promise<ConnectionPool<MongoDBClient>> {
+    const key = `mongodb:${poolConfig.name}`;
+    
+    if (this.pools.has(key)) {
+      this.logger.debug('Reusing existing MongoDB pool', { name: poolConfig.name });
+      return this.pools.get(key)!;
+    }
+
+    this.logger.info('Creating MongoDB connection pool', {
+      name: poolConfig.name,
+      poolSize: poolConfig.poolSize,
+    });
+
+    try {
+      const events: ConnectionEvents = {
+        onConnect: () => {
+          this.logger.debug('MongoDB connected', { pool: poolConfig.name });
+        },
+        onDisconnect: () => {
+          this.logger.debug('MongoDB disconnected', { pool: poolConfig.name });
+        },
+        onError: (error) => {
+          this.logger.error('MongoDB error', { pool: poolConfig.name, error });
+        },
+      };
+
+      const client = createMongoDBClient({
+        ...poolConfig.config,
+        poolSettings: {
+          maxPoolSize: poolConfig.maxConnections || poolConfig.poolSize || 10,
+          minPoolSize: poolConfig.minConnections || 2,
+          maxIdleTime: 30000,
+        }
+      }, events);
+
+      await client.connect();
+      
+      // Warm up the pool
+      if (poolConfig.minConnections) {
+        await client.warmupPool();
+      }
+
+      const pool: ConnectionPool<MongoDBClient> = {
+        name: poolConfig.name,
+        client,
+        metrics: client.getPoolMetrics(),
+        health: async () => {
+          try {
+            await client.getDatabase().admin().ping();
+            return true;
+          } catch {
+            return false;
+          }
+        },
+        dispose: async () => {
+          await client.disconnect();
+          this.pools.delete(key);
+        },
+      };
+
+      this.pools.set(key, pool);
+      return pool;
+    } catch (error) {
+      this.logger.error('Failed to create MongoDB pool', { name: poolConfig.name, error });
+      throw error;
+    }
+  }
+
+  async createPostgreSQL(poolConfig: PostgreSQLPoolConfig): Promise<ConnectionPool<PostgreSQLClient>> {
+    const key = `postgres:${poolConfig.name}`;
+    
+    if (this.pools.has(key)) {
+      this.logger.debug('Reusing existing PostgreSQL pool', { name: poolConfig.name });
+      return this.pools.get(key)!;
+    }
+
+    this.logger.info('Creating PostgreSQL connection pool', {
+      name: poolConfig.name,
+      poolSize: poolConfig.poolSize,
+    });
+
+    try {
+      const events: ConnectionEvents = {
+        onConnect: () => {
+          this.logger.debug('PostgreSQL connected', { pool: poolConfig.name });
+        },
+        onDisconnect: () => {
+          this.logger.debug('PostgreSQL disconnected', { pool: poolConfig.name });
+        },
+        onError: (error) => {
+          this.logger.error('PostgreSQL error', { pool: poolConfig.name, error });
+        },
+      };
+
+      const client = createPostgreSQLClient({
+        ...poolConfig.config,
+        poolSettings: {
+          max: poolConfig.maxConnections || poolConfig.poolSize || 10,
+          min: poolConfig.minConnections || 2,
+          idleTimeoutMillis: poolConfig.idleTimeoutMillis || 30000,
+        },
+      }, undefined, events);
+
+      await client.connect();
+      
+      // Warm up the pool
+      if (poolConfig.minConnections) {
+        await client.warmupPool();
+      }
+
+      const pool: ConnectionPool<PostgreSQLClient> = {
+        name: poolConfig.name,
+        client,
+        metrics: client.getPoolMetrics(),
+        health: async () => client.connected,
+        dispose: async () => {
+          await client.disconnect();
+          this.pools.delete(key);
+        },
+      };
+
+      this.pools.set(key, pool);
+      return pool;
+    } catch (error) {
+      this.logger.error('Failed to create PostgreSQL pool', { name: poolConfig.name, error });
+      throw error;
+    }
+  }
+
+  createCache(poolConfig: CachePoolConfig): ConnectionPool<CacheProvider> {
+    const key = `cache:${poolConfig.name}`;
+    
+    if (this.pools.has(key)) {
+      this.logger.debug('Reusing existing cache pool', { name: poolConfig.name });
+      return this.pools.get(key)!;
+    }
+
+    this.logger.info('Creating cache connection pool', {
+      name: poolConfig.name,
+    });
+
+    try {
+      const cache = createCache({
+        ...poolConfig.config,
+        keyPrefix: `${this.config.service}:${poolConfig.name}:`,
+        shared: false, // Each pool gets its own connection
+      });
+
+      const pool: ConnectionPool<CacheProvider> = {
+        name: poolConfig.name,
+        client: cache,
+        metrics: this.createInitialMetrics(),
+        health: async () => cache.health(),
+        dispose: async () => {
+          // Cache disposal handled internally
+          this.pools.delete(key);
+        },
+      };
+
+      this.pools.set(key, pool);
+      return pool;
+    } catch (error) {
+      this.logger.error('Failed to create cache pool', { name: poolConfig.name, error });
+      throw error;
+    }
+  }
+
+  getPool(type: 'mongodb' | 'postgres' | 'cache', name: string): ConnectionPool<any> | undefined {
+    const key = `${type}:${name}`;
+    return this.pools.get(key);
+  }
+
+  listPools(): Array<{ type: string; name: string; metrics: PoolMetrics }> {
+    const result: Array<{ type: string; name: string; metrics: PoolMetrics }> = [];
+    
+    for (const [key, pool] of this.pools.entries()) {
+      const [type, ...nameParts] = key.split(':');
+      result.push({
+        type: type || 'unknown',
+        name: nameParts.join(':'),
+        metrics: pool.metrics,
+      });
+    }
+    
+    return result;
+  }
+
+  async disposeAll(): Promise<void> {
+    this.logger.info('Disposing all connection pools', { count: this.pools.size });
+    
+    const disposePromises: Promise<void>[] = [];
+    for (const pool of this.pools.values()) {
+      disposePromises.push(pool.dispose());
+    }
+    
+    await Promise.all(disposePromises);
+    this.pools.clear();
+  }
+
+  private createInitialMetrics(): PoolMetrics {
+    return {
+      created: new Date(),
+      totalConnections: 0,
+      activeConnections: 0,
+      idleConnections: 0,
+      waitingRequests: 0,
+      errors: 0,
+    };
+  }
+}

libs/connection-factory/src/index.ts

@@ -0,0 +1,23 @@
+export { ConnectionFactory } from './connection-factory';
+export { ServiceContainer, createServiceContainer } from './service-container';
+export { PoolSizeCalculator } from './pool-size-calculator';
+
+export type {
+  ConnectionPoolConfig,
+  MongoDBPoolConfig,
+  PostgreSQLPoolConfig,
+  CachePoolConfig,
+  ConnectionFactoryConfig,
+  ConnectionPool,
+  PoolMetrics,
+  ConnectionFactory as IConnectionFactory,
+} from './types';
+
+export type {
+  ServiceRegistration,
+  ServiceResolver,
+} from './service-container';
+
+export type {
+  PoolSizeRecommendation,
+} from './pool-size-calculator';

libs/connection-factory/src/pool-size-calculator.ts

@@ -0,0 +1,80 @@
+import type { ConnectionPoolConfig } from './types';
+
+export interface PoolSizeRecommendation {
+  min: number;
+  max: number;
+  idle: number;
+}
+
+export class PoolSizeCalculator {
+  private static readonly DEFAULT_SIZES: Record<string, PoolSizeRecommendation> = {
+    // Service-level defaults
+    'data-ingestion': { min: 5, max: 50, idle: 10 },
+    'data-pipeline': { min: 3, max: 30, idle: 5 },
+    'processing-service': { min: 2, max: 20, idle: 3 },
+    'web-api': { min: 2, max: 10, idle: 2 },
+    'portfolio-service': { min: 2, max: 15, idle: 3 },
+    'strategy-service': { min: 3, max: 25, idle: 5 },
+    'execution-service': { min: 2, max: 10, idle: 2 },
+    
+    // Handler-level defaults
+    'batch-import': { min: 10, max: 100, idle: 20 },
+    'real-time': { min: 2, max: 10, idle: 3 },
+    'analytics': { min: 5, max: 30, idle: 10 },
+    'reporting': { min: 3, max: 20, idle: 5 },
+  };
+
+  static calculate(
+    serviceName: string,
+    handlerName?: string,
+    customConfig?: Partial<ConnectionPoolConfig>
+  ): PoolSizeRecommendation {
+    // Check for custom configuration first
+    if (customConfig?.minConnections && customConfig?.maxConnections) {
+      return {
+        min: customConfig.minConnections,
+        max: customConfig.maxConnections,
+        idle: Math.floor((customConfig.minConnections + customConfig.maxConnections) / 4),
+      };
+    }
+
+    // Try handler-specific sizes first, then service-level
+    const key = handlerName || serviceName;
+    const recommendation = this.DEFAULT_SIZES[key] || this.DEFAULT_SIZES[serviceName];
+
+    if (recommendation) {
+      return { ...recommendation };
+    }
+
+    // Fall back to generic defaults
+    return {
+      min: 2,
+      max: 10,
+      idle: 3,
+    };
+  }
+
+  static getOptimalPoolSize(
+    expectedConcurrency: number,
+    averageQueryTimeMs: number,
+    targetLatencyMs: number
+  ): number {
+    // Little's Law: L = λ * W
+    // L = number of connections needed
+    // λ = arrival rate (requests per second)
+    // W = average time in system (seconds)
+    
+    const requestsPerSecond = expectedConcurrency;
+    const averageTimeInSystem = averageQueryTimeMs / 1000;
+    
+    const minConnections = Math.ceil(requestsPerSecond * averageTimeInSystem);
+    
+    // Add buffer for burst traffic (20% overhead)
+    const recommendedSize = Math.ceil(minConnections * 1.2);
+    
+    // Ensure we meet target latency
+    const latencyBasedSize = Math.ceil(expectedConcurrency * (averageQueryTimeMs / targetLatencyMs));
+    
+    return Math.max(recommendedSize, latencyBasedSize, 2); // Minimum 2 connections
+  }
+}

libs/connection-factory/src/service-container.ts

@@ -0,0 +1,147 @@
+import { getLogger, type Logger } from '@stock-bot/logger';
+import type { ConnectionFactory } from './connection-factory';
+
+export interface ServiceRegistration<T = any> {
+  name: string;
+  factory: () => T | Promise<T>;
+  singleton?: boolean;
+  dispose?: (instance: T) => Promise<void>;
+}
+
+export interface ServiceResolver {
+  resolve<T>(name: string, options?: any): T;
+  resolveAsync<T>(name: string, options?: any): Promise<T>;
+}
+
+export class ServiceContainer implements ServiceResolver {
+  private readonly logger: Logger;
+  private readonly registrations = new Map<string, ServiceRegistration>();
+  private readonly instances = new Map<string, any>();
+  private readonly scopedInstances = new Map<string, any>();
+  private readonly parent?: ServiceContainer;
+
+  constructor(name: string, parent?: ServiceContainer) {
+    this.logger = getLogger(`service-container:${name}`);
+    this.parent = parent;
+  }
+
+  register<T>(registration: ServiceRegistration<T>): void {
+    this.registrations.set(registration.name, registration);
+    this.logger.debug('Service registered', { name: registration.name, singleton: registration.singleton });
+  }
+
+  resolve<T>(name: string, options?: any): T {
+    const instance = this.resolveAsync<T>(name, options);
+    if (instance instanceof Promise) {
+      throw new Error(`Service ${name} is async. Use resolveAsync() instead.`);
+    }
+    return instance as T;
+  }
+
+  async resolveAsync<T>(name: string, _options?: any): Promise<T> {
+    // Check scoped instances first
+    if (this.scopedInstances.has(name)) {
+      return this.scopedInstances.get(name);
+    }
+
+    // Check singleton instances
+    if (this.instances.has(name)) {
+      return this.instances.get(name);
+    }
+
+    // Get registration from this container or parent
+    const registration = this.getRegistration(name);
+    if (!registration) {
+      throw new Error(`Service ${name} not registered`);
+    }
+
+    // Create instance
+    const instance = await Promise.resolve(registration.factory());
+
+    // Store based on singleton flag
+    if (registration.singleton) {
+      this.instances.set(name, instance);
+    } else {
+      this.scopedInstances.set(name, instance);
+    }
+
+    return instance as T;
+  }
+
+  createScope(): ServiceContainer {
+    return new ServiceContainer('scoped', this);
+  }
+
+  async dispose(): Promise<void> {
+    // Dispose scoped instances
+    for (const [name, instance] of this.scopedInstances.entries()) {
+      const registration = this.getRegistration(name);
+      if (registration?.dispose) {
+        await registration.dispose(instance);
+      }
+    }
+    this.scopedInstances.clear();
+
+    // Only dispose singletons if this is the root container
+    if (!this.parent) {
+      for (const [name, instance] of this.instances.entries()) {
+        const registration = this.registrations.get(name);
+        if (registration?.dispose) {
+          await registration.dispose(instance);
+        }
+      }
+      this.instances.clear();
+    }
+  }
+
+  private getRegistration(name: string): ServiceRegistration | undefined {
+    return this.registrations.get(name) || this.parent?.getRegistration(name);
+  }
+}
+
+// Helper to create pre-configured containers for services
+export function createServiceContainer(
+  serviceName: string,
+  connectionFactory: ConnectionFactory
+): ServiceContainer {
+  const container = new ServiceContainer(serviceName);
+
+  // Register connection factories
+  container.register({
+    name: 'mongodb',
+    factory: async () => {
+      const pool = await connectionFactory.createMongoDB({
+        name: 'default',
+        config: {} as any, // Config injected by factory
+      });
+      return pool.client;
+    },
+    singleton: true,
+  });
+
+  container.register({
+    name: 'postgres',
+    factory: async () => {
+      const pool = await connectionFactory.createPostgreSQL({
+        name: 'default',
+        config: {} as any, // Config injected by factory
+      });
+      return pool.client;
+    },
+    singleton: true,
+  });
+
+  container.register({
+    name: 'cache',
+    factory: () => {
+      const pool = connectionFactory.createCache({
+        name: 'default',
+        config: {} as any, // Config injected by factory
+      });
+      return pool.client;
+    },
+    singleton: true,
+  });
+
+  return container;
+}

libs/connection-factory/src/types.ts

@@ -0,0 +1,61 @@
+import type { MongoDBClientConfig } from '@stock-bot/mongodb-client';
+import type { PostgreSQLClientConfig } from '@stock-bot/postgres-client';
+import type { CacheOptions } from '@stock-bot/cache';
+
+export interface ConnectionPoolConfig {
+  name: string;
+  poolSize?: number;
+  minConnections?: number;
+  maxConnections?: number;
+  idleTimeoutMillis?: number;
+  connectionTimeoutMillis?: number;
+  enableMetrics?: boolean;
+}
+
+export interface MongoDBPoolConfig extends ConnectionPoolConfig {
+  config: MongoDBClientConfig;
+}
+
+export interface PostgreSQLPoolConfig extends ConnectionPoolConfig {
+  config: PostgreSQLClientConfig;
+}
+
+export interface CachePoolConfig extends ConnectionPoolConfig {
+  config: CacheOptions;
+}
+
+export interface ConnectionFactoryConfig {
+  service: string;
+  environment: 'development' | 'production' | 'test';
+  pools?: {
+    mongodb?: Partial<MongoDBPoolConfig>;
+    postgres?: Partial<PostgreSQLPoolConfig>;
+    cache?: Partial<CachePoolConfig>;
+  };
+}
+
+export interface ConnectionPool<T> {
+  name: string;
+  client: T;
+  metrics: PoolMetrics;
+  health(): Promise<boolean>;
+  dispose(): Promise<void>;
+}
+
+export interface PoolMetrics {
+  created: Date;
+  totalConnections: number;
+  activeConnections: number;
+  idleConnections: number;
+  waitingRequests: number;
+  errors: number;
+}
+
+export interface ConnectionFactory {
+  createMongoDB(config: MongoDBPoolConfig): Promise<ConnectionPool<any>>;
+  createPostgreSQL(config: PostgreSQLPoolConfig): Promise<ConnectionPool<any>>;
+  createCache(config: CachePoolConfig): ConnectionPool<any>;
+  getPool(type: 'mongodb' | 'postgres' | 'cache', name: string): ConnectionPool<any> | undefined;
+  listPools(): Array<{ type: string; name: string; metrics: PoolMetrics }>;
+  disposeAll(): Promise<void>;
+}

libs/connection-factory/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "composite": true
+  },
+  "include": ["src/**/*"],
+  "references": [
+    { "path": "../config" },
+    { "path": "../logger" },
+    { "path": "../mongodb-client" },
+    { "path": "../postgres-client" },
+    { "path": "../cache" }
+  ]
+}