added new di with connection pool and rebuild of the database/cache services

apps/data-ingestion/src/setup/dynamic-pool-example.ts

@@ -0,0 +1,116 @@
+import { getDatabaseConfig } from '@stock-bot/config';
+import { getLogger } from '@stock-bot/logger';
+import { createMongoDBClient, createPostgreSQLClient } from '@stock-bot/connection-factory';
+import type { DynamicPoolConfig } from '@stock-bot/mongodb-client';
+
+const logger = getLogger('dynamic-pool-example');
+
+/**
+ * Example of setting up dynamic pool sizing for high-load scenarios
+ */
+export async function setupDynamicPools() {
+  const dbConfig = getDatabaseConfig();
+  
+  // Dynamic pool configuration for batch processing
+  const dynamicConfig: DynamicPoolConfig = {
+    enabled: true,
+    minSize: 5,
+    maxSize: 100,
+    scaleUpThreshold: 70,    // Scale up when 70% of connections are in use
+    scaleDownThreshold: 30,  // Scale down when only 30% are in use
+    scaleUpIncrement: 10,     // Add 10 connections at a time
+    scaleDownIncrement: 5,    // Remove 5 connections at a time
+    evaluationInterval: 10000 // Check every 10 seconds
+  };
+  
+  // Create MongoDB client with dynamic pooling
+  const mongoClient = createMongoDBClient({
+    uri: dbConfig.mongodb.uri,
+    database: dbConfig.mongodb.database,
+    poolSettings: {
+      minPoolSize: dynamicConfig.minSize,
+      maxPoolSize: dynamicConfig.maxSize,
+    }
+  }, {
+    onConnect: () => logger.info('MongoDB connected with dynamic pooling'),
+    onError: (error) => logger.error('MongoDB pool error', { error }),
+  });
+  
+  await mongoClient.connect();
+  mongoClient.setDynamicPoolConfig(dynamicConfig);
+  
+  // Create PostgreSQL client with dynamic pooling
+  const pgClient = createPostgreSQLClient({
+    host: dbConfig.postgresql.host,
+    port: dbConfig.postgresql.port,
+    database: dbConfig.postgresql.database,
+    username: dbConfig.postgresql.user,
+    password: dbConfig.postgresql.password,
+    poolSettings: {
+      min: dynamicConfig.minSize,
+      max: dynamicConfig.maxSize,
+    }
+  }, undefined, {
+    onConnect: () => logger.info('PostgreSQL connected with dynamic pooling'),
+    onError: (error) => logger.error('PostgreSQL pool error', { error }),
+  });
+  
+  await pgClient.connect();
+  pgClient.setDynamicPoolConfig(dynamicConfig);
+  
+  // Monitor pool metrics
+  setInterval(() => {
+    const mongoMetrics = mongoClient.getPoolMetrics();
+    const pgMetrics = pgClient.getPoolMetrics();
+    
+    logger.info('Pool metrics', {
+      mongodb: {
+        total: mongoMetrics.totalConnections,
+        active: mongoMetrics.activeConnections,
+        idle: mongoMetrics.idleConnections,
+        waiting: mongoMetrics.waitingRequests,
+      },
+      postgresql: {
+        total: pgMetrics.totalConnections,
+        active: pgMetrics.activeConnections,
+        idle: pgMetrics.idleConnections,
+        waiting: pgMetrics.waitingRequests,
+      }
+    });
+  }, 30000); // Log metrics every 30 seconds
+  
+  return { mongoClient, pgClient };
+}
+
+/**
+ * Example of adaptive pool sizing based on time of day
+ */
+export function getTimeBasedPoolConfig(): DynamicPoolConfig {
+  const hour = new Date().getHours();
+  
+  // High load hours (9 AM - 5 PM)
+  if (hour >= 9 && hour <= 17) {
+    return {
+      enabled: true,
+      minSize: 10,
+      maxSize: 150,
+      scaleUpThreshold: 60,
+      scaleDownThreshold: 20,
+      scaleUpIncrement: 20,
+      scaleDownIncrement: 10,
+      evaluationInterval: 5000 // More frequent checks during peak
+    };
+  }
+  
+  // Low load hours (night time)
+  return {
+    enabled: true,
+    minSize: 2,
+    maxSize: 50,
+    scaleUpThreshold: 80,
+    scaleDownThreshold: 40,
+    scaleUpIncrement: 5,
+    scaleDownIncrement: 2,
+    evaluationInterval: 30000 // Less frequent checks at night
+  };
+}