refactored monorepo for more projects

libs/core/config/src/config-manager.ts

@@ -73,6 +73,17 @@ export class ConfigManager<T = Record<string, unknown>> {
         this.config = this.schema.parse(mergedConfig) as T;
       } catch (error) {
         if (error instanceof z.ZodError) {
+          const errorDetails = error.errors.map(err => ({
+            path: err.path.join('.'),
+            message: err.message,
+            code: err.code,
+            expected: (err as any).expected,
+            received: (err as any).received,
+          }));
+          
+          console.error('Configuration validation failed:');
+          console.error(JSON.stringify(errorDetails, null, 2));
+          
           throw new ConfigValidationError('Configuration validation failed', error.errors);
         }
         throw error;

libs/core/config/src/index.ts

@@ -1,11 +1,12 @@
-// Import necessary types for singleton
+// Import necessary types
 import { EnvLoader } from './loaders/env.loader';
 import { FileLoader } from './loaders/file.loader';
 import { ConfigManager } from './config-manager';
 import type { AppConfig } from './schemas';
 import { appConfigSchema } from './schemas';
+import { z } from 'zod';
 
-// Create singleton instance
+// Legacy singleton instance for backward compatibility
 let configInstance: ConfigManager<AppConfig> | null = null;
 
 // Synchronously load critical env vars for early initialization
@@ -57,6 +58,7 @@ loadCriticalEnvVarsSync();
 
 /**
  * Initialize the global configuration synchronously.
+ * @deprecated Use initializeAppConfig with your own schema instead
  *
  * This loads configuration from all sources in the correct hierarchy:
  * 1. Schema defaults (lowest priority)
@@ -143,8 +145,11 @@ export function getLoggingConfig() {
   return getConfig().log;
 }
 
+// Deprecated - provider configs should be app-specific
+// @deprecated Move provider configs to your app-specific config
 export function getProviderConfig(provider: string) {
-  const providers = getConfig().providers;
+  const config = getConfig() as any;
+  const providers = config.providers;
   if (!providers || !(provider in providers)) {
     throw new Error(`Provider configuration not found: ${provider}`);
   }
@@ -168,6 +173,39 @@ export function isTest(): boolean {
   return getConfig().environment === 'test';
 }
 
+/**
+ * Generic config builder for creating app-specific configurations
+ * @param schema - Zod schema for your app config
+ * @param options - Config manager options
+ * @returns Initialized config manager instance
+ */
+export function createAppConfig<T extends z.ZodSchema>(
+  schema: T,
+  options?: {
+    configPath?: string;
+    environment?: 'development' | 'test' | 'production';
+    loaders?: any[];
+  }
+): ConfigManager<z.infer<T>> {
+  const manager = new ConfigManager<z.infer<T>>(options);
+  return manager;
+}
+
+/**
+ * Create and initialize app config in one step
+ */
+export function initializeAppConfig<T extends z.ZodSchema>(
+  schema: T,
+  options?: {
+    configPath?: string;
+    environment?: 'development' | 'test' | 'production';
+    loaders?: any[];
+  }
+): z.infer<T> {
+  const manager = createAppConfig(schema, options);
+  return manager.initialize(schema);
+}
+
 // Export all schemas
 export * from './schemas';
 

libs/core/config/src/loaders/env.loader.ts

@@ -28,7 +28,7 @@ export class EnvLoader implements ConfigLoader {
   load(): Record<string, unknown> {
     try {
       // Load root .env file - try multiple possible locations
-      const possiblePaths = ['./.env', '../.env', '../../.env'];
+      const possiblePaths = ['./.env', '../.env', '../../.env', '../../../.env'];
       for (const path of possiblePaths) {
         this.loadEnvFile(path);
       }

libs/core/config/src/schemas/base-app.schema.ts

@@ -0,0 +1,61 @@
+import { z } from 'zod';
+import { baseConfigSchema, environmentSchema } from './base.schema';
+import { 
+  postgresConfigSchema, 
+  mongodbConfigSchema, 
+  questdbConfigSchema, 
+  dragonflyConfigSchema 
+} from './database.schema';
+import { 
+  serviceConfigSchema, 
+  loggingConfigSchema, 
+  queueConfigSchema, 
+  httpConfigSchema, 
+  webshareConfigSchema,
+  browserConfigSchema,
+  proxyConfigSchema
+} from './service.schema';
+
+/**
+ * Generic base application schema that can be extended by specific apps
+ */
+export const baseAppSchema = z.object({
+  // Basic app info
+  name: z.string(),
+  version: z.string(),
+  environment: environmentSchema.default('development'),
+  
+  // Service configuration
+  service: serviceConfigSchema,
+  
+  // Logging configuration
+  log: loggingConfigSchema,
+  
+  // Database configuration - apps can choose which databases they need
+  database: z.object({
+    postgres: postgresConfigSchema.optional(),
+    mongodb: mongodbConfigSchema.optional(),
+    questdb: questdbConfigSchema.optional(),
+    dragonfly: dragonflyConfigSchema.optional(),
+  }).optional(),
+  
+  // Redis configuration (used for cache and queue)
+  redis: dragonflyConfigSchema.optional(),
+  
+  // Queue configuration
+  queue: queueConfigSchema.optional(),
+  
+  // HTTP client configuration
+  http: httpConfigSchema.optional(),
+  
+  // WebShare proxy configuration
+  webshare: webshareConfigSchema.optional(),
+  
+  // Browser configuration
+  browser: browserConfigSchema.optional(),
+  
+  // Proxy manager configuration
+  proxy: proxyConfigSchema.optional(),
+});
+
+export type BaseAppConfig = z.infer<typeof baseAppSchema>;

libs/core/config/src/schemas/database.schema.ts

@@ -31,15 +31,16 @@ export const questdbConfigSchema = z.object({
 // MongoDB configuration
 export const mongodbConfigSchema = z.object({
   enabled: z.boolean().default(true),
-  uri: z.string().url().optional(),
-  host: z.string().default('localhost'),
-  port: z.number().default(27017),
-  database: z.string(),
+  uri: z.string().url(), // URI is required and contains all connection info
+  database: z.string(), // Database name for reference
+  poolSize: z.number().min(1).max(100).default(10),
+  // Optional fields for cases where URI parsing might fail
+  host: z.string().default('localhost').optional(),
+  port: z.number().default(27017).optional(),
   user: z.string().optional(),
   password: z.string().optional(),
-  authSource: z.string().default('admin'),
+  authSource: z.string().default('admin').optional(),
   replicaSet: z.string().optional(),
-  poolSize: z.number().min(1).max(100).default(10),
 });
 
 // Dragonfly/Redis configuration

libs/core/config/src/schemas/index.ts

@@ -1,116 +1,25 @@
-import { z } from 'zod';
-import { baseConfigSchema, environmentSchema } from './base.schema';
-import { providerConfigSchema, webshareProviderConfigSchema } from './provider.schema';
-import { httpConfigSchema, queueConfigSchema } from './service.schema';
-
+// Export all schema modules
 export * from './base.schema';
 export * from './database.schema';
-export * from './provider.schema';
 export * from './service.schema';
+export * from './base-app.schema';
 
-// Flexible service schema with defaults
-const flexibleServiceConfigSchema = z
-  .object({
-    name: z.string().default('default-service'),
-    port: z.number().min(1).max(65535).default(3000),
-    host: z.string().default('0.0.0.0'),
-    healthCheckPath: z.string().default('/health'),
-    metricsPath: z.string().default('/metrics'),
-    shutdownTimeout: z.number().default(30000),
-    cors: z
-      .object({
-        enabled: z.boolean().default(true),
-        origin: z.union([z.string(), z.array(z.string())]).default('*'),
-        credentials: z.boolean().default(true),
-      })
-      .default({}),
-  })
-  .default({});
+// Export provider schemas temporarily for backward compatibility
+// These will be moved to stock-specific config
+export * from './provider.schema';
 
-// Flexible database schema with defaults
-const flexibleDatabaseConfigSchema = z
-  .object({
-    postgres: z
-      .object({
-        host: z.string().default('localhost'),
-        port: z.number().default(5432),
-        database: z.string().default('test_db'),
-        user: z.string().default('test_user'),
-        password: z.string().default('test_pass'),
-        ssl: z.boolean().default(false),
-        poolSize: z.number().min(1).max(100).default(10),
-        connectionTimeout: z.number().default(30000),
-        idleTimeout: z.number().default(10000),
-      })
-      .default({}),
-    questdb: z
-      .object({
-        host: z.string().default('localhost'),
-        ilpPort: z.number().default(9009),
-        httpPort: z.number().default(9000),
-        pgPort: z.number().default(8812),
-        database: z.string().default('questdb'),
-        user: z.string().default('admin'),
-        password: z.string().default('quest'),
-        bufferSize: z.number().default(65536),
-        flushInterval: z.number().default(1000),
-      })
-      .default({}),
-    mongodb: z
-      .object({
-        uri: z.string().url().optional(),
-        host: z.string().default('localhost'),
-        port: z.number().default(27017),
-        database: z.string().default('test_mongo'),
-        user: z.string().optional(),
-        password: z.string().optional(),
-        authSource: z.string().default('admin'),
-        replicaSet: z.string().optional(),
-        poolSize: z.number().min(1).max(100).default(10),
-      })
-      .default({}),
-    dragonfly: z
-      .object({
-        host: z.string().default('localhost'),
-        port: z.number().default(6379),
-        password: z.string().optional(),
-        db: z.number().min(0).max(15).default(0),
-        keyPrefix: z.string().optional(),
-        ttl: z.number().optional(),
-        maxRetries: z.number().default(3),
-        retryDelay: z.number().default(100),
-      })
-      .default({}),
-  })
-  .default({});
+// Re-export commonly used schemas for convenience
+export { baseAppSchema } from './base-app.schema';
+export type { BaseAppConfig } from './base-app.schema';
 
-// Flexible log schema with defaults (renamed from logging)
-const flexibleLogConfigSchema = z
-  .object({
-    level: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'),
-    format: z.enum(['json', 'pretty']).default('json'),
-    hideObject: z.boolean().default(false),
-    loki: z
-      .object({
-        enabled: z.boolean().default(false),
-        host: z.string().default('localhost'),
-        port: z.number().default(3100),
-        labels: z.record(z.string()).default({}),
-      })
-      .optional(),
-  })
-  .default({});
+// Keep AppConfig for backward compatibility (deprecated)
+// @deprecated Use baseAppSchema and extend it for your specific app
+import { z } from 'zod';
+import { baseAppSchema } from './base-app.schema';
+import { providerConfigSchema } from './provider.schema';
 
-// Complete application configuration schema
-export const appConfigSchema = baseConfigSchema.extend({
-  environment: environmentSchema.default('development'),
-  service: flexibleServiceConfigSchema,
-  log: flexibleLogConfigSchema,
-  database: flexibleDatabaseConfigSchema,
-  queue: queueConfigSchema.optional(),
-  http: httpConfigSchema.optional(),
+export const appConfigSchema = baseAppSchema.extend({
   providers: providerConfigSchema.optional(),
-  webshare: webshareProviderConfigSchema.optional(),
 });
 
 export type AppConfig = z.infer<typeof appConfigSchema>;

libs/core/config/src/schemas/service.schema.ts

@@ -21,6 +21,7 @@ export const serviceConfigSchema = z.object({
 export const loggingConfigSchema = z.object({
   level: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'),
   format: z.enum(['json', 'pretty']).default('json'),
+  hideObject: z.boolean().default(false),
   loki: z
     .object({
       enabled: z.boolean().default(false),
@@ -33,12 +34,17 @@ export const loggingConfigSchema = z.object({
 
 // Queue configuration
 export const queueConfigSchema = z.object({
+  enabled: z.boolean().default(true),
   redis: z.object({
     host: z.string().default('localhost'),
     port: z.number().default(6379),
     password: z.string().optional(),
     db: z.number().default(1),
   }),
+  workers: z.number().default(1),
+  concurrency: z.number().default(1),
+  enableScheduledJobs: z.boolean().default(true),
+  delayWorkerStart: z.boolean().default(false),
   defaultJobOptions: z
     .object({
       attempts: z.number().default(3),
@@ -48,8 +54,9 @@ export const queueConfigSchema = z.object({
           delay: z.number().default(1000),
         })
         .default({}),
-      removeOnComplete: z.number().default(10),
-      removeOnFail: z.number().default(5),
+      removeOnComplete: z.number().default(100),
+      removeOnFail: z.number().default(50),
+      timeout: z.number().optional(),
     })
     .default({}),
 });
@@ -73,3 +80,22 @@ export const httpConfigSchema = z.object({
     })
     .optional(),
 });
+
+// WebShare proxy service configuration
+export const webshareConfigSchema = z.object({
+  apiKey: z.string().optional(),
+  apiUrl: z.string().default('https://proxy.webshare.io/api/v2/'),
+  enabled: z.boolean().default(true),
+});
+
+// Browser configuration
+export const browserConfigSchema = z.object({
+  headless: z.boolean().default(true),
+  timeout: z.number().default(30000),
+});
+
+// Proxy manager configuration
+export const proxyConfigSchema = z.object({
+  cachePrefix: z.string().default('proxy:'),
+  ttl: z.number().default(3600),
+});

libs/core/di/src/config/schemas/service.schema.ts

@@ -12,6 +12,20 @@ export const browserConfigSchema = z.object({
 
 export const queueConfigSchema = z.object({
   enabled: z.boolean().optional().default(true),
+  workers: z.number().optional().default(1),
+  concurrency: z.number().optional().default(1),
+  enableScheduledJobs: z.boolean().optional().default(true),
+  delayWorkerStart: z.boolean().optional().default(false),
+  defaultJobOptions: z.object({
+    attempts: z.number().default(3),
+    backoff: z.object({
+      type: z.enum(['exponential', 'fixed']).default('exponential'),
+      delay: z.number().default(1000),
+    }).default({}),
+    removeOnComplete: z.number().default(100),
+    removeOnFail: z.number().default(50),
+    timeout: z.number().optional(),
+  }).optional().default({}),
 });
 
 export type ProxyConfig = z.infer<typeof proxyConfigSchema>;

libs/core/di/src/container/builder.ts

@@ -73,26 +73,24 @@ export class ServiceContainerBuilder {
 
   private applyServiceOptions(config: Partial<AppConfig>): AppConfig {
     return {
-      redis: {
+      redis: config.redis || {
         enabled: this.options.enableCache ?? true,
-        host: config.redis?.host || 'localhost',
-        port: config.redis?.port || 6379,
-        password: config.redis?.password,
-        username: config.redis?.username,
-        db: config.redis?.db || 0,
+        host: 'localhost',
+        port: 6379,
+        db: 0,
       },
-      mongodb: {
+      mongodb: config.mongodb || {
         enabled: this.options.enableMongoDB ?? true,
-        uri: config.mongodb?.uri || '',
-        database: config.mongodb?.database || '',
+        uri: '',
+        database: '',
       },
-      postgres: {
+      postgres: config.postgres || {
         enabled: this.options.enablePostgres ?? true,
-        host: config.postgres?.host || 'localhost',
-        port: config.postgres?.port || 5432,
-        database: config.postgres?.database || '',
-        user: config.postgres?.user || '',
-        password: config.postgres?.password || '',
+        host: 'localhost',
+        port: 5432,
+        database: 'postgres',
+        user: 'postgres',
+        password: 'postgres',
       },
       questdb: this.options.enableQuestDB ? (config.questdb || {
         enabled: true,
@@ -104,7 +102,19 @@ export class ServiceContainerBuilder {
       }) : undefined,
       proxy: this.options.enableProxy ? (config.proxy || { cachePrefix: 'proxy:', ttl: 3600 }) : undefined,
       browser: this.options.enableBrowser ? (config.browser || { headless: true, timeout: 30000 }) : undefined,
-      queue: this.options.enableQueue ? { enabled: this.options.enableQueue } : undefined,
+      queue: this.options.enableQueue ? (config.queue || { 
+        enabled: true,
+        workers: 1,
+        concurrency: 1,
+        enableScheduledJobs: true,
+        delayWorkerStart: false,
+        defaultJobOptions: {
+          attempts: 3,
+          backoff: { type: 'exponential' as const, delay: 1000 },
+          removeOnComplete: 100,
+          removeOnFail: 50,
+        }
+      }) : undefined,
     };
   }
 
@@ -133,8 +143,8 @@ export class ServiceContainerBuilder {
   }
 
   private transformStockBotConfig(config: AppConfig | StockBotAppConfig): Partial<AppConfig> {
-    // If it's already in the new format, return as is
-    if ('redis' in config) {
+    // If it's already in the new format (has redis AND postgres at top level), return as is
+    if ('redis' in config && 'postgres' in config && 'mongodb' in config) {
       return config as AppConfig;
     }
 
@@ -149,18 +159,17 @@ export class ServiceContainerBuilder {
         db: stockBotConfig.database.dragonfly.db || 0,
       } : undefined,
       mongodb: stockBotConfig.database?.mongodb ? {
-        enabled: true,
-        uri: stockBotConfig.database.mongodb.uri || 
-          `mongodb://${stockBotConfig.database.mongodb.user || ''}:${stockBotConfig.database.mongodb.password || ''}@${stockBotConfig.database.mongodb.host || 'localhost'}:${stockBotConfig.database.mongodb.port || 27017}/${stockBotConfig.database.mongodb.database || 'test'}?authSource=${stockBotConfig.database.mongodb.authSource || 'admin'}`,
-        database: stockBotConfig.database.mongodb.database || 'test',
+        enabled: stockBotConfig.database.mongodb.enabled ?? true,
+        uri: stockBotConfig.database.mongodb.uri,
+        database: stockBotConfig.database.mongodb.database,
       } : undefined,
       postgres: stockBotConfig.database?.postgres ? {
-        enabled: true,
-        host: stockBotConfig.database.postgres.host || 'localhost',
-        port: stockBotConfig.database.postgres.port || 5432,
-        database: stockBotConfig.database.postgres.database || 'test',
-        user: stockBotConfig.database.postgres.user || 'test',
-        password: stockBotConfig.database.postgres.password || 'test',
+        enabled: stockBotConfig.database.postgres.enabled ?? true,
+        host: stockBotConfig.database.postgres.host,
+        port: stockBotConfig.database.postgres.port,
+        database: stockBotConfig.database.postgres.database,
+        user: stockBotConfig.database.postgres.user,
+        password: stockBotConfig.database.postgres.password,
       } : undefined,
       questdb: stockBotConfig.database?.questdb ? {
         enabled: true,
@@ -170,6 +179,9 @@ export class ServiceContainerBuilder {
         influxPort: stockBotConfig.database.questdb.ilpPort || 9009,
         database: stockBotConfig.database.questdb.database || 'questdb',
       } : undefined,
+      queue: stockBotConfig.queue,
+      browser: stockBotConfig.browser,
+      proxy: stockBotConfig.proxy,
     };
   }
 }

libs/core/di/src/registrations/database.registration.ts

@@ -20,7 +20,7 @@ export function registerDatabaseServices(
           port: parseInt(uriMatch?.[4] || '27017'),
           database: config.mongodb.database,
           username: uriMatch?.[1],
-          password: uriMatch?.[2],
+          password: uriMatch?.[2] ? String(uriMatch?.[2]) : undefined,
           authSource: uriMatch?.[6] || 'admin',
           uri: config.mongodb.uri,
         };
@@ -37,16 +37,19 @@ export function registerDatabaseServices(
   if (config.postgres.enabled) {
     container.register({
       postgresClient: asFunction(({ logger }) => {
-        return new PostgreSQLClient(
-          {
-            host: config.postgres.host,
-            port: config.postgres.port,
-            database: config.postgres.database,
-            username: config.postgres.user,
-            password: config.postgres.password,
-          },
-          logger
-        );
+        const pgConfig = {
+          host: config.postgres.host,
+          port: config.postgres.port,
+          database: config.postgres.database,
+          username: config.postgres.user,
+          password: String(config.postgres.password), // Ensure password is a string
+        };
+        
+        logger.debug('PostgreSQL config:', { 
+          ...pgConfig, 
+          password: pgConfig.password ? '***' : 'NO_PASSWORD',
+        });
+        return new PostgreSQLClient(pgConfig, logger);
       }).singleton(),
     });
   } else {

libs/core/di/src/registrations/service.registration.ts

@@ -56,19 +56,12 @@ export function registerApplicationServices(
             db: config.redis.db,
           },
           defaultQueueOptions: {
-            workers: 1,
-            concurrency: 1,
-            defaultJobOptions: {
-              removeOnComplete: 100,
-              removeOnFail: 50,
-              attempts: 3,
-              backoff: {
-                type: 'exponential',
-                delay: 1000,
-              },
-            },
+            workers: config.queue!.workers || 1,
+            concurrency: config.queue!.concurrency || 1,
+            defaultJobOptions: config.queue!.defaultJobOptions,
           },
-          enableScheduledJobs: true,
+          enableScheduledJobs: config.queue!.enableScheduledJobs ?? true,
+          delayWorkerStart: config.queue!.delayWorkerStart ?? false,
         };
         return new QueueManager(queueConfig, logger);
       }).singleton(),