created new config lib

libs/config-new/src/cli.ts

@@ -0,0 +1,194 @@
+#!/usr/bin/env bun
+import { parseArgs } from 'util';
+import { join } from 'path';
+import { ConfigManager } from './config-manager';
+import { appConfigSchema } from './schemas';
+import { 
+  validateConfig, 
+  formatValidationResult, 
+  checkDeprecations,
+  checkRequiredEnvVars,
+  validateCompleteness
+} from './utils/validation';
+import { redactSecrets } from './utils/secrets';
+
+interface CliOptions {
+  config?: string;
+  env?: string;
+  validate?: boolean;
+  show?: boolean;
+  check?: boolean;
+  json?: boolean;
+  help?: boolean;
+}
+
+const DEPRECATIONS = {
+  'service.legacyMode': 'Use service.mode instead',
+  'database.redis': 'Use database.dragonfly instead',
+};
+
+const REQUIRED_PATHS = [
+  'service.name',
+  'service.port',
+  'database.postgres.host',
+  'database.postgres.database',
+];
+
+const REQUIRED_ENV_VARS = [
+  'NODE_ENV',
+];
+
+const SECRET_PATHS = [
+  'database.postgres.password',
+  'database.mongodb.uri',
+  'providers.quoteMedia.apiKey',
+  'providers.interactiveBrokers.clientId',
+];
+
+function printUsage() {
+  console.log(`
+Stock Bot Configuration CLI
+
+Usage: bun run config-cli [options]
+
+Options:
+  --config <path>   Path to config directory (default: ./config)
+  --env <env>       Environment to use (development, test, production)
+  --validate        Validate configuration against schema
+  --show            Show current configuration (secrets redacted)
+  --check           Run all configuration checks
+  --json            Output in JSON format
+  --help            Show this help message
+
+Examples:
+  # Validate configuration
+  bun run config-cli --validate
+
+  # Show configuration for production
+  bun run config-cli --env production --show
+
+  # Run all checks
+  bun run config-cli --check
+
+  # Output configuration as JSON
+  bun run config-cli --show --json
+`);
+}
+
+async function main() {
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      config: { type: 'string' },
+      env: { type: 'string' },
+      validate: { type: 'boolean' },
+      show: { type: 'boolean' },
+      check: { type: 'boolean' },
+      json: { type: 'boolean' },
+      help: { type: 'boolean' },
+    },
+  }) as { values: CliOptions };
+
+  if (values.help) {
+    printUsage();
+    process.exit(0);
+  }
+
+  const configPath = values.config || join(process.cwd(), 'config');
+  const environment = values.env as any;
+
+  try {
+    const manager = new ConfigManager({
+      configPath,
+      environment,
+    });
+
+    const config = await manager.initialize(appConfigSchema);
+
+    if (values.validate) {
+      const result = validateConfig(config, appConfigSchema);
+      
+      if (values.json) {
+        console.log(JSON.stringify(result, null, 2));
+      } else {
+        console.log(formatValidationResult(result));
+      }
+      
+      process.exit(result.valid ? 0 : 1);
+    }
+
+    if (values.show) {
+      const redacted = redactSecrets(config, SECRET_PATHS);
+      
+      if (values.json) {
+        console.log(JSON.stringify(redacted, null, 2));
+      } else {
+        console.log('Current Configuration:');
+        console.log(JSON.stringify(redacted, null, 2));
+      }
+    }
+
+    if (values.check) {
+      console.log('Running configuration checks...\n');
+      
+      // Schema validation
+      console.log('1. Schema Validation:');
+      const schemaResult = validateConfig(config, appConfigSchema);
+      console.log(formatValidationResult(schemaResult));
+      console.log();
+      
+      // Environment variables
+      console.log('2. Required Environment Variables:');
+      const envResult = checkRequiredEnvVars(REQUIRED_ENV_VARS);
+      console.log(formatValidationResult(envResult));
+      console.log();
+      
+      // Required paths
+      console.log('3. Required Configuration Paths:');
+      const pathResult = validateCompleteness(config, REQUIRED_PATHS);
+      console.log(formatValidationResult(pathResult));
+      console.log();
+      
+      // Deprecations
+      console.log('4. Deprecation Warnings:');
+      const warnings = checkDeprecations(config, DEPRECATIONS);
+      if (warnings && warnings.length > 0) {
+        for (const warning of warnings) {
+          console.log(`  ⚠️  ${warning.path}: ${warning.message}`);
+        }
+      } else {
+        console.log('  ✅ No deprecated options found');
+      }
+      console.log();
+      
+      // Overall result
+      const allValid = schemaResult.valid && envResult.valid && pathResult.valid;
+      
+      if (allValid) {
+        console.log('✅ All configuration checks passed!');
+        process.exit(0);
+      } else {
+        console.log('❌ Some configuration checks failed');
+        process.exit(1);
+      }
+    }
+
+    if (!values.validate && !values.show && !values.check) {
+      console.log('No action specified. Use --help for usage information.');
+      process.exit(1);
+    }
+
+  } catch (error) {
+    if (values.json) {
+      console.error(JSON.stringify({ error: String(error) }));
+    } else {
+      console.error('Error:', error);
+    }
+    process.exit(1);
+  }
+}
+
+// Run CLI
+if (import.meta.main) {
+  main();
+}

libs/config-new/src/config-manager.ts

@@ -0,0 +1,220 @@
+import { join } from 'path';
+import { z } from 'zod';
+import { 
+  ConfigManagerOptions, 
+  Environment, 
+  ConfigLoader, 
+  DeepPartial,
+  ConfigSchema
+} from './types';
+import { ConfigError, ConfigValidationError } from './errors';
+import { EnvLoader } from './loaders/env.loader';
+import { FileLoader } from './loaders/file.loader';
+
+export class ConfigManager<T = Record<string, unknown>> {
+  private config: T | null = null;
+  private loaders: ConfigLoader[];
+  private environment: Environment;
+  private schema?: ConfigSchema;
+
+  constructor(options: ConfigManagerOptions = {}) {
+    this.environment = options.environment || this.detectEnvironment();
+    
+    // Default loaders if none provided
+    if (options.loaders) {
+      this.loaders = options.loaders;
+    } else {
+      const configPath = options.configPath || join(process.cwd(), 'config');
+      this.loaders = [
+        new FileLoader(configPath, this.environment),
+        new EnvLoader('STOCKBOT_'), // Prefix for env vars
+      ];
+    }
+  }
+
+  /**
+   * Initialize the configuration by loading from all sources
+   */
+  async initialize(schema?: ConfigSchema): Promise<T> {
+    if (this.config) {
+      return this.config;
+    }
+
+    this.schema = schema;
+
+    // Sort loaders by priority (higher priority last)
+    const sortedLoaders = [...this.loaders].sort((a, b) => a.priority - b.priority);
+
+    // Load configurations from all sources
+    const configs: Record<string, unknown>[] = [];
+    for (const loader of sortedLoaders) {
+      const config = await loader.load();
+      if (config && Object.keys(config).length > 0) {
+        configs.push(config);
+      }
+    }
+
+    // Merge all configurations
+    const mergedConfig = this.deepMerge(...configs) as T;
+
+    // Add environment if not present
+    if (typeof mergedConfig === 'object' && mergedConfig !== null && !('environment' in mergedConfig)) {
+      (mergedConfig as any).environment = this.environment;
+    }
+
+    // Validate if schema provided
+    if (this.schema) {
+      try {
+        this.config = this.schema.parse(mergedConfig) as T;
+      } catch (error) {
+        if (error instanceof z.ZodError) {
+          throw new ConfigValidationError(
+            'Configuration validation failed',
+            error.errors
+          );
+        }
+        throw error;
+      }
+    } else {
+      this.config = mergedConfig;
+    }
+
+    return this.config;
+  }
+
+  /**
+   * Get the current configuration
+   */
+  get(): T {
+    if (!this.config) {
+      throw new ConfigError('Configuration not initialized. Call initialize() first.');
+    }
+    return this.config;
+  }
+
+  /**
+   * Get a specific configuration value by path
+   */
+  getValue<R = unknown>(path: string): R {
+    const config = this.get();
+    const keys = path.split('.');
+    let value: any = config;
+
+    for (const key of keys) {
+      if (value && typeof value === 'object' && key in value) {
+        value = value[key];
+      } else {
+        throw new ConfigError(`Configuration key not found: ${path}`);
+      }
+    }
+
+    return value as R;
+  }
+
+  /**
+   * Check if a configuration path exists
+   */
+  has(path: string): boolean {
+    try {
+      this.getValue(path);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Update configuration at runtime (useful for testing)
+   */
+  set(updates: DeepPartial<T>): void {
+    if (!this.config) {
+      throw new ConfigError('Configuration not initialized. Call initialize() first.');
+    }
+
+    const updated = this.deepMerge(this.config as any, updates as any) as T;
+
+    // Re-validate if schema is present
+    if (this.schema) {
+      try {
+        this.config = this.schema.parse(updated) as T;
+      } catch (error) {
+        if (error instanceof z.ZodError) {
+          throw new ConfigValidationError(
+            'Configuration validation failed after update',
+            error.errors
+          );
+        }
+        throw error;
+      }
+    } else {
+      this.config = updated;
+    }
+  }
+
+  /**
+   * Get the current environment
+   */
+  getEnvironment(): Environment {
+    return this.environment;
+  }
+
+  /**
+   * Reset configuration (useful for testing)
+   */
+  reset(): void {
+    this.config = null;
+  }
+
+  /**
+   * Validate configuration against a schema
+   */
+  validate<S extends ConfigSchema>(schema: S): z.infer<S> {
+    const config = this.get();
+    return schema.parse(config);
+  }
+
+  /**
+   * Create a typed configuration getter
+   */
+  createTypedGetter<S extends z.ZodSchema>(schema: S): () => z.infer<S> {
+    return () => this.validate(schema);
+  }
+
+  private detectEnvironment(): Environment {
+    const env = process.env.NODE_ENV?.toLowerCase();
+    switch (env) {
+      case 'production':
+      case 'prod':
+        return 'production';
+      case 'test':
+        return 'test';
+      case 'development':
+      case 'dev':
+      default:
+        return 'development';
+    }
+  }
+
+  private deepMerge(...objects: Record<string, any>[]): Record<string, any> {
+    const result: Record<string, any> = {};
+
+    for (const obj of objects) {
+      for (const [key, value] of Object.entries(obj)) {
+        if (value === null || value === undefined) {
+          result[key] = value;
+        } else if (
+          typeof value === 'object' && 
+          !Array.isArray(value) &&
+          !(value instanceof Date) &&
+          !(value instanceof RegExp)
+        ) {
+          result[key] = this.deepMerge(result[key] || {}, value);
+        } else {
+          result[key] = value;
+        }
+      }
+    }
+
+    return result;
+  }
+}

libs/config-new/src/errors.ts

@@ -0,0 +1,20 @@
+export class ConfigError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ConfigError';
+  }
+}
+
+export class ConfigValidationError extends ConfigError {
+  constructor(message: string, public errors: unknown) {
+    super(message);
+    this.name = 'ConfigValidationError';
+  }
+}
+
+export class ConfigLoaderError extends ConfigError {
+  constructor(message: string, public loader: string) {
+    super(`${loader}: ${message}`);
+    this.name = 'ConfigLoaderError';
+  }
+}

libs/config-new/src/index.ts

@@ -0,0 +1,104 @@
+// Export all schemas
+export * from './schemas';
+
+// Export types
+export * from './types';
+
+// Export errors
+export * from './errors';
+
+// Export loaders
+export { EnvLoader } from './loaders/env.loader';
+export { FileLoader } from './loaders/file.loader';
+
+// Export ConfigManager
+export { ConfigManager } from './config-manager';
+
+// Export utilities
+export * from './utils/secrets';
+export * from './utils/validation';
+
+// Import necessary types for singleton
+import { ConfigManager } from './config-manager';
+import { AppConfig, appConfigSchema } from './schemas';
+
+// Create singleton instance
+let configInstance: ConfigManager<AppConfig> | null = null;
+
+/**
+ * Initialize the global configuration
+ */
+export async function initializeConfig(
+  configPath?: string
+): Promise<AppConfig> {
+  if (!configInstance) {
+    configInstance = new ConfigManager<AppConfig>({
+      configPath,
+    });
+  }
+  return configInstance.initialize(appConfigSchema);
+}
+
+/**
+ * Get the current configuration
+ */
+export function getConfig(): AppConfig {
+  if (!configInstance) {
+    throw new Error('Configuration not initialized. Call initializeConfig() first.');
+  }
+  return configInstance.get();
+}
+
+/**
+ * Get configuration manager instance
+ */
+export function getConfigManager(): ConfigManager<AppConfig> {
+  if (!configInstance) {
+    throw new Error('Configuration not initialized. Call initializeConfig() first.');
+  }
+  return configInstance;
+}
+
+/**
+ * Reset configuration (useful for testing)
+ */
+export function resetConfig(): void {
+  if (configInstance) {
+    configInstance.reset();
+    configInstance = null;
+  }
+}
+
+// Export convenience functions for common configs
+export function getDatabaseConfig() {
+  return getConfig().database;
+}
+
+export function getServiceConfig() {
+  return getConfig().service;
+}
+
+export function getLoggingConfig() {
+  return getConfig().logging;
+}
+
+export function getProviderConfig(provider: string) {
+  const providers = getConfig().providers;
+  if (!providers || !(provider in providers)) {
+    throw new Error(`Provider configuration not found: ${provider}`);
+  }
+  return (providers as any)[provider];
+}
+
+// Export environment helpers
+export function isDevelopment(): boolean {
+  return getConfig().environment === 'development';
+}
+
+export function isProduction(): boolean {
+  return getConfig().environment === 'production';
+}
+
+export function isTest(): boolean {
+  return getConfig().environment === 'test';
+}

libs/config-new/src/loaders/env.loader.ts

@@ -0,0 +1,127 @@
+import { ConfigLoader } from '../types';
+import { ConfigLoaderError } from '../errors';
+
+export interface EnvLoaderOptions {
+  convertCase?: boolean;
+  parseJson?: boolean;
+  parseValues?: boolean;
+  nestedDelimiter?: string;
+}
+
+export class EnvLoader implements ConfigLoader {
+  readonly priority = 100; // Highest priority
+
+  constructor(
+    private prefix = '',
+    private options: EnvLoaderOptions = {}
+  ) {
+    this.options = {
+      convertCase: false,
+      parseJson: true,
+      parseValues: true,
+      nestedDelimiter: '_',
+      ...options
+    };
+  }
+
+  async load(): Promise<Record<string, unknown>> {
+    try {
+      const config: Record<string, unknown> = {};
+      const envVars = process.env;
+
+      for (const [key, value] of Object.entries(envVars)) {
+        if (this.prefix && !key.startsWith(this.prefix)) {
+          continue;
+        }
+
+        const configKey = this.prefix
+          ? key.slice(this.prefix.length)
+          : key;
+
+        if (!this.options.convertCase && !this.options.nestedDelimiter) {
+          // Simple case - just keep the key as is
+          config[configKey] = this.parseValue(value || '');
+        } else {
+          // Handle nested structure or case conversion
+          this.setConfigValue(config, configKey, value || '');
+        }
+      }
+
+      return config;
+    } catch (error) {
+      throw new ConfigLoaderError(
+        `Failed to load environment variables: ${error}`,
+        'EnvLoader'
+      );
+    }
+  }
+
+  private setConfigValue(config: Record<string, any>, key: string, value: string): void {
+    const parsedValue = this.parseValue(value);
+
+    if (this.options.nestedDelimiter && key.includes(this.options.nestedDelimiter)) {
+      // Handle nested delimiter (e.g., APP__NAME -> { APP: { NAME: value } })
+      const parts = key.split(this.options.nestedDelimiter);
+      this.setNestedValue(config, parts, parsedValue);
+    } else if (this.options.convertCase) {
+      // Convert to camelCase
+      const camelKey = this.toCamelCase(key);
+      config[camelKey] = parsedValue;
+    } else {
+      // Convert to nested structure based on underscores
+      const path = key.toLowerCase().split('_');
+      this.setNestedValue(config, path, parsedValue);
+    }
+  }
+
+  private setNestedValue(obj: Record<string, any>, path: string[], value: unknown): void {
+    const lastKey = path.pop()!;
+    const target = path.reduce((acc, key) => {
+      if (!acc[key]) {
+        acc[key] = {};
+      }
+      return acc[key];
+    }, obj);
+    target[lastKey] = value;
+  }
+
+  private toCamelCase(str: string): string {
+    return str
+      .toLowerCase()
+      .replace(/_([a-z])/g, (_, char) => char.toUpperCase());
+  }
+
+  private parseValue(value: string): unknown {
+    if (!this.options.parseValues && !this.options.parseJson) {
+      return value;
+    }
+
+    // Try to parse as JSON first if enabled
+    if (this.options.parseJson) {
+      try {
+        return JSON.parse(value);
+      } catch {
+        // Not JSON, continue with other parsing
+      }
+    }
+
+    if (!this.options.parseValues) {
+      return value;
+    }
+
+    // Handle booleans
+    if (value.toLowerCase() === 'true') return true;
+    if (value.toLowerCase() === 'false') return false;
+
+    // Handle numbers
+    const num = Number(value);
+    if (!isNaN(num) && value !== '') return num;
+
+    // Handle null/undefined
+    if (value.toLowerCase() === 'null') return null;
+    if (value.toLowerCase() === 'undefined') return undefined;
+
+    // Return as string
+    return value;
+  }
+}

libs/config-new/src/loaders/file.loader.ts

@@ -0,0 +1,72 @@
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+import { ConfigLoader } from '../types';
+import { ConfigLoaderError } from '../errors';
+
+export class FileLoader implements ConfigLoader {
+  readonly priority = 50; // Medium priority
+
+  constructor(
+    private configPath: string,
+    private environment: string
+  ) {}
+
+  async load(): Promise<Record<string, unknown>> {
+    try {
+      const configs: Record<string, unknown>[] = [];
+
+      // Load default config
+      const defaultConfig = await this.loadFile('default.json');
+      if (defaultConfig) {
+        configs.push(defaultConfig);
+      }
+
+      // Load environment-specific config
+      const envConfig = await this.loadFile(`${this.environment}.json`);
+      if (envConfig) {
+        configs.push(envConfig);
+      }
+
+      // Merge configs (later configs override earlier ones)
+      return this.deepMerge(...configs);
+    } catch (error) {
+      throw new ConfigLoaderError(
+        `Failed to load configuration files: ${error}`,
+        'FileLoader'
+      );
+    }
+  }
+
+  private async loadFile(filename: string): Promise<Record<string, unknown> | null> {
+    const filepath = join(this.configPath, filename);
+    
+    try {
+      const content = await readFile(filepath, 'utf-8');
+      return JSON.parse(content);
+    } catch (error: any) {
+      // File not found is not an error (configs are optional)
+      if (error.code === 'ENOENT') {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  private deepMerge(...objects: Record<string, any>[]): Record<string, any> {
+    const result: Record<string, any> = {};
+
+    for (const obj of objects) {
+      for (const [key, value] of Object.entries(obj)) {
+        if (value === null || value === undefined) {
+          result[key] = value;
+        } else if (typeof value === 'object' && !Array.isArray(value)) {
+          result[key] = this.deepMerge(result[key] || {}, value);
+        } else {
+          result[key] = value;
+        }
+      }
+    }
+
+    return result;
+  }
+}

libs/config-new/src/schemas/base.schema.ts

@@ -0,0 +1,10 @@
+import { z } from 'zod';
+
+export const environmentSchema = z.enum(['development', 'test', 'production']);
+
+export const baseConfigSchema = z.object({
+  environment: environmentSchema.optional(),
+  name: z.string().optional(),
+  version: z.string().optional(),
+  debug: z.boolean().default(false),
+});

libs/config-new/src/schemas/database.schema.ts

@@ -0,0 +1,60 @@
+import { z } from 'zod';
+
+// PostgreSQL configuration
+export const postgresConfigSchema = z.object({
+  host: z.string().default('localhost'),
+  port: z.number().default(5432),
+  database: z.string(),
+  user: z.string(),
+  password: z.string(),
+  ssl: z.boolean().default(false),
+  poolSize: z.number().min(1).max(100).default(10),
+  connectionTimeout: z.number().default(30000),
+  idleTimeout: z.number().default(10000),
+});
+
+// QuestDB configuration
+export const questdbConfigSchema = z.object({
+  host: z.string().default('localhost'),
+  ilpPort: z.number().default(9009),
+  httpPort: z.number().default(9000),
+  pgPort: z.number().default(8812),
+  database: z.string().default('questdb'),
+  user: z.string().default('admin'),
+  password: z.string().default('quest'),
+  bufferSize: z.number().default(65536),
+  flushInterval: z.number().default(1000),
+});
+
+// MongoDB configuration
+export const mongodbConfigSchema = z.object({
+  uri: z.string().url().optional(),
+  host: z.string().default('localhost'),
+  port: z.number().default(27017),
+  database: z.string(),
+  user: z.string().optional(),
+  password: z.string().optional(),
+  authSource: z.string().default('admin'),
+  replicaSet: z.string().optional(),
+  poolSize: z.number().min(1).max(100).default(10),
+});
+
+// Dragonfly/Redis configuration
+export const dragonflyConfigSchema = z.object({
+  host: z.string().default('localhost'),
+  port: z.number().default(6379),
+  password: z.string().optional(),
+  db: z.number().min(0).max(15).default(0),
+  keyPrefix: z.string().optional(),
+  ttl: z.number().optional(),
+  maxRetries: z.number().default(3),
+  retryDelay: z.number().default(100),
+});
+
+// Combined database configuration
+export const databaseConfigSchema = z.object({
+  postgres: postgresConfigSchema,
+  questdb: questdbConfigSchema,
+  mongodb: mongodbConfigSchema,
+  dragonfly: dragonflyConfigSchema,
+});

libs/config-new/src/schemas/index.ts

@@ -0,0 +1,23 @@
+export * from './base.schema';
+export * from './database.schema';
+export * from './service.schema';
+export * from './provider.schema';
+
+import { z } from 'zod';
+import { baseConfigSchema, environmentSchema } from './base.schema';
+import { databaseConfigSchema } from './database.schema';
+import { serviceConfigSchema, loggingConfigSchema, queueConfigSchema, httpConfigSchema } from './service.schema';
+import { providerConfigSchema } from './provider.schema';
+
+// Complete application configuration schema
+export const appConfigSchema = baseConfigSchema.extend({
+  environment: environmentSchema.default('development'),
+  service: serviceConfigSchema,
+  logging: loggingConfigSchema,
+  database: databaseConfigSchema,
+  queue: queueConfigSchema.optional(),
+  http: httpConfigSchema.optional(),
+  providers: providerConfigSchema.optional(),
+});
+
+export type AppConfig = z.infer<typeof appConfigSchema>;

libs/config-new/src/schemas/provider.schema.ts

@@ -0,0 +1,65 @@
+import { z } from 'zod';
+
+// Base provider configuration
+export const baseProviderConfigSchema = z.object({
+  name: z.string(),
+  enabled: z.boolean().default(true),
+  priority: z.number().default(0),
+  rateLimit: z.object({
+    maxRequests: z.number().default(100),
+    windowMs: z.number().default(60000),
+  }).optional(),
+  timeout: z.number().default(30000),
+  retries: z.number().default(3),
+});
+
+// EOD Historical Data provider
+export const eodProviderConfigSchema = baseProviderConfigSchema.extend({
+  apiKey: z.string(),
+  baseUrl: z.string().default('https://eodhistoricaldata.com/api'),
+  tier: z.enum(['free', 'fundamentals', 'all-in-one']).default('free'),
+});
+
+// Interactive Brokers provider
+export const ibProviderConfigSchema = baseProviderConfigSchema.extend({
+  gateway: z.object({
+    host: z.string().default('localhost'),
+    port: z.number().default(5000),
+    clientId: z.number().default(1),
+  }),
+  account: z.string().optional(),
+  marketDataType: z.enum(['live', 'delayed', 'frozen']).default('delayed'),
+});
+
+// QuoteMedia provider
+export const qmProviderConfigSchema = baseProviderConfigSchema.extend({
+  username: z.string(),
+  password: z.string(),
+  baseUrl: z.string().default('https://app.quotemedia.com/quotetools'),
+  webmasterId: z.string(),
+});
+
+// Yahoo Finance provider
+export const yahooProviderConfigSchema = baseProviderConfigSchema.extend({
+  baseUrl: z.string().default('https://query1.finance.yahoo.com'),
+  cookieJar: z.boolean().default(true),
+  crumb: z.string().optional(),
+});
+
+// Combined provider configuration
+export const providerConfigSchema = z.object({
+  eod: eodProviderConfigSchema.optional(),
+  ib: ibProviderConfigSchema.optional(),
+  qm: qmProviderConfigSchema.optional(),
+  yahoo: yahooProviderConfigSchema.optional(),
+});
+
+// Dynamic provider configuration type
+export type ProviderName = 'eod' | 'ib' | 'qm' | 'yahoo';
+
+export const providerSchemas = {
+  eod: eodProviderConfigSchema,
+  ib: ibProviderConfigSchema,
+  qm: qmProviderConfigSchema,
+  yahoo: yahooProviderConfigSchema,
+} as const;

libs/config-new/src/schemas/service.schema.ts

@@ -0,0 +1,63 @@
+import { z } from 'zod';
+
+// Common service configuration
+export const serviceConfigSchema = z.object({
+  name: z.string(),
+  port: z.number().min(1).max(65535),
+  host: z.string().default('0.0.0.0'),
+  healthCheckPath: z.string().default('/health'),
+  metricsPath: z.string().default('/metrics'),
+  shutdownTimeout: z.number().default(30000),
+  cors: z.object({
+    enabled: z.boolean().default(true),
+    origin: z.union([z.string(), z.array(z.string())]).default('*'),
+    credentials: z.boolean().default(true),
+  }).default({}),
+});
+
+// Logging configuration
+export const loggingConfigSchema = z.object({
+  level: z.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal']).default('info'),
+  format: z.enum(['json', 'pretty']).default('json'),
+  loki: z.object({
+    enabled: z.boolean().default(false),
+    host: z.string().default('localhost'),
+    port: z.number().default(3100),
+    labels: z.record(z.string()).default({}),
+  }).optional(),
+});
+
+// Queue configuration
+export const queueConfigSchema = z.object({
+  redis: z.object({
+    host: z.string().default('localhost'),
+    port: z.number().default(6379),
+    password: z.string().optional(),
+    db: z.number().default(1),
+  }),
+  defaultJobOptions: z.object({
+    attempts: z.number().default(3),
+    backoff: z.object({
+      type: z.enum(['exponential', 'fixed']).default('exponential'),
+      delay: z.number().default(1000),
+    }).default({}),
+    removeOnComplete: z.boolean().default(true),
+    removeOnFail: z.boolean().default(false),
+  }).default({}),
+});
+
+// HTTP client configuration
+export const httpConfigSchema = z.object({
+  timeout: z.number().default(30000),
+  retries: z.number().default(3),
+  retryDelay: z.number().default(1000),
+  userAgent: z.string().optional(),
+  proxy: z.object({
+    enabled: z.boolean().default(false),
+    url: z.string().url().optional(),
+    auth: z.object({
+      username: z.string(),
+      password: z.string(),
+    }).optional(),
+  }).optional(),
+});

libs/config-new/src/types/index.ts

@@ -0,0 +1,28 @@
+import { z } from 'zod';
+
+export type Environment = 'development' | 'test' | 'production';
+
+export interface ConfigLoader {
+  load(): Promise<Record<string, unknown>>;
+  readonly priority: number;
+}
+
+export interface ConfigManagerOptions {
+  environment?: Environment;
+  configPath?: string;
+  loaders?: ConfigLoader[];
+}
+
+export type DeepPartial<T> = T extends object
+  ? {
+      [P in keyof T]?: DeepPartial<T[P]>;
+    }
+  : T;
+
+export type ConfigSchema = z.ZodSchema<any>;
+
+export interface ProviderConfig {
+  name: string;
+  enabled: boolean;
+  [key: string]: unknown;
+}

libs/config-new/src/utils/secrets.ts

@@ -0,0 +1,182 @@
+import { z } from 'zod';
+
+/**
+ * Secret value wrapper to prevent accidental logging
+ */
+export class SecretValue<T = string> {
+  private readonly value: T;
+  private readonly masked: string;
+
+  constructor(value: T, mask: string = '***') {
+    this.value = value;
+    this.masked = mask;
+  }
+
+  /**
+   * Get the actual secret value
+   * @param reason - Required reason for accessing the secret
+   */
+  reveal(reason: string): T {
+    if (!reason) {
+      throw new Error('Reason required for revealing secret value');
+    }
+    return this.value;
+  }
+
+  /**
+   * Get masked representation
+   */
+  toString(): string {
+    return this.masked;
+  }
+
+  /**
+   * Prevent JSON serialization of actual value
+   */
+  toJSON(): string {
+    return this.masked;
+  }
+
+  /**
+   * Check if value matches without revealing it
+   */
+  equals(other: T): boolean {
+    return this.value === other;
+  }
+
+  /**
+   * Transform the secret value
+   */
+  map<R>(fn: (value: T) => R, reason: string): SecretValue<R> {
+    return new SecretValue(fn(this.reveal(reason)));
+  }
+}
+
+/**
+ * Zod schema for secret values
+ */
+export const secretSchema = <T extends z.ZodTypeAny>(schema: T) => {
+  return z.custom<SecretValue<z.infer<T>>>(
+    (val) => val instanceof SecretValue,
+    {
+      message: 'Expected SecretValue instance',
+    }
+  );
+};
+
+/**
+ * Transform string to SecretValue in Zod schema
+ */
+export const secretStringSchema = z
+  .string()
+  .transform((val) => new SecretValue(val));
+
+/**
+ * Create a secret value
+ */
+export function secret<T = string>(value: T, mask?: string): SecretValue<T> {
+  return new SecretValue(value, mask);
+}
+
+/**
+ * Check if a value is a secret
+ */
+export function isSecret(value: unknown): value is SecretValue {
+  return value instanceof SecretValue;
+}
+
+/**
+ * Redact secrets from an object
+ */
+export function redactSecrets<T extends Record<string, any>>(
+  obj: T,
+  secretPaths: string[] = []
+): T {
+  const result = { ...obj };
+
+  // Redact known secret paths
+  for (const path of secretPaths) {
+    const keys = path.split('.');
+    let current: any = result;
+    
+    for (let i = 0; i < keys.length - 1; i++) {
+      if (current[keys[i]] && typeof current[keys[i]] === 'object') {
+        current = current[keys[i]];
+      } else {
+        break;
+      }
+    }
+    
+    const lastKey = keys[keys.length - 1];
+    if (current && lastKey in current) {
+      current[lastKey] = '***REDACTED***';
+    }
+  }
+
+  // Recursively redact SecretValue instances
+  function redactSecretValues(obj: any): any {
+    if (obj === null || obj === undefined) {
+      return obj;
+    }
+
+    if (isSecret(obj)) {
+      return obj.toString();
+    }
+
+    if (Array.isArray(obj)) {
+      return obj.map(redactSecretValues);
+    }
+
+    if (typeof obj === 'object') {
+      const result: any = {};
+      for (const [key, value] of Object.entries(obj)) {
+        result[key] = redactSecretValues(value);
+      }
+      return result;
+    }
+
+    return obj;
+  }
+
+  return redactSecretValues(result);
+}
+
+/**
+ * Environment variable names that should be treated as secrets
+ */
+export const COMMON_SECRET_PATTERNS = [
+  /password/i,
+  /secret/i,
+  /key/i,
+  /token/i,
+  /credential/i,
+  /private/i,
+  /auth/i,
+  /api[-_]?key/i,
+];
+
+/**
+ * Check if an environment variable name indicates a secret
+ */
+export function isSecretEnvVar(name: string): boolean {
+  return COMMON_SECRET_PATTERNS.some(pattern => pattern.test(name));
+}
+
+/**
+ * Wrap environment variables that look like secrets
+ */
+export function wrapSecretEnvVars(
+  env: Record<string, string | undefined>
+): Record<string, string | SecretValue | undefined> {
+  const result: Record<string, string | SecretValue | undefined> = {};
+
+  for (const [key, value] of Object.entries(env)) {
+    if (value !== undefined && isSecretEnvVar(key)) {
+      result[key] = new SecretValue(value, `***${key}***`);
+    } else {
+      result[key] = value;
+    }
+  }
+
+  return result;
+}

libs/config-new/src/utils/validation.ts

@@ -0,0 +1,193 @@
+import { z } from 'zod';
+import { ConfigValidationError } from '../errors';
+
+export interface ValidationResult {
+  valid: boolean;
+  errors?: Array<{
+    path: string;
+    message: string;
+    expected?: string;
+    received?: string;
+  }>;
+  warnings?: Array<{
+    path: string;
+    message: string;
+  }>;
+}
+
+/**
+ * Validate configuration against a schema
+ */
+export function validateConfig<T>(
+  config: unknown,
+  schema: z.ZodSchema<T>
+): ValidationResult {
+  try {
+    schema.parse(config);
+    return { valid: true };
+  } catch (error) {
+    if (error instanceof z.ZodError) {
+      const errors = error.errors.map(err => ({
+        path: err.path.join('.'),
+        message: err.message,
+        expected: 'expected' in err ? String(err.expected) : undefined,
+        received: 'received' in err ? String(err.received) : undefined,
+      }));
+      
+      return { valid: false, errors };
+    }
+    
+    throw error;
+  }
+}
+
+/**
+ * Check for deprecated configuration options
+ */
+export function checkDeprecations(
+  config: Record<string, any>,
+  deprecations: Record<string, string>
+): ValidationResult['warnings'] {
+  const warnings: ValidationResult['warnings'] = [];
+  
+  function checkObject(obj: any, path: string[] = []): void {
+    for (const [key, value] of Object.entries(obj)) {
+      const currentPath = [...path, key];
+      const pathStr = currentPath.join('.');
+      
+      if (pathStr in deprecations) {
+        warnings?.push({
+          path: pathStr,
+          message: deprecations[pathStr],
+        });
+      }
+      
+      if (value && typeof value === 'object' && !Array.isArray(value)) {
+        checkObject(value, currentPath);
+      }
+    }
+  }
+  
+  checkObject(config);
+  return warnings;
+}
+
+/**
+ * Check for required environment variables
+ */
+export function checkRequiredEnvVars(
+  required: string[]
+): ValidationResult {
+  const errors: ValidationResult['errors'] = [];
+  
+  for (const envVar of required) {
+    if (!process.env[envVar]) {
+      errors.push({
+        path: `env.${envVar}`,
+        message: `Required environment variable ${envVar} is not set`,
+      });
+    }
+  }
+  
+  return {
+    valid: errors.length === 0,
+    errors: errors.length > 0 ? errors : undefined,
+  };
+}
+
+/**
+ * Validate configuration completeness
+ */
+export function validateCompleteness(
+  config: Record<string, any>,
+  required: string[]
+): ValidationResult {
+  const errors: ValidationResult['errors'] = [];
+  
+  for (const path of required) {
+    const keys = path.split('.');
+    let current: any = config;
+    let found = true;
+    
+    for (const key of keys) {
+      if (current && typeof current === 'object' && key in current) {
+        current = current[key];
+      } else {
+        found = false;
+        break;
+      }
+    }
+    
+    if (!found || current === undefined || current === null) {
+      errors.push({
+        path,
+        message: `Required configuration value is missing`,
+      });
+    }
+  }
+  
+  return {
+    valid: errors.length === 0,
+    errors: errors.length > 0 ? errors : undefined,
+  };
+}
+
+/**
+ * Format validation result for display
+ */
+export function formatValidationResult(result: ValidationResult): string {
+  const lines: string[] = [];
+  
+  if (result.valid) {
+    lines.push('✅ Configuration is valid');
+  } else {
+    lines.push('❌ Configuration validation failed');
+  }
+  
+  if (result.errors && result.errors.length > 0) {
+    lines.push('\nErrors:');
+    for (const error of result.errors) {
+      lines.push(`  - ${error.path}: ${error.message}`);
+      if (error.expected && error.received) {
+        lines.push(`    Expected: ${error.expected}, Received: ${error.received}`);
+      }
+    }
+  }
+  
+  if (result.warnings && result.warnings.length > 0) {
+    lines.push('\nWarnings:');
+    for (const warning of result.warnings) {
+      lines.push(`  - ${warning.path}: ${warning.message}`);
+    }
+  }
+  
+  return lines.join('\n');
+}
+
+/**
+ * Create a strict schema that doesn't allow extra properties
+ */
+export function createStrictSchema<T extends z.ZodRawShape>(
+  shape: T
+): z.ZodObject<T, 'strict'> {
+  return z.object(shape).strict();
+}
+
+/**
+ * Merge multiple schemas
+ */
+export function mergeSchemas<T extends z.ZodSchema[]>(
+  ...schemas: T
+): z.ZodIntersection<T[0], T[1]> {
+  if (schemas.length < 2) {
+    throw new Error('At least two schemas required for merge');
+  }
+  
+  let result = schemas[0].and(schemas[1]);
+  
+  for (let i = 2; i < schemas.length; i++) {
+    result = result.and(schemas[i]) as any;
+  }
+  
+  return result as any;
+}