unified config

This commit is contained in:
Boki 2025-06-23 11:16:34 -04:00
parent e7c0fe2798
commit 3877902ff4
13 changed files with 856 additions and 476 deletions

View file

@ -20,12 +20,14 @@ export function initializeStockConfig(serviceName?: 'dataIngestion' | 'dataPipel
// If a service name is provided, override the service port
if (serviceName && config.services?.[serviceName]) {
const kebabName = serviceName.replace(/([A-Z])/g, '-$1').toLowerCase().replace(/^-/, '');
return {
...config,
service: {
...config.service,
port: config.services[serviceName].port,
name: serviceName.replace(/([A-Z])/g, '-$1').toLowerCase() // Convert camelCase to kebab-case
name: serviceName, // Keep original for backward compatibility
serviceName: kebabName // Standard kebab-case name
}
};
}

View file

@ -47,17 +47,6 @@ const app = new ServiceApplication(
},
{
// Custom lifecycle hooks
onContainerReady: (container) => {
// Override queue configuration to disable workers
const config = container.cradle.config;
if (config.queue) {
config.queue.workers = 0;
config.queue.concurrency = 0;
config.queue.enableScheduledJobs = false;
config.queue.delayWorkerStart = true;
}
return container;
},
onStarted: (port) => {
const logger = getLogger('web-api');
logger.info('Web API service startup initiated with ServiceApplication framework');

View file

@ -0,0 +1,133 @@
# Configuration Standardization
## Overview
The Stock Bot system now uses a unified configuration approach that standardizes how services receive and use configuration. This eliminates the previous confusion between `StockBotAppConfig` and `AppConfig`, providing a single source of truth for all configuration needs.
## Key Changes
### 1. Unified Configuration Schema
The new `UnifiedAppConfig` schema:
- Provides both nested (backward compatible) and flat (DI-friendly) database configurations
- Automatically standardizes service names to kebab-case
- Handles field name mappings (e.g., `ilpPort``influxPort`)
- Ensures all required fields are present for DI system
### 2. Service Name Standardization
All service names are now standardized to kebab-case:
- `dataIngestion``data-ingestion`
- `dataPipeline``data-pipeline`
- `webApi``web-api`
This happens automatically in:
- `initializeStockConfig()` when passing service name
- `ServiceApplication` constructor
- `toUnifiedConfig()` transformation
### 3. Single Configuration Object
Services now use a single configuration object (`this.config`) that contains:
- All service-specific settings
- Database configurations (both nested and flat)
- Service metadata including standardized name
- All settings required by the DI system
## Migration Guide
### For Service Implementations
Before:
```typescript
const app = new ServiceApplication(
config,
{
serviceName: 'web-api',
// other options
}
);
// In container factory
const configWithService = {
...this.config,
service: { name: this.serviceConfig.serviceName }
};
```
After:
```typescript
const app = new ServiceApplication(
config, // Config already has service.serviceName
{
serviceName: 'web-api', // Still needed for logger
// other options
}
);
// In container factory
// No manual service name addition needed
this.container = await containerFactory(this.config);
```
### For DI Container Usage
Before:
```typescript
const serviceName = config.service?.name || 'unknown';
// Had to handle different naming conventions
```
After:
```typescript
const serviceName = config.service?.serviceName || config.service?.name || 'unknown';
// Standardized kebab-case name is always available
```
### For Configuration Files
The configuration structure remains the same, but the system now ensures:
- Service names are standardized automatically
- Database configs are available in both formats
- All required fields are properly mapped
## Benefits
1. **Simplicity**: One configuration object with all necessary information
2. **Consistency**: Standardized service naming across the system
3. **Type Safety**: Unified schema provides better TypeScript support
4. **Backward Compatibility**: Old configuration formats still work
5. **Reduced Complexity**: No more manual config transformations
## Technical Details
### UnifiedAppConfig Schema
```typescript
export const unifiedAppSchema = baseAppSchema.extend({
// Flat database configs for DI system
redis: dragonflyConfigSchema.optional(),
mongodb: mongodbConfigSchema.optional(),
postgres: postgresConfigSchema.optional(),
questdb: questdbConfigSchema.optional(),
}).transform((data) => {
// Auto-standardize service name
// Sync nested and flat configs
// Handle field mappings
});
```
### Service Registry
The `SERVICE_REGISTRY` now includes aliases for different naming conventions:
```typescript
'web-api': { db: 3, ... },
'webApi': { db: 3, ... }, // Alias for backward compatibility
```
## Future Improvements
1. Remove service name aliases after full migration
2. Deprecate old configuration formats
3. Add configuration validation at startup
4. Provide migration tooling for existing services

View file

@ -0,0 +1,155 @@
import { describe, expect, it } from 'bun:test';
import { unifiedAppSchema, toUnifiedConfig, getStandardServiceName } from '../unified-app.schema';
describe('UnifiedAppConfig', () => {
describe('getStandardServiceName', () => {
it('should convert camelCase to kebab-case', () => {
expect(getStandardServiceName('dataIngestion')).toBe('data-ingestion');
expect(getStandardServiceName('dataPipeline')).toBe('data-pipeline');
expect(getStandardServiceName('webApi')).toBe('web-api');
});
it('should handle already kebab-case names', () => {
expect(getStandardServiceName('data-ingestion')).toBe('data-ingestion');
expect(getStandardServiceName('web-api')).toBe('web-api');
});
it('should handle single word names', () => {
expect(getStandardServiceName('api')).toBe('api');
expect(getStandardServiceName('worker')).toBe('worker');
});
});
describe('unifiedAppSchema transform', () => {
it('should set serviceName from name if not provided', () => {
const config = {
name: 'test-app',
version: '1.0.0',
service: {
name: 'webApi',
port: 3000,
},
log: { level: 'info' },
};
const result = unifiedAppSchema.parse(config);
expect(result.service.serviceName).toBe('web-api');
});
it('should keep existing serviceName if provided', () => {
const config = {
name: 'test-app',
version: '1.0.0',
service: {
name: 'webApi',
serviceName: 'custom-name',
port: 3000,
},
log: { level: 'info' },
};
const result = unifiedAppSchema.parse(config);
expect(result.service.serviceName).toBe('custom-name');
});
it('should sync nested and flat database configs', () => {
const config = {
name: 'test-app',
version: '1.0.0',
service: { name: 'test', port: 3000 },
log: { level: 'info' },
database: {
postgres: {
host: 'localhost',
port: 5432,
database: 'test',
user: 'user',
password: 'pass',
},
mongodb: {
uri: 'mongodb://localhost:27017',
database: 'test',
},
},
};
const result = unifiedAppSchema.parse(config);
// Should have both nested and flat structure
expect(result.postgres).toBeDefined();
expect(result.mongodb).toBeDefined();
expect(result.database?.postgres).toBeDefined();
expect(result.database?.mongodb).toBeDefined();
// Values should match
expect(result.postgres?.host).toBe('localhost');
expect(result.postgres?.port).toBe(5432);
expect(result.mongodb?.uri).toBe('mongodb://localhost:27017');
});
it('should handle questdb ilpPort to influxPort mapping', () => {
const config = {
name: 'test-app',
version: '1.0.0',
service: { name: 'test', port: 3000 },
log: { level: 'info' },
database: {
questdb: {
host: 'localhost',
ilpPort: 9009,
httpPort: 9000,
pgPort: 8812,
database: 'questdb',
},
},
};
const result = unifiedAppSchema.parse(config);
expect(result.questdb).toBeDefined();
expect((result.questdb as any).influxPort).toBe(9009);
});
});
describe('toUnifiedConfig', () => {
it('should convert StockBotAppConfig to UnifiedAppConfig', () => {
const stockBotConfig = {
name: 'stock-bot',
version: '1.0.0',
environment: 'development',
service: {
name: 'dataIngestion',
port: 3001,
host: '0.0.0.0',
},
log: {
level: 'info',
format: 'json',
},
database: {
postgres: {
enabled: true,
host: 'localhost',
port: 5432,
database: 'stock',
user: 'user',
password: 'pass',
},
dragonfly: {
enabled: true,
host: 'localhost',
port: 6379,
db: 0,
},
},
};
const unified = toUnifiedConfig(stockBotConfig);
expect(unified.service.serviceName).toBe('data-ingestion');
expect(unified.redis).toBeDefined();
expect(unified.redis?.host).toBe('localhost');
expect(unified.postgres).toBeDefined();
expect(unified.postgres?.host).toBe('localhost');
});
});
});

View file

@ -12,6 +12,10 @@ export * from './provider.schema';
export { baseAppSchema } from './base-app.schema';
export type { BaseAppConfig } from './base-app.schema';
// Export unified schema for standardized configuration
export { unifiedAppSchema, toUnifiedConfig, getStandardServiceName } from './unified-app.schema';
export type { UnifiedAppConfig } from './unified-app.schema';
// Keep AppConfig for backward compatibility (deprecated)
// @deprecated Use baseAppSchema and extend it for your specific app
import { z } from 'zod';

View file

@ -3,6 +3,7 @@ import { z } from 'zod';
// Common service configuration
export const serviceConfigSchema = z.object({
name: z.string(),
serviceName: z.string().optional(), // Standard service name (kebab-case)
port: z.number().min(1).max(65535),
host: z.string().default('0.0.0.0'),
healthCheckPath: z.string().default('/health'),
@ -96,6 +97,11 @@ export const browserConfigSchema = z.object({
// Proxy manager configuration
export const proxyConfigSchema = z.object({
enabled: z.boolean().default(false),
cachePrefix: z.string().default('proxy:'),
ttl: z.number().default(3600),
webshare: z.object({
apiKey: z.string(),
apiUrl: z.string().default('https://proxy.webshare.io/api/v2/'),
}).optional(),
});

View file

@ -0,0 +1,76 @@
import { z } from 'zod';
import { baseAppSchema } from './base-app.schema';
import {
postgresConfigSchema,
mongodbConfigSchema,
questdbConfigSchema,
dragonflyConfigSchema
} from './database.schema';
/**
* Unified application configuration schema that provides both nested and flat access
* to database configurations for backward compatibility while maintaining a clean structure
*/
export const unifiedAppSchema = baseAppSchema.extend({
// Flat database configs for DI system (these take precedence)
redis: dragonflyConfigSchema.optional(),
mongodb: mongodbConfigSchema.optional(),
postgres: postgresConfigSchema.optional(),
questdb: questdbConfigSchema.optional(),
}).transform((data) => {
// Ensure service.serviceName is set from service.name if not provided
if (data.service && !data.service.serviceName) {
data.service.serviceName = data.service.name.replace(/([A-Z])/g, '-$1').toLowerCase().replace(/^-/, '');
}
// If flat configs exist, ensure they're also in the nested database object
if (data.redis || data.mongodb || data.postgres || data.questdb) {
data.database = {
...data.database,
dragonfly: data.redis || data.database?.dragonfly,
mongodb: data.mongodb || data.database?.mongodb,
postgres: data.postgres || data.database?.postgres,
questdb: data.questdb || data.database?.questdb,
};
}
// If nested configs exist but flat ones don't, copy them to flat structure
if (data.database) {
if (data.database.dragonfly && !data.redis) {
data.redis = data.database.dragonfly;
}
if (data.database.mongodb && !data.mongodb) {
data.mongodb = data.database.mongodb;
}
if (data.database.postgres && !data.postgres) {
data.postgres = data.database.postgres;
}
if (data.database.questdb && !data.questdb) {
// Handle the ilpPort -> influxPort mapping for DI system
const questdbConfig = { ...data.database.questdb };
if ('ilpPort' in questdbConfig && !('influxPort' in questdbConfig)) {
(questdbConfig as any).influxPort = questdbConfig.ilpPort;
}
data.questdb = questdbConfig;
}
}
return data;
});
export type UnifiedAppConfig = z.infer<typeof unifiedAppSchema>;
/**
* Helper to convert StockBotAppConfig to UnifiedAppConfig
*/
export function toUnifiedConfig(config: any): UnifiedAppConfig {
return unifiedAppSchema.parse(config);
}
/**
* Helper to get standardized service name
*/
export function getStandardServiceName(serviceName: string): string {
// Convert camelCase to kebab-case
return serviceName.replace(/([A-Z])/g, '-$1').toLowerCase().replace(/^-/, '');
}

View file

@ -15,6 +15,8 @@ export const appConfigSchema = z.object({
queue: queueConfigSchema.optional(),
service: z.object({
name: z.string(),
serviceName: z.string().optional(), // Standard kebab-case service name
port: z.number().optional(),
}).optional(),
});

View file

@ -1,6 +1,7 @@
import { createContainer, InjectionMode, asFunction, type AwilixContainer } from 'awilix';
import type { AppConfig as StockBotAppConfig } from '@stock-bot/config';
import type { AppConfig as StockBotAppConfig, UnifiedAppConfig } from '@stock-bot/config';
import { appConfigSchema, type AppConfig } from '../config/schemas';
import { toUnifiedConfig } from '@stock-bot/config';
import {
registerCoreServices,
registerCacheServices,
@ -12,6 +13,7 @@ import type { ServiceDefinitions, ContainerBuildOptions } from './types';
export class ServiceContainerBuilder {
private config: Partial<AppConfig> = {};
private unifiedConfig: UnifiedAppConfig | null = null;
private options: ContainerBuildOptions = {
enableCache: true,
enableQueue: true,
@ -24,8 +26,10 @@ export class ServiceContainerBuilder {
initializationTimeout: 30000,
};
withConfig(config: AppConfig | StockBotAppConfig): this {
this.config = this.transformStockBotConfig(config);
withConfig(config: AppConfig | StockBotAppConfig | UnifiedAppConfig): this {
// Convert to unified config format
this.unifiedConfig = toUnifiedConfig(config);
this.config = this.transformStockBotConfig(this.unifiedConfig);
return this;
}
@ -72,6 +76,19 @@ export class ServiceContainerBuilder {
}
private applyServiceOptions(config: Partial<AppConfig>): AppConfig {
// Ensure questdb config has the right field names for DI
const questdbConfig = config.questdb ? {
...config.questdb,
influxPort: (config.questdb as any).influxPort || (config.questdb as any).ilpPort || 9009,
} : {
enabled: true,
host: 'localhost',
httpPort: 9000,
pgPort: 8812,
influxPort: 9009,
database: 'questdb',
};
return {
redis: config.redis || {
enabled: this.options.enableCache ?? true,
@ -92,14 +109,7 @@ export class ServiceContainerBuilder {
user: 'postgres',
password: 'postgres',
},
questdb: this.options.enableQuestDB ? (config.questdb || {
enabled: true,
host: 'localhost',
httpPort: 9000,
pgPort: 8812,
influxPort: 9009,
database: 'questdb',
}) : undefined,
questdb: this.options.enableQuestDB ? questdbConfig : undefined,
proxy: this.options.enableProxy ? (config.proxy || { enabled: false, cachePrefix: 'proxy:', ttl: 3600 }) : undefined,
browser: this.options.enableBrowser ? (config.browser || { headless: true, timeout: 30000 }) : undefined,
queue: this.options.enableQueue ? (config.queue || {
@ -115,6 +125,7 @@ export class ServiceContainerBuilder {
removeOnFail: 50,
}
}) : undefined,
service: config.service,
};
}
@ -143,53 +154,27 @@ export class ServiceContainerBuilder {
});
}
private transformStockBotConfig(config: AppConfig | StockBotAppConfig): Partial<AppConfig> {
// If it's already in the new format (has redis AND postgres at top level), return as is
if ('redis' in config && 'postgres' in config && 'mongodb' in config) {
return config as AppConfig;
}
private transformStockBotConfig(config: UnifiedAppConfig): Partial<AppConfig> {
// Unified config already has flat structure, just extract what we need
// Handle questdb field name mapping
const questdb = config.questdb ? {
enabled: config.questdb.enabled || true,
host: config.questdb.host || 'localhost',
httpPort: config.questdb.httpPort || 9000,
pgPort: config.questdb.pgPort || 8812,
influxPort: (config.questdb as any).influxPort || (config.questdb as any).ilpPort || 9009,
database: config.questdb.database || 'questdb',
} : undefined;
// Transform from StockBotAppConfig format
const stockBotConfig = config as StockBotAppConfig;
return {
redis: stockBotConfig.database?.dragonfly ? {
enabled: true,
host: stockBotConfig.database.dragonfly.host || 'localhost',
port: stockBotConfig.database.dragonfly.port || 6379,
password: stockBotConfig.database.dragonfly.password,
db: stockBotConfig.database.dragonfly.db || 0,
} : undefined,
mongodb: stockBotConfig.database?.mongodb ? {
enabled: stockBotConfig.database.mongodb.enabled ?? true,
uri: stockBotConfig.database.mongodb.uri,
database: stockBotConfig.database.mongodb.database,
} : undefined,
postgres: stockBotConfig.database?.postgres ? {
enabled: stockBotConfig.database.postgres.enabled ?? true,
host: stockBotConfig.database.postgres.host,
port: stockBotConfig.database.postgres.port,
database: stockBotConfig.database.postgres.database,
user: stockBotConfig.database.postgres.user,
password: stockBotConfig.database.postgres.password,
} : undefined,
questdb: stockBotConfig.database?.questdb ? {
enabled: true,
host: stockBotConfig.database.questdb.host || 'localhost',
httpPort: stockBotConfig.database.questdb.httpPort || 9000,
pgPort: stockBotConfig.database.questdb.pgPort || 8812,
influxPort: stockBotConfig.database.questdb.ilpPort || 9009,
database: stockBotConfig.database.questdb.database || 'questdb',
} : undefined,
queue: stockBotConfig.queue,
browser: stockBotConfig.browser,
proxy: stockBotConfig.proxy ? {
...{
enabled: false,
cachePrefix: 'proxy:',
ttl: 3600,
},
...stockBotConfig.proxy
} : undefined,
redis: config.redis,
mongodb: config.mongodb,
postgres: config.postgres,
questdb,
queue: config.queue,
browser: config.browser,
proxy: config.proxy,
service: config.service,
};
}
}

View file

@ -11,7 +11,8 @@ export function registerCacheServices(
container.register({
cache: asFunction(() => {
const { createServiceCache } = require('@stock-bot/queue');
const serviceName = config.service?.name || 'unknown';
// Get standardized service name from config
const serviceName = config.service?.serviceName || config.service?.name || 'unknown';
// Create service-specific cache that uses the service's Redis DB
return createServiceCache(serviceName, {
@ -25,7 +26,7 @@ export function registerCacheServices(
// Also provide global cache for shared data
globalCache: asFunction(() => {
const { createServiceCache } = require('@stock-bot/queue');
const serviceName = config.service?.name || 'unknown';
const serviceName = config.service?.serviceName || config.service?.name || 'unknown';
return createServiceCache(serviceName, {
host: config.redis.host,

View file

@ -53,7 +53,7 @@ export function registerApplicationServices(
queueManager: asFunction(({ logger }) => {
const { SmartQueueManager } = require('@stock-bot/queue');
const queueConfig = {
serviceName: config.service?.name || 'unknown',
serviceName: config.service?.serviceName || config.service?.name || 'unknown',
redis: {
host: config.redis.host,
port: config.redis.port,

View file

@ -7,7 +7,8 @@ import { Hono } from 'hono';
import { cors } from 'hono/cors';
import { getLogger, setLoggerConfig, shutdownLoggers, type Logger } from '@stock-bot/logger';
import { Shutdown } from '@stock-bot/shutdown';
import type { AppConfig as StockBotAppConfig } from '@stock-bot/config';
import type { AppConfig as StockBotAppConfig, UnifiedAppConfig } from '@stock-bot/config';
import { toUnifiedConfig } from '@stock-bot/config';
import type { IServiceContainer } from '@stock-bot/handlers';
import type { ServiceContainer } from './awilix-container';
@ -65,7 +66,7 @@ export interface ServiceLifecycleHooks {
* ServiceApplication - Manages the complete lifecycle of a microservice
*/
export class ServiceApplication {
private config: StockBotAppConfig;
private config: UnifiedAppConfig;
private serviceConfig: ServiceApplicationConfig;
private hooks: ServiceLifecycleHooks;
private logger: Logger;
@ -77,11 +78,18 @@ export class ServiceApplication {
private shutdown: Shutdown;
constructor(
config: StockBotAppConfig,
config: StockBotAppConfig | UnifiedAppConfig,
serviceConfig: ServiceApplicationConfig,
hooks: ServiceLifecycleHooks = {}
) {
this.config = config;
// Convert to unified config
this.config = toUnifiedConfig(config);
// Ensure service name is set in config
if (!this.config.service.serviceName) {
this.config.service.serviceName = serviceConfig.serviceName;
}
this.serviceConfig = {
shutdownTimeout: 15000,
enableHandlers: false,
@ -238,7 +246,7 @@ export class ServiceApplication {
* Start the service with full initialization
*/
async start(
containerFactory: (config: StockBotAppConfig) => Promise<ServiceContainer>,
containerFactory: (config: UnifiedAppConfig) => Promise<ServiceContainer>,
routeFactory: (container: IServiceContainer) => Hono,
handlerInitializer?: (container: IServiceContainer) => Promise<void>
): Promise<void> {
@ -247,6 +255,7 @@ export class ServiceApplication {
try {
// Create and initialize container
this.logger.debug('Creating DI container...');
// Config already has service name from constructor
this.container = await containerFactory(this.config);
this.serviceContainer = this.container.resolve('serviceContainer');
this.logger.info('DI container created and initialized');

View file

@ -45,7 +45,25 @@ export const SERVICE_REGISTRY: Record<string, ServiceConfig> = {
cachePrefix: 'cache:api',
producerOnly: true,
},
// Add more services as needed
// Add aliases for services with different naming conventions
'webApi': {
db: 3,
queuePrefix: 'bull:api',
cachePrefix: 'cache:api',
producerOnly: true,
},
'dataIngestion': {
db: 1,
queuePrefix: 'bull:di',
cachePrefix: 'cache:di',
handlers: ['ceo', 'qm', 'webshare', 'ib', 'proxy'],
},
'dataPipeline': {
db: 2,
queuePrefix: 'bull:dp',
cachePrefix: 'cache:dp',
handlers: ['exchanges', 'symbols'],
},
};
/**