boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

initial data-ingestion refactor

Browse source
This commit is contained in:
Boki 2025-06-21 15:18:25 -04:00
parent 09d907a10c
commit 4f89affc2b
19 changed files with 309 additions and 549 deletions
Download patch file Download diff file Expand all files Collapse all files

64
apps/data-ingestion/src/setup/database-setup.ts
View file

@ -2,11 +2,10 @@ import { getDatabaseConfig } from '@stock-bot/config';
import { getLogger } from '@stock-bot/logger';
import {
ConnectionFactory,
ServiceContainer,
createServiceContainer,
ServiceContainer,
PoolSizeCalculator
} from '@stock-bot/connection-factory';
import type { ConnectionFactoryConfig } from '@stock-bot/connection-factory';
import type { ConnectionFactoryConfig, DynamicPoolConfig } from '@stock-bot/mongodb-client';
const logger = getLogger('database-setup');
@ -55,10 +54,18 @@ export async function setupServiceContainer(): Promise<ServiceContainer> {
const pool = await connectionFactory.createMongoDB({
name: 'default',
config: {
connectionString: dbConfig.mongodb.uri,
uri: dbConfig.mongodb.uri,
database: dbConfig.mongodb.database,
maxPoolSize: poolSize.max,
minPoolSize: poolSize.min,
host: dbConfig.mongodb.host,
port: dbConfig.mongodb.port,
username: dbConfig.mongodb.username,
password: dbConfig.mongodb.password,
authSource: dbConfig.mongodb.authSource,
poolSettings: {
maxPoolSize: poolSize.max,
minPoolSize: poolSize.min,
maxIdleTime: 30000,
}
},
maxConnections: poolSize.max,
minConnections: poolSize.min,
@ -82,11 +89,12 @@ export async function setupServiceContainer(): Promise<ServiceContainer> {
host: dbConfig.postgresql.host,
port: dbConfig.postgresql.port,
database: dbConfig.postgresql.database,
user: dbConfig.postgresql.user,
username: dbConfig.postgresql.user,
password: dbConfig.postgresql.password,
pool: {
poolSettings: {
max: poolSize.max,
min: poolSize.min,
idleTimeoutMillis: 30000,
}
},
maxConnections: poolSize.max,
@ -133,5 +141,45 @@ export async function setupServiceContainer(): Promise<ServiceContainer> {
});
logger.info('Service container setup complete');
// Optional: Enable dynamic pool sizing for production
if (process.env.NODE_ENV === 'production') {
await enableDynamicPoolSizing(container);
}
return container;
}
/**
* Enable dynamic pool sizing for production workloads
*/
async function enableDynamicPoolSizing(container: ServiceContainer): Promise<void> {
const dynamicConfig: DynamicPoolConfig = {
enabled: true,
minSize: 5,
maxSize: 100,
scaleUpThreshold: 70,
scaleDownThreshold: 30,
scaleUpIncrement: 10,
scaleDownIncrement: 5,
evaluationInterval: 30000, // Check every 30 seconds
};
try {
// Set dynamic config for MongoDB
const mongoClient = await container.resolveAsync('mongodb');
if (mongoClient && typeof mongoClient.setDynamicPoolConfig === 'function') {
mongoClient.setDynamicPoolConfig(dynamicConfig);
logger.info('Dynamic pool sizing enabled for MongoDB');
}
// Set dynamic config for PostgreSQL
const pgClient = await container.resolveAsync('postgres');
if (pgClient && typeof pgClient.setDynamicPoolConfig === 'function') {
pgClient.setDynamicPoolConfig(dynamicConfig);
logger.info('Dynamic pool sizing enabled for PostgreSQL');
}
} catch (error) {
logger.warn('Failed to enable dynamic pool sizing', { error });
}
}