This commit is contained in:
Boki 2025-06-22 17:55:51 -04:00
parent d858222af7
commit 7d9044ab29
202 changed files with 10755 additions and 10972 deletions

View file

@ -1,7 +1,13 @@
import type { Logger } from '@stock-bot/core/logger';
import type { OptionalUnlessRequiredId } from 'mongodb';
import { Collection, Db, MongoClient } from 'mongodb';
import type { ConnectionEvents, DocumentBase, DynamicPoolConfig, MongoDBClientConfig, PoolMetrics } from './types';
import type { Logger } from '@stock-bot/core/logger';
import type {
ConnectionEvents,
DocumentBase,
DynamicPoolConfig,
MongoDBClientConfig,
PoolMetrics,
} from './types';
/**
* MongoDB Client for Stock Bot Data Service
@ -71,7 +77,7 @@ export class MongoDBClient {
if (this.events?.onConnect) {
await Promise.resolve(this.events.onConnect());
}
// Fire pool created event
if (this.events?.onPoolCreated) {
await Promise.resolve(this.events.onPoolCreated());
@ -89,12 +95,12 @@ export class MongoDBClient {
} catch (error) {
this.metrics.errors++;
this.metrics.lastError = error instanceof Error ? error.message : 'Unknown error';
// Fire error event
if (this.events?.onError) {
await Promise.resolve(this.events.onError(error as Error));
}
this.logger.error('MongoDB connection failed:', error);
if (this.client) {
await this.client.close();
@ -123,12 +129,12 @@ export class MongoDBClient {
this.isConnected = false;
this.client = null;
this.db = null;
// Fire disconnect event
if (this.events?.onDisconnect) {
await Promise.resolve(this.events.onDisconnect());
}
this.logger.info('Disconnected from MongoDB');
} catch (error) {
this.logger.error('Error disconnecting from MongoDB:', error);
@ -206,13 +212,16 @@ export class MongoDBClient {
let totalUpdated = 0;
const errors: unknown[] = [];
this.logger.info(`Starting batch upsert operation [${collectionName}-${documents.length}][${operationId}]`, {
database: dbName,
collection: collectionName,
totalDocuments: documents.length,
uniqueKeys: keyFields,
chunkSize,
});
this.logger.info(
`Starting batch upsert operation [${collectionName}-${documents.length}][${operationId}]`,
{
database: dbName,
collection: collectionName,
totalDocuments: documents.length,
uniqueKeys: keyFields,
chunkSize,
}
);
// Process documents in chunks to avoid memory issues
for (let i = 0; i < documents.length; i += chunkSize) {
@ -422,7 +431,7 @@ export class MongoDBClient {
getPoolMetrics(): PoolMetrics {
// Update last used timestamp
this.metrics.lastUsed = new Date();
// Note: MongoDB driver doesn't expose detailed pool metrics
// These are estimates based on configuration
return { ...this.metrics };
@ -433,7 +442,7 @@ export class MongoDBClient {
*/
setDynamicPoolConfig(config: DynamicPoolConfig): void {
this.dynamicPoolConfig = config;
if (config.enabled && this.isConnected && !this.poolMonitorInterval) {
this.startPoolMonitoring();
} else if (!config.enabled && this.poolMonitorInterval) {
@ -465,7 +474,7 @@ export class MongoDBClient {
const { minSize, maxSize, scaleUpThreshold, scaleDownThreshold } = this.dynamicPoolConfig;
const currentSize = this.metrics.totalConnections;
const utilization = ((this.metrics.activeConnections / currentSize) * 100);
const utilization = (this.metrics.activeConnections / currentSize) * 100;
this.logger.debug('Pool utilization', {
utilization: `${utilization.toFixed(1)}%`,
@ -477,13 +486,21 @@ export class MongoDBClient {
if (utilization > scaleUpThreshold && currentSize < maxSize) {
const newSize = Math.min(currentSize + this.dynamicPoolConfig.scaleUpIncrement, maxSize);
await this.resizePool(newSize);
this.logger.info('Scaling up connection pool', { from: currentSize, to: newSize, utilization });
this.logger.info('Scaling up connection pool', {
from: currentSize,
to: newSize,
utilization,
});
}
// Scale down if utilization is low
else if (utilization < scaleDownThreshold && currentSize > minSize) {
const newSize = Math.max(currentSize - this.dynamicPoolConfig.scaleDownIncrement, minSize);
await this.resizePool(newSize);
this.logger.info('Scaling down connection pool', { from: currentSize, to: newSize, utilization });
this.logger.info('Scaling down connection pool', {
from: currentSize,
to: newSize,
utilization,
});
}
}
@ -494,8 +511,10 @@ export class MongoDBClient {
private async resizePool(newSize: number): Promise<void> {
// MongoDB doesn't support dynamic pool resizing
// This is a placeholder for future implementation
this.logger.warn('Dynamic pool resizing not yet implemented for MongoDB', { requestedSize: newSize });
this.logger.warn('Dynamic pool resizing not yet implemented for MongoDB', {
requestedSize: newSize,
});
// Update metrics to reflect desired state
this.metrics.totalConnections = newSize;
}
@ -514,7 +533,10 @@ export class MongoDBClient {
// Create minimum connections by running parallel pings
for (let i = 0; i < minSize; i++) {
promises.push(
this.client.db(this.defaultDatabase).admin().ping()
this.client
.db(this.defaultDatabase)
.admin()
.ping()
.then(() => {
this.logger.debug(`Warmed up connection ${i + 1}/${minSize}`);
})

View file

@ -9,14 +9,20 @@ export { MongoDBClient } from './client';
// Types
export type {
AnalystReport, ConnectionEvents, DocumentBase, DynamicPoolConfig, EarningsTranscript,
ExchangeSourceMapping,
MasterExchange,
MongoDBClientConfig,
MongoDBConnectionOptions,
NewsArticle, PoolMetrics, RawDocument,
SecFiling,
SentimentData
AnalystReport,
ConnectionEvents,
DocumentBase,
DynamicPoolConfig,
EarningsTranscript,
ExchangeSourceMapping,
MasterExchange,
MongoDBClientConfig,
MongoDBConnectionOptions,
NewsArticle,
PoolMetrics,
RawDocument,
SecFiling,
SentimentData,
} from './types';
// Note: Factory functions removed - use Awilix DI container instead