boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

format

Browse source
This commit is contained in:
Boki 2025-06-22 17:55:51 -04:00
parent d858222af7
commit 7d9044ab29
202 changed files with 10755 additions and 10972 deletions
Download patch file Download diff file Expand all files Collapse all files

8
libs/data/cache/src/index.ts vendored
View file

@ -39,7 +39,13 @@ export function createCache(options: CacheOptions): CacheProvider {
// Export types and classes
export type {
CacheConfig, CacheKey, CacheOptions, CacheProvider, CacheStats, RedisConfig, SerializationOptions
CacheConfig,
CacheKey,
CacheOptions,
CacheProvider,
CacheStats,
RedisConfig,
SerializationOptions,
} from './types';
export { RedisConnectionManager } from './connection-manager';

4
libs/data/cache/tsconfig.json vendored
View file

@ -6,7 +6,5 @@
"composite": true
},
"include": ["src/**/*"],
"references": [
{ "path": "../../core/logger" }
]
"references": [{ "path": "../../core/logger" }]
}

66
libs/data/mongodb/src/client.ts
View file

@ -1,7 +1,13 @@
import type { Logger } from '@stock-bot/core/logger';
import type { OptionalUnlessRequiredId } from 'mongodb';
import { Collection, Db, MongoClient } from 'mongodb';
import type { ConnectionEvents, DocumentBase, DynamicPoolConfig, MongoDBClientConfig, PoolMetrics } from './types';
import type { Logger } from '@stock-bot/core/logger';
import type {
ConnectionEvents,
DocumentBase,
DynamicPoolConfig,
MongoDBClientConfig,
PoolMetrics,
} from './types';
/**
* MongoDB Client for Stock Bot Data Service
@ -71,7 +77,7 @@ export class MongoDBClient {
if (this.events?.onConnect) {
await Promise.resolve(this.events.onConnect());
}
// Fire pool created event
if (this.events?.onPoolCreated) {
await Promise.resolve(this.events.onPoolCreated());
@ -89,12 +95,12 @@ export class MongoDBClient {
} catch (error) {
this.metrics.errors++;
this.metrics.lastError = error instanceof Error ? error.message : 'Unknown error';
// Fire error event
if (this.events?.onError) {
await Promise.resolve(this.events.onError(error as Error));
}
this.logger.error('MongoDB connection failed:', error);
if (this.client) {
await this.client.close();
@ -123,12 +129,12 @@ export class MongoDBClient {
this.isConnected = false;
this.client = null;
this.db = null;
// Fire disconnect event
if (this.events?.onDisconnect) {
await Promise.resolve(this.events.onDisconnect());
}
this.logger.info('Disconnected from MongoDB');
} catch (error) {
this.logger.error('Error disconnecting from MongoDB:', error);
@ -206,13 +212,16 @@ export class MongoDBClient {
let totalUpdated = 0;
const errors: unknown[] = [];
this.logger.info(`Starting batch upsert operation [${collectionName}-${documents.length}][${operationId}]`, {
database: dbName,
collection: collectionName,
totalDocuments: documents.length,
uniqueKeys: keyFields,
chunkSize,
});
this.logger.info(
`Starting batch upsert operation [${collectionName}-${documents.length}][${operationId}]`,
{
database: dbName,
collection: collectionName,
totalDocuments: documents.length,
uniqueKeys: keyFields,
chunkSize,
}
);
// Process documents in chunks to avoid memory issues
for (let i = 0; i < documents.length; i += chunkSize) {
@ -422,7 +431,7 @@ export class MongoDBClient {
getPoolMetrics(): PoolMetrics {
// Update last used timestamp
this.metrics.lastUsed = new Date();
// Note: MongoDB driver doesn't expose detailed pool metrics
// These are estimates based on configuration
return { ...this.metrics };
@ -433,7 +442,7 @@ export class MongoDBClient {
*/
setDynamicPoolConfig(config: DynamicPoolConfig): void {
this.dynamicPoolConfig = config;
if (config.enabled && this.isConnected && !this.poolMonitorInterval) {
this.startPoolMonitoring();
} else if (!config.enabled && this.poolMonitorInterval) {
@ -465,7 +474,7 @@ export class MongoDBClient {
const { minSize, maxSize, scaleUpThreshold, scaleDownThreshold } = this.dynamicPoolConfig;
const currentSize = this.metrics.totalConnections;
const utilization = ((this.metrics.activeConnections / currentSize) * 100);
const utilization = (this.metrics.activeConnections / currentSize) * 100;
this.logger.debug('Pool utilization', {
utilization: `${utilization.toFixed(1)}%`,
@ -477,13 +486,21 @@ export class MongoDBClient {
if (utilization > scaleUpThreshold && currentSize < maxSize) {
const newSize = Math.min(currentSize + this.dynamicPoolConfig.scaleUpIncrement, maxSize);
await this.resizePool(newSize);
this.logger.info('Scaling up connection pool', { from: currentSize, to: newSize, utilization });
this.logger.info('Scaling up connection pool', {
from: currentSize,
to: newSize,
utilization,
});
}
// Scale down if utilization is low
else if (utilization < scaleDownThreshold && currentSize > minSize) {
const newSize = Math.max(currentSize - this.dynamicPoolConfig.scaleDownIncrement, minSize);
await this.resizePool(newSize);
this.logger.info('Scaling down connection pool', { from: currentSize, to: newSize, utilization });
this.logger.info('Scaling down connection pool', {
from: currentSize,
to: newSize,
utilization,
});
}
}
@ -494,8 +511,10 @@ export class MongoDBClient {
private async resizePool(newSize: number): Promise<void> {
// MongoDB doesn't support dynamic pool resizing
// This is a placeholder for future implementation
this.logger.warn('Dynamic pool resizing not yet implemented for MongoDB', { requestedSize: newSize });
this.logger.warn('Dynamic pool resizing not yet implemented for MongoDB', {
requestedSize: newSize,
});
// Update metrics to reflect desired state
this.metrics.totalConnections = newSize;
}
@ -514,7 +533,10 @@ export class MongoDBClient {
// Create minimum connections by running parallel pings
for (let i = 0; i < minSize; i++) {
promises.push(
this.client.db(this.defaultDatabase).admin().ping()
this.client
.db(this.defaultDatabase)
.admin()
.ping()
.then(() => {
this.logger.debug(`Warmed up connection ${i + 1}/${minSize}`);
})

22
libs/data/mongodb/src/index.ts
View file

@ -9,14 +9,20 @@ export { MongoDBClient } from './client';
// Types
export type {
AnalystReport, ConnectionEvents, DocumentBase, DynamicPoolConfig, EarningsTranscript,
ExchangeSourceMapping,
MasterExchange,
MongoDBClientConfig,
MongoDBConnectionOptions,
NewsArticle, PoolMetrics, RawDocument,
SecFiling,
SentimentData
AnalystReport,
ConnectionEvents,
DocumentBase,
DynamicPoolConfig,
EarningsTranscript,
ExchangeSourceMapping,
MasterExchange,
MongoDBClientConfig,
MongoDBConnectionOptions,
NewsArticle,
PoolMetrics,
RawDocument,
SecFiling,
SentimentData,
} from './types';
// Note: Factory functions removed - use Awilix DI container instead

5
libs/data/mongodb/tsconfig.json
View file

@ -6,8 +6,5 @@
"composite": true
},
"include": ["src/**/*"],
"references": [
{ "path": "../../core/logger" },
{ "path": "../../core/types" }
]
"references": [{ "path": "../../core/logger" }, { "path": "../../core/types" }]
}

61
libs/data/postgres/src/client.ts
View file

@ -4,13 +4,13 @@ import { PostgreSQLHealthMonitor } from './health';
import { PostgreSQLQueryBuilder } from './query-builder';
import { PostgreSQLTransactionManager } from './transactions';
import type {
ConnectionEvents,
DynamicPoolConfig,
PoolMetrics,
PostgreSQLClientConfig,
PostgreSQLConnectionOptions,
QueryResult,
TransactionCallback,
PoolMetrics,
ConnectionEvents,
DynamicPoolConfig,
} from './types';
/**
@ -32,7 +32,12 @@ export class PostgreSQLClient {
private dynamicPoolConfig?: DynamicPoolConfig;
private poolMonitorInterval?: NodeJS.Timeout;
constructor(config: PostgreSQLClientConfig, logger?: any, options?: PostgreSQLConnectionOptions, events?: ConnectionEvents) {
constructor(
config: PostgreSQLClientConfig,
logger?: any,
options?: PostgreSQLConnectionOptions,
events?: ConnectionEvents
) {
this.config = config;
this.options = {
retryAttempts: 3,
@ -45,7 +50,7 @@ export class PostgreSQLClient {
this.logger = logger || console;
this.healthMonitor = new PostgreSQLHealthMonitor(this);
this.transactionManager = new PostgreSQLTransactionManager(this);
this.metrics = {
totalConnections: 0,
activeConnections: 0,
@ -80,22 +85,22 @@ export class PostgreSQLClient {
client.release();
this.isConnected = true;
// Update metrics
const poolConfig = this.config.poolSettings;
this.metrics.totalConnections = poolConfig?.max || 10;
this.metrics.idleConnections = poolConfig?.min || 2;
// Fire connection event
if (this.events?.onConnect) {
await Promise.resolve(this.events.onConnect());
}
// Fire pool created event
if (this.events?.onPoolCreated) {
await Promise.resolve(this.events.onPoolCreated());
}
this.logger.info('Successfully connected to PostgreSQL', {
poolSize: this.metrics.totalConnections,
});
@ -105,10 +110,10 @@ export class PostgreSQLClient {
// Setup error handlers
this.setupErrorHandlers();
// Setup pool event listeners for metrics
this.setupPoolMetrics();
// Start dynamic pool monitoring if enabled
if (this.dynamicPoolConfig?.enabled) {
this.startPoolMonitoring();
@ -119,12 +124,12 @@ export class PostgreSQLClient {
lastError = error as Error;
this.metrics.errors++;
this.metrics.lastError = lastError.message;
// Fire error event
if (this.events?.onError) {
await Promise.resolve(this.events.onError(lastError));
}
this.logger.error(`PostgreSQL connection attempt ${attempt} failed:`, error);
if (this.pool) {
@ -157,17 +162,17 @@ export class PostgreSQLClient {
clearInterval(this.poolMonitorInterval);
this.poolMonitorInterval = undefined;
}
this.healthMonitor.stop();
await this.pool.end();
this.isConnected = false;
this.pool = null;
// Fire disconnect event
if (this.events?.onDisconnect) {
await Promise.resolve(this.events.onDisconnect());
}
this.logger.info('Disconnected from PostgreSQL');
} catch (error) {
this.logger.error('Error disconnecting from PostgreSQL:', error);
@ -429,7 +434,6 @@ export class PostgreSQLClient {
return this.pool;
}
private buildPoolConfig(): any {
return {
host: this.config.host,
@ -481,7 +485,7 @@ export class PostgreSQLClient {
getPoolMetrics(): PoolMetrics {
// Update last used timestamp
this.metrics.lastUsed = new Date();
// Update metrics from pool if available
if (this.pool) {
this.metrics.totalConnections = this.pool.totalCount;
@ -489,7 +493,7 @@ export class PostgreSQLClient {
this.metrics.waitingRequests = this.pool.waitingCount;
this.metrics.activeConnections = this.metrics.totalConnections - this.metrics.idleConnections;
}
return { ...this.metrics };
}
@ -498,7 +502,7 @@ export class PostgreSQLClient {
*/
setDynamicPoolConfig(config: DynamicPoolConfig): void {
this.dynamicPoolConfig = config;
if (config.enabled && this.isConnected && !this.poolMonitorInterval) {
this.startPoolMonitoring();
} else if (!config.enabled && this.poolMonitorInterval) {
@ -552,7 +556,7 @@ export class PostgreSQLClient {
const metrics = this.getPoolMetrics();
const { minSize, maxSize, scaleUpThreshold, scaleDownThreshold } = this.dynamicPoolConfig;
const currentSize = metrics.totalConnections;
const utilization = currentSize > 0 ? ((metrics.activeConnections / currentSize) * 100) : 0;
const utilization = currentSize > 0 ? (metrics.activeConnections / currentSize) * 100 : 0;
this.logger.debug('Pool utilization', {
utilization: `${utilization.toFixed(1)}%`,
@ -564,13 +568,21 @@ export class PostgreSQLClient {
// Scale up if utilization is high or there are waiting requests
if ((utilization > scaleUpThreshold || metrics.waitingRequests > 0) && currentSize < maxSize) {
const newSize = Math.min(currentSize + this.dynamicPoolConfig.scaleUpIncrement, maxSize);
this.logger.info('Would scale up connection pool', { from: currentSize, to: newSize, utilization });
this.logger.info('Would scale up connection pool', {
from: currentSize,
to: newSize,
utilization,
});
// Note: pg module doesn't support dynamic resizing, would need reconnection
}
// Scale down if utilization is low
else if (utilization < scaleDownThreshold && currentSize > minSize) {
const newSize = Math.max(currentSize - this.dynamicPoolConfig.scaleDownIncrement, minSize);
this.logger.info('Would scale down connection pool', { from: currentSize, to: newSize, utilization });
this.logger.info('Would scale down connection pool', {
from: currentSize,
to: newSize,
utilization,
});
// Note: pg module doesn't support dynamic resizing, would need reconnection
}
}
@ -589,7 +601,8 @@ export class PostgreSQLClient {
// Create minimum connections by running parallel queries
for (let i = 0; i < minSize; i++) {
promises.push(
this.pool.query('SELECT 1')
this.pool
.query('SELECT 1')
.then(() => {
this.logger.debug(`Warmed up connection ${i + 1}/${minSize}`);
})

5
libs/data/postgres/tsconfig.json
View file

@ -6,8 +6,5 @@
"composite": true
},
"include": ["src/**/*"],
"references": [
{ "path": "../../core/logger" },
{ "path": "../../core/types" }
]
"references": [{ "path": "../../core/logger" }, { "path": "../../core/types" }]
}

17
libs/data/questdb/src/client.ts
View file

@ -41,12 +41,12 @@ export class QuestDBClient {
console.log('DEBUG: QuestDB client constructor called with config:', {
...config,
user: config.user || '[NOT PROVIDED]',
password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]'
password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]',
});
this.logger.debug('QuestDB client created with config:', {
...config,
user: config.user || '[NOT PROVIDED]',
password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]'
password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]',
});
this.healthMonitor = new QuestDBHealthMonitor(this);
@ -417,7 +417,6 @@ export class QuestDBClient {
return { ...this.config };
}
private buildPgPoolConfig(): any {
const config: any = {
host: this.config.host,
@ -443,7 +442,7 @@ export class QuestDBClient {
console.log('DEBUG: No user provided for QuestDB connection');
this.logger.debug('No user provided for QuestDB connection');
}
if (this.config.password) {
console.log('DEBUG: Adding password to QuestDB pool config');
this.logger.debug('Adding password to QuestDB pool config');
@ -453,8 +452,14 @@ export class QuestDBClient {
this.logger.debug('No password provided for QuestDB connection');
}
console.log('DEBUG: Final QuestDB pool config:', { ...config, password: config.password ? '[REDACTED]' : undefined });
this.logger.debug('Final QuestDB pool config:', { ...config, password: config.password ? '[REDACTED]' : undefined });
console.log('DEBUG: Final QuestDB pool config:', {
...config,
password: config.password ? '[REDACTED]' : undefined,
});
this.logger.debug('Final QuestDB pool config:', {
...config,
password: config.password ? '[REDACTED]' : undefined,
});
return config;
}

6
libs/data/questdb/src/query-builder.ts
View file

@ -1,9 +1,5 @@
import { getLogger } from '@stock-bot/logger';
import type {
QueryResult,
TableNames,
TimeRange,
} from './types';
import type { QueryResult, TableNames, TimeRange } from './types';
// Interface to avoid circular dependency
interface QuestDBClientInterface {

1
libs/data/questdb/src/schema.ts
View file

@ -337,7 +337,6 @@ export class QuestDBSchemaManager {
return sql;
}
/**
* Validate schema definition
*/

5
libs/data/questdb/tsconfig.json
View file

@ -6,8 +6,5 @@
"composite": true
},
"include": ["src/**/*"],
"references": [
{ "path": "../../core/logger" },
{ "path": "../../core/types" }
]
"references": [{ "path": "../../core/logger" }, { "path": "../../core/types" }]
}