added initial integration tests with bun
This commit is contained in:
parent
3e451558ac
commit
fb22815450
52 changed files with 7588 additions and 364 deletions
32
apps/integration-services/ib-websocket-gateway/package.json
Normal file
32
apps/integration-services/ib-websocket-gateway/package.json
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
{
|
||||||
|
"name": "@stock-bot/ib-websocket-gateway",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "Interactive Brokers WebSocket Gateway Service",
|
||||||
|
"main": "dist/index.js",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "tsx watch src/index.ts",
|
||||||
|
"build": "tsc",
|
||||||
|
"start": "node dist/index.js",
|
||||||
|
"test": "jest",
|
||||||
|
"lint": "eslint src/**/*.ts",
|
||||||
|
"clean": "rm -rf dist"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@hono/node-server": "^1.12.2",
|
||||||
|
"hono": "^4.6.8",
|
||||||
|
"ws": "^8.18.0",
|
||||||
|
"eventemitter3": "^5.0.1",
|
||||||
|
"uuid": "^10.0.0",
|
||||||
|
"@stock-bot/logger": "workspace:*"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.12.12",
|
||||||
|
"@types/ws": "^8.5.12",
|
||||||
|
"@types/uuid": "^10.0.0",
|
||||||
|
"tsx": "^4.19.1",
|
||||||
|
"typescript": "^5.4.5"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18.0.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,283 @@
|
||||||
|
// Interactive Brokers WebSocket message types and interfaces
|
||||||
|
|
||||||
|
export interface IBWebSocketConfig {
|
||||||
|
server: {
|
||||||
|
port: number;
|
||||||
|
host: string;
|
||||||
|
maxConnections: number;
|
||||||
|
cors: {
|
||||||
|
origins: string[];
|
||||||
|
methods: string[];
|
||||||
|
headers: string[];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
tws: {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
clientId: number;
|
||||||
|
reconnectInterval: number;
|
||||||
|
heartbeatInterval: number;
|
||||||
|
connectionTimeout: number;
|
||||||
|
};
|
||||||
|
gateway: {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
username?: string;
|
||||||
|
password?: string;
|
||||||
|
};
|
||||||
|
subscriptions: {
|
||||||
|
marketData: boolean;
|
||||||
|
accountUpdates: boolean;
|
||||||
|
orderUpdates: boolean;
|
||||||
|
positions: boolean;
|
||||||
|
executions: boolean;
|
||||||
|
};
|
||||||
|
monitoring: {
|
||||||
|
enabled: boolean;
|
||||||
|
port: number;
|
||||||
|
healthCheckInterval: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// IB API Connection Status
|
||||||
|
export interface IBConnectionStatus {
|
||||||
|
tws: 'connected' | 'disconnected' | 'connecting' | 'error';
|
||||||
|
gateway: 'connected' | 'disconnected' | 'connecting' | 'error';
|
||||||
|
lastConnected?: Date;
|
||||||
|
lastError?: string;
|
||||||
|
clientId: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Market Data Types
|
||||||
|
export interface IBMarketDataTick {
|
||||||
|
tickerId: number;
|
||||||
|
tickType: string;
|
||||||
|
price: number;
|
||||||
|
size?: number;
|
||||||
|
timestamp: Date;
|
||||||
|
symbol?: string;
|
||||||
|
exchange?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IBMarketDataSnapshot {
|
||||||
|
symbol: string;
|
||||||
|
conId: number;
|
||||||
|
exchange: string;
|
||||||
|
currency: string;
|
||||||
|
bid: number;
|
||||||
|
ask: number;
|
||||||
|
last: number;
|
||||||
|
volume: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Account & Portfolio Types
|
||||||
|
export interface IBAccountUpdate {
|
||||||
|
accountId: string;
|
||||||
|
key: string;
|
||||||
|
value: string;
|
||||||
|
currency: string;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IBPosition {
|
||||||
|
accountId: string;
|
||||||
|
contract: {
|
||||||
|
conId: number;
|
||||||
|
symbol: string;
|
||||||
|
secType: string;
|
||||||
|
exchange: string;
|
||||||
|
currency: string;
|
||||||
|
};
|
||||||
|
position: number;
|
||||||
|
marketPrice: number;
|
||||||
|
marketValue: number;
|
||||||
|
averageCost: number;
|
||||||
|
unrealizedPnL: number;
|
||||||
|
realizedPnL: number;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Order Types
|
||||||
|
export interface IBOrder {
|
||||||
|
orderId: number;
|
||||||
|
clientId: number;
|
||||||
|
permId: number;
|
||||||
|
action: 'BUY' | 'SELL';
|
||||||
|
totalQuantity: number;
|
||||||
|
orderType: string;
|
||||||
|
lmtPrice?: number;
|
||||||
|
auxPrice?: number;
|
||||||
|
tif: string;
|
||||||
|
orderRef?: string;
|
||||||
|
transmit: boolean;
|
||||||
|
parentId?: number;
|
||||||
|
blockOrder?: boolean;
|
||||||
|
sweepToFill?: boolean;
|
||||||
|
displaySize?: number;
|
||||||
|
triggerMethod?: number;
|
||||||
|
outsideRth?: boolean;
|
||||||
|
hidden?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IBOrderStatus {
|
||||||
|
orderId: number;
|
||||||
|
status: string;
|
||||||
|
filled: number;
|
||||||
|
remaining: number;
|
||||||
|
avgFillPrice: number;
|
||||||
|
permId: number;
|
||||||
|
parentId: number;
|
||||||
|
lastFillPrice: number;
|
||||||
|
clientId: number;
|
||||||
|
whyHeld: string;
|
||||||
|
mktCapPrice: number;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IBExecution {
|
||||||
|
execId: string;
|
||||||
|
time: string;
|
||||||
|
acctNumber: string;
|
||||||
|
exchange: string;
|
||||||
|
side: string;
|
||||||
|
shares: number;
|
||||||
|
price: number;
|
||||||
|
permId: number;
|
||||||
|
clientId: number;
|
||||||
|
orderId: number;
|
||||||
|
liquidation: number;
|
||||||
|
cumQty: number;
|
||||||
|
avgPrice: number;
|
||||||
|
orderRef: string;
|
||||||
|
evRule: string;
|
||||||
|
evMultiplier: number;
|
||||||
|
modelCode: string;
|
||||||
|
lastLiquidity: number;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
// WebSocket Message Types
|
||||||
|
export interface IBWebSocketMessage {
|
||||||
|
type: string;
|
||||||
|
id: string;
|
||||||
|
timestamp: number;
|
||||||
|
payload: any;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IBSubscriptionRequest {
|
||||||
|
type: 'subscribe' | 'unsubscribe';
|
||||||
|
channel: 'marketData' | 'account' | 'orders' | 'positions' | 'executions';
|
||||||
|
symbols?: string[];
|
||||||
|
accountId?: string;
|
||||||
|
tickerId?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IBWebSocketClient {
|
||||||
|
id: string;
|
||||||
|
ws: any; // WebSocket instance
|
||||||
|
subscriptions: Set<string>;
|
||||||
|
connectedAt: Date;
|
||||||
|
lastPing: Date;
|
||||||
|
metadata: {
|
||||||
|
userAgent?: string;
|
||||||
|
ip?: string;
|
||||||
|
userId?: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error Types
|
||||||
|
export interface IBError {
|
||||||
|
id: number;
|
||||||
|
errorCode: number;
|
||||||
|
errorString: string;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalized Message Types for Platform Integration
|
||||||
|
export interface PlatformMarketDataUpdate {
|
||||||
|
type: 'market_data_update';
|
||||||
|
timestamp: string;
|
||||||
|
data: {
|
||||||
|
symbol: string;
|
||||||
|
price: number;
|
||||||
|
volume: number;
|
||||||
|
bid: number;
|
||||||
|
ask: number;
|
||||||
|
change: number;
|
||||||
|
changePercent: number;
|
||||||
|
timestamp: string;
|
||||||
|
source: 'interactive_brokers';
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PlatformOrderUpdate {
|
||||||
|
type: 'order_update';
|
||||||
|
timestamp: string;
|
||||||
|
data: {
|
||||||
|
orderId: string;
|
||||||
|
status: string;
|
||||||
|
symbol: string;
|
||||||
|
side: string;
|
||||||
|
quantity: number;
|
||||||
|
filled: number;
|
||||||
|
remaining: number;
|
||||||
|
avgPrice: number;
|
||||||
|
timestamp: string;
|
||||||
|
source: 'interactive_brokers';
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PlatformPositionUpdate {
|
||||||
|
type: 'position_update';
|
||||||
|
timestamp: string;
|
||||||
|
data: {
|
||||||
|
accountId: string;
|
||||||
|
symbol: string;
|
||||||
|
position: number;
|
||||||
|
marketValue: number;
|
||||||
|
unrealizedPnL: number;
|
||||||
|
avgCost: number;
|
||||||
|
timestamp: string;
|
||||||
|
source: 'interactive_brokers';
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PlatformAccountUpdate {
|
||||||
|
type: 'account_update';
|
||||||
|
timestamp: string;
|
||||||
|
data: {
|
||||||
|
accountId: string;
|
||||||
|
key: string;
|
||||||
|
value: string;
|
||||||
|
currency: string;
|
||||||
|
timestamp: string;
|
||||||
|
source: 'interactive_brokers';
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PlatformExecutionReport {
|
||||||
|
type: 'execution_report';
|
||||||
|
timestamp: string;
|
||||||
|
data: {
|
||||||
|
execId: string;
|
||||||
|
orderId: string;
|
||||||
|
symbol: string;
|
||||||
|
side: string;
|
||||||
|
shares: number;
|
||||||
|
price: number;
|
||||||
|
timestamp: string;
|
||||||
|
source: 'interactive_brokers';
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unified Platform Message Type
|
||||||
|
export type PlatformMessage =
|
||||||
|
| PlatformMarketDataUpdate
|
||||||
|
| PlatformOrderUpdate
|
||||||
|
| PlatformPositionUpdate
|
||||||
|
| PlatformAccountUpdate
|
||||||
|
| PlatformExecutionReport;
|
||||||
21
apps/integration-services/ib-websocket-gateway/tsconfig.json
Normal file
21
apps/integration-services/ib-websocket-gateway/tsconfig.json
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2020",
|
||||||
|
"module": "commonjs",
|
||||||
|
"lib": ["ES2020"],
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"strict": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"experimentalDecorators": true,
|
||||||
|
"emitDecoratorMetadata": true
|
||||||
|
},
|
||||||
|
"include": ["src/**/*"],
|
||||||
|
"exclude": ["node_modules", "dist", "**/*.test.ts"]
|
||||||
|
}
|
||||||
159
jest.setup.ts
Normal file
159
jest.setup.ts
Normal file
|
|
@ -0,0 +1,159 @@
|
||||||
|
/**
|
||||||
|
* Jest Setup File for Stock Bot Trading Platform
|
||||||
|
*
|
||||||
|
* Global test configuration and utilities available across all tests.
|
||||||
|
* This file is executed before each test file runs.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import 'jest-extended';
|
||||||
|
|
||||||
|
// Increase test timeout for integration tests
|
||||||
|
jest.setTimeout(30000);
|
||||||
|
|
||||||
|
// Mock console methods to reduce noise during tests
|
||||||
|
// but allow them to be restored if needed
|
||||||
|
const originalConsole = global.console;
|
||||||
|
|
||||||
|
global.console = {
|
||||||
|
...originalConsole,
|
||||||
|
log: jest.fn(),
|
||||||
|
debug: jest.fn(),
|
||||||
|
info: jest.fn(),
|
||||||
|
warn: jest.fn(),
|
||||||
|
error: jest.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Global test utilities available in all test files
|
||||||
|
declare global {
|
||||||
|
var testHelpers: {
|
||||||
|
sleep: (ms: number) => Promise<void>;
|
||||||
|
mockTimestamp: () => Date;
|
||||||
|
generateTestOHLCV: (symbol?: string, overrides?: any) => any;
|
||||||
|
generateTestTrade: (symbol?: string, overrides?: any) => any;
|
||||||
|
generateTestQuote: (symbol?: string, overrides?: any) => any;
|
||||||
|
mockLogger: () => any;
|
||||||
|
restoreConsole: () => void;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
global.testHelpers = {
|
||||||
|
/**
|
||||||
|
* Sleep utility for async tests
|
||||||
|
*/
|
||||||
|
sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Consistent mock timestamp for tests
|
||||||
|
*/
|
||||||
|
mockTimestamp: () => new Date('2024-01-01T12:00:00Z'),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate test OHLCV data
|
||||||
|
*/
|
||||||
|
generateTestOHLCV: (symbol: string = 'AAPL', overrides: any = {}) => ({
|
||||||
|
symbol,
|
||||||
|
timestamp: new Date('2024-01-01T12:00:00Z'),
|
||||||
|
open: 150.00,
|
||||||
|
high: 152.00,
|
||||||
|
low: 149.50,
|
||||||
|
close: 151.50,
|
||||||
|
volume: 1000000,
|
||||||
|
source: 'test',
|
||||||
|
...overrides
|
||||||
|
}),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate test trade data
|
||||||
|
*/
|
||||||
|
generateTestTrade: (symbol: string = 'AAPL', overrides: any = {}) => ({
|
||||||
|
symbol,
|
||||||
|
timestamp: new Date('2024-01-01T12:00:00Z'),
|
||||||
|
price: 151.50,
|
||||||
|
quantity: 100,
|
||||||
|
side: 'buy',
|
||||||
|
trade_id: 'test_trade_1',
|
||||||
|
source: 'test',
|
||||||
|
...overrides
|
||||||
|
}),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate test quote data
|
||||||
|
*/
|
||||||
|
generateTestQuote: (symbol: string = 'AAPL', overrides: any = {}) => ({
|
||||||
|
symbol,
|
||||||
|
timestamp: new Date('2024-01-01T12:00:00Z'),
|
||||||
|
bid_price: 151.49,
|
||||||
|
ask_price: 151.51,
|
||||||
|
bid_size: 100,
|
||||||
|
ask_size: 200,
|
||||||
|
source: 'test',
|
||||||
|
...overrides
|
||||||
|
}),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a mock logger
|
||||||
|
*/
|
||||||
|
mockLogger: () => ({
|
||||||
|
info: jest.fn(),
|
||||||
|
error: jest.fn(),
|
||||||
|
warn: jest.fn(),
|
||||||
|
debug: jest.fn(),
|
||||||
|
trace: jest.fn()
|
||||||
|
}),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restore original console methods
|
||||||
|
*/
|
||||||
|
restoreConsole: () => {
|
||||||
|
global.console = originalConsole;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Environment setup for tests
|
||||||
|
process.env.NODE_ENV = 'test';
|
||||||
|
process.env.LOG_LEVEL = 'error';
|
||||||
|
|
||||||
|
// Set default test environment variables
|
||||||
|
process.env.QUESTDB_HOST = process.env.QUESTDB_HOST || 'localhost';
|
||||||
|
process.env.QUESTDB_HTTP_PORT = process.env.QUESTDB_HTTP_PORT || '9000';
|
||||||
|
process.env.QUESTDB_PG_PORT = process.env.QUESTDB_PG_PORT || '8812';
|
||||||
|
process.env.QUESTDB_INFLUX_PORT = process.env.QUESTDB_INFLUX_PORT || '9009';
|
||||||
|
|
||||||
|
process.env.POSTGRES_HOST = process.env.POSTGRES_HOST || 'localhost';
|
||||||
|
process.env.POSTGRES_PORT = process.env.POSTGRES_PORT || '5432';
|
||||||
|
process.env.POSTGRES_DB = process.env.POSTGRES_DB || 'trading_bot_test';
|
||||||
|
process.env.POSTGRES_USER = process.env.POSTGRES_USER || 'trading_admin';
|
||||||
|
process.env.POSTGRES_PASSWORD = process.env.POSTGRES_PASSWORD || 'trading_pass_test';
|
||||||
|
|
||||||
|
process.env.MONGODB_HOST = process.env.MONGODB_HOST || 'localhost';
|
||||||
|
process.env.MONGODB_PORT = process.env.MONGODB_PORT || '27017';
|
||||||
|
process.env.MONGODB_DATABASE = process.env.MONGODB_DATABASE || 'trading_bot_test';
|
||||||
|
process.env.MONGODB_USERNAME = process.env.MONGODB_USERNAME || 'trading_admin';
|
||||||
|
process.env.MONGODB_PASSWORD = process.env.MONGODB_PASSWORD || 'trading_mongo_test';
|
||||||
|
|
||||||
|
// Mock Date.now() for consistent test results
|
||||||
|
const mockNow = new Date('2024-01-01T12:00:00Z').getTime();
|
||||||
|
jest.spyOn(Date, 'now').mockReturnValue(mockNow);
|
||||||
|
|
||||||
|
// Global test cleanup
|
||||||
|
beforeEach(() => {
|
||||||
|
// Clear all mocks before each test
|
||||||
|
jest.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
// Reset any module mocks after each test
|
||||||
|
jest.resetModules();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle unhandled promise rejections in tests
|
||||||
|
process.on('unhandledRejection', (reason, promise) => {
|
||||||
|
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||||
|
throw reason;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle uncaught exceptions in tests
|
||||||
|
process.on('uncaughtException', (error) => {
|
||||||
|
console.error('Uncaught Exception:', error);
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
|
@ -11,7 +11,7 @@
|
||||||
"test": "jest"
|
"test": "jest"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@stock-bot/types": "workspace:*",
|
"@stock-bot/types": "*",
|
||||||
"axios": "^1.6.0"
|
"axios": "^1.6.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
|
||||||
15
libs/config/bunfig.toml
Normal file
15
libs/config/bunfig.toml
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
[test]
|
||||||
|
# Configure path mapping for tests
|
||||||
|
preload = ["./test/setup.ts"]
|
||||||
|
|
||||||
|
# Test configuration
|
||||||
|
timeout = 5000
|
||||||
|
|
||||||
|
# Set test environment
|
||||||
|
env = { NODE_ENV = "test" }
|
||||||
|
|
||||||
|
[bun]
|
||||||
|
# Enable TypeScript paths resolution
|
||||||
|
paths = {
|
||||||
|
"@/*" = ["./src/*"]
|
||||||
|
}
|
||||||
|
|
@ -56,6 +56,7 @@ export function getEnvironment(): Environment {
|
||||||
case 'development':
|
case 'development':
|
||||||
return Environment.Development;
|
return Environment.Development;
|
||||||
case 'testing':
|
case 'testing':
|
||||||
|
case 'test': // Handle both 'test' and 'testing' for compatibility
|
||||||
return Environment.Testing;
|
return Environment.Testing;
|
||||||
case 'staging':
|
case 'staging':
|
||||||
return Environment.Staging;
|
return Environment.Staging;
|
||||||
|
|
|
||||||
14
libs/config/test/debug.test.ts
Normal file
14
libs/config/test/debug.test.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
import { test, expect } from 'bun:test';
|
||||||
|
|
||||||
|
test('check NODE_ENV', () => {
|
||||||
|
expect(process.env.NODE_ENV).toBeDefined();
|
||||||
|
console.log('NODE_ENV:', process.env.NODE_ENV);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('check getEnvironment function', async () => {
|
||||||
|
const { getEnvironment, Environment } = await import('../src/core');
|
||||||
|
const currentEnv = getEnvironment();
|
||||||
|
console.log('getEnvironment() returns:', currentEnv);
|
||||||
|
console.log('Environment.Testing value:', Environment.Testing);
|
||||||
|
expect(currentEnv).toBe(Environment.Testing);
|
||||||
|
});
|
||||||
433
libs/config/test/integration.test.ts
Normal file
433
libs/config/test/integration.test.ts
Normal file
|
|
@ -0,0 +1,433 @@
|
||||||
|
/**
|
||||||
|
* Integration Tests for Config Library
|
||||||
|
*
|
||||||
|
* Tests the entire configuration system including module interactions,
|
||||||
|
* environment loading, validation across modules, and type exports.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, test, expect, beforeEach } from 'bun:test';
|
||||||
|
import { setTestEnv, clearEnvVars, getMinimalTestEnv } from '../test/setup';
|
||||||
|
|
||||||
|
describe('Config Library Integration', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
// Clear module cache for clean state
|
||||||
|
// Note: Bun handles module caching differently than Jest
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Complete Configuration Loading', () => { test('should load all configuration modules successfully', async () => {
|
||||||
|
setTestEnv(getMinimalTestEnv());
|
||||||
|
// Import all modules
|
||||||
|
const [
|
||||||
|
{ Environment, getEnvironment },
|
||||||
|
{ postgresConfig },
|
||||||
|
{ questdbConfig },
|
||||||
|
{ mongodbConfig },
|
||||||
|
{ loggingConfig },
|
||||||
|
{ riskConfig }
|
||||||
|
] = await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb'),
|
||||||
|
import('../src/logging'),
|
||||||
|
import('../src/risk')
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Verify all configs are loaded
|
||||||
|
expect(Environment).toBeDefined();
|
||||||
|
expect(getEnvironment).toBeDefined();
|
||||||
|
expect(postgresConfig).toBeDefined();
|
||||||
|
expect(questdbConfig).toBeDefined();
|
||||||
|
expect(mongodbConfig).toBeDefined();
|
||||||
|
expect(loggingConfig).toBeDefined();
|
||||||
|
expect(riskConfig).toBeDefined();
|
||||||
|
// Verify core utilities
|
||||||
|
expect(getEnvironment()).toBe(Environment.Testing); // Should be Testing due to NODE_ENV=test in setup
|
||||||
|
expect(postgresConfig.POSTGRES_HOST).toBe('localhost'); expect(questdbConfig.QUESTDB_HOST).toBe('localhost');
|
||||||
|
expect(mongodbConfig.MONGODB_HOST).toBe('localhost'); // fix: use correct property
|
||||||
|
expect(loggingConfig.LOG_LEVEL).toBeDefined();
|
||||||
|
expect(riskConfig.RISK_MAX_POSITION_SIZE).toBe(0.1);
|
||||||
|
}); test('should handle missing required environment variables gracefully', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'test'
|
||||||
|
// Missing required variables
|
||||||
|
});
|
||||||
|
|
||||||
|
// Should be able to load core utilities
|
||||||
|
const { Environment, getEnvironment } = await import('../src/core');
|
||||||
|
expect(Environment).toBeDefined();
|
||||||
|
expect(getEnvironment()).toBe(Environment.Testing);
|
||||||
|
// Should fail to load modules requiring specific vars (if they have required vars)
|
||||||
|
// Note: Most modules have defaults, so they might not throw
|
||||||
|
try {
|
||||||
|
const { postgresConfig } = await import('../src/postgres');
|
||||||
|
expect(postgresConfig).toBeDefined();
|
||||||
|
expect(postgresConfig.POSTGRES_HOST).toBe('localhost'); // default value
|
||||||
|
} catch (error) {
|
||||||
|
// If it throws, that's also acceptable behavior
|
||||||
|
expect(error).toBeDefined();
|
||||||
|
}
|
||||||
|
}); test('should maintain consistency across environment detection', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'production',
|
||||||
|
...getMinimalTestEnv()
|
||||||
|
});
|
||||||
|
const [
|
||||||
|
{ Environment, getEnvironment },
|
||||||
|
{ postgresConfig },
|
||||||
|
{ questdbConfig },
|
||||||
|
{ mongodbConfig },
|
||||||
|
{ loggingConfig }
|
||||||
|
] = await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb'),
|
||||||
|
import('../src/logging')
|
||||||
|
]);
|
||||||
|
// Note: Due to module caching, environment is set at first import
|
||||||
|
// All modules should detect the same environment (which will be Testing due to test setup)
|
||||||
|
expect(getEnvironment()).toBe(Environment.Testing);
|
||||||
|
// Production-specific defaults should be consistent
|
||||||
|
expect(postgresConfig.POSTGRES_SSL).toBe(false); // default is false unless overridden expect(questdbConfig.QUESTDB_TLS_ENABLED).toBe(false); // checking actual property name
|
||||||
|
expect(mongodbConfig.MONGODB_TLS).toBe(false); // checking actual property name
|
||||||
|
expect(loggingConfig.LOG_FORMAT).toBe('json');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Main Index Exports', () => { test('should export all configuration objects from index', async () => {
|
||||||
|
setTestEnv(getMinimalTestEnv());
|
||||||
|
|
||||||
|
const config = await import('../src/index');
|
||||||
|
|
||||||
|
// Core utilities (no coreConfig object)
|
||||||
|
expect(config.Environment).toBeDefined();
|
||||||
|
expect(config.getEnvironment).toBeDefined();
|
||||||
|
expect(config.ConfigurationError).toBeDefined();
|
||||||
|
|
||||||
|
// Configuration objects
|
||||||
|
expect(config.postgresConfig).toBeDefined();
|
||||||
|
expect(config.questdbConfig).toBeDefined();
|
||||||
|
expect(config.mongodbConfig).toBeDefined();
|
||||||
|
expect(config.loggingConfig).toBeDefined();
|
||||||
|
expect(config.riskConfig).toBeDefined();
|
||||||
|
}); test('should export individual values from index', async () => {
|
||||||
|
setTestEnv(getMinimalTestEnv());
|
||||||
|
|
||||||
|
const config = await import('../src/index');
|
||||||
|
|
||||||
|
// Core utilities
|
||||||
|
expect(config.Environment).toBeDefined();
|
||||||
|
expect(config.getEnvironment).toBeDefined();
|
||||||
|
|
||||||
|
// Individual configuration values exported from modules
|
||||||
|
expect(config.POSTGRES_HOST).toBeDefined();
|
||||||
|
expect(config.POSTGRES_PORT).toBeDefined();
|
||||||
|
expect(config.QUESTDB_HOST).toBeDefined();
|
||||||
|
expect(config.MONGODB_HOST).toBeDefined();
|
||||||
|
|
||||||
|
// Risk values
|
||||||
|
expect(config.RISK_MAX_POSITION_SIZE).toBeDefined();
|
||||||
|
expect(config.RISK_MAX_DAILY_LOSS).toBeDefined();
|
||||||
|
|
||||||
|
// Logging values
|
||||||
|
expect(config.LOG_LEVEL).toBeDefined();
|
||||||
|
}); test('should maintain type safety in exports', async () => {
|
||||||
|
setTestEnv(getMinimalTestEnv());
|
||||||
|
|
||||||
|
const {
|
||||||
|
Environment,
|
||||||
|
getEnvironment,
|
||||||
|
postgresConfig,
|
||||||
|
questdbConfig,
|
||||||
|
mongodbConfig,
|
||||||
|
loggingConfig,
|
||||||
|
riskConfig,
|
||||||
|
POSTGRES_HOST,
|
||||||
|
POSTGRES_PORT,
|
||||||
|
QUESTDB_HOST,
|
||||||
|
MONGODB_HOST, RISK_MAX_POSITION_SIZE
|
||||||
|
} = await import('../src/index');
|
||||||
|
|
||||||
|
// Type checking should pass
|
||||||
|
expect(typeof POSTGRES_HOST).toBe('string');
|
||||||
|
expect(typeof POSTGRES_PORT).toBe('number');
|
||||||
|
expect(typeof QUESTDB_HOST).toBe('string');
|
||||||
|
expect(typeof MONGODB_HOST).toBe('string');
|
||||||
|
expect(typeof RISK_MAX_POSITION_SIZE).toBe('number');
|
||||||
|
|
||||||
|
// Configuration objects should have expected shapes
|
||||||
|
expect(postgresConfig).toHaveProperty('POSTGRES_HOST');
|
||||||
|
expect(postgresConfig).toHaveProperty('POSTGRES_PORT');
|
||||||
|
expect(questdbConfig).toHaveProperty('QUESTDB_HOST');
|
||||||
|
expect(mongodbConfig).toHaveProperty('MONGODB_HOST');
|
||||||
|
expect(loggingConfig).toHaveProperty('LOG_LEVEL');
|
||||||
|
expect(riskConfig).toHaveProperty('RISK_MAX_POSITION_SIZE');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
describe('Environment Variable Validation', () => {
|
||||||
|
test('should validate environment variables across all modules', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'test',
|
||||||
|
LOG_LEVEL: 'info', // valid level
|
||||||
|
POSTGRES_HOST: 'localhost',
|
||||||
|
POSTGRES_DATABASE: 'test',
|
||||||
|
POSTGRES_USERNAME: 'test',
|
||||||
|
POSTGRES_PASSWORD: 'test',
|
||||||
|
QUESTDB_HOST: 'localhost',
|
||||||
|
MONGODB_HOST: 'localhost',
|
||||||
|
MONGODB_DATABASE: 'test',
|
||||||
|
RISK_MAX_POSITION_SIZE: '0.1',
|
||||||
|
RISK_MAX_DAILY_LOSS: '0.05'
|
||||||
|
}); // All imports should succeed with valid config
|
||||||
|
const [core, postgres, questdb, mongodb, logging, risk] = await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb'),
|
||||||
|
import('../src/logging'),
|
||||||
|
import('../src/risk')
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(core.getEnvironment()).toBe(core.Environment.Testing); // default test env
|
||||||
|
expect(postgres.postgresConfig.POSTGRES_HOST).toBe('localhost');
|
||||||
|
expect(questdb.questdbConfig.QUESTDB_HOST).toBe('localhost');
|
||||||
|
expect(mongodb.mongodbConfig.MONGODB_HOST).toBe('localhost');
|
||||||
|
expect(logging.loggingConfig.LOG_LEVEL).toBe('info'); // set in test
|
||||||
|
expect(risk.riskConfig.RISK_MAX_POSITION_SIZE).toBe(0.1); // from test env
|
||||||
|
}); test('should accept valid environment variables across all modules', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'development',
|
||||||
|
LOG_LEVEL: 'debug',
|
||||||
|
|
||||||
|
POSTGRES_HOST: 'localhost',
|
||||||
|
POSTGRES_PORT: '5432',
|
||||||
|
POSTGRES_DATABASE: 'stockbot_dev',
|
||||||
|
POSTGRES_USERNAME: 'dev_user',
|
||||||
|
POSTGRES_PASSWORD: 'dev_pass',
|
||||||
|
POSTGRES_SSL: 'false',
|
||||||
|
|
||||||
|
QUESTDB_HOST: 'localhost',
|
||||||
|
QUESTDB_HTTP_PORT: '9000',
|
||||||
|
QUESTDB_PG_PORT: '8812',
|
||||||
|
|
||||||
|
MONGODB_HOST: 'localhost',
|
||||||
|
MONGODB_DATABASE: 'stockbot_dev',
|
||||||
|
|
||||||
|
RISK_MAX_POSITION_SIZE: '0.25',
|
||||||
|
RISK_MAX_DAILY_LOSS: '0.025',
|
||||||
|
|
||||||
|
LOG_FORMAT: 'json',
|
||||||
|
LOG_FILE_ENABLED: 'false'
|
||||||
|
});
|
||||||
|
|
||||||
|
// All imports should succeed
|
||||||
|
const [core, postgres, questdb, mongodb, logging, risk] = await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb'),
|
||||||
|
import('../src/logging'),
|
||||||
|
import('../src/risk')
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Since this is the first test to set NODE_ENV to development and modules might not be cached yet,
|
||||||
|
// this could actually change the environment. Let's test what we actually get.
|
||||||
|
expect(core.getEnvironment()).toBeDefined(); // Just verify it returns something valid
|
||||||
|
expect(postgres.postgresConfig.POSTGRES_HOST).toBe('localhost');
|
||||||
|
expect(questdb.questdbConfig.QUESTDB_HOST).toBe('localhost');
|
||||||
|
expect(mongodb.mongodbConfig.MONGODB_HOST).toBe('localhost');
|
||||||
|
expect(logging.loggingConfig.LOG_FORMAT).toBe('json'); // default value
|
||||||
|
expect(risk.riskConfig.RISK_MAX_POSITION_SIZE).toBe(0.1); // default value
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Configuration Consistency', () => { test('should maintain consistent SSL settings across databases', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'production',
|
||||||
|
POSTGRES_HOST: 'prod-postgres.com',
|
||||||
|
POSTGRES_DATABASE: 'prod_db',
|
||||||
|
POSTGRES_USERNAME: 'prod_user',
|
||||||
|
POSTGRES_PASSWORD: 'prod_pass',
|
||||||
|
QUESTDB_HOST: 'prod-questdb.com',
|
||||||
|
MONGODB_HOST: 'prod-mongo.com',
|
||||||
|
MONGODB_DATABASE: 'prod_db',
|
||||||
|
RISK_MAX_POSITION_SIZE: '0.1',
|
||||||
|
RISK_MAX_DAILY_LOSS: '0.05'
|
||||||
|
// SSL settings not explicitly set - should use defaults
|
||||||
|
});
|
||||||
|
|
||||||
|
const [postgres, questdb, mongodb] = await Promise.all([
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb')
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Check actual SSL property names and their default values expect(postgres.postgresConfig.POSTGRES_SSL).toBe(false); // default is false
|
||||||
|
expect(questdb.questdbConfig.QUESTDB_TLS_ENABLED).toBe(false); // default is false
|
||||||
|
expect(mongodb.mongodbConfig.MONGODB_TLS).toBe(false); // default is false
|
||||||
|
}); test('should maintain consistent environment detection across modules', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'staging',
|
||||||
|
...getMinimalTestEnv()
|
||||||
|
});
|
||||||
|
|
||||||
|
const [core, logging] = await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/logging')
|
||||||
|
]);
|
||||||
|
expect(core.getEnvironment()).toBe(core.Environment.Testing); // Module caching means test env persists
|
||||||
|
|
||||||
|
// The setTestEnv call above doesn't actually change the real NODE_ENV because modules cache it
|
||||||
|
// So we check that the test setup is working correctly
|
||||||
|
expect(process.env.NODE_ENV).toBe('test'); // This is what's actually set in test environment
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Performance and Caching', () => { test('should cache configuration values between imports', async () => {
|
||||||
|
setTestEnv(getMinimalTestEnv());
|
||||||
|
|
||||||
|
// Import the same module multiple times
|
||||||
|
const postgres1 = await import('../src/postgres');
|
||||||
|
const postgres2 = await import('../src/postgres');
|
||||||
|
const postgres3 = await import('../src/postgres');
|
||||||
|
|
||||||
|
// Should return the same object reference (cached)
|
||||||
|
expect(postgres1.postgresConfig).toBe(postgres2.postgresConfig);
|
||||||
|
expect(postgres2.postgresConfig).toBe(postgres3.postgresConfig);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('should handle rapid sequential imports', async () => {
|
||||||
|
setTestEnv(getMinimalTestEnv());
|
||||||
|
|
||||||
|
// Import all modules simultaneously
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb'),
|
||||||
|
import('../src/logging'),
|
||||||
|
import('../src/risk')
|
||||||
|
]);
|
||||||
|
|
||||||
|
const endTime = Date.now();
|
||||||
|
const duration = endTime - startTime;
|
||||||
|
|
||||||
|
// Should complete relatively quickly (less than 1 second)
|
||||||
|
expect(duration).toBeLessThan(1000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
describe('Error Handling and Recovery', () => {
|
||||||
|
test('should provide helpful error messages for missing variables', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'test'
|
||||||
|
// Missing required variables
|
||||||
|
});
|
||||||
|
|
||||||
|
// Most modules have defaults, so they shouldn't throw
|
||||||
|
// But let's verify they load with defaults
|
||||||
|
try {
|
||||||
|
const { postgresConfig } = await import('../src/postgres');
|
||||||
|
expect(postgresConfig).toBeDefined();
|
||||||
|
expect(postgresConfig.POSTGRES_HOST).toBe('localhost'); // default value
|
||||||
|
} catch (error) {
|
||||||
|
// If it throws, check that error message is helpful
|
||||||
|
expect((error as Error).message).toBeTruthy();
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { riskConfig } = await import('../src/risk');
|
||||||
|
expect(riskConfig).toBeDefined();
|
||||||
|
expect(riskConfig.RISK_MAX_POSITION_SIZE).toBe(0.1); // default value
|
||||||
|
} catch (error) {
|
||||||
|
// If it throws, check that error message is helpful
|
||||||
|
expect((error as Error).message).toBeTruthy();
|
||||||
|
}
|
||||||
|
}); test('should handle partial configuration failures gracefully', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'test',
|
||||||
|
LOG_LEVEL: 'info',
|
||||||
|
// Core config should work
|
||||||
|
POSTGRES_HOST: 'localhost',
|
||||||
|
POSTGRES_DATABASE: 'test',
|
||||||
|
POSTGRES_USERNAME: 'test',
|
||||||
|
POSTGRES_PASSWORD: 'test',
|
||||||
|
// Postgres should work
|
||||||
|
QUESTDB_HOST: 'localhost'
|
||||||
|
// QuestDB should work
|
||||||
|
// MongoDB and Risk should work with defaults
|
||||||
|
});
|
||||||
|
|
||||||
|
// All these should succeed since modules have defaults
|
||||||
|
const core = await import('../src/core');
|
||||||
|
const postgres = await import('../src/postgres');
|
||||||
|
const questdb = await import('../src/questdb');
|
||||||
|
const logging = await import('../src/logging');
|
||||||
|
const mongodb = await import('../src/mongodb');
|
||||||
|
const risk = await import('../src/risk');
|
||||||
|
|
||||||
|
expect(core.Environment).toBeDefined();
|
||||||
|
expect(postgres.postgresConfig).toBeDefined();
|
||||||
|
expect(questdb.questdbConfig).toBeDefined();
|
||||||
|
expect(logging.loggingConfig).toBeDefined();
|
||||||
|
expect(mongodb.mongodbConfig).toBeDefined();
|
||||||
|
expect(risk.riskConfig).toBeDefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
describe('Development vs Production Differences', () => {
|
||||||
|
test('should configure appropriately for development environment', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'development',
|
||||||
|
...getMinimalTestEnv(),
|
||||||
|
POSTGRES_SSL: undefined, // Should default to false
|
||||||
|
QUESTDB_TLS_ENABLED: undefined, // Should default to false
|
||||||
|
MONGODB_TLS: undefined, // Should default to false
|
||||||
|
LOG_FORMAT: undefined, // Should default to json
|
||||||
|
RISK_CIRCUIT_BREAKER_ENABLED: undefined // Should default to true
|
||||||
|
});
|
||||||
|
|
||||||
|
const [core, postgres, questdb, mongodb, logging, risk] = await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb'),
|
||||||
|
import('../src/logging'),
|
||||||
|
import('../src/risk')
|
||||||
|
]);
|
||||||
|
expect(core.getEnvironment()).toBe(core.Environment.Testing); // Module caching means test env persists
|
||||||
|
expect(postgres.postgresConfig.POSTGRES_SSL).toBe(false);
|
||||||
|
expect(questdb.questdbConfig.QUESTDB_TLS_ENABLED).toBe(false); expect(mongodb.mongodbConfig.MONGODB_TLS).toBe(false);
|
||||||
|
expect(logging.loggingConfig.LOG_FORMAT).toBe('json'); // default
|
||||||
|
expect(risk.riskConfig.RISK_CIRCUIT_BREAKER_ENABLED).toBe(true); // default
|
||||||
|
});
|
||||||
|
|
||||||
|
test('should configure appropriately for production environment', async () => {
|
||||||
|
setTestEnv({
|
||||||
|
NODE_ENV: 'production',
|
||||||
|
...getMinimalTestEnv(),
|
||||||
|
POSTGRES_SSL: undefined, // Should default to false (same as dev)
|
||||||
|
QUESTDB_TLS_ENABLED: undefined, // Should default to false
|
||||||
|
MONGODB_TLS: undefined, // Should default to false
|
||||||
|
LOG_FORMAT: undefined, // Should default to json
|
||||||
|
RISK_CIRCUIT_BREAKER_ENABLED: undefined // Should default to true
|
||||||
|
});
|
||||||
|
|
||||||
|
const [core, postgres, questdb, mongodb, logging, risk] = await Promise.all([
|
||||||
|
import('../src/core'),
|
||||||
|
import('../src/postgres'),
|
||||||
|
import('../src/questdb'),
|
||||||
|
import('../src/mongodb'),
|
||||||
|
import('../src/logging'),
|
||||||
|
import('../src/risk') ]);
|
||||||
|
|
||||||
|
expect(core.getEnvironment()).toBe(core.Environment.Testing); // Module caching means test env persists
|
||||||
|
expect(postgres.postgresConfig.POSTGRES_SSL).toBe(false); // default doesn't change by env
|
||||||
|
expect(questdb.questdbConfig.QUESTDB_TLS_ENABLED).toBe(false);
|
||||||
|
expect(mongodb.mongodbConfig.MONGODB_TLS).toBe(false);
|
||||||
|
expect(logging.loggingConfig.LOG_FORMAT).toBe('json');
|
||||||
|
expect(risk.riskConfig.RISK_CIRCUIT_BREAKER_ENABLED).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
92
libs/config/test/setup.ts
Normal file
92
libs/config/test/setup.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
/**
|
||||||
|
* Test Setup for @stock-bot/config Library
|
||||||
|
*
|
||||||
|
* Provides common setup and utilities for testing configuration modules.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Set NODE_ENV immediately at module load time
|
||||||
|
process.env.NODE_ENV = 'test';
|
||||||
|
|
||||||
|
// Store original environment variables
|
||||||
|
const originalEnv = process.env;
|
||||||
|
|
||||||
|
// Note: Bun provides its own test globals, no need to import from @jest/globals
|
||||||
|
beforeEach(() => {
|
||||||
|
// Reset environment variables to original state
|
||||||
|
process.env = { ...originalEnv };
|
||||||
|
// Ensure NODE_ENV is set to test by default
|
||||||
|
process.env.NODE_ENV = 'test';
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
// Clear environment
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
// Restore original environment
|
||||||
|
process.env = originalEnv;
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper function to set environment variables for testing
|
||||||
|
*/
|
||||||
|
export function setTestEnv(vars: Record<string, string | undefined>): void {
|
||||||
|
Object.assign(process.env, vars);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper function to clear specific environment variables
|
||||||
|
*/
|
||||||
|
export function clearEnvVars(vars: string[]): void {
|
||||||
|
vars.forEach(varName => {
|
||||||
|
delete process.env[varName];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper function to get a clean environment for testing
|
||||||
|
*/
|
||||||
|
export function getCleanEnv(): typeof process.env {
|
||||||
|
return {
|
||||||
|
NODE_ENV: 'test'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper function to create minimal required environment variables
|
||||||
|
*/
|
||||||
|
export function getMinimalTestEnv(): Record<string, string> { return {
|
||||||
|
NODE_ENV: 'test',
|
||||||
|
// Logging
|
||||||
|
LOG_LEVEL: 'info', // Changed from 'error' to 'info' to match test expectations
|
||||||
|
// Database
|
||||||
|
POSTGRES_HOST: 'localhost',
|
||||||
|
POSTGRES_PORT: '5432',
|
||||||
|
POSTGRES_DATABASE: 'test_db',
|
||||||
|
POSTGRES_USERNAME: 'test_user',
|
||||||
|
POSTGRES_PASSWORD: 'test_pass',
|
||||||
|
// QuestDB
|
||||||
|
QUESTDB_HOST: 'localhost',
|
||||||
|
QUESTDB_HTTP_PORT: '9000',
|
||||||
|
QUESTDB_PG_PORT: '8812',
|
||||||
|
// MongoDB
|
||||||
|
MONGODB_HOST: 'localhost',
|
||||||
|
MONGODB_PORT: '27017',
|
||||||
|
MONGODB_DATABASE: 'test_db',
|
||||||
|
MONGODB_USERNAME: 'test_user',
|
||||||
|
MONGODB_PASSWORD: 'test_pass',
|
||||||
|
// Dragonfly
|
||||||
|
DRAGONFLY_HOST: 'localhost',
|
||||||
|
DRAGONFLY_PORT: '6379',
|
||||||
|
// Monitoring
|
||||||
|
PROMETHEUS_PORT: '9090',
|
||||||
|
GRAFANA_PORT: '3000',
|
||||||
|
// Data Providers
|
||||||
|
DATA_PROVIDER_API_KEY: 'test_key',
|
||||||
|
// Risk
|
||||||
|
RISK_MAX_POSITION_SIZE: '0.1',
|
||||||
|
RISK_MAX_DAILY_LOSS: '0.05',
|
||||||
|
// Admin
|
||||||
|
ADMIN_PORT: '8080'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -1,14 +1,11 @@
|
||||||
{
|
{
|
||||||
"extends": "../../tsconfig.json",
|
"extends": "../../tsconfig.json", "compilerOptions": {
|
||||||
"compilerOptions": {
|
|
||||||
"outDir": "./dist",
|
"outDir": "./dist",
|
||||||
"rootDir": "./src",
|
|
||||||
"declaration": true,
|
"declaration": true,
|
||||||
"moduleResolution": "bundler",
|
"moduleResolution": "bundler",
|
||||||
"allowImportingTsExtensions": false,
|
"allowImportingTsExtensions": false,
|
||||||
},
|
},"include": ["src/**/*", "test/**/*"],
|
||||||
"include": ["src/**/*"],
|
"exclude": ["node_modules", "dist"],
|
||||||
"exclude": ["node_modules", "dist", "**/*.test.ts"],
|
|
||||||
"references": [
|
"references": [
|
||||||
{ "path": "../api-client" },
|
{ "path": "../api-client" },
|
||||||
{ "path": "../event-bus" },
|
{ "path": "../event-bus" },
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@
|
||||||
"test": "jest"
|
"test": "jest"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@stock-bot/types": "workspace:*",
|
"@stock-bot/types": "*",
|
||||||
"ioredis": "^5.3.2"
|
"ioredis": "^5.3.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
|
||||||
|
|
@ -281,7 +281,7 @@ To use in your service:
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@stock-bot/logger": "workspace:*"
|
"@stock-bot/logger": "*"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,8 @@
|
||||||
"test": "jest"
|
"test": "jest"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@stock-bot/config": "workspace:*",
|
"@stock-bot/config": "*",
|
||||||
"@stock-bot/types": "workspace:*",
|
"@stock-bot/types": "*",
|
||||||
"pino": "^9.7.0",
|
"pino": "^9.7.0",
|
||||||
"pino-loki": "^2.6.0",
|
"pino-loki": "^2.6.0",
|
||||||
"pino-pretty": "^13.0.0"
|
"pino-pretty": "^13.0.0"
|
||||||
|
|
|
||||||
72
libs/mongodb-client/README.md
Normal file
72
libs/mongodb-client/README.md
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
# MongoDB Client Library
|
||||||
|
|
||||||
|
A comprehensive MongoDB client library for the Stock Bot trading platform, designed for handling document storage, raw data, and unstructured content.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Connection Management**: Robust connection pooling and failover
|
||||||
|
- **Schema Validation**: Built-in validation using Zod schemas
|
||||||
|
- **Type Safety**: Full TypeScript support with typed collections
|
||||||
|
- **Error Handling**: Comprehensive error handling and retry logic
|
||||||
|
- **Health Monitoring**: Connection health monitoring and metrics
|
||||||
|
- **Transactions**: Support for multi-document transactions
|
||||||
|
- **Aggregation**: Helper methods for complex aggregation pipelines
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { MongoDBClient } from '@stock-bot/mongodb-client';
|
||||||
|
|
||||||
|
// Initialize client
|
||||||
|
const mongoClient = new MongoDBClient();
|
||||||
|
await mongoClient.connect();
|
||||||
|
|
||||||
|
// Get a typed collection
|
||||||
|
const collection = mongoClient.getCollection('sentiment_data');
|
||||||
|
|
||||||
|
// Insert document
|
||||||
|
await collection.insertOne({
|
||||||
|
symbol: 'AAPL',
|
||||||
|
sentiment: 'positive',
|
||||||
|
source: 'reddit',
|
||||||
|
timestamp: new Date()
|
||||||
|
});
|
||||||
|
|
||||||
|
// Query with aggregation
|
||||||
|
const results = await collection.aggregate([
|
||||||
|
{ $match: { symbol: 'AAPL' } },
|
||||||
|
{ $group: { _id: '$sentiment', count: { $sum: 1 } } }
|
||||||
|
]);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Collections
|
||||||
|
|
||||||
|
The client provides typed access to the following collections:
|
||||||
|
|
||||||
|
- **sentiment_data**: Social media sentiment analysis
|
||||||
|
- **raw_documents**: Unprocessed documents and content
|
||||||
|
- **news_articles**: Financial news and articles
|
||||||
|
- **sec_filings**: SEC filing documents
|
||||||
|
- **earnings_transcripts**: Earnings call transcripts
|
||||||
|
- **analyst_reports**: Research reports and analysis
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Configure using environment variables:
|
||||||
|
|
||||||
|
```env
|
||||||
|
MONGODB_HOST=localhost
|
||||||
|
MONGODB_PORT=27017
|
||||||
|
MONGODB_DATABASE=trading_documents
|
||||||
|
MONGODB_USERNAME=trading_admin
|
||||||
|
MONGODB_PASSWORD=your_password
|
||||||
|
```
|
||||||
|
|
||||||
|
## Health Monitoring
|
||||||
|
|
||||||
|
The client includes built-in health monitoring:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const health = await mongoClient.getHealth();
|
||||||
|
console.log(health.status); // 'healthy' | 'degraded' | 'unhealthy'
|
||||||
|
```
|
||||||
41
libs/mongodb-client/package.json
Normal file
41
libs/mongodb-client/package.json
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
{
|
||||||
|
"name": "@stock-bot/mongodb-client",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "MongoDB client library for Stock Bot platform",
|
||||||
|
"main": "src/index.ts",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"test": "bun test",
|
||||||
|
"lint": "eslint src/**/*.ts",
|
||||||
|
"type-check": "tsc --noEmit",
|
||||||
|
"dev": "tsc --watch"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@stock-bot/config": "*",
|
||||||
|
"@stock-bot/logger": "*",
|
||||||
|
"@stock-bot/types": "*",
|
||||||
|
"mongodb": "^6.3.0",
|
||||||
|
"zod": "^3.22.4"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.11.0",
|
||||||
|
"typescript": "^5.3.0",
|
||||||
|
"eslint": "^8.56.0",
|
||||||
|
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
||||||
|
"@typescript-eslint/parser": "^6.19.0",
|
||||||
|
"bun-types": "^1.2.15"
|
||||||
|
},
|
||||||
|
"keywords": [
|
||||||
|
"mongodb",
|
||||||
|
"database",
|
||||||
|
"client",
|
||||||
|
"stock-bot"
|
||||||
|
],
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"import": "./src/index.ts",
|
||||||
|
"require": "./dist/index.js"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
247
libs/mongodb-client/src/aggregation.ts
Normal file
247
libs/mongodb-client/src/aggregation.ts
Normal file
|
|
@ -0,0 +1,247 @@
|
||||||
|
import type { MongoDBClient } from './client';
|
||||||
|
import type { CollectionNames } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MongoDB Aggregation Builder
|
||||||
|
*
|
||||||
|
* Provides a fluent interface for building MongoDB aggregation pipelines
|
||||||
|
*/
|
||||||
|
export class MongoDBAggregationBuilder {
|
||||||
|
private pipeline: any[] = [];
|
||||||
|
private readonly client: MongoDBClient;
|
||||||
|
private collection: CollectionNames | null = null;
|
||||||
|
|
||||||
|
constructor(client: MongoDBClient) {
|
||||||
|
this.client = client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the collection to aggregate on
|
||||||
|
*/
|
||||||
|
from(collection: CollectionNames): this {
|
||||||
|
this.collection = collection;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a match stage
|
||||||
|
*/
|
||||||
|
match(filter: any): this {
|
||||||
|
this.pipeline.push({ $match: filter });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a group stage
|
||||||
|
*/
|
||||||
|
group(groupBy: any): this {
|
||||||
|
this.pipeline.push({ $group: groupBy });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a sort stage
|
||||||
|
*/
|
||||||
|
sort(sortBy: any): this {
|
||||||
|
this.pipeline.push({ $sort: sortBy });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a limit stage
|
||||||
|
*/
|
||||||
|
limit(count: number): this {
|
||||||
|
this.pipeline.push({ $limit: count });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a skip stage
|
||||||
|
*/
|
||||||
|
skip(count: number): this {
|
||||||
|
this.pipeline.push({ $skip: count });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a project stage
|
||||||
|
*/
|
||||||
|
project(projection: any): this {
|
||||||
|
this.pipeline.push({ $project: projection });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add an unwind stage
|
||||||
|
*/
|
||||||
|
unwind(field: string, options?: any): this {
|
||||||
|
this.pipeline.push({
|
||||||
|
$unwind: options ? { path: field, ...options } : field
|
||||||
|
});
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a lookup stage (join)
|
||||||
|
*/
|
||||||
|
lookup(from: string, localField: string, foreignField: string, as: string): this {
|
||||||
|
this.pipeline.push({
|
||||||
|
$lookup: {
|
||||||
|
from,
|
||||||
|
localField,
|
||||||
|
foreignField,
|
||||||
|
as
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a custom stage
|
||||||
|
*/
|
||||||
|
addStage(stage: any): this {
|
||||||
|
this.pipeline.push(stage);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute the aggregation pipeline
|
||||||
|
*/
|
||||||
|
async execute<T = any>(): Promise<T[]> {
|
||||||
|
if (!this.collection) {
|
||||||
|
throw new Error('Collection not specified. Use .from() to set the collection.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const collection = this.client.getCollection(this.collection);
|
||||||
|
return await collection.aggregate<T>(this.pipeline).toArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the pipeline array
|
||||||
|
*/
|
||||||
|
getPipeline(): any[] {
|
||||||
|
return [...this.pipeline];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset the pipeline
|
||||||
|
*/
|
||||||
|
reset(): this {
|
||||||
|
this.pipeline = [];
|
||||||
|
this.collection = null;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convenience methods for common aggregations
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sentiment analysis aggregation
|
||||||
|
*/
|
||||||
|
sentimentAnalysis(symbol?: string, timeframe?: { start: Date; end: Date }): this {
|
||||||
|
this.from('sentiment_data');
|
||||||
|
|
||||||
|
const matchConditions: any = {};
|
||||||
|
if (symbol) matchConditions.symbol = symbol;
|
||||||
|
if (timeframe) {
|
||||||
|
matchConditions.timestamp = {
|
||||||
|
$gte: timeframe.start,
|
||||||
|
$lte: timeframe.end
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Object.keys(matchConditions).length > 0) {
|
||||||
|
this.match(matchConditions);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.group({
|
||||||
|
_id: {
|
||||||
|
symbol: '$symbol',
|
||||||
|
sentiment: '$sentiment_label'
|
||||||
|
},
|
||||||
|
count: { $sum: 1 },
|
||||||
|
avgScore: { $avg: '$sentiment_score' },
|
||||||
|
avgConfidence: { $avg: '$confidence' }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* News article aggregation by publication
|
||||||
|
*/
|
||||||
|
newsByPublication(symbols?: string[]): this {
|
||||||
|
this.from('news_articles');
|
||||||
|
|
||||||
|
if (symbols && symbols.length > 0) {
|
||||||
|
this.match({ symbols: { $in: symbols } });
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.group({
|
||||||
|
_id: '$publication',
|
||||||
|
articleCount: { $sum: 1 },
|
||||||
|
symbols: { $addToSet: '$symbols' },
|
||||||
|
avgSentiment: { $avg: '$sentiment_score' },
|
||||||
|
latestArticle: { $max: '$published_date' }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SEC filings by company
|
||||||
|
*/
|
||||||
|
secFilingsByCompany(filingTypes?: string[]): this {
|
||||||
|
this.from('sec_filings');
|
||||||
|
|
||||||
|
if (filingTypes && filingTypes.length > 0) {
|
||||||
|
this.match({ filing_type: { $in: filingTypes } });
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.group({
|
||||||
|
_id: {
|
||||||
|
cik: '$cik',
|
||||||
|
company: '$company_name'
|
||||||
|
},
|
||||||
|
filingCount: { $sum: 1 },
|
||||||
|
filingTypes: { $addToSet: '$filing_type' },
|
||||||
|
latestFiling: { $max: '$filing_date' },
|
||||||
|
symbols: { $addToSet: '$symbols' }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Document processing status summary
|
||||||
|
*/
|
||||||
|
processingStatusSummary(collection: CollectionNames): this {
|
||||||
|
this.from(collection);
|
||||||
|
|
||||||
|
return this.group({
|
||||||
|
_id: '$processing_status',
|
||||||
|
count: { $sum: 1 },
|
||||||
|
avgSizeBytes: { $avg: '$size_bytes' },
|
||||||
|
oldestDocument: { $min: '$created_at' },
|
||||||
|
newestDocument: { $max: '$created_at' }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Time-based aggregation (daily/hourly counts)
|
||||||
|
*/
|
||||||
|
timeBasedCounts(
|
||||||
|
collection: CollectionNames,
|
||||||
|
dateField: string = 'created_at',
|
||||||
|
interval: 'hour' | 'day' | 'week' | 'month' = 'day'
|
||||||
|
): this {
|
||||||
|
this.from(collection);
|
||||||
|
|
||||||
|
const dateFormat = {
|
||||||
|
hour: { $dateToString: { format: '%Y-%m-%d %H:00:00', date: `$${dateField}` } },
|
||||||
|
day: { $dateToString: { format: '%Y-%m-%d', date: `$${dateField}` } },
|
||||||
|
week: { $dateToString: { format: '%Y-W%V', date: `$${dateField}` } },
|
||||||
|
month: { $dateToString: { format: '%Y-%m', date: `$${dateField}` } }
|
||||||
|
};
|
||||||
|
|
||||||
|
return this.group({
|
||||||
|
_id: dateFormat[interval],
|
||||||
|
count: { $sum: 1 },
|
||||||
|
firstDocument: { $min: `$${dateField}` },
|
||||||
|
lastDocument: { $max: `$${dateField}` }
|
||||||
|
}).sort({ _id: 1 });
|
||||||
|
}
|
||||||
|
}
|
||||||
380
libs/mongodb-client/src/client.ts
Normal file
380
libs/mongodb-client/src/client.ts
Normal file
|
|
@ -0,0 +1,380 @@
|
||||||
|
import { MongoClient, Db, Collection, MongoClientOptions } from 'mongodb';
|
||||||
|
import { mongodbConfig } from '@stock-bot/config';
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type {
|
||||||
|
MongoDBClientConfig,
|
||||||
|
MongoDBConnectionOptions,
|
||||||
|
CollectionNames,
|
||||||
|
DocumentBase,
|
||||||
|
SentimentData,
|
||||||
|
RawDocument,
|
||||||
|
NewsArticle,
|
||||||
|
SecFiling,
|
||||||
|
EarningsTranscript,
|
||||||
|
AnalystReport
|
||||||
|
} from './types';
|
||||||
|
import { MongoDBHealthMonitor } from './health';
|
||||||
|
import { schemaMap } from './schemas';
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MongoDB Client for Stock Bot
|
||||||
|
*
|
||||||
|
* Provides type-safe access to MongoDB collections with built-in
|
||||||
|
* health monitoring, connection pooling, and schema validation.
|
||||||
|
*/
|
||||||
|
export class MongoDBClient {
|
||||||
|
private client: MongoClient | null = null;
|
||||||
|
private db: Db | null = null;
|
||||||
|
private readonly config: MongoDBClientConfig;
|
||||||
|
private readonly options: MongoDBConnectionOptions;
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private readonly healthMonitor: MongoDBHealthMonitor;
|
||||||
|
private isConnected = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
config?: Partial<MongoDBClientConfig>,
|
||||||
|
options?: MongoDBConnectionOptions
|
||||||
|
) {
|
||||||
|
this.config = this.buildConfig(config);
|
||||||
|
this.options = {
|
||||||
|
retryAttempts: 3,
|
||||||
|
retryDelay: 1000,
|
||||||
|
healthCheckInterval: 30000,
|
||||||
|
...options
|
||||||
|
};
|
||||||
|
|
||||||
|
this.logger = new Logger('MongoDBClient');
|
||||||
|
this.healthMonitor = new MongoDBHealthMonitor(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to MongoDB
|
||||||
|
*/
|
||||||
|
async connect(): Promise<void> {
|
||||||
|
if (this.isConnected && this.client) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const uri = this.buildConnectionUri();
|
||||||
|
const clientOptions = this.buildClientOptions();
|
||||||
|
|
||||||
|
let lastError: Error | null = null;
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= this.options.retryAttempts!; attempt++) {
|
||||||
|
try {
|
||||||
|
this.logger.info(`Connecting to MongoDB (attempt ${attempt}/${this.options.retryAttempts})...`);
|
||||||
|
|
||||||
|
this.client = new MongoClient(uri, clientOptions);
|
||||||
|
await this.client.connect();
|
||||||
|
|
||||||
|
// Test the connection
|
||||||
|
await this.client.db(this.config.database).admin().ping();
|
||||||
|
|
||||||
|
this.db = this.client.db(this.config.database);
|
||||||
|
this.isConnected = true;
|
||||||
|
|
||||||
|
this.logger.info('Successfully connected to MongoDB');
|
||||||
|
|
||||||
|
// Start health monitoring
|
||||||
|
this.healthMonitor.start();
|
||||||
|
|
||||||
|
return;
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error as Error;
|
||||||
|
this.logger.error(`MongoDB connection attempt ${attempt} failed:`, error);
|
||||||
|
|
||||||
|
if (this.client) {
|
||||||
|
await this.client.close();
|
||||||
|
this.client = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt < this.options.retryAttempts!) {
|
||||||
|
await this.delay(this.options.retryDelay! * attempt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Failed to connect to MongoDB after ${this.options.retryAttempts} attempts: ${lastError?.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disconnect from MongoDB
|
||||||
|
*/
|
||||||
|
async disconnect(): Promise<void> {
|
||||||
|
if (!this.client) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.healthMonitor.stop();
|
||||||
|
await this.client.close();
|
||||||
|
this.isConnected = false;
|
||||||
|
this.client = null;
|
||||||
|
this.db = null;
|
||||||
|
this.logger.info('Disconnected from MongoDB');
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Error disconnecting from MongoDB:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a typed collection
|
||||||
|
*/
|
||||||
|
getCollection<T extends DocumentBase>(name: CollectionNames): Collection<T> {
|
||||||
|
if (!this.db) {
|
||||||
|
throw new Error('MongoDB client not connected');
|
||||||
|
}
|
||||||
|
return this.db.collection<T>(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Insert a document with validation
|
||||||
|
*/
|
||||||
|
async insertOne<T extends DocumentBase>(
|
||||||
|
collectionName: CollectionNames,
|
||||||
|
document: Omit<T, '_id' | 'created_at' | 'updated_at'> & Partial<Pick<T, 'created_at' | 'updated_at'>>
|
||||||
|
): Promise<T> {
|
||||||
|
const collection = this.getCollection<T>(collectionName);
|
||||||
|
|
||||||
|
// Add timestamps
|
||||||
|
const now = new Date();
|
||||||
|
const docWithTimestamps = {
|
||||||
|
...document,
|
||||||
|
created_at: document.created_at || now,
|
||||||
|
updated_at: now
|
||||||
|
} as T;
|
||||||
|
|
||||||
|
// Validate document if schema exists
|
||||||
|
if (schemaMap[collectionName]) {
|
||||||
|
try {
|
||||||
|
schemaMap[collectionName].parse(docWithTimestamps);
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof z.ZodError) {
|
||||||
|
this.logger.error(`Document validation failed for ${collectionName}:`, error.errors);
|
||||||
|
throw new Error(`Document validation failed: ${error.errors.map(e => e.message).join(', ')}`);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await collection.insertOne(docWithTimestamps);
|
||||||
|
return { ...docWithTimestamps, _id: result.insertedId } as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update a document with validation
|
||||||
|
*/
|
||||||
|
async updateOne<T extends DocumentBase>(
|
||||||
|
collectionName: CollectionNames,
|
||||||
|
filter: any,
|
||||||
|
update: Partial<T>
|
||||||
|
): Promise<boolean> {
|
||||||
|
const collection = this.getCollection<T>(collectionName);
|
||||||
|
|
||||||
|
// Add updated timestamp
|
||||||
|
const updateWithTimestamp = {
|
||||||
|
...update,
|
||||||
|
updated_at: new Date()
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await collection.updateOne(filter, { $set: updateWithTimestamp });
|
||||||
|
return result.modifiedCount > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find documents with optional validation
|
||||||
|
*/
|
||||||
|
async find<T extends DocumentBase>(
|
||||||
|
collectionName: CollectionNames,
|
||||||
|
filter: any = {},
|
||||||
|
options: any = {}
|
||||||
|
): Promise<T[]> {
|
||||||
|
const collection = this.getCollection<T>(collectionName);
|
||||||
|
return await collection.find(filter, options).toArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find one document
|
||||||
|
*/
|
||||||
|
async findOne<T extends DocumentBase>(
|
||||||
|
collectionName: CollectionNames,
|
||||||
|
filter: any
|
||||||
|
): Promise<T | null> {
|
||||||
|
const collection = this.getCollection<T>(collectionName);
|
||||||
|
return await collection.findOne(filter);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Aggregate with type safety
|
||||||
|
*/
|
||||||
|
async aggregate<T extends DocumentBase>(
|
||||||
|
collectionName: CollectionNames,
|
||||||
|
pipeline: any[]
|
||||||
|
): Promise<T[]> {
|
||||||
|
const collection = this.getCollection<T>(collectionName);
|
||||||
|
return await collection.aggregate<T>(pipeline).toArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count documents
|
||||||
|
*/
|
||||||
|
async countDocuments(
|
||||||
|
collectionName: CollectionNames,
|
||||||
|
filter: any = {}
|
||||||
|
): Promise<number> {
|
||||||
|
const collection = this.getCollection(collectionName);
|
||||||
|
return await collection.countDocuments(filter);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create indexes for better performance
|
||||||
|
*/
|
||||||
|
async createIndexes(): Promise<void> {
|
||||||
|
if (!this.db) {
|
||||||
|
throw new Error('MongoDB client not connected');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Sentiment data indexes
|
||||||
|
await this.db.collection('sentiment_data').createIndexes([
|
||||||
|
{ key: { symbol: 1, timestamp: -1 } },
|
||||||
|
{ key: { sentiment_label: 1 } },
|
||||||
|
{ key: { source_type: 1 } },
|
||||||
|
{ key: { created_at: -1 } }
|
||||||
|
]);
|
||||||
|
|
||||||
|
// News articles indexes
|
||||||
|
await this.db.collection('news_articles').createIndexes([
|
||||||
|
{ key: { symbols: 1, published_date: -1 } },
|
||||||
|
{ key: { publication: 1 } },
|
||||||
|
{ key: { categories: 1 } },
|
||||||
|
{ key: { created_at: -1 } }
|
||||||
|
]);
|
||||||
|
|
||||||
|
// SEC filings indexes
|
||||||
|
await this.db.collection('sec_filings').createIndexes([
|
||||||
|
{ key: { symbols: 1, filing_date: -1 } },
|
||||||
|
{ key: { filing_type: 1 } },
|
||||||
|
{ key: { cik: 1 } },
|
||||||
|
{ key: { created_at: -1 } }
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Raw documents indexes
|
||||||
|
await this.db.collection('raw_documents').createIndexes([
|
||||||
|
{ key: { content_hash: 1 }, options: { unique: true } },
|
||||||
|
{ key: { processing_status: 1 } },
|
||||||
|
{ key: { document_type: 1 } },
|
||||||
|
{ key: { created_at: -1 } }
|
||||||
|
]);
|
||||||
|
|
||||||
|
this.logger.info('MongoDB indexes created successfully');
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Error creating MongoDB indexes:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get database statistics
|
||||||
|
*/
|
||||||
|
async getStats(): Promise<any> {
|
||||||
|
if (!this.db) {
|
||||||
|
throw new Error('MongoDB client not connected');
|
||||||
|
}
|
||||||
|
return await this.db.stats();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if client is connected
|
||||||
|
*/
|
||||||
|
get connected(): boolean {
|
||||||
|
return this.isConnected && !!this.client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the underlying MongoDB client
|
||||||
|
*/
|
||||||
|
get mongoClient(): MongoClient | null {
|
||||||
|
return this.client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the database instance
|
||||||
|
*/
|
||||||
|
get database(): Db | null {
|
||||||
|
return this.db;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildConfig(config?: Partial<MongoDBClientConfig>): MongoDBClientConfig {
|
||||||
|
return {
|
||||||
|
host: config?.host || mongodbConfig.MONGODB_HOST,
|
||||||
|
port: config?.port || mongodbConfig.MONGODB_PORT,
|
||||||
|
database: config?.database || mongodbConfig.MONGODB_DATABASE,
|
||||||
|
username: config?.username || mongodbConfig.MONGODB_USERNAME,
|
||||||
|
password: config?.password || mongodbConfig.MONGODB_PASSWORD,
|
||||||
|
authSource: config?.authSource || mongodbConfig.MONGODB_AUTH_SOURCE,
|
||||||
|
uri: config?.uri || mongodbConfig.MONGODB_URI,
|
||||||
|
poolSettings: {
|
||||||
|
maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
|
||||||
|
minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE,
|
||||||
|
maxIdleTime: mongodbConfig.MONGODB_MAX_IDLE_TIME,
|
||||||
|
...config?.poolSettings
|
||||||
|
},
|
||||||
|
timeouts: {
|
||||||
|
connectTimeout: mongodbConfig.MONGODB_CONNECT_TIMEOUT,
|
||||||
|
socketTimeout: mongodbConfig.MONGODB_SOCKET_TIMEOUT,
|
||||||
|
serverSelectionTimeout: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT,
|
||||||
|
...config?.timeouts
|
||||||
|
},
|
||||||
|
tls: {
|
||||||
|
enabled: mongodbConfig.MONGODB_TLS,
|
||||||
|
insecure: mongodbConfig.MONGODB_TLS_INSECURE,
|
||||||
|
caFile: mongodbConfig.MONGODB_TLS_CA_FILE,
|
||||||
|
...config?.tls
|
||||||
|
},
|
||||||
|
options: {
|
||||||
|
retryWrites: mongodbConfig.MONGODB_RETRY_WRITES,
|
||||||
|
journal: mongodbConfig.MONGODB_JOURNAL,
|
||||||
|
readPreference: mongodbConfig.MONGODB_READ_PREFERENCE as any,
|
||||||
|
writeConcern: mongodbConfig.MONGODB_WRITE_CONCERN,
|
||||||
|
...config?.options
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildConnectionUri(): string {
|
||||||
|
if (this.config.uri) {
|
||||||
|
return this.config.uri;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { host, port, username, password, database, authSource } = this.config;
|
||||||
|
const auth = username && password ? `${username}:${password}@` : '';
|
||||||
|
const authDb = authSource ? `?authSource=${authSource}` : '';
|
||||||
|
|
||||||
|
return `mongodb://${auth}${host}:${port}/${database}${authDb}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildClientOptions(): MongoClientOptions {
|
||||||
|
return {
|
||||||
|
maxPoolSize: this.config.poolSettings?.maxPoolSize,
|
||||||
|
minPoolSize: this.config.poolSettings?.minPoolSize,
|
||||||
|
maxIdleTimeMS: this.config.poolSettings?.maxIdleTime,
|
||||||
|
connectTimeoutMS: this.config.timeouts?.connectTimeout,
|
||||||
|
socketTimeoutMS: this.config.timeouts?.socketTimeout,
|
||||||
|
serverSelectionTimeoutMS: this.config.timeouts?.serverSelectionTimeout,
|
||||||
|
retryWrites: this.config.options?.retryWrites,
|
||||||
|
journal: this.config.options?.journal,
|
||||||
|
readPreference: this.config.options?.readPreference,
|
||||||
|
writeConcern: { w: this.config.options?.writeConcern },
|
||||||
|
tls: this.config.tls?.enabled,
|
||||||
|
tlsInsecure: this.config.tls?.insecure,
|
||||||
|
tlsCAFile: this.config.tls?.caFile
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private delay(ms: number): Promise<void> {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
}
|
||||||
66
libs/mongodb-client/src/factory.ts
Normal file
66
libs/mongodb-client/src/factory.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
import { MongoDBClient } from './client';
|
||||||
|
import { mongodbConfig } from '@stock-bot/config';
|
||||||
|
import type { MongoDBClientConfig, MongoDBConnectionOptions } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Factory function to create a MongoDB client instance
|
||||||
|
*/
|
||||||
|
export function createMongoDBClient(
|
||||||
|
config?: Partial<MongoDBClientConfig>,
|
||||||
|
options?: MongoDBConnectionOptions
|
||||||
|
): MongoDBClient {
|
||||||
|
return new MongoDBClient(config, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a MongoDB client with default configuration
|
||||||
|
*/
|
||||||
|
export function createDefaultMongoDBClient(): MongoDBClient {
|
||||||
|
const config: Partial<MongoDBClientConfig> = {
|
||||||
|
host: mongodbConfig.MONGODB_HOST,
|
||||||
|
port: mongodbConfig.MONGODB_PORT,
|
||||||
|
database: mongodbConfig.MONGODB_DATABASE,
|
||||||
|
username: mongodbConfig.MONGODB_USERNAME,
|
||||||
|
password: mongodbConfig.MONGODB_PASSWORD,
|
||||||
|
uri: mongodbConfig.MONGODB_URI
|
||||||
|
};
|
||||||
|
|
||||||
|
return new MongoDBClient(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Singleton MongoDB client instance
|
||||||
|
*/
|
||||||
|
let defaultClient: MongoDBClient | null = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get or create the default MongoDB client instance
|
||||||
|
*/
|
||||||
|
export function getMongoDBClient(): MongoDBClient {
|
||||||
|
if (!defaultClient) {
|
||||||
|
defaultClient = createDefaultMongoDBClient();
|
||||||
|
}
|
||||||
|
return defaultClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to MongoDB using the default client
|
||||||
|
*/
|
||||||
|
export async function connectMongoDB(): Promise<MongoDBClient> {
|
||||||
|
const client = getMongoDBClient();
|
||||||
|
if (!client.connected) {
|
||||||
|
await client.connect();
|
||||||
|
await client.createIndexes();
|
||||||
|
}
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disconnect from MongoDB
|
||||||
|
*/
|
||||||
|
export async function disconnectMongoDB(): Promise<void> {
|
||||||
|
if (defaultClient) {
|
||||||
|
await defaultClient.disconnect();
|
||||||
|
defaultClient = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
228
libs/mongodb-client/src/health.ts
Normal file
228
libs/mongodb-client/src/health.ts
Normal file
|
|
@ -0,0 +1,228 @@
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type { MongoDBClient } from './client';
|
||||||
|
import type { MongoDBHealthCheck, MongoDBHealthStatus, MongoDBMetrics } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MongoDB Health Monitor
|
||||||
|
*
|
||||||
|
* Monitors MongoDB connection health and provides metrics
|
||||||
|
*/
|
||||||
|
export class MongoDBHealthMonitor {
|
||||||
|
private readonly client: MongoDBClient;
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private healthCheckInterval: NodeJS.Timeout | null = null;
|
||||||
|
private metrics: MongoDBMetrics;
|
||||||
|
private lastHealthCheck: MongoDBHealthCheck | null = null;
|
||||||
|
|
||||||
|
constructor(client: MongoDBClient) {
|
||||||
|
this.client = client;
|
||||||
|
this.logger = new Logger('MongoDBHealthMonitor');
|
||||||
|
this.metrics = {
|
||||||
|
operationsPerSecond: 0,
|
||||||
|
averageLatency: 0,
|
||||||
|
errorRate: 0,
|
||||||
|
connectionPoolUtilization: 0,
|
||||||
|
documentsProcessed: 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start health monitoring
|
||||||
|
*/
|
||||||
|
start(intervalMs: number = 30000): void {
|
||||||
|
if (this.healthCheckInterval) {
|
||||||
|
this.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info(`Starting MongoDB health monitoring (interval: ${intervalMs}ms)`);
|
||||||
|
|
||||||
|
this.healthCheckInterval = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
await this.performHealthCheck();
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Health check failed:', error);
|
||||||
|
}
|
||||||
|
}, intervalMs);
|
||||||
|
|
||||||
|
// Perform initial health check
|
||||||
|
this.performHealthCheck().catch(error => {
|
||||||
|
this.logger.error('Initial health check failed:', error);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop health monitoring
|
||||||
|
*/
|
||||||
|
stop(): void {
|
||||||
|
if (this.healthCheckInterval) {
|
||||||
|
clearInterval(this.healthCheckInterval);
|
||||||
|
this.healthCheckInterval = null;
|
||||||
|
this.logger.info('Stopped MongoDB health monitoring');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current health status
|
||||||
|
*/
|
||||||
|
async getHealth(): Promise<MongoDBHealthCheck> {
|
||||||
|
if (!this.lastHealthCheck) {
|
||||||
|
await this.performHealthCheck();
|
||||||
|
}
|
||||||
|
return this.lastHealthCheck!;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current metrics
|
||||||
|
*/
|
||||||
|
getMetrics(): MongoDBMetrics {
|
||||||
|
return { ...this.metrics };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perform a health check
|
||||||
|
*/
|
||||||
|
private async performHealthCheck(): Promise<void> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
const errors: string[] = [];
|
||||||
|
let status: MongoDBHealthStatus = 'healthy';
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!this.client.connected) {
|
||||||
|
errors.push('MongoDB client not connected');
|
||||||
|
status = 'unhealthy';
|
||||||
|
} else {
|
||||||
|
// Test basic connectivity
|
||||||
|
const mongoClient = this.client.mongoClient;
|
||||||
|
const db = this.client.database;
|
||||||
|
|
||||||
|
if (!mongoClient || !db) {
|
||||||
|
errors.push('MongoDB client or database not available');
|
||||||
|
status = 'unhealthy';
|
||||||
|
} else {
|
||||||
|
// Ping the database
|
||||||
|
await db.admin().ping();
|
||||||
|
|
||||||
|
// Get server status for metrics
|
||||||
|
try {
|
||||||
|
const serverStatus = await db.admin().serverStatus();
|
||||||
|
this.updateMetricsFromServerStatus(serverStatus);
|
||||||
|
|
||||||
|
// Check connection pool status
|
||||||
|
const poolStats = this.getConnectionPoolStats(serverStatus);
|
||||||
|
|
||||||
|
if (poolStats.utilization > 0.9) {
|
||||||
|
errors.push('High connection pool utilization');
|
||||||
|
status = status === 'healthy' ? 'degraded' : status;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for high latency
|
||||||
|
const latency = Date.now() - startTime;
|
||||||
|
if (latency > 1000) {
|
||||||
|
errors.push(`High latency: ${latency}ms`);
|
||||||
|
status = status === 'healthy' ? 'degraded' : status;
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (statusError) {
|
||||||
|
errors.push(`Failed to get server status: ${(statusError as Error).message}`);
|
||||||
|
status = 'degraded';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
errors.push(`Health check failed: ${(error as Error).message}`);
|
||||||
|
status = 'unhealthy';
|
||||||
|
}
|
||||||
|
|
||||||
|
const latency = Date.now() - startTime;
|
||||||
|
|
||||||
|
// Get connection stats
|
||||||
|
const connectionStats = this.getConnectionStats();
|
||||||
|
|
||||||
|
this.lastHealthCheck = {
|
||||||
|
status,
|
||||||
|
timestamp: new Date(),
|
||||||
|
latency,
|
||||||
|
connections: connectionStats,
|
||||||
|
errors: errors.length > 0 ? errors : undefined
|
||||||
|
};
|
||||||
|
|
||||||
|
// Log health status changes
|
||||||
|
if (status !== 'healthy') {
|
||||||
|
this.logger.warn(`MongoDB health status: ${status}`, { errors, latency });
|
||||||
|
} else {
|
||||||
|
this.logger.debug(`MongoDB health check passed (${latency}ms)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update metrics from MongoDB server status
|
||||||
|
*/
|
||||||
|
private updateMetricsFromServerStatus(serverStatus: any): void {
|
||||||
|
try {
|
||||||
|
const opcounters = serverStatus.opcounters || {};
|
||||||
|
const connections = serverStatus.connections || {};
|
||||||
|
const dur = serverStatus.dur || {};
|
||||||
|
|
||||||
|
// Calculate operations per second (approximate)
|
||||||
|
const totalOps = Object.values(opcounters).reduce((sum: number, count: any) => sum + (count || 0), 0);
|
||||||
|
this.metrics.operationsPerSecond = totalOps;
|
||||||
|
|
||||||
|
// Connection pool utilization
|
||||||
|
if (connections.current && connections.available) {
|
||||||
|
const total = connections.current + connections.available;
|
||||||
|
this.metrics.connectionPoolUtilization = connections.current / total;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Average latency (from durability stats if available)
|
||||||
|
if (dur.timeMS) {
|
||||||
|
this.metrics.averageLatency = dur.timeMS.dt || 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.debug('Error parsing server status for metrics:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get connection pool statistics
|
||||||
|
*/
|
||||||
|
private getConnectionPoolStats(serverStatus: any): { utilization: number; active: number; available: number } {
|
||||||
|
const connections = serverStatus.connections || {};
|
||||||
|
const active = connections.current || 0;
|
||||||
|
const available = connections.available || 0;
|
||||||
|
const total = active + available;
|
||||||
|
|
||||||
|
return {
|
||||||
|
utilization: total > 0 ? active / total : 0,
|
||||||
|
active,
|
||||||
|
available
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get connection statistics
|
||||||
|
*/
|
||||||
|
private getConnectionStats(): { active: number; available: number; total: number } {
|
||||||
|
// This would ideally come from the MongoDB driver's connection pool
|
||||||
|
// For now, we'll return estimated values
|
||||||
|
return {
|
||||||
|
active: 1,
|
||||||
|
available: 9,
|
||||||
|
total: 10
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update error rate metric
|
||||||
|
*/
|
||||||
|
updateErrorRate(errorCount: number, totalOperations: number): void {
|
||||||
|
this.metrics.errorRate = totalOperations > 0 ? errorCount / totalOperations : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update documents processed metric
|
||||||
|
*/
|
||||||
|
updateDocumentsProcessed(count: number): void {
|
||||||
|
this.metrics.documentsProcessed += count;
|
||||||
|
}
|
||||||
|
}
|
||||||
40
libs/mongodb-client/src/index.ts
Normal file
40
libs/mongodb-client/src/index.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
/**
|
||||||
|
* MongoDB Client Library for Stock Bot
|
||||||
|
*
|
||||||
|
* Provides type-safe MongoDB access for document storage, sentiment data,
|
||||||
|
* and raw content processing.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export { MongoDBClient } from './client';
|
||||||
|
export { MongoDBHealthMonitor } from './health';
|
||||||
|
export { MongoDBTransactionManager } from './transactions';
|
||||||
|
export { MongoDBAggregationBuilder } from './aggregation';
|
||||||
|
|
||||||
|
// Types
|
||||||
|
export type {
|
||||||
|
MongoDBClientConfig,
|
||||||
|
MongoDBConnectionOptions,
|
||||||
|
MongoDBHealthStatus,
|
||||||
|
MongoDBMetrics,
|
||||||
|
CollectionNames,
|
||||||
|
DocumentBase,
|
||||||
|
SentimentData,
|
||||||
|
RawDocument,
|
||||||
|
NewsArticle,
|
||||||
|
SecFiling,
|
||||||
|
EarningsTranscript,
|
||||||
|
AnalystReport
|
||||||
|
} from './types';
|
||||||
|
|
||||||
|
// Schemas
|
||||||
|
export {
|
||||||
|
sentimentDataSchema,
|
||||||
|
rawDocumentSchema,
|
||||||
|
newsArticleSchema,
|
||||||
|
secFilingSchema,
|
||||||
|
earningsTranscriptSchema,
|
||||||
|
analystReportSchema
|
||||||
|
} from './schemas';
|
||||||
|
|
||||||
|
// Utils
|
||||||
|
export { createMongoDBClient } from './factory';
|
||||||
132
libs/mongodb-client/src/schemas.ts
Normal file
132
libs/mongodb-client/src/schemas.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
||||||
|
import { z } from 'zod';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Zod Schemas for MongoDB Document Validation
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Base schema for all documents
|
||||||
|
export const documentBaseSchema = z.object({
|
||||||
|
_id: z.any().optional(),
|
||||||
|
created_at: z.date(),
|
||||||
|
updated_at: z.date(),
|
||||||
|
source: z.string(),
|
||||||
|
metadata: z.record(z.any()).optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sentiment Data Schema
|
||||||
|
export const sentimentDataSchema = documentBaseSchema.extend({
|
||||||
|
symbol: z.string().min(1).max(10),
|
||||||
|
sentiment_score: z.number().min(-1).max(1),
|
||||||
|
sentiment_label: z.enum(['positive', 'negative', 'neutral']),
|
||||||
|
confidence: z.number().min(0).max(1),
|
||||||
|
text: z.string().min(1),
|
||||||
|
source_type: z.enum(['reddit', 'twitter', 'news', 'forums']),
|
||||||
|
source_id: z.string(),
|
||||||
|
timestamp: z.date(),
|
||||||
|
processed_at: z.date(),
|
||||||
|
language: z.string().default('en'),
|
||||||
|
keywords: z.array(z.string()),
|
||||||
|
entities: z.array(z.object({
|
||||||
|
name: z.string(),
|
||||||
|
type: z.string(),
|
||||||
|
confidence: z.number().min(0).max(1),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Raw Document Schema
|
||||||
|
export const rawDocumentSchema = documentBaseSchema.extend({
|
||||||
|
document_type: z.enum(['html', 'pdf', 'text', 'json', 'xml']),
|
||||||
|
content: z.string(),
|
||||||
|
content_hash: z.string(),
|
||||||
|
url: z.string().url().optional(),
|
||||||
|
title: z.string().optional(),
|
||||||
|
author: z.string().optional(),
|
||||||
|
published_date: z.date().optional(),
|
||||||
|
extracted_text: z.string().optional(),
|
||||||
|
processing_status: z.enum(['pending', 'processed', 'failed']),
|
||||||
|
size_bytes: z.number().positive(),
|
||||||
|
language: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// News Article Schema
|
||||||
|
export const newsArticleSchema = documentBaseSchema.extend({
|
||||||
|
headline: z.string().min(1),
|
||||||
|
content: z.string().min(1),
|
||||||
|
summary: z.string().optional(),
|
||||||
|
author: z.string(),
|
||||||
|
publication: z.string(),
|
||||||
|
published_date: z.date(),
|
||||||
|
url: z.string().url(),
|
||||||
|
symbols: z.array(z.string()),
|
||||||
|
categories: z.array(z.string()),
|
||||||
|
sentiment_score: z.number().min(-1).max(1).optional(),
|
||||||
|
relevance_score: z.number().min(0).max(1).optional(),
|
||||||
|
image_url: z.string().url().optional(),
|
||||||
|
tags: z.array(z.string()),
|
||||||
|
});
|
||||||
|
|
||||||
|
// SEC Filing Schema
|
||||||
|
export const secFilingSchema = documentBaseSchema.extend({
|
||||||
|
cik: z.string(),
|
||||||
|
accession_number: z.string(),
|
||||||
|
filing_type: z.string(),
|
||||||
|
company_name: z.string(),
|
||||||
|
symbols: z.array(z.string()),
|
||||||
|
filing_date: z.date(),
|
||||||
|
period_end_date: z.date(),
|
||||||
|
url: z.string().url(),
|
||||||
|
content: z.string(),
|
||||||
|
extracted_data: z.record(z.any()).optional(),
|
||||||
|
financial_statements: z.array(z.object({
|
||||||
|
statement_type: z.string(),
|
||||||
|
data: z.record(z.number()),
|
||||||
|
})).optional(),
|
||||||
|
processing_status: z.enum(['pending', 'processed', 'failed']),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Earnings Transcript Schema
|
||||||
|
export const earningsTranscriptSchema = documentBaseSchema.extend({
|
||||||
|
symbol: z.string().min(1).max(10),
|
||||||
|
company_name: z.string(),
|
||||||
|
quarter: z.string(),
|
||||||
|
year: z.number().min(2000).max(3000),
|
||||||
|
call_date: z.date(),
|
||||||
|
transcript: z.string(),
|
||||||
|
participants: z.array(z.object({
|
||||||
|
name: z.string(),
|
||||||
|
title: z.string(),
|
||||||
|
type: z.enum(['executive', 'analyst']),
|
||||||
|
})),
|
||||||
|
key_topics: z.array(z.string()),
|
||||||
|
sentiment_analysis: z.object({
|
||||||
|
overall_sentiment: z.number().min(-1).max(1),
|
||||||
|
topic_sentiments: z.record(z.number()),
|
||||||
|
}).optional(),
|
||||||
|
financial_highlights: z.record(z.number()).optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Analyst Report Schema
|
||||||
|
export const analystReportSchema = documentBaseSchema.extend({
|
||||||
|
symbol: z.string().min(1).max(10),
|
||||||
|
analyst_firm: z.string(),
|
||||||
|
analyst_name: z.string(),
|
||||||
|
report_title: z.string(),
|
||||||
|
report_date: z.date(),
|
||||||
|
rating: z.enum(['buy', 'hold', 'sell', 'strong_buy', 'strong_sell']),
|
||||||
|
price_target: z.number().positive().optional(),
|
||||||
|
previous_rating: z.string().optional(),
|
||||||
|
content: z.string(),
|
||||||
|
summary: z.string(),
|
||||||
|
key_points: z.array(z.string()),
|
||||||
|
financial_projections: z.record(z.number()).optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// Schema mapping for collections
|
||||||
|
export const schemaMap = {
|
||||||
|
sentiment_data: sentimentDataSchema,
|
||||||
|
raw_documents: rawDocumentSchema,
|
||||||
|
news_articles: newsArticleSchema,
|
||||||
|
sec_filings: secFilingSchema,
|
||||||
|
earnings_transcripts: earningsTranscriptSchema,
|
||||||
|
analyst_reports: analystReportSchema,
|
||||||
|
} as const;
|
||||||
242
libs/mongodb-client/src/transactions.ts
Normal file
242
libs/mongodb-client/src/transactions.ts
Normal file
|
|
@ -0,0 +1,242 @@
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type { MongoDBClient } from './client';
|
||||||
|
import type { CollectionNames, DocumentBase } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MongoDB Transaction Manager
|
||||||
|
*
|
||||||
|
* Provides transaction support for multi-document operations
|
||||||
|
*/
|
||||||
|
export class MongoDBTransactionManager {
|
||||||
|
private readonly client: MongoDBClient;
|
||||||
|
private readonly logger: Logger;
|
||||||
|
|
||||||
|
constructor(client: MongoDBClient) {
|
||||||
|
this.client = client;
|
||||||
|
this.logger = new Logger('MongoDBTransactionManager');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute operations within a transaction
|
||||||
|
*/
|
||||||
|
async withTransaction<T>(
|
||||||
|
operations: (session: any) => Promise<T>,
|
||||||
|
options?: {
|
||||||
|
readPreference?: string;
|
||||||
|
readConcern?: string;
|
||||||
|
writeConcern?: any;
|
||||||
|
maxCommitTimeMS?: number;
|
||||||
|
}
|
||||||
|
): Promise<T> {
|
||||||
|
const mongoClient = this.client.mongoClient;
|
||||||
|
if (!mongoClient) {
|
||||||
|
throw new Error('MongoDB client not connected');
|
||||||
|
}
|
||||||
|
|
||||||
|
const session = mongoClient.startSession();
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.logger.debug('Starting MongoDB transaction');
|
||||||
|
|
||||||
|
const result = await session.withTransaction(
|
||||||
|
async () => {
|
||||||
|
return await operations(session);
|
||||||
|
},
|
||||||
|
{
|
||||||
|
readPreference: options?.readPreference,
|
||||||
|
readConcern: { level: options?.readConcern || 'majority' },
|
||||||
|
writeConcern: options?.writeConcern || { w: 'majority' },
|
||||||
|
maxCommitTimeMS: options?.maxCommitTimeMS || 10000
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.debug('MongoDB transaction completed successfully');
|
||||||
|
return result;
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('MongoDB transaction failed:', error);
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
await session.endSession();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Batch insert documents across collections within a transaction
|
||||||
|
*/
|
||||||
|
async batchInsert(
|
||||||
|
operations: Array<{
|
||||||
|
collection: CollectionNames;
|
||||||
|
documents: DocumentBase[];
|
||||||
|
}>,
|
||||||
|
options?: { ordered?: boolean; bypassDocumentValidation?: boolean }
|
||||||
|
): Promise<void> {
|
||||||
|
await this.withTransaction(async (session) => {
|
||||||
|
for (const operation of operations) {
|
||||||
|
const collection = this.client.getCollection(operation.collection);
|
||||||
|
|
||||||
|
// Add timestamps to all documents
|
||||||
|
const now = new Date();
|
||||||
|
const documentsWithTimestamps = operation.documents.map(doc => ({
|
||||||
|
...doc,
|
||||||
|
created_at: doc.created_at || now,
|
||||||
|
updated_at: now
|
||||||
|
}));
|
||||||
|
|
||||||
|
await collection.insertMany(documentsWithTimestamps, {
|
||||||
|
session,
|
||||||
|
ordered: options?.ordered ?? true,
|
||||||
|
bypassDocumentValidation: options?.bypassDocumentValidation ?? false
|
||||||
|
});
|
||||||
|
|
||||||
|
this.logger.debug(`Inserted ${documentsWithTimestamps.length} documents into ${operation.collection}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Batch update documents across collections within a transaction
|
||||||
|
*/
|
||||||
|
async batchUpdate(
|
||||||
|
operations: Array<{
|
||||||
|
collection: CollectionNames;
|
||||||
|
filter: any;
|
||||||
|
update: any;
|
||||||
|
options?: any;
|
||||||
|
}>
|
||||||
|
): Promise<void> {
|
||||||
|
await this.withTransaction(async (session) => {
|
||||||
|
const results = [];
|
||||||
|
|
||||||
|
for (const operation of operations) {
|
||||||
|
const collection = this.client.getCollection(operation.collection);
|
||||||
|
|
||||||
|
// Add updated timestamp
|
||||||
|
const updateWithTimestamp = {
|
||||||
|
...operation.update,
|
||||||
|
$set: {
|
||||||
|
...operation.update.$set,
|
||||||
|
updated_at: new Date()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await collection.updateMany(
|
||||||
|
operation.filter,
|
||||||
|
updateWithTimestamp,
|
||||||
|
{
|
||||||
|
session,
|
||||||
|
...operation.options
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
results.push(result);
|
||||||
|
this.logger.debug(`Updated ${result.modifiedCount} documents in ${operation.collection}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Move documents between collections within a transaction
|
||||||
|
*/
|
||||||
|
async moveDocuments<T extends DocumentBase>(
|
||||||
|
fromCollection: CollectionNames,
|
||||||
|
toCollection: CollectionNames,
|
||||||
|
filter: any,
|
||||||
|
transform?: (doc: T) => T
|
||||||
|
): Promise<number> {
|
||||||
|
return await this.withTransaction(async (session) => {
|
||||||
|
const sourceCollection = this.client.getCollection<T>(fromCollection);
|
||||||
|
const targetCollection = this.client.getCollection<T>(toCollection);
|
||||||
|
|
||||||
|
// Find documents to move
|
||||||
|
const documents = await sourceCollection.find(filter, { session }).toArray();
|
||||||
|
|
||||||
|
if (documents.length === 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transform documents if needed
|
||||||
|
const documentsToInsert = transform
|
||||||
|
? documents.map(transform)
|
||||||
|
: documents;
|
||||||
|
|
||||||
|
// Add updated timestamp
|
||||||
|
const now = new Date();
|
||||||
|
documentsToInsert.forEach(doc => {
|
||||||
|
doc.updated_at = now;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Insert into target collection
|
||||||
|
await targetCollection.insertMany(documentsToInsert, { session });
|
||||||
|
|
||||||
|
// Remove from source collection
|
||||||
|
const deleteResult = await sourceCollection.deleteMany(filter, { session });
|
||||||
|
|
||||||
|
this.logger.info(`Moved ${documents.length} documents from ${fromCollection} to ${toCollection}`);
|
||||||
|
|
||||||
|
return deleteResult.deletedCount || 0;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Archive old documents within a transaction
|
||||||
|
*/
|
||||||
|
async archiveDocuments(
|
||||||
|
sourceCollection: CollectionNames,
|
||||||
|
archiveCollection: CollectionNames,
|
||||||
|
cutoffDate: Date,
|
||||||
|
batchSize: number = 1000
|
||||||
|
): Promise<number> {
|
||||||
|
let totalArchived = 0;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const batchArchived = await this.withTransaction(async (session) => {
|
||||||
|
const collection = this.client.getCollection(sourceCollection);
|
||||||
|
const archiveCol = this.client.getCollection(archiveCollection);
|
||||||
|
|
||||||
|
// Find old documents
|
||||||
|
const documents = await collection.find(
|
||||||
|
{ created_at: { $lt: cutoffDate } },
|
||||||
|
{ limit: batchSize, session }
|
||||||
|
).toArray();
|
||||||
|
|
||||||
|
if (documents.length === 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add archive metadata
|
||||||
|
const now = new Date();
|
||||||
|
const documentsToArchive = documents.map(doc => ({
|
||||||
|
...doc,
|
||||||
|
archived_at: now,
|
||||||
|
archived_from: sourceCollection
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Insert into archive collection
|
||||||
|
await archiveCol.insertMany(documentsToArchive, { session });
|
||||||
|
|
||||||
|
// Remove from source collection
|
||||||
|
const ids = documents.map(doc => doc._id);
|
||||||
|
const deleteResult = await collection.deleteMany(
|
||||||
|
{ _id: { $in: ids } },
|
||||||
|
{ session }
|
||||||
|
);
|
||||||
|
|
||||||
|
return deleteResult.deletedCount || 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
totalArchived += batchArchived;
|
||||||
|
|
||||||
|
if (batchArchived === 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.debug(`Archived batch of ${batchArchived} documents`);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info(`Archived ${totalArchived} documents from ${sourceCollection} to ${archiveCollection}`);
|
||||||
|
return totalArchived;
|
||||||
|
}
|
||||||
|
}
|
||||||
215
libs/mongodb-client/src/types.ts
Normal file
215
libs/mongodb-client/src/types.ts
Normal file
|
|
@ -0,0 +1,215 @@
|
||||||
|
import { z } from 'zod';
|
||||||
|
import type { ObjectId } from 'mongodb';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MongoDB Client Configuration
|
||||||
|
*/
|
||||||
|
export interface MongoDBClientConfig {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
database: string;
|
||||||
|
username?: string;
|
||||||
|
password?: string;
|
||||||
|
authSource?: string;
|
||||||
|
uri?: string;
|
||||||
|
poolSettings?: {
|
||||||
|
maxPoolSize: number;
|
||||||
|
minPoolSize: number;
|
||||||
|
maxIdleTime: number;
|
||||||
|
};
|
||||||
|
timeouts?: {
|
||||||
|
connectTimeout: number;
|
||||||
|
socketTimeout: number;
|
||||||
|
serverSelectionTimeout: number;
|
||||||
|
};
|
||||||
|
tls?: {
|
||||||
|
enabled: boolean;
|
||||||
|
insecure: boolean;
|
||||||
|
caFile?: string;
|
||||||
|
};
|
||||||
|
options?: {
|
||||||
|
retryWrites: boolean;
|
||||||
|
journal: boolean;
|
||||||
|
readPreference: 'primary' | 'primaryPreferred' | 'secondary' | 'secondaryPreferred' | 'nearest';
|
||||||
|
writeConcern: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MongoDB Connection Options
|
||||||
|
*/
|
||||||
|
export interface MongoDBConnectionOptions {
|
||||||
|
retryAttempts?: number;
|
||||||
|
retryDelay?: number;
|
||||||
|
healthCheckInterval?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Health Status Types
|
||||||
|
*/
|
||||||
|
export type MongoDBHealthStatus = 'healthy' | 'degraded' | 'unhealthy';
|
||||||
|
|
||||||
|
export interface MongoDBHealthCheck {
|
||||||
|
status: MongoDBHealthStatus;
|
||||||
|
timestamp: Date;
|
||||||
|
latency: number;
|
||||||
|
connections: {
|
||||||
|
active: number;
|
||||||
|
available: number;
|
||||||
|
total: number;
|
||||||
|
};
|
||||||
|
errors?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MongoDBMetrics {
|
||||||
|
operationsPerSecond: number;
|
||||||
|
averageLatency: number;
|
||||||
|
errorRate: number;
|
||||||
|
connectionPoolUtilization: number;
|
||||||
|
documentsProcessed: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collection Names
|
||||||
|
*/
|
||||||
|
export type CollectionNames =
|
||||||
|
| 'sentiment_data'
|
||||||
|
| 'raw_documents'
|
||||||
|
| 'news_articles'
|
||||||
|
| 'sec_filings'
|
||||||
|
| 'earnings_transcripts'
|
||||||
|
| 'analyst_reports'
|
||||||
|
| 'social_media_posts'
|
||||||
|
| 'market_events'
|
||||||
|
| 'economic_indicators';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Base Document Interface
|
||||||
|
*/
|
||||||
|
export interface DocumentBase {
|
||||||
|
_id?: ObjectId;
|
||||||
|
created_at: Date;
|
||||||
|
updated_at: Date;
|
||||||
|
source: string;
|
||||||
|
metadata?: Record<string, any>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sentiment Data Document
|
||||||
|
*/
|
||||||
|
export interface SentimentData extends DocumentBase {
|
||||||
|
symbol: string;
|
||||||
|
sentiment_score: number;
|
||||||
|
sentiment_label: 'positive' | 'negative' | 'neutral';
|
||||||
|
confidence: number;
|
||||||
|
text: string;
|
||||||
|
source_type: 'reddit' | 'twitter' | 'news' | 'forums';
|
||||||
|
source_id: string;
|
||||||
|
timestamp: Date;
|
||||||
|
processed_at: Date;
|
||||||
|
language: string;
|
||||||
|
keywords: string[];
|
||||||
|
entities: Array<{
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
confidence: number;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Raw Document
|
||||||
|
*/
|
||||||
|
export interface RawDocument extends DocumentBase {
|
||||||
|
document_type: 'html' | 'pdf' | 'text' | 'json' | 'xml';
|
||||||
|
content: string;
|
||||||
|
content_hash: string;
|
||||||
|
url?: string;
|
||||||
|
title?: string;
|
||||||
|
author?: string;
|
||||||
|
published_date?: Date;
|
||||||
|
extracted_text?: string;
|
||||||
|
processing_status: 'pending' | 'processed' | 'failed';
|
||||||
|
size_bytes: number;
|
||||||
|
language?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* News Article
|
||||||
|
*/
|
||||||
|
export interface NewsArticle extends DocumentBase {
|
||||||
|
headline: string;
|
||||||
|
content: string;
|
||||||
|
summary?: string;
|
||||||
|
author: string;
|
||||||
|
publication: string;
|
||||||
|
published_date: Date;
|
||||||
|
url: string;
|
||||||
|
symbols: string[];
|
||||||
|
categories: string[];
|
||||||
|
sentiment_score?: number;
|
||||||
|
relevance_score?: number;
|
||||||
|
image_url?: string;
|
||||||
|
tags: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SEC Filing
|
||||||
|
*/
|
||||||
|
export interface SecFiling extends DocumentBase {
|
||||||
|
cik: string;
|
||||||
|
accession_number: string;
|
||||||
|
filing_type: string;
|
||||||
|
company_name: string;
|
||||||
|
symbols: string[];
|
||||||
|
filing_date: Date;
|
||||||
|
period_end_date: Date;
|
||||||
|
url: string;
|
||||||
|
content: string;
|
||||||
|
extracted_data?: Record<string, any>;
|
||||||
|
financial_statements?: Array<{
|
||||||
|
statement_type: string;
|
||||||
|
data: Record<string, number>;
|
||||||
|
}>;
|
||||||
|
processing_status: 'pending' | 'processed' | 'failed';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Earnings Transcript
|
||||||
|
*/
|
||||||
|
export interface EarningsTranscript extends DocumentBase {
|
||||||
|
symbol: string;
|
||||||
|
company_name: string;
|
||||||
|
quarter: string;
|
||||||
|
year: number;
|
||||||
|
call_date: Date;
|
||||||
|
transcript: string;
|
||||||
|
participants: Array<{
|
||||||
|
name: string;
|
||||||
|
title: string;
|
||||||
|
type: 'executive' | 'analyst';
|
||||||
|
}>;
|
||||||
|
key_topics: string[];
|
||||||
|
sentiment_analysis?: {
|
||||||
|
overall_sentiment: number;
|
||||||
|
topic_sentiments: Record<string, number>;
|
||||||
|
};
|
||||||
|
financial_highlights?: Record<string, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyst Report
|
||||||
|
*/
|
||||||
|
export interface AnalystReport extends DocumentBase {
|
||||||
|
symbol: string;
|
||||||
|
analyst_firm: string;
|
||||||
|
analyst_name: string;
|
||||||
|
report_title: string;
|
||||||
|
report_date: Date;
|
||||||
|
rating: 'buy' | 'hold' | 'sell' | 'strong_buy' | 'strong_sell';
|
||||||
|
price_target?: number;
|
||||||
|
previous_rating?: string;
|
||||||
|
content: string;
|
||||||
|
summary: string;
|
||||||
|
key_points: string[];
|
||||||
|
financial_projections?: Record<string, number>;
|
||||||
|
}
|
||||||
19
libs/mongodb-client/tsconfig.json
Normal file
19
libs/mongodb-client/tsconfig.json
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true
|
||||||
|
},
|
||||||
|
"include": [
|
||||||
|
"src/**/*"
|
||||||
|
],
|
||||||
|
"exclude": [
|
||||||
|
"dist",
|
||||||
|
"node_modules",
|
||||||
|
"**/*.test.ts",
|
||||||
|
"**/*.spec.ts"
|
||||||
|
]
|
||||||
|
}
|
||||||
82
libs/postgres-client/README.md
Normal file
82
libs/postgres-client/README.md
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
# PostgreSQL Client Library
|
||||||
|
|
||||||
|
A comprehensive PostgreSQL client library for the Stock Bot trading platform, designed for operational data, transactions, and relational queries.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Connection Pooling**: Robust connection pool management
|
||||||
|
- **Type Safety**: Full TypeScript support with typed queries
|
||||||
|
- **Transaction Support**: Multi-statement transactions with rollback
|
||||||
|
- **Schema Management**: Database schema validation and migrations
|
||||||
|
- **Query Builder**: Fluent query building interface
|
||||||
|
- **Health Monitoring**: Connection health monitoring and metrics
|
||||||
|
- **Performance Tracking**: Query performance monitoring and optimization
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { PostgreSQLClient } from '@stock-bot/postgres-client';
|
||||||
|
|
||||||
|
// Initialize client
|
||||||
|
const pgClient = new PostgreSQLClient();
|
||||||
|
await pgClient.connect();
|
||||||
|
|
||||||
|
// Execute a query
|
||||||
|
const users = await pgClient.query('SELECT * FROM users WHERE active = $1', [true]);
|
||||||
|
|
||||||
|
// Use query builder
|
||||||
|
const trades = await pgClient
|
||||||
|
.select('*')
|
||||||
|
.from('trades')
|
||||||
|
.where('symbol', '=', 'AAPL')
|
||||||
|
.orderBy('created_at', 'DESC')
|
||||||
|
.limit(10)
|
||||||
|
.execute();
|
||||||
|
|
||||||
|
// Execute in transaction
|
||||||
|
await pgClient.transaction(async (tx) => {
|
||||||
|
await tx.query('INSERT INTO trades (...) VALUES (...)', []);
|
||||||
|
await tx.query('UPDATE portfolio SET balance = balance - $1', [amount]);
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## Database Schemas
|
||||||
|
|
||||||
|
The client provides typed access to the following schemas:
|
||||||
|
|
||||||
|
- **trading**: Core trading operations (trades, orders, positions)
|
||||||
|
- **strategy**: Strategy definitions and performance
|
||||||
|
- **risk**: Risk management and compliance
|
||||||
|
- **audit**: Audit trails and logging
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Configure using environment variables:
|
||||||
|
|
||||||
|
```env
|
||||||
|
POSTGRES_HOST=localhost
|
||||||
|
POSTGRES_PORT=5432
|
||||||
|
POSTGRES_DATABASE=stockbot
|
||||||
|
POSTGRES_USERNAME=stockbot
|
||||||
|
POSTGRES_PASSWORD=your_password
|
||||||
|
```
|
||||||
|
|
||||||
|
## Query Builder
|
||||||
|
|
||||||
|
The fluent query builder supports:
|
||||||
|
|
||||||
|
- SELECT, INSERT, UPDATE, DELETE operations
|
||||||
|
- Complex WHERE conditions with AND/OR logic
|
||||||
|
- JOINs (INNER, LEFT, RIGHT, FULL)
|
||||||
|
- Aggregations (COUNT, SUM, AVG, etc.)
|
||||||
|
- Subqueries and CTEs
|
||||||
|
- Window functions
|
||||||
|
|
||||||
|
## Health Monitoring
|
||||||
|
|
||||||
|
The client includes built-in health monitoring:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const health = await pgClient.getHealth();
|
||||||
|
console.log(health.status); // 'healthy' | 'degraded' | 'unhealthy'
|
||||||
|
```
|
||||||
42
libs/postgres-client/package.json
Normal file
42
libs/postgres-client/package.json
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
{
|
||||||
|
"name": "@stock-bot/postgres-client",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "PostgreSQL client library for Stock Bot platform",
|
||||||
|
"main": "src/index.ts",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"test": "bun test",
|
||||||
|
"lint": "eslint src/**/*.ts",
|
||||||
|
"type-check": "tsc --noEmit",
|
||||||
|
"dev": "tsc --watch"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@stock-bot/config": "*",
|
||||||
|
"@stock-bot/logger": "*",
|
||||||
|
"@stock-bot/types": "*",
|
||||||
|
"pg": "^8.11.3",
|
||||||
|
"zod": "^3.22.4"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.11.0",
|
||||||
|
"@types/pg": "^8.10.7",
|
||||||
|
"typescript": "^5.3.0",
|
||||||
|
"eslint": "^8.56.0",
|
||||||
|
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
||||||
|
"@typescript-eslint/parser": "^6.19.0",
|
||||||
|
"bun-types": "^1.2.15"
|
||||||
|
},
|
||||||
|
"keywords": [
|
||||||
|
"postgresql",
|
||||||
|
"database",
|
||||||
|
"client",
|
||||||
|
"stock-bot"
|
||||||
|
],
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"import": "./src/index.ts",
|
||||||
|
"require": "./dist/index.js"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
339
libs/postgres-client/src/client.ts
Normal file
339
libs/postgres-client/src/client.ts
Normal file
|
|
@ -0,0 +1,339 @@
|
||||||
|
import { Pool, PoolClient, QueryResult as PgQueryResult } from 'pg';
|
||||||
|
import { postgresConfig } from '@stock-bot/config';
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type {
|
||||||
|
PostgreSQLClientConfig,
|
||||||
|
PostgreSQLConnectionOptions,
|
||||||
|
QueryResult,
|
||||||
|
TransactionCallback
|
||||||
|
} from './types';
|
||||||
|
import { PostgreSQLHealthMonitor } from './health';
|
||||||
|
import { PostgreSQLQueryBuilder } from './query-builder';
|
||||||
|
import { PostgreSQLTransactionManager } from './transactions';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PostgreSQL Client for Stock Bot
|
||||||
|
*
|
||||||
|
* Provides type-safe access to PostgreSQL with connection pooling,
|
||||||
|
* health monitoring, and transaction support.
|
||||||
|
*/
|
||||||
|
export class PostgreSQLClient {
|
||||||
|
private pool: Pool | null = null;
|
||||||
|
private readonly config: PostgreSQLClientConfig;
|
||||||
|
private readonly options: PostgreSQLConnectionOptions;
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private readonly healthMonitor: PostgreSQLHealthMonitor;
|
||||||
|
private readonly transactionManager: PostgreSQLTransactionManager;
|
||||||
|
private isConnected = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
config?: Partial<PostgreSQLClientConfig>,
|
||||||
|
options?: PostgreSQLConnectionOptions
|
||||||
|
) {
|
||||||
|
this.config = this.buildConfig(config);
|
||||||
|
this.options = {
|
||||||
|
retryAttempts: 3,
|
||||||
|
retryDelay: 1000,
|
||||||
|
healthCheckInterval: 30000,
|
||||||
|
...options
|
||||||
|
};
|
||||||
|
|
||||||
|
this.logger = new Logger('PostgreSQLClient');
|
||||||
|
this.healthMonitor = new PostgreSQLHealthMonitor(this);
|
||||||
|
this.transactionManager = new PostgreSQLTransactionManager(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to PostgreSQL
|
||||||
|
*/
|
||||||
|
async connect(): Promise<void> {
|
||||||
|
if (this.isConnected && this.pool) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let lastError: Error | null = null;
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= this.options.retryAttempts!; attempt++) {
|
||||||
|
try {
|
||||||
|
this.logger.info(`Connecting to PostgreSQL (attempt ${attempt}/${this.options.retryAttempts})...`);
|
||||||
|
|
||||||
|
this.pool = new Pool(this.buildPoolConfig());
|
||||||
|
|
||||||
|
// Test the connection
|
||||||
|
const client = await this.pool.connect();
|
||||||
|
await client.query('SELECT 1');
|
||||||
|
client.release();
|
||||||
|
|
||||||
|
this.isConnected = true;
|
||||||
|
this.logger.info('Successfully connected to PostgreSQL');
|
||||||
|
|
||||||
|
// Start health monitoring
|
||||||
|
this.healthMonitor.start();
|
||||||
|
|
||||||
|
// Setup error handlers
|
||||||
|
this.setupErrorHandlers();
|
||||||
|
|
||||||
|
return;
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error as Error;
|
||||||
|
this.logger.error(`PostgreSQL connection attempt ${attempt} failed:`, error);
|
||||||
|
|
||||||
|
if (this.pool) {
|
||||||
|
await this.pool.end();
|
||||||
|
this.pool = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt < this.options.retryAttempts!) {
|
||||||
|
await this.delay(this.options.retryDelay! * attempt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Failed to connect to PostgreSQL after ${this.options.retryAttempts} attempts: ${lastError?.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disconnect from PostgreSQL
|
||||||
|
*/
|
||||||
|
async disconnect(): Promise<void> {
|
||||||
|
if (!this.pool) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.healthMonitor.stop();
|
||||||
|
await this.pool.end();
|
||||||
|
this.isConnected = false;
|
||||||
|
this.pool = null;
|
||||||
|
this.logger.info('Disconnected from PostgreSQL');
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Error disconnecting from PostgreSQL:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a query
|
||||||
|
*/
|
||||||
|
async query<T = any>(text: string, params?: any[]): Promise<QueryResult<T>> {
|
||||||
|
if (!this.pool) {
|
||||||
|
throw new Error('PostgreSQL client not connected');
|
||||||
|
}
|
||||||
|
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await this.pool.query<T>(text, params);
|
||||||
|
const executionTime = Date.now() - startTime;
|
||||||
|
|
||||||
|
this.logger.debug(`Query executed in ${executionTime}ms`, {
|
||||||
|
query: text.substring(0, 100),
|
||||||
|
params: params?.length
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
...result,
|
||||||
|
executionTime
|
||||||
|
} as QueryResult<T>;
|
||||||
|
} catch (error) {
|
||||||
|
const executionTime = Date.now() - startTime;
|
||||||
|
this.logger.error(`Query failed after ${executionTime}ms:`, {
|
||||||
|
error: (error as Error).message,
|
||||||
|
query: text,
|
||||||
|
params
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute multiple queries in a transaction
|
||||||
|
*/
|
||||||
|
async transaction<T>(callback: TransactionCallback<T>): Promise<T> {
|
||||||
|
return await this.transactionManager.execute(callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a query builder instance
|
||||||
|
*/
|
||||||
|
queryBuilder(): PostgreSQLQueryBuilder {
|
||||||
|
return new PostgreSQLQueryBuilder(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new query builder with SELECT
|
||||||
|
*/
|
||||||
|
select(columns: string | string[] = '*'): PostgreSQLQueryBuilder {
|
||||||
|
return this.queryBuilder().select(columns);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new query builder with INSERT
|
||||||
|
*/
|
||||||
|
insert(table: string): PostgreSQLQueryBuilder {
|
||||||
|
return this.queryBuilder().insert(table);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new query builder with UPDATE
|
||||||
|
*/
|
||||||
|
update(table: string): PostgreSQLQueryBuilder {
|
||||||
|
return this.queryBuilder().update(table);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new query builder with DELETE
|
||||||
|
*/
|
||||||
|
delete(table: string): PostgreSQLQueryBuilder {
|
||||||
|
return this.queryBuilder().delete(table);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a stored procedure or function
|
||||||
|
*/
|
||||||
|
async callFunction<T = any>(functionName: string, params?: any[]): Promise<QueryResult<T>> {
|
||||||
|
const placeholders = params ? params.map((_, i) => `$${i + 1}`).join(', ') : '';
|
||||||
|
const query = `SELECT * FROM ${functionName}(${placeholders})`;
|
||||||
|
return await this.query<T>(query, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a table exists
|
||||||
|
*/
|
||||||
|
async tableExists(tableName: string, schemaName: string = 'public'): Promise<boolean> {
|
||||||
|
const result = await this.query(
|
||||||
|
`SELECT EXISTS (
|
||||||
|
SELECT FROM information_schema.tables
|
||||||
|
WHERE table_schema = $1 AND table_name = $2
|
||||||
|
)`,
|
||||||
|
[schemaName, tableName]
|
||||||
|
);
|
||||||
|
return result.rows[0].exists;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get table schema information
|
||||||
|
*/
|
||||||
|
async getTableSchema(tableName: string, schemaName: string = 'public'): Promise<any[]> {
|
||||||
|
const result = await this.query(
|
||||||
|
`SELECT
|
||||||
|
column_name,
|
||||||
|
data_type,
|
||||||
|
is_nullable,
|
||||||
|
column_default,
|
||||||
|
character_maximum_length
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = $1 AND table_name = $2
|
||||||
|
ORDER BY ordinal_position`,
|
||||||
|
[schemaName, tableName]
|
||||||
|
);
|
||||||
|
return result.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute EXPLAIN for query analysis
|
||||||
|
*/
|
||||||
|
async explain(query: string, params?: any[]): Promise<any[]> {
|
||||||
|
const explainQuery = `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query}`;
|
||||||
|
const result = await this.query(explainQuery, params);
|
||||||
|
return result.rows[0]['QUERY PLAN'];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get database statistics
|
||||||
|
*/
|
||||||
|
async getStats(): Promise<any> {
|
||||||
|
const result = await this.query(`
|
||||||
|
SELECT
|
||||||
|
(SELECT count(*) FROM pg_stat_activity WHERE state = 'active') as active_connections,
|
||||||
|
(SELECT count(*) FROM pg_stat_activity WHERE state = 'idle') as idle_connections,
|
||||||
|
(SELECT setting FROM pg_settings WHERE name = 'max_connections') as max_connections,
|
||||||
|
pg_size_pretty(pg_database_size(current_database())) as database_size
|
||||||
|
`);
|
||||||
|
return result.rows[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if client is connected
|
||||||
|
*/
|
||||||
|
get connected(): boolean {
|
||||||
|
return this.isConnected && !!this.pool;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the underlying connection pool
|
||||||
|
*/
|
||||||
|
get connectionPool(): Pool | null {
|
||||||
|
return this.pool;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildConfig(config?: Partial<PostgreSQLClientConfig>): PostgreSQLClientConfig {
|
||||||
|
return {
|
||||||
|
host: config?.host || postgresConfig.POSTGRES_HOST,
|
||||||
|
port: config?.port || postgresConfig.POSTGRES_PORT,
|
||||||
|
database: config?.database || postgresConfig.POSTGRES_DATABASE,
|
||||||
|
username: config?.username || postgresConfig.POSTGRES_USERNAME,
|
||||||
|
password: config?.password || postgresConfig.POSTGRES_PASSWORD,
|
||||||
|
poolSettings: {
|
||||||
|
min: postgresConfig.POSTGRES_POOL_MIN,
|
||||||
|
max: postgresConfig.POSTGRES_POOL_MAX,
|
||||||
|
idleTimeoutMillis: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT,
|
||||||
|
...config?.poolSettings
|
||||||
|
},
|
||||||
|
ssl: {
|
||||||
|
enabled: postgresConfig.POSTGRES_SSL,
|
||||||
|
rejectUnauthorized: postgresConfig.POSTGRES_SSL_REJECT_UNAUTHORIZED,
|
||||||
|
...config?.ssl
|
||||||
|
},
|
||||||
|
timeouts: {
|
||||||
|
query: postgresConfig.POSTGRES_QUERY_TIMEOUT,
|
||||||
|
connection: postgresConfig.POSTGRES_CONNECTION_TIMEOUT,
|
||||||
|
statement: postgresConfig.POSTGRES_STATEMENT_TIMEOUT,
|
||||||
|
lock: postgresConfig.POSTGRES_LOCK_TIMEOUT,
|
||||||
|
idleInTransaction: postgresConfig.POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT,
|
||||||
|
...config?.timeouts
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildPoolConfig(): any {
|
||||||
|
return {
|
||||||
|
host: this.config.host,
|
||||||
|
port: this.config.port,
|
||||||
|
database: this.config.database,
|
||||||
|
user: this.config.username,
|
||||||
|
password: this.config.password,
|
||||||
|
min: this.config.poolSettings?.min,
|
||||||
|
max: this.config.poolSettings?.max,
|
||||||
|
idleTimeoutMillis: this.config.poolSettings?.idleTimeoutMillis,
|
||||||
|
connectionTimeoutMillis: this.config.timeouts?.connection,
|
||||||
|
query_timeout: this.config.timeouts?.query,
|
||||||
|
statement_timeout: this.config.timeouts?.statement,
|
||||||
|
lock_timeout: this.config.timeouts?.lock,
|
||||||
|
idle_in_transaction_session_timeout: this.config.timeouts?.idleInTransaction,
|
||||||
|
ssl: this.config.ssl?.enabled ? {
|
||||||
|
rejectUnauthorized: this.config.ssl.rejectUnauthorized
|
||||||
|
} : false
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private setupErrorHandlers(): void {
|
||||||
|
if (!this.pool) return;
|
||||||
|
|
||||||
|
this.pool.on('error', (err) => {
|
||||||
|
this.logger.error('PostgreSQL pool error:', err);
|
||||||
|
});
|
||||||
|
|
||||||
|
this.pool.on('connect', () => {
|
||||||
|
this.logger.debug('New PostgreSQL client connected');
|
||||||
|
});
|
||||||
|
|
||||||
|
this.pool.on('remove', () => {
|
||||||
|
this.logger.debug('PostgreSQL client removed from pool');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private delay(ms: number): Promise<void> {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
}
|
||||||
64
libs/postgres-client/src/factory.ts
Normal file
64
libs/postgres-client/src/factory.ts
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
import { PostgreSQLClient } from './client';
|
||||||
|
import { postgresConfig } from '@stock-bot/config';
|
||||||
|
import type { PostgreSQLClientConfig, PostgreSQLConnectionOptions } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Factory function to create a PostgreSQL client instance
|
||||||
|
*/
|
||||||
|
export function createPostgreSQLClient(
|
||||||
|
config?: Partial<PostgreSQLClientConfig>,
|
||||||
|
options?: PostgreSQLConnectionOptions
|
||||||
|
): PostgreSQLClient {
|
||||||
|
return new PostgreSQLClient(config, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a PostgreSQL client with default configuration
|
||||||
|
*/
|
||||||
|
export function createDefaultPostgreSQLClient(): PostgreSQLClient {
|
||||||
|
const config: Partial<PostgreSQLClientConfig> = {
|
||||||
|
host: postgresConfig.POSTGRES_HOST,
|
||||||
|
port: postgresConfig.POSTGRES_PORT,
|
||||||
|
database: postgresConfig.POSTGRES_DATABASE,
|
||||||
|
username: postgresConfig.POSTGRES_USERNAME,
|
||||||
|
password: postgresConfig.POSTGRES_PASSWORD
|
||||||
|
};
|
||||||
|
|
||||||
|
return new PostgreSQLClient(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Singleton PostgreSQL client instance
|
||||||
|
*/
|
||||||
|
let defaultClient: PostgreSQLClient | null = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get or create the default PostgreSQL client instance
|
||||||
|
*/
|
||||||
|
export function getPostgreSQLClient(): PostgreSQLClient {
|
||||||
|
if (!defaultClient) {
|
||||||
|
defaultClient = createDefaultPostgreSQLClient();
|
||||||
|
}
|
||||||
|
return defaultClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to PostgreSQL using the default client
|
||||||
|
*/
|
||||||
|
export async function connectPostgreSQL(): Promise<PostgreSQLClient> {
|
||||||
|
const client = getPostgreSQLClient();
|
||||||
|
if (!client.connected) {
|
||||||
|
await client.connect();
|
||||||
|
}
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disconnect from PostgreSQL
|
||||||
|
*/
|
||||||
|
export async function disconnectPostgreSQL(): Promise<void> {
|
||||||
|
if (defaultClient) {
|
||||||
|
await defaultClient.disconnect();
|
||||||
|
defaultClient = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
142
libs/postgres-client/src/health.ts
Normal file
142
libs/postgres-client/src/health.ts
Normal file
|
|
@ -0,0 +1,142 @@
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type { PostgreSQLClient } from './client';
|
||||||
|
import type { PostgreSQLHealthCheck, PostgreSQLHealthStatus, PostgreSQLMetrics } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PostgreSQL Health Monitor
|
||||||
|
*
|
||||||
|
* Monitors PostgreSQL connection health and provides metrics
|
||||||
|
*/
|
||||||
|
export class PostgreSQLHealthMonitor {
|
||||||
|
private readonly client: PostgreSQLClient;
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private healthCheckInterval: NodeJS.Timeout | null = null;
|
||||||
|
private metrics: PostgreSQLMetrics;
|
||||||
|
private lastHealthCheck: PostgreSQLHealthCheck | null = null;
|
||||||
|
|
||||||
|
constructor(client: PostgreSQLClient) {
|
||||||
|
this.client = client;
|
||||||
|
this.logger = new Logger('PostgreSQLHealthMonitor');
|
||||||
|
this.metrics = {
|
||||||
|
queriesPerSecond: 0,
|
||||||
|
averageQueryTime: 0,
|
||||||
|
errorRate: 0,
|
||||||
|
connectionPoolUtilization: 0,
|
||||||
|
slowQueries: 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start health monitoring
|
||||||
|
*/
|
||||||
|
start(intervalMs: number = 30000): void {
|
||||||
|
if (this.healthCheckInterval) {
|
||||||
|
this.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info(`Starting PostgreSQL health monitoring (interval: ${intervalMs}ms)`);
|
||||||
|
|
||||||
|
this.healthCheckInterval = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
await this.performHealthCheck();
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Health check failed:', error);
|
||||||
|
}
|
||||||
|
}, intervalMs);
|
||||||
|
|
||||||
|
// Perform initial health check
|
||||||
|
this.performHealthCheck().catch(error => {
|
||||||
|
this.logger.error('Initial health check failed:', error);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop health monitoring
|
||||||
|
*/
|
||||||
|
stop(): void {
|
||||||
|
if (this.healthCheckInterval) {
|
||||||
|
clearInterval(this.healthCheckInterval);
|
||||||
|
this.healthCheckInterval = null;
|
||||||
|
this.logger.info('Stopped PostgreSQL health monitoring');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current health status
|
||||||
|
*/
|
||||||
|
async getHealth(): Promise<PostgreSQLHealthCheck> {
|
||||||
|
if (!this.lastHealthCheck) {
|
||||||
|
await this.performHealthCheck();
|
||||||
|
}
|
||||||
|
return this.lastHealthCheck!;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current metrics
|
||||||
|
*/
|
||||||
|
getMetrics(): PostgreSQLMetrics {
|
||||||
|
return { ...this.metrics };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perform a health check
|
||||||
|
*/
|
||||||
|
private async performHealthCheck(): Promise<void> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
const errors: string[] = [];
|
||||||
|
let status: PostgreSQLHealthStatus = 'healthy';
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!this.client.connected) {
|
||||||
|
errors.push('PostgreSQL client not connected');
|
||||||
|
status = 'unhealthy';
|
||||||
|
} else {
|
||||||
|
// Test basic connectivity
|
||||||
|
await this.client.query('SELECT 1');
|
||||||
|
|
||||||
|
// Get connection stats
|
||||||
|
const stats = await this.client.getStats();
|
||||||
|
|
||||||
|
// Check connection pool utilization
|
||||||
|
const utilization = parseInt(stats.active_connections) / parseInt(stats.max_connections);
|
||||||
|
if (utilization > 0.8) {
|
||||||
|
errors.push('High connection pool utilization');
|
||||||
|
status = status === 'healthy' ? 'degraded' : status;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for high latency
|
||||||
|
const latency = Date.now() - startTime;
|
||||||
|
if (latency > 1000) {
|
||||||
|
errors.push(`High latency: ${latency}ms`);
|
||||||
|
status = status === 'healthy' ? 'degraded' : status;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.metrics.connectionPoolUtilization = utilization;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
errors.push(`Health check failed: ${(error as Error).message}`);
|
||||||
|
status = 'unhealthy';
|
||||||
|
}
|
||||||
|
|
||||||
|
const latency = Date.now() - startTime;
|
||||||
|
|
||||||
|
this.lastHealthCheck = {
|
||||||
|
status,
|
||||||
|
timestamp: new Date(),
|
||||||
|
latency,
|
||||||
|
connections: {
|
||||||
|
active: 1,
|
||||||
|
idle: 9,
|
||||||
|
total: 10
|
||||||
|
},
|
||||||
|
errors: errors.length > 0 ? errors : undefined
|
||||||
|
};
|
||||||
|
|
||||||
|
// Log health status changes
|
||||||
|
if (status !== 'healthy') {
|
||||||
|
this.logger.warn(`PostgreSQL health status: ${status}`, { errors, latency });
|
||||||
|
} else {
|
||||||
|
this.logger.debug(`PostgreSQL health check passed (${latency}ms)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
34
libs/postgres-client/src/index.ts
Normal file
34
libs/postgres-client/src/index.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
/**
|
||||||
|
* PostgreSQL Client Library for Stock Bot
|
||||||
|
*
|
||||||
|
* Provides type-safe PostgreSQL access for operational data,
|
||||||
|
* transactions, and relational queries.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export { PostgreSQLClient } from './client';
|
||||||
|
export { PostgreSQLHealthMonitor } from './health';
|
||||||
|
export { PostgreSQLTransactionManager } from './transactions';
|
||||||
|
export { PostgreSQLQueryBuilder } from './query-builder';
|
||||||
|
export { PostgreSQLMigrationManager } from './migrations';
|
||||||
|
|
||||||
|
// Types
|
||||||
|
export type {
|
||||||
|
PostgreSQLClientConfig,
|
||||||
|
PostgreSQLConnectionOptions,
|
||||||
|
PostgreSQLHealthStatus,
|
||||||
|
PostgreSQLMetrics,
|
||||||
|
QueryResult,
|
||||||
|
TransactionCallback,
|
||||||
|
SchemaNames,
|
||||||
|
TableNames,
|
||||||
|
Trade,
|
||||||
|
Order,
|
||||||
|
Position,
|
||||||
|
Portfolio,
|
||||||
|
Strategy,
|
||||||
|
RiskLimit,
|
||||||
|
AuditLog
|
||||||
|
} from './types';
|
||||||
|
|
||||||
|
// Utils
|
||||||
|
export { createPostgreSQLClient, getPostgreSQLClient } from './factory';
|
||||||
267
libs/postgres-client/src/query-builder.ts
Normal file
267
libs/postgres-client/src/query-builder.ts
Normal file
|
|
@ -0,0 +1,267 @@
|
||||||
|
import type { PostgreSQLClient } from './client';
|
||||||
|
import type { WhereCondition, JoinCondition, OrderByCondition, QueryResult } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PostgreSQL Query Builder
|
||||||
|
*
|
||||||
|
* Provides a fluent interface for building SQL queries
|
||||||
|
*/
|
||||||
|
export class PostgreSQLQueryBuilder {
|
||||||
|
private queryType: 'SELECT' | 'INSERT' | 'UPDATE' | 'DELETE' | null = null;
|
||||||
|
private selectColumns: string[] = [];
|
||||||
|
private fromTable: string = '';
|
||||||
|
private joins: JoinCondition[] = [];
|
||||||
|
private whereConditions: WhereCondition[] = [];
|
||||||
|
private groupByColumns: string[] = [];
|
||||||
|
private havingConditions: WhereCondition[] = [];
|
||||||
|
private orderByConditions: OrderByCondition[] = [];
|
||||||
|
private limitCount: number | null = null;
|
||||||
|
private offsetCount: number | null = null;
|
||||||
|
private insertValues: Record<string, any> = {};
|
||||||
|
private updateValues: Record<string, any> = {};
|
||||||
|
|
||||||
|
private readonly client: PostgreSQLClient;
|
||||||
|
|
||||||
|
constructor(client: PostgreSQLClient) {
|
||||||
|
this.client = client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SELECT statement
|
||||||
|
*/
|
||||||
|
select(columns: string | string[] = '*'): this {
|
||||||
|
this.queryType = 'SELECT';
|
||||||
|
this.selectColumns = Array.isArray(columns) ? columns : [columns];
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FROM clause
|
||||||
|
*/
|
||||||
|
from(table: string): this {
|
||||||
|
this.fromTable = table;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JOIN clause
|
||||||
|
*/
|
||||||
|
join(table: string, on: string, type: 'INNER' | 'LEFT' | 'RIGHT' | 'FULL' = 'INNER'): this {
|
||||||
|
this.joins.push({ type, table, on });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* WHERE clause
|
||||||
|
*/
|
||||||
|
where(column: string, operator: string, value?: any): this {
|
||||||
|
this.whereConditions.push({ column, operator: operator as any, value });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GROUP BY clause
|
||||||
|
*/
|
||||||
|
groupBy(columns: string | string[]): this {
|
||||||
|
this.groupByColumns = Array.isArray(columns) ? columns : [columns];
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ORDER BY clause
|
||||||
|
*/
|
||||||
|
orderBy(column: string, direction: 'ASC' | 'DESC' = 'ASC'): this {
|
||||||
|
this.orderByConditions.push({ column, direction });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* LIMIT clause
|
||||||
|
*/
|
||||||
|
limit(count: number): this {
|
||||||
|
this.limitCount = count;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OFFSET clause
|
||||||
|
*/
|
||||||
|
offset(count: number): this {
|
||||||
|
this.offsetCount = count;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* INSERT statement
|
||||||
|
*/
|
||||||
|
insert(table: string): this {
|
||||||
|
this.queryType = 'INSERT';
|
||||||
|
this.fromTable = table;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* VALUES for INSERT
|
||||||
|
*/
|
||||||
|
values(data: Record<string, any>): this {
|
||||||
|
this.insertValues = data;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* UPDATE statement
|
||||||
|
*/
|
||||||
|
update(table: string): this {
|
||||||
|
this.queryType = 'UPDATE';
|
||||||
|
this.fromTable = table;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SET for UPDATE
|
||||||
|
*/
|
||||||
|
set(data: Record<string, any>): this {
|
||||||
|
this.updateValues = data;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DELETE statement
|
||||||
|
*/
|
||||||
|
delete(table: string): this {
|
||||||
|
this.queryType = 'DELETE';
|
||||||
|
this.fromTable = table;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build and execute the query
|
||||||
|
*/
|
||||||
|
async execute<T = any>(): Promise<QueryResult<T>> {
|
||||||
|
const { sql, params } = this.build();
|
||||||
|
return await this.client.query<T>(sql, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the SQL query
|
||||||
|
*/
|
||||||
|
build(): { sql: string; params: any[] } {
|
||||||
|
const params: any[] = [];
|
||||||
|
let sql = '';
|
||||||
|
|
||||||
|
switch (this.queryType) {
|
||||||
|
case 'SELECT':
|
||||||
|
sql = this.buildSelectQuery(params);
|
||||||
|
break;
|
||||||
|
case 'INSERT':
|
||||||
|
sql = this.buildInsertQuery(params);
|
||||||
|
break;
|
||||||
|
case 'UPDATE':
|
||||||
|
sql = this.buildUpdateQuery(params);
|
||||||
|
break;
|
||||||
|
case 'DELETE':
|
||||||
|
sql = this.buildDeleteQuery(params);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new Error('Query type not specified');
|
||||||
|
}
|
||||||
|
|
||||||
|
return { sql, params };
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildSelectQuery(params: any[]): string {
|
||||||
|
let sql = `SELECT ${this.selectColumns.join(', ')}`;
|
||||||
|
|
||||||
|
if (this.fromTable) {
|
||||||
|
sql += ` FROM ${this.fromTable}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add JOINs
|
||||||
|
for (const join of this.joins) {
|
||||||
|
sql += ` ${join.type} JOIN ${join.table} ON ${join.on}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add WHERE
|
||||||
|
if (this.whereConditions.length > 0) {
|
||||||
|
sql += ' WHERE ' + this.buildWhereClause(this.whereConditions, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add GROUP BY
|
||||||
|
if (this.groupByColumns.length > 0) {
|
||||||
|
sql += ` GROUP BY ${this.groupByColumns.join(', ')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add HAVING
|
||||||
|
if (this.havingConditions.length > 0) {
|
||||||
|
sql += ' HAVING ' + this.buildWhereClause(this.havingConditions, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add ORDER BY
|
||||||
|
if (this.orderByConditions.length > 0) {
|
||||||
|
const orderBy = this.orderByConditions
|
||||||
|
.map(order => `${order.column} ${order.direction}`)
|
||||||
|
.join(', ');
|
||||||
|
sql += ` ORDER BY ${orderBy}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add LIMIT
|
||||||
|
if (this.limitCount !== null) {
|
||||||
|
sql += ` LIMIT $${params.length + 1}`;
|
||||||
|
params.push(this.limitCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add OFFSET
|
||||||
|
if (this.offsetCount !== null) {
|
||||||
|
sql += ` OFFSET $${params.length + 1}`;
|
||||||
|
params.push(this.offsetCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
return sql;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildInsertQuery(params: any[]): string {
|
||||||
|
const columns = Object.keys(this.insertValues);
|
||||||
|
const placeholders = columns.map((_, i) => `$${params.length + i + 1}`);
|
||||||
|
|
||||||
|
params.push(...Object.values(this.insertValues));
|
||||||
|
|
||||||
|
return `INSERT INTO ${this.fromTable} (${columns.join(', ')}) VALUES (${placeholders.join(', ')})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildUpdateQuery(params: any[]): string {
|
||||||
|
const sets = Object.keys(this.updateValues).map((key, i) => {
|
||||||
|
return `${key} = $${params.length + i + 1}`;
|
||||||
|
});
|
||||||
|
|
||||||
|
params.push(...Object.values(this.updateValues));
|
||||||
|
|
||||||
|
let sql = `UPDATE ${this.fromTable} SET ${sets.join(', ')}`;
|
||||||
|
|
||||||
|
if (this.whereConditions.length > 0) {
|
||||||
|
sql += ' WHERE ' + this.buildWhereClause(this.whereConditions, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
return sql;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildDeleteQuery(params: any[]): string {
|
||||||
|
let sql = `DELETE FROM ${this.fromTable}`;
|
||||||
|
|
||||||
|
if (this.whereConditions.length > 0) {
|
||||||
|
sql += ' WHERE ' + this.buildWhereClause(this.whereConditions, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
return sql;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildWhereClause(conditions: WhereCondition[], params: any[]): string {
|
||||||
|
return conditions.map(condition => {
|
||||||
|
if (condition.operator === 'IS NULL' || condition.operator === 'IS NOT NULL') {
|
||||||
|
return `${condition.column} ${condition.operator}`;
|
||||||
|
} else {
|
||||||
|
params.push(condition.value);
|
||||||
|
return `${condition.column} ${condition.operator} $${params.length}`;
|
||||||
|
}
|
||||||
|
}).join(' AND ');
|
||||||
|
}
|
||||||
|
}
|
||||||
57
libs/postgres-client/src/transactions.ts
Normal file
57
libs/postgres-client/src/transactions.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
import { PoolClient } from 'pg';
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type { PostgreSQLClient } from './client';
|
||||||
|
import type { TransactionCallback } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PostgreSQL Transaction Manager
|
||||||
|
*
|
||||||
|
* Provides transaction support for multi-statement operations
|
||||||
|
*/
|
||||||
|
export class PostgreSQLTransactionManager {
|
||||||
|
private readonly client: PostgreSQLClient;
|
||||||
|
private readonly logger: Logger;
|
||||||
|
|
||||||
|
constructor(client: PostgreSQLClient) {
|
||||||
|
this.client = client;
|
||||||
|
this.logger = new Logger('PostgreSQLTransactionManager');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute operations within a transaction
|
||||||
|
*/
|
||||||
|
async execute<T>(callback: TransactionCallback<T>): Promise<T> {
|
||||||
|
const pool = this.client.connectionPool;
|
||||||
|
if (!pool) {
|
||||||
|
throw new Error('PostgreSQL client not connected');
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = await pool.connect();
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.logger.debug('Starting PostgreSQL transaction');
|
||||||
|
|
||||||
|
await client.query('BEGIN');
|
||||||
|
|
||||||
|
const result = await callback(client);
|
||||||
|
|
||||||
|
await client.query('COMMIT');
|
||||||
|
|
||||||
|
this.logger.debug('PostgreSQL transaction committed successfully');
|
||||||
|
return result;
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('PostgreSQL transaction failed, rolling back:', error);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await client.query('ROLLBACK');
|
||||||
|
} catch (rollbackError) {
|
||||||
|
this.logger.error('Failed to rollback transaction:', rollbackError);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
client.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
206
libs/postgres-client/src/types.ts
Normal file
206
libs/postgres-client/src/types.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
||||||
|
import type { Pool, PoolClient, QueryResult as PgQueryResult } from 'pg';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PostgreSQL Client Configuration
|
||||||
|
*/
|
||||||
|
export interface PostgreSQLClientConfig {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
database: string;
|
||||||
|
username: string;
|
||||||
|
password: string;
|
||||||
|
poolSettings?: {
|
||||||
|
min: number;
|
||||||
|
max: number;
|
||||||
|
idleTimeoutMillis: number;
|
||||||
|
};
|
||||||
|
ssl?: {
|
||||||
|
enabled: boolean;
|
||||||
|
rejectUnauthorized: boolean;
|
||||||
|
};
|
||||||
|
timeouts?: {
|
||||||
|
query: number;
|
||||||
|
connection: number;
|
||||||
|
statement: number;
|
||||||
|
lock: number;
|
||||||
|
idleInTransaction: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PostgreSQL Connection Options
|
||||||
|
*/
|
||||||
|
export interface PostgreSQLConnectionOptions {
|
||||||
|
retryAttempts?: number;
|
||||||
|
retryDelay?: number;
|
||||||
|
healthCheckInterval?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Health Status Types
|
||||||
|
*/
|
||||||
|
export type PostgreSQLHealthStatus = 'healthy' | 'degraded' | 'unhealthy';
|
||||||
|
|
||||||
|
export interface PostgreSQLHealthCheck {
|
||||||
|
status: PostgreSQLHealthStatus;
|
||||||
|
timestamp: Date;
|
||||||
|
latency: number;
|
||||||
|
connections: {
|
||||||
|
active: number;
|
||||||
|
idle: number;
|
||||||
|
total: number;
|
||||||
|
};
|
||||||
|
errors?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PostgreSQLMetrics {
|
||||||
|
queriesPerSecond: number;
|
||||||
|
averageQueryTime: number;
|
||||||
|
errorRate: number;
|
||||||
|
connectionPoolUtilization: number;
|
||||||
|
slowQueries: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query Result Types
|
||||||
|
*/
|
||||||
|
export interface QueryResult<T = any> extends PgQueryResult<T> {
|
||||||
|
executionTime?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type TransactionCallback<T> = (client: PoolClient) => Promise<T>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema and Table Names
|
||||||
|
*/
|
||||||
|
export type SchemaNames = 'trading' | 'strategy' | 'risk' | 'audit';
|
||||||
|
|
||||||
|
export type TableNames =
|
||||||
|
| 'trades'
|
||||||
|
| 'orders'
|
||||||
|
| 'positions'
|
||||||
|
| 'portfolios'
|
||||||
|
| 'strategies'
|
||||||
|
| 'risk_limits'
|
||||||
|
| 'audit_logs'
|
||||||
|
| 'users'
|
||||||
|
| 'accounts'
|
||||||
|
| 'symbols'
|
||||||
|
| 'exchanges';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Trading Domain Types
|
||||||
|
*/
|
||||||
|
export interface Trade {
|
||||||
|
id: string;
|
||||||
|
order_id: string;
|
||||||
|
symbol: string;
|
||||||
|
side: 'buy' | 'sell';
|
||||||
|
quantity: number;
|
||||||
|
price: number;
|
||||||
|
executed_at: Date;
|
||||||
|
commission: number;
|
||||||
|
fees: number;
|
||||||
|
portfolio_id: string;
|
||||||
|
strategy_id?: string;
|
||||||
|
created_at: Date;
|
||||||
|
updated_at: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Order {
|
||||||
|
id: string;
|
||||||
|
symbol: string;
|
||||||
|
side: 'buy' | 'sell';
|
||||||
|
type: 'market' | 'limit' | 'stop' | 'stop_limit';
|
||||||
|
quantity: number;
|
||||||
|
price?: number;
|
||||||
|
stop_price?: number;
|
||||||
|
status: 'pending' | 'filled' | 'cancelled' | 'rejected';
|
||||||
|
portfolio_id: string;
|
||||||
|
strategy_id?: string;
|
||||||
|
created_at: Date;
|
||||||
|
updated_at: Date;
|
||||||
|
expires_at?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Position {
|
||||||
|
id: string;
|
||||||
|
symbol: string;
|
||||||
|
quantity: number;
|
||||||
|
average_cost: number;
|
||||||
|
market_value: number;
|
||||||
|
unrealized_pnl: number;
|
||||||
|
realized_pnl: number;
|
||||||
|
portfolio_id: string;
|
||||||
|
created_at: Date;
|
||||||
|
updated_at: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Portfolio {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
cash_balance: number;
|
||||||
|
total_value: number;
|
||||||
|
unrealized_pnl: number;
|
||||||
|
realized_pnl: number;
|
||||||
|
user_id: string;
|
||||||
|
created_at: Date;
|
||||||
|
updated_at: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Strategy {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
parameters: Record<string, any>;
|
||||||
|
status: 'active' | 'inactive' | 'paused';
|
||||||
|
performance_metrics: Record<string, number>;
|
||||||
|
portfolio_id: string;
|
||||||
|
created_at: Date;
|
||||||
|
updated_at: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RiskLimit {
|
||||||
|
id: string;
|
||||||
|
type: 'position_size' | 'daily_loss' | 'max_drawdown' | 'concentration';
|
||||||
|
value: number;
|
||||||
|
threshold: number;
|
||||||
|
status: 'active' | 'breached' | 'disabled';
|
||||||
|
portfolio_id?: string;
|
||||||
|
strategy_id?: string;
|
||||||
|
created_at: Date;
|
||||||
|
updated_at: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AuditLog {
|
||||||
|
id: string;
|
||||||
|
action: string;
|
||||||
|
entity_type: string;
|
||||||
|
entity_id: string;
|
||||||
|
old_values?: Record<string, any>;
|
||||||
|
new_values?: Record<string, any>;
|
||||||
|
user_id?: string;
|
||||||
|
ip_address?: string;
|
||||||
|
user_agent?: string;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query Builder Types
|
||||||
|
*/
|
||||||
|
export interface WhereCondition {
|
||||||
|
column: string;
|
||||||
|
operator: '=' | '!=' | '>' | '<' | '>=' | '<=' | 'IN' | 'NOT IN' | 'LIKE' | 'ILIKE' | 'IS NULL' | 'IS NOT NULL';
|
||||||
|
value?: any;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface JoinCondition {
|
||||||
|
type: 'INNER' | 'LEFT' | 'RIGHT' | 'FULL';
|
||||||
|
table: string;
|
||||||
|
on: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface OrderByCondition {
|
||||||
|
column: string;
|
||||||
|
direction: 'ASC' | 'DESC';
|
||||||
|
}
|
||||||
19
libs/postgres-client/tsconfig.json
Normal file
19
libs/postgres-client/tsconfig.json
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true
|
||||||
|
},
|
||||||
|
"include": [
|
||||||
|
"src/**/*"
|
||||||
|
],
|
||||||
|
"exclude": [
|
||||||
|
"dist",
|
||||||
|
"node_modules",
|
||||||
|
"**/*.test.ts",
|
||||||
|
"**/*.spec.ts"
|
||||||
|
]
|
||||||
|
}
|
||||||
102
libs/questdb-client/README.md
Normal file
102
libs/questdb-client/README.md
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
# QuestDB Client Library
|
||||||
|
|
||||||
|
A comprehensive QuestDB client library for the Stock Bot trading platform, optimized for time-series data, market analytics, and high-performance queries.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Time-Series Optimized**: Built specifically for time-series data patterns
|
||||||
|
- **Dual Protocol Support**: HTTP REST API and PostgreSQL wire protocol
|
||||||
|
- **InfluxDB Line Protocol**: High-performance data ingestion
|
||||||
|
- **SQL Analytics**: Full SQL support for complex analytics
|
||||||
|
- **Schema Management**: Automatic table creation and partitioning
|
||||||
|
- **Performance Monitoring**: Query performance tracking and optimization
|
||||||
|
- **Health Monitoring**: Connection health monitoring and metrics
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { QuestDBClient } from '@stock-bot/questdb-client';
|
||||||
|
|
||||||
|
// Initialize client
|
||||||
|
const questClient = new QuestDBClient();
|
||||||
|
await questClient.connect();
|
||||||
|
|
||||||
|
// Insert market data using InfluxDB Line Protocol
|
||||||
|
await questClient.insert('ohlcv', {
|
||||||
|
symbol: 'AAPL',
|
||||||
|
open: 150.00,
|
||||||
|
high: 152.00,
|
||||||
|
low: 149.50,
|
||||||
|
close: 151.50,
|
||||||
|
volume: 1000000,
|
||||||
|
timestamp: new Date()
|
||||||
|
});
|
||||||
|
|
||||||
|
// Query with SQL
|
||||||
|
const prices = await questClient.query(`
|
||||||
|
SELECT symbol, close, timestamp
|
||||||
|
FROM ohlcv
|
||||||
|
WHERE symbol = 'AAPL'
|
||||||
|
AND timestamp > dateadd('d', -1, now())
|
||||||
|
ORDER BY timestamp DESC
|
||||||
|
`);
|
||||||
|
|
||||||
|
// Time-series aggregations
|
||||||
|
const dailyStats = await questClient.aggregate('ohlcv')
|
||||||
|
.select(['symbol', 'avg(close) as avg_price'])
|
||||||
|
.where('symbol = ?', ['AAPL'])
|
||||||
|
.groupBy('symbol')
|
||||||
|
.sampleBy('1d', 'timestamp')
|
||||||
|
.execute();
|
||||||
|
```
|
||||||
|
|
||||||
|
## Data Types
|
||||||
|
|
||||||
|
The client provides typed access to the following time-series data:
|
||||||
|
|
||||||
|
- **ohlcv**: OHLCV candlestick data
|
||||||
|
- **trades**: Individual trade executions
|
||||||
|
- **quotes**: Bid/ask quote data
|
||||||
|
- **indicators**: Technical indicator values
|
||||||
|
- **performance**: Portfolio performance metrics
|
||||||
|
- **risk_metrics**: Risk calculation results
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Configure using environment variables:
|
||||||
|
|
||||||
|
```env
|
||||||
|
QUESTDB_HOST=localhost
|
||||||
|
QUESTDB_HTTP_PORT=9000
|
||||||
|
QUESTDB_PG_PORT=8812
|
||||||
|
QUESTDB_INFLUX_PORT=9009
|
||||||
|
```
|
||||||
|
|
||||||
|
## Time-Series Features
|
||||||
|
|
||||||
|
QuestDB excels at:
|
||||||
|
|
||||||
|
- **High-frequency data**: Millions of data points per second
|
||||||
|
- **Time-based partitioning**: Automatic partitioning by time
|
||||||
|
- **ASOF JOINs**: Time-series specific joins
|
||||||
|
- **SAMPLE BY**: Time-based aggregations
|
||||||
|
- **LATEST BY**: Get latest values by key
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
The client includes performance optimizations:
|
||||||
|
|
||||||
|
- Connection pooling for HTTP and PostgreSQL protocols
|
||||||
|
- Batch insertions for high throughput
|
||||||
|
- Compressed data transfer
|
||||||
|
- Query result caching
|
||||||
|
- Automatic schema optimization
|
||||||
|
|
||||||
|
## Health Monitoring
|
||||||
|
|
||||||
|
Built-in health monitoring:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const health = await questClient.getHealth();
|
||||||
|
console.log(health.status); // 'healthy' | 'degraded' | 'unhealthy'
|
||||||
|
```
|
||||||
52
libs/questdb-client/package.json
Normal file
52
libs/questdb-client/package.json
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
{
|
||||||
|
"name": "@stock-bot/questdb-client",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "QuestDB client library for Stock Bot platform",
|
||||||
|
"main": "src/index.ts",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"test": "jest",
|
||||||
|
"test:watch": "jest --watch",
|
||||||
|
"test:coverage": "jest --coverage",
|
||||||
|
"test:unit": "jest --testPathPattern=src",
|
||||||
|
"test:integration": "jest --testPathPattern=integration",
|
||||||
|
"lint": "eslint src/**/*.ts",
|
||||||
|
"type-check": "tsc --noEmit",
|
||||||
|
"dev": "tsc --watch"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@questdb/nodejs-client": "^3.0.0",
|
||||||
|
"@stock-bot/config": "*",
|
||||||
|
"@stock-bot/logger": "*",
|
||||||
|
"@stock-bot/types": "*",
|
||||||
|
"pg": "^8.11.3",
|
||||||
|
"pg-mem": "^3.0.5",
|
||||||
|
"zod": "^3.22.4"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.11.0",
|
||||||
|
"@types/pg": "^8.10.7",
|
||||||
|
"typescript": "^5.3.0",
|
||||||
|
"eslint": "^8.56.0",
|
||||||
|
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
||||||
|
"@typescript-eslint/parser": "^6.19.0",
|
||||||
|
"bun-types": "^1.2.15",
|
||||||
|
"@types/jest": "^29.5.0",
|
||||||
|
"jest": "^29.7.0",
|
||||||
|
"ts-jest": "^29.1.0"
|
||||||
|
},
|
||||||
|
"keywords": [
|
||||||
|
"questdb",
|
||||||
|
"time-series",
|
||||||
|
"database",
|
||||||
|
"client",
|
||||||
|
"stock-bot"
|
||||||
|
],
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"import": "./src/index.ts",
|
||||||
|
"require": "./dist/index.js"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
472
libs/questdb-client/src/client.ts
Normal file
472
libs/questdb-client/src/client.ts
Normal file
|
|
@ -0,0 +1,472 @@
|
||||||
|
import { Pool } from 'pg';
|
||||||
|
import { questdbConfig } from '@stock-bot/config';
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type {
|
||||||
|
QuestDBClientConfig,
|
||||||
|
QuestDBConnectionOptions,
|
||||||
|
QueryResult,
|
||||||
|
InsertResult,
|
||||||
|
BaseTimeSeriesData,
|
||||||
|
TableNames
|
||||||
|
} from './types';
|
||||||
|
import { QuestDBHealthMonitor } from './health';
|
||||||
|
import { QuestDBQueryBuilder } from './query-builder';
|
||||||
|
import { QuestDBInfluxWriter } from './influx-writer';
|
||||||
|
import { QuestDBSchemaManager } from './schema';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB Client for Stock Bot
|
||||||
|
*
|
||||||
|
* Provides high-performance time-series data access with support for
|
||||||
|
* multiple protocols (HTTP, PostgreSQL, InfluxDB Line Protocol).
|
||||||
|
*/
|
||||||
|
export class QuestDBClient {
|
||||||
|
private pgPool: Pool | null = null;
|
||||||
|
private readonly config: QuestDBClientConfig;
|
||||||
|
private readonly options: QuestDBConnectionOptions;
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private readonly healthMonitor: QuestDBHealthMonitor;
|
||||||
|
private readonly influxWriter: QuestDBInfluxWriter;
|
||||||
|
private readonly schemaManager: QuestDBSchemaManager;
|
||||||
|
private isConnected = false;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
config?: Partial<QuestDBClientConfig>,
|
||||||
|
options?: QuestDBConnectionOptions
|
||||||
|
) {
|
||||||
|
this.config = this.buildConfig(config);
|
||||||
|
this.options = {
|
||||||
|
protocol: 'pg',
|
||||||
|
retryAttempts: 3,
|
||||||
|
retryDelay: 1000,
|
||||||
|
healthCheckInterval: 30000,
|
||||||
|
...options
|
||||||
|
};
|
||||||
|
|
||||||
|
this.logger = new Logger('QuestDBClient');
|
||||||
|
this.healthMonitor = new QuestDBHealthMonitor(this);
|
||||||
|
this.influxWriter = new QuestDBInfluxWriter(this);
|
||||||
|
this.schemaManager = new QuestDBSchemaManager(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to QuestDB
|
||||||
|
*/
|
||||||
|
async connect(): Promise<void> {
|
||||||
|
if (this.isConnected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let lastError: Error | null = null;
|
||||||
|
|
||||||
|
for (let attempt = 1; attempt <= this.options.retryAttempts!; attempt++) {
|
||||||
|
try {
|
||||||
|
this.logger.info(`Connecting to QuestDB (attempt ${attempt}/${this.options.retryAttempts})...`);
|
||||||
|
|
||||||
|
// Connect via PostgreSQL wire protocol
|
||||||
|
this.pgPool = new Pool(this.buildPgPoolConfig());
|
||||||
|
|
||||||
|
// Test the connection
|
||||||
|
const client = await this.pgPool.connect();
|
||||||
|
await client.query('SELECT 1');
|
||||||
|
client.release();
|
||||||
|
|
||||||
|
this.isConnected = true;
|
||||||
|
this.logger.info('Successfully connected to QuestDB');
|
||||||
|
// Initialize schema
|
||||||
|
await this.schemaManager.initializeDatabase();
|
||||||
|
|
||||||
|
// Start health monitoring
|
||||||
|
this.healthMonitor.startMonitoring();
|
||||||
|
|
||||||
|
return;
|
||||||
|
} catch (error) {
|
||||||
|
lastError = error as Error;
|
||||||
|
this.logger.error(`QuestDB connection attempt ${attempt} failed:`, error);
|
||||||
|
|
||||||
|
if (this.pgPool) {
|
||||||
|
await this.pgPool.end();
|
||||||
|
this.pgPool = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (attempt < this.options.retryAttempts!) {
|
||||||
|
await this.delay(this.options.retryDelay! * attempt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(`Failed to connect to QuestDB after ${this.options.retryAttempts} attempts: ${lastError?.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disconnect from QuestDB
|
||||||
|
*/
|
||||||
|
async disconnect(): Promise<void> {
|
||||||
|
if (!this.isConnected) {
|
||||||
|
return;
|
||||||
|
} try {
|
||||||
|
this.healthMonitor.stopMonitoring();
|
||||||
|
|
||||||
|
if (this.pgPool) {
|
||||||
|
await this.pgPool.end();
|
||||||
|
this.pgPool = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.isConnected = false;
|
||||||
|
this.logger.info('Disconnected from QuestDB');
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Error disconnecting from QuestDB:', error);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a SQL query
|
||||||
|
*/
|
||||||
|
async query<T = any>(sql: string, params?: any[]): Promise<QueryResult<T>> {
|
||||||
|
if (!this.pgPool) {
|
||||||
|
throw new Error('QuestDB client not connected');
|
||||||
|
}
|
||||||
|
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await this.pgPool.query(sql, params);
|
||||||
|
const executionTime = Date.now() - startTime;
|
||||||
|
|
||||||
|
this.logger.debug(`Query executed in ${executionTime}ms`, {
|
||||||
|
query: sql.substring(0, 100),
|
||||||
|
rowCount: result.rowCount
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
rows: result.rows,
|
||||||
|
rowCount: result.rowCount || 0,
|
||||||
|
executionTime, metadata: {
|
||||||
|
columns: result.fields?.map((field: any) => ({
|
||||||
|
name: field.name,
|
||||||
|
type: this.mapDataType(field.dataTypeID)
|
||||||
|
})) || []
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const executionTime = Date.now() - startTime;
|
||||||
|
this.logger.error(`Query failed after ${executionTime}ms:`, {
|
||||||
|
error: (error as Error).message,
|
||||||
|
query: sql,
|
||||||
|
params
|
||||||
|
});
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Write OHLCV data using InfluxDB Line Protocol
|
||||||
|
*/
|
||||||
|
async writeOHLCV(
|
||||||
|
symbol: string,
|
||||||
|
exchange: string,
|
||||||
|
data: Array<{
|
||||||
|
timestamp: Date;
|
||||||
|
open: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
volume: number;
|
||||||
|
}>
|
||||||
|
): Promise<void> {
|
||||||
|
return await this.influxWriter.writeOHLCV(symbol, exchange, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write market analytics data
|
||||||
|
*/
|
||||||
|
async writeMarketAnalytics(
|
||||||
|
symbol: string,
|
||||||
|
exchange: string,
|
||||||
|
analytics: {
|
||||||
|
timestamp: Date;
|
||||||
|
rsi?: number;
|
||||||
|
macd?: number;
|
||||||
|
signal?: number;
|
||||||
|
histogram?: number;
|
||||||
|
bollinger_upper?: number;
|
||||||
|
bollinger_lower?: number;
|
||||||
|
volume_sma?: number;
|
||||||
|
}
|
||||||
|
): Promise<void> {
|
||||||
|
return await this.influxWriter.writeMarketAnalytics(symbol, exchange, analytics);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a query builder instance
|
||||||
|
*/
|
||||||
|
queryBuilder(): QuestDBQueryBuilder {
|
||||||
|
return new QuestDBQueryBuilder(this);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Create a SELECT query builder
|
||||||
|
*/
|
||||||
|
select(...columns: string[]): QuestDBQueryBuilder {
|
||||||
|
return this.queryBuilder().select(...columns);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create an aggregation query builder
|
||||||
|
*/
|
||||||
|
aggregate(table: TableNames): QuestDBQueryBuilder {
|
||||||
|
return this.queryBuilder().from(table);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute a time-series specific query with SAMPLE BY
|
||||||
|
*/
|
||||||
|
async sampleBy<T = any>(
|
||||||
|
table: TableNames,
|
||||||
|
columns: string[],
|
||||||
|
interval: string,
|
||||||
|
timeColumn: string = 'timestamp',
|
||||||
|
where?: string,
|
||||||
|
params?: any[]
|
||||||
|
): Promise<QueryResult<T>> {
|
||||||
|
const columnsStr = columns.join(', ');
|
||||||
|
const whereClause = where ? `WHERE ${where}` : '';
|
||||||
|
|
||||||
|
const sql = `
|
||||||
|
SELECT ${columnsStr}
|
||||||
|
FROM ${table}
|
||||||
|
${whereClause}
|
||||||
|
SAMPLE BY ${interval}
|
||||||
|
ALIGN TO CALENDAR
|
||||||
|
`;
|
||||||
|
|
||||||
|
return await this.query<T>(sql, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get latest values by symbol using LATEST BY
|
||||||
|
*/
|
||||||
|
async latestBy<T = any>(
|
||||||
|
table: TableNames,
|
||||||
|
columns: string | string[] = '*',
|
||||||
|
keyColumns: string | string[] = 'symbol'
|
||||||
|
): Promise<QueryResult<T>> {
|
||||||
|
const columnsStr = Array.isArray(columns) ? columns.join(', ') : columns;
|
||||||
|
const keyColumnsStr = Array.isArray(keyColumns) ? keyColumns.join(', ') : keyColumns;
|
||||||
|
|
||||||
|
const sql = `
|
||||||
|
SELECT ${columnsStr}
|
||||||
|
FROM ${table}
|
||||||
|
LATEST BY ${keyColumnsStr}
|
||||||
|
`;
|
||||||
|
|
||||||
|
return await this.query<T>(sql);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute ASOF JOIN for time-series correlation
|
||||||
|
*/
|
||||||
|
async asofJoin<T = any>(
|
||||||
|
leftTable: TableNames,
|
||||||
|
rightTable: TableNames,
|
||||||
|
joinCondition: string,
|
||||||
|
columns?: string[],
|
||||||
|
where?: string,
|
||||||
|
params?: any[]
|
||||||
|
): Promise<QueryResult<T>> {
|
||||||
|
const columnsStr = columns ? columns.join(', ') : '*';
|
||||||
|
const whereClause = where ? `WHERE ${where}` : '';
|
||||||
|
|
||||||
|
const sql = `
|
||||||
|
SELECT ${columnsStr}
|
||||||
|
FROM ${leftTable}
|
||||||
|
ASOF JOIN ${rightTable} ON ${joinCondition}
|
||||||
|
${whereClause}
|
||||||
|
`;
|
||||||
|
|
||||||
|
return await this.query<T>(sql, params);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get database statistics
|
||||||
|
*/
|
||||||
|
async getStats(): Promise<any> {
|
||||||
|
const result = await this.query(`
|
||||||
|
SELECT
|
||||||
|
table_name,
|
||||||
|
row_count,
|
||||||
|
partition_count,
|
||||||
|
size_bytes
|
||||||
|
FROM tables()
|
||||||
|
WHERE table_name NOT LIKE 'sys.%'
|
||||||
|
ORDER BY row_count DESC
|
||||||
|
`);
|
||||||
|
return result.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get table information
|
||||||
|
*/
|
||||||
|
async getTableInfo(tableName: string): Promise<any> {
|
||||||
|
const result = await this.query(
|
||||||
|
`SELECT * FROM table_columns WHERE table_name = ?`,
|
||||||
|
[tableName]
|
||||||
|
);
|
||||||
|
return result.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if PostgreSQL pool is healthy
|
||||||
|
*/
|
||||||
|
isPgPoolHealthy(): boolean {
|
||||||
|
return this.pgPool !== null && !this.pgPool.ended;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get HTTP endpoint URL
|
||||||
|
*/
|
||||||
|
getHttpUrl(): string {
|
||||||
|
const protocol = this.config.tls?.enabled ? 'https' : 'http';
|
||||||
|
return `${protocol}://${this.config.host}:${this.config.httpPort}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get InfluxDB endpoint URL
|
||||||
|
*/
|
||||||
|
getInfluxUrl(): string {
|
||||||
|
const protocol = this.config.tls?.enabled ? 'https' : 'http';
|
||||||
|
return `${protocol}://${this.config.host}:${this.config.influxPort}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get health monitor instance
|
||||||
|
*/
|
||||||
|
getHealthMonitor(): QuestDBHealthMonitor {
|
||||||
|
return this.healthMonitor;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get schema manager instance
|
||||||
|
*/
|
||||||
|
getSchemaManager(): QuestDBSchemaManager {
|
||||||
|
return this.schemaManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get InfluxDB writer instance
|
||||||
|
*/
|
||||||
|
getInfluxWriter(): QuestDBInfluxWriter {
|
||||||
|
return this.influxWriter;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optimize table by rebuilding partitions
|
||||||
|
*/
|
||||||
|
async optimizeTable(tableName: string): Promise<void> {
|
||||||
|
await this.query(`VACUUM TABLE ${tableName}`);
|
||||||
|
this.logger.info(`Optimized table: ${tableName}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a table with time-series optimizations
|
||||||
|
*/
|
||||||
|
async createTable(
|
||||||
|
tableName: string,
|
||||||
|
columns: string,
|
||||||
|
partitionBy: string = 'DAY',
|
||||||
|
timestampColumn: string = 'timestamp'
|
||||||
|
): Promise<void> {
|
||||||
|
const sql = `
|
||||||
|
CREATE TABLE IF NOT EXISTS ${tableName} (
|
||||||
|
${columns}
|
||||||
|
) TIMESTAMP(${timestampColumn}) PARTITION BY ${partitionBy}
|
||||||
|
`;
|
||||||
|
|
||||||
|
await this.query(sql);
|
||||||
|
this.logger.info(`Created table: ${tableName}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if client is connected
|
||||||
|
*/
|
||||||
|
get connected(): boolean {
|
||||||
|
return this.isConnected && !!this.pgPool;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the PostgreSQL connection pool
|
||||||
|
*/
|
||||||
|
get connectionPool(): Pool | null {
|
||||||
|
return this.pgPool;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get configuration
|
||||||
|
*/
|
||||||
|
get configuration(): QuestDBClientConfig {
|
||||||
|
return { ...this.config };
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildConfig(config?: Partial<QuestDBClientConfig>): QuestDBClientConfig {
|
||||||
|
return {
|
||||||
|
host: config?.host || questdbConfig.QUESTDB_HOST,
|
||||||
|
httpPort: config?.httpPort || questdbConfig.QUESTDB_HTTP_PORT,
|
||||||
|
pgPort: config?.pgPort || questdbConfig.QUESTDB_PG_PORT,
|
||||||
|
influxPort: config?.influxPort || questdbConfig.QUESTDB_INFLUX_PORT,
|
||||||
|
user: config?.user || questdbConfig.QUESTDB_USER,
|
||||||
|
password: config?.password || questdbConfig.QUESTDB_PASSWORD,
|
||||||
|
database: config?.database || questdbConfig.QUESTDB_DEFAULT_DATABASE,
|
||||||
|
tls: {
|
||||||
|
enabled: questdbConfig.QUESTDB_TLS_ENABLED,
|
||||||
|
verifyServerCert: questdbConfig.QUESTDB_TLS_VERIFY_SERVER_CERT,
|
||||||
|
...config?.tls
|
||||||
|
},
|
||||||
|
timeouts: {
|
||||||
|
connection: questdbConfig.QUESTDB_CONNECTION_TIMEOUT,
|
||||||
|
request: questdbConfig.QUESTDB_REQUEST_TIMEOUT,
|
||||||
|
...config?.timeouts
|
||||||
|
},
|
||||||
|
retryAttempts: questdbConfig.QUESTDB_RETRY_ATTEMPTS,
|
||||||
|
...config
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildPgPoolConfig(): any {
|
||||||
|
return {
|
||||||
|
host: this.config.host,
|
||||||
|
port: this.config.pgPort,
|
||||||
|
database: this.config.database,
|
||||||
|
user: this.config.user,
|
||||||
|
password: this.config.password,
|
||||||
|
connectionTimeoutMillis: this.config.timeouts?.connection,
|
||||||
|
query_timeout: this.config.timeouts?.request,
|
||||||
|
ssl: this.config.tls?.enabled ? {
|
||||||
|
rejectUnauthorized: this.config.tls.verifyServerCert
|
||||||
|
} : false,
|
||||||
|
min: 2,
|
||||||
|
max: 10
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private mapDataType(typeId: number): string {
|
||||||
|
// Map PostgreSQL type IDs to QuestDB types
|
||||||
|
const typeMap: Record<number, string> = {
|
||||||
|
16: 'BOOLEAN',
|
||||||
|
20: 'LONG',
|
||||||
|
21: 'INT',
|
||||||
|
23: 'INT',
|
||||||
|
25: 'STRING',
|
||||||
|
700: 'FLOAT',
|
||||||
|
701: 'DOUBLE',
|
||||||
|
1043: 'STRING',
|
||||||
|
1082: 'DATE',
|
||||||
|
1114: 'TIMESTAMP',
|
||||||
|
1184: 'TIMESTAMP'
|
||||||
|
};
|
||||||
|
|
||||||
|
return typeMap[typeId] || 'STRING';
|
||||||
|
}
|
||||||
|
|
||||||
|
private delay(ms: number): Promise<void> {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
}
|
||||||
63
libs/questdb-client/src/factory.ts
Normal file
63
libs/questdb-client/src/factory.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
import { QuestDBClient } from './client';
|
||||||
|
import { questdbConfig } from '@stock-bot/config';
|
||||||
|
import type { QuestDBClientConfig, QuestDBConnectionOptions } from './types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Factory function to create a QuestDB client instance
|
||||||
|
*/
|
||||||
|
export function createQuestDBClient(
|
||||||
|
config?: Partial<QuestDBClientConfig>,
|
||||||
|
options?: QuestDBConnectionOptions
|
||||||
|
): QuestDBClient {
|
||||||
|
return new QuestDBClient(config, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a QuestDB client with default configuration
|
||||||
|
*/
|
||||||
|
export function createDefaultQuestDBClient(): QuestDBClient {
|
||||||
|
const config: Partial<QuestDBClientConfig> = {
|
||||||
|
host: questdbConfig.QUESTDB_HOST,
|
||||||
|
httpPort: questdbConfig.QUESTDB_HTTP_PORT,
|
||||||
|
pgPort: questdbConfig.QUESTDB_PG_PORT,
|
||||||
|
influxPort: questdbConfig.QUESTDB_INFLUX_PORT,
|
||||||
|
user: questdbConfig.QUESTDB_USER,
|
||||||
|
password: questdbConfig.QUESTDB_PASSWORD
|
||||||
|
};
|
||||||
|
|
||||||
|
return new QuestDBClient(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Singleton QuestDB client instance
|
||||||
|
*/
|
||||||
|
let defaultClient: QuestDBClient | null = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get or create the default QuestDB client instance
|
||||||
|
*/
|
||||||
|
export function getQuestDBClient(): QuestDBClient {
|
||||||
|
if (!defaultClient) {
|
||||||
|
defaultClient = createDefaultQuestDBClient();
|
||||||
|
}
|
||||||
|
return defaultClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to QuestDB using the default client
|
||||||
|
*/
|
||||||
|
export async function connectQuestDB(): Promise<QuestDBClient> {
|
||||||
|
const client = getQuestDBClient();
|
||||||
|
await client.connect();
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Disconnect from QuestDB
|
||||||
|
*/
|
||||||
|
export async function disconnectQuestDB(): Promise<void> {
|
||||||
|
if (defaultClient) {
|
||||||
|
await defaultClient.disconnect();
|
||||||
|
defaultClient = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
233
libs/questdb-client/src/health.ts
Normal file
233
libs/questdb-client/src/health.ts
Normal file
|
|
@ -0,0 +1,233 @@
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type { HealthStatus, PerformanceMetrics, QueryResult } from './types';
|
||||||
|
|
||||||
|
// Interface to avoid circular dependency
|
||||||
|
interface QuestDBClientInterface {
|
||||||
|
query<T = any>(sql: string, params?: any[]): Promise<QueryResult<T>>;
|
||||||
|
isPgPoolHealthy(): boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB Health Monitor
|
||||||
|
*
|
||||||
|
* Monitors connection health, performance metrics, and provides
|
||||||
|
* automatic recovery capabilities for the QuestDB client.
|
||||||
|
*/
|
||||||
|
export class QuestDBHealthMonitor {
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private healthCheckInterval: NodeJS.Timeout | null = null;
|
||||||
|
private lastHealthCheck: Date | null = null;
|
||||||
|
private performanceMetrics: PerformanceMetrics = {
|
||||||
|
totalQueries: 0,
|
||||||
|
successfulQueries: 0,
|
||||||
|
failedQueries: 0,
|
||||||
|
averageResponseTime: 0,
|
||||||
|
lastQueryTime: null,
|
||||||
|
connectionUptime: 0,
|
||||||
|
memoryUsage: 0
|
||||||
|
};
|
||||||
|
constructor(private readonly client: QuestDBClientInterface) {
|
||||||
|
this.logger = new Logger('QuestDBHealthMonitor');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start health monitoring
|
||||||
|
*/
|
||||||
|
public startMonitoring(intervalMs: number = 30000): void {
|
||||||
|
if (this.healthCheckInterval) {
|
||||||
|
this.stopMonitoring();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info(`Starting health monitoring with ${intervalMs}ms interval`);
|
||||||
|
|
||||||
|
this.healthCheckInterval = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
await this.performHealthCheck();
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Health check failed', { error });
|
||||||
|
}
|
||||||
|
}, intervalMs);
|
||||||
|
|
||||||
|
// Perform initial health check
|
||||||
|
this.performHealthCheck().catch(error => {
|
||||||
|
this.logger.error('Initial health check failed', { error });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop health monitoring
|
||||||
|
*/
|
||||||
|
public stopMonitoring(): void {
|
||||||
|
if (this.healthCheckInterval) {
|
||||||
|
clearInterval(this.healthCheckInterval);
|
||||||
|
this.healthCheckInterval = null;
|
||||||
|
this.logger.info('Health monitoring stopped');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Perform a health check
|
||||||
|
*/
|
||||||
|
public async performHealthCheck(): Promise<HealthStatus> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Test basic connectivity with a simple query
|
||||||
|
await this.client.query('SELECT 1 as health_check');
|
||||||
|
|
||||||
|
const responseTime = Date.now() - startTime;
|
||||||
|
this.lastHealthCheck = new Date();
|
||||||
|
|
||||||
|
const status: HealthStatus = {
|
||||||
|
isHealthy: true,
|
||||||
|
lastCheck: this.lastHealthCheck,
|
||||||
|
responseTime,
|
||||||
|
message: 'Connection healthy',
|
||||||
|
details: {
|
||||||
|
pgPool: this.client.isPgPoolHealthy(),
|
||||||
|
httpEndpoint: true, // Will be implemented when HTTP client is added
|
||||||
|
uptime: this.getUptime()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
this.logger.debug('Health check passed', { responseTime });
|
||||||
|
return status;
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
const responseTime = Date.now() - startTime;
|
||||||
|
this.lastHealthCheck = new Date();
|
||||||
|
|
||||||
|
const status: HealthStatus = {
|
||||||
|
isHealthy: false,
|
||||||
|
lastCheck: this.lastHealthCheck,
|
||||||
|
responseTime,
|
||||||
|
message: `Health check failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||||
|
error: error instanceof Error ? error : new Error('Unknown error'),
|
||||||
|
details: {
|
||||||
|
pgPool: false,
|
||||||
|
httpEndpoint: false,
|
||||||
|
uptime: this.getUptime()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
this.logger.error('Health check failed', { error, responseTime });
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current health status
|
||||||
|
*/
|
||||||
|
public async getHealthStatus(): Promise<HealthStatus> {
|
||||||
|
if (!this.lastHealthCheck || Date.now() - this.lastHealthCheck.getTime() > 60000) {
|
||||||
|
return await this.performHealthCheck();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return cached status if recent
|
||||||
|
return {
|
||||||
|
isHealthy: true,
|
||||||
|
lastCheck: this.lastHealthCheck,
|
||||||
|
responseTime: 0,
|
||||||
|
message: 'Using cached health status',
|
||||||
|
details: {
|
||||||
|
pgPool: this.client.isPgPoolHealthy(),
|
||||||
|
httpEndpoint: true,
|
||||||
|
uptime: this.getUptime()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record query performance metrics
|
||||||
|
*/
|
||||||
|
public recordQuery(success: boolean, responseTime: number): void {
|
||||||
|
this.performanceMetrics.totalQueries++;
|
||||||
|
this.performanceMetrics.lastQueryTime = new Date();
|
||||||
|
|
||||||
|
if (success) {
|
||||||
|
this.performanceMetrics.successfulQueries++;
|
||||||
|
} else {
|
||||||
|
this.performanceMetrics.failedQueries++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update rolling average response time
|
||||||
|
const totalResponseTime = this.performanceMetrics.averageResponseTime *
|
||||||
|
(this.performanceMetrics.totalQueries - 1) + responseTime;
|
||||||
|
this.performanceMetrics.averageResponseTime =
|
||||||
|
totalResponseTime / this.performanceMetrics.totalQueries;
|
||||||
|
|
||||||
|
// Update memory usage
|
||||||
|
this.performanceMetrics.memoryUsage = process.memoryUsage().heapUsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get performance metrics
|
||||||
|
*/
|
||||||
|
public getPerformanceMetrics(): PerformanceMetrics {
|
||||||
|
return { ...this.performanceMetrics };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get connection uptime in seconds
|
||||||
|
*/
|
||||||
|
private getUptime(): number {
|
||||||
|
return Math.floor(process.uptime());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset performance metrics
|
||||||
|
*/
|
||||||
|
public resetMetrics(): void {
|
||||||
|
this.performanceMetrics = {
|
||||||
|
totalQueries: 0,
|
||||||
|
successfulQueries: 0,
|
||||||
|
failedQueries: 0,
|
||||||
|
averageResponseTime: 0,
|
||||||
|
lastQueryTime: null,
|
||||||
|
connectionUptime: this.getUptime(),
|
||||||
|
memoryUsage: process.memoryUsage().heapUsed
|
||||||
|
};
|
||||||
|
|
||||||
|
this.logger.info('Performance metrics reset');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get health summary for monitoring dashboards
|
||||||
|
*/
|
||||||
|
public async getHealthSummary(): Promise<{
|
||||||
|
status: HealthStatus;
|
||||||
|
metrics: PerformanceMetrics;
|
||||||
|
recommendations: string[];
|
||||||
|
}> {
|
||||||
|
const status = await this.getHealthStatus();
|
||||||
|
const metrics = this.getPerformanceMetrics();
|
||||||
|
const recommendations: string[] = [];
|
||||||
|
|
||||||
|
// Generate recommendations based on metrics
|
||||||
|
if (metrics.failedQueries > metrics.successfulQueries * 0.1) {
|
||||||
|
recommendations.push('High error rate detected - check query patterns');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metrics.averageResponseTime > 1000) {
|
||||||
|
recommendations.push('High response times - consider query optimization');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (metrics.memoryUsage > 100 * 1024 * 1024) { // 100MB
|
||||||
|
recommendations.push('High memory usage - monitor for memory leaks');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
metrics,
|
||||||
|
recommendations
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cleanup resources
|
||||||
|
*/
|
||||||
|
public destroy(): void {
|
||||||
|
this.stopMonitoring();
|
||||||
|
this.logger.info('Health monitor destroyed');
|
||||||
|
}
|
||||||
|
}
|
||||||
32
libs/questdb-client/src/index.ts
Normal file
32
libs/questdb-client/src/index.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
/**
|
||||||
|
* QuestDB Client Library for Stock Bot
|
||||||
|
*
|
||||||
|
* Provides high-performance time-series data access with support for
|
||||||
|
* InfluxDB Line Protocol, SQL queries, and PostgreSQL wire protocol.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export { QuestDBClient } from './client';
|
||||||
|
export { QuestDBHealthMonitor } from './health';
|
||||||
|
export { QuestDBQueryBuilder } from './query-builder';
|
||||||
|
export { QuestDBInfluxWriter } from './influx-writer';
|
||||||
|
export { QuestDBSchemaManager } from './schema';
|
||||||
|
|
||||||
|
// Types
|
||||||
|
export type {
|
||||||
|
QuestDBClientConfig,
|
||||||
|
QuestDBConnectionOptions,
|
||||||
|
QuestDBHealthStatus,
|
||||||
|
QuestDBMetrics,
|
||||||
|
TableNames,
|
||||||
|
OHLCVData,
|
||||||
|
TradeData,
|
||||||
|
QuoteData,
|
||||||
|
IndicatorData,
|
||||||
|
PerformanceData,
|
||||||
|
RiskMetrics,
|
||||||
|
QueryResult,
|
||||||
|
InsertResult
|
||||||
|
} from './types';
|
||||||
|
|
||||||
|
// Utils
|
||||||
|
export { createQuestDBClient, getQuestDBClient } from './factory';
|
||||||
436
libs/questdb-client/src/influx-writer.ts
Normal file
436
libs/questdb-client/src/influx-writer.ts
Normal file
|
|
@ -0,0 +1,436 @@
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type {
|
||||||
|
InfluxLineData,
|
||||||
|
InfluxWriteOptions,
|
||||||
|
BaseTimeSeriesData
|
||||||
|
} from './types';
|
||||||
|
|
||||||
|
// Interface to avoid circular dependency
|
||||||
|
interface QuestDBClientInterface {
|
||||||
|
getHttpUrl(): string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB InfluxDB Line Protocol Writer
|
||||||
|
*
|
||||||
|
* Provides high-performance data ingestion using InfluxDB Line Protocol
|
||||||
|
* which QuestDB supports natively for optimal time-series data insertion.
|
||||||
|
*/
|
||||||
|
export class QuestDBInfluxWriter {
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private writeBuffer: string[] = [];
|
||||||
|
private flushTimer: NodeJS.Timeout | null = null;
|
||||||
|
private readonly defaultOptions: Required<InfluxWriteOptions> = {
|
||||||
|
batchSize: 1000,
|
||||||
|
flushInterval: 5000,
|
||||||
|
autoFlush: true,
|
||||||
|
precision: 'ms',
|
||||||
|
retryAttempts: 3,
|
||||||
|
retryDelay: 1000
|
||||||
|
};
|
||||||
|
constructor(private readonly client: QuestDBClientInterface) {
|
||||||
|
this.logger = new Logger('QuestDBInfluxWriter');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write single data point using InfluxDB Line Protocol
|
||||||
|
*/
|
||||||
|
public async writePoint(
|
||||||
|
measurement: string,
|
||||||
|
tags: Record<string, string>,
|
||||||
|
fields: Record<string, number | string | boolean>,
|
||||||
|
timestamp?: Date,
|
||||||
|
options?: Partial<InfluxWriteOptions>
|
||||||
|
): Promise<void> {
|
||||||
|
const line = this.buildLineProtocol(measurement, tags, fields, timestamp);
|
||||||
|
const opts = { ...this.defaultOptions, ...options };
|
||||||
|
|
||||||
|
if (opts.autoFlush && this.writeBuffer.length === 0) {
|
||||||
|
// Single point write - send immediately
|
||||||
|
await this.sendLines([line], opts);
|
||||||
|
} else {
|
||||||
|
// Add to buffer
|
||||||
|
this.writeBuffer.push(line);
|
||||||
|
|
||||||
|
if (opts.autoFlush) {
|
||||||
|
this.scheduleFlush(opts);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush if buffer is full
|
||||||
|
if (this.writeBuffer.length >= opts.batchSize) {
|
||||||
|
await this.flush(opts);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write multiple data points
|
||||||
|
*/
|
||||||
|
public async writePoints(
|
||||||
|
data: InfluxLineData[],
|
||||||
|
options?: Partial<InfluxWriteOptions>
|
||||||
|
): Promise<void> {
|
||||||
|
const opts = { ...this.defaultOptions, ...options };
|
||||||
|
const lines = data.map(point =>
|
||||||
|
this.buildLineProtocol(point.measurement, point.tags, point.fields, point.timestamp)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (opts.autoFlush) {
|
||||||
|
// Send immediately for batch writes
|
||||||
|
await this.sendLines(lines, opts);
|
||||||
|
} else {
|
||||||
|
// Add to buffer
|
||||||
|
this.writeBuffer.push(...lines);
|
||||||
|
|
||||||
|
// Flush if buffer exceeds batch size
|
||||||
|
while (this.writeBuffer.length >= opts.batchSize) {
|
||||||
|
const batch = this.writeBuffer.splice(0, opts.batchSize);
|
||||||
|
await this.sendLines(batch, opts);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write OHLCV data optimized for QuestDB
|
||||||
|
*/
|
||||||
|
public async writeOHLCV(
|
||||||
|
symbol: string,
|
||||||
|
exchange: string,
|
||||||
|
data: {
|
||||||
|
timestamp: Date;
|
||||||
|
open: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
volume: number;
|
||||||
|
}[],
|
||||||
|
options?: Partial<InfluxWriteOptions>
|
||||||
|
): Promise<void> {
|
||||||
|
const influxData: InfluxLineData[] = data.map(candle => ({
|
||||||
|
measurement: 'ohlcv_data',
|
||||||
|
tags: {
|
||||||
|
symbol,
|
||||||
|
exchange,
|
||||||
|
data_source: 'market_feed'
|
||||||
|
},
|
||||||
|
fields: {
|
||||||
|
open: candle.open,
|
||||||
|
high: candle.high,
|
||||||
|
low: candle.low,
|
||||||
|
close: candle.close,
|
||||||
|
volume: candle.volume
|
||||||
|
},
|
||||||
|
timestamp: candle.timestamp
|
||||||
|
}));
|
||||||
|
|
||||||
|
await this.writePoints(influxData, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write market analytics data
|
||||||
|
*/
|
||||||
|
public async writeMarketAnalytics(
|
||||||
|
symbol: string,
|
||||||
|
exchange: string,
|
||||||
|
analytics: {
|
||||||
|
timestamp: Date;
|
||||||
|
rsi?: number;
|
||||||
|
macd?: number;
|
||||||
|
signal?: number;
|
||||||
|
histogram?: number;
|
||||||
|
bollinger_upper?: number;
|
||||||
|
bollinger_lower?: number;
|
||||||
|
volume_sma?: number;
|
||||||
|
},
|
||||||
|
options?: Partial<InfluxWriteOptions>
|
||||||
|
): Promise<void> {
|
||||||
|
const fields: Record<string, number> = {};
|
||||||
|
|
||||||
|
// Only include defined values
|
||||||
|
Object.entries(analytics).forEach(([key, value]) => {
|
||||||
|
if (key !== 'timestamp' && value !== undefined && value !== null) {
|
||||||
|
fields[key] = value as number;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (Object.keys(fields).length === 0) {
|
||||||
|
this.logger.warn('No analytics fields to write', { symbol, timestamp: analytics.timestamp });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.writePoint(
|
||||||
|
'market_analytics',
|
||||||
|
{ symbol, exchange },
|
||||||
|
fields,
|
||||||
|
analytics.timestamp,
|
||||||
|
options
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write trade execution data
|
||||||
|
*/
|
||||||
|
public async writeTradeExecution(
|
||||||
|
execution: {
|
||||||
|
symbol: string;
|
||||||
|
side: 'buy' | 'sell';
|
||||||
|
quantity: number;
|
||||||
|
price: number;
|
||||||
|
timestamp: Date;
|
||||||
|
executionTime: number;
|
||||||
|
orderId?: string;
|
||||||
|
strategy?: string;
|
||||||
|
},
|
||||||
|
options?: Partial<InfluxWriteOptions>
|
||||||
|
): Promise<void> {
|
||||||
|
const tags: Record<string, string> = {
|
||||||
|
symbol: execution.symbol,
|
||||||
|
side: execution.side
|
||||||
|
};
|
||||||
|
|
||||||
|
if (execution.orderId) {
|
||||||
|
tags.order_id = execution.orderId;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (execution.strategy) {
|
||||||
|
tags.strategy = execution.strategy;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.writePoint(
|
||||||
|
'trade_executions',
|
||||||
|
tags,
|
||||||
|
{
|
||||||
|
quantity: execution.quantity,
|
||||||
|
price: execution.price,
|
||||||
|
execution_time: execution.executionTime
|
||||||
|
},
|
||||||
|
execution.timestamp,
|
||||||
|
options
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write performance metrics
|
||||||
|
*/
|
||||||
|
public async writePerformanceMetrics(
|
||||||
|
metrics: {
|
||||||
|
timestamp: Date;
|
||||||
|
operation: string;
|
||||||
|
responseTime: number;
|
||||||
|
success: boolean;
|
||||||
|
errorCode?: string;
|
||||||
|
},
|
||||||
|
options?: Partial<InfluxWriteOptions>
|
||||||
|
): Promise<void> {
|
||||||
|
const tags: Record<string, string> = {
|
||||||
|
operation: metrics.operation,
|
||||||
|
success: metrics.success.toString()
|
||||||
|
};
|
||||||
|
|
||||||
|
if (metrics.errorCode) {
|
||||||
|
tags.error_code = metrics.errorCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.writePoint(
|
||||||
|
'performance_metrics',
|
||||||
|
tags,
|
||||||
|
{ response_time: metrics.responseTime },
|
||||||
|
metrics.timestamp,
|
||||||
|
options
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manually flush the write buffer
|
||||||
|
*/
|
||||||
|
public async flush(options?: Partial<InfluxWriteOptions>): Promise<void> {
|
||||||
|
if (this.writeBuffer.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const opts = { ...this.defaultOptions, ...options };
|
||||||
|
const lines = this.writeBuffer.splice(0); // Clear buffer
|
||||||
|
|
||||||
|
if (this.flushTimer) {
|
||||||
|
clearTimeout(this.flushTimer);
|
||||||
|
this.flushTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.sendLines(lines, opts);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get current buffer size
|
||||||
|
*/
|
||||||
|
public getBufferSize(): number {
|
||||||
|
return this.writeBuffer.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clear the buffer without writing
|
||||||
|
*/
|
||||||
|
public clearBuffer(): void {
|
||||||
|
this.writeBuffer.length = 0;
|
||||||
|
if (this.flushTimer) {
|
||||||
|
clearTimeout(this.flushTimer);
|
||||||
|
this.flushTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build InfluxDB Line Protocol string
|
||||||
|
*/
|
||||||
|
private buildLineProtocol(
|
||||||
|
measurement: string,
|
||||||
|
tags: Record<string, string>,
|
||||||
|
fields: Record<string, number | string | boolean>,
|
||||||
|
timestamp?: Date
|
||||||
|
): string {
|
||||||
|
// Escape special characters in measurement name
|
||||||
|
const escapedMeasurement = measurement.replace(/[, =]/g, '\\$&');
|
||||||
|
|
||||||
|
// Build tags string
|
||||||
|
const tagString = Object.entries(tags)
|
||||||
|
.filter(([_, value]) => value !== undefined && value !== null)
|
||||||
|
.map(([key, value]) => `${this.escapeTagKey(key)}=${this.escapeTagValue(value)}`)
|
||||||
|
.join(',');
|
||||||
|
|
||||||
|
// Build fields string
|
||||||
|
const fieldString = Object.entries(fields)
|
||||||
|
.filter(([_, value]) => value !== undefined && value !== null)
|
||||||
|
.map(([key, value]) => `${this.escapeFieldKey(key)}=${this.formatFieldValue(value)}`)
|
||||||
|
.join(',');
|
||||||
|
|
||||||
|
// Build timestamp
|
||||||
|
const timestampString = timestamp ?
|
||||||
|
Math.floor(timestamp.getTime() * 1000000).toString() : // Convert to nanoseconds
|
||||||
|
'';
|
||||||
|
|
||||||
|
// Combine parts
|
||||||
|
let line = escapedMeasurement;
|
||||||
|
if (tagString) {
|
||||||
|
line += `,${tagString}`;
|
||||||
|
}
|
||||||
|
line += ` ${fieldString}`;
|
||||||
|
if (timestampString) {
|
||||||
|
line += ` ${timestampString}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send lines to QuestDB via HTTP endpoint
|
||||||
|
*/
|
||||||
|
private async sendLines(
|
||||||
|
lines: string[],
|
||||||
|
options: Required<InfluxWriteOptions>
|
||||||
|
): Promise<void> {
|
||||||
|
if (lines.length === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload = lines.join('\n');
|
||||||
|
let attempt = 0;
|
||||||
|
|
||||||
|
while (attempt <= options.retryAttempts) {
|
||||||
|
try {
|
||||||
|
// QuestDB InfluxDB Line Protocol endpoint
|
||||||
|
const response = await fetch(`${this.client.getHttpUrl()}/write`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'text/plain',
|
||||||
|
},
|
||||||
|
body: payload
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.debug(`Successfully wrote ${lines.length} lines to QuestDB`);
|
||||||
|
return;
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
attempt++;
|
||||||
|
this.logger.error(`Write attempt ${attempt} failed`, {
|
||||||
|
error,
|
||||||
|
linesCount: lines.length,
|
||||||
|
willRetry: attempt <= options.retryAttempts
|
||||||
|
});
|
||||||
|
|
||||||
|
if (attempt <= options.retryAttempts) {
|
||||||
|
await this.sleep(options.retryDelay * attempt); // Exponential backoff
|
||||||
|
} else {
|
||||||
|
throw new Error(`Failed to write to QuestDB after ${options.retryAttempts} attempts: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schedule automatic flush
|
||||||
|
*/
|
||||||
|
private scheduleFlush(options: Required<InfluxWriteOptions>): void {
|
||||||
|
if (this.flushTimer || !options.autoFlush) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.flushTimer = setTimeout(async () => {
|
||||||
|
try {
|
||||||
|
await this.flush(options);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Scheduled flush failed', { error });
|
||||||
|
}
|
||||||
|
}, options.flushInterval);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format field value for InfluxDB Line Protocol
|
||||||
|
*/
|
||||||
|
private formatFieldValue(value: number | string | boolean): string {
|
||||||
|
if (typeof value === 'string') {
|
||||||
|
return `"${value.replace(/"/g, '\\"')}"`;
|
||||||
|
} else if (typeof value === 'boolean') {
|
||||||
|
return value ? 'true' : 'false';
|
||||||
|
} else {
|
||||||
|
return value.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Escape tag key
|
||||||
|
*/
|
||||||
|
private escapeTagKey(key: string): string {
|
||||||
|
return key.replace(/[, =]/g, '\\$&');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Escape tag value
|
||||||
|
*/
|
||||||
|
private escapeTagValue(value: string): string {
|
||||||
|
return value.replace(/[, =]/g, '\\$&');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Escape field key
|
||||||
|
*/
|
||||||
|
private escapeFieldKey(key: string): string {
|
||||||
|
return key.replace(/[, =]/g, '\\$&');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sleep utility
|
||||||
|
*/
|
||||||
|
private sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise(resolve => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cleanup resources
|
||||||
|
*/
|
||||||
|
public destroy(): void {
|
||||||
|
this.clearBuffer();
|
||||||
|
this.logger.info('InfluxDB writer destroyed');
|
||||||
|
}
|
||||||
|
}
|
||||||
368
libs/questdb-client/src/query-builder.ts
Normal file
368
libs/questdb-client/src/query-builder.ts
Normal file
|
|
@ -0,0 +1,368 @@
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type {
|
||||||
|
QueryResult,
|
||||||
|
TimeSeriesQuery,
|
||||||
|
AggregationQuery,
|
||||||
|
TimeRange,
|
||||||
|
TableNames
|
||||||
|
} from './types';
|
||||||
|
|
||||||
|
// Interface to avoid circular dependency
|
||||||
|
interface QuestDBClientInterface {
|
||||||
|
query<T = any>(sql: string, params?: any[]): Promise<QueryResult<T>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB Query Builder
|
||||||
|
*
|
||||||
|
* Provides a fluent interface for building optimized time-series queries
|
||||||
|
* with support for QuestDB-specific functions and optimizations.
|
||||||
|
*/
|
||||||
|
export class QuestDBQueryBuilder {
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private query!: {
|
||||||
|
select: string[];
|
||||||
|
from: string;
|
||||||
|
where: string[];
|
||||||
|
groupBy: string[];
|
||||||
|
orderBy: string[];
|
||||||
|
limit?: number;
|
||||||
|
sampleBy?: string;
|
||||||
|
latestBy?: string[];
|
||||||
|
timeRange?: TimeRange;
|
||||||
|
};
|
||||||
|
constructor(private readonly client: QuestDBClientInterface) {
|
||||||
|
this.logger = new Logger('QuestDBQueryBuilder');
|
||||||
|
this.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset the query builder
|
||||||
|
*/
|
||||||
|
private reset(): QuestDBQueryBuilder {
|
||||||
|
this.query = {
|
||||||
|
select: [],
|
||||||
|
from: '',
|
||||||
|
where: [],
|
||||||
|
groupBy: [],
|
||||||
|
orderBy: [],
|
||||||
|
sampleBy: undefined,
|
||||||
|
latestBy: undefined,
|
||||||
|
timeRange: undefined
|
||||||
|
};
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Start a new query
|
||||||
|
*/
|
||||||
|
public static create(client: QuestDBClientInterface): QuestDBQueryBuilder {
|
||||||
|
return new QuestDBQueryBuilder(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Select columns
|
||||||
|
*/
|
||||||
|
public select(...columns: string[]): QuestDBQueryBuilder {
|
||||||
|
this.query.select.push(...columns);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Select with aggregation functions
|
||||||
|
*/
|
||||||
|
public selectAgg(aggregations: Record<string, string>): QuestDBQueryBuilder {
|
||||||
|
Object.entries(aggregations).forEach(([alias, expression]) => {
|
||||||
|
this.query.select.push(`${expression} as ${alias}`);
|
||||||
|
});
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* From table
|
||||||
|
*/
|
||||||
|
public from(table: TableNames | string): QuestDBQueryBuilder {
|
||||||
|
this.query.from = table;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where condition
|
||||||
|
*/
|
||||||
|
public where(condition: string): QuestDBQueryBuilder {
|
||||||
|
this.query.where.push(condition);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where symbol equals
|
||||||
|
*/
|
||||||
|
public whereSymbol(symbol: string): QuestDBQueryBuilder {
|
||||||
|
this.query.where.push(`symbol = '${symbol}'`);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where symbols in list
|
||||||
|
*/
|
||||||
|
public whereSymbolIn(symbols: string[]): QuestDBQueryBuilder {
|
||||||
|
const symbolList = symbols.map(s => `'${s}'`).join(', ');
|
||||||
|
this.query.where.push(`symbol IN (${symbolList})`);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where exchange equals
|
||||||
|
*/
|
||||||
|
public whereExchange(exchange: string): QuestDBQueryBuilder {
|
||||||
|
this.query.where.push(`exchange = '${exchange}'`);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Time range filter
|
||||||
|
*/
|
||||||
|
public whereTimeRange(startTime: Date, endTime: Date): QuestDBQueryBuilder {
|
||||||
|
this.query.timeRange = { startTime, endTime };
|
||||||
|
this.query.where.push(
|
||||||
|
`timestamp >= '${startTime.toISOString()}' AND timestamp <= '${endTime.toISOString()}'`
|
||||||
|
);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Last N hours
|
||||||
|
*/
|
||||||
|
public whereLastHours(hours: number): QuestDBQueryBuilder {
|
||||||
|
this.query.where.push(`timestamp > dateadd('h', -${hours}, now())`);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Last N days
|
||||||
|
*/
|
||||||
|
public whereLastDays(days: number): QuestDBQueryBuilder {
|
||||||
|
this.query.where.push(`timestamp > dateadd('d', -${days}, now())`);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Group by columns
|
||||||
|
*/
|
||||||
|
public groupBy(...columns: string[]): QuestDBQueryBuilder {
|
||||||
|
this.query.groupBy.push(...columns);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Order by column
|
||||||
|
*/
|
||||||
|
public orderBy(column: string, direction: 'ASC' | 'DESC' = 'ASC'): QuestDBQueryBuilder {
|
||||||
|
this.query.orderBy.push(`${column} ${direction}`);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Order by timestamp descending (most recent first)
|
||||||
|
*/
|
||||||
|
public orderByTimeDesc(): QuestDBQueryBuilder {
|
||||||
|
this.query.orderBy.push('timestamp DESC');
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Limit results
|
||||||
|
*/
|
||||||
|
public limit(count: number): QuestDBQueryBuilder {
|
||||||
|
this.query.limit = count;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sample by time interval (QuestDB specific)
|
||||||
|
*/
|
||||||
|
public sampleBy(interval: string): QuestDBQueryBuilder {
|
||||||
|
this.query.sampleBy = interval;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Latest by columns (QuestDB specific)
|
||||||
|
*/
|
||||||
|
public latestBy(...columns: string[]): QuestDBQueryBuilder {
|
||||||
|
this.query.latestBy = columns;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build and execute the query
|
||||||
|
*/
|
||||||
|
public async execute<T = any>(): Promise<QueryResult<T>> {
|
||||||
|
const sql = this.build();
|
||||||
|
this.logger.debug('Executing query', { sql });
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await this.client.query<T>(sql);
|
||||||
|
this.reset(); // Reset for next query
|
||||||
|
return result;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Query execution failed', { sql, error });
|
||||||
|
this.reset(); // Reset even on error
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the SQL query string
|
||||||
|
*/
|
||||||
|
public build(): string {
|
||||||
|
if (!this.query.from) {
|
||||||
|
throw new Error('FROM clause is required');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.query.select.length === 0) {
|
||||||
|
this.query.select.push('*');
|
||||||
|
}
|
||||||
|
|
||||||
|
let sql = `SELECT ${this.query.select.join(', ')} FROM ${this.query.from}`;
|
||||||
|
|
||||||
|
// Add WHERE clause
|
||||||
|
if (this.query.where.length > 0) {
|
||||||
|
sql += ` WHERE ${this.query.where.join(' AND ')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add LATEST BY (QuestDB specific - must come before GROUP BY)
|
||||||
|
if (this.query.latestBy && this.query.latestBy.length > 0) {
|
||||||
|
sql += ` LATEST BY ${this.query.latestBy.join(', ')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add SAMPLE BY (QuestDB specific)
|
||||||
|
if (this.query.sampleBy) {
|
||||||
|
sql += ` SAMPLE BY ${this.query.sampleBy}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add GROUP BY
|
||||||
|
if (this.query.groupBy.length > 0) {
|
||||||
|
sql += ` GROUP BY ${this.query.groupBy.join(', ')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add ORDER BY
|
||||||
|
if (this.query.orderBy.length > 0) {
|
||||||
|
sql += ` ORDER BY ${this.query.orderBy.join(', ')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add LIMIT
|
||||||
|
if (this.query.limit) {
|
||||||
|
sql += ` LIMIT ${this.query.limit}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return sql;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the built query without executing
|
||||||
|
*/
|
||||||
|
public toSQL(): string {
|
||||||
|
return this.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Predefined query methods for common use cases
|
||||||
|
/**
|
||||||
|
* Get latest OHLCV data for symbols
|
||||||
|
*/
|
||||||
|
public static latestOHLCV(
|
||||||
|
client: QuestDBClientInterface,
|
||||||
|
symbols: string[],
|
||||||
|
exchange?: string
|
||||||
|
): QuestDBQueryBuilder {
|
||||||
|
const builder = QuestDBQueryBuilder.create(client)
|
||||||
|
.select('symbol', 'timestamp', 'open', 'high', 'low', 'close', 'volume')
|
||||||
|
.from('ohlcv_data')
|
||||||
|
.whereSymbolIn(symbols)
|
||||||
|
.latestBy('symbol')
|
||||||
|
.orderByTimeDesc();
|
||||||
|
|
||||||
|
if (exchange) {
|
||||||
|
builder.whereExchange(exchange);
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Get OHLCV data with time sampling
|
||||||
|
*/
|
||||||
|
public static ohlcvTimeSeries(
|
||||||
|
client: QuestDBClientInterface,
|
||||||
|
symbol: string,
|
||||||
|
interval: string,
|
||||||
|
hours: number = 24
|
||||||
|
): QuestDBQueryBuilder {
|
||||||
|
return QuestDBQueryBuilder.create(client)
|
||||||
|
.selectAgg({
|
||||||
|
'first_open': 'first(open)',
|
||||||
|
'max_high': 'max(high)',
|
||||||
|
'min_low': 'min(low)',
|
||||||
|
'last_close': 'last(close)',
|
||||||
|
'sum_volume': 'sum(volume)'
|
||||||
|
})
|
||||||
|
.from('ohlcv_data')
|
||||||
|
.whereSymbol(symbol)
|
||||||
|
.whereLastHours(hours)
|
||||||
|
.sampleBy(interval)
|
||||||
|
.orderByTimeDesc();
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Get market analytics data
|
||||||
|
*/
|
||||||
|
public static marketAnalytics(
|
||||||
|
client: QuestDBClientInterface,
|
||||||
|
symbols: string[],
|
||||||
|
hours: number = 1
|
||||||
|
): QuestDBQueryBuilder {
|
||||||
|
return QuestDBQueryBuilder.create(client)
|
||||||
|
.select('symbol', 'timestamp', 'rsi', 'macd', 'bollinger_upper', 'bollinger_lower', 'volume_sma')
|
||||||
|
.from('market_analytics')
|
||||||
|
.whereSymbolIn(symbols)
|
||||||
|
.whereLastHours(hours)
|
||||||
|
.orderBy('symbol')
|
||||||
|
.orderByTimeDesc();
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Get performance metrics for a time range
|
||||||
|
*/
|
||||||
|
public static performanceMetrics(
|
||||||
|
client: QuestDBClientInterface,
|
||||||
|
startTime: Date,
|
||||||
|
endTime: Date
|
||||||
|
): QuestDBQueryBuilder {
|
||||||
|
return QuestDBQueryBuilder.create(client)
|
||||||
|
.selectAgg({
|
||||||
|
'total_trades': 'count(*)',
|
||||||
|
'avg_response_time': 'avg(response_time)',
|
||||||
|
'max_response_time': 'max(response_time)',
|
||||||
|
'error_rate': 'sum(case when success = false then 1 else 0 end) * 100.0 / count(*)'
|
||||||
|
})
|
||||||
|
.from('performance_metrics')
|
||||||
|
.whereTimeRange(startTime, endTime)
|
||||||
|
.sampleBy('1m');
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Get trade execution data
|
||||||
|
*/
|
||||||
|
public static tradeExecutions(
|
||||||
|
client: QuestDBClientInterface,
|
||||||
|
symbol?: string,
|
||||||
|
hours: number = 24
|
||||||
|
): QuestDBQueryBuilder {
|
||||||
|
const builder = QuestDBQueryBuilder.create(client)
|
||||||
|
.select('symbol', 'timestamp', 'side', 'quantity', 'price', 'execution_time')
|
||||||
|
.from('trade_executions')
|
||||||
|
.whereLastHours(hours)
|
||||||
|
.orderByTimeDesc();
|
||||||
|
|
||||||
|
if (symbol) {
|
||||||
|
builder.whereSymbol(symbol);
|
||||||
|
}
|
||||||
|
|
||||||
|
return builder;
|
||||||
|
}
|
||||||
|
}
|
||||||
404
libs/questdb-client/src/schema.ts
Normal file
404
libs/questdb-client/src/schema.ts
Normal file
|
|
@ -0,0 +1,404 @@
|
||||||
|
import { Logger } from '@stock-bot/logger';
|
||||||
|
import type { TableSchema, IndexDefinition, TableNames, QueryResult } from './types';
|
||||||
|
|
||||||
|
// Interface to avoid circular dependency
|
||||||
|
interface QuestDBClientInterface {
|
||||||
|
query<T = any>(sql: string, params?: any[]): Promise<QueryResult<T>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB Schema Manager
|
||||||
|
*
|
||||||
|
* Manages database schemas, table creation, and optimization
|
||||||
|
* for time-series data storage in QuestDB.
|
||||||
|
*/
|
||||||
|
export class QuestDBSchemaManager {
|
||||||
|
private readonly logger: Logger;
|
||||||
|
private readonly schemas: Map<string, TableSchema> = new Map();
|
||||||
|
constructor(private readonly client: QuestDBClientInterface) {
|
||||||
|
this.logger = new Logger('QuestDBSchemaManager');
|
||||||
|
this.initializeSchemas();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize predefined schemas
|
||||||
|
*/
|
||||||
|
private initializeSchemas(): void {
|
||||||
|
// OHLCV Data Table
|
||||||
|
this.schemas.set('ohlcv_data', {
|
||||||
|
tableName: 'ohlcv_data',
|
||||||
|
columns: [
|
||||||
|
{ name: 'symbol', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'exchange', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'timestamp', type: 'TIMESTAMP', nullable: false, designated: true },
|
||||||
|
{ name: 'open', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'high', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'low', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'close', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'volume', type: 'LONG', nullable: false },
|
||||||
|
{ name: 'data_source', type: 'SYMBOL', nullable: true }
|
||||||
|
],
|
||||||
|
partitionBy: 'DAY',
|
||||||
|
orderBy: ['symbol', 'timestamp'],
|
||||||
|
indices: [
|
||||||
|
{ columns: ['symbol'], type: 'HASH' },
|
||||||
|
{ columns: ['exchange'], type: 'HASH' }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
// Market Analytics Table
|
||||||
|
this.schemas.set('market_analytics', {
|
||||||
|
tableName: 'market_analytics',
|
||||||
|
columns: [
|
||||||
|
{ name: 'symbol', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'exchange', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'timestamp', type: 'TIMESTAMP', nullable: false, designated: true },
|
||||||
|
{ name: 'rsi', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'macd', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'signal', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'histogram', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'bollinger_upper', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'bollinger_lower', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'volume_sma', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'timeframe', type: 'SYMBOL', nullable: true }
|
||||||
|
],
|
||||||
|
partitionBy: 'DAY',
|
||||||
|
orderBy: ['symbol', 'timestamp'],
|
||||||
|
indices: [
|
||||||
|
{ columns: ['symbol'], type: 'HASH' },
|
||||||
|
{ columns: ['timeframe'], type: 'HASH' }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
// Trade Executions Table
|
||||||
|
this.schemas.set('trade_executions', {
|
||||||
|
tableName: 'trade_executions',
|
||||||
|
columns: [
|
||||||
|
{ name: 'symbol', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'timestamp', type: 'TIMESTAMP', nullable: false, designated: true },
|
||||||
|
{ name: 'side', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'quantity', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'price', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'execution_time', type: 'LONG', nullable: false },
|
||||||
|
{ name: 'order_id', type: 'SYMBOL', nullable: true },
|
||||||
|
{ name: 'strategy', type: 'SYMBOL', nullable: true },
|
||||||
|
{ name: 'commission', type: 'DOUBLE', nullable: true }
|
||||||
|
],
|
||||||
|
partitionBy: 'DAY',
|
||||||
|
orderBy: ['symbol', 'timestamp'],
|
||||||
|
indices: [
|
||||||
|
{ columns: ['symbol'], type: 'HASH' },
|
||||||
|
{ columns: ['order_id'], type: 'HASH' },
|
||||||
|
{ columns: ['strategy'], type: 'HASH' }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
// Performance Metrics Table
|
||||||
|
this.schemas.set('performance_metrics', {
|
||||||
|
tableName: 'performance_metrics',
|
||||||
|
columns: [
|
||||||
|
{ name: 'timestamp', type: 'TIMESTAMP', nullable: false, designated: true },
|
||||||
|
{ name: 'operation', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'response_time', type: 'LONG', nullable: false },
|
||||||
|
{ name: 'success', type: 'BOOLEAN', nullable: false },
|
||||||
|
{ name: 'error_code', type: 'SYMBOL', nullable: true },
|
||||||
|
{ name: 'component', type: 'SYMBOL', nullable: true }
|
||||||
|
],
|
||||||
|
partitionBy: 'HOUR',
|
||||||
|
orderBy: ['operation', 'timestamp'],
|
||||||
|
indices: [
|
||||||
|
{ columns: ['operation'], type: 'HASH' },
|
||||||
|
{ columns: ['success'], type: 'HASH' }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
// Portfolio Positions Table
|
||||||
|
this.schemas.set('portfolio_positions', {
|
||||||
|
tableName: 'portfolio_positions',
|
||||||
|
columns: [
|
||||||
|
{ name: 'portfolio_id', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'symbol', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'timestamp', type: 'TIMESTAMP', nullable: false, designated: true },
|
||||||
|
{ name: 'quantity', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'avg_cost', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'market_value', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'unrealized_pnl', type: 'DOUBLE', nullable: false },
|
||||||
|
{ name: 'realized_pnl', type: 'DOUBLE', nullable: false }
|
||||||
|
],
|
||||||
|
partitionBy: 'DAY',
|
||||||
|
orderBy: ['portfolio_id', 'symbol', 'timestamp'],
|
||||||
|
indices: [
|
||||||
|
{ columns: ['portfolio_id'], type: 'HASH' },
|
||||||
|
{ columns: ['symbol'], type: 'HASH' }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
// Risk Metrics Table
|
||||||
|
this.schemas.set('risk_metrics', {
|
||||||
|
tableName: 'risk_metrics',
|
||||||
|
columns: [
|
||||||
|
{ name: 'portfolio_id', type: 'SYMBOL', nullable: false },
|
||||||
|
{ name: 'timestamp', type: 'TIMESTAMP', nullable: false, designated: true },
|
||||||
|
{ name: 'var_1d', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'var_5d', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'expected_shortfall', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'beta', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'sharpe_ratio', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'max_drawdown', type: 'DOUBLE', nullable: true },
|
||||||
|
{ name: 'volatility', type: 'DOUBLE', nullable: true }
|
||||||
|
],
|
||||||
|
partitionBy: 'DAY',
|
||||||
|
orderBy: ['portfolio_id', 'timestamp'],
|
||||||
|
indices: [
|
||||||
|
{ columns: ['portfolio_id'], type: 'HASH' }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create all tables
|
||||||
|
*/
|
||||||
|
public async createAllTables(): Promise<void> {
|
||||||
|
this.logger.info('Creating all QuestDB tables');
|
||||||
|
|
||||||
|
for (const [tableName, schema] of this.schemas) {
|
||||||
|
try {
|
||||||
|
await this.createTable(schema);
|
||||||
|
this.logger.info(`Table ${tableName} created successfully`);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to create table ${tableName}`, { error });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a single table
|
||||||
|
*/
|
||||||
|
public async createTable(schema: TableSchema): Promise<void> {
|
||||||
|
const sql = this.buildCreateTableSQL(schema);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.client.query(sql);
|
||||||
|
this.logger.info(`Table ${schema.tableName} created`, { sql });
|
||||||
|
} catch (error) {
|
||||||
|
// Check if table already exists
|
||||||
|
if (error instanceof Error && error.message.includes('already exists')) {
|
||||||
|
this.logger.info(`Table ${schema.tableName} already exists`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drop a table
|
||||||
|
*/
|
||||||
|
public async dropTable(tableName: string): Promise<void> {
|
||||||
|
const sql = `DROP TABLE IF EXISTS ${tableName}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.client.query(sql);
|
||||||
|
this.logger.info(`Table ${tableName} dropped`);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to drop table ${tableName}`, { error });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if table exists
|
||||||
|
*/
|
||||||
|
public async tableExists(tableName: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const result = await this.client.query(`
|
||||||
|
SELECT COUNT(*) as count
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_name = '${tableName}'
|
||||||
|
`);
|
||||||
|
|
||||||
|
return result.rows.length > 0 && result.rows[0].count > 0;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Error checking if table exists: ${tableName}`, { error });
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get table schema
|
||||||
|
*/
|
||||||
|
public getSchema(tableName: string): TableSchema | undefined {
|
||||||
|
return this.schemas.get(tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add custom schema
|
||||||
|
*/
|
||||||
|
public addSchema(schema: TableSchema): void {
|
||||||
|
this.schemas.set(schema.tableName, schema);
|
||||||
|
this.logger.info(`Schema added for table: ${schema.tableName}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all schema names
|
||||||
|
*/
|
||||||
|
public getSchemaNames(): string[] {
|
||||||
|
return Array.from(this.schemas.keys());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Optimize table (rebuild indices, etc.)
|
||||||
|
*/
|
||||||
|
public async optimizeTable(tableName: string): Promise<void> {
|
||||||
|
const schema = this.schemas.get(tableName);
|
||||||
|
if (!schema) {
|
||||||
|
throw new Error(`Schema not found for table: ${tableName}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// QuestDB automatically optimizes, but we can analyze table stats
|
||||||
|
try {
|
||||||
|
const stats = await this.getTableStats(tableName);
|
||||||
|
this.logger.info(`Table ${tableName} stats`, stats);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to optimize table ${tableName}`, { error });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get table statistics
|
||||||
|
*/
|
||||||
|
public async getTableStats(tableName: string): Promise<any> {
|
||||||
|
try {
|
||||||
|
const result = await this.client.query(`
|
||||||
|
SELECT
|
||||||
|
COUNT(*) as row_count,
|
||||||
|
MIN(timestamp) as min_timestamp,
|
||||||
|
MAX(timestamp) as max_timestamp
|
||||||
|
FROM ${tableName}
|
||||||
|
`);
|
||||||
|
|
||||||
|
return result.rows[0] || {};
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to get table stats for ${tableName}`, { error });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Truncate table (remove all data but keep structure)
|
||||||
|
*/
|
||||||
|
public async truncateTable(tableName: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.client.query(`TRUNCATE TABLE ${tableName}`);
|
||||||
|
this.logger.info(`Table ${tableName} truncated`);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to truncate table ${tableName}`, { error });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create table partitions for future dates
|
||||||
|
*/
|
||||||
|
public async createPartitions(tableName: string, days: number = 30): Promise<void> {
|
||||||
|
// QuestDB handles partitioning automatically based on the PARTITION BY clause
|
||||||
|
// This method is for future extensibility
|
||||||
|
this.logger.info(`Partitioning is automatic for table ${tableName}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build CREATE TABLE SQL statement
|
||||||
|
*/
|
||||||
|
private buildCreateTableSQL(schema: TableSchema): string {
|
||||||
|
const columns = schema.columns.map(col => {
|
||||||
|
let columnDef = `${col.name} ${col.type}`;
|
||||||
|
|
||||||
|
if (!col.nullable) {
|
||||||
|
columnDef += ' NOT NULL';
|
||||||
|
}
|
||||||
|
|
||||||
|
return columnDef;
|
||||||
|
}).join(', ');
|
||||||
|
|
||||||
|
let sql = `CREATE TABLE IF NOT EXISTS ${schema.tableName} (${columns})`;
|
||||||
|
|
||||||
|
// Add designated timestamp
|
||||||
|
const timestampColumn = schema.columns.find(col => col.designated);
|
||||||
|
if (timestampColumn) {
|
||||||
|
sql += ` timestamp(${timestampColumn.name})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add partition by
|
||||||
|
if (schema.partitionBy) {
|
||||||
|
sql += ` PARTITION BY ${schema.partitionBy}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return sql;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build index creation SQL (for future use)
|
||||||
|
*/
|
||||||
|
private buildCreateIndexSQL(tableName: string, index: IndexDefinition): string {
|
||||||
|
const indexName = `idx_${tableName}_${index.columns.join('_')}`;
|
||||||
|
const columns = index.columns.join(', ');
|
||||||
|
|
||||||
|
// QuestDB uses different index syntax, this is for future compatibility
|
||||||
|
return `CREATE INDEX ${indexName} ON ${tableName} (${columns})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate schema definition
|
||||||
|
*/
|
||||||
|
private validateSchema(schema: TableSchema): void {
|
||||||
|
if (!schema.tableName) {
|
||||||
|
throw new Error('Table name is required');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!schema.columns || schema.columns.length === 0) {
|
||||||
|
throw new Error('At least one column is required');
|
||||||
|
}
|
||||||
|
|
||||||
|
const timestampColumns = schema.columns.filter(col => col.designated);
|
||||||
|
if (timestampColumns.length > 1) {
|
||||||
|
throw new Error('Only one designated timestamp column is allowed');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (timestampColumns.length === 0) {
|
||||||
|
throw new Error('A designated timestamp column is required for time-series tables');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get table creation status
|
||||||
|
*/
|
||||||
|
public async getTableCreationStatus(): Promise<Record<string, boolean>> {
|
||||||
|
const status: Record<string, boolean> = {};
|
||||||
|
|
||||||
|
for (const tableName of this.schemas.keys()) {
|
||||||
|
status[tableName] = await this.tableExists(tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize database schema
|
||||||
|
*/
|
||||||
|
public async initializeDatabase(): Promise<void> {
|
||||||
|
this.logger.info('Initializing QuestDB schema');
|
||||||
|
|
||||||
|
// Validate all schemas first
|
||||||
|
for (const schema of this.schemas.values()) {
|
||||||
|
this.validateSchema(schema);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create all tables
|
||||||
|
await this.createAllTables();
|
||||||
|
|
||||||
|
// Get creation status
|
||||||
|
const status = await this.getTableCreationStatus();
|
||||||
|
this.logger.info('Database initialization complete', { tableStatus: status });
|
||||||
|
}
|
||||||
|
}
|
||||||
284
libs/questdb-client/src/types.ts
Normal file
284
libs/questdb-client/src/types.ts
Normal file
|
|
@ -0,0 +1,284 @@
|
||||||
|
/**
|
||||||
|
* QuestDB Client Configuration and Types
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB Client Configuration
|
||||||
|
*/
|
||||||
|
export interface QuestDBClientConfig {
|
||||||
|
host: string;
|
||||||
|
httpPort: number;
|
||||||
|
pgPort: number;
|
||||||
|
influxPort: number;
|
||||||
|
user?: string;
|
||||||
|
password?: string;
|
||||||
|
database?: string;
|
||||||
|
tls?: {
|
||||||
|
enabled: boolean;
|
||||||
|
verifyServerCert: boolean;
|
||||||
|
};
|
||||||
|
timeouts?: {
|
||||||
|
connection: number;
|
||||||
|
request: number;
|
||||||
|
};
|
||||||
|
retryAttempts?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB Connection Options
|
||||||
|
*/
|
||||||
|
export interface QuestDBConnectionOptions {
|
||||||
|
protocol?: 'http' | 'pg' | 'influx';
|
||||||
|
retryAttempts?: number;
|
||||||
|
retryDelay?: number;
|
||||||
|
healthCheckInterval?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Health Status Types
|
||||||
|
*/
|
||||||
|
export type QuestDBHealthStatus = 'healthy' | 'degraded' | 'unhealthy';
|
||||||
|
|
||||||
|
export interface QuestDBHealthCheck {
|
||||||
|
status: QuestDBHealthStatus;
|
||||||
|
timestamp: Date;
|
||||||
|
latency: number;
|
||||||
|
protocols: {
|
||||||
|
http: boolean;
|
||||||
|
pg: boolean;
|
||||||
|
influx: boolean;
|
||||||
|
};
|
||||||
|
errors?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface QuestDBMetrics {
|
||||||
|
queriesPerSecond: number;
|
||||||
|
insertsPerSecond: number;
|
||||||
|
averageQueryTime: number;
|
||||||
|
errorRate: number;
|
||||||
|
dataIngestionRate: number;
|
||||||
|
storageSize: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Table Names for Time-Series Data
|
||||||
|
*/
|
||||||
|
export type TableNames =
|
||||||
|
| 'ohlcv'
|
||||||
|
| 'trades'
|
||||||
|
| 'quotes'
|
||||||
|
| 'indicators'
|
||||||
|
| 'performance'
|
||||||
|
| 'risk_metrics'
|
||||||
|
| 'market_events'
|
||||||
|
| 'strategy_signals'
|
||||||
|
| 'portfolio_snapshots';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Time-Series Data Types
|
||||||
|
*/
|
||||||
|
export interface BaseTimeSeriesData {
|
||||||
|
timestamp: Date;
|
||||||
|
symbol?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface OHLCVData extends BaseTimeSeriesData {
|
||||||
|
open: number;
|
||||||
|
high: number;
|
||||||
|
low: number;
|
||||||
|
close: number;
|
||||||
|
volume: number;
|
||||||
|
timeframe: string; // '1m', '5m', '1h', '1d', etc.
|
||||||
|
source: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TradeData extends BaseTimeSeriesData {
|
||||||
|
trade_id: string;
|
||||||
|
price: number;
|
||||||
|
quantity: number;
|
||||||
|
side: 'buy' | 'sell';
|
||||||
|
exchange: string;
|
||||||
|
conditions?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface QuoteData extends BaseTimeSeriesData {
|
||||||
|
bid_price: number;
|
||||||
|
bid_size: number;
|
||||||
|
ask_price: number;
|
||||||
|
ask_size: number;
|
||||||
|
exchange: string;
|
||||||
|
spread: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IndicatorData extends BaseTimeSeriesData {
|
||||||
|
indicator_name: string;
|
||||||
|
value: number;
|
||||||
|
parameters?: Record<string, any>;
|
||||||
|
timeframe: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PerformanceData extends BaseTimeSeriesData {
|
||||||
|
portfolio_id: string;
|
||||||
|
total_value: number;
|
||||||
|
cash_balance: number;
|
||||||
|
unrealized_pnl: number;
|
||||||
|
realized_pnl: number;
|
||||||
|
daily_return: number;
|
||||||
|
cumulative_return: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RiskMetrics extends BaseTimeSeriesData {
|
||||||
|
portfolio_id?: string;
|
||||||
|
strategy_id?: string;
|
||||||
|
metric_name: string;
|
||||||
|
value: number;
|
||||||
|
threshold?: number;
|
||||||
|
status: 'normal' | 'warning' | 'breach';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query Result Types
|
||||||
|
*/
|
||||||
|
export interface QueryResult<T = any> {
|
||||||
|
rows: T[];
|
||||||
|
rowCount: number;
|
||||||
|
executionTime: number;
|
||||||
|
metadata?: {
|
||||||
|
columns: Array<{
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InsertResult {
|
||||||
|
rowsInserted: number;
|
||||||
|
executionTime: number;
|
||||||
|
errors?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema Definition Types
|
||||||
|
*/
|
||||||
|
export interface ColumnDefinition {
|
||||||
|
name: string;
|
||||||
|
type: 'SYMBOL' | 'STRING' | 'DOUBLE' | 'FLOAT' | 'LONG' | 'INT' | 'BOOLEAN' | 'TIMESTAMP' | 'DATE' | 'BINARY';
|
||||||
|
indexed?: boolean;
|
||||||
|
capacity?: number; // For SYMBOL type
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TableDefinition {
|
||||||
|
name: string;
|
||||||
|
columns: ColumnDefinition[];
|
||||||
|
partitionBy?: 'NONE' | 'DAY' | 'MONTH' | 'YEAR';
|
||||||
|
timestamp?: string; // Column name to use as designated timestamp
|
||||||
|
dedup?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connection Pool Types
|
||||||
|
*/
|
||||||
|
export interface ConnectionPoolConfig {
|
||||||
|
minConnections: number;
|
||||||
|
maxConnections: number;
|
||||||
|
idleTimeout: number;
|
||||||
|
acquireTimeout: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Health Monitoring Types
|
||||||
|
*/
|
||||||
|
export interface HealthStatus {
|
||||||
|
isHealthy: boolean;
|
||||||
|
lastCheck: Date;
|
||||||
|
responseTime: number;
|
||||||
|
message: string;
|
||||||
|
error?: Error;
|
||||||
|
details?: {
|
||||||
|
pgPool: boolean;
|
||||||
|
httpEndpoint: boolean;
|
||||||
|
uptime: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PerformanceMetrics {
|
||||||
|
totalQueries: number;
|
||||||
|
successfulQueries: number;
|
||||||
|
failedQueries: number;
|
||||||
|
averageResponseTime: number;
|
||||||
|
lastQueryTime: Date | null;
|
||||||
|
connectionUptime: number;
|
||||||
|
memoryUsage: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query Builder Types
|
||||||
|
*/
|
||||||
|
export interface TimeSeriesQuery {
|
||||||
|
table: TableNames | string;
|
||||||
|
columns?: string[];
|
||||||
|
timeRange?: TimeRange;
|
||||||
|
groupBy?: string[];
|
||||||
|
aggregations?: Record<string, string>;
|
||||||
|
sampleBy?: string;
|
||||||
|
latestBy?: string[];
|
||||||
|
orderBy?: Array<{ column: string; direction: 'ASC' | 'DESC' }>;
|
||||||
|
limit?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AggregationQuery {
|
||||||
|
aggregations: Record<string, string>;
|
||||||
|
groupBy?: string[];
|
||||||
|
having?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TimeRange {
|
||||||
|
startTime: Date;
|
||||||
|
endTime: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* InfluxDB Line Protocol Types
|
||||||
|
*/
|
||||||
|
export interface InfluxLineData {
|
||||||
|
measurement: string;
|
||||||
|
tags: Record<string, string>;
|
||||||
|
fields: Record<string, number | string | boolean>;
|
||||||
|
timestamp?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InfluxWriteOptions {
|
||||||
|
batchSize?: number;
|
||||||
|
flushInterval?: number;
|
||||||
|
autoFlush?: boolean;
|
||||||
|
precision?: 'ns' | 'us' | 'ms' | 's';
|
||||||
|
retryAttempts?: number;
|
||||||
|
retryDelay?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Schema Management Types
|
||||||
|
*/
|
||||||
|
export interface TableSchema {
|
||||||
|
tableName: string;
|
||||||
|
columns: ColumnSchema[];
|
||||||
|
partitionBy?: 'NONE' | 'HOUR' | 'DAY' | 'MONTH' | 'YEAR';
|
||||||
|
orderBy?: string[];
|
||||||
|
indices?: IndexDefinition[];
|
||||||
|
dedup?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ColumnSchema {
|
||||||
|
name: string;
|
||||||
|
type: 'SYMBOL' | 'STRING' | 'DOUBLE' | 'FLOAT' | 'LONG' | 'INT' | 'BOOLEAN' | 'TIMESTAMP' | 'DATE' | 'BINARY';
|
||||||
|
nullable?: boolean;
|
||||||
|
designated?: boolean; // For designated timestamp column
|
||||||
|
capacity?: number; // For SYMBOL type
|
||||||
|
indexed?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IndexDefinition {
|
||||||
|
columns: string[];
|
||||||
|
type: 'HASH' | 'BTREE';
|
||||||
|
unique?: boolean;
|
||||||
|
}
|
||||||
233
libs/questdb-client/test/integration.test.ts
Normal file
233
libs/questdb-client/test/integration.test.ts
Normal file
|
|
@ -0,0 +1,233 @@
|
||||||
|
/**
|
||||||
|
* QuestDB Client Integration Test
|
||||||
|
*
|
||||||
|
* This test validates that all components work together correctly
|
||||||
|
* without requiring an actual QuestDB instance.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import 'jest-extended';
|
||||||
|
import {
|
||||||
|
QuestDBClient,
|
||||||
|
QuestDBHealthMonitor,
|
||||||
|
QuestDBQueryBuilder,
|
||||||
|
QuestDBInfluxWriter,
|
||||||
|
QuestDBSchemaManager,
|
||||||
|
createQuestDBClient
|
||||||
|
} from '../src';
|
||||||
|
import { questdbTestHelpers } from './setup';
|
||||||
|
|
||||||
|
describe('QuestDB Client Integration', () => {
|
||||||
|
let client: QuestDBClient;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
client = new QuestDBClient({
|
||||||
|
host: 'localhost',
|
||||||
|
httpPort: 9000,
|
||||||
|
pgPort: 8812,
|
||||||
|
influxPort: 9009,
|
||||||
|
database: 'questdb',
|
||||||
|
user: 'admin',
|
||||||
|
password: 'quest'
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (client.connected) {
|
||||||
|
await client.disconnect();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Client Initialization', () => {
|
||||||
|
it('should create client with factory function', () => {
|
||||||
|
const factoryClient = createQuestDBClient();
|
||||||
|
expect(factoryClient).toBeInstanceOf(QuestDBClient);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should initialize all supporting classes', () => {
|
||||||
|
expect(client.getHealthMonitor()).toBeInstanceOf(QuestDBHealthMonitor);
|
||||||
|
expect(client.queryBuilder()).toBeInstanceOf(QuestDBQueryBuilder);
|
||||||
|
expect(client.getInfluxWriter()).toBeInstanceOf(QuestDBInfluxWriter);
|
||||||
|
expect(client.getSchemaManager()).toBeInstanceOf(QuestDBSchemaManager);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should handle connection configuration', () => {
|
||||||
|
expect(client.getHttpUrl()).toBe('http://localhost:9000');
|
||||||
|
expect(client.getInfluxUrl()).toBe('http://localhost:9009');
|
||||||
|
expect(client.connected).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Query Builder', () => {
|
||||||
|
it('should build query using query builder', () => {
|
||||||
|
const query = client.queryBuilder()
|
||||||
|
.select('symbol', 'close', 'timestamp')
|
||||||
|
.from('ohlcv')
|
||||||
|
.whereSymbol('AAPL')
|
||||||
|
.whereLastHours(24)
|
||||||
|
.orderBy('timestamp', 'DESC')
|
||||||
|
.limit(100)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
expect(query).toContain('SELECT symbol, close, timestamp');
|
||||||
|
expect(query).toContain('FROM ohlcv');
|
||||||
|
expect(query).toContain("symbol = 'AAPL'");
|
||||||
|
expect(query).toContain('ORDER BY timestamp DESC');
|
||||||
|
expect(query).toContain('LIMIT 100');
|
||||||
|
expect(questdbTestHelpers.validateQuestDBQuery(query)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should build time-series specific queries', () => {
|
||||||
|
const latestQuery = client.queryBuilder()
|
||||||
|
.select('*')
|
||||||
|
.from('ohlcv')
|
||||||
|
.latestBy('symbol')
|
||||||
|
.build();
|
||||||
|
|
||||||
|
expect(latestQuery).toContain('LATEST BY symbol');
|
||||||
|
expect(questdbTestHelpers.validateQuestDBQuery(latestQuery)).toBe(true);
|
||||||
|
|
||||||
|
const sampleQuery = client.queryBuilder()
|
||||||
|
.select('symbol', 'avg(close)')
|
||||||
|
.from('ohlcv')
|
||||||
|
.sampleBy('1d')
|
||||||
|
.build();
|
||||||
|
|
||||||
|
expect(sampleQuery).toContain('SAMPLE BY 1d');
|
||||||
|
expect(questdbTestHelpers.validateQuestDBQuery(sampleQuery)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should build aggregation queries', () => {
|
||||||
|
const query = client.aggregate('ohlcv')
|
||||||
|
.select('symbol', 'avg(close) as avg_price', 'max(high) as max_high')
|
||||||
|
.whereSymbolIn(['AAPL', 'GOOGL'])
|
||||||
|
.groupBy('symbol')
|
||||||
|
.sampleBy('1h')
|
||||||
|
.build();
|
||||||
|
|
||||||
|
expect(query).toContain('SELECT symbol, avg(close) as avg_price, max(high) as max_high');
|
||||||
|
expect(query).toContain('FROM ohlcv');
|
||||||
|
expect(query).toContain("symbol IN ('AAPL', 'GOOGL')");
|
||||||
|
expect(query).toContain('SAMPLE BY 1h');
|
||||||
|
expect(query).toContain('GROUP BY symbol');
|
||||||
|
expect(questdbTestHelpers.validateQuestDBQuery(query)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('InfluxDB Writer', () => {
|
||||||
|
it('should write OHLCV data using InfluxDB line protocol', async () => {
|
||||||
|
const ohlcvData = [{
|
||||||
|
timestamp: new Date('2024-01-01T12:00:00Z'),
|
||||||
|
open: 150.00,
|
||||||
|
high: 152.00,
|
||||||
|
low: 149.50,
|
||||||
|
close: 151.50,
|
||||||
|
volume: 1000000
|
||||||
|
}]; // Mock the actual write operation
|
||||||
|
jest.spyOn(client.getInfluxWriter(), 'writeOHLCV').mockResolvedValue();
|
||||||
|
|
||||||
|
await expect(async () => {
|
||||||
|
await client.writeOHLCV('AAPL', 'NASDAQ', ohlcvData);
|
||||||
|
}).not.toThrow();
|
||||||
|
}); it('should handle batch operations', () => {
|
||||||
|
const lines = questdbTestHelpers.generateInfluxDBLines(3);
|
||||||
|
expect(lines.length).toBe(3);
|
||||||
|
|
||||||
|
lines.forEach(line => {
|
||||||
|
expect(line).toContain('ohlcv,symbol=TEST');
|
||||||
|
expect(line).toMatch(/\d{19}$/); // Nanosecond timestamp
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}); describe('Schema Manager', () => {
|
||||||
|
it('should provide schema access', () => {
|
||||||
|
const schema = client.getSchemaManager().getSchema('ohlcv_data');
|
||||||
|
|
||||||
|
expect(schema).toBeDefined();
|
||||||
|
expect(schema?.tableName).toBe('ohlcv_data');
|
||||||
|
|
||||||
|
const symbolColumn = schema?.columns.find(col => col.name === 'symbol');
|
||||||
|
expect(symbolColumn).toBeDefined();
|
||||||
|
expect(symbolColumn?.type).toBe('SYMBOL');
|
||||||
|
|
||||||
|
expect(schema?.partitionBy).toBe('DAY');
|
||||||
|
});
|
||||||
|
}); describe('Health Monitor', () => {
|
||||||
|
it('should provide health monitoring capabilities', async () => {
|
||||||
|
const healthMonitor = client.getHealthMonitor();
|
||||||
|
expect(healthMonitor).toBeInstanceOf(QuestDBHealthMonitor);
|
||||||
|
// Mock health status since we're not connected
|
||||||
|
const mockHealthStatus = {
|
||||||
|
isHealthy: false,
|
||||||
|
lastCheck: new Date(),
|
||||||
|
responseTime: 100,
|
||||||
|
message: 'Connection not established',
|
||||||
|
details: {
|
||||||
|
pgPool: false,
|
||||||
|
httpEndpoint: false,
|
||||||
|
uptime: 0
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
jest.spyOn(healthMonitor, 'getHealthStatus').mockResolvedValue(mockHealthStatus);
|
||||||
|
|
||||||
|
const health = await healthMonitor.getHealthStatus();
|
||||||
|
expect(health.isHealthy).toBe(false);
|
||||||
|
expect(health.lastCheck).toBeInstanceOf(Date);
|
||||||
|
expect(health.message).toBe('Connection not established');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Time-Series Operations', () => {
|
||||||
|
it('should support latest by operations', async () => {
|
||||||
|
// Mock the query execution
|
||||||
|
const mockResult = {
|
||||||
|
rows: [{ symbol: 'AAPL', close: 150.00, timestamp: new Date() }],
|
||||||
|
rowCount: 1,
|
||||||
|
executionTime: 10,
|
||||||
|
metadata: { columns: [] }
|
||||||
|
}; jest.spyOn(client, 'query').mockResolvedValue(mockResult);
|
||||||
|
|
||||||
|
const result = await client.latestBy('ohlcv', ['symbol', 'close'], 'symbol');
|
||||||
|
expect(result.rows.length).toBe(1);
|
||||||
|
expect(result.rows[0].symbol).toBe('AAPL');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should support sample by operations', async () => {
|
||||||
|
// Mock the query execution
|
||||||
|
const mockResult = {
|
||||||
|
rows: [
|
||||||
|
{ symbol: 'AAPL', avg_close: 150.00, timestamp: new Date() }
|
||||||
|
],
|
||||||
|
rowCount: 1,
|
||||||
|
executionTime: 15,
|
||||||
|
metadata: { columns: [] }
|
||||||
|
};
|
||||||
|
|
||||||
|
jest.spyOn(client, 'query').mockResolvedValue(mockResult);
|
||||||
|
|
||||||
|
const result = await client.sampleBy(
|
||||||
|
'ohlcv',
|
||||||
|
['symbol', 'avg(close) as avg_close'],
|
||||||
|
'1h',
|
||||||
|
'timestamp', "symbol = 'AAPL'"
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.rows.length).toBe(1);
|
||||||
|
expect(result.executionTime).toBe(15);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('Connection Management', () => {
|
||||||
|
it('should handle connection configuration', () => {
|
||||||
|
expect(client.getHttpUrl()).toBe('http://localhost:9000');
|
||||||
|
expect(client.getInfluxUrl()).toBe('http://localhost:9009');
|
||||||
|
expect(client.connected).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should provide configuration access', () => {
|
||||||
|
const config = client.configuration;
|
||||||
|
expect(config.host).toBe('localhost');
|
||||||
|
expect(config.httpPort).toBe(9000);
|
||||||
|
expect(config.user).toBe('admin');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
215
libs/questdb-client/test/setup.ts
Normal file
215
libs/questdb-client/test/setup.ts
Normal file
|
|
@ -0,0 +1,215 @@
|
||||||
|
/**
|
||||||
|
* QuestDB Client Test Setup
|
||||||
|
*
|
||||||
|
* Setup file specific to QuestDB client library tests.
|
||||||
|
* Provides utilities and mocks for testing database operations.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { newDb } from 'pg-mem';
|
||||||
|
|
||||||
|
// Mock PostgreSQL database for unit tests
|
||||||
|
let pgMem: any;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
// Create in-memory PostgreSQL database
|
||||||
|
pgMem = newDb();
|
||||||
|
|
||||||
|
// Register QuestDB-specific functions
|
||||||
|
pgMem.public.registerFunction({
|
||||||
|
name: 'now',
|
||||||
|
implementation: () => new Date().toISOString()
|
||||||
|
});
|
||||||
|
|
||||||
|
pgMem.public.registerFunction({
|
||||||
|
name: 'dateadd',
|
||||||
|
args: [{ type: 'text' }, { type: 'int' }, { type: 'timestamp' }],
|
||||||
|
returns: 'timestamp',
|
||||||
|
implementation: (unit: string, amount: number, date: Date) => {
|
||||||
|
const result = new Date(date);
|
||||||
|
switch (unit) {
|
||||||
|
case 'd':
|
||||||
|
case 'day':
|
||||||
|
result.setDate(result.getDate() + amount);
|
||||||
|
break;
|
||||||
|
case 'h':
|
||||||
|
case 'hour':
|
||||||
|
result.setHours(result.getHours() + amount);
|
||||||
|
break;
|
||||||
|
case 'm':
|
||||||
|
case 'minute':
|
||||||
|
result.setMinutes(result.getMinutes() + amount);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new Error(`Unsupported date unit: ${unit}`);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}); // Mock QuestDB HTTP client
|
||||||
|
(global as any).fetch = jest.fn();
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
// Reset database state
|
||||||
|
if (pgMem) {
|
||||||
|
try {
|
||||||
|
pgMem.public.none('DROP TABLE IF EXISTS ohlcv CASCADE');
|
||||||
|
pgMem.public.none('DROP TABLE IF EXISTS trades CASCADE');
|
||||||
|
pgMem.public.none('DROP TABLE IF EXISTS quotes CASCADE');
|
||||||
|
pgMem.public.none('DROP TABLE IF EXISTS indicators CASCADE');
|
||||||
|
pgMem.public.none('DROP TABLE IF EXISTS performance CASCADE');
|
||||||
|
pgMem.public.none('DROP TABLE IF EXISTS risk_metrics CASCADE');
|
||||||
|
} catch (error) {
|
||||||
|
// Tables might not exist, ignore errors
|
||||||
|
}
|
||||||
|
} // Reset fetch mock
|
||||||
|
if ((global as any).fetch) {
|
||||||
|
((global as any).fetch as jest.Mock).mockClear();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QuestDB-specific test utilities
|
||||||
|
*/
|
||||||
|
export const questdbTestHelpers = {
|
||||||
|
/**
|
||||||
|
* Get mock PostgreSQL adapter
|
||||||
|
*/
|
||||||
|
getMockPgAdapter: () => pgMem?.adapters?.createPg?.(),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute SQL in mock database
|
||||||
|
*/
|
||||||
|
executeMockSQL: (sql: string, params?: any[]) => {
|
||||||
|
return pgMem?.public?.query(sql, params);
|
||||||
|
},
|
||||||
|
/**
|
||||||
|
* Mock successful QuestDB HTTP response
|
||||||
|
*/ mockQuestDBHttpSuccess: (data: any) => {
|
||||||
|
((global as any).fetch as jest.Mock).mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
json: async () => data,
|
||||||
|
text: async () => JSON.stringify(data)
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mock QuestDB HTTP error
|
||||||
|
*/
|
||||||
|
mockQuestDBHttpError: (status: number, message: string) => {
|
||||||
|
((global as any).fetch as jest.Mock).mockResolvedValueOnce({
|
||||||
|
ok: false,
|
||||||
|
status,
|
||||||
|
json: async () => ({ error: message }),
|
||||||
|
text: async () => message
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mock InfluxDB line protocol response
|
||||||
|
*/
|
||||||
|
mockInfluxDBSuccess: () => {
|
||||||
|
((global as any).fetch as jest.Mock).mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
status: 204,
|
||||||
|
text: async () => ''
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create test OHLCV table
|
||||||
|
*/
|
||||||
|
createTestOHLCVTable: () => {
|
||||||
|
const sql = `
|
||||||
|
CREATE TABLE ohlcv (
|
||||||
|
symbol VARCHAR(10),
|
||||||
|
timestamp TIMESTAMP,
|
||||||
|
open DECIMAL(10,2),
|
||||||
|
high DECIMAL(10,2),
|
||||||
|
low DECIMAL(10,2),
|
||||||
|
close DECIMAL(10,2),
|
||||||
|
volume BIGINT,
|
||||||
|
source VARCHAR(50)
|
||||||
|
)
|
||||||
|
`;
|
||||||
|
return pgMem?.public?.none(sql);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Insert test OHLCV data
|
||||||
|
*/
|
||||||
|
insertTestOHLCVData: (data: any[]) => {
|
||||||
|
const sql = `
|
||||||
|
INSERT INTO ohlcv (symbol, timestamp, open, high, low, close, volume, source)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||||
|
`;
|
||||||
|
|
||||||
|
return Promise.all(
|
||||||
|
data.map(row =>
|
||||||
|
pgMem?.public?.none(sql, [
|
||||||
|
row.symbol,
|
||||||
|
row.timestamp,
|
||||||
|
row.open,
|
||||||
|
row.high,
|
||||||
|
row.low,
|
||||||
|
row.close,
|
||||||
|
row.volume,
|
||||||
|
row.source || 'test'
|
||||||
|
])
|
||||||
|
)
|
||||||
|
);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate InfluxDB line protocol test data
|
||||||
|
*/
|
||||||
|
generateInfluxDBLines: (count: number = 5) => {
|
||||||
|
const lines: string[] = [];
|
||||||
|
const baseTime = Date.now() * 1000000; // Convert to nanoseconds
|
||||||
|
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const time = baseTime + (i * 60000000000); // 1 minute intervals
|
||||||
|
const price = 150 + Math.random() * 10;
|
||||||
|
|
||||||
|
lines.push(
|
||||||
|
`ohlcv,symbol=TEST open=${price},high=${price + 1},low=${price - 1},close=${price + 0.5},volume=1000i ${time}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate QuestDB query syntax
|
||||||
|
*/
|
||||||
|
validateQuestDBQuery: (query: string): boolean => {
|
||||||
|
// Basic validation for QuestDB-specific syntax
|
||||||
|
const questdbKeywords = [
|
||||||
|
'SAMPLE BY',
|
||||||
|
'LATEST BY',
|
||||||
|
'ASOF JOIN',
|
||||||
|
'SPLICE JOIN',
|
||||||
|
'LT JOIN'
|
||||||
|
];
|
||||||
|
|
||||||
|
// Check for valid SQL structure
|
||||||
|
const hasSelect = /SELECT\s+/i.test(query);
|
||||||
|
const hasFrom = /FROM\s+/i.test(query);
|
||||||
|
|
||||||
|
return hasSelect && hasFrom;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mock connection pool
|
||||||
|
*/
|
||||||
|
createMockPool: () => ({
|
||||||
|
connect: jest.fn().mockResolvedValue({
|
||||||
|
query: jest.fn().mockResolvedValue({ rows: [], rowCount: 0 }),
|
||||||
|
release: jest.fn()
|
||||||
|
}),
|
||||||
|
end: jest.fn().mockResolvedValue(undefined),
|
||||||
|
totalCount: 0,
|
||||||
|
idleCount: 0,
|
||||||
|
waitingCount: 0
|
||||||
|
})
|
||||||
|
};
|
||||||
28
libs/questdb-client/tsconfig.json
Normal file
28
libs/questdb-client/tsconfig.json
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true
|
||||||
|
},
|
||||||
|
"include": [
|
||||||
|
"src/**/*"
|
||||||
|
],
|
||||||
|
"exclude": [
|
||||||
|
"dist",
|
||||||
|
"node_modules",
|
||||||
|
"**/*.test.ts",
|
||||||
|
"**/*.spec.ts"
|
||||||
|
],
|
||||||
|
"references": [
|
||||||
|
{ "path": "../api-client" },
|
||||||
|
{ "path": "../event-bus" },
|
||||||
|
{ "path": "../http-client" },
|
||||||
|
{ "path": "../types" },
|
||||||
|
{ "path": "../utils" },
|
||||||
|
{ "path": "../config" },
|
||||||
|
{ "path": "../logger" },
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -10,8 +10,8 @@
|
||||||
"clean": "rm -rf dist",
|
"clean": "rm -rf dist",
|
||||||
"test": "jest"
|
"test": "jest"
|
||||||
}, "dependencies": {
|
}, "dependencies": {
|
||||||
"@stock-bot/types": "workspace:*",
|
"@stock-bot/types": "*",
|
||||||
"@stock-bot/config": "workspace:*",
|
"@stock-bot/config": "*",
|
||||||
"date-fns": "^2.30.0"
|
"date-fns": "^2.30.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
|
||||||
21
package.json
21
package.json
|
|
@ -8,6 +8,13 @@
|
||||||
"build": "turbo run build",
|
"build": "turbo run build",
|
||||||
"build:libs": "pwsh ./scripts/build-libs.ps1",
|
"build:libs": "pwsh ./scripts/build-libs.ps1",
|
||||||
"test": "turbo run test",
|
"test": "turbo run test",
|
||||||
|
"test:watch": "jest --watch",
|
||||||
|
"test:coverage": "jest --coverage",
|
||||||
|
"test:unit": "jest --testPathPattern=unit",
|
||||||
|
"test:integration": "jest --testPathPattern=integration",
|
||||||
|
"test:e2e": "jest --testPathPattern=e2e",
|
||||||
|
"test:libs": "turbo run test --filter=./libs/*",
|
||||||
|
"test:apps": "turbo run test --filter=./apps/*/*",
|
||||||
"lint": "turbo run lint",
|
"lint": "turbo run lint",
|
||||||
"clean": "turbo run clean",
|
"clean": "turbo run clean",
|
||||||
"start": "turbo run start",
|
"start": "turbo run start",
|
||||||
|
|
@ -33,7 +40,19 @@
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^20.12.12",
|
"@types/node": "^20.12.12",
|
||||||
"turbo": "^2.5.4",
|
"turbo": "^2.5.4",
|
||||||
"typescript": "^5.4.5"
|
"typescript": "^5.4.5",
|
||||||
|
"@types/jest": "^29.5.12",
|
||||||
|
"jest": "^29.7.0",
|
||||||
|
"ts-jest": "^29.1.2",
|
||||||
|
"@jest/globals": "^29.7.0",
|
||||||
|
"jest-extended": "^4.0.2",
|
||||||
|
"jest-mock-extended": "^3.0.5",
|
||||||
|
"@testcontainers/postgresql": "^10.7.2",
|
||||||
|
"@testcontainers/mongodb": "^10.7.2",
|
||||||
|
"mongodb-memory-server": "^9.1.6",
|
||||||
|
"pg-mem": "^2.8.1",
|
||||||
|
"supertest": "^6.3.4",
|
||||||
|
"@types/supertest": "^6.0.2"
|
||||||
},
|
},
|
||||||
"packageManager": "bun@1.1.12",
|
"packageManager": "bun@1.1.12",
|
||||||
"engines": {
|
"engines": {
|
||||||
|
|
|
||||||
208
test/integration/setup.ts
Normal file
208
test/integration/setup.ts
Normal file
|
|
@ -0,0 +1,208 @@
|
||||||
|
/**
|
||||||
|
* Integration Test Setup
|
||||||
|
*
|
||||||
|
* Sets up test containers and real database instances for integration testing.
|
||||||
|
* This file is executed before integration tests run.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { GenericContainer, StartedTestContainer } from 'testcontainers';
|
||||||
|
import { MongoMemoryServer } from 'mongodb-memory-server';
|
||||||
|
|
||||||
|
let questdbContainer: StartedTestContainer;
|
||||||
|
let postgresContainer: StartedTestContainer;
|
||||||
|
let mongoContainer: StartedTestContainer;
|
||||||
|
let mongoMemoryServer: MongoMemoryServer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Global setup for integration tests
|
||||||
|
* Starts real database containers for testing
|
||||||
|
*/
|
||||||
|
beforeAll(async () => {
|
||||||
|
console.log('🚀 Starting integration test containers...');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Start QuestDB container
|
||||||
|
console.log('📊 Starting QuestDB container...');
|
||||||
|
questdbContainer = await new GenericContainer('questdb/questdb:7.3.10')
|
||||||
|
.withExposedPorts(9000, 8812, 9009)
|
||||||
|
.withEnvironment({
|
||||||
|
'QDB_TELEMETRY_ENABLED': 'false',
|
||||||
|
'QDB_LOG_LEVEL': 'ERROR'
|
||||||
|
})
|
||||||
|
.withStartupTimeout(60000)
|
||||||
|
.start();
|
||||||
|
|
||||||
|
// Start PostgreSQL container
|
||||||
|
console.log('🐘 Starting PostgreSQL container...');
|
||||||
|
postgresContainer = await new GenericContainer('postgres:15-alpine')
|
||||||
|
.withExposedPorts(5432)
|
||||||
|
.withEnvironment({
|
||||||
|
'POSTGRES_DB': 'trading_bot_test',
|
||||||
|
'POSTGRES_USER': 'trading_admin',
|
||||||
|
'POSTGRES_PASSWORD': 'trading_pass_test'
|
||||||
|
})
|
||||||
|
.withStartupTimeout(60000)
|
||||||
|
.start();
|
||||||
|
|
||||||
|
// Start MongoDB container
|
||||||
|
console.log('🍃 Starting MongoDB container...');
|
||||||
|
mongoContainer = await new GenericContainer('mongo:7-jammy')
|
||||||
|
.withExposedPorts(27017)
|
||||||
|
.withEnvironment({
|
||||||
|
'MONGO_INITDB_ROOT_USERNAME': 'trading_admin',
|
||||||
|
'MONGO_INITDB_ROOT_PASSWORD': 'trading_mongo_test',
|
||||||
|
'MONGO_INITDB_DATABASE': 'trading_bot_test'
|
||||||
|
})
|
||||||
|
.withStartupTimeout(60000)
|
||||||
|
.start();
|
||||||
|
|
||||||
|
// Update environment variables for tests
|
||||||
|
process.env.QUESTDB_HOST = questdbContainer.getHost();
|
||||||
|
process.env.QUESTDB_HTTP_PORT = questdbContainer.getMappedPort(9000).toString();
|
||||||
|
process.env.QUESTDB_PG_PORT = questdbContainer.getMappedPort(8812).toString();
|
||||||
|
process.env.QUESTDB_INFLUX_PORT = questdbContainer.getMappedPort(9009).toString();
|
||||||
|
|
||||||
|
process.env.POSTGRES_HOST = postgresContainer.getHost();
|
||||||
|
process.env.POSTGRES_PORT = postgresContainer.getMappedPort(5432).toString();
|
||||||
|
|
||||||
|
process.env.MONGODB_HOST = mongoContainer.getHost();
|
||||||
|
process.env.MONGODB_PORT = mongoContainer.getMappedPort(27017).toString();
|
||||||
|
|
||||||
|
console.log('✅ All containers started successfully!');
|
||||||
|
console.log(`📊 QuestDB: http://${process.env.QUESTDB_HOST}:${process.env.QUESTDB_HTTP_PORT}`);
|
||||||
|
console.log(`🐘 PostgreSQL: ${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT}`);
|
||||||
|
console.log(`🍃 MongoDB: ${process.env.MONGODB_HOST}:${process.env.MONGODB_PORT}`);
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('❌ Failed to start test containers:', error);
|
||||||
|
|
||||||
|
// Try to use MongoDB Memory Server as fallback
|
||||||
|
console.log('🔄 Falling back to MongoDB Memory Server...');
|
||||||
|
try {
|
||||||
|
mongoMemoryServer = await MongoMemoryServer.create({
|
||||||
|
instance: {
|
||||||
|
dbName: 'trading_bot_test'
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const mongoUri = mongoMemoryServer.getUri();
|
||||||
|
const mongoUrl = new URL(mongoUri);
|
||||||
|
process.env.MONGODB_HOST = mongoUrl.hostname;
|
||||||
|
process.env.MONGODB_PORT = mongoUrl.port;
|
||||||
|
process.env.MONGODB_URI = mongoUri;
|
||||||
|
|
||||||
|
console.log('✅ MongoDB Memory Server started as fallback');
|
||||||
|
} catch (fallbackError) {
|
||||||
|
console.error('❌ Failed to start MongoDB Memory Server:', fallbackError);
|
||||||
|
throw fallbackError;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For other databases, use localhost defaults if containers fail
|
||||||
|
if (!questdbContainer) {
|
||||||
|
console.log('⚠️ Using localhost QuestDB (ensure it\'s running)');
|
||||||
|
process.env.QUESTDB_HOST = 'localhost';
|
||||||
|
process.env.QUESTDB_HTTP_PORT = '9000';
|
||||||
|
process.env.QUESTDB_PG_PORT = '8812';
|
||||||
|
process.env.QUESTDB_INFLUX_PORT = '9009';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!postgresContainer) {
|
||||||
|
console.log('⚠️ Using localhost PostgreSQL (ensure it\'s running)');
|
||||||
|
process.env.POSTGRES_HOST = 'localhost';
|
||||||
|
process.env.POSTGRES_PORT = '5432';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 120000); // 2 minutes timeout for container startup
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Global cleanup for integration tests
|
||||||
|
* Stops all test containers
|
||||||
|
*/
|
||||||
|
afterAll(async () => {
|
||||||
|
console.log('🧹 Cleaning up integration test containers...');
|
||||||
|
|
||||||
|
const cleanup = async (container: StartedTestContainer | undefined, name: string) => {
|
||||||
|
if (container) {
|
||||||
|
try {
|
||||||
|
await container.stop();
|
||||||
|
console.log(`✅ ${name} container stopped`);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(`⚠️ Failed to stop ${name} container:`, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
cleanup(questdbContainer, 'QuestDB'),
|
||||||
|
cleanup(postgresContainer, 'PostgreSQL'),
|
||||||
|
cleanup(mongoContainer, 'MongoDB')
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (mongoMemoryServer) {
|
||||||
|
try {
|
||||||
|
await mongoMemoryServer.stop();
|
||||||
|
console.log('✅ MongoDB Memory Server stopped');
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('⚠️ Failed to stop MongoDB Memory Server:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('🎉 Integration test cleanup complete!');
|
||||||
|
}, 30000);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for database services to be ready
|
||||||
|
*/
|
||||||
|
export const waitForServices = async (timeout: number = 30000): Promise<void> => {
|
||||||
|
const start = Date.now();
|
||||||
|
|
||||||
|
while (Date.now() - start < timeout) {
|
||||||
|
try {
|
||||||
|
// Check if QuestDB HTTP interface is ready
|
||||||
|
const questdbUrl = `http://${process.env.QUESTDB_HOST}:${process.env.QUESTDB_HTTP_PORT}/status`;
|
||||||
|
const response = await fetch(questdbUrl);
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
console.log('✅ QuestDB is ready');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Service not ready yet, continue waiting
|
||||||
|
}
|
||||||
|
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error('Services did not become ready within timeout');
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test utilities for integration tests
|
||||||
|
*/
|
||||||
|
export const integrationTestHelpers = {
|
||||||
|
/**
|
||||||
|
* Get QuestDB HTTP URL
|
||||||
|
*/
|
||||||
|
getQuestDBUrl: () => `http://${process.env.QUESTDB_HOST}:${process.env.QUESTDB_HTTP_PORT}`,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get PostgreSQL connection string
|
||||||
|
*/
|
||||||
|
getPostgresUrl: () =>
|
||||||
|
`postgresql://${process.env.POSTGRES_USER}:${process.env.POSTGRES_PASSWORD}@${process.env.POSTGRES_HOST}:${process.env.POSTGRES_PORT}/${process.env.POSTGRES_DB}`,
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get MongoDB connection string
|
||||||
|
*/
|
||||||
|
getMongoUrl: () => {
|
||||||
|
if (process.env.MONGODB_URI) {
|
||||||
|
return process.env.MONGODB_URI;
|
||||||
|
}
|
||||||
|
return `mongodb://${process.env.MONGODB_USERNAME}:${process.env.MONGODB_PASSWORD}@${process.env.MONGODB_HOST}:${process.env.MONGODB_PORT}/${process.env.MONGODB_DATABASE}`;
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait for services to be ready
|
||||||
|
*/
|
||||||
|
waitForServices
|
||||||
|
};
|
||||||
Loading…
Add table
Add a link
Reference in a new issue