boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

work up marketdatagateway

Browse source
This commit is contained in:
Bojan Kucera 2025-06-03 19:43:22 -04:00
parent 58ae897e90
commit d22f7aafa0
17 changed files with 653 additions and 494 deletions
Download patch file Download diff file Expand all files Collapse all files

16
apps/core-services/market-data-gateway/src/index.ts
View file

@ -55,11 +55,11 @@ const config: GatewayConfig = {
methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
headers: ['Content-Type', 'Authorization'],
},
},
dataSources: getEnabledProviders().map(provider => ({
}, dataSources: getEnabledProviders().map(provider => ({
id: provider.name.toLowerCase().replace(/\s+/g, '-'),
name: provider.name,
type: provider.type === 'both' ? 'websocket' : provider.type as any,
provider: provider.name,
enabled: provider.enabled,
priority: provider.priority,
rateLimit: {
@ -114,21 +114,17 @@ const config: GatewayConfig = {
candles: 86400000, // 24 hours
orderbook: 30000, // 30 seconds
},
},
monitoring: {
}, monitoring: {
metrics: {
enabled: true,
port: parseInt(process.env.METRICS_PORT || '9090'),
path: '/metrics',
},
logging: {
level: 'info',
format: 'json',
outputs: ['console'],
intervalMs: 5000,
retention: '24h',
},
alerts: {
enabled: true,
thresholds: {
latency: 1000,
latencyMs: 1000,
errorRate: 0.05,
connectionLoss: 3,

12
apps/core-services/market-data-gateway/src/index_clean.ts
View file

@ -55,21 +55,17 @@ const config: GatewayConfig = {
candles: 86400000, // 24 hours
orderbook: 30000, // 30 seconds
},
},
monitoring: {
}, monitoring: {
metrics: {
enabled: true,
port: parseInt(process.env.METRICS_PORT || '9090'),
path: '/metrics',
},
logging: {
level: 'info',
format: 'json',
outputs: ['console'],
intervalMs: 5000,
retention: '24h',
},
alerts: {
enabled: true,
thresholds: {
latency: 1000,
latencyMs: 1000,
errorRate: 0.05,
connectionLoss: 3,

380
apps/core-services/market-data-gateway/src/services/MarketDataGatewayService.ts
View file

@ -1,380 +0,0 @@
import { EventEmitter } from 'eventemitter3';
// Local logger interface to avoid pino dependency issues
interface Logger {
info(msg: string, ...args: any[]): void;
error(msg: string, ...args: any[]): void;
warn(msg: string, ...args: any[]): void;
debug(msg: string, ...args: any[]): void;
child(options: any): Logger;
}
// Simple logger implementation
const createLogger = (name: string): Logger => ({
info: (msg: string, ...args: any[]) => console.log(`[${name}] INFO:`, msg, ...args),
error: (msg: string, ...args: any[]) => console.error(`[${name}] ERROR:`, msg, ...args),
warn: (msg: string, ...args: any[]) => console.warn(`[${name}] WARN:`, msg, ...args),
debug: (msg: string, ...args: any[]) => console.debug(`[${name}] DEBUG:`, msg, ...args),
child: (options: any) => createLogger(`${name}.${options.component || 'child'}`)
});
import {
GatewayConfig,
DataSourceConfig,
ProcessingPipeline,
ClientSubscription,
SubscriptionRequest,
DataSourceMetrics,
GatewayMetrics,
MarketDataTick,
MarketDataCandle,
MarketDataTrade,
MarketDataOrder,
HealthStatus
} from '../types/MarketDataGateway';
import { DataSourceManager } from './DataSourceManager';
import { ProcessingEngine } from './ProcessingEngine';
import { SubscriptionManager } from './SubscriptionManager';
import { CacheManager } from './CacheManager';
import { MetricsCollector } from './MetricsCollector';
import { ServiceIntegrationManager } from './ServiceIntegrationManager';
export class MarketDataGatewayService extends EventEmitter {
private config: GatewayConfig;
private logger: Logger;
private dataSourceManager: DataSourceManager;
private processingEngine: ProcessingEngine;
private subscriptionManager: SubscriptionManager;
private cacheManager: CacheManager;
private metricsCollector: MetricsCollector;
private serviceIntegration: ServiceIntegrationManager;
private isRunning = false;
private startTime: Date = new Date();
constructor(config: GatewayConfig, logger: Logger) {
super();
this.config = config;
this.logger = logger;
this.initializeComponents();
this.setupEventHandlers();
}
private initializeComponents() {
this.logger.info('Initializing Market Data Gateway components');
// Initialize core components
this.dataSourceManager = new DataSourceManager(
this.config.dataSources,
this.logger.child({ component: 'DataSourceManager' })
);
this.processingEngine = new ProcessingEngine(
this.config.processing,
this.logger.child({ component: 'ProcessingEngine' })
);
this.subscriptionManager = new SubscriptionManager(
this.logger.child({ component: 'SubscriptionManager' })
);
this.cacheManager = new CacheManager(
this.config.cache,
this.logger.child({ component: 'CacheManager' })
);
this.metricsCollector = new MetricsCollector(
this.logger.child({ component: 'MetricsCollector' })
);
this.serviceIntegration = new ServiceIntegrationManager(
this.logger.child({ component: 'ServiceIntegration' })
);
}
private setupEventHandlers() {
// Data source events
this.dataSourceManager.on('data', this.handleIncomingData.bind(this));
this.dataSourceManager.on('error', this.handleDataSourceError.bind(this));
this.dataSourceManager.on('connected', this.handleDataSourceConnected.bind(this));
this.dataSourceManager.on('disconnected', this.handleDataSourceDisconnected.bind(this));
// Processing engine events
this.processingEngine.on('processed', this.handleProcessedData.bind(this));
this.processingEngine.on('error', this.handleProcessingError.bind(this));
// Subscription events
this.subscriptionManager.on('subscribed', this.handleClientSubscribed.bind(this));
this.subscriptionManager.on('unsubscribed', this.handleClientUnsubscribed.bind(this));
this.subscriptionManager.on('error', this.handleSubscriptionError.bind(this));
// Cache events
this.cacheManager.on('cached', this.handleDataCached.bind(this));
this.cacheManager.on('error', this.handleCacheError.bind(this));
// Service integration events
this.serviceIntegration.on('data-forwarded', this.handleDataForwarded.bind(this));
this.serviceIntegration.on('integration-error', this.handleIntegrationError.bind(this));
}
public async start(): Promise<void> {
if (this.isRunning) {
this.logger.warn('Gateway is already running');
return;
}
try {
this.logger.info('Starting Market Data Gateway');
this.startTime = new Date();
// Start components in order
await this.cacheManager.start();
await this.metricsCollector.start();
await this.serviceIntegration.start();
await this.processingEngine.start();
await this.subscriptionManager.start();
await this.dataSourceManager.start();
this.isRunning = true;
this.logger.info('Market Data Gateway started successfully');
this.emit('started');
} catch (error) {
this.logger.error({ error }, 'Failed to start Market Data Gateway');
await this.stop();
throw error;
}
}
public async stop(): Promise<void> {
if (!this.isRunning) {
return;
}
try {
this.logger.info('Stopping Market Data Gateway');
// Stop components in reverse order
await this.dataSourceManager.stop();
await this.subscriptionManager.stop();
await this.processingEngine.stop();
await this.serviceIntegration.stop();
await this.metricsCollector.stop();
await this.cacheManager.stop();
this.isRunning = false;
this.logger.info('Market Data Gateway stopped');
this.emit('stopped');
} catch (error) {
this.logger.error({ error }, 'Error stopping Market Data Gateway');
throw error;
}
}
// Data handling methods
private async handleIncomingData(sourceId: string, data: any): Promise<void> {
try {
this.metricsCollector.recordMessage(sourceId, 'received');
// Process data through pipeline
const processedData = await this.processingEngine.process(data);
// Cache processed data
await this.cacheManager.cache(processedData);
// Forward to subscribers
await this.subscriptionManager.broadcast(processedData);
// Forward to integrated services
await this.serviceIntegration.forwardData(processedData);
this.emit('data-processed', { sourceId, data: processedData });
} catch (error) {
this.logger.error({ error, sourceId, data }, 'Error handling incoming data');
this.metricsCollector.recordError(sourceId);
}
}
private async handleProcessedData(data: any): Promise<void> {
this.logger.debug({ data }, 'Data processed successfully');
this.metricsCollector.recordMessage('processing', 'processed');
}
private handleDataSourceError(sourceId: string, error: Error): void {
this.logger.error({ sourceId, error }, 'Data source error');
this.metricsCollector.recordError(sourceId);
this.emit('source-error', { sourceId, error });
}
private handleDataSourceConnected(sourceId: string): void {
this.logger.info({ sourceId }, 'Data source connected');
this.metricsCollector.recordConnection(sourceId, 'connected');
}
private handleDataSourceDisconnected(sourceId: string): void {
this.logger.warn({ sourceId }, 'Data source disconnected');
this.metricsCollector.recordConnection(sourceId, 'disconnected');
}
private handleProcessingError(error: Error, data: any): void {
this.logger.error({ error, data }, 'Processing error');
this.emit('processing-error', { error, data });
}
private handleClientSubscribed(subscription: ClientSubscription): void {
this.logger.info({
clientId: subscription.request.clientId,
symbols: subscription.request.symbols
}, 'Client subscribed');
}
private handleClientUnsubscribed(clientId: string): void {
this.logger.info({ clientId }, 'Client unsubscribed');
}
private handleSubscriptionError(error: Error, clientId: string): void {
this.logger.error({ error, clientId }, 'Subscription error');
}
private handleDataCached(key: string, data: any): void {
this.logger.debug({ key }, 'Data cached');
}
private handleCacheError(error: Error, operation: string): void {
this.logger.error({ error, operation }, 'Cache error');
}
private handleDataForwarded(service: string, data: any): void {
this.logger.debug({ service }, 'Data forwarded to service');
}
private handleIntegrationError(service: string, error: Error): void {
this.logger.error({ service, error }, 'Service integration error');
}
// Public API methods
public async subscribe(request: SubscriptionRequest): Promise<string> {
return this.subscriptionManager.subscribe(request);
}
public async unsubscribe(subscriptionId: string): Promise<void> {
return this.subscriptionManager.unsubscribe(subscriptionId);
}
public async getSubscriptions(clientId?: string): Promise<ClientSubscription[]> {
return this.subscriptionManager.getSubscriptions(clientId);
}
public async addDataSource(config: DataSourceConfig): Promise<void> {
return this.dataSourceManager.addDataSource(config);
}
public async removeDataSource(sourceId: string): Promise<void> {
return this.dataSourceManager.removeDataSource(sourceId);
}
public async updateDataSource(sourceId: string, config: Partial<DataSourceConfig>): Promise<void> {
return this.dataSourceManager.updateDataSource(sourceId, config);
}
public async getDataSources(): Promise<DataSourceConfig[]> {
return this.dataSourceManager.getDataSources();
}
public async addProcessingPipeline(pipeline: ProcessingPipeline): Promise<void> {
return this.processingEngine.addPipeline(pipeline);
}
public async removeProcessingPipeline(pipelineId: string): Promise<void> {
return this.processingEngine.removePipeline(pipelineId);
}
public async getProcessingPipelines(): Promise<ProcessingPipeline[]> {
return this.processingEngine.getPipelines();
}
public async getMetrics(): Promise<GatewayMetrics> {
return this.metricsCollector.getMetrics();
}
public async getDataSourceMetrics(sourceId?: string): Promise<DataSourceMetrics[]> {
return this.metricsCollector.getDataSourceMetrics(sourceId);
}
public async getHealthStatus(): Promise<HealthStatus> {
const metrics = await this.getMetrics();
const dataSources = await this.getDataSources();
// Check component health
const dependencies = [
{
name: 'cache',
status: await this.cacheManager.isHealthy() ? 'healthy' : 'unhealthy' as const
},
{
name: 'processing-engine',
status: this.processingEngine.isHealthy() ? 'healthy' : 'unhealthy' as const
},
{
name: 'data-sources',
status: dataSources.every(ds => ds.enabled) ? 'healthy' : 'unhealthy' as const
}
];
const hasUnhealthyDependencies = dependencies.some(dep => dep.status === 'unhealthy');
return {
service: 'market-data-gateway',
status: hasUnhealthyDependencies ? 'degraded' : 'healthy',
timestamp: new Date(),
uptime: Date.now() - this.startTime.getTime(),
version: process.env.SERVICE_VERSION || '1.0.0',
dependencies,
metrics: {
connectionsActive: metrics.subscriptions.active,
messagesPerSecond: metrics.processing.messagesPerSecond,
errorRate: metrics.processing.errorRate,
avgLatencyMs: metrics.dataSources.reduce((sum, ds) => sum + ds.latency.avgMs, 0) / metrics.dataSources.length || 0
}
};
}
// Cache operations
public async getCachedData(key: string): Promise<any> {
return this.cacheManager.get(key);
}
public async setCachedData(key: string, data: any, ttl?: number): Promise<void> {
return this.cacheManager.set(key, data, ttl);
}
// Configuration management
public getConfig(): GatewayConfig {
return { ...this.config };
}
public async updateConfig(updates: Partial<GatewayConfig>): Promise<void> {
this.config = { ...this.config, ...updates };
this.logger.info('Gateway configuration updated');
// Notify components of config changes
if (updates.dataSources) {
await this.dataSourceManager.updateConfig(updates.dataSources);
}
if (updates.processing) {
await this.processingEngine.updateConfig(updates.processing);
}
this.emit('config-updated', this.config);
}
// Utility methods
public isRunning(): boolean {
return this.isRunning;
}
public getUptime(): number {
return Date.now() - this.startTime.getTime();
}
}

404
apps/core-services/market-data-gateway/src/services/MarketDataGatewayService.ts.backup Normal file
View file

@ -0,0 +1,404 @@
import { EventEmitter } from 'eventemitter3';
// Local logger interface to avoid pino dependency issues
interface Logger {
info(msg: string | object, ...args: any[]): void;
error(msg: string | object, ...args: any[]): void;
warn(msg: string | object, ...args: any[]): void;
debug(msg: string | object, ...args: any[]): void;
child(options: any): Logger;
}
// Simple logger implementation
const createLogger = (name: string): Logger => ({
info: (msg: string | object, ...args: any[]) => {
if (typeof msg === 'object') {
console.log(`[${name}] INFO:`, JSON.stringify(msg), ...args);
} else {
console.log(`[${name}] INFO:`, msg, ...args);
}
},
error: (msg: string | object, ...args: any[]) => {
if (typeof msg === 'object') {
console.error(`[${name}] ERROR:`, JSON.stringify(msg), ...args);
} else {
console.error(`[${name}] ERROR:`, msg, ...args);
}
},
warn: (msg: string | object, ...args: any[]) => {
if (typeof msg === 'object') {
console.warn(`[${name}] WARN:`, JSON.stringify(msg), ...args);
} else {
console.warn(`[${name}] WARN:`, msg, ...args);
}
},
debug: (msg: string | object, ...args: any[]) => {
if (typeof msg === 'object') {
console.debug(`[${name}] DEBUG:`, JSON.stringify(msg), ...args);
} else {
console.debug(`[${name}] DEBUG:`, msg, ...args);
}
},
child: (options: any) => createLogger(`${name}.${options.component || 'child'}`)
});
import {
GatewayConfig,
DataSourceConfig,
ProcessingPipeline,
ClientSubscription,
SubscriptionRequest,
DataSourceMetrics,
GatewayMetrics,
MarketDataTick,
MarketDataCandle,
MarketDataTrade,
MarketDataOrder,
HealthStatus
} from '../types/MarketDataGateway';
import { DataSourceManager } from './DataSourceManager';
import { ProcessingEngine } from './ProcessingEngine';
import { SubscriptionManager } from './SubscriptionManager';
import { CacheManager } from './CacheManager';
import { MetricsCollector } from './MetricsCollector';
import { ServiceIntegrationManager } from './ServiceIntegrationManager';
export class MarketDataGatewayService extends EventEmitter {
private config: GatewayConfig;
private logger: Logger;
private dataSourceManager!: DataSourceManager;
private processingEngine!: ProcessingEngine;
private subscriptionManager!: SubscriptionManager;
private cacheManager!: CacheManager;
private metricsCollector!: MetricsCollector;
private serviceIntegration!: ServiceIntegrationManager;
private _isRunning = false;
private startTime: Date = new Date();
constructor(config: GatewayConfig, logger: Logger) {
super();
this.config = config;
this.logger = logger;
this.initializeComponents();
this.setupEventHandlers();
}
private initializeComponents() {
this.logger.info('Initializing Market Data Gateway components');
// Initialize core components
this.dataSourceManager = new DataSourceManager(
this.config.dataSources,
this.logger.child({ component: 'DataSourceManager' })
);
this.processingEngine = new ProcessingEngine(
this.config.processing,
this.logger.child({ component: 'ProcessingEngine' })
);
this.subscriptionManager = new SubscriptionManager(
this.logger.child({ component: 'SubscriptionManager' })
);
this.cacheManager = new CacheManager(
this.config.cache,
this.logger.child({ component: 'CacheManager' })
);
this.metricsCollector = new MetricsCollector(
this.logger.child({ component: 'MetricsCollector' })
);
this.serviceIntegration = new ServiceIntegrationManager(
this.logger.child({ component: 'ServiceIntegration' })
);
}
private setupEventHandlers() {
// Data source events
this.dataSourceManager.on('data', this.handleIncomingData.bind(this));
this.dataSourceManager.on('error', this.handleDataSourceError.bind(this));
this.dataSourceManager.on('connected', this.handleDataSourceConnected.bind(this));
this.dataSourceManager.on('disconnected', this.handleDataSourceDisconnected.bind(this));
// Processing engine events
this.processingEngine.on('processed', this.handleProcessedData.bind(this));
this.processingEngine.on('error', this.handleProcessingError.bind(this));
// Subscription events
this.subscriptionManager.on('subscribed', this.handleClientSubscribed.bind(this));
this.subscriptionManager.on('unsubscribed', this.handleClientUnsubscribed.bind(this));
this.subscriptionManager.on('error', this.handleSubscriptionError.bind(this));
// Cache events
this.cacheManager.on('cached', this.handleDataCached.bind(this));
this.cacheManager.on('error', this.handleCacheError.bind(this));
// Service integration events
this.serviceIntegration.on('data-forwarded', this.handleDataForwarded.bind(this));
this.serviceIntegration.on('integration-error', this.handleIntegrationError.bind(this));
}
public async start(): Promise<void> {
if (this.isRunning) {
this.logger.warn('Gateway is already running');
return;
}
try {
this.logger.info('Starting Market Data Gateway');
this.startTime = new Date();
// Start components in order
await this.cacheManager.start();
await this.metricsCollector.start();
await this.serviceIntegration.start();
await this.processingEngine.start();
await this.subscriptionManager.start();
await this.dataSourceManager.start();
this.isRunning = true;
this.logger.info('Market Data Gateway started successfully');
this.emit('started');
} catch (error) {
this.logger.error({ error }, 'Failed to start Market Data Gateway');
await this.stop();
throw error;
}
}
public async stop(): Promise<void> {
if (!this.isRunning) {
return;
}
try {
this.logger.info('Stopping Market Data Gateway');
// Stop components in reverse order
await this.dataSourceManager.stop();
await this.subscriptionManager.stop();
await this.processingEngine.stop();
await this.serviceIntegration.stop();
await this.metricsCollector.stop();
await this.cacheManager.stop();
this.isRunning = false;
this.logger.info('Market Data Gateway stopped');
this.emit('stopped');
} catch (error) {
this.logger.error({ error }, 'Error stopping Market Data Gateway');
throw error;
}
}
// Data handling methods
private async handleIncomingData(sourceId: string, data: any): Promise<void> {
try {
this.metricsCollector.recordMessage(sourceId, 'received');
// Process data through pipeline
const processedData = await this.processingEngine.process(data);
// Cache processed data
await this.cacheManager.cache(processedData);
// Forward to subscribers
await this.subscriptionManager.broadcast(processedData);
// Forward to integrated services
await this.serviceIntegration.forwardData(processedData);
this.emit('data-processed', { sourceId, data: processedData });
} catch (error) {
this.logger.error({ error, sourceId, data }, 'Error handling incoming data');
this.metricsCollector.recordError(sourceId);
}
}
private async handleProcessedData(data: any): Promise<void> {
this.logger.debug({ data }, 'Data processed successfully');
this.metricsCollector.recordMessage('processing', 'processed');
}
private handleDataSourceError(sourceId: string, error: Error): void {
this.logger.error({ sourceId, error }, 'Data source error');
this.metricsCollector.recordError(sourceId);
this.emit('source-error', { sourceId, error });
}
private handleDataSourceConnected(sourceId: string): void {
this.logger.info({ sourceId }, 'Data source connected');
this.metricsCollector.recordConnection(sourceId, 'connected');
}
private handleDataSourceDisconnected(sourceId: string): void {
this.logger.warn({ sourceId }, 'Data source disconnected');
this.metricsCollector.recordConnection(sourceId, 'disconnected');
}
private handleProcessingError(error: Error, data: any): void {
this.logger.error({ error, data }, 'Processing error');
this.emit('processing-error', { error, data });
}
private handleClientSubscribed(subscription: ClientSubscription): void {
this.logger.info({
clientId: subscription.request.clientId,
symbols: subscription.request.symbols
}, 'Client subscribed');
}
private handleClientUnsubscribed(clientId: string): void {
this.logger.info({ clientId }, 'Client unsubscribed');
}
private handleSubscriptionError(error: Error, clientId: string): void {
this.logger.error({ error, clientId }, 'Subscription error');
}
private handleDataCached(key: string, data: any): void {
this.logger.debug({ key }, 'Data cached');
}
private handleCacheError(error: Error, operation: string): void {
this.logger.error({ error, operation }, 'Cache error');
}
private handleDataForwarded(service: string, data: any): void {
this.logger.debug({ service }, 'Data forwarded to service');
}
private handleIntegrationError(service: string, error: Error): void {
this.logger.error({ service, error }, 'Service integration error');
}
// Public API methods
public async subscribe(request: SubscriptionRequest): Promise<string> {
return this.subscriptionManager.subscribe(request);
}
public async unsubscribe(subscriptionId: string): Promise<void> {
return this.subscriptionManager.unsubscribe(subscriptionId);
}
public async getSubscriptions(clientId?: string): Promise<ClientSubscription[]> {
return this.subscriptionManager.getSubscriptions(clientId);
}
public async addDataSource(config: DataSourceConfig): Promise<void> {
return this.dataSourceManager.addDataSource(config);
}
public async removeDataSource(sourceId: string): Promise<void> {
return this.dataSourceManager.removeDataSource(sourceId);
}
public async updateDataSource(sourceId: string, config: Partial<DataSourceConfig>): Promise<void> {
return this.dataSourceManager.updateDataSource(sourceId, config);
}
public async getDataSources(): Promise<DataSourceConfig[]> {
return this.dataSourceManager.getDataSources();
}
public async addProcessingPipeline(pipeline: ProcessingPipeline): Promise<void> {
return this.processingEngine.addPipeline(pipeline);
}
public async removeProcessingPipeline(pipelineId: string): Promise<void> {
return this.processingEngine.removePipeline(pipelineId);
}
public async getProcessingPipelines(): Promise<ProcessingPipeline[]> {
return this.processingEngine.getPipelines();
}
public async getMetrics(): Promise<GatewayMetrics> {
return this.metricsCollector.getMetrics();
}
public async getDataSourceMetrics(sourceId?: string): Promise<DataSourceMetrics[]> {
return this.metricsCollector.getDataSourceMetrics(sourceId);
}
public async getHealthStatus(): Promise<HealthStatus> {
const metrics = await this.getMetrics();
const dataSources = await this.getDataSources();
// Check component health
const dependencies = [
{
name: 'cache',
status: await this.cacheManager.isHealthy() ? 'healthy' : 'unhealthy' as const
},
{
name: 'processing-engine',
status: this.processingEngine.isHealthy() ? 'healthy' : 'unhealthy' as const
},
{
name: 'data-sources',
status: dataSources.every(ds => ds.enabled) ? 'healthy' : 'unhealthy' as const
}
];
const hasUnhealthyDependencies = dependencies.some(dep => dep.status === 'unhealthy');
return {
service: 'market-data-gateway',
status: hasUnhealthyDependencies ? 'degraded' : 'healthy',
timestamp: new Date(),
uptime: Date.now() - this.startTime.getTime(),
version: process.env.SERVICE_VERSION || '1.0.0',
dependencies,
metrics: {
connectionsActive: metrics.subscriptions.active,
messagesPerSecond: metrics.processing.messagesPerSecond,
errorRate: metrics.processing.errorRate,
avgLatencyMs: metrics.dataSources.reduce((sum, ds) => sum + ds.latency.avgMs, 0) / metrics.dataSources.length || 0
}
};
}
// Cache operations
public async getCachedData(key: string): Promise<any> {
return this.cacheManager.get(key);
}
public async setCachedData(key: string, data: any, ttl?: number): Promise<void> {
return this.cacheManager.set(key, data, ttl);
}
// Configuration management
public getConfig(): GatewayConfig {
return { ...this.config };
}
public async updateConfig(updates: Partial<GatewayConfig>): Promise<void> {
this.config = { ...this.config, ...updates };
this.logger.info('Gateway configuration updated');
// Notify components of config changes
if (updates.dataSources) {
await this.dataSourceManager.updateConfig(updates.dataSources);
}
if (updates.processing) {
await this.processingEngine.updateConfig(updates.processing);
}
this.emit('config-updated', this.config);
}
// Utility methods
public isRunning(): boolean {
return this.isRunning;
}
public getUptime(): number {
return Date.now() - this.startTime.getTime();
}
}

144
apps/core-services/market-data-gateway/src/types/MarketDataGateway.ts
View file

@ -66,6 +66,7 @@ export interface DataSourceConfig {
id: string;
name: string;
type: 'websocket' | 'rest' | 'fix' | 'stream';
provider: string;
enabled: boolean;
priority: number;
rateLimit: {
@ -131,35 +132,8 @@ export interface ProcessingPipeline {
};
}
// Data Processing Pipeline
export interface DataProcessor {
id: string;
name: string;
type: 'enrichment' | 'validation' | 'normalization' | 'aggregation' | 'filter';
enabled: boolean;
priority: number;
config: Record<string, any>;
process(data: MarketDataTick | MarketDataCandle | MarketDataTrade): Promise<any>;
}
export interface ProcessingPipeline {
id: string;
name: string;
processors: DataProcessor[];
inputFilter: {
symbols?: string[];
sources?: string[];
dataTypes?: string[];
};
outputTargets: {
eventBus?: boolean;
database?: boolean;
cache?: boolean;
websocket?: boolean;
dataProcessor?: boolean;
featureStore?: boolean;
};
}
// ProcessingPipelineConfig is an alias for ProcessingPipeline
export type ProcessingPipelineConfig = ProcessingPipeline;
// Subscription Management
export interface SubscriptionRequest {
@ -228,10 +202,10 @@ export interface GatewayConfig {
candles: number;
orderbook: number;
};
};
monitoring: {
}; monitoring: {
metrics: {
enabled: boolean;
port: number;
intervalMs: number;
retention: string;
};
@ -240,6 +214,7 @@ export interface GatewayConfig {
thresholds: {
errorRate: number;
latency: number;
latencyMs: number;
connectionLoss: number;
};
};
@ -320,6 +295,7 @@ export interface WebSocketSubscribeMessage extends WebSocketMessage {
export interface WebSocketDataMessage extends WebSocketMessage {
type: 'data';
payload: MarketDataTick | MarketDataTrade | MarketDataCandle | MarketDataOrder;
dataType?: string;
}
// Error Types
@ -342,3 +318,109 @@ export interface MarketDataEvent {
data: MarketDataTick | MarketDataTrade | MarketDataCandle | MarketDataOrder;
metadata?: Record<string, any>;
}
// Processing and Integration Types
export interface ProcessingError {
code: string;
message: string;
timestamp: Date;
data?: any;
source?: string;
}
export interface ServiceIntegration {
serviceName: string;
endpoint: string;
enabled: boolean;
config: Record<string, any>;
dataProcessor: {
enabled: boolean;
endpoint: string;
timeout: number;
retries: number;
};
featureStore: {
enabled: boolean;
endpoint: string;
timeout: number;
retries: number;
};
dataCatalog: {
enabled: boolean;
endpoint: string;
timeout: number;
retries: number;
};
}
export interface Logger {
info(message: string, ...args: any[]): void;
error(message: string, ...args: any[]): void;
warn(message: string, ...args: any[]): void;
debug(message: string, ...args: any[]): void;
}
export interface ProcessedData {
source: string;
timestamp: Date;
data: any;
processedAt: Date;
metadata?: Record<string, any>;
}
export interface DataPipelineJob {
id: string;
type: string;
status: 'pending' | 'running' | 'completed' | 'failed';
data: any;
createdAt: Date;
startedAt?: Date;
completedAt?: Date;
}
export interface FeatureComputationRequest {
featureGroupId: string;
features: string[];
data: any;
timestamp: Date;
metadata?: Record<string, any>;
}
export interface DataAsset {
id: string;
name: string;
type: string;
source: string;
metadata: Record<string, any>;
createdAt: Date;
updatedAt: Date;
}
// Add missing types
export interface CacheConfig {
redis: {
host: string;
port: number;
password?: string;
db: number;
};
ttl: {
quotes: number;
trades: number;
candles: number;
orderbook: number;
};
}
export interface ProcessingMetrics {
totalProcessed: number;
processedPerSecond: number;
processingLatency: number;
errorCount: number;
}
export interface SubscriptionMetrics {
totalSubscriptions: number;
messagesSent: number;
sendRate: number;
}

28
apps/core-services/market-data-gateway/tsconfig.json
View file

@ -2,11 +2,31 @@
"extends": "../../../tsconfig.json",
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src",
"module": "ESNext",
"moduleResolution": "bundler",
"types": ["bun-types"]
"types": ["bun-types"],
"baseUrl": "../../../",
"paths": {
"@stock-bot/*": ["libs/*/src", "libs/*/dist"]
},
"rootDir": "../../../"
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
"include": [
"src/**/*",
"../../../libs/*/src/**/*"
],
"exclude": [
"node_modules",
"dist",
"../../../libs/*/examples/**/*",
"../../../libs/**/*.test.ts",
"../../../libs/**/*.spec.ts"
],
"references": [
{ "path": "../../../libs/config" },
{ "path": "../../../libs/types" },
{ "path": "../../../libs/logger" },
{ "path": "../../../libs/http-client" },
{ "path": "../../../libs/event-bus" }
]
}