deleted a lot of the stuff

This commit is contained in:
Bojan Kucera 2025-06-04 10:50:05 -04:00
parent d22f7aafa0
commit 3e451558ac
173 changed files with 1313 additions and 30205 deletions

View file

@ -0,0 +1,6 @@
// Data Processing and Pipeline Types
export * from './sources';
export * from './processors';
export * from './pipelines';
export * from './transformations';
export * from './quality';

View file

@ -0,0 +1,181 @@
import { AuthenticationConfig } from '../config/security';
import { ProcessingError } from './processors';
// Data Pipeline Types
export interface ProcessingPipeline {
id: string;
name: string;
description?: string;
version: string;
processors: PipelineProcessor[];
inputFilter: InputFilter;
outputTargets: OutputTargets;
schedule?: PipelineSchedule;
status: PipelineStatus;
metadata: PipelineMetadata;
}
export interface PipelineProcessor {
processorId: string;
order: number;
config?: Record<string, any>;
conditions?: ProcessingCondition[];
parallelBranches?: PipelineProcessor[][];
}
export interface ProcessingCondition {
field: string;
operator: 'equals' | 'not_equals' | 'contains' | 'regex' | 'greater_than' | 'less_than';
value: any;
logicalOperator?: 'AND' | 'OR';
}
export interface InputFilter {
symbols?: string[];
sources?: string[];
dataTypes?: string[];
timeRange?: TimeRange;
conditions?: ProcessingCondition[];
}
export interface TimeRange {
start?: Date;
end?: Date;
duration?: string;
timezone?: string;
}
export interface OutputTargets {
eventBus?: boolean;
database?: DatabaseTarget;
cache?: CacheTarget;
websocket?: WebSocketTarget;
dataProcessor?: ServiceTarget;
featureStore?: ServiceTarget;
file?: FileTarget;
api?: ApiTarget;
}
export interface DatabaseTarget {
enabled: boolean;
connectionId: string;
table: string;
batchSize?: number;
flushInterval?: number;
}
export interface CacheTarget {
enabled: boolean;
connectionId: string;
keyPattern: string;
ttl?: number;
compression?: boolean;
}
export interface WebSocketTarget {
enabled: boolean;
rooms?: string[];
filters?: Record<string, any>;
}
export interface ServiceTarget {
enabled: boolean;
serviceId: string;
endpoint: string;
batchSize?: number;
timeout?: number;
}
export interface FileTarget {
enabled: boolean;
path: string;
format: 'json' | 'csv' | 'parquet' | 'avro';
compression?: 'gzip' | 'bzip2' | 'lz4';
rotation?: FileRotation;
}
export interface FileRotation {
size?: string;
time?: string;
count?: number;
}
export interface ApiTarget {
enabled: boolean;
url: string;
method: string;
headers?: Record<string, string>;
authentication?: AuthenticationConfig;
}
export interface PipelineSchedule {
enabled: boolean;
cronExpression?: string;
interval?: string;
timezone?: string;
startDate?: Date;
endDate?: Date;
}
export enum PipelineStatus {
DRAFT = 'draft',
ACTIVE = 'active',
PAUSED = 'paused',
ERROR = 'error',
COMPLETED = 'completed',
ARCHIVED = 'archived'
}
export interface PipelineMetadata {
createdBy: string;
createdAt: Date;
updatedBy: string;
updatedAt: Date;
tags: string[];
documentation?: string;
changeLog?: ChangeLogEntry[];
}
export interface ChangeLogEntry {
version: string;
timestamp: Date;
author: string;
changes: string[];
breaking: boolean;
}
export interface PipelineExecution {
id: string;
pipelineId: string;
status: ExecutionStatus;
startTime: Date;
endTime?: Date;
recordsProcessed: number;
recordsSuccess: number;
recordsError: number;
errors: ProcessingError[];
metadata: ExecutionMetadata;
}
export enum ExecutionStatus {
PENDING = 'pending',
RUNNING = 'running',
COMPLETED = 'completed',
FAILED = 'failed',
CANCELLED = 'cancelled',
TIMEOUT = 'timeout'
}
export interface ExecutionMetadata {
triggeredBy: 'schedule' | 'manual' | 'event';
trigger?: string;
parameters?: Record<string, any>;
resources?: ResourceUsage;
}
export interface ResourceUsage {
cpuTimeMs: number;
memoryPeakMB: number;
diskIOBytes: number;
networkIOBytes: number;
}

View file

@ -0,0 +1,103 @@
// Data Processing Types
export interface DataProcessor {
id: string;
name: string;
type: ProcessorType;
enabled: boolean;
priority: number;
config: ProcessorConfig;
inputSchema?: DataSchema;
outputSchema?: DataSchema;
metrics: ProcessorMetrics;
}
export enum ProcessorType {
ENRICHMENT = 'enrichment',
VALIDATION = 'validation',
NORMALIZATION = 'normalization',
AGGREGATION = 'aggregation',
FILTER = 'filter',
TRANSFORMATION = 'transformation',
QUALITY_CHECK = 'quality_check',
DEDUPLICATION = 'deduplication'
}
export interface ProcessorConfig {
batchSize?: number;
timeoutMs?: number;
retryAttempts?: number;
parallelism?: number;
parameters?: Record<string, any>;
errorHandling?: ErrorHandlingConfig;
}
export interface ErrorHandlingConfig {
strategy: 'fail_fast' | 'skip_invalid' | 'retry' | 'quarantine';
maxErrors?: number;
quarantineLocation?: string;
notificationChannels?: string[];
}
export interface ProcessorMetrics {
recordsProcessed: number;
recordsSuccess: number;
recordsError: number;
avgProcessingTimeMs: number;
lastProcessed: Date;
errorRate: number;
throughputPerSecond: number;
}
export interface ProcessingError {
processorId: string;
pipelineId: string;
timestamp: Date;
inputData: any;
error: string;
stackTrace?: string;
severity: 'low' | 'medium' | 'high' | 'critical';
recoverable: boolean;
}
export interface DataSchema {
version: string;
fields: SchemaField[];
primaryKeys?: string[];
foreignKeys?: ForeignKey[];
indexes?: Index[];
constraints?: SchemaConstraint[];
}
export interface SchemaField {
name: string;
type: string;
nullable: boolean;
description?: string;
defaultValue?: any;
format?: string;
pattern?: string;
enum?: any[];
}
export interface ForeignKey {
fields: string[];
referencedTable: string;
referencedFields: string[];
onDelete?: 'cascade' | 'restrict' | 'set_null';
onUpdate?: 'cascade' | 'restrict' | 'set_null';
}
export interface Index {
name: string;
fields: string[];
unique: boolean;
type: 'btree' | 'hash' | 'gin' | 'gist';
}
export interface SchemaConstraint {
name: string;
type: 'check' | 'unique' | 'not_null' | 'range';
fields: string[];
condition?: string;
parameters?: Record<string, any>;
}

View file

@ -0,0 +1,193 @@
// Data Quality Types
export interface DataQuality {
id: string;
assetId: string;
overallScore: number;
dimensions: QualityDimension[];
rules: QualityRule[];
issues: QualityIssue[];
trend: QualityTrend;
lastAssessment: Date;
nextAssessment?: Date;
}
export interface QualityDimension {
name: QualityDimensionType;
score: number;
weight: number;
description: string;
metrics: QualityMetric[];
status: 'pass' | 'warn' | 'fail';
}
export enum QualityDimensionType {
COMPLETENESS = 'completeness',
ACCURACY = 'accuracy',
CONSISTENCY = 'consistency',
VALIDITY = 'validity',
UNIQUENESS = 'uniqueness',
TIMELINESS = 'timeliness',
INTEGRITY = 'integrity',
CONFORMITY = 'conformity',
RELEVANCE = 'relevance'
}
export interface QualityRule {
id: string;
name: string;
description: string;
dimension: QualityDimensionType;
type: QualityRuleType;
field?: string;
condition: string;
threshold: number;
severity: 'low' | 'medium' | 'high' | 'critical';
enabled: boolean;
metadata?: Record<string, any>;
}
export enum QualityRuleType {
NULL_CHECK = 'null_check',
RANGE_CHECK = 'range_check',
PATTERN_CHECK = 'pattern_check',
REFERENCE_CHECK = 'reference_check',
DUPLICATE_CHECK = 'duplicate_check',
FRESHNESS_CHECK = 'freshness_check',
OUTLIER_CHECK = 'outlier_check',
CUSTOM = 'custom'
}
export interface QualityMetric {
name: string;
value: number;
unit?: string;
threshold?: number;
status: 'pass' | 'warn' | 'fail';
timestamp: Date;
trend?: 'improving' | 'stable' | 'degrading';
}
export interface QualityIssue {
id: string;
ruleId: string;
severity: 'low' | 'medium' | 'high' | 'critical';
description: string;
field?: string;
affectedRows?: number;
sampleValues?: any[];
detectedAt: Date;
status: 'open' | 'acknowledged' | 'resolved' | 'false_positive';
assignee?: string;
resolution?: string;
resolvedAt?: Date;
impact?: QualityImpact;
}
export interface QualityImpact {
scope: 'field' | 'table' | 'dataset' | 'system';
affectedRecords: number;
downstreamAssets: string[];
businessImpact: 'low' | 'medium' | 'high' | 'critical';
estimatedCost?: number;
}
export interface QualityTrend {
timeframe: 'day' | 'week' | 'month' | 'quarter';
dataPoints: QualityDataPoint[];
trend: 'improving' | 'stable' | 'degrading';
changeRate: number;
projectedScore?: number;
}
export interface QualityDataPoint {
timestamp: Date;
score: number;
dimensionScores: Record<QualityDimensionType, number>;
issueCount: number;
rulesPassed: number;
rulesTotal: number;
}
export interface QualityProfile {
assetId: string;
fieldProfiles: FieldProfile[];
rowCount: number;
columnCount: number;
dataTypes: Record<string, string>;
nullCounts: Record<string, number>;
uniqueCounts: Record<string, number>;
profiledAt: Date;
}
export interface FieldProfile {
fieldName: string;
dataType: string;
nullCount: number;
uniqueCount: number;
minValue?: any;
maxValue?: any;
avgValue?: number;
medianValue?: number;
standardDeviation?: number;
topValues?: ValueFrequency[];
histogram?: HistogramBucket[];
}
export interface ValueFrequency {
value: any;
count: number;
percentage: number;
}
export interface HistogramBucket {
min: number;
max: number;
count: number;
percentage: number;
}
export interface QualityReport {
id: string;
assetId: string;
reportType: 'summary' | 'detailed' | 'trend' | 'comparison';
generatedAt: Date;
period: string;
summary: QualitySummary;
details?: QualityDetails;
recommendations?: QualityRecommendation[];
}
export interface QualitySummary {
overallScore: number;
scoreTrend: 'improving' | 'stable' | 'degrading';
totalIssues: number;
criticalIssues: number;
resolvedIssues: number;
rulesPassed: number;
rulesTotal: number;
}
export interface QualityDetails {
dimensionBreakdown: Record<QualityDimensionType, QualityDimension>;
topIssues: QualityIssue[];
ruleResults: QualityRuleResult[];
historicalTrend: QualityDataPoint[];
}
export interface QualityRuleResult {
ruleId: string;
ruleName: string;
status: 'pass' | 'fail';
score: number;
violations: number;
executionTime: number;
}
export interface QualityRecommendation {
type: 'rule_adjustment' | 'data_cleanup' | 'process_improvement' | 'monitoring';
priority: 'low' | 'medium' | 'high';
description: string;
impact: string;
effort: string;
steps: string[];
}

View file

@ -0,0 +1,84 @@
// Data Source Configuration Types
export interface DataSourceConfig {
id: string;
name: string;
type: 'websocket' | 'rest' | 'fix' | 'stream' | 'file' | 'database';
enabled: boolean;
priority: number;
rateLimit: DataRateLimitConfig;
connection: DataConnectionConfig;
subscriptions: SubscriptionConfig;
symbols: string[];
retryPolicy: RetryPolicyConfig;
healthCheck: HealthCheckConfig;
metadata?: Record<string, any>;
}
export interface DataRateLimitConfig {
requestsPerSecond: number;
burstLimit: number;
windowMs?: number;
}
export interface DataConnectionConfig {
url: string;
headers?: Record<string, string>;
queryParams?: Record<string, string>;
authentication?: DataAuthenticationConfig;
timeout?: number;
keepAlive?: boolean;
compression?: boolean;
}
export interface DataAuthenticationConfig {
type: 'apikey' | 'oauth' | 'basic' | 'jwt' | 'cert';
credentials: Record<string, string>;
refreshInterval?: number;
}
export interface SubscriptionConfig {
quotes: boolean;
trades: boolean;
orderbook: boolean;
candles: boolean;
news: boolean;
fundamentals?: boolean;
options?: boolean;
futures?: boolean;
}
export interface RetryPolicyConfig {
maxRetries: number;
backoffMultiplier: number;
maxBackoffMs: number;
enableJitter?: boolean;
}
export interface HealthCheckConfig {
intervalMs: number;
timeoutMs: number;
expectedLatencyMs: number;
failureThreshold?: number;
}
export interface DataSourceStatus {
id: string;
status: 'connected' | 'disconnected' | 'connecting' | 'error';
lastUpdate: Date;
connectionTime?: Date;
disconnectionTime?: Date;
errorCount: number;
messagesReceived: number;
latencyMs: number;
throughputPerSecond: number;
}
export interface DataSourceError {
sourceId: string;
timestamp: Date;
type: 'connection' | 'authentication' | 'ratelimit' | 'data' | 'timeout' | 'parsing';
message: string;
details?: Record<string, any>;
severity: 'low' | 'medium' | 'high' | 'critical';
recoverable: boolean;
}

View file

@ -0,0 +1,119 @@
// Data Transformation Types
export interface DataTransformation {
id: string;
name: string;
type: TransformationType;
description?: string;
code?: string;
inputFields: string[];
outputFields: string[];
logic: string;
parameters?: Record<string, any>;
validationRules?: ValidationRule[];
}
export enum TransformationType {
FILTER = 'filter',
AGGREGATE = 'aggregate',
JOIN = 'join',
UNION = 'union',
PIVOT = 'pivot',
UNPIVOT = 'unpivot',
SORT = 'sort',
DEDUPLICATE = 'deduplicate',
CALCULATE = 'calculate',
CAST = 'cast',
RENAME = 'rename',
SPLIT = 'split',
MERGE = 'merge',
NORMALIZE = 'normalize',
ENRICH = 'enrich'
}
export interface ValidationRule {
field: string;
type: ValidationType;
parameters?: Record<string, any>;
errorMessage?: string;
severity: 'warning' | 'error' | 'critical';
}
export enum ValidationType {
REQUIRED = 'required',
TYPE_CHECK = 'type_check',
RANGE = 'range',
PATTERN = 'pattern',
LENGTH = 'length',
UNIQUE = 'unique',
REFERENCE = 'reference',
CUSTOM = 'custom'
}
export interface TransformationRule {
id: string;
sourceField: string;
targetField: string;
transformation: string;
condition?: string;
parameters?: Record<string, any>;
}
export interface MappingRule {
sourceField: string;
targetField: string;
transformation?: string;
defaultValue?: any;
required: boolean;
}
export interface AggregationRule {
groupByFields: string[];
aggregations: AggregationFunction[];
having?: string;
windowFunction?: WindowFunction;
}
export interface AggregationFunction {
function: 'sum' | 'avg' | 'count' | 'min' | 'max' | 'first' | 'last' | 'stddev' | 'variance';
field: string;
alias?: string;
distinct?: boolean;
}
export interface WindowFunction {
type: 'row_number' | 'rank' | 'dense_rank' | 'lag' | 'lead' | 'first_value' | 'last_value';
partitionBy?: string[];
orderBy?: OrderByClause[];
frame?: WindowFrame;
}
export interface OrderByClause {
field: string;
direction: 'asc' | 'desc';
nulls?: 'first' | 'last';
}
export interface WindowFrame {
type: 'rows' | 'range';
start: FrameBoundary;
end: FrameBoundary;
}
export interface FrameBoundary {
type: 'unbounded_preceding' | 'current_row' | 'unbounded_following' | 'preceding' | 'following';
offset?: number;
}
export interface JoinRule {
type: 'inner' | 'left' | 'right' | 'full' | 'cross';
leftTable: string;
rightTable: string;
joinConditions: JoinCondition[];
selectFields?: string[];
}
export interface JoinCondition {
leftField: string;
rightField: string;
operator: '=' | '<>' | '<' | '>' | '<=' | '>=';
}