added new exchanges system
This commit is contained in:
parent
95eda4a842
commit
263e9513b7
98 changed files with 4643 additions and 1496 deletions
13
database/postgres/init/01-init-database.sql
Normal file
13
database/postgres/init/01-init-database.sql
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
-- Initialize main database and user
|
||||
-- This runs first when PostgreSQL container starts
|
||||
|
||||
-- Create main database if it doesn't exist
|
||||
SELECT 'CREATE DATABASE trading_bot'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'trading_bot')\gexec
|
||||
|
||||
-- Connect to the trading_bot database
|
||||
\c trading_bot;
|
||||
|
||||
-- Create extensions we'll need
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_trgm"; -- For fuzzy string matching
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
-- Trading Bot Database Schema Initialization
|
||||
|
||||
-- Create schemas
|
||||
CREATE SCHEMA IF NOT EXISTS trading;
|
||||
CREATE SCHEMA IF NOT EXISTS strategy;
|
||||
CREATE SCHEMA IF NOT EXISTS risk;
|
||||
CREATE SCHEMA IF NOT EXISTS audit;
|
||||
|
||||
-- Set search path for the database
|
||||
ALTER DATABASE trading_bot SET search_path TO trading, strategy, risk, audit, public;
|
||||
|
||||
-- Create extensions
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "btree_gin";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_stat_statements";
|
||||
|
||||
-- Create a read-only user for analytics
|
||||
CREATE USER trading_reader WITH PASSWORD 'reader_pass_dev';
|
||||
GRANT CONNECT ON DATABASE trading_bot TO trading_reader;
|
||||
GRANT USAGE ON SCHEMA trading, strategy, risk, audit TO trading_reader;
|
||||
63
database/postgres/init/02-master-schema.sql
Normal file
63
database/postgres/init/02-master-schema.sql
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
-- Simple Master Schema for Symbol Resolution
|
||||
-- Connect to trading_bot database
|
||||
\c trading_bot;
|
||||
|
||||
-- Exchanges Table
|
||||
CREATE TABLE IF NOT EXISTS exchanges (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
code VARCHAR(10) UNIQUE NOT NULL, -- NASDAQ, NYSE, TSX, etc.
|
||||
name VARCHAR(255) NOT NULL, -- Full exchange name
|
||||
country CHAR(2) NOT NULL, -- US, CA, GB
|
||||
currency CHAR(3) NOT NULL DEFAULT 'USD', -- USD, CAD, GBP
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Symbols Table
|
||||
CREATE TABLE IF NOT EXISTS symbols (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
symbol VARCHAR(20) NOT NULL, -- AAPL, SHOP, etc.
|
||||
exchange_id UUID REFERENCES exchanges(id),
|
||||
company_name VARCHAR(255),
|
||||
sector VARCHAR(100),
|
||||
country CHAR(2),
|
||||
currency CHAR(3),
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
UNIQUE(symbol, exchange_id)
|
||||
);
|
||||
|
||||
-- Provider Symbol Mappings (How each provider refers to our symbols)
|
||||
CREATE TABLE IF NOT EXISTS provider_mappings (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
symbol_id UUID REFERENCES symbols(id),
|
||||
provider VARCHAR(50) NOT NULL, -- 'qm', 'yahoo', 'ib', etc.
|
||||
provider_symbol VARCHAR(100) NOT NULL, -- How provider names it: "AAPL:NASDAQ", "SHOP.TO"
|
||||
provider_exchange VARCHAR(50), -- Provider's exchange code
|
||||
confidence DECIMAL(3,2) DEFAULT 1.0, -- 0.0 to 1.0 matching confidence
|
||||
verified BOOLEAN DEFAULT false, -- Manually verified flag
|
||||
last_seen TIMESTAMP DEFAULT NOW(),
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
UNIQUE(provider, provider_symbol)
|
||||
);
|
||||
|
||||
-- Simple sync tracking table
|
||||
CREATE TABLE IF NOT EXISTS sync_status (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
provider VARCHAR(50) NOT NULL,
|
||||
data_type VARCHAR(50) NOT NULL, -- 'symbols', 'exchanges'
|
||||
last_sync_at TIMESTAMP,
|
||||
last_sync_count INTEGER DEFAULT 0,
|
||||
sync_errors TEXT,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
UNIQUE(provider, data_type)
|
||||
);
|
||||
|
||||
-- Create basic indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_symbols_symbol ON symbols(symbol);
|
||||
CREATE INDEX IF NOT EXISTS idx_symbols_exchange ON symbols(exchange_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_mappings_provider ON provider_mappings(provider, provider_symbol);
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_mappings_symbol ON provider_mappings(symbol_id);
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
-- Core trading tables
|
||||
|
||||
-- Symbols and instruments
|
||||
CREATE TABLE trading.symbols (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
symbol VARCHAR(20) NOT NULL UNIQUE,
|
||||
name VARCHAR(255),
|
||||
exchange VARCHAR(50),
|
||||
asset_type VARCHAR(20) DEFAULT 'equity',
|
||||
sector VARCHAR(100),
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
metadata JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Orders
|
||||
CREATE TABLE trading.orders (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
symbol_id UUID REFERENCES trading.symbols(id),
|
||||
strategy_id UUID,
|
||||
order_type VARCHAR(20) NOT NULL, -- 'market', 'limit', 'stop', etc.
|
||||
side VARCHAR(10) NOT NULL CHECK (side IN ('buy', 'sell')),
|
||||
quantity DECIMAL(18,8) NOT NULL CHECK (quantity > 0),
|
||||
price DECIMAL(18,8),
|
||||
stop_price DECIMAL(18,8),
|
||||
status VARCHAR(20) DEFAULT 'pending' CHECK (status IN ('pending', 'submitted', 'filled', 'cancelled', 'rejected')),
|
||||
broker_order_id VARCHAR(100),
|
||||
filled_quantity DECIMAL(18,8) DEFAULT 0,
|
||||
avg_fill_price DECIMAL(18,8),
|
||||
commission DECIMAL(18,8) DEFAULT 0,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
CONSTRAINT valid_prices CHECK (
|
||||
(order_type = 'market') OR
|
||||
(order_type = 'limit' AND price IS NOT NULL) OR
|
||||
(order_type = 'stop' AND stop_price IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
-- Positions
|
||||
CREATE TABLE trading.positions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
symbol_id UUID REFERENCES trading.symbols(id),
|
||||
strategy_id UUID,
|
||||
quantity DECIMAL(18,8) NOT NULL,
|
||||
avg_cost DECIMAL(18,8) NOT NULL,
|
||||
market_value DECIMAL(18,8),
|
||||
unrealized_pnl DECIMAL(18,8),
|
||||
realized_pnl DECIMAL(18,8) DEFAULT 0,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
UNIQUE(symbol_id, strategy_id)
|
||||
);
|
||||
|
||||
-- Executions/Fills
|
||||
CREATE TABLE trading.executions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
order_id UUID REFERENCES trading.orders(id),
|
||||
symbol_id UUID REFERENCES trading.symbols(id),
|
||||
side VARCHAR(10) NOT NULL CHECK (side IN ('buy', 'sell')),
|
||||
quantity DECIMAL(18,8) NOT NULL,
|
||||
price DECIMAL(18,8) NOT NULL,
|
||||
commission DECIMAL(18,8) DEFAULT 0,
|
||||
broker_execution_id VARCHAR(100),
|
||||
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Accounts/Portfolios
|
||||
CREATE TABLE trading.accounts (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
account_type VARCHAR(50) DEFAULT 'paper', -- 'paper', 'live'
|
||||
broker VARCHAR(50),
|
||||
cash_balance DECIMAL(18,2) DEFAULT 0,
|
||||
buying_power DECIMAL(18,2) DEFAULT 0,
|
||||
total_value DECIMAL(18,2) DEFAULT 0,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Create indexes for performance
|
||||
CREATE INDEX idx_orders_symbol_created ON trading.orders(symbol_id, created_at);
|
||||
CREATE INDEX idx_orders_status ON trading.orders(status);
|
||||
CREATE INDEX idx_orders_strategy ON trading.orders(strategy_id);
|
||||
CREATE INDEX idx_positions_strategy ON trading.positions(strategy_id);
|
||||
CREATE INDEX idx_executions_order ON trading.executions(order_id);
|
||||
CREATE INDEX idx_executions_symbol_time ON trading.executions(symbol_id, executed_at);
|
||||
|
||||
-- Grant permissions to reader
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA trading TO trading_reader;
|
||||
23
database/postgres/init/03-initial-data.sql
Normal file
23
database/postgres/init/03-initial-data.sql
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
-- Insert initial reference data
|
||||
\c trading_bot;
|
||||
|
||||
-- Insert basic exchanges to start with
|
||||
INSERT INTO exchanges (code, name, country, currency) VALUES
|
||||
('NASDAQ', 'NASDAQ Stock Market', 'US', 'USD'),
|
||||
('NYSE', 'New York Stock Exchange', 'US', 'USD'),
|
||||
('TSX', 'Toronto Stock Exchange', 'CA', 'CAD'),
|
||||
('LSE', 'London Stock Exchange', 'GB', 'GBP'),
|
||||
('CME', 'Chicago Mercantile Exchange', 'US', 'USD')
|
||||
ON CONFLICT (code) DO NOTHING;
|
||||
|
||||
-- Insert initial sync status records for QM provider
|
||||
INSERT INTO sync_status (provider, data_type) VALUES
|
||||
('qm', 'symbols'),
|
||||
('qm', 'exchanges')
|
||||
ON CONFLICT (provider, data_type) DO NOTHING;
|
||||
|
||||
-- Show what we created
|
||||
SELECT 'Database setup complete. Tables created:' as status;
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
-- Strategy and Risk Management Tables
|
||||
|
||||
-- Strategies
|
||||
CREATE TABLE strategy.strategies (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
version VARCHAR(20) DEFAULT '1.0.0',
|
||||
config JSONB DEFAULT '{}',
|
||||
parameters JSONB DEFAULT '{}',
|
||||
is_active BOOLEAN DEFAULT false,
|
||||
is_enabled BOOLEAN DEFAULT true,
|
||||
created_by VARCHAR(255),
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Strategy executions/runs
|
||||
CREATE TABLE strategy.executions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
strategy_id UUID REFERENCES strategy.strategies(id),
|
||||
status VARCHAR(20) DEFAULT 'running' CHECK (status IN ('running', 'stopped', 'error', 'completed')),
|
||||
started_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
stopped_at TIMESTAMP WITH TIME ZONE,
|
||||
error_message TEXT,
|
||||
execution_stats JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Strategy signals
|
||||
CREATE TABLE strategy.signals (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
strategy_id UUID REFERENCES strategy.strategies(id),
|
||||
symbol_id UUID REFERENCES trading.symbols(id),
|
||||
signal_type VARCHAR(20) NOT NULL CHECK (signal_type IN ('buy', 'sell', 'hold')),
|
||||
strength DECIMAL(3,2) CHECK (strength >= 0 AND strength <= 1), -- 0.0 to 1.0
|
||||
confidence DECIMAL(3,2) CHECK (confidence >= 0 AND confidence <= 1),
|
||||
target_price DECIMAL(18,8),
|
||||
stop_loss DECIMAL(18,8),
|
||||
take_profit DECIMAL(18,8),
|
||||
metadata JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Risk limits
|
||||
CREATE TABLE risk.limits (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
strategy_id UUID REFERENCES strategy.strategies(id),
|
||||
account_id UUID REFERENCES trading.accounts(id),
|
||||
limit_type VARCHAR(50) NOT NULL, -- 'max_position_size', 'max_daily_loss', 'max_drawdown', etc.
|
||||
limit_value DECIMAL(18,8) NOT NULL,
|
||||
current_value DECIMAL(18,8) DEFAULT 0,
|
||||
threshold_warning DECIMAL(18,8), -- Warning at X% of limit
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Risk events/alerts
|
||||
CREATE TABLE risk.events (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
limit_id UUID REFERENCES risk.limits(id),
|
||||
strategy_id UUID,
|
||||
event_type VARCHAR(50) NOT NULL, -- 'warning', 'breach', 'resolved'
|
||||
severity VARCHAR(20) DEFAULT 'medium' CHECK (severity IN ('low', 'medium', 'high', 'critical')),
|
||||
message TEXT NOT NULL,
|
||||
current_value DECIMAL(18,8),
|
||||
limit_value DECIMAL(18,8),
|
||||
metadata JSONB DEFAULT '{}',
|
||||
acknowledged BOOLEAN DEFAULT false,
|
||||
acknowledged_by VARCHAR(255),
|
||||
acknowledged_at TIMESTAMP WITH TIME ZONE,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Performance tracking
|
||||
CREATE TABLE strategy.performance (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
strategy_id UUID REFERENCES strategy.strategies(id),
|
||||
date DATE NOT NULL,
|
||||
total_return DECIMAL(10,4),
|
||||
daily_return DECIMAL(10,4),
|
||||
sharpe_ratio DECIMAL(10,4),
|
||||
max_drawdown DECIMAL(10,4),
|
||||
win_rate DECIMAL(5,4),
|
||||
profit_factor DECIMAL(10,4),
|
||||
total_trades INTEGER DEFAULT 0,
|
||||
winning_trades INTEGER DEFAULT 0,
|
||||
losing_trades INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
UNIQUE(strategy_id, date)
|
||||
);
|
||||
|
||||
-- Create indexes
|
||||
CREATE INDEX idx_strategies_active ON strategy.strategies(is_active, is_enabled);
|
||||
CREATE INDEX idx_executions_strategy ON strategy.executions(strategy_id);
|
||||
CREATE INDEX idx_signals_strategy_time ON strategy.signals(strategy_id, created_at);
|
||||
CREATE INDEX idx_signals_symbol ON strategy.signals(symbol_id);
|
||||
CREATE INDEX idx_limits_strategy ON risk.limits(strategy_id);
|
||||
CREATE INDEX idx_risk_events_severity ON risk.events(severity, created_at);
|
||||
CREATE INDEX idx_performance_strategy_date ON strategy.performance(strategy_id, date);
|
||||
|
||||
-- Grant permissions
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA strategy TO trading_reader;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA risk TO trading_reader;
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
-- Audit and System Tables
|
||||
|
||||
-- System events audit
|
||||
CREATE TABLE audit.system_events (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
service_name VARCHAR(100) NOT NULL,
|
||||
event_type VARCHAR(50) NOT NULL,
|
||||
event_data JSONB DEFAULT '{}',
|
||||
user_id VARCHAR(255),
|
||||
ip_address INET,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Trading events audit
|
||||
CREATE TABLE audit.trading_events (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
event_type VARCHAR(50) NOT NULL, -- 'order_created', 'order_filled', 'position_opened', etc.
|
||||
entity_type VARCHAR(50) NOT NULL, -- 'order', 'position', 'execution'
|
||||
entity_id UUID NOT NULL,
|
||||
old_values JSONB,
|
||||
new_values JSONB,
|
||||
changed_by VARCHAR(255),
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Service health monitoring
|
||||
CREATE TABLE audit.service_health (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
service_name VARCHAR(100) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL CHECK (status IN ('healthy', 'unhealthy', 'degraded')),
|
||||
version VARCHAR(50),
|
||||
uptime_seconds INTEGER,
|
||||
memory_usage_mb INTEGER,
|
||||
cpu_usage_percent DECIMAL(5,2),
|
||||
last_health_check TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
metadata JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Configuration changes
|
||||
CREATE TABLE audit.config_changes (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
config_key VARCHAR(255) NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by VARCHAR(255),
|
||||
reason TEXT,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Create indexes
|
||||
CREATE INDEX idx_system_events_service_time ON audit.system_events(service_name, created_at);
|
||||
CREATE INDEX idx_trading_events_type_time ON audit.trading_events(event_type, created_at);
|
||||
CREATE INDEX idx_trading_events_entity ON audit.trading_events(entity_type, entity_id);
|
||||
CREATE INDEX idx_service_health_name_time ON audit.service_health(service_name, created_at);
|
||||
CREATE INDEX idx_config_changes_key_time ON audit.config_changes(config_key, created_at);
|
||||
|
||||
-- Grant permissions
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA audit TO trading_reader;
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
-- Enhanced Schema with Provider Exchange Mappings
|
||||
-- Connect to trading_bot database
|
||||
\c trading_bot;
|
||||
|
||||
-- First, rename is_active to active in existing tables
|
||||
ALTER TABLE IF EXISTS exchanges RENAME COLUMN is_active TO active;
|
||||
ALTER TABLE IF EXISTS symbols RENAME COLUMN is_active TO active;
|
||||
|
||||
-- Provider Exchange Mappings Table
|
||||
-- Maps provider-specific exchange codes to our master exchanges
|
||||
CREATE TABLE IF NOT EXISTS provider_exchange_mappings (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
provider VARCHAR(50) NOT NULL, -- 'qm', 'eod', 'ib', etc.
|
||||
provider_exchange_code VARCHAR(50) NOT NULL, -- Provider's exchange code: 'NYE', 'US', 'NASDAQ'
|
||||
provider_exchange_name VARCHAR(255), -- Provider's exchange name
|
||||
master_exchange_id UUID REFERENCES exchanges(id),
|
||||
country_code CHAR(2), -- Provider's country code
|
||||
currency CHAR(3), -- Provider's currency
|
||||
confidence DECIMAL(3,2) DEFAULT 0.8, -- Mapping confidence (0.0 to 1.0)
|
||||
active BOOLEAN DEFAULT false, -- Manual activation flag
|
||||
verified BOOLEAN DEFAULT false, -- Manually verified flag
|
||||
auto_mapped BOOLEAN DEFAULT true, -- Was this auto-created by sync?
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
UNIQUE(provider, provider_exchange_code)
|
||||
);
|
||||
|
||||
-- Create indexes for provider exchange mappings
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_exchange_mappings_provider ON provider_exchange_mappings(provider);
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_exchange_mappings_master ON provider_exchange_mappings(master_exchange_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_exchange_mappings_active ON provider_exchange_mappings(provider, active);
|
||||
|
||||
-- Update existing exchanges to be inactive by default (for new syncs)
|
||||
-- But preserve any existing active status
|
||||
-- This only affects future INSERTs, not existing data
|
||||
|
||||
-- Add some useful views for management
|
||||
CREATE OR REPLACE VIEW exchange_provider_summary AS
|
||||
SELECT
|
||||
e.code as master_code,
|
||||
e.name as master_name,
|
||||
e.country,
|
||||
e.currency,
|
||||
e.active as master_active,
|
||||
COUNT(pem.id) as provider_mappings,
|
||||
COUNT(CASE WHEN pem.active = true THEN 1 END) as active_mappings,
|
||||
COUNT(CASE WHEN pem.verified = true THEN 1 END) as verified_mappings,
|
||||
STRING_AGG(DISTINCT pem.provider, ', ') as providers
|
||||
FROM exchanges e
|
||||
LEFT JOIN provider_exchange_mappings pem ON e.id = pem.master_exchange_id
|
||||
GROUP BY e.id, e.code, e.name, e.country, e.currency, e.active
|
||||
ORDER BY e.code;
|
||||
|
||||
CREATE OR REPLACE VIEW provider_exchange_details AS
|
||||
SELECT
|
||||
pem.provider,
|
||||
pem.provider_exchange_code,
|
||||
pem.provider_exchange_name,
|
||||
pem.country_code,
|
||||
pem.currency,
|
||||
pem.active,
|
||||
pem.verified,
|
||||
pem.auto_mapped,
|
||||
pem.confidence,
|
||||
e.code as master_exchange_code,
|
||||
e.name as master_exchange_name,
|
||||
e.active as master_active
|
||||
FROM provider_exchange_mappings pem
|
||||
JOIN exchanges e ON pem.master_exchange_id = e.id
|
||||
ORDER BY pem.provider, pem.provider_exchange_code;
|
||||
|
||||
-- Show what we created
|
||||
SELECT 'Enhanced provider exchange mapping schema created' as status;
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
-- Insert initial reference data
|
||||
|
||||
-- Insert common symbols
|
||||
INSERT INTO trading.symbols (symbol, name, exchange, asset_type, sector) VALUES
|
||||
('AAPL', 'Apple Inc.', 'NASDAQ', 'equity', 'Technology'),
|
||||
('GOOGL', 'Alphabet Inc.', 'NASDAQ', 'equity', 'Technology'),
|
||||
('MSFT', 'Microsoft Corporation', 'NASDAQ', 'equity', 'Technology'),
|
||||
('AMZN', 'Amazon.com Inc.', 'NASDAQ', 'equity', 'Consumer Discretionary'),
|
||||
('TSLA', 'Tesla Inc.', 'NASDAQ', 'equity', 'Consumer Discretionary'),
|
||||
('NVDA', 'NVIDIA Corporation', 'NASDAQ', 'equity', 'Technology'),
|
||||
('META', 'Meta Platforms Inc.', 'NASDAQ', 'equity', 'Technology'),
|
||||
('NFLX', 'Netflix Inc.', 'NASDAQ', 'equity', 'Communication Services'),
|
||||
('SPY', 'SPDR S&P 500 ETF Trust', 'NYSE', 'etf', 'Broad Market'),
|
||||
('QQQ', 'Invesco QQQ Trust', 'NASDAQ', 'etf', 'Technology'),
|
||||
('BTC-USD', 'Bitcoin USD', 'CRYPTO', 'cryptocurrency', 'Digital Assets'),
|
||||
('ETH-USD', 'Ethereum USD', 'CRYPTO', 'cryptocurrency', 'Digital Assets');
|
||||
|
||||
-- Insert default trading account
|
||||
INSERT INTO trading.accounts (name, account_type, broker, cash_balance, buying_power, total_value) VALUES
|
||||
('Demo Account', 'paper', 'demo', 100000.00, 100000.00, 100000.00);
|
||||
|
||||
-- Insert demo strategy
|
||||
INSERT INTO strategy.strategies (name, description, config, parameters, is_active) VALUES
|
||||
('Demo Mean Reversion', 'Simple mean reversion strategy for demonstration',
|
||||
'{"timeframe": "1h", "lookback_period": 20}',
|
||||
'{"rsi_oversold": 30, "rsi_overbought": 70, "position_size": 0.1}',
|
||||
false);
|
||||
|
||||
-- Insert basic risk limits
|
||||
INSERT INTO risk.limits (strategy_id, limit_type, limit_value, threshold_warning)
|
||||
SELECT s.id, 'max_position_size', 10000.00, 8000.00
|
||||
FROM strategy.strategies s
|
||||
WHERE s.name = 'Demo Mean Reversion';
|
||||
|
||||
INSERT INTO risk.limits (strategy_id, limit_type, limit_value, threshold_warning)
|
||||
SELECT s.id, 'max_daily_loss', 5000.00, 4000.00
|
||||
FROM strategy.strategies s
|
||||
WHERE s.name = 'Demo Mean Reversion';
|
||||
|
||||
-- Create updated_at trigger function
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- Apply updated_at triggers
|
||||
CREATE TRIGGER update_symbols_updated_at BEFORE UPDATE ON trading.symbols FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
CREATE TRIGGER update_orders_updated_at BEFORE UPDATE ON trading.orders FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
CREATE TRIGGER update_positions_updated_at BEFORE UPDATE ON trading.positions FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
CREATE TRIGGER update_accounts_updated_at BEFORE UPDATE ON trading.accounts FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
CREATE TRIGGER update_strategies_updated_at BEFORE UPDATE ON strategy.strategies FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
CREATE TRIGGER update_limits_updated_at BEFORE UPDATE ON risk.limits FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
-- =============================================================================
|
||||
-- Interactive Brokers Simple Schema Setup
|
||||
-- =============================================================================
|
||||
|
||||
-- Create dedicated schema for IB data
|
||||
CREATE SCHEMA IF NOT EXISTS ib_data;
|
||||
|
||||
-- =============================================================================
|
||||
-- Simple Exchanges Table
|
||||
-- =============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ib_data.exchanges (
|
||||
id SERIAL PRIMARY KEY,
|
||||
exchange_code VARCHAR(20) NOT NULL UNIQUE,
|
||||
exchange_name TEXT NOT NULL,
|
||||
country VARCHAR(100),
|
||||
region VARCHAR(50),
|
||||
country_code VARCHAR(3),
|
||||
assets TEXT,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Create indexes for performance
|
||||
CREATE INDEX IF NOT EXISTS idx_exchanges_code ON ib_data.exchanges(exchange_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_exchanges_country ON ib_data.exchanges(country_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_exchanges_region ON ib_data.exchanges(region);
|
||||
CREATE INDEX IF NOT EXISTS idx_exchanges_active ON ib_data.exchanges(is_active);
|
||||
|
||||
-- =============================================================================
|
||||
-- Permissions
|
||||
-- =============================================================================
|
||||
|
||||
-- Grant usage on schema
|
||||
GRANT USAGE ON SCHEMA ib_data TO PUBLIC;
|
||||
|
||||
-- Grant permissions on tables
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA ib_data TO PUBLIC;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA ib_data TO PUBLIC;
|
||||
|
||||
-- Set default permissions for future tables
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA ib_data GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO PUBLIC;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA ib_data GRANT USAGE, SELECT ON SEQUENCES TO PUBLIC;
|
||||
|
||||
-- =============================================================================
|
||||
-- Comments
|
||||
-- =============================================================================
|
||||
|
||||
COMMENT ON SCHEMA ib_data IS 'Interactive Brokers market data schema (simplified)';
|
||||
COMMENT ON TABLE ib_data.exchanges IS 'Trading exchanges from Interactive Brokers';
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
# Interactive Brokers Database Setup
|
||||
|
||||
This directory contains the PostgreSQL schema setup for Interactive Brokers data.
|
||||
|
||||
## Quick Setup
|
||||
|
||||
### 1. **Create the Schema and Tables**
|
||||
```bash
|
||||
# Run the SQL schema setup
|
||||
bun run db:setup
|
||||
|
||||
# Or manually with psql:
|
||||
psql -U postgres -d stock_bot -f database/postgres/providers/01-ib.sql
|
||||
```
|
||||
|
||||
### 2. **Populate with Exchange Data**
|
||||
```bash
|
||||
# Populate exchanges from ib-exchanges.json
|
||||
bun run db:populate-ib
|
||||
|
||||
# Or run the complete setup (schema + data):
|
||||
bun run db:setup-ib
|
||||
```
|
||||
|
||||
## What Gets Created
|
||||
|
||||
### 📊 **Schema: `ib_data`**
|
||||
- `exchanges` - All IB trading exchanges with metadata
|
||||
- `asset_types` - Types of financial instruments (Stocks, Options, etc.)
|
||||
- `exchange_assets` - Many-to-many mapping of exchanges to asset types
|
||||
- `securities` - Individual tradeable instruments
|
||||
- `market_data` - Real-time and historical price data
|
||||
- `data_fetch_jobs` - Queue for data collection tasks
|
||||
|
||||
### 🔍 **Views**
|
||||
- `exchanges_with_assets` - Exchanges with their supported asset types
|
||||
- `latest_market_data` - Most recent market data per security
|
||||
- `securities_full_view` - Securities with full exchange and asset type info
|
||||
|
||||
### ⚡ **Functions**
|
||||
- `get_or_create_exchange()` - Utility to insert/update exchanges
|
||||
- `add_assets_to_exchange()` - Parse and add asset types to exchanges
|
||||
|
||||
## Database Structure
|
||||
|
||||
```sql
|
||||
-- Example queries you can run after setup:
|
||||
|
||||
-- View all exchanges with their supported assets
|
||||
SELECT * FROM ib_data.exchanges_with_assets LIMIT 10;
|
||||
|
||||
-- Count exchanges by region
|
||||
SELECT region, COUNT(*)
|
||||
FROM ib_data.exchanges
|
||||
GROUP BY region
|
||||
ORDER BY COUNT(*) DESC;
|
||||
|
||||
-- Find exchanges that support stocks
|
||||
SELECT e.exchange_code, e.exchange_name, e.country
|
||||
FROM ib_data.exchanges e
|
||||
JOIN ib_data.exchange_assets ea ON e.id = ea.exchange_id
|
||||
JOIN ib_data.asset_types at ON ea.asset_type_id = at.id
|
||||
WHERE at.code = 'Stocks'
|
||||
ORDER BY e.exchange_code;
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Set these in your `.env` file for the populate script:
|
||||
|
||||
```bash
|
||||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_NAME=stock_bot
|
||||
DB_USER=postgres
|
||||
DB_PASSWORD=your_password
|
||||
```
|
||||
|
||||
## Integration with Your Code
|
||||
|
||||
The schema is designed to work with your existing `ib.tasks.ts` file:
|
||||
|
||||
```typescript
|
||||
// Your fetchSession() function can now store data like:
|
||||
import { query } from '@stock-bot/postgres-client';
|
||||
|
||||
// Create a fetch job
|
||||
await query(`
|
||||
INSERT INTO ib_data.data_fetch_jobs (job_type, status, metadata)
|
||||
VALUES ('SYMBOL_SUMMARY', 'PENDING', $1)
|
||||
`, [{ url: 'https://...', proxy: '...' }]);
|
||||
|
||||
// Store exchange data
|
||||
const exchangeId = await query(`
|
||||
SELECT ib_data.get_or_create_exchange($1, $2, $3, $4, $5)
|
||||
`, ['NASDAQ', 'NASDAQ Global Select Market', 'United States', 'Americas', 'US']);
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. ✅ Run the setup scripts
|
||||
2. 🔧 Update your IB tasks to use the database
|
||||
3. 📊 Start collecting market data
|
||||
4. 🚀 Build your trading strategies on top of this data layer!
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
# Database Scripts
|
||||
|
||||
**Simplified database initialization system for Interactive Brokers data.**
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Initialize everything (recommended)
|
||||
bun run db:init
|
||||
|
||||
# Or run Interactive Brokers setup directly:
|
||||
bun run db:setup-ib # Create schema and populate IB data
|
||||
```
|
||||
|
||||
## What We Built
|
||||
|
||||
✅ **Simplified from complex multi-table schema to exchanges-only**
|
||||
✅ **Single script setup** - `setup-ib.ts` handles both schema and data
|
||||
✅ **Structured logging** with `@stock-bot/logger`
|
||||
✅ **184 exchanges populated** from JSON data
|
||||
✅ **Proper error handling** with helpful troubleshooting messages
|
||||
|
||||
## Scripts
|
||||
|
||||
### `setup-ib.ts` - Interactive Brokers Complete Setup
|
||||
**Main script for IB setup** - Sets up schema and populates exchange data in one go.
|
||||
|
||||
### `init.ts`
|
||||
Main initialization script that orchestrates setup for all providers.
|
||||
|
||||
## Database Schema
|
||||
|
||||
### IB Data (`ib_data` schema)
|
||||
- `exchanges` - Trading exchanges with metadata
|
||||
- `upsert_exchange()` - Function to insert/update exchanges
|
||||
|
||||
## Package.json Commands
|
||||
|
||||
```json
|
||||
{
|
||||
"db:init": "Run complete database initialization",
|
||||
"db:setup-ib": "Complete IB setup (schema + data)"
|
||||
}
|
||||
```
|
||||
|
||||
## Adding New Providers
|
||||
|
||||
1. Create `{provider}.sql` in `database/postgres/providers/`
|
||||
2. Create `{provider}.ts` script
|
||||
3. Add to `init.ts` and `package.json`
|
||||
|
||||
## Requirements
|
||||
|
||||
- PostgreSQL running
|
||||
- Database configured in `.env`
|
||||
- `ib-exchanges.json` file in `apps/data-service/src/setup/`
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Main database initialization script
|
||||
* Sets up the database schema and populates with initial data
|
||||
*/
|
||||
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { setupIB } from './setup-ib';
|
||||
|
||||
const logger = getLogger('db-init');
|
||||
|
||||
async function main() {
|
||||
logger.info('Starting database initialization');
|
||||
|
||||
try {
|
||||
// Step 1: Setup Interactive Brokers (schema + data)
|
||||
logger.info('Setting up Interactive Brokers (schema + data)');
|
||||
await setupIB();
|
||||
logger.info('IB setup completed');
|
||||
|
||||
// Future providers can be added here:
|
||||
// await setupAlpaca();
|
||||
// await setupPolygon();
|
||||
|
||||
logger.info('Database initialization completed successfully');
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Database initialization failed', { error });
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error('Init script failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { main as initDatabase };
|
||||
|
|
@ -1,366 +0,0 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Interactive Brokers complete setup script
|
||||
* Sets up schema and populates IB exchanges from ib-exchanges.json into PostgreSQL
|
||||
*/
|
||||
|
||||
import { postgresConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { PostgreSQLClient } from '@stock-bot/postgres-client';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
// Initialize logger
|
||||
const logger = getLogger('ib-setup');
|
||||
|
||||
// Type definitions based on the JSON structure
|
||||
interface IBExchange {
|
||||
id: string;
|
||||
name: string;
|
||||
country: string;
|
||||
region: string;
|
||||
assets: string;
|
||||
country_code: string;
|
||||
}
|
||||
|
||||
async function connectToDatabase(): Promise<PostgreSQLClient> {
|
||||
logger.info('Connecting to PostgreSQL', {
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
database: postgresConfig.POSTGRES_DATABASE
|
||||
});
|
||||
|
||||
try {
|
||||
const client = new PostgreSQLClient();
|
||||
await client.connect();
|
||||
|
||||
logger.info('Connected to PostgreSQL database');
|
||||
|
||||
// Test the connection
|
||||
const result = await client.query('SELECT version()');
|
||||
const version = result.rows[0].version.split(' ')[0];
|
||||
logger.info('PostgreSQL connection verified', { version });
|
||||
|
||||
return client;
|
||||
} catch (error) {
|
||||
logger.error('Failed to connect to PostgreSQL', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function runSchemaSetup(client: PostgreSQLClient) {
|
||||
try {
|
||||
logger.info('Loading schema SQL file');
|
||||
|
||||
const schemaPath = join(process.cwd(), 'database/postgres/providers/01-ib.sql');
|
||||
const schemaSql = readFileSync(schemaPath, 'utf-8');
|
||||
|
||||
logger.info('Executing schema setup');
|
||||
|
||||
// Execute the entire SQL file as one statement to handle multi-line functions
|
||||
try {
|
||||
await client.query(schemaSql);
|
||||
logger.info('Schema setup completed successfully');
|
||||
} catch (error: unknown) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
// Check if it's just "already exists" errors
|
||||
if (errorMessage.includes('already exists')) {
|
||||
logger.info('Schema setup completed (some objects already existed)');
|
||||
} else {
|
||||
logger.error('Error executing schema setup', { error: errorMessage });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify the setup
|
||||
await verifySchemaSetup(client);
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Schema setup failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function verifySchemaSetup(client: PostgreSQLClient) {
|
||||
logger.info('Verifying schema setup');
|
||||
|
||||
try {
|
||||
// Check if schema exists
|
||||
const schemaCheck = await client.query(`
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name = 'ib_data'
|
||||
`);
|
||||
|
||||
if (schemaCheck.rows.length === 0) {
|
||||
throw new Error('ib_data schema was not created');
|
||||
}
|
||||
|
||||
// Check tables
|
||||
const tableCheck = await client.query(`
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'ib_data'
|
||||
ORDER BY table_name
|
||||
`);
|
||||
|
||||
const actualTables = tableCheck.rows.map((row: any) => row.table_name);
|
||||
|
||||
// Check functions
|
||||
const functionCheck = await client.query(`
|
||||
SELECT routine_name
|
||||
FROM information_schema.routines
|
||||
WHERE routine_schema = 'ib_data'
|
||||
ORDER BY routine_name
|
||||
`);
|
||||
|
||||
const functions = functionCheck.rows.map((row: any) => row.routine_name);
|
||||
|
||||
logger.info('Schema verification completed', {
|
||||
schema: 'ib_data',
|
||||
tables: actualTables,
|
||||
functions: functions
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Schema verification failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadExchangesData(): Promise<IBExchange[]> {
|
||||
try {
|
||||
// Look for the JSON file in the project root
|
||||
const jsonPath = join(process.cwd(), 'apps/data-service/src/setup/ib-exchanges.json');
|
||||
|
||||
logger.info('Loading exchanges from file', { path: jsonPath });
|
||||
const jsonData = readFileSync(jsonPath, 'utf-8');
|
||||
|
||||
// Remove comment lines if they exist
|
||||
const cleanJsonData = jsonData.replace(/^\/\/.*$/gm, '');
|
||||
const exchanges: IBExchange[] = JSON.parse(cleanJsonData);
|
||||
|
||||
// Filter out incomplete entries and deduplicate by exchange code
|
||||
const validExchanges = exchanges.filter(exchange =>
|
||||
exchange.id &&
|
||||
exchange.name &&
|
||||
exchange.country_code &&
|
||||
exchange.id.trim() !== '' &&
|
||||
exchange.name.trim() !== '' &&
|
||||
exchange.country_code.trim() !== ''
|
||||
);
|
||||
|
||||
// Deduplicate by exchange code (keep the first occurrence)
|
||||
const exchangeMap = new Map<string, IBExchange>();
|
||||
validExchanges.forEach(exchange => {
|
||||
if (!exchangeMap.has(exchange.id)) {
|
||||
exchangeMap.set(exchange.id, exchange);
|
||||
}
|
||||
});
|
||||
const uniqueExchanges = Array.from(exchangeMap.values());
|
||||
|
||||
logger.info('Exchanges loaded successfully', {
|
||||
totalExchanges: exchanges.length,
|
||||
validExchanges: validExchanges.length,
|
||||
uniqueExchanges: uniqueExchanges.length,
|
||||
duplicatesRemoved: validExchanges.length - uniqueExchanges.length,
|
||||
filteredOut: exchanges.length - validExchanges.length
|
||||
});
|
||||
|
||||
if (validExchanges.length !== exchanges.length) {
|
||||
logger.warn('Some exchanges were filtered out due to incomplete data', {
|
||||
filteredCount: exchanges.length - validExchanges.length
|
||||
});
|
||||
}
|
||||
|
||||
if (uniqueExchanges.length !== validExchanges.length) {
|
||||
logger.warn('Duplicate exchange codes found and removed', {
|
||||
duplicateCount: validExchanges.length - uniqueExchanges.length
|
||||
});
|
||||
}
|
||||
|
||||
return uniqueExchanges;
|
||||
} catch (error) {
|
||||
logger.error('Error loading exchanges JSON', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function populateExchanges(client: PostgreSQLClient, exchanges: IBExchange[]): Promise<void> {
|
||||
logger.info('Starting batch exchange population', {
|
||||
totalExchanges: exchanges.length
|
||||
});
|
||||
|
||||
try {
|
||||
// Use the new batchUpsert method for fast population
|
||||
const result = await client.batchUpsert(
|
||||
'ib_data.exchanges',
|
||||
exchanges.map(ex => ({
|
||||
exchange_code: ex.id,
|
||||
exchange_name: ex.name,
|
||||
country: ex.country || null,
|
||||
region: ex.region || null,
|
||||
country_code: ex.country_code,
|
||||
assets: ex.assets || null
|
||||
})),
|
||||
'exchange_code',
|
||||
{ chunkSize: 100 }
|
||||
);
|
||||
|
||||
logger.info('Batch exchange population completed', {
|
||||
insertedCount: result.insertedCount,
|
||||
updatedCount: result.updatedCount,
|
||||
totalProcessed: result.insertedCount + result.updatedCount
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Batch exchange population failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyData(client: PostgreSQLClient) {
|
||||
logger.info('Verifying populated data');
|
||||
|
||||
try {
|
||||
// Count exchanges
|
||||
const exchangeCount = await client.query(`
|
||||
SELECT COUNT(*) as count FROM ib_data.exchanges
|
||||
`);
|
||||
|
||||
// Get exchanges by region
|
||||
const regionStats = await client.query(`
|
||||
SELECT region, COUNT(*) as count
|
||||
FROM ib_data.exchanges
|
||||
WHERE region IS NOT NULL
|
||||
GROUP BY region
|
||||
ORDER BY count DESC
|
||||
`);
|
||||
|
||||
// Get sample exchanges
|
||||
const sampleExchanges = await client.query(`
|
||||
SELECT
|
||||
exchange_code,
|
||||
exchange_name,
|
||||
country,
|
||||
region,
|
||||
country_code,
|
||||
assets
|
||||
FROM ib_data.exchanges
|
||||
ORDER BY exchange_code
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
const totalExchanges = exchangeCount.rows[0].count;
|
||||
logger.info('Data verification completed', { totalExchanges });
|
||||
|
||||
if (regionStats.rows.length > 0) {
|
||||
logger.info('Exchanges by region', {
|
||||
regions: regionStats.rows.map((row: any) => ({
|
||||
region: row.region,
|
||||
count: row.count
|
||||
}))
|
||||
});
|
||||
}
|
||||
|
||||
logger.info('Sample exchanges', {
|
||||
samples: sampleExchanges.rows.slice(0, 5).map((row: any) => ({
|
||||
code: row.exchange_code,
|
||||
name: row.exchange_name,
|
||||
country: row.country,
|
||||
region: row.region,
|
||||
assets: row.assets
|
||||
}))
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Data verification failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
logger.info('Starting Interactive Brokers complete setup (schema + data)');
|
||||
logger.info('Database configuration', {
|
||||
database: postgresConfig.POSTGRES_DATABASE,
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
user: postgresConfig.POSTGRES_USERNAME,
|
||||
ssl: postgresConfig.POSTGRES_SSL
|
||||
});
|
||||
|
||||
let client: PostgreSQLClient | null = null;
|
||||
|
||||
try {
|
||||
// Connect to database
|
||||
client = await connectToDatabase();
|
||||
|
||||
// Step 1: Setup schema
|
||||
logger.info('Step 1: Setting up database schema');
|
||||
await runSchemaSetup(client);
|
||||
|
||||
// Step 2: Load exchange data
|
||||
logger.info('Step 2: Loading exchange data');
|
||||
const exchanges = await loadExchangesData();
|
||||
|
||||
if (exchanges.length === 0) {
|
||||
logger.warn('No valid exchanges found to process');
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 3: Populate exchanges with batch upsert
|
||||
logger.info('Step 3: Populating exchanges (batch mode)');
|
||||
await populateExchanges(client, exchanges);
|
||||
|
||||
// Step 4: Verify the data
|
||||
logger.info('Step 4: Verifying setup and data');
|
||||
await verifyData(client);
|
||||
|
||||
logger.info('Interactive Brokers setup completed successfully');
|
||||
logger.info('Next steps', {
|
||||
suggestions: [
|
||||
'Start your data service',
|
||||
'Begin collecting market data',
|
||||
'Connect to Interactive Brokers API'
|
||||
]
|
||||
});
|
||||
|
||||
} catch (error: unknown) {
|
||||
logger.error('IB setup failed', { error });
|
||||
|
||||
// Provide helpful error messages
|
||||
if (error && typeof error === 'object' && 'code' in error && error.code === 'ECONNREFUSED') {
|
||||
logger.error('Database connection refused', {
|
||||
troubleshooting: [
|
||||
'Make sure PostgreSQL is running',
|
||||
'Check your database configuration in .env file',
|
||||
'Verify the database connection details'
|
||||
]
|
||||
});
|
||||
} else if (error && typeof error === 'object' && 'message' in error &&
|
||||
typeof error.message === 'string' &&
|
||||
error.message.includes('database') &&
|
||||
error.message.includes('does not exist')) {
|
||||
logger.error('Database does not exist', {
|
||||
suggestion: `Create database first: createdb ${postgresConfig.POSTGRES_DATABASE}`
|
||||
});
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
} finally {
|
||||
if (client) {
|
||||
await client.disconnect();
|
||||
logger.info('Database connection closed');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error('IB setup script failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { main as setupIB };
|
||||
Loading…
Add table
Add a link
Reference in a new issue