work on postgress / will prob remove and work on ib exchanges and symbols
This commit is contained in:
parent
cce5126cb7
commit
a20a11c1aa
16 changed files with 1441 additions and 95 deletions
56
database/postgres/scripts/README.md
Normal file
56
database/postgres/scripts/README.md
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
# Database Scripts
|
||||
|
||||
**Simplified database initialization system for Interactive Brokers data.**
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Initialize everything (recommended)
|
||||
bun run db:init
|
||||
|
||||
# Or run Interactive Brokers setup directly:
|
||||
bun run db:setup-ib # Create schema and populate IB data
|
||||
```
|
||||
|
||||
## What We Built
|
||||
|
||||
✅ **Simplified from complex multi-table schema to exchanges-only**
|
||||
✅ **Single script setup** - `setup-ib.ts` handles both schema and data
|
||||
✅ **Structured logging** with `@stock-bot/logger`
|
||||
✅ **184 exchanges populated** from JSON data
|
||||
✅ **Proper error handling** with helpful troubleshooting messages
|
||||
|
||||
## Scripts
|
||||
|
||||
### `setup-ib.ts` - Interactive Brokers Complete Setup
|
||||
**Main script for IB setup** - Sets up schema and populates exchange data in one go.
|
||||
|
||||
### `init.ts`
|
||||
Main initialization script that orchestrates setup for all providers.
|
||||
|
||||
## Database Schema
|
||||
|
||||
### IB Data (`ib_data` schema)
|
||||
- `exchanges` - Trading exchanges with metadata
|
||||
- `upsert_exchange()` - Function to insert/update exchanges
|
||||
|
||||
## Package.json Commands
|
||||
|
||||
```json
|
||||
{
|
||||
"db:init": "Run complete database initialization",
|
||||
"db:setup-ib": "Complete IB setup (schema + data)"
|
||||
}
|
||||
```
|
||||
|
||||
## Adding New Providers
|
||||
|
||||
1. Create `{provider}.sql` in `database/postgres/providers/`
|
||||
2. Create `{provider}.ts` script
|
||||
3. Add to `init.ts` and `package.json`
|
||||
|
||||
## Requirements
|
||||
|
||||
- PostgreSQL running
|
||||
- Database configured in `.env`
|
||||
- `ib-exchanges.json` file in `apps/data-service/src/setup/`
|
||||
41
database/postgres/scripts/init.ts
Normal file
41
database/postgres/scripts/init.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Main database initialization script
|
||||
* Sets up the database schema and populates with initial data
|
||||
*/
|
||||
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { setupIB } from './setup-ib';
|
||||
|
||||
const logger = getLogger('db-init');
|
||||
|
||||
async function main() {
|
||||
logger.info('Starting database initialization');
|
||||
|
||||
try {
|
||||
// Step 1: Setup Interactive Brokers (schema + data)
|
||||
logger.info('Setting up Interactive Brokers (schema + data)');
|
||||
await setupIB();
|
||||
logger.info('IB setup completed');
|
||||
|
||||
// Future providers can be added here:
|
||||
// await setupAlpaca();
|
||||
// await setupPolygon();
|
||||
|
||||
logger.info('Database initialization completed successfully');
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Database initialization failed', { error });
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error('Init script failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { main as initDatabase };
|
||||
366
database/postgres/scripts/setup-ib.ts
Normal file
366
database/postgres/scripts/setup-ib.ts
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Interactive Brokers complete setup script
|
||||
* Sets up schema and populates IB exchanges from ib-exchanges.json into PostgreSQL
|
||||
*/
|
||||
|
||||
import { postgresConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { PostgreSQLClient } from '@stock-bot/postgres-client';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
// Initialize logger
|
||||
const logger = getLogger('ib-setup');
|
||||
|
||||
// Type definitions based on the JSON structure
|
||||
interface IBExchange {
|
||||
id: string;
|
||||
name: string;
|
||||
country: string;
|
||||
region: string;
|
||||
assets: string;
|
||||
country_code: string;
|
||||
}
|
||||
|
||||
async function connectToDatabase(): Promise<PostgreSQLClient> {
|
||||
logger.info('Connecting to PostgreSQL', {
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
database: postgresConfig.POSTGRES_DATABASE
|
||||
});
|
||||
|
||||
try {
|
||||
const client = new PostgreSQLClient();
|
||||
await client.connect();
|
||||
|
||||
logger.info('Connected to PostgreSQL database');
|
||||
|
||||
// Test the connection
|
||||
const result = await client.query('SELECT version()');
|
||||
const version = result.rows[0].version.split(' ')[0];
|
||||
logger.info('PostgreSQL connection verified', { version });
|
||||
|
||||
return client;
|
||||
} catch (error) {
|
||||
logger.error('Failed to connect to PostgreSQL', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function runSchemaSetup(client: PostgreSQLClient) {
|
||||
try {
|
||||
logger.info('Loading schema SQL file');
|
||||
|
||||
const schemaPath = join(process.cwd(), 'database/postgres/providers/01-ib.sql');
|
||||
const schemaSql = readFileSync(schemaPath, 'utf-8');
|
||||
|
||||
logger.info('Executing schema setup');
|
||||
|
||||
// Execute the entire SQL file as one statement to handle multi-line functions
|
||||
try {
|
||||
await client.query(schemaSql);
|
||||
logger.info('Schema setup completed successfully');
|
||||
} catch (error: unknown) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
// Check if it's just "already exists" errors
|
||||
if (errorMessage.includes('already exists')) {
|
||||
logger.info('Schema setup completed (some objects already existed)');
|
||||
} else {
|
||||
logger.error('Error executing schema setup', { error: errorMessage });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify the setup
|
||||
await verifySchemaSetup(client);
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Schema setup failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function verifySchemaSetup(client: PostgreSQLClient) {
|
||||
logger.info('Verifying schema setup');
|
||||
|
||||
try {
|
||||
// Check if schema exists
|
||||
const schemaCheck = await client.query(`
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name = 'ib_data'
|
||||
`);
|
||||
|
||||
if (schemaCheck.rows.length === 0) {
|
||||
throw new Error('ib_data schema was not created');
|
||||
}
|
||||
|
||||
// Check tables
|
||||
const tableCheck = await client.query(`
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'ib_data'
|
||||
ORDER BY table_name
|
||||
`);
|
||||
|
||||
const actualTables = tableCheck.rows.map((row: any) => row.table_name);
|
||||
|
||||
// Check functions
|
||||
const functionCheck = await client.query(`
|
||||
SELECT routine_name
|
||||
FROM information_schema.routines
|
||||
WHERE routine_schema = 'ib_data'
|
||||
ORDER BY routine_name
|
||||
`);
|
||||
|
||||
const functions = functionCheck.rows.map((row: any) => row.routine_name);
|
||||
|
||||
logger.info('Schema verification completed', {
|
||||
schema: 'ib_data',
|
||||
tables: actualTables,
|
||||
functions: functions
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Schema verification failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadExchangesData(): Promise<IBExchange[]> {
|
||||
try {
|
||||
// Look for the JSON file in the project root
|
||||
const jsonPath = join(process.cwd(), 'apps/data-service/src/setup/ib-exchanges.json');
|
||||
|
||||
logger.info('Loading exchanges from file', { path: jsonPath });
|
||||
const jsonData = readFileSync(jsonPath, 'utf-8');
|
||||
|
||||
// Remove comment lines if they exist
|
||||
const cleanJsonData = jsonData.replace(/^\/\/.*$/gm, '');
|
||||
const exchanges: IBExchange[] = JSON.parse(cleanJsonData);
|
||||
|
||||
// Filter out incomplete entries and deduplicate by exchange code
|
||||
const validExchanges = exchanges.filter(exchange =>
|
||||
exchange.id &&
|
||||
exchange.name &&
|
||||
exchange.country_code &&
|
||||
exchange.id.trim() !== '' &&
|
||||
exchange.name.trim() !== '' &&
|
||||
exchange.country_code.trim() !== ''
|
||||
);
|
||||
|
||||
// Deduplicate by exchange code (keep the first occurrence)
|
||||
const exchangeMap = new Map<string, IBExchange>();
|
||||
validExchanges.forEach(exchange => {
|
||||
if (!exchangeMap.has(exchange.id)) {
|
||||
exchangeMap.set(exchange.id, exchange);
|
||||
}
|
||||
});
|
||||
const uniqueExchanges = Array.from(exchangeMap.values());
|
||||
|
||||
logger.info('Exchanges loaded successfully', {
|
||||
totalExchanges: exchanges.length,
|
||||
validExchanges: validExchanges.length,
|
||||
uniqueExchanges: uniqueExchanges.length,
|
||||
duplicatesRemoved: validExchanges.length - uniqueExchanges.length,
|
||||
filteredOut: exchanges.length - validExchanges.length
|
||||
});
|
||||
|
||||
if (validExchanges.length !== exchanges.length) {
|
||||
logger.warn('Some exchanges were filtered out due to incomplete data', {
|
||||
filteredCount: exchanges.length - validExchanges.length
|
||||
});
|
||||
}
|
||||
|
||||
if (uniqueExchanges.length !== validExchanges.length) {
|
||||
logger.warn('Duplicate exchange codes found and removed', {
|
||||
duplicateCount: validExchanges.length - uniqueExchanges.length
|
||||
});
|
||||
}
|
||||
|
||||
return uniqueExchanges;
|
||||
} catch (error) {
|
||||
logger.error('Error loading exchanges JSON', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function populateExchanges(client: PostgreSQLClient, exchanges: IBExchange[]): Promise<void> {
|
||||
logger.info('Starting batch exchange population', {
|
||||
totalExchanges: exchanges.length
|
||||
});
|
||||
|
||||
try {
|
||||
// Use the new batchUpsert method for fast population
|
||||
const result = await client.batchUpsert(
|
||||
'ib_data.exchanges',
|
||||
exchanges.map(ex => ({
|
||||
exchange_code: ex.id,
|
||||
exchange_name: ex.name,
|
||||
country: ex.country || null,
|
||||
region: ex.region || null,
|
||||
country_code: ex.country_code,
|
||||
assets: ex.assets || null
|
||||
})),
|
||||
'exchange_code',
|
||||
{ chunkSize: 100 }
|
||||
);
|
||||
|
||||
logger.info('Batch exchange population completed', {
|
||||
insertedCount: result.insertedCount,
|
||||
updatedCount: result.updatedCount,
|
||||
totalProcessed: result.insertedCount + result.updatedCount
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Batch exchange population failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyData(client: PostgreSQLClient) {
|
||||
logger.info('Verifying populated data');
|
||||
|
||||
try {
|
||||
// Count exchanges
|
||||
const exchangeCount = await client.query(`
|
||||
SELECT COUNT(*) as count FROM ib_data.exchanges
|
||||
`);
|
||||
|
||||
// Get exchanges by region
|
||||
const regionStats = await client.query(`
|
||||
SELECT region, COUNT(*) as count
|
||||
FROM ib_data.exchanges
|
||||
WHERE region IS NOT NULL
|
||||
GROUP BY region
|
||||
ORDER BY count DESC
|
||||
`);
|
||||
|
||||
// Get sample exchanges
|
||||
const sampleExchanges = await client.query(`
|
||||
SELECT
|
||||
exchange_code,
|
||||
exchange_name,
|
||||
country,
|
||||
region,
|
||||
country_code,
|
||||
assets
|
||||
FROM ib_data.exchanges
|
||||
ORDER BY exchange_code
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
const totalExchanges = exchangeCount.rows[0].count;
|
||||
logger.info('Data verification completed', { totalExchanges });
|
||||
|
||||
if (regionStats.rows.length > 0) {
|
||||
logger.info('Exchanges by region', {
|
||||
regions: regionStats.rows.map((row: any) => ({
|
||||
region: row.region,
|
||||
count: row.count
|
||||
}))
|
||||
});
|
||||
}
|
||||
|
||||
logger.info('Sample exchanges', {
|
||||
samples: sampleExchanges.rows.slice(0, 5).map((row: any) => ({
|
||||
code: row.exchange_code,
|
||||
name: row.exchange_name,
|
||||
country: row.country,
|
||||
region: row.region,
|
||||
assets: row.assets
|
||||
}))
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Data verification failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
logger.info('Starting Interactive Brokers complete setup (schema + data)');
|
||||
logger.info('Database configuration', {
|
||||
database: postgresConfig.POSTGRES_DATABASE,
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
user: postgresConfig.POSTGRES_USERNAME,
|
||||
ssl: postgresConfig.POSTGRES_SSL
|
||||
});
|
||||
|
||||
let client: PostgreSQLClient | null = null;
|
||||
|
||||
try {
|
||||
// Connect to database
|
||||
client = await connectToDatabase();
|
||||
|
||||
// Step 1: Setup schema
|
||||
logger.info('Step 1: Setting up database schema');
|
||||
await runSchemaSetup(client);
|
||||
|
||||
// Step 2: Load exchange data
|
||||
logger.info('Step 2: Loading exchange data');
|
||||
const exchanges = await loadExchangesData();
|
||||
|
||||
if (exchanges.length === 0) {
|
||||
logger.warn('No valid exchanges found to process');
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 3: Populate exchanges with batch upsert
|
||||
logger.info('Step 3: Populating exchanges (batch mode)');
|
||||
await populateExchanges(client, exchanges);
|
||||
|
||||
// Step 4: Verify the data
|
||||
logger.info('Step 4: Verifying setup and data');
|
||||
await verifyData(client);
|
||||
|
||||
logger.info('Interactive Brokers setup completed successfully');
|
||||
logger.info('Next steps', {
|
||||
suggestions: [
|
||||
'Start your data service',
|
||||
'Begin collecting market data',
|
||||
'Connect to Interactive Brokers API'
|
||||
]
|
||||
});
|
||||
|
||||
} catch (error: unknown) {
|
||||
logger.error('IB setup failed', { error });
|
||||
|
||||
// Provide helpful error messages
|
||||
if (error && typeof error === 'object' && 'code' in error && error.code === 'ECONNREFUSED') {
|
||||
logger.error('Database connection refused', {
|
||||
troubleshooting: [
|
||||
'Make sure PostgreSQL is running',
|
||||
'Check your database configuration in .env file',
|
||||
'Verify the database connection details'
|
||||
]
|
||||
});
|
||||
} else if (error && typeof error === 'object' && 'message' in error &&
|
||||
typeof error.message === 'string' &&
|
||||
error.message.includes('database') &&
|
||||
error.message.includes('does not exist')) {
|
||||
logger.error('Database does not exist', {
|
||||
suggestion: `Create database first: createdb ${postgresConfig.POSTGRES_DATABASE}`
|
||||
});
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
} finally {
|
||||
if (client) {
|
||||
await client.disconnect();
|
||||
logger.info('Database connection closed');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error('IB setup script failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { main as setupIB };
|
||||
Loading…
Add table
Add a link
Reference in a new issue