working on proxy service. getting closer
This commit is contained in:
parent
6380165a94
commit
afc1843fdb
5 changed files with 91 additions and 892 deletions
|
|
@ -7,50 +7,62 @@ export class ProxyService {
|
||||||
private logger = new Logger('proxy-service');
|
private logger = new Logger('proxy-service');
|
||||||
private cache: CacheProvider = createCache('hybrid');
|
private cache: CacheProvider = createCache('hybrid');
|
||||||
private httpClient: HttpClient;
|
private httpClient: HttpClient;
|
||||||
private readonly concurrencyLimit = pLimit(250);
|
private readonly concurrencyLimit = pLimit(300);
|
||||||
private readonly CACHE_KEY = 'proxy';
|
private readonly CACHE_KEY = 'proxy';
|
||||||
private readonly CACHE_TTL = 86400; // 24 hours
|
private readonly CACHE_TTL = 86400; // 24 hours
|
||||||
private readonly CHECK_TIMEOUT = 7000;
|
private readonly CHECK_TIMEOUT = 10000;
|
||||||
private readonly CHECK_IP = '99.246.102.205'
|
private readonly CHECK_IP = '99.246.102.205'
|
||||||
private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
|
private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
|
||||||
private readonly PROXY_SOURCES = [
|
private readonly PROXY_SOURCES = [
|
||||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http' },
|
{url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
// {url: 'https://github.com/zloi-user/hideip.me/raw/refs/heads/master/http.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http' },
|
||||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/http.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
||||||
// {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
{url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', },
|
||||||
|
{url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
||||||
|
{url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http' },
|
||||||
|
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
||||||
|
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
||||||
|
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
||||||
|
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
||||||
|
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
||||||
|
|
||||||
// {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
|
|
||||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
|
{url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
|
||||||
// {url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' },
|
{url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
|
||||||
// {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', },
|
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
|
||||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
{url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' },
|
||||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
{url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', },
|
||||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
{url: 'https://github.com/officialputuid/KangProxy/blob/KangProxy/https/https.txt',protocol: 'https', },
|
||||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', },
|
||||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
// {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', },
|
||||||
{url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks4.txt',protocol: 'socks4', },
|
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
||||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||||
{url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks5.txt',protocol: 'socks5', },
|
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
||||||
|
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
||||||
|
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
||||||
|
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||||
|
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
||||||
|
// {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||||
]
|
]
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
|
|
@ -62,14 +74,15 @@ export class ProxyService {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async fetchProxiesFromSources() : Promise<boolean> {
|
async fetchProxiesFromSources() : Promise<number> {
|
||||||
const sources = this.PROXY_SOURCES.map(source =>
|
const sources = this.PROXY_SOURCES.map(source =>
|
||||||
this.concurrencyLimit(() => this.fetchProxiesFromSource(source))
|
this.concurrencyLimit(() => this.fetchProxiesFromSource(source))
|
||||||
)
|
)
|
||||||
const result = await Promise.all(sources);
|
const result = await Promise.all(sources);
|
||||||
const allProxies: ProxyInfo[] = result.flat();
|
let allProxies: ProxyInfo[] = result.flat();
|
||||||
await this.checkProxies(this.removeDuplicateProxies(allProxies))
|
allProxies = this.removeDuplicateProxies(allProxies)
|
||||||
return true
|
await this.checkProxies(allProxies)
|
||||||
|
return allProxies.length
|
||||||
}
|
}
|
||||||
|
|
||||||
private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
|
private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
|
||||||
|
|
@ -106,8 +119,10 @@ export class ProxyService {
|
||||||
const text = response.data;
|
const text = response.data;
|
||||||
const lines = text.split('\n').filter((line: string) => line.trim());
|
const lines = text.split('\n').filter((line: string) => line.trim());
|
||||||
|
|
||||||
|
|
||||||
for (const line of lines) {
|
for (const line of lines) {
|
||||||
const trimmed = line.trim();
|
let trimmed = line.trim();
|
||||||
|
trimmed = this.cleanProxyUrl(trimmed);
|
||||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||||
|
|
||||||
// Parse formats like "host:port" or "host:port:user:pass"
|
// Parse formats like "host:port" or "host:port:user:pass"
|
||||||
|
|
@ -132,15 +147,24 @@ export class ProxyService {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
this.logger.info(`Total proxies fetched: ${allProxies.length}`);
|
this.logger.info(`Total proxies fetched: ${allProxies.length}`);
|
||||||
return allProxies;
|
return allProxies;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private cleanProxyUrl(url: string): string {
|
||||||
|
// Remove http:// or https:// and any leading zeros from the host part
|
||||||
|
return url
|
||||||
|
.replace(/^https?:\/\//, '') // Remove protocol
|
||||||
|
.replace(/^0+/, '') // Remove leading zeros at start
|
||||||
|
.replace(/:0+(\d)/g, ':$1'); // Remove leading zeros from port numbers
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a proxy is working
|
* Check if a proxy is working
|
||||||
*/
|
*/
|
||||||
async checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
async checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||||
console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount);
|
// console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount);
|
||||||
try {
|
try {
|
||||||
|
|
||||||
// Test the proxy
|
// Test the proxy
|
||||||
|
|
@ -157,12 +181,13 @@ export class ProxyService {
|
||||||
checkedAt: new Date(),
|
checkedAt: new Date(),
|
||||||
responseTime: response.responseTime,
|
responseTime: response.responseTime,
|
||||||
};
|
};
|
||||||
|
// console.log('Proxy check result:', proxy);
|
||||||
if (isWorking && !response.data.includes('CHECK_IP')) {
|
if (isWorking && !JSON.stringify(response.data).includes(this.CHECK_IP)) {
|
||||||
await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL);
|
await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL);
|
||||||
} else {
|
|
||||||
await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
|
||||||
}
|
}
|
||||||
|
// else { // TODO
|
||||||
|
// await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||||
|
// }
|
||||||
|
|
||||||
this.logger.debug('Proxy check completed', {
|
this.logger.debug('Proxy check completed', {
|
||||||
host: proxy.host,
|
host: proxy.host,
|
||||||
|
|
|
||||||
|
|
@ -1,817 +0,0 @@
|
||||||
/**
|
|
||||||
* Example usage of the Stock Bot configuration library
|
|
||||||
*
|
|
||||||
* This file demonstrates how to use the Zod-based configuration
|
|
||||||
* system for various services in the Stock Bot platform.
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Import all the configuration modules
|
|
||||||
import {
|
|
||||||
// Core utilities
|
|
||||||
loadEnvVariables,
|
|
||||||
getEnvironment,
|
|
||||||
Environment,
|
|
||||||
ConfigurationError,
|
|
||||||
} from './core';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// PostgreSQL configuration
|
|
||||||
postgresConfig,
|
|
||||||
PostgresConfig,
|
|
||||||
POSTGRES_HOST,
|
|
||||||
POSTGRES_PORT,
|
|
||||||
POSTGRES_DATABASE,
|
|
||||||
POSTGRES_USERNAME,
|
|
||||||
POSTGRES_PASSWORD,
|
|
||||||
} from './postgres';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// QuestDB configuration
|
|
||||||
questdbConfig,
|
|
||||||
QuestDbConfig,
|
|
||||||
QUESTDB_HOST,
|
|
||||||
QUESTDB_HTTP_PORT,
|
|
||||||
QUESTDB_PG_PORT,
|
|
||||||
} from './questdb';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// MongoDB configuration
|
|
||||||
mongodbConfig,
|
|
||||||
MongoDbConfig,
|
|
||||||
MONGODB_HOST,
|
|
||||||
MONGODB_PORT,
|
|
||||||
MONGODB_DATABASE,
|
|
||||||
MONGODB_USERNAME,
|
|
||||||
} from './mongodb';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// Dragonfly configuration
|
|
||||||
dragonflyConfig,
|
|
||||||
DragonflyConfig,
|
|
||||||
DRAGONFLY_HOST,
|
|
||||||
DRAGONFLY_PORT,
|
|
||||||
DRAGONFLY_DATABASE,
|
|
||||||
} from './dragonfly';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// Monitoring configuration
|
|
||||||
prometheusConfig,
|
|
||||||
grafanaConfig,
|
|
||||||
PrometheusConfig,
|
|
||||||
GrafanaConfig,
|
|
||||||
PROMETHEUS_HOST,
|
|
||||||
PROMETHEUS_PORT,
|
|
||||||
GRAFANA_HOST,
|
|
||||||
GRAFANA_PORT,
|
|
||||||
} from './monitoring';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// Loki configuration
|
|
||||||
lokiConfig,
|
|
||||||
LokiConfig,
|
|
||||||
LOKI_HOST,
|
|
||||||
LOKI_PORT,
|
|
||||||
LOKI_SERVICE_LABEL,
|
|
||||||
LOKI_BATCH_SIZE,
|
|
||||||
} from './loki';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// Logging configuration
|
|
||||||
loggingConfig,
|
|
||||||
LoggingConfig,
|
|
||||||
LOG_LEVEL,
|
|
||||||
LOG_FORMAT,
|
|
||||||
LOG_CONSOLE,
|
|
||||||
LOG_FILE,
|
|
||||||
LOG_SERVICE_NAME,
|
|
||||||
} from './logging';
|
|
||||||
|
|
||||||
|
|
||||||
import {
|
|
||||||
// Risk management configuration
|
|
||||||
riskConfig,
|
|
||||||
RiskConfig,
|
|
||||||
RISK_MAX_POSITION_SIZE,
|
|
||||||
RISK_DEFAULT_STOP_LOSS,
|
|
||||||
RISK_CIRCUIT_BREAKER_ENABLED,
|
|
||||||
} from './risk';
|
|
||||||
|
|
||||||
import {
|
|
||||||
// Data provider configuration
|
|
||||||
dataProvidersConfig,
|
|
||||||
DataProvidersConfig,
|
|
||||||
getProviderConfig,
|
|
||||||
getEnabledProviders,
|
|
||||||
getDefaultProvider,
|
|
||||||
DEFAULT_DATA_PROVIDER,
|
|
||||||
ALPACA_API_KEY,
|
|
||||||
} from './data-providers';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 1: Basic usage - Load environment variables and get configuration
|
|
||||||
*/
|
|
||||||
function basicUsageExample() {
|
|
||||||
console.log('=== Basic Usage Example ===');
|
|
||||||
|
|
||||||
// Load environment variables (optional - they're loaded automatically)
|
|
||||||
loadEnvVariables();
|
|
||||||
|
|
||||||
// Get the current environment
|
|
||||||
const env = getEnvironment();
|
|
||||||
console.log(`Current environment: ${env}`);
|
|
||||||
// Access individual configuration values
|
|
||||||
console.log(`Database host: ${POSTGRES_HOST}`);
|
|
||||||
console.log(`Database port: ${POSTGRES_PORT}`);
|
|
||||||
console.log(`Log level: ${LOG_LEVEL}`);
|
|
||||||
|
|
||||||
// Access full database config objects
|
|
||||||
console.log(`Full database config:`, {
|
|
||||||
host: postgresConfig.POSTGRES_HOST,
|
|
||||||
port: postgresConfig.POSTGRES_PORT,
|
|
||||||
name: postgresConfig.POSTGRES_DATABASE,
|
|
||||||
ssl: postgresConfig.POSTGRES_SSL,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 2: Using configuration in a database connection
|
|
||||||
*/
|
|
||||||
async function databaseConnectionExample() {
|
|
||||||
console.log('=== Database Connection Example ===');
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Use the database configuration to create a connection string
|
|
||||||
const connectionString = `postgresql://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}`;
|
|
||||||
|
|
||||||
console.log('Database connection settings:');
|
|
||||||
console.log(`- Host: ${postgresConfig.POSTGRES_HOST}`);
|
|
||||||
console.log(`- Port: ${postgresConfig.POSTGRES_PORT}`);
|
|
||||||
console.log(`- Database: ${postgresConfig.POSTGRES_DATABASE}`);
|
|
||||||
console.log(`- SSL enabled: ${postgresConfig.POSTGRES_SSL}`);
|
|
||||||
console.log(`- Pool max connections: ${postgresConfig.POSTGRES_POOL_MAX}`);
|
|
||||||
console.log(`- Query timeout: ${postgresConfig.POSTGRES_QUERY_TIMEOUT}ms`);
|
|
||||||
|
|
||||||
// Example pool configuration
|
|
||||||
const poolConfig = {
|
|
||||||
host: postgresConfig.POSTGRES_HOST,
|
|
||||||
port: postgresConfig.POSTGRES_PORT,
|
|
||||||
database: postgresConfig.POSTGRES_DATABASE,
|
|
||||||
user: postgresConfig.POSTGRES_USERNAME,
|
|
||||||
password: postgresConfig.POSTGRES_PASSWORD,
|
|
||||||
ssl: postgresConfig.POSTGRES_SSL,
|
|
||||||
min: postgresConfig.POSTGRES_POOL_MIN,
|
|
||||||
max: postgresConfig.POSTGRES_POOL_MAX,
|
|
||||||
idleTimeoutMillis: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT,
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('Pool configuration:', poolConfig);
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Database configuration error:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 3: Logging setup example
|
|
||||||
*/
|
|
||||||
function loggingSetupExample() {
|
|
||||||
console.log('=== Logging Setup Example ===');
|
|
||||||
|
|
||||||
// Access logging configuration
|
|
||||||
console.log('Logging settings:');
|
|
||||||
console.log(`- Level: ${loggingConfig.LOG_LEVEL}`);
|
|
||||||
console.log(`- Format: ${loggingConfig.LOG_FORMAT}`);
|
|
||||||
console.log(`- Console enabled: ${loggingConfig.LOG_CONSOLE}`);
|
|
||||||
console.log(`- File logging: ${loggingConfig.LOG_FILE}`);
|
|
||||||
console.log(`- Service name: ${loggingConfig.LOG_SERVICE_NAME}`);
|
|
||||||
// Example logger configuration
|
|
||||||
const loggerConfig = {
|
|
||||||
level: loggingConfig.LOG_LEVEL,
|
|
||||||
format: loggingConfig.LOG_FORMAT,
|
|
||||||
transports: [] as any[],
|
|
||||||
defaultMeta: {
|
|
||||||
service: loggingConfig.LOG_SERVICE_NAME,
|
|
||||||
version: loggingConfig.LOG_SERVICE_VERSION,
|
|
||||||
environment: loggingConfig.LOG_ENVIRONMENT,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
if (loggingConfig.LOG_CONSOLE) {
|
|
||||||
loggerConfig.transports.push({
|
|
||||||
type: 'console',
|
|
||||||
format: loggingConfig.LOG_FORMAT,
|
|
||||||
timestamp: loggingConfig.LOG_TIMESTAMP,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (loggingConfig.LOG_FILE) {
|
|
||||||
loggerConfig.transports.push({
|
|
||||||
type: 'file',
|
|
||||||
filename: `${loggingConfig.LOG_FILE_PATH}/application.log`,
|
|
||||||
maxSize: loggingConfig.LOG_FILE_MAX_SIZE,
|
|
||||||
maxFiles: loggingConfig.LOG_FILE_MAX_FILES,
|
|
||||||
datePattern: loggingConfig.LOG_FILE_DATE_PATTERN,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example Loki transport configuration
|
|
||||||
if (lokiConfig.LOKI_HOST) {
|
|
||||||
loggerConfig.transports.push({
|
|
||||||
type: 'loki',
|
|
||||||
host: lokiConfig.LOKI_HOST,
|
|
||||||
port: lokiConfig.LOKI_PORT,
|
|
||||||
batchSize: lokiConfig.LOKI_BATCH_SIZE,
|
|
||||||
labels: {
|
|
||||||
service: lokiConfig.LOKI_SERVICE_LABEL,
|
|
||||||
environment: lokiConfig.LOKI_ENVIRONMENT_LABEL,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
console.log('Logger configuration:', loggerConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 4: Risk management configuration
|
|
||||||
*/
|
|
||||||
function riskManagementExample() {
|
|
||||||
console.log('=== Risk Management Example ===');
|
|
||||||
|
|
||||||
// Access risk configuration
|
|
||||||
console.log('Risk management settings:');
|
|
||||||
console.log(`- Max position size: ${RISK_MAX_POSITION_SIZE * 100}%`);
|
|
||||||
console.log(`- Default stop loss: ${RISK_DEFAULT_STOP_LOSS * 100}%`);
|
|
||||||
console.log(`- Circuit breaker enabled: ${RISK_CIRCUIT_BREAKER_ENABLED}`);
|
|
||||||
console.log(`- Max leverage: ${riskConfig.RISK_MAX_LEVERAGE}x`);
|
|
||||||
|
|
||||||
// Example risk calculator
|
|
||||||
function calculatePositionSize(portfolioValue: number, riskPerTrade: number = RISK_DEFAULT_STOP_LOSS) {
|
|
||||||
const maxPositionValue = portfolioValue * RISK_MAX_POSITION_SIZE;
|
|
||||||
const riskAmount = portfolioValue * riskPerTrade;
|
|
||||||
|
|
||||||
return {
|
|
||||||
maxPositionValue,
|
|
||||||
riskAmount,
|
|
||||||
maxShares: Math.floor(maxPositionValue / 100), // Assuming $100 per share
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const portfolioValue = 100000; // $100k portfolio
|
|
||||||
const position = calculatePositionSize(portfolioValue);
|
|
||||||
console.log(`Position sizing for $${portfolioValue} portfolio:`, position);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 5: Data provider configuration
|
|
||||||
*/
|
|
||||||
function dataProviderExample() {
|
|
||||||
console.log('=== Data Provider Example ===');
|
|
||||||
|
|
||||||
// Get the default provider
|
|
||||||
const defaultProvider = getDefaultProvider();
|
|
||||||
console.log('Default provider:', defaultProvider);
|
|
||||||
|
|
||||||
// Get all enabled providers
|
|
||||||
const enabledProviders = getEnabledProviders();
|
|
||||||
console.log('Enabled providers:', enabledProviders.map(p => p.name));
|
|
||||||
|
|
||||||
// Get specific provider configuration
|
|
||||||
try {
|
|
||||||
const alpacaConfig = getProviderConfig('alpaca');
|
|
||||||
console.log('Alpaca configuration:', {
|
|
||||||
enabled: alpacaConfig.enabled,
|
|
||||||
baseUrl: alpacaConfig.baseUrl,
|
|
||||||
hasApiKey: !!alpacaConfig.apiKey,
|
|
||||||
rateLimit: alpacaConfig.rateLimits,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error getting Alpaca config:', error);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example API client setup
|
|
||||||
const apiClients = enabledProviders.map(provider => ({
|
|
||||||
name: provider.name,
|
|
||||||
client: {
|
|
||||||
baseURL: provider.baseUrl,
|
|
||||||
timeout: dataProvidersConfig.DATA_PROVIDER_TIMEOUT,
|
|
||||||
retries: dataProvidersConfig.DATA_PROVIDER_RETRIES,
|
|
||||||
retryDelay: dataProvidersConfig.DATA_PROVIDER_RETRY_DELAY,
|
|
||||||
headers: provider.apiKey ? {
|
|
||||||
'Authorization': `Bearer ${provider.apiKey}`
|
|
||||||
} : {},
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
console.log('API clients configuration:', apiClients);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 6: Environment-specific configuration
|
|
||||||
*/
|
|
||||||
function environmentSpecificExample() {
|
|
||||||
console.log('=== Environment-Specific Example ===');
|
|
||||||
|
|
||||||
const env = getEnvironment();
|
|
||||||
|
|
||||||
switch (env) {
|
|
||||||
case Environment.Development:
|
|
||||||
console.log('Development environment detected');
|
|
||||||
console.log('- Using local database');
|
|
||||||
console.log('- Verbose logging enabled');
|
|
||||||
console.log('- Paper trading mode');
|
|
||||||
break;
|
|
||||||
|
|
||||||
case Environment.Testing:
|
|
||||||
console.log('Testing environment detected');
|
|
||||||
console.log('- Using test database');
|
|
||||||
console.log('- Structured logging');
|
|
||||||
console.log('- Mock data providers');
|
|
||||||
break;
|
|
||||||
|
|
||||||
case Environment.Staging:
|
|
||||||
console.log('Staging environment detected');
|
|
||||||
console.log('- Using staging database');
|
|
||||||
console.log('- Production-like settings');
|
|
||||||
console.log('- Real data providers (limited)');
|
|
||||||
break;
|
|
||||||
|
|
||||||
case Environment.Production:
|
|
||||||
console.log('Production environment detected');
|
|
||||||
console.log('- Using production database');
|
|
||||||
console.log('- Optimized logging');
|
|
||||||
console.log('- Live trading enabled');
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Example of environment-specific behavior
|
|
||||||
const isProduction = env === Environment.Production;
|
|
||||||
const tradingMode = isProduction ? 'live' : 'paper';
|
|
||||||
const logLevel = isProduction ? 'info' : 'debug';
|
|
||||||
|
|
||||||
console.log(`Trading mode: ${tradingMode}`);
|
|
||||||
console.log(`Recommended log level: ${logLevel}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 7: Configuration validation and error handling
|
|
||||||
*/
|
|
||||||
function configurationValidationExample() {
|
|
||||||
console.log('=== Configuration Validation Example ===');
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Check required configurations
|
|
||||||
if (!ALPACA_API_KEY && DEFAULT_DATA_PROVIDER === 'alpaca') {
|
|
||||||
throw new ConfigurationError('Alpaca API key is required when using Alpaca as default provider');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate risk settings
|
|
||||||
if (RISK_MAX_POSITION_SIZE > 1.0) {
|
|
||||||
throw new ConfigurationError('Maximum position size cannot exceed 100%');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (riskConfig.RISK_DEFAULT_STOP_LOSS > riskConfig.RISK_DEFAULT_TAKE_PROFIT) {
|
|
||||||
console.warn('Warning: Stop loss is greater than take profit - check your risk settings');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate database connection settings
|
|
||||||
if (postgresConfig.POSTGRES_POOL_MAX < postgresConfig.POSTGRES_POOL_MIN) {
|
|
||||||
throw new ConfigurationError('Database max pool size must be greater than min pool size');
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('✅ All configuration validations passed');
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
if (error instanceof ConfigurationError) {
|
|
||||||
console.error('❌ Configuration error:', error.message);
|
|
||||||
} else {
|
|
||||||
console.error('❌ Unexpected error:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 8: QuestDB time-series database configuration
|
|
||||||
*/
|
|
||||||
function questdbConfigurationExample() {
|
|
||||||
console.log('=== QuestDB Configuration Example ===');
|
|
||||||
|
|
||||||
// Access QuestDB configuration
|
|
||||||
console.log('QuestDB settings:');
|
|
||||||
console.log(`- Host: ${questdbConfig.QUESTDB_HOST}`);
|
|
||||||
console.log(`- HTTP port (web console): ${questdbConfig.QUESTDB_HTTP_PORT}`);
|
|
||||||
console.log(`- PostgreSQL port: ${questdbConfig.QUESTDB_PG_PORT}`);
|
|
||||||
console.log(`- InfluxDB port: ${questdbConfig.QUESTDB_INFLUX_PORT}`);
|
|
||||||
console.log(`- TLS enabled: ${questdbConfig.QUESTDB_TLS_ENABLED}`);
|
|
||||||
|
|
||||||
// Example QuestDB client configuration
|
|
||||||
const questdbClientConfig = {
|
|
||||||
http: {
|
|
||||||
host: questdbConfig.QUESTDB_HOST,
|
|
||||||
port: questdbConfig.QUESTDB_HTTP_PORT,
|
|
||||||
tls: questdbConfig.QUESTDB_TLS_ENABLED,
|
|
||||||
timeout: questdbConfig.QUESTDB_REQUEST_TIMEOUT,
|
|
||||||
},
|
|
||||||
postgresql: {
|
|
||||||
host: questdbConfig.QUESTDB_HOST,
|
|
||||||
port: questdbConfig.QUESTDB_PG_PORT,
|
|
||||||
database: questdbConfig.QUESTDB_DEFAULT_DATABASE,
|
|
||||||
user: questdbConfig.QUESTDB_USER,
|
|
||||||
password: questdbConfig.QUESTDB_PASSWORD,
|
|
||||||
},
|
|
||||||
influxdb: {
|
|
||||||
host: questdbConfig.QUESTDB_HOST,
|
|
||||||
port: questdbConfig.QUESTDB_INFLUX_PORT,
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('QuestDB client configuration:', questdbClientConfig);
|
|
||||||
|
|
||||||
// Example time-series table creation
|
|
||||||
const createTableQuery = `
|
|
||||||
CREATE TABLE IF NOT EXISTS ohlcv_data (
|
|
||||||
timestamp TIMESTAMP,
|
|
||||||
symbol SYMBOL,
|
|
||||||
open DOUBLE,
|
|
||||||
high DOUBLE,
|
|
||||||
low DOUBLE,
|
|
||||||
close DOUBLE,
|
|
||||||
volume LONG
|
|
||||||
) timestamp(timestamp) PARTITION BY DAY;
|
|
||||||
`;
|
|
||||||
|
|
||||||
console.log('Example table creation query:', createTableQuery);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 9: MongoDB document database configuration
|
|
||||||
*/
|
|
||||||
function mongodbConfigurationExample() {
|
|
||||||
console.log('=== MongoDB Configuration Example ===');
|
|
||||||
|
|
||||||
// Access MongoDB configuration
|
|
||||||
console.log('MongoDB settings:');
|
|
||||||
console.log(`- Host: ${mongodbConfig.MONGODB_HOST}`);
|
|
||||||
console.log(`- Port: ${mongodbConfig.MONGODB_PORT}`);
|
|
||||||
console.log(`- Database: ${mongodbConfig.MONGODB_DATABASE}`);
|
|
||||||
console.log(`- Username: ${mongodbConfig.MONGODB_USERNAME}`);
|
|
||||||
console.log(`- TLS enabled: ${mongodbConfig.MONGODB_TLS}`);
|
|
||||||
console.log(`- Max pool size: ${mongodbConfig.MONGODB_MAX_POOL_SIZE}`);
|
|
||||||
|
|
||||||
// Build connection URI
|
|
||||||
const buildMongoUri = () => {
|
|
||||||
if (mongodbConfig.MONGODB_URI) {
|
|
||||||
return mongodbConfig.MONGODB_URI;
|
|
||||||
}
|
|
||||||
|
|
||||||
const auth = mongodbConfig.MONGODB_USERNAME && mongodbConfig.MONGODB_PASSWORD
|
|
||||||
? `${mongodbConfig.MONGODB_USERNAME}:${mongodbConfig.MONGODB_PASSWORD}@`
|
|
||||||
: '';
|
|
||||||
|
|
||||||
const tls = mongodbConfig.MONGODB_TLS ? '?tls=true' : '';
|
|
||||||
|
|
||||||
return `mongodb://${auth}${mongodbConfig.MONGODB_HOST}:${mongodbConfig.MONGODB_PORT}/${mongodbConfig.MONGODB_DATABASE}${tls}`;
|
|
||||||
};
|
|
||||||
|
|
||||||
const mongoUri = buildMongoUri();
|
|
||||||
console.log('MongoDB connection URI:', mongoUri.replace(/:[^:@]*@/, ':***@')); // Hide password
|
|
||||||
|
|
||||||
// Example MongoDB client configuration
|
|
||||||
const mongoClientConfig = {
|
|
||||||
maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
|
|
||||||
minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE,
|
|
||||||
maxIdleTimeMS: mongodbConfig.MONGODB_MAX_IDLE_TIME,
|
|
||||||
connectTimeoutMS: mongodbConfig.MONGODB_CONNECT_TIMEOUT,
|
|
||||||
socketTimeoutMS: mongodbConfig.MONGODB_SOCKET_TIMEOUT,
|
|
||||||
serverSelectionTimeoutMS: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT,
|
|
||||||
retryWrites: mongodbConfig.MONGODB_RETRY_WRITES,
|
|
||||||
w: mongodbConfig.MONGODB_WRITE_CONCERN,
|
|
||||||
readPreference: mongodbConfig.MONGODB_READ_PREFERENCE,
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('MongoDB client configuration:', mongoClientConfig);
|
|
||||||
|
|
||||||
// Example collections structure
|
|
||||||
const collections = [
|
|
||||||
'sentiment_data', // News sentiment analysis
|
|
||||||
'market_news', // Raw news articles
|
|
||||||
'social_signals', // Social media signals
|
|
||||||
'earnings_reports', // Earnings data
|
|
||||||
'analyst_ratings', // Analyst recommendations
|
|
||||||
];
|
|
||||||
|
|
||||||
console.log('Example collections:', collections);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 10: Dragonfly (Redis replacement) configuration
|
|
||||||
*/
|
|
||||||
function dragonflyConfigurationExample() {
|
|
||||||
console.log('=== Dragonfly Configuration Example ===');
|
|
||||||
|
|
||||||
// Access Dragonfly configuration
|
|
||||||
console.log('Dragonfly settings:');
|
|
||||||
console.log(`- Host: ${dragonflyConfig.DRAGONFLY_HOST}`);
|
|
||||||
console.log(`- Port: ${dragonflyConfig.DRAGONFLY_PORT}`);
|
|
||||||
console.log(`- Database: ${dragonflyConfig.DRAGONFLY_DATABASE}`);
|
|
||||||
console.log(`- Cache mode: ${dragonflyConfig.DRAGONFLY_CACHE_MODE}`);
|
|
||||||
console.log(`- Max memory: ${dragonflyConfig.DRAGONFLY_MAX_MEMORY}`);
|
|
||||||
console.log(`- Pool size: ${dragonflyConfig.DRAGONFLY_POOL_SIZE}`);
|
|
||||||
// Example Dragonfly client configuration
|
|
||||||
const dragonflyClientConfig = {
|
|
||||||
host: dragonflyConfig.DRAGONFLY_HOST,
|
|
||||||
port: dragonflyConfig.DRAGONFLY_PORT,
|
|
||||||
db: dragonflyConfig.DRAGONFLY_DATABASE,
|
|
||||||
password: dragonflyConfig.DRAGONFLY_PASSWORD || undefined,
|
|
||||||
username: dragonflyConfig.DRAGONFLY_USERNAME || undefined,
|
|
||||||
retryDelayOnFailover: dragonflyConfig.DRAGONFLY_RETRY_DELAY,
|
|
||||||
maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES,
|
|
||||||
connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT,
|
|
||||||
commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT,
|
|
||||||
enableAutoPipelining: true,
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('Dragonfly client configuration:', dragonflyClientConfig);
|
|
||||||
|
|
||||||
// Example cache key patterns
|
|
||||||
const cachePatterns = {
|
|
||||||
marketData: 'market:{symbol}:{timeframe}',
|
|
||||||
indicators: 'indicators:{symbol}:{indicator}:{period}',
|
|
||||||
positions: 'positions:{account_id}',
|
|
||||||
orders: 'orders:{order_id}',
|
|
||||||
rateLimit: 'rate_limit:{provider}:{endpoint}',
|
|
||||||
sessions: 'session:{user_id}',
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('Example cache key patterns:', cachePatterns);
|
|
||||||
|
|
||||||
// Example TTL configurations
|
|
||||||
const ttlConfigs = {
|
|
||||||
marketData: 60, // 1 minute for real-time data
|
|
||||||
indicators: 300, // 5 minutes for calculated indicators
|
|
||||||
positions: 30, // 30 seconds for positions
|
|
||||||
orders: 86400, // 1 day for order history
|
|
||||||
rateLimit: 3600, // 1 hour for rate limiting
|
|
||||||
sessions: 1800, // 30 minutes for user sessions
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('Example TTL configurations (seconds):', ttlConfigs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 11: Monitoring stack configuration (Prometheus, Grafana, Loki)
|
|
||||||
*/
|
|
||||||
function monitoringConfigurationExample() {
|
|
||||||
console.log('=== Monitoring Configuration Example ===');
|
|
||||||
|
|
||||||
// Prometheus configuration
|
|
||||||
console.log('Prometheus settings:');
|
|
||||||
console.log(`- Host: ${prometheusConfig.PROMETHEUS_HOST}`);
|
|
||||||
console.log(`- Port: ${prometheusConfig.PROMETHEUS_PORT}`);
|
|
||||||
console.log(`- Scrape interval: ${prometheusConfig.PROMETHEUS_SCRAPE_INTERVAL}`);
|
|
||||||
console.log(`- Retention time: ${prometheusConfig.PROMETHEUS_RETENTION_TIME}`);
|
|
||||||
|
|
||||||
// Grafana configuration
|
|
||||||
console.log('\nGrafana settings:');
|
|
||||||
console.log(`- Host: ${grafanaConfig.GRAFANA_HOST}`);
|
|
||||||
console.log(`- Port: ${grafanaConfig.GRAFANA_PORT}`);
|
|
||||||
console.log(`- Admin user: ${grafanaConfig.GRAFANA_ADMIN_USER}`);
|
|
||||||
console.log(`- Allow sign up: ${grafanaConfig.GRAFANA_ALLOW_SIGN_UP}`);
|
|
||||||
console.log(`- Database type: ${grafanaConfig.GRAFANA_DATABASE_TYPE}`);
|
|
||||||
|
|
||||||
// Loki configuration
|
|
||||||
console.log('\nLoki settings:');
|
|
||||||
console.log(`- Host: ${lokiConfig.LOKI_HOST}`);
|
|
||||||
console.log(`- Port: ${lokiConfig.LOKI_PORT}`);
|
|
||||||
console.log(`- Batch size: ${lokiConfig.LOKI_BATCH_SIZE}`);
|
|
||||||
console.log(`- Retention period: ${lokiConfig.LOKI_RETENTION_PERIOD}`);
|
|
||||||
|
|
||||||
// Example monitoring endpoints
|
|
||||||
const monitoringEndpoints = {
|
|
||||||
prometheus: `http://${prometheusConfig.PROMETHEUS_HOST}:${prometheusConfig.PROMETHEUS_PORT}`,
|
|
||||||
grafana: `http://${grafanaConfig.GRAFANA_HOST}:${grafanaConfig.GRAFANA_PORT}`,
|
|
||||||
loki: `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`,
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('\nMonitoring endpoints:', monitoringEndpoints);
|
|
||||||
|
|
||||||
// Example metrics configuration
|
|
||||||
const metricsConfig = {
|
|
||||||
defaultLabels: {
|
|
||||||
service: 'stock-bot',
|
|
||||||
environment: getEnvironment(),
|
|
||||||
version: process.env.npm_package_version || '1.0.0',
|
|
||||||
},
|
|
||||||
collectDefaultMetrics: true,
|
|
||||||
prefix: 'stockbot_',
|
|
||||||
buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60], // Response time buckets in seconds
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('Example metrics configuration:', metricsConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 12: Multi-database service configuration
|
|
||||||
*/
|
|
||||||
function multiDatabaseServiceExample() {
|
|
||||||
console.log('=== Multi-Database Service Example ===');
|
|
||||||
|
|
||||||
// Complete database configuration for a microservice
|
|
||||||
const serviceConfig = {
|
|
||||||
service: {
|
|
||||||
name: 'market-data-processor',
|
|
||||||
version: '1.0.0',
|
|
||||||
environment: getEnvironment(),
|
|
||||||
},
|
|
||||||
|
|
||||||
// PostgreSQL for operational data
|
|
||||||
postgresql: {
|
|
||||||
host: postgresConfig.POSTGRES_HOST,
|
|
||||||
port: postgresConfig.POSTGRES_PORT,
|
|
||||||
database: postgresConfig.POSTGRES_DATABASE,
|
|
||||||
username: postgresConfig.POSTGRES_USERNAME,
|
|
||||||
password: postgresConfig.POSTGRES_PASSWORD,
|
|
||||||
ssl: postgresConfig.POSTGRES_SSL,
|
|
||||||
pool: {
|
|
||||||
min: postgresConfig.POSTGRES_POOL_MIN,
|
|
||||||
max: postgresConfig.POSTGRES_POOL_MAX,
|
|
||||||
idleTimeout: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
|
|
||||||
// QuestDB for time-series data
|
|
||||||
questdb: {
|
|
||||||
host: questdbConfig.QUESTDB_HOST,
|
|
||||||
httpPort: questdbConfig.QUESTDB_HTTP_PORT,
|
|
||||||
pgPort: questdbConfig.QUESTDB_PG_PORT,
|
|
||||||
database: questdbConfig.QUESTDB_DEFAULT_DATABASE,
|
|
||||||
timeout: questdbConfig.QUESTDB_REQUEST_TIMEOUT,
|
|
||||||
},
|
|
||||||
|
|
||||||
// MongoDB for document storage
|
|
||||||
mongodb: {
|
|
||||||
host: mongodbConfig.MONGODB_HOST,
|
|
||||||
port: mongodbConfig.MONGODB_PORT,
|
|
||||||
database: mongodbConfig.MONGODB_DATABASE,
|
|
||||||
username: mongodbConfig.MONGODB_USERNAME,
|
|
||||||
maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
|
|
||||||
readPreference: mongodbConfig.MONGODB_READ_PREFERENCE,
|
|
||||||
},
|
|
||||||
|
|
||||||
// Dragonfly for caching
|
|
||||||
dragonfly: {
|
|
||||||
host: dragonflyConfig.DRAGONFLY_HOST,
|
|
||||||
port: dragonflyConfig.DRAGONFLY_PORT,
|
|
||||||
database: dragonflyConfig.DRAGONFLY_DATABASE,
|
|
||||||
poolSize: dragonflyConfig.DRAGONFLY_POOL_SIZE,
|
|
||||||
commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT,
|
|
||||||
},
|
|
||||||
|
|
||||||
// Monitoring
|
|
||||||
monitoring: {
|
|
||||||
prometheus: {
|
|
||||||
pushGateway: `http://${prometheusConfig.PROMETHEUS_HOST}:${prometheusConfig.PROMETHEUS_PORT}`,
|
|
||||||
scrapeInterval: prometheusConfig.PROMETHEUS_SCRAPE_INTERVAL,
|
|
||||||
}, loki: {
|
|
||||||
host: lokiConfig.LOKI_HOST,
|
|
||||||
port: lokiConfig.LOKI_PORT,
|
|
||||||
batchSize: lokiConfig.LOKI_BATCH_SIZE,
|
|
||||||
labels: {
|
|
||||||
service: 'market-data-processor',
|
|
||||||
environment: getEnvironment(),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('Complete service configuration:', JSON.stringify(serviceConfig, null, 2));
|
|
||||||
|
|
||||||
// Example data flow
|
|
||||||
const dataFlow = {
|
|
||||||
ingestion: 'Market data → Dragonfly (cache) → QuestDB (storage)',
|
|
||||||
processing: 'QuestDB → Analysis → PostgreSQL (results) → MongoDB (metadata)',
|
|
||||||
serving: 'Dragonfly (cache) ← PostgreSQL/QuestDB ← API requests',
|
|
||||||
monitoring: 'All services → Prometheus → Grafana dashboards',
|
|
||||||
logging: 'All services → Loki → Grafana log viewer',
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('\nData flow patterns:', dataFlow);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Example 8: Creating a service configuration object
|
|
||||||
*/
|
|
||||||
function serviceConfigurationExample() {
|
|
||||||
console.log('=== Service Configuration Example ===');
|
|
||||||
|
|
||||||
// Example: Market Data Gateway service configuration
|
|
||||||
const marketDataGatewayConfig = {
|
|
||||||
service: {
|
|
||||||
name: 'market-data-gateway',
|
|
||||||
port: 3001,
|
|
||||||
environment: getEnvironment(),
|
|
||||||
},
|
|
||||||
database: {
|
|
||||||
host: postgresConfig.POSTGRES_HOST,
|
|
||||||
port: postgresConfig.POSTGRES_PORT,
|
|
||||||
name: postgresConfig.POSTGRES_DATABASE,
|
|
||||||
ssl: postgresConfig.POSTGRES_SSL,
|
|
||||||
}, logging: {
|
|
||||||
level: loggingConfig.LOG_LEVEL,
|
|
||||||
console: loggingConfig.LOG_CONSOLE,
|
|
||||||
loki: {
|
|
||||||
host: lokiConfig.LOKI_HOST,
|
|
||||||
port: lokiConfig.LOKI_PORT,
|
|
||||||
labels: {
|
|
||||||
service: 'market-data-gateway',
|
|
||||||
environment: getEnvironment(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
dataProviders: {
|
|
||||||
default: DEFAULT_DATA_PROVIDER,
|
|
||||||
enabled: getEnabledProviders(),
|
|
||||||
timeout: dataProvidersConfig.DATA_PROVIDER_TIMEOUT,
|
|
||||||
retries: dataProvidersConfig.DATA_PROVIDER_RETRIES,
|
|
||||||
},
|
|
||||||
cache: {
|
|
||||||
enabled: dataProvidersConfig.DATA_CACHE_ENABLED,
|
|
||||||
ttl: dataProvidersConfig.DATA_CACHE_TTL,
|
|
||||||
maxSize: dataProvidersConfig.DATA_CACHE_MAX_SIZE,
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
console.log('Market Data Gateway configuration:', JSON.stringify(marketDataGatewayConfig, null, 2));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Main example runner
|
|
||||||
*/
|
|
||||||
function runAllExamples() {
|
|
||||||
console.log('🚀 Stock Bot Configuration Examples\n');
|
|
||||||
|
|
||||||
try {
|
|
||||||
basicUsageExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
databaseConnectionExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
loggingSetupExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
riskManagementExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
dataProviderExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
environmentSpecificExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
configurationValidationExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
questdbConfigurationExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
mongodbConfigurationExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
dragonflyConfigurationExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
monitoringConfigurationExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
multiDatabaseServiceExample();
|
|
||||||
console.log('\n');
|
|
||||||
|
|
||||||
serviceConfigurationExample();
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Example execution error:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export the examples for use in other files
|
|
||||||
export {
|
|
||||||
basicUsageExample,
|
|
||||||
databaseConnectionExample,
|
|
||||||
loggingSetupExample,
|
|
||||||
riskManagementExample,
|
|
||||||
dataProviderExample,
|
|
||||||
environmentSpecificExample,
|
|
||||||
configurationValidationExample,
|
|
||||||
questdbConfigurationExample,
|
|
||||||
mongodbConfigurationExample,
|
|
||||||
dragonflyConfigurationExample,
|
|
||||||
monitoringConfigurationExample,
|
|
||||||
multiDatabaseServiceExample,
|
|
||||||
serviceConfigurationExample,
|
|
||||||
runAllExamples,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Run examples if this file is executed directly
|
|
||||||
if (require.main === module) {
|
|
||||||
runAllExamples();
|
|
||||||
}
|
|
||||||
|
|
@ -7,7 +7,7 @@ import type {
|
||||||
import { HttpError } from './types.js';
|
import { HttpError } from './types.js';
|
||||||
import { ProxyManager } from './proxy-manager.js';
|
import { ProxyManager } from './proxy-manager.js';
|
||||||
import axios, { type AxiosResponse, AxiosError } from 'axios';
|
import axios, { type AxiosResponse, AxiosError } from 'axios';
|
||||||
import { clear } from 'node:console';
|
import { loggingConfig } from '@stock-bot/config';
|
||||||
|
|
||||||
export class HttpClient {
|
export class HttpClient {
|
||||||
private readonly config: HttpClientConfig;
|
private readonly config: HttpClientConfig;
|
||||||
|
|
@ -15,8 +15,9 @@ export class HttpClient {
|
||||||
|
|
||||||
constructor(config: HttpClientConfig = {}, logger?: Logger) {
|
constructor(config: HttpClientConfig = {}, logger?: Logger) {
|
||||||
this.config = config;
|
this.config = config;
|
||||||
this.logger = logger?.child({
|
this.logger = logger?.child({ //TODO fix pino levels
|
||||||
component: 'http',
|
component: 'http',
|
||||||
|
// level: loggingConfig?.LOG_LEVEL || 'info',
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -68,11 +69,11 @@ export class HttpClient {
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger?.warn('HTTP request failed', {
|
// this.logger?.warn('HTTP request failed', {
|
||||||
method: finalConfig.method,
|
// method: finalConfig.method,
|
||||||
url: finalConfig.url,
|
// url: finalConfig.url,
|
||||||
error: (error as Error).message,
|
// error: (error as Error).message,
|
||||||
});
|
// });
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -125,9 +126,9 @@ export class HttpClient {
|
||||||
const options = this.buildFetchOptions(config, signal);
|
const options = this.buildFetchOptions(config, signal);
|
||||||
|
|
||||||
this.logger?.debug('Making request with fetch: ', { url: config.url, options })
|
this.logger?.debug('Making request with fetch: ', { url: config.url, options })
|
||||||
|
// console.log(options)
|
||||||
const response = await fetch(config.url, options);
|
const response = await fetch(config.url, options);
|
||||||
|
// console.log('Fetch response:', response.status);
|
||||||
return this.parseFetchResponse<T>(response);
|
return this.parseFetchResponse<T>(response);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw signal.aborted
|
throw signal.aborted
|
||||||
|
|
@ -201,7 +202,6 @@ export class HttpClient {
|
||||||
if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) {
|
if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) {
|
||||||
(options as any).proxy = ProxyManager.createProxyUrl(config.proxy);
|
(options as any).proxy = ProxyManager.createProxyUrl(config.proxy);
|
||||||
}
|
}
|
||||||
|
|
||||||
return options;
|
return options;
|
||||||
} /**
|
} /**
|
||||||
* Build Axios options (for reference, though we're creating instance in ProxyManager)
|
* Build Axios options (for reference, though we're creating instance in ProxyManager)
|
||||||
|
|
|
||||||
|
|
@ -16,9 +16,6 @@ export class ProxyManager {
|
||||||
*/
|
*/
|
||||||
static createProxyUrl(proxy: ProxyInfo): string {
|
static createProxyUrl(proxy: ProxyInfo): string {
|
||||||
const { protocol, host, port, username, password } = proxy;
|
const { protocol, host, port, username, password } = proxy;
|
||||||
if(protocol.includes('socks')) {
|
|
||||||
return `${protocol}://${host}:${port}`;
|
|
||||||
}
|
|
||||||
if (username && password) {
|
if (username && password) {
|
||||||
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||||
}
|
}
|
||||||
|
|
@ -32,11 +29,10 @@ export class ProxyManager {
|
||||||
this.validateConfig(proxy);
|
this.validateConfig(proxy);
|
||||||
|
|
||||||
const proxyUrl = this.createProxyUrl(proxy);
|
const proxyUrl = this.createProxyUrl(proxy);
|
||||||
console.log(`Using proxy url: ${proxyUrl}`);
|
|
||||||
switch (proxy.protocol) {
|
switch (proxy.protocol) {
|
||||||
case 'socks4':
|
case 'socks4':
|
||||||
case 'socks5':
|
case 'socks5':
|
||||||
console.log(`Using SOCKS proxy: ${proxyUrl}`);
|
// console.log(`Using SOCKS proxy: ${proxyUrl}`);
|
||||||
return new SocksProxyAgent(proxyUrl);
|
return new SocksProxyAgent(proxyUrl);
|
||||||
case 'http':
|
case 'http':
|
||||||
return new HttpProxyAgent(proxyUrl);
|
return new HttpProxyAgent(proxyUrl);
|
||||||
|
|
@ -54,12 +50,6 @@ export class ProxyManager {
|
||||||
return {
|
return {
|
||||||
httpAgent: agent,
|
httpAgent: agent,
|
||||||
httpsAgent: agent,
|
httpsAgent: agent,
|
||||||
// timeout: 30000,
|
|
||||||
// validateStatus: () => true, // Don't throw errors on HTTP status codes
|
|
||||||
// maxRedirects: 5,
|
|
||||||
// headers: {
|
|
||||||
// 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
|
||||||
// }
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -200,9 +200,10 @@ export class Logger {
|
||||||
* Create child logger with additional context
|
* Create child logger with additional context
|
||||||
*/
|
*/
|
||||||
child(context: LogContext): Logger {
|
child(context: LogContext): Logger {
|
||||||
const childLogger = new Logger((this.pino.bindings() as any).service, { ...this.context, ...context });
|
// Create child logger that shares the same pino instance with additional context
|
||||||
// Use the pino child logger to properly propagate context
|
const childLogger = Object.create(Logger.prototype);
|
||||||
childLogger.pino = this.pino.child(context);
|
childLogger.context = { ...this.context, ...context };
|
||||||
|
childLogger.pino = this.pino.child(context); // Let pino handle level inheritance naturally
|
||||||
return childLogger;
|
return childLogger;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue