working on proxy service. getting closer
This commit is contained in:
parent
6380165a94
commit
afc1843fdb
5 changed files with 91 additions and 892 deletions
|
|
@ -7,50 +7,62 @@ export class ProxyService {
|
|||
private logger = new Logger('proxy-service');
|
||||
private cache: CacheProvider = createCache('hybrid');
|
||||
private httpClient: HttpClient;
|
||||
private readonly concurrencyLimit = pLimit(250);
|
||||
private readonly concurrencyLimit = pLimit(300);
|
||||
private readonly CACHE_KEY = 'proxy';
|
||||
private readonly CACHE_TTL = 86400; // 24 hours
|
||||
private readonly CHECK_TIMEOUT = 7000;
|
||||
private readonly CHECK_TIMEOUT = 10000;
|
||||
private readonly CHECK_IP = '99.246.102.205'
|
||||
private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955';
|
||||
private readonly PROXY_SOURCES = [
|
||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http' },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',protocol: 'http', },
|
||||
// {url: 'https://github.com/zloi-user/hideip.me/raw/refs/heads/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http' },
|
||||
{url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', },
|
||||
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http' },
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', },
|
||||
|
||||
// {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
|
||||
// {url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' },
|
||||
// {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
{url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
||||
{url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
|
||||
{url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
|
||||
{url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
|
||||
{url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
|
||||
{url: 'https://github.com/vakhov/fresh-proxy-list/blob/master/https.txt', protocol: 'https' },
|
||||
{url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', },
|
||||
{url: 'https://github.com/officialputuid/KangProxy/blob/KangProxy/https/https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks4',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt',protocol: 'socks4', },
|
||||
// {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/socks5',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks5.txt',protocol: 'socks5', },
|
||||
// {url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/socks5.txt',protocol: 'socks5', },
|
||||
]
|
||||
|
||||
constructor() {
|
||||
|
|
@ -62,14 +74,15 @@ export class ProxyService {
|
|||
}
|
||||
|
||||
|
||||
async fetchProxiesFromSources() : Promise<boolean> {
|
||||
async fetchProxiesFromSources() : Promise<number> {
|
||||
const sources = this.PROXY_SOURCES.map(source =>
|
||||
this.concurrencyLimit(() => this.fetchProxiesFromSource(source))
|
||||
)
|
||||
)
|
||||
const result = await Promise.all(sources);
|
||||
const allProxies: ProxyInfo[] = result.flat();
|
||||
await this.checkProxies(this.removeDuplicateProxies(allProxies))
|
||||
return true
|
||||
let allProxies: ProxyInfo[] = result.flat();
|
||||
allProxies = this.removeDuplicateProxies(allProxies)
|
||||
await this.checkProxies(allProxies)
|
||||
return allProxies.length
|
||||
}
|
||||
|
||||
private removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
|
||||
|
|
@ -106,8 +119,10 @@ export class ProxyService {
|
|||
const text = response.data;
|
||||
const lines = text.split('\n').filter((line: string) => line.trim());
|
||||
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
let trimmed = line.trim();
|
||||
trimmed = this.cleanProxyUrl(trimmed);
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
|
||||
// Parse formats like "host:port" or "host:port:user:pass"
|
||||
|
|
@ -131,16 +146,25 @@ export class ProxyService {
|
|||
this.logger.error(`Error fetching proxies from ${source.url}`, error);
|
||||
return [];
|
||||
}
|
||||
|
||||
|
||||
this.logger.info(`Total proxies fetched: ${allProxies.length}`);
|
||||
return allProxies;
|
||||
}
|
||||
|
||||
private cleanProxyUrl(url: string): string {
|
||||
// Remove http:// or https:// and any leading zeros from the host part
|
||||
return url
|
||||
.replace(/^https?:\/\//, '') // Remove protocol
|
||||
.replace(/^0+/, '') // Remove leading zeros at start
|
||||
.replace(/:0+(\d)/g, ':$1'); // Remove leading zeros from port numbers
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a proxy is working
|
||||
*/
|
||||
async checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||
console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount);
|
||||
// console.log('Checking proxy:', `${proxy.protocol}://${proxy.host}:${proxy.port}`, this.concurrencyLimit.activeCount, this.concurrencyLimit.pendingCount);
|
||||
try {
|
||||
|
||||
// Test the proxy
|
||||
|
|
@ -157,12 +181,13 @@ export class ProxyService {
|
|||
checkedAt: new Date(),
|
||||
responseTime: response.responseTime,
|
||||
};
|
||||
|
||||
if (isWorking && !response.data.includes('CHECK_IP')) {
|
||||
// console.log('Proxy check result:', proxy);
|
||||
if (isWorking && !JSON.stringify(response.data).includes(this.CHECK_IP)) {
|
||||
await this.cache.set(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, this.CACHE_TTL);
|
||||
} else {
|
||||
await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
}
|
||||
}
|
||||
// else { // TODO
|
||||
// await this.cache.del(`${this.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
// }
|
||||
|
||||
this.logger.debug('Proxy check completed', {
|
||||
host: proxy.host,
|
||||
|
|
|
|||
|
|
@ -1,817 +0,0 @@
|
|||
/**
|
||||
* Example usage of the Stock Bot configuration library
|
||||
*
|
||||
* This file demonstrates how to use the Zod-based configuration
|
||||
* system for various services in the Stock Bot platform.
|
||||
*/
|
||||
|
||||
// Import all the configuration modules
|
||||
import {
|
||||
// Core utilities
|
||||
loadEnvVariables,
|
||||
getEnvironment,
|
||||
Environment,
|
||||
ConfigurationError,
|
||||
} from './core';
|
||||
|
||||
import {
|
||||
// PostgreSQL configuration
|
||||
postgresConfig,
|
||||
PostgresConfig,
|
||||
POSTGRES_HOST,
|
||||
POSTGRES_PORT,
|
||||
POSTGRES_DATABASE,
|
||||
POSTGRES_USERNAME,
|
||||
POSTGRES_PASSWORD,
|
||||
} from './postgres';
|
||||
|
||||
import {
|
||||
// QuestDB configuration
|
||||
questdbConfig,
|
||||
QuestDbConfig,
|
||||
QUESTDB_HOST,
|
||||
QUESTDB_HTTP_PORT,
|
||||
QUESTDB_PG_PORT,
|
||||
} from './questdb';
|
||||
|
||||
import {
|
||||
// MongoDB configuration
|
||||
mongodbConfig,
|
||||
MongoDbConfig,
|
||||
MONGODB_HOST,
|
||||
MONGODB_PORT,
|
||||
MONGODB_DATABASE,
|
||||
MONGODB_USERNAME,
|
||||
} from './mongodb';
|
||||
|
||||
import {
|
||||
// Dragonfly configuration
|
||||
dragonflyConfig,
|
||||
DragonflyConfig,
|
||||
DRAGONFLY_HOST,
|
||||
DRAGONFLY_PORT,
|
||||
DRAGONFLY_DATABASE,
|
||||
} from './dragonfly';
|
||||
|
||||
import {
|
||||
// Monitoring configuration
|
||||
prometheusConfig,
|
||||
grafanaConfig,
|
||||
PrometheusConfig,
|
||||
GrafanaConfig,
|
||||
PROMETHEUS_HOST,
|
||||
PROMETHEUS_PORT,
|
||||
GRAFANA_HOST,
|
||||
GRAFANA_PORT,
|
||||
} from './monitoring';
|
||||
|
||||
import {
|
||||
// Loki configuration
|
||||
lokiConfig,
|
||||
LokiConfig,
|
||||
LOKI_HOST,
|
||||
LOKI_PORT,
|
||||
LOKI_SERVICE_LABEL,
|
||||
LOKI_BATCH_SIZE,
|
||||
} from './loki';
|
||||
|
||||
import {
|
||||
// Logging configuration
|
||||
loggingConfig,
|
||||
LoggingConfig,
|
||||
LOG_LEVEL,
|
||||
LOG_FORMAT,
|
||||
LOG_CONSOLE,
|
||||
LOG_FILE,
|
||||
LOG_SERVICE_NAME,
|
||||
} from './logging';
|
||||
|
||||
|
||||
import {
|
||||
// Risk management configuration
|
||||
riskConfig,
|
||||
RiskConfig,
|
||||
RISK_MAX_POSITION_SIZE,
|
||||
RISK_DEFAULT_STOP_LOSS,
|
||||
RISK_CIRCUIT_BREAKER_ENABLED,
|
||||
} from './risk';
|
||||
|
||||
import {
|
||||
// Data provider configuration
|
||||
dataProvidersConfig,
|
||||
DataProvidersConfig,
|
||||
getProviderConfig,
|
||||
getEnabledProviders,
|
||||
getDefaultProvider,
|
||||
DEFAULT_DATA_PROVIDER,
|
||||
ALPACA_API_KEY,
|
||||
} from './data-providers';
|
||||
|
||||
/**
|
||||
* Example 1: Basic usage - Load environment variables and get configuration
|
||||
*/
|
||||
function basicUsageExample() {
|
||||
console.log('=== Basic Usage Example ===');
|
||||
|
||||
// Load environment variables (optional - they're loaded automatically)
|
||||
loadEnvVariables();
|
||||
|
||||
// Get the current environment
|
||||
const env = getEnvironment();
|
||||
console.log(`Current environment: ${env}`);
|
||||
// Access individual configuration values
|
||||
console.log(`Database host: ${POSTGRES_HOST}`);
|
||||
console.log(`Database port: ${POSTGRES_PORT}`);
|
||||
console.log(`Log level: ${LOG_LEVEL}`);
|
||||
|
||||
// Access full database config objects
|
||||
console.log(`Full database config:`, {
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
name: postgresConfig.POSTGRES_DATABASE,
|
||||
ssl: postgresConfig.POSTGRES_SSL,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 2: Using configuration in a database connection
|
||||
*/
|
||||
async function databaseConnectionExample() {
|
||||
console.log('=== Database Connection Example ===');
|
||||
|
||||
try {
|
||||
// Use the database configuration to create a connection string
|
||||
const connectionString = `postgresql://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}`;
|
||||
|
||||
console.log('Database connection settings:');
|
||||
console.log(`- Host: ${postgresConfig.POSTGRES_HOST}`);
|
||||
console.log(`- Port: ${postgresConfig.POSTGRES_PORT}`);
|
||||
console.log(`- Database: ${postgresConfig.POSTGRES_DATABASE}`);
|
||||
console.log(`- SSL enabled: ${postgresConfig.POSTGRES_SSL}`);
|
||||
console.log(`- Pool max connections: ${postgresConfig.POSTGRES_POOL_MAX}`);
|
||||
console.log(`- Query timeout: ${postgresConfig.POSTGRES_QUERY_TIMEOUT}ms`);
|
||||
|
||||
// Example pool configuration
|
||||
const poolConfig = {
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
database: postgresConfig.POSTGRES_DATABASE,
|
||||
user: postgresConfig.POSTGRES_USERNAME,
|
||||
password: postgresConfig.POSTGRES_PASSWORD,
|
||||
ssl: postgresConfig.POSTGRES_SSL,
|
||||
min: postgresConfig.POSTGRES_POOL_MIN,
|
||||
max: postgresConfig.POSTGRES_POOL_MAX,
|
||||
idleTimeoutMillis: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT,
|
||||
};
|
||||
|
||||
console.log('Pool configuration:', poolConfig);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Database configuration error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Example 3: Logging setup example
|
||||
*/
|
||||
function loggingSetupExample() {
|
||||
console.log('=== Logging Setup Example ===');
|
||||
|
||||
// Access logging configuration
|
||||
console.log('Logging settings:');
|
||||
console.log(`- Level: ${loggingConfig.LOG_LEVEL}`);
|
||||
console.log(`- Format: ${loggingConfig.LOG_FORMAT}`);
|
||||
console.log(`- Console enabled: ${loggingConfig.LOG_CONSOLE}`);
|
||||
console.log(`- File logging: ${loggingConfig.LOG_FILE}`);
|
||||
console.log(`- Service name: ${loggingConfig.LOG_SERVICE_NAME}`);
|
||||
// Example logger configuration
|
||||
const loggerConfig = {
|
||||
level: loggingConfig.LOG_LEVEL,
|
||||
format: loggingConfig.LOG_FORMAT,
|
||||
transports: [] as any[],
|
||||
defaultMeta: {
|
||||
service: loggingConfig.LOG_SERVICE_NAME,
|
||||
version: loggingConfig.LOG_SERVICE_VERSION,
|
||||
environment: loggingConfig.LOG_ENVIRONMENT,
|
||||
},
|
||||
};
|
||||
|
||||
if (loggingConfig.LOG_CONSOLE) {
|
||||
loggerConfig.transports.push({
|
||||
type: 'console',
|
||||
format: loggingConfig.LOG_FORMAT,
|
||||
timestamp: loggingConfig.LOG_TIMESTAMP,
|
||||
});
|
||||
}
|
||||
|
||||
if (loggingConfig.LOG_FILE) {
|
||||
loggerConfig.transports.push({
|
||||
type: 'file',
|
||||
filename: `${loggingConfig.LOG_FILE_PATH}/application.log`,
|
||||
maxSize: loggingConfig.LOG_FILE_MAX_SIZE,
|
||||
maxFiles: loggingConfig.LOG_FILE_MAX_FILES,
|
||||
datePattern: loggingConfig.LOG_FILE_DATE_PATTERN,
|
||||
});
|
||||
}
|
||||
|
||||
// Example Loki transport configuration
|
||||
if (lokiConfig.LOKI_HOST) {
|
||||
loggerConfig.transports.push({
|
||||
type: 'loki',
|
||||
host: lokiConfig.LOKI_HOST,
|
||||
port: lokiConfig.LOKI_PORT,
|
||||
batchSize: lokiConfig.LOKI_BATCH_SIZE,
|
||||
labels: {
|
||||
service: lokiConfig.LOKI_SERVICE_LABEL,
|
||||
environment: lokiConfig.LOKI_ENVIRONMENT_LABEL,
|
||||
},
|
||||
});
|
||||
}
|
||||
console.log('Logger configuration:', loggerConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 4: Risk management configuration
|
||||
*/
|
||||
function riskManagementExample() {
|
||||
console.log('=== Risk Management Example ===');
|
||||
|
||||
// Access risk configuration
|
||||
console.log('Risk management settings:');
|
||||
console.log(`- Max position size: ${RISK_MAX_POSITION_SIZE * 100}%`);
|
||||
console.log(`- Default stop loss: ${RISK_DEFAULT_STOP_LOSS * 100}%`);
|
||||
console.log(`- Circuit breaker enabled: ${RISK_CIRCUIT_BREAKER_ENABLED}`);
|
||||
console.log(`- Max leverage: ${riskConfig.RISK_MAX_LEVERAGE}x`);
|
||||
|
||||
// Example risk calculator
|
||||
function calculatePositionSize(portfolioValue: number, riskPerTrade: number = RISK_DEFAULT_STOP_LOSS) {
|
||||
const maxPositionValue = portfolioValue * RISK_MAX_POSITION_SIZE;
|
||||
const riskAmount = portfolioValue * riskPerTrade;
|
||||
|
||||
return {
|
||||
maxPositionValue,
|
||||
riskAmount,
|
||||
maxShares: Math.floor(maxPositionValue / 100), // Assuming $100 per share
|
||||
};
|
||||
}
|
||||
|
||||
const portfolioValue = 100000; // $100k portfolio
|
||||
const position = calculatePositionSize(portfolioValue);
|
||||
console.log(`Position sizing for $${portfolioValue} portfolio:`, position);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 5: Data provider configuration
|
||||
*/
|
||||
function dataProviderExample() {
|
||||
console.log('=== Data Provider Example ===');
|
||||
|
||||
// Get the default provider
|
||||
const defaultProvider = getDefaultProvider();
|
||||
console.log('Default provider:', defaultProvider);
|
||||
|
||||
// Get all enabled providers
|
||||
const enabledProviders = getEnabledProviders();
|
||||
console.log('Enabled providers:', enabledProviders.map(p => p.name));
|
||||
|
||||
// Get specific provider configuration
|
||||
try {
|
||||
const alpacaConfig = getProviderConfig('alpaca');
|
||||
console.log('Alpaca configuration:', {
|
||||
enabled: alpacaConfig.enabled,
|
||||
baseUrl: alpacaConfig.baseUrl,
|
||||
hasApiKey: !!alpacaConfig.apiKey,
|
||||
rateLimit: alpacaConfig.rateLimits,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error getting Alpaca config:', error);
|
||||
}
|
||||
|
||||
// Example API client setup
|
||||
const apiClients = enabledProviders.map(provider => ({
|
||||
name: provider.name,
|
||||
client: {
|
||||
baseURL: provider.baseUrl,
|
||||
timeout: dataProvidersConfig.DATA_PROVIDER_TIMEOUT,
|
||||
retries: dataProvidersConfig.DATA_PROVIDER_RETRIES,
|
||||
retryDelay: dataProvidersConfig.DATA_PROVIDER_RETRY_DELAY,
|
||||
headers: provider.apiKey ? {
|
||||
'Authorization': `Bearer ${provider.apiKey}`
|
||||
} : {},
|
||||
}
|
||||
}));
|
||||
|
||||
console.log('API clients configuration:', apiClients);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 6: Environment-specific configuration
|
||||
*/
|
||||
function environmentSpecificExample() {
|
||||
console.log('=== Environment-Specific Example ===');
|
||||
|
||||
const env = getEnvironment();
|
||||
|
||||
switch (env) {
|
||||
case Environment.Development:
|
||||
console.log('Development environment detected');
|
||||
console.log('- Using local database');
|
||||
console.log('- Verbose logging enabled');
|
||||
console.log('- Paper trading mode');
|
||||
break;
|
||||
|
||||
case Environment.Testing:
|
||||
console.log('Testing environment detected');
|
||||
console.log('- Using test database');
|
||||
console.log('- Structured logging');
|
||||
console.log('- Mock data providers');
|
||||
break;
|
||||
|
||||
case Environment.Staging:
|
||||
console.log('Staging environment detected');
|
||||
console.log('- Using staging database');
|
||||
console.log('- Production-like settings');
|
||||
console.log('- Real data providers (limited)');
|
||||
break;
|
||||
|
||||
case Environment.Production:
|
||||
console.log('Production environment detected');
|
||||
console.log('- Using production database');
|
||||
console.log('- Optimized logging');
|
||||
console.log('- Live trading enabled');
|
||||
break;
|
||||
}
|
||||
|
||||
// Example of environment-specific behavior
|
||||
const isProduction = env === Environment.Production;
|
||||
const tradingMode = isProduction ? 'live' : 'paper';
|
||||
const logLevel = isProduction ? 'info' : 'debug';
|
||||
|
||||
console.log(`Trading mode: ${tradingMode}`);
|
||||
console.log(`Recommended log level: ${logLevel}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 7: Configuration validation and error handling
|
||||
*/
|
||||
function configurationValidationExample() {
|
||||
console.log('=== Configuration Validation Example ===');
|
||||
|
||||
try {
|
||||
// Check required configurations
|
||||
if (!ALPACA_API_KEY && DEFAULT_DATA_PROVIDER === 'alpaca') {
|
||||
throw new ConfigurationError('Alpaca API key is required when using Alpaca as default provider');
|
||||
}
|
||||
|
||||
// Validate risk settings
|
||||
if (RISK_MAX_POSITION_SIZE > 1.0) {
|
||||
throw new ConfigurationError('Maximum position size cannot exceed 100%');
|
||||
}
|
||||
|
||||
if (riskConfig.RISK_DEFAULT_STOP_LOSS > riskConfig.RISK_DEFAULT_TAKE_PROFIT) {
|
||||
console.warn('Warning: Stop loss is greater than take profit - check your risk settings');
|
||||
}
|
||||
|
||||
// Validate database connection settings
|
||||
if (postgresConfig.POSTGRES_POOL_MAX < postgresConfig.POSTGRES_POOL_MIN) {
|
||||
throw new ConfigurationError('Database max pool size must be greater than min pool size');
|
||||
}
|
||||
|
||||
console.log('✅ All configuration validations passed');
|
||||
|
||||
} catch (error) {
|
||||
if (error instanceof ConfigurationError) {
|
||||
console.error('❌ Configuration error:', error.message);
|
||||
} else {
|
||||
console.error('❌ Unexpected error:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 8: QuestDB time-series database configuration
|
||||
*/
|
||||
function questdbConfigurationExample() {
|
||||
console.log('=== QuestDB Configuration Example ===');
|
||||
|
||||
// Access QuestDB configuration
|
||||
console.log('QuestDB settings:');
|
||||
console.log(`- Host: ${questdbConfig.QUESTDB_HOST}`);
|
||||
console.log(`- HTTP port (web console): ${questdbConfig.QUESTDB_HTTP_PORT}`);
|
||||
console.log(`- PostgreSQL port: ${questdbConfig.QUESTDB_PG_PORT}`);
|
||||
console.log(`- InfluxDB port: ${questdbConfig.QUESTDB_INFLUX_PORT}`);
|
||||
console.log(`- TLS enabled: ${questdbConfig.QUESTDB_TLS_ENABLED}`);
|
||||
|
||||
// Example QuestDB client configuration
|
||||
const questdbClientConfig = {
|
||||
http: {
|
||||
host: questdbConfig.QUESTDB_HOST,
|
||||
port: questdbConfig.QUESTDB_HTTP_PORT,
|
||||
tls: questdbConfig.QUESTDB_TLS_ENABLED,
|
||||
timeout: questdbConfig.QUESTDB_REQUEST_TIMEOUT,
|
||||
},
|
||||
postgresql: {
|
||||
host: questdbConfig.QUESTDB_HOST,
|
||||
port: questdbConfig.QUESTDB_PG_PORT,
|
||||
database: questdbConfig.QUESTDB_DEFAULT_DATABASE,
|
||||
user: questdbConfig.QUESTDB_USER,
|
||||
password: questdbConfig.QUESTDB_PASSWORD,
|
||||
},
|
||||
influxdb: {
|
||||
host: questdbConfig.QUESTDB_HOST,
|
||||
port: questdbConfig.QUESTDB_INFLUX_PORT,
|
||||
}
|
||||
};
|
||||
|
||||
console.log('QuestDB client configuration:', questdbClientConfig);
|
||||
|
||||
// Example time-series table creation
|
||||
const createTableQuery = `
|
||||
CREATE TABLE IF NOT EXISTS ohlcv_data (
|
||||
timestamp TIMESTAMP,
|
||||
symbol SYMBOL,
|
||||
open DOUBLE,
|
||||
high DOUBLE,
|
||||
low DOUBLE,
|
||||
close DOUBLE,
|
||||
volume LONG
|
||||
) timestamp(timestamp) PARTITION BY DAY;
|
||||
`;
|
||||
|
||||
console.log('Example table creation query:', createTableQuery);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 9: MongoDB document database configuration
|
||||
*/
|
||||
function mongodbConfigurationExample() {
|
||||
console.log('=== MongoDB Configuration Example ===');
|
||||
|
||||
// Access MongoDB configuration
|
||||
console.log('MongoDB settings:');
|
||||
console.log(`- Host: ${mongodbConfig.MONGODB_HOST}`);
|
||||
console.log(`- Port: ${mongodbConfig.MONGODB_PORT}`);
|
||||
console.log(`- Database: ${mongodbConfig.MONGODB_DATABASE}`);
|
||||
console.log(`- Username: ${mongodbConfig.MONGODB_USERNAME}`);
|
||||
console.log(`- TLS enabled: ${mongodbConfig.MONGODB_TLS}`);
|
||||
console.log(`- Max pool size: ${mongodbConfig.MONGODB_MAX_POOL_SIZE}`);
|
||||
|
||||
// Build connection URI
|
||||
const buildMongoUri = () => {
|
||||
if (mongodbConfig.MONGODB_URI) {
|
||||
return mongodbConfig.MONGODB_URI;
|
||||
}
|
||||
|
||||
const auth = mongodbConfig.MONGODB_USERNAME && mongodbConfig.MONGODB_PASSWORD
|
||||
? `${mongodbConfig.MONGODB_USERNAME}:${mongodbConfig.MONGODB_PASSWORD}@`
|
||||
: '';
|
||||
|
||||
const tls = mongodbConfig.MONGODB_TLS ? '?tls=true' : '';
|
||||
|
||||
return `mongodb://${auth}${mongodbConfig.MONGODB_HOST}:${mongodbConfig.MONGODB_PORT}/${mongodbConfig.MONGODB_DATABASE}${tls}`;
|
||||
};
|
||||
|
||||
const mongoUri = buildMongoUri();
|
||||
console.log('MongoDB connection URI:', mongoUri.replace(/:[^:@]*@/, ':***@')); // Hide password
|
||||
|
||||
// Example MongoDB client configuration
|
||||
const mongoClientConfig = {
|
||||
maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
|
||||
minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE,
|
||||
maxIdleTimeMS: mongodbConfig.MONGODB_MAX_IDLE_TIME,
|
||||
connectTimeoutMS: mongodbConfig.MONGODB_CONNECT_TIMEOUT,
|
||||
socketTimeoutMS: mongodbConfig.MONGODB_SOCKET_TIMEOUT,
|
||||
serverSelectionTimeoutMS: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT,
|
||||
retryWrites: mongodbConfig.MONGODB_RETRY_WRITES,
|
||||
w: mongodbConfig.MONGODB_WRITE_CONCERN,
|
||||
readPreference: mongodbConfig.MONGODB_READ_PREFERENCE,
|
||||
};
|
||||
|
||||
console.log('MongoDB client configuration:', mongoClientConfig);
|
||||
|
||||
// Example collections structure
|
||||
const collections = [
|
||||
'sentiment_data', // News sentiment analysis
|
||||
'market_news', // Raw news articles
|
||||
'social_signals', // Social media signals
|
||||
'earnings_reports', // Earnings data
|
||||
'analyst_ratings', // Analyst recommendations
|
||||
];
|
||||
|
||||
console.log('Example collections:', collections);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 10: Dragonfly (Redis replacement) configuration
|
||||
*/
|
||||
function dragonflyConfigurationExample() {
|
||||
console.log('=== Dragonfly Configuration Example ===');
|
||||
|
||||
// Access Dragonfly configuration
|
||||
console.log('Dragonfly settings:');
|
||||
console.log(`- Host: ${dragonflyConfig.DRAGONFLY_HOST}`);
|
||||
console.log(`- Port: ${dragonflyConfig.DRAGONFLY_PORT}`);
|
||||
console.log(`- Database: ${dragonflyConfig.DRAGONFLY_DATABASE}`);
|
||||
console.log(`- Cache mode: ${dragonflyConfig.DRAGONFLY_CACHE_MODE}`);
|
||||
console.log(`- Max memory: ${dragonflyConfig.DRAGONFLY_MAX_MEMORY}`);
|
||||
console.log(`- Pool size: ${dragonflyConfig.DRAGONFLY_POOL_SIZE}`);
|
||||
// Example Dragonfly client configuration
|
||||
const dragonflyClientConfig = {
|
||||
host: dragonflyConfig.DRAGONFLY_HOST,
|
||||
port: dragonflyConfig.DRAGONFLY_PORT,
|
||||
db: dragonflyConfig.DRAGONFLY_DATABASE,
|
||||
password: dragonflyConfig.DRAGONFLY_PASSWORD || undefined,
|
||||
username: dragonflyConfig.DRAGONFLY_USERNAME || undefined,
|
||||
retryDelayOnFailover: dragonflyConfig.DRAGONFLY_RETRY_DELAY,
|
||||
maxRetriesPerRequest: dragonflyConfig.DRAGONFLY_MAX_RETRIES,
|
||||
connectTimeout: dragonflyConfig.DRAGONFLY_CONNECT_TIMEOUT,
|
||||
commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT,
|
||||
enableAutoPipelining: true,
|
||||
};
|
||||
|
||||
console.log('Dragonfly client configuration:', dragonflyClientConfig);
|
||||
|
||||
// Example cache key patterns
|
||||
const cachePatterns = {
|
||||
marketData: 'market:{symbol}:{timeframe}',
|
||||
indicators: 'indicators:{symbol}:{indicator}:{period}',
|
||||
positions: 'positions:{account_id}',
|
||||
orders: 'orders:{order_id}',
|
||||
rateLimit: 'rate_limit:{provider}:{endpoint}',
|
||||
sessions: 'session:{user_id}',
|
||||
};
|
||||
|
||||
console.log('Example cache key patterns:', cachePatterns);
|
||||
|
||||
// Example TTL configurations
|
||||
const ttlConfigs = {
|
||||
marketData: 60, // 1 minute for real-time data
|
||||
indicators: 300, // 5 minutes for calculated indicators
|
||||
positions: 30, // 30 seconds for positions
|
||||
orders: 86400, // 1 day for order history
|
||||
rateLimit: 3600, // 1 hour for rate limiting
|
||||
sessions: 1800, // 30 minutes for user sessions
|
||||
};
|
||||
|
||||
console.log('Example TTL configurations (seconds):', ttlConfigs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 11: Monitoring stack configuration (Prometheus, Grafana, Loki)
|
||||
*/
|
||||
function monitoringConfigurationExample() {
|
||||
console.log('=== Monitoring Configuration Example ===');
|
||||
|
||||
// Prometheus configuration
|
||||
console.log('Prometheus settings:');
|
||||
console.log(`- Host: ${prometheusConfig.PROMETHEUS_HOST}`);
|
||||
console.log(`- Port: ${prometheusConfig.PROMETHEUS_PORT}`);
|
||||
console.log(`- Scrape interval: ${prometheusConfig.PROMETHEUS_SCRAPE_INTERVAL}`);
|
||||
console.log(`- Retention time: ${prometheusConfig.PROMETHEUS_RETENTION_TIME}`);
|
||||
|
||||
// Grafana configuration
|
||||
console.log('\nGrafana settings:');
|
||||
console.log(`- Host: ${grafanaConfig.GRAFANA_HOST}`);
|
||||
console.log(`- Port: ${grafanaConfig.GRAFANA_PORT}`);
|
||||
console.log(`- Admin user: ${grafanaConfig.GRAFANA_ADMIN_USER}`);
|
||||
console.log(`- Allow sign up: ${grafanaConfig.GRAFANA_ALLOW_SIGN_UP}`);
|
||||
console.log(`- Database type: ${grafanaConfig.GRAFANA_DATABASE_TYPE}`);
|
||||
|
||||
// Loki configuration
|
||||
console.log('\nLoki settings:');
|
||||
console.log(`- Host: ${lokiConfig.LOKI_HOST}`);
|
||||
console.log(`- Port: ${lokiConfig.LOKI_PORT}`);
|
||||
console.log(`- Batch size: ${lokiConfig.LOKI_BATCH_SIZE}`);
|
||||
console.log(`- Retention period: ${lokiConfig.LOKI_RETENTION_PERIOD}`);
|
||||
|
||||
// Example monitoring endpoints
|
||||
const monitoringEndpoints = {
|
||||
prometheus: `http://${prometheusConfig.PROMETHEUS_HOST}:${prometheusConfig.PROMETHEUS_PORT}`,
|
||||
grafana: `http://${grafanaConfig.GRAFANA_HOST}:${grafanaConfig.GRAFANA_PORT}`,
|
||||
loki: `http://${lokiConfig.LOKI_HOST}:${lokiConfig.LOKI_PORT}`,
|
||||
};
|
||||
|
||||
console.log('\nMonitoring endpoints:', monitoringEndpoints);
|
||||
|
||||
// Example metrics configuration
|
||||
const metricsConfig = {
|
||||
defaultLabels: {
|
||||
service: 'stock-bot',
|
||||
environment: getEnvironment(),
|
||||
version: process.env.npm_package_version || '1.0.0',
|
||||
},
|
||||
collectDefaultMetrics: true,
|
||||
prefix: 'stockbot_',
|
||||
buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60], // Response time buckets in seconds
|
||||
};
|
||||
|
||||
console.log('Example metrics configuration:', metricsConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 12: Multi-database service configuration
|
||||
*/
|
||||
function multiDatabaseServiceExample() {
|
||||
console.log('=== Multi-Database Service Example ===');
|
||||
|
||||
// Complete database configuration for a microservice
|
||||
const serviceConfig = {
|
||||
service: {
|
||||
name: 'market-data-processor',
|
||||
version: '1.0.0',
|
||||
environment: getEnvironment(),
|
||||
},
|
||||
|
||||
// PostgreSQL for operational data
|
||||
postgresql: {
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
database: postgresConfig.POSTGRES_DATABASE,
|
||||
username: postgresConfig.POSTGRES_USERNAME,
|
||||
password: postgresConfig.POSTGRES_PASSWORD,
|
||||
ssl: postgresConfig.POSTGRES_SSL,
|
||||
pool: {
|
||||
min: postgresConfig.POSTGRES_POOL_MIN,
|
||||
max: postgresConfig.POSTGRES_POOL_MAX,
|
||||
idleTimeout: postgresConfig.POSTGRES_POOL_IDLE_TIMEOUT,
|
||||
},
|
||||
},
|
||||
|
||||
// QuestDB for time-series data
|
||||
questdb: {
|
||||
host: questdbConfig.QUESTDB_HOST,
|
||||
httpPort: questdbConfig.QUESTDB_HTTP_PORT,
|
||||
pgPort: questdbConfig.QUESTDB_PG_PORT,
|
||||
database: questdbConfig.QUESTDB_DEFAULT_DATABASE,
|
||||
timeout: questdbConfig.QUESTDB_REQUEST_TIMEOUT,
|
||||
},
|
||||
|
||||
// MongoDB for document storage
|
||||
mongodb: {
|
||||
host: mongodbConfig.MONGODB_HOST,
|
||||
port: mongodbConfig.MONGODB_PORT,
|
||||
database: mongodbConfig.MONGODB_DATABASE,
|
||||
username: mongodbConfig.MONGODB_USERNAME,
|
||||
maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
|
||||
readPreference: mongodbConfig.MONGODB_READ_PREFERENCE,
|
||||
},
|
||||
|
||||
// Dragonfly for caching
|
||||
dragonfly: {
|
||||
host: dragonflyConfig.DRAGONFLY_HOST,
|
||||
port: dragonflyConfig.DRAGONFLY_PORT,
|
||||
database: dragonflyConfig.DRAGONFLY_DATABASE,
|
||||
poolSize: dragonflyConfig.DRAGONFLY_POOL_SIZE,
|
||||
commandTimeout: dragonflyConfig.DRAGONFLY_COMMAND_TIMEOUT,
|
||||
},
|
||||
|
||||
// Monitoring
|
||||
monitoring: {
|
||||
prometheus: {
|
||||
pushGateway: `http://${prometheusConfig.PROMETHEUS_HOST}:${prometheusConfig.PROMETHEUS_PORT}`,
|
||||
scrapeInterval: prometheusConfig.PROMETHEUS_SCRAPE_INTERVAL,
|
||||
}, loki: {
|
||||
host: lokiConfig.LOKI_HOST,
|
||||
port: lokiConfig.LOKI_PORT,
|
||||
batchSize: lokiConfig.LOKI_BATCH_SIZE,
|
||||
labels: {
|
||||
service: 'market-data-processor',
|
||||
environment: getEnvironment(),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
console.log('Complete service configuration:', JSON.stringify(serviceConfig, null, 2));
|
||||
|
||||
// Example data flow
|
||||
const dataFlow = {
|
||||
ingestion: 'Market data → Dragonfly (cache) → QuestDB (storage)',
|
||||
processing: 'QuestDB → Analysis → PostgreSQL (results) → MongoDB (metadata)',
|
||||
serving: 'Dragonfly (cache) ← PostgreSQL/QuestDB ← API requests',
|
||||
monitoring: 'All services → Prometheus → Grafana dashboards',
|
||||
logging: 'All services → Loki → Grafana log viewer',
|
||||
};
|
||||
|
||||
console.log('\nData flow patterns:', dataFlow);
|
||||
}
|
||||
|
||||
/**
|
||||
* Example 8: Creating a service configuration object
|
||||
*/
|
||||
function serviceConfigurationExample() {
|
||||
console.log('=== Service Configuration Example ===');
|
||||
|
||||
// Example: Market Data Gateway service configuration
|
||||
const marketDataGatewayConfig = {
|
||||
service: {
|
||||
name: 'market-data-gateway',
|
||||
port: 3001,
|
||||
environment: getEnvironment(),
|
||||
},
|
||||
database: {
|
||||
host: postgresConfig.POSTGRES_HOST,
|
||||
port: postgresConfig.POSTGRES_PORT,
|
||||
name: postgresConfig.POSTGRES_DATABASE,
|
||||
ssl: postgresConfig.POSTGRES_SSL,
|
||||
}, logging: {
|
||||
level: loggingConfig.LOG_LEVEL,
|
||||
console: loggingConfig.LOG_CONSOLE,
|
||||
loki: {
|
||||
host: lokiConfig.LOKI_HOST,
|
||||
port: lokiConfig.LOKI_PORT,
|
||||
labels: {
|
||||
service: 'market-data-gateway',
|
||||
environment: getEnvironment(),
|
||||
}
|
||||
}
|
||||
},
|
||||
dataProviders: {
|
||||
default: DEFAULT_DATA_PROVIDER,
|
||||
enabled: getEnabledProviders(),
|
||||
timeout: dataProvidersConfig.DATA_PROVIDER_TIMEOUT,
|
||||
retries: dataProvidersConfig.DATA_PROVIDER_RETRIES,
|
||||
},
|
||||
cache: {
|
||||
enabled: dataProvidersConfig.DATA_CACHE_ENABLED,
|
||||
ttl: dataProvidersConfig.DATA_CACHE_TTL,
|
||||
maxSize: dataProvidersConfig.DATA_CACHE_MAX_SIZE,
|
||||
}
|
||||
};
|
||||
|
||||
console.log('Market Data Gateway configuration:', JSON.stringify(marketDataGatewayConfig, null, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
* Main example runner
|
||||
*/
|
||||
function runAllExamples() {
|
||||
console.log('🚀 Stock Bot Configuration Examples\n');
|
||||
|
||||
try {
|
||||
basicUsageExample();
|
||||
console.log('\n');
|
||||
|
||||
databaseConnectionExample();
|
||||
console.log('\n');
|
||||
|
||||
loggingSetupExample();
|
||||
console.log('\n');
|
||||
|
||||
riskManagementExample();
|
||||
console.log('\n');
|
||||
|
||||
dataProviderExample();
|
||||
console.log('\n');
|
||||
|
||||
environmentSpecificExample();
|
||||
console.log('\n');
|
||||
|
||||
configurationValidationExample();
|
||||
console.log('\n');
|
||||
|
||||
questdbConfigurationExample();
|
||||
console.log('\n');
|
||||
|
||||
mongodbConfigurationExample();
|
||||
console.log('\n');
|
||||
|
||||
dragonflyConfigurationExample();
|
||||
console.log('\n');
|
||||
|
||||
monitoringConfigurationExample();
|
||||
console.log('\n');
|
||||
|
||||
multiDatabaseServiceExample();
|
||||
console.log('\n');
|
||||
|
||||
serviceConfigurationExample();
|
||||
|
||||
} catch (error) {
|
||||
console.error('Example execution error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Export the examples for use in other files
|
||||
export {
|
||||
basicUsageExample,
|
||||
databaseConnectionExample,
|
||||
loggingSetupExample,
|
||||
riskManagementExample,
|
||||
dataProviderExample,
|
||||
environmentSpecificExample,
|
||||
configurationValidationExample,
|
||||
questdbConfigurationExample,
|
||||
mongodbConfigurationExample,
|
||||
dragonflyConfigurationExample,
|
||||
monitoringConfigurationExample,
|
||||
multiDatabaseServiceExample,
|
||||
serviceConfigurationExample,
|
||||
runAllExamples,
|
||||
};
|
||||
|
||||
// Run examples if this file is executed directly
|
||||
if (require.main === module) {
|
||||
runAllExamples();
|
||||
}
|
||||
|
|
@ -7,7 +7,7 @@ import type {
|
|||
import { HttpError } from './types.js';
|
||||
import { ProxyManager } from './proxy-manager.js';
|
||||
import axios, { type AxiosResponse, AxiosError } from 'axios';
|
||||
import { clear } from 'node:console';
|
||||
import { loggingConfig } from '@stock-bot/config';
|
||||
|
||||
export class HttpClient {
|
||||
private readonly config: HttpClientConfig;
|
||||
|
|
@ -15,8 +15,9 @@ export class HttpClient {
|
|||
|
||||
constructor(config: HttpClientConfig = {}, logger?: Logger) {
|
||||
this.config = config;
|
||||
this.logger = logger?.child({
|
||||
this.logger = logger?.child({ //TODO fix pino levels
|
||||
component: 'http',
|
||||
// level: loggingConfig?.LOG_LEVEL || 'info',
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -68,11 +69,11 @@ export class HttpClient {
|
|||
|
||||
return response;
|
||||
} catch (error) {
|
||||
this.logger?.warn('HTTP request failed', {
|
||||
method: finalConfig.method,
|
||||
url: finalConfig.url,
|
||||
error: (error as Error).message,
|
||||
});
|
||||
// this.logger?.warn('HTTP request failed', {
|
||||
// method: finalConfig.method,
|
||||
// url: finalConfig.url,
|
||||
// error: (error as Error).message,
|
||||
// });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
|
@ -124,10 +125,10 @@ export class HttpClient {
|
|||
try {
|
||||
const options = this.buildFetchOptions(config, signal);
|
||||
|
||||
this.logger?.debug('Making request with fetch: ', { url: config.url, options })
|
||||
|
||||
this.logger?.debug('Making request with fetch: ', { url: config.url, options })
|
||||
// console.log(options)
|
||||
const response = await fetch(config.url, options);
|
||||
|
||||
// console.log('Fetch response:', response.status);
|
||||
return this.parseFetchResponse<T>(response);
|
||||
} catch (error) {
|
||||
throw signal.aborted
|
||||
|
|
@ -201,7 +202,6 @@ export class HttpClient {
|
|||
if (config.proxy && ProxyManager.shouldUseBunFetch(config.proxy)) {
|
||||
(options as any).proxy = ProxyManager.createProxyUrl(config.proxy);
|
||||
}
|
||||
|
||||
return options;
|
||||
} /**
|
||||
* Build Axios options (for reference, though we're creating instance in ProxyManager)
|
||||
|
|
|
|||
|
|
@ -16,9 +16,6 @@ export class ProxyManager {
|
|||
*/
|
||||
static createProxyUrl(proxy: ProxyInfo): string {
|
||||
const { protocol, host, port, username, password } = proxy;
|
||||
if(protocol.includes('socks')) {
|
||||
return `${protocol}://${host}:${port}`;
|
||||
}
|
||||
if (username && password) {
|
||||
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||
}
|
||||
|
|
@ -32,11 +29,10 @@ export class ProxyManager {
|
|||
this.validateConfig(proxy);
|
||||
|
||||
const proxyUrl = this.createProxyUrl(proxy);
|
||||
console.log(`Using proxy url: ${proxyUrl}`);
|
||||
switch (proxy.protocol) {
|
||||
case 'socks4':
|
||||
case 'socks5':
|
||||
console.log(`Using SOCKS proxy: ${proxyUrl}`);
|
||||
// console.log(`Using SOCKS proxy: ${proxyUrl}`);
|
||||
return new SocksProxyAgent(proxyUrl);
|
||||
case 'http':
|
||||
return new HttpProxyAgent(proxyUrl);
|
||||
|
|
@ -54,12 +50,6 @@ export class ProxyManager {
|
|||
return {
|
||||
httpAgent: agent,
|
||||
httpsAgent: agent,
|
||||
// timeout: 30000,
|
||||
// validateStatus: () => true, // Don't throw errors on HTTP status codes
|
||||
// maxRedirects: 5,
|
||||
// headers: {
|
||||
// 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
// }
|
||||
};
|
||||
}
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -200,9 +200,10 @@ export class Logger {
|
|||
* Create child logger with additional context
|
||||
*/
|
||||
child(context: LogContext): Logger {
|
||||
const childLogger = new Logger((this.pino.bindings() as any).service, { ...this.context, ...context });
|
||||
// Use the pino child logger to properly propagate context
|
||||
childLogger.pino = this.pino.child(context);
|
||||
// Create child logger that shares the same pino instance with additional context
|
||||
const childLogger = Object.create(Logger.prototype);
|
||||
childLogger.context = { ...this.context, ...context };
|
||||
childLogger.pino = this.pino.child(context); // Let pino handle level inheritance naturally
|
||||
return childLogger;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue