From 8165994fde4af3e7c5ca5a036bf2d60cc2e991a6 Mon Sep 17 00:00:00 2001 From: Boki Date: Sun, 22 Jun 2025 09:57:38 -0400 Subject: [PATCH] almost working --- .env | 2 +- apps/data-ingestion/src/index.ts | 7 ++++ .../config/src/schemas/database.schema.ts | 4 +- libs/core/di/src/awilix-container.ts | 27 ++++++++----- libs/data/questdb/src/client.ts | 39 +++++++++++++++++-- libs/data/questdb/src/schema.ts | 2 +- libs/services/browser/src/browser-pool.ts | 0 7 files changed, 64 insertions(+), 17 deletions(-) delete mode 100644 libs/services/browser/src/browser-pool.ts diff --git a/.env b/.env index 1ffd587..77fea79 100644 --- a/.env +++ b/.env @@ -39,7 +39,7 @@ POSTGRES_SSL=false QUESTDB_HOST=localhost QUESTDB_PORT=9000 QUESTDB_DB=qdb -QUESTDB_USER=admin +QUESTDB_USERNAME=admin QUESTDB_PASSWORD=quest # MongoDB Configuration diff --git a/apps/data-ingestion/src/index.ts b/apps/data-ingestion/src/index.ts index 655e19c..f4bd010 100644 --- a/apps/data-ingestion/src/index.ts +++ b/apps/data-ingestion/src/index.ts @@ -76,6 +76,13 @@ async function initializeServices() { user: config.database.postgres.user, password: config.database.postgres.password, }, + questdb: { + host: config.database.questdb.host, + httpPort: config.database.questdb.httpPort, + pgPort: config.database.questdb.pgPort, + influxPort: config.database.questdb.ilpPort, + database: config.database.questdb.database, + }, proxy: { cachePrefix: 'proxy:', ttl: 3600, diff --git a/libs/core/config/src/schemas/database.schema.ts b/libs/core/config/src/schemas/database.schema.ts index d0b1666..88e027b 100644 --- a/libs/core/config/src/schemas/database.schema.ts +++ b/libs/core/config/src/schemas/database.schema.ts @@ -20,8 +20,8 @@ export const questdbConfigSchema = z.object({ httpPort: z.number().default(9000), pgPort: z.number().default(8812), database: z.string().default('questdb'), - user: z.string().default('admin'), - password: z.string().default('quest'), + user: z.string().optional(), // No default - QuestDB doesn't require auth by default + password: z.string().optional(), // No default - QuestDB doesn't require auth by default bufferSize: z.number().default(65536), flushInterval: z.number().default(1000), }); diff --git a/libs/core/di/src/awilix-container.ts b/libs/core/di/src/awilix-container.ts index 1d59b16..f3da283 100644 --- a/libs/core/di/src/awilix-container.ts +++ b/libs/core/di/src/awilix-container.ts @@ -3,15 +3,15 @@ * Creates a decoupled, reusable dependency injection container */ -import { createContainer, asFunction, asValue, InjectionMode, type AwilixContainer } from 'awilix'; +import { Browser } from '@stock-bot/browser'; import { createCache, type CacheProvider } from '@stock-bot/cache'; -import { ProxyManager } from '@stock-bot/proxy'; +import type { IServiceContainer } from '@stock-bot/handlers'; import { getLogger } from '@stock-bot/logger'; import { createMongoDBClient } from '@stock-bot/mongodb'; import { createPostgreSQLClient } from '@stock-bot/postgres'; +import { ProxyManager } from '@stock-bot/proxy'; import { createQuestDBClient } from '@stock-bot/questdb'; -import { Browser } from '@stock-bot/browser'; -import type { IServiceContainer } from '@stock-bot/handlers'; +import { asFunction, asValue, createContainer, InjectionMode, type AwilixContainer } from 'awilix'; // Configuration types export interface AppConfig { @@ -35,7 +35,10 @@ export interface AppConfig { }; questdb?: { host: string; - port: number; + httpPort?: number; + pgPort?: number; + influxPort?: number; + database?: string; }; proxy?: { cachePrefix?: string; @@ -62,7 +65,7 @@ export function createServiceContainer(config: AppConfig): AwilixContainer { redisConfig: asValue(config.redis), mongoConfig: asValue(config.mongodb), postgresConfig: asValue(config.postgres), - questdbConfig: asValue(config.questdb || { host: 'localhost', port: 9009 }), + questdbConfig: asValue(config.questdb || { host: 'localhost', httpPort: 9000, pgPort: 8812, influxPort: 9009 }), // Core services with dependency injection logger: asFunction(() => getLogger('app')).singleton(), @@ -119,13 +122,17 @@ export function createServiceContainer(config: AppConfig): AwilixContainer { }).singleton(), questdbClient: asFunction(({ questdbConfig, logger }) => { + console.log('Creating QuestDB client with config:', questdbConfig); return createQuestDBClient( { host: questdbConfig.host, - httpPort: 9000, - pgPort: questdbConfig.port || 8812, - influxPort: 9009, - database: 'questdb', + httpPort: questdbConfig.httpPort, + pgPort: questdbConfig.pgPort, + influxPort: questdbConfig.influxPort, + database: questdbConfig.database, + // QuestDB appears to require default credentials + user: 'admin', + password: 'quest', }, logger ); diff --git a/libs/data/questdb/src/client.ts b/libs/data/questdb/src/client.ts index 0fc5bc9..ef9411c 100644 --- a/libs/data/questdb/src/client.ts +++ b/libs/data/questdb/src/client.ts @@ -37,6 +37,18 @@ export class QuestDBClient { ...options, }; + // Debug: log the received config + console.log('DEBUG: QuestDB client constructor called with config:', { + ...config, + user: config.user || '[NOT PROVIDED]', + password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]' + }); + this.logger.debug('QuestDB client created with config:', { + ...config, + user: config.user || '[NOT PROVIDED]', + password: config.password ? '[PROVIDED]' : '[NOT PROVIDED]' + }); + this.healthMonitor = new QuestDBHealthMonitor(this); this.influxWriter = new QuestDBInfluxWriter(this); this.schemaManager = new QuestDBSchemaManager(this); @@ -407,12 +419,10 @@ export class QuestDBClient { private buildPgPoolConfig(): any { - return { + const config: any = { host: this.config.host, port: this.config.pgPort, database: this.config.database, - user: this.config.user, - password: this.config.password, connectionTimeoutMillis: this.config.timeouts?.connection, query_timeout: this.config.timeouts?.request, ssl: this.config.tls?.enabled @@ -423,6 +433,29 @@ export class QuestDBClient { min: 2, max: 10, }; + + // Only add user/password if they are provided + if (this.config.user) { + console.log('DEBUG: Adding user to QuestDB pool config:', this.config.user); + this.logger.debug('Adding user to QuestDB pool config:', this.config.user); + config.user = this.config.user; + } else { + console.log('DEBUG: No user provided for QuestDB connection'); + this.logger.debug('No user provided for QuestDB connection'); + } + + if (this.config.password) { + console.log('DEBUG: Adding password to QuestDB pool config'); + this.logger.debug('Adding password to QuestDB pool config'); + config.password = this.config.password; + } else { + console.log('DEBUG: No password provided for QuestDB connection'); + this.logger.debug('No password provided for QuestDB connection'); + } + + console.log('DEBUG: Final QuestDB pool config:', { ...config, password: config.password ? '[REDACTED]' : undefined }); + this.logger.debug('Final QuestDB pool config:', { ...config, password: config.password ? '[REDACTED]' : undefined }); + return config; } private mapDataType(typeId: number): string { diff --git a/libs/data/questdb/src/schema.ts b/libs/data/questdb/src/schema.ts index 281f3f5..e584e5e 100644 --- a/libs/data/questdb/src/schema.ts +++ b/libs/data/questdb/src/schema.ts @@ -326,7 +326,7 @@ export class QuestDBSchemaManager { // Add designated timestamp const timestampColumn = schema.columns.find(col => col.designated); if (timestampColumn) { - sql += ` timestamp(${timestampColumn.name})`; + sql += ` TIMESTAMP(${timestampColumn.name})`; } // Add partition by diff --git a/libs/services/browser/src/browser-pool.ts b/libs/services/browser/src/browser-pool.ts deleted file mode 100644 index e69de29..0000000