refactoring to remove a lot of junk

apps/data-pipeline/src/clients.ts

@@ -0,0 +1,27 @@
+import { PostgreSQLClient } from '@stock-bot/postgres';
+import { MongoDBClient } from '@stock-bot/mongodb';
+
+let postgresClient: PostgreSQLClient | null = null;
+let mongodbClient: MongoDBClient | null = null;
+
+export function setPostgreSQLClient(client: PostgreSQLClient): void {
+  postgresClient = client;
+}
+
+export function getPostgreSQLClient(): PostgreSQLClient {
+  if (!postgresClient) {
+    throw new Error('PostgreSQL client not initialized. Call setPostgreSQLClient first.');
+  }
+  return postgresClient;
+}
+
+export function setMongoDBClient(client: MongoDBClient): void {
+  mongodbClient = client;
+}
+
+export function getMongoDBClient(): MongoDBClient {
+  if (!mongodbClient) {
+    throw new Error('MongoDB client not initialized. Call setMongoDBClient first.');
+  }
+  return mongodbClient;
+}

apps/data-pipeline/src/handlers/exchanges/operations/clear-postgresql-data.operations.ts

@@ -1,5 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getPostgreSQLClient } from '../../../clients';
 import type { JobPayload } from '../../../types/job-payloads';
 
 const logger = getLogger('enhanced-sync-clear-postgresql-data');

apps/data-pipeline/src/handlers/exchanges/operations/enhanced-sync-status.operations.ts

@@ -1,5 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getPostgreSQLClient } from '../../../clients';
 import type { JobPayload, SyncStatus } from '../../../types/job-payloads';
 
 const logger = getLogger('enhanced-sync-status');

apps/data-pipeline/src/handlers/exchanges/operations/exchange-stats.operations.ts

@@ -1,5 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getPostgreSQLClient } from '../../../clients';
 import type { JobPayload } from '../../../types/job-payloads';
 
 const logger = getLogger('enhanced-sync-exchange-stats');

apps/data-pipeline/src/handlers/exchanges/operations/provider-mapping-stats.operations.ts

@@ -1,5 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getPostgreSQLClient } from '../../../clients';
 import type { JobPayload } from '../../../types/job-payloads';
 
 const logger = getLogger('enhanced-sync-provider-mapping-stats');

apps/data-pipeline/src/handlers/exchanges/operations/qm-exchanges.operations.ts

@@ -1,6 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getMongoDBClient } from '@stock-bot/mongodb-client';
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
 import type { JobPayload } from '../../../types/job-payloads';
 
 const logger = getLogger('sync-qm-exchanges');

apps/data-pipeline/src/handlers/exchanges/operations/sync-all-exchanges.operations.ts

@@ -1,6 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getMongoDBClient } from "@stock-bot/mongodb-client";
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
 import type { JobPayload, SyncResult } from '../../../types/job-payloads';
 
 const logger = getLogger('enhanced-sync-all-exchanges');

apps/data-pipeline/src/handlers/exchanges/operations/sync-ib-exchanges.operations.ts

@@ -1,7 +1,7 @@
 import { getLogger } from '@stock-bot/logger';
-import { getMongoDBClient } from '@stock-bot/mongodb-client';
+import { getMongoDBClient } from '../../../clients';
 import type { JobPayload } from '../../../types/job-payloads';
-import type { MasterExchange } from '@stock-bot/mongodb-client';
+import type { MasterExchange } from '@stock-bot/mongodb';
 
 const logger = getLogger('sync-ib-exchanges');
 

apps/data-pipeline/src/handlers/exchanges/operations/sync-qm-provider-mappings.operations.ts

@@ -1,6 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getMongoDBClient } from "@stock-bot/mongodb-client";
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
 import type { JobPayload, SyncResult } from '../../../types/job-payloads';
 
 const logger = getLogger('enhanced-sync-qm-provider-mappings');

apps/data-pipeline/src/handlers/symbols/operations/qm-symbols.operations.ts

@@ -1,6 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getMongoDBClient } from '@stock-bot/mongodb-client';
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
 import type { JobPayload } from '../../../types/job-payloads';
 
 const logger = getLogger('sync-qm-symbols');

apps/data-pipeline/src/handlers/symbols/operations/sync-status.operations.ts

@@ -1,5 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getPostgreSQLClient } from '../../../clients';
 import type { JobPayload } from '../../../types/job-payloads';
 
 const logger = getLogger('sync-status');

apps/data-pipeline/src/handlers/symbols/operations/sync-symbols-from-provider.operations.ts

@@ -1,6 +1,5 @@
 import { getLogger } from '@stock-bot/logger';
-import { getMongoDBClient } from "@stock-bot/mongodb-client";
-import { getPostgreSQLClient } from '@stock-bot/postgres-client';
+import { getMongoDBClient, getPostgreSQLClient } from '../../../clients';
 import type { JobPayload, SyncResult } from '../../../types/job-payloads';
 
 const logger = getLogger('enhanced-sync-symbols-from-provider');

apps/data-pipeline/src/index.ts

@@ -4,12 +4,13 @@ import { Hono } from 'hono';
 import { cors } from 'hono/cors';
 // Library imports
 import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
-import { connectMongoDB } from '@stock-bot/mongodb-client';
-import { connectPostgreSQL } from '@stock-bot/postgres-client';
+import { MongoDBClient } from '@stock-bot/mongodb';
+import { PostgreSQLClient } from '@stock-bot/postgres';
 import { QueueManager, type QueueManagerConfig } from '@stock-bot/queue';
 import { Shutdown } from '@stock-bot/shutdown';
 // Local imports
 import { enhancedSyncRoutes, healthRoutes, statsRoutes, syncRoutes } from './routes';
+import { setMongoDBClient, setPostgreSQLClient } from './clients';
 
 const config = initializeServiceConfig();
 console.log('Data Sync Service Configuration:', JSON.stringify(config, null, 2));
@@ -44,7 +45,8 @@ app.use(
 );
 const PORT = serviceConfig.port;
 let server: ReturnType<typeof Bun.serve> | null = null;
-// Singleton clients are managed in libraries
+let mongoClient: MongoDBClient | null = null;
+let postgresClient: PostgreSQLClient | null = null;
 let queueManager: QueueManager | null = null;
 
 // Initialize shutdown manager
@@ -61,10 +63,10 @@ async function initializeServices() {
   logger.info('Initializing data sync service...');
 
   try {
-    // Initialize MongoDB client singleton
+    // Initialize MongoDB client
     logger.debug('Connecting to MongoDB...');
     const mongoConfig = databaseConfig.mongodb;
-    await connectMongoDB({
+    mongoClient = new MongoDBClient({
       uri: mongoConfig.uri,
       database: mongoConfig.database,
       host: mongoConfig.host || 'localhost',
@@ -74,13 +76,15 @@ async function initializeServices() {
         socketTimeout: 30000,
         serverSelectionTimeout: 5000,
       },
-    });
+    }, logger);
+    await mongoClient.connect();
+    setMongoDBClient(mongoClient);
     logger.info('MongoDB connected');
 
-    // Initialize PostgreSQL client singleton
+    // Initialize PostgreSQL client
     logger.debug('Connecting to PostgreSQL...');
     const pgConfig = databaseConfig.postgres;
-    await connectPostgreSQL({
+    postgresClient = new PostgreSQLClient({
       host: pgConfig.host,
       port: pgConfig.port,
       database: pgConfig.database,
@@ -91,7 +95,9 @@ async function initializeServices() {
         max: pgConfig.poolSize || 10,
         idleTimeoutMillis: pgConfig.idleTimeout || 30000,
       },
-    });
+    }, logger);
+    await postgresClient.connect();
+    setPostgreSQLClient(postgresClient);
     logger.info('PostgreSQL connected');
 
     // Initialize queue system (with delayed worker start)
@@ -236,11 +242,12 @@ shutdown.onShutdownHigh(async () => {
 shutdown.onShutdownMedium(async () => {
   logger.info('Disconnecting from databases...');
   try {
-    const { disconnectMongoDB } = await import('@stock-bot/mongodb-client');
-    const { disconnectPostgreSQL } = await import('@stock-bot/postgres-client');
-
-    await disconnectMongoDB();
-    await disconnectPostgreSQL();
+    if (mongoClient) {
+      await mongoClient.disconnect();
+    }
+    if (postgresClient) {
+      await postgresClient.disconnect();
+    }
     logger.info('Database connections closed');
   } catch (error) {
     logger.error('Error closing database connections', { error });