fixed up data-service
This commit is contained in:
parent
68d977f9e0
commit
6b69bcbcaa
8 changed files with 135 additions and 535 deletions
|
|
@ -1,17 +1,36 @@
|
|||
/**
|
||||
* Data Service - Combined live and historical data ingestion with queue-based architecture
|
||||
* Data Service - Market data ingestion service
|
||||
*/
|
||||
|
||||
// Framework imports
|
||||
import { Hono } from 'hono';
|
||||
import { cors } from 'hono/cors';
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
import { getLogger, shutdownLoggers } from '@stock-bot/logger';
|
||||
import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client';
|
||||
import { processItems, QueueManager, type QueueConfig } from '@stock-bot/queue';
|
||||
|
||||
// Library imports
|
||||
import { initializeServiceConfig } from '@stock-bot/config-new';
|
||||
import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
|
||||
import { createAndConnectMongoDBClient, MongoDBClient } from '@stock-bot/mongodb-client';
|
||||
import { createAndConnectPostgreSQLClient, PostgreSQLClient } from '@stock-bot/postgres-client';
|
||||
import { Shutdown } from '@stock-bot/shutdown';
|
||||
import { initializeIBResources } from './providers/ib.tasks';
|
||||
import { initializeProxyResources } from './providers/proxy.tasks';
|
||||
|
||||
// Local imports
|
||||
import { exchangeRoutes, healthRoutes, queueRoutes } from './routes';
|
||||
|
||||
// Initialize configuration with automatic monorepo config inheritance
|
||||
const config = await initializeServiceConfig();
|
||||
const serviceConfig = config.service;
|
||||
const databaseConfig = config.database;
|
||||
|
||||
// Initialize logger with config
|
||||
const loggingConfig = config.logging;
|
||||
if (loggingConfig) {
|
||||
setLoggerConfig({
|
||||
logLevel: loggingConfig.level,
|
||||
logConsole: true,
|
||||
logFile: false,
|
||||
environment: config.environment,
|
||||
});
|
||||
}
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
|
|
@ -27,95 +46,56 @@ app.use(
|
|||
);
|
||||
|
||||
const logger = getLogger('data-service');
|
||||
const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002');
|
||||
const PORT = serviceConfig.port;
|
||||
let server: ReturnType<typeof Bun.serve> | null = null;
|
||||
let postgresClient: PostgreSQLClient | null = null;
|
||||
let mongoClient: MongoDBClient | null = null;
|
||||
|
||||
// Initialize shutdown manager with 15 second timeout
|
||||
// Initialize shutdown manager
|
||||
const shutdown = Shutdown.getInstance({ timeout: 15000 });
|
||||
|
||||
/**
|
||||
* Create and configure the enhanced queue manager for data service
|
||||
*/
|
||||
function createDataServiceQueueManager(): QueueManager {
|
||||
const config: QueueConfig = {
|
||||
queueName: 'data-service-queue',
|
||||
workers: parseInt(process.env.WORKER_COUNT || '5'),
|
||||
concurrency: parseInt(process.env.WORKER_CONCURRENCY || '20'),
|
||||
redis: {
|
||||
host: process.env.DRAGONFLY_HOST || 'localhost',
|
||||
port: parseInt(process.env.DRAGONFLY_PORT || '6379'),
|
||||
},
|
||||
providers: [
|
||||
// Import and initialize providers lazily
|
||||
async () => {
|
||||
const { initializeWebShareProvider } = await import('./providers/webshare.provider');
|
||||
return initializeWebShareProvider();
|
||||
},
|
||||
async () => {
|
||||
const { initializeIBProvider } = await import('./providers/ib.provider');
|
||||
return initializeIBProvider();
|
||||
},
|
||||
async () => {
|
||||
const { initializeProxyProvider } = await import('./providers/proxy.provider');
|
||||
return initializeProxyProvider();
|
||||
},
|
||||
async () => {
|
||||
const { initializeQMProvider } = await import('./providers/qm.provider');
|
||||
return initializeQMProvider();
|
||||
},
|
||||
async () => {
|
||||
const { initializeExchangeSyncProvider } = await import(
|
||||
'./providers/exchange-sync.provider'
|
||||
);
|
||||
return initializeExchangeSyncProvider();
|
||||
},
|
||||
],
|
||||
enableScheduledJobs: true,
|
||||
};
|
||||
|
||||
return new QueueManager(config);
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
export const queueManager = createDataServiceQueueManager();
|
||||
|
||||
// Export processItems for direct use
|
||||
export { processItems };
|
||||
|
||||
// Register all routes
|
||||
app.route('', exchangeRoutes);
|
||||
app.route('', healthRoutes);
|
||||
app.route('', queueRoutes);
|
||||
// Mount routes
|
||||
app.route('/health', healthRoutes);
|
||||
app.route('/api/exchanges', exchangeRoutes);
|
||||
app.route('/api/queue', queueRoutes);
|
||||
|
||||
// Initialize services
|
||||
async function initializeServices() {
|
||||
logger.info('Initializing data service...');
|
||||
|
||||
try {
|
||||
// Initialize MongoDB client first
|
||||
logger.info('Starting MongoDB client initialization...');
|
||||
await connectMongoDB();
|
||||
logger.info('MongoDB client initialized');
|
||||
// Initialize MongoDB client
|
||||
logger.info('Connecting to MongoDB...');
|
||||
const mongoConfig = databaseConfig.mongodb;
|
||||
mongoClient = await createAndConnectMongoDBClient({
|
||||
uri: mongoConfig.uri,
|
||||
database: mongoConfig.database,
|
||||
host: mongoConfig.host || 'localhost',
|
||||
port: mongoConfig.port || 27017,
|
||||
timeouts: {
|
||||
connectTimeout: 30000,
|
||||
socketTimeout: 30000,
|
||||
serverSelectionTimeout: 5000,
|
||||
},
|
||||
});
|
||||
logger.info('MongoDB connected');
|
||||
|
||||
// Initialize browser resources
|
||||
logger.info('Starting browser resources initialization...');
|
||||
await Browser.initialize();
|
||||
logger.info('Browser resources initialized');
|
||||
|
||||
// Initialize proxy resources
|
||||
logger.info('Starting proxy resources initialization...');
|
||||
await initializeProxyResources();
|
||||
logger.info('Proxy resources initialized');
|
||||
|
||||
// Initialize IB resources
|
||||
logger.info('Starting IB resources initialization...');
|
||||
await initializeIBResources();
|
||||
logger.info('IB resources initialized');
|
||||
|
||||
// Initialize queue manager (includes batch cache initialization)
|
||||
logger.info('Starting queue manager initialization...');
|
||||
await queueManager.initialize();
|
||||
logger.info('Queue manager initialized');
|
||||
// Initialize PostgreSQL client
|
||||
logger.info('Connecting to PostgreSQL...');
|
||||
const pgConfig = databaseConfig.postgres;
|
||||
postgresClient = await createAndConnectPostgreSQLClient({
|
||||
host: pgConfig.host,
|
||||
port: pgConfig.port,
|
||||
database: pgConfig.database,
|
||||
username: pgConfig.user,
|
||||
password: pgConfig.password,
|
||||
poolSettings: {
|
||||
min: 2,
|
||||
max: pgConfig.poolSize || 10,
|
||||
idleTimeoutMillis: pgConfig.idleTimeout || 30000,
|
||||
},
|
||||
});
|
||||
logger.info('PostgreSQL connected');
|
||||
|
||||
logger.info('All services initialized successfully');
|
||||
} catch (error) {
|
||||
|
|
@ -127,12 +107,13 @@ async function initializeServices() {
|
|||
// Start server
|
||||
async function startServer() {
|
||||
await initializeServices();
|
||||
// Start the HTTP server using Bun's native serve
|
||||
|
||||
server = Bun.serve({
|
||||
port: PORT,
|
||||
fetch: app.fetch,
|
||||
development: process.env.NODE_ENV === 'development',
|
||||
development: config.environment === 'development',
|
||||
});
|
||||
|
||||
logger.info(`Data Service started on port ${PORT}`);
|
||||
}
|
||||
|
||||
|
|
@ -142,7 +123,7 @@ shutdown.onShutdown(async () => {
|
|||
logger.info('Stopping HTTP server...');
|
||||
try {
|
||||
server.stop();
|
||||
logger.info('HTTP server stopped successfully');
|
||||
logger.info('HTTP server stopped');
|
||||
} catch (error) {
|
||||
logger.error('Error stopping HTTP server', { error });
|
||||
}
|
||||
|
|
@ -150,63 +131,33 @@ shutdown.onShutdown(async () => {
|
|||
});
|
||||
|
||||
shutdown.onShutdown(async () => {
|
||||
logger.info('Shutting down queue manager...');
|
||||
logger.info('Disconnecting from databases...');
|
||||
try {
|
||||
await queueManager.shutdown();
|
||||
logger.info('Queue manager shut down successfully');
|
||||
} catch (error) {
|
||||
logger.error('Error shutting down queue manager', { error });
|
||||
// Don't re-throw to allow other shutdown handlers to complete
|
||||
// The shutdown library tracks failures internally
|
||||
}
|
||||
});
|
||||
|
||||
// Add Browser shutdown handler
|
||||
shutdown.onShutdown(async () => {
|
||||
logger.info('Shutting down browser resources...');
|
||||
try {
|
||||
await Browser.close();
|
||||
logger.info('Browser resources shut down successfully');
|
||||
} catch (error) {
|
||||
// Browser might already be closed by running tasks, this is expected
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
if (errorMessage.includes('Target page, context or browser has been closed')) {
|
||||
logger.info('Browser was already closed by running tasks');
|
||||
} else {
|
||||
logger.error('Error shutting down browser resources', { error });
|
||||
if (mongoClient) {
|
||||
await mongoClient.disconnect();
|
||||
}
|
||||
// Don't throw here as browser shutdown shouldn't block app shutdown
|
||||
}
|
||||
});
|
||||
|
||||
// Add MongoDB shutdown handler
|
||||
shutdown.onShutdown(async () => {
|
||||
logger.info('Shutting down MongoDB client...');
|
||||
try {
|
||||
await disconnectMongoDB();
|
||||
logger.info('MongoDB client shut down successfully');
|
||||
if (postgresClient) {
|
||||
await postgresClient.disconnect();
|
||||
}
|
||||
logger.info('Database connections closed');
|
||||
} catch (error) {
|
||||
logger.error('Error shutting down MongoDB client', { error });
|
||||
// Don't throw here to allow other shutdown handlers to complete
|
||||
logger.error('Error closing database connections', { error });
|
||||
}
|
||||
});
|
||||
|
||||
// Add logger shutdown handler (should be last)
|
||||
shutdown.onShutdown(async () => {
|
||||
try {
|
||||
await shutdownLoggers();
|
||||
// Use process.stdout since loggers are being shut down
|
||||
process.stdout.write('All loggers flushed and shut down successfully\n');
|
||||
process.stdout.write('Data service loggers shut down\n');
|
||||
} catch (error) {
|
||||
process.stderr.write(`Error shutting down loggers: ${error}\n`);
|
||||
// Don't throw here as this is the final cleanup
|
||||
}
|
||||
});
|
||||
|
||||
// Start the application
|
||||
// Start the service
|
||||
startServer().catch(error => {
|
||||
logger.error('Failed to start server', { error });
|
||||
logger.error('Failed to start data service', { error });
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
logger.info('Data service startup initiated with graceful shutdown handlers');
|
||||
logger.info('Data service startup initiated');
|
||||
Loading…
Add table
Add a link
Reference in a new issue