boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

handler to auto register and removed service registry, cleaned up queues and cache naming

Browse source
This commit is contained in:
Boki 2025-06-23 21:23:38 -04:00
parent 0d1be9e3cb
commit 34c6c36695
19 changed files with 474 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

1
apps/stock/data-ingestion/src/handlers/index.ts
View file

@ -27,6 +27,7 @@ export async function initializeAllHandlers(serviceContainer: IServiceContainer)
pattern: '.handler.',
exclude: ['test', 'spec'],
dryRun: false,
serviceName: 'data-ingestion',
});
logger.info('Handler auto-registration complete', {

5
apps/stock/web-api/src/routes/monitoring.routes.ts
View file

@ -234,17 +234,16 @@ export function createMonitoringRoutes(container: IServiceContainer) {
const connection = {
host: 'localhost',
port: 6379,
db: 1,
db: 0, // All queues in DB 0
};
const queue = new Queue(`{${queueName}}`, { connection });
const queue = new Queue(queueName, { connection });
try {
const counts = await queue.getJobCounts();
await queue.close();
return c.json({
queueName,
bullmqName: `{${queueName}}`,
counts
});
} catch (error: any) {

126
apps/stock/web-api/src/services/monitoring.service.ts
View file

@ -91,28 +91,42 @@ export class MonitoringService {
});
// Always use the known queue names since web-api doesn't create worker queues
const queueNames = ['proxy', 'qm', 'ib', 'ceo', 'webshare', 'exchanges', 'symbols'];
const handlerMapping = {
'proxy': 'data-ingestion',
'qm': 'data-ingestion',
'ib': 'data-ingestion',
'ceo': 'data-ingestion',
'webshare': 'data-ingestion',
'exchanges': 'data-pipeline',
'symbols': 'data-pipeline',
};
const queueNames = Object.keys(handlerMapping);
this.logger.debug('Using known queue names', { count: queueNames.length, names: queueNames });
// Create BullMQ queues directly with the correct format
for (const queueName of queueNames) {
for (const handlerName of queueNames) {
try {
// Import BullMQ directly to create queue instances
const { Queue: BullMQQueue } = await import('bullmq');
const connection = {
host: 'localhost',
port: 6379,
db: 1, // Queue DB
db: 0, // All queues now in DB 0
};
// Create BullMQ queue with the correct format
const bullQueue = new BullMQQueue(`{${queueName}}`, { connection });
// Get the service that owns this handler
const serviceName = handlerMapping[handlerName as keyof typeof handlerMapping];
// Create BullMQ queue with the new naming format {service_handler}
const fullQueueName = `{${serviceName}_${handlerName}}`;
const bullQueue = new BullMQQueue(fullQueueName, { connection });
// Get stats directly from BullMQ
const queueStats = await this.getQueueStatsForBullQueue(bullQueue, queueName);
const queueStats = await this.getQueueStatsForBullQueue(bullQueue, handlerName);
stats.push({
name: queueName,
name: handlerName,
connected: true,
jobs: queueStats,
workers: {
@ -124,9 +138,9 @@ export class MonitoringService {
// Close the queue connection after getting stats
await bullQueue.close();
} catch (error) {
this.logger.warn(`Failed to get stats for queue ${queueName}`, { error });
this.logger.warn(`Failed to get stats for queue ${handlerName}`, { error });
stats.push({
name: queueName,
name: handlerName,
connected: false,
jobs: {
waiting: 0,
@ -535,6 +549,20 @@ export class MonitoringService {
for (const service of serviceEndpoints) {
try {
// For the current service (web-api), add it directly without health check
if (service.name === 'web-api') {
services.push({
name: 'web-api',
version: '1.0.0',
status: 'running',
port: process.env.PORT ? parseInt(process.env.PORT) : 2003,
uptime: Date.now() - this.startTime,
lastCheck: new Date().toISOString(),
healthy: true,
});
continue;
}
const startTime = Date.now();
const response = await fetch(`http://localhost:${service.port}${service.path}`, {
signal: AbortSignal.timeout(5000), // 5 second timeout
@ -578,17 +606,6 @@ export class MonitoringService {
}
}
// Add current service (web-api)
services.push({
name: 'web-api',
version: '1.0.0',
status: 'running',
port: process.env.PORT ? parseInt(process.env.PORT) : 2003,
uptime: Date.now() - this.startTime,
lastCheck: new Date().toISOString(),
healthy: true,
});
return services;
}
@ -597,7 +614,9 @@ export class MonitoringService {
*/
async getProxyStats(): Promise<ProxyStats | null> {
try {
if (!this.container.proxy) {
// Since web-api doesn't have proxy manager, query the cache directly
// The proxy manager stores data with cache:proxy: prefix
if (!this.container.cache) {
return {
enabled: false,
totalProxies: 0,
@ -606,10 +625,55 @@ export class MonitoringService {
};
}
const proxyManager = this.container.proxy as any;
// Check if proxy manager is ready
if (!proxyManager.isReady || !proxyManager.isReady()) {
try {
// Get proxy data from cache using getRaw method
// The proxy manager uses cache:proxy: prefix, but web-api cache uses cache:api:
const cacheProvider = this.container.cache;
if (cacheProvider.getRaw) {
// Use getRaw to access data with different cache prefix
// The proxy manager now uses a global cache:proxy: prefix
this.logger.debug('Attempting to fetch proxy data from cache');
const [cachedProxies, lastUpdateStr] = await Promise.all([
cacheProvider.getRaw<any[]>('cache:proxy:active'),
cacheProvider.getRaw<string>('cache:proxy:last-update')
]);
this.logger.debug('Proxy cache data retrieved', {
hasProxies: !!cachedProxies,
isArray: Array.isArray(cachedProxies),
proxyCount: cachedProxies ? cachedProxies.length : 0,
lastUpdate: lastUpdateStr
});
if (cachedProxies && Array.isArray(cachedProxies)) {
const workingCount = cachedProxies.filter((p: any) => p.isWorking !== false).length;
const failedCount = cachedProxies.filter((p: any) => p.isWorking === false).length;
return {
enabled: true,
totalProxies: cachedProxies.length,
workingProxies: workingCount,
failedProxies: failedCount,
lastUpdate: lastUpdateStr || undefined,
};
}
} else {
this.logger.debug('Cache provider does not support getRaw method');
}
// No cached data found - proxies might not be initialized yet
return {
enabled: true,
totalProxies: 0,
workingProxies: 0,
failedProxies: 0,
};
} catch (cacheError) {
this.logger.debug('Could not retrieve proxy data from cache', { error: cacheError });
// Return basic stats if cache query fails
return {
enabled: true,
totalProxies: 0,
@ -617,18 +681,6 @@ export class MonitoringService {
failedProxies: 0,
};
}
const stats = proxyManager.getStats ? proxyManager.getStats() : null;
const lastFetchTime = proxyManager.getLastFetchTime ? proxyManager.getLastFetchTime() : null;
return {
enabled: true,
totalProxies: stats?.total || 0,
workingProxies: stats?.working || 0,
failedProxies: stats?.failed || 0,
lastUpdate: stats?.lastUpdate ? new Date(stats.lastUpdate).toISOString() : undefined,
lastFetchTime: lastFetchTime ? new Date(lastFetchTime).toISOString() : undefined,
};
} catch (error) {
this.logger.error('Failed to get proxy stats', { error });
return null;