removed old working on ceo handler
This commit is contained in:
parent
acf66dbfb6
commit
5009ccbeda
8 changed files with 243 additions and 213 deletions
|
|
@ -1,6 +1,7 @@
|
|||
import {
|
||||
BaseHandler,
|
||||
Handler,
|
||||
Operation,
|
||||
ScheduledOperation,
|
||||
type IServiceContainer
|
||||
} from '@stock-bot/handlers';
|
||||
|
|
@ -81,4 +82,135 @@ export class CeoHandler extends BaseHandler {
|
|||
this.logger.info(`Fetched CEO channels for page ${page}/${totalPages}`);
|
||||
return { page, totalPages };
|
||||
}
|
||||
|
||||
@Operation('process-unique-symbols')
|
||||
@ScheduledOperation('process-unique-symbols', '0 */30 * * *', {
|
||||
priority: 5,
|
||||
immediately: false,
|
||||
description: 'Process unique CEO symbols and schedule individual jobs'
|
||||
})
|
||||
async processUniqueSymbols(_payload: unknown, _context: any): Promise<unknown> {
|
||||
this.logger.info('Starting process to get unique CEO symbols by ceoId');
|
||||
|
||||
try {
|
||||
// Get unique ceoId values from the ceoSymbols collection
|
||||
const uniqueCeoIds = await this.mongodb.collection('ceoSymbols').distinct('ceoId');
|
||||
|
||||
this.logger.info(`Found ${uniqueCeoIds.length} unique CEO IDs`);
|
||||
|
||||
// Get detailed records for each unique ceoId (latest/first record)
|
||||
const uniqueSymbols = [];
|
||||
for (const ceoId of uniqueCeoIds) {
|
||||
const symbol = await this.mongodb.collection('ceoSymbols')
|
||||
.findOne({ ceoId }, { sort: { _id: -1 } }); // Get latest record
|
||||
|
||||
if (symbol) {
|
||||
uniqueSymbols.push(symbol);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info(`Retrieved ${uniqueSymbols.length} unique symbol records`);
|
||||
|
||||
// Schedule individual jobs for each unique symbol
|
||||
let scheduledJobs = 0;
|
||||
for (const symbol of uniqueSymbols) {
|
||||
// Schedule a job to process this individual symbol
|
||||
await this.scheduleOperation('process-individual-symbol', {
|
||||
ceoId: symbol.ceoId,
|
||||
symbol: symbol.symbol,
|
||||
exchange: symbol.exchange,
|
||||
name: symbol.name
|
||||
});
|
||||
scheduledJobs++;
|
||||
|
||||
// Add small delay to avoid overwhelming the queue
|
||||
if (scheduledJobs % 10 === 0) {
|
||||
this.logger.debug(`Scheduled ${scheduledJobs} jobs so far`);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info(`Successfully scheduled ${scheduledJobs} individual symbol processing jobs`);
|
||||
|
||||
// Cache the results for monitoring
|
||||
await this.cacheSet('unique-symbols-last-run', {
|
||||
timestamp: new Date().toISOString(),
|
||||
totalUniqueIds: uniqueCeoIds.length,
|
||||
totalRecords: uniqueSymbols.length,
|
||||
scheduledJobs
|
||||
}, 1800); // Cache for 30 minutes
|
||||
|
||||
return {
|
||||
success: true,
|
||||
uniqueCeoIds: uniqueCeoIds.length,
|
||||
uniqueRecords: uniqueSymbols.length,
|
||||
scheduledJobs,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to process unique CEO symbols', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@Operation('process-individual-symbol')
|
||||
async processIndividualSymbol(payload: any, _context: any): Promise<unknown> {
|
||||
const { ceoId, symbol, exchange, name } = payload;
|
||||
|
||||
this.logger.debug('Processing individual CEO symbol', {
|
||||
ceoId,
|
||||
symbol,
|
||||
exchange,
|
||||
name
|
||||
});
|
||||
|
||||
try {
|
||||
// Here you can add specific processing logic for each symbol
|
||||
// For now, just log and potentially fetch additional data
|
||||
|
||||
// Example: Get all historical records for this ceoId
|
||||
const allRecords = await this.mongodb.collection('ceoSymbols')
|
||||
.find({ ceoId })
|
||||
.sort({ _id: -1 })
|
||||
.toArray();
|
||||
|
||||
this.logger.debug(`Found ${allRecords.length} records for CEO ID ${ceoId}`);
|
||||
|
||||
// Example: Update processing status
|
||||
await this.mongodb.collection('ceoSymbols').updateMany(
|
||||
{ ceoId },
|
||||
{
|
||||
$set: {
|
||||
lastProcessed: new Date(),
|
||||
processedBy: 'individual-symbol-processor'
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Cache individual symbol data
|
||||
await this.cacheSet(`symbol-${ceoId}`, {
|
||||
symbol,
|
||||
exchange,
|
||||
name,
|
||||
recordCount: allRecords.length,
|
||||
lastProcessed: new Date().toISOString()
|
||||
}, 3600); // Cache for 1 hour
|
||||
|
||||
return {
|
||||
success: true,
|
||||
ceoId,
|
||||
symbol,
|
||||
recordsProcessed: allRecords.length,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to process individual symbol', {
|
||||
error,
|
||||
ceoId,
|
||||
symbol
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { getLogger } from '@stock-bot/logger';
|
|||
// Import handlers for bundling (ensures they're included in the build)
|
||||
import './qm/qm.handler';
|
||||
import './webshare/webshare.handler';
|
||||
import './ceo/ceo.handler';
|
||||
// Add more handler imports as needed
|
||||
|
||||
const logger = getLogger('handler-init');
|
||||
|
|
|
|||
101
apps/data-ingestion/test-ceo-operations.ts
Executable file
101
apps/data-ingestion/test-ceo-operations.ts
Executable file
|
|
@ -0,0 +1,101 @@
|
|||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Test script for CEO handler operations
|
||||
*/
|
||||
|
||||
import { initializeServiceConfig } from '@stock-bot/config';
|
||||
import { createServiceContainer, initializeServices } from '@stock-bot/di';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
|
||||
const logger = getLogger('test-ceo-operations');
|
||||
|
||||
async function testCeoOperations() {
|
||||
logger.info('Testing CEO handler operations...');
|
||||
|
||||
try {
|
||||
// Initialize config
|
||||
const config = initializeServiceConfig();
|
||||
|
||||
// Create Awilix container
|
||||
const awilixConfig = {
|
||||
redis: {
|
||||
host: config.database.dragonfly.host,
|
||||
port: config.database.dragonfly.port,
|
||||
db: config.database.dragonfly.db,
|
||||
},
|
||||
mongodb: {
|
||||
uri: config.database.mongodb.uri,
|
||||
database: config.database.mongodb.database,
|
||||
},
|
||||
postgres: {
|
||||
host: config.database.postgres.host,
|
||||
port: config.database.postgres.port,
|
||||
database: config.database.postgres.database,
|
||||
user: config.database.postgres.user,
|
||||
password: config.database.postgres.password,
|
||||
},
|
||||
questdb: {
|
||||
enabled: false,
|
||||
host: config.database.questdb.host,
|
||||
httpPort: config.database.questdb.httpPort,
|
||||
pgPort: config.database.questdb.pgPort,
|
||||
influxPort: config.database.questdb.ilpPort,
|
||||
database: config.database.questdb.database,
|
||||
},
|
||||
};
|
||||
|
||||
const container = createServiceContainer(awilixConfig);
|
||||
await initializeServices(container);
|
||||
|
||||
const serviceContainer = container.resolve('serviceContainer');
|
||||
|
||||
// Import and create CEO handler
|
||||
const { CeoHandler } = await import('./src/handlers/ceo/ceo.handler');
|
||||
const ceoHandler = new CeoHandler(serviceContainer);
|
||||
|
||||
// Test 1: Check if there are any CEO symbols in the database
|
||||
logger.info('Checking for existing CEO symbols...');
|
||||
const collection = serviceContainer.mongodb.collection('ceoSymbols');
|
||||
const count = await collection.countDocuments();
|
||||
logger.info(`Found ${count} CEO symbols in database`);
|
||||
|
||||
if (count > 0) {
|
||||
// Test 2: Run process-unique-symbols operation
|
||||
logger.info('Testing process-unique-symbols operation...');
|
||||
const result = await ceoHandler.processUniqueSymbols(undefined, {});
|
||||
logger.info('Process unique symbols result:', result);
|
||||
|
||||
// Test 3: Test individual symbol processing
|
||||
logger.info('Testing process-individual-symbol operation...');
|
||||
const sampleSymbol = await collection.findOne({});
|
||||
if (sampleSymbol) {
|
||||
const individualResult = await ceoHandler.processIndividualSymbol({
|
||||
ceoId: sampleSymbol.ceoId,
|
||||
symbol: sampleSymbol.symbol,
|
||||
exchange: sampleSymbol.exchange,
|
||||
name: sampleSymbol.name,
|
||||
}, {});
|
||||
logger.info('Process individual symbol result:', individualResult);
|
||||
}
|
||||
} else {
|
||||
logger.warn('No CEO symbols found. Run the service to populate data first.');
|
||||
}
|
||||
|
||||
// Clean up
|
||||
await serviceContainer.mongodb.disconnect();
|
||||
await serviceContainer.postgres.disconnect();
|
||||
if (serviceContainer.cache) {
|
||||
await serviceContainer.cache.disconnect();
|
||||
}
|
||||
|
||||
logger.info('Test completed successfully!');
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
logger.error('Test failed:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the test
|
||||
testCeoOperations();
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
// Export all dependency injection components
|
||||
export * from './service-container';
|
||||
export * from './operation-context';
|
||||
export * from './pool-size-calculator';
|
||||
export * from './types';
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@
|
|||
*/
|
||||
|
||||
import { getLogger, type Logger } from '@stock-bot/logger';
|
||||
import type { ServiceResolver } from './service-container';
|
||||
interface ServiceResolver {
|
||||
resolve<T>(serviceName: string): T;
|
||||
resolveAsync<T>(serviceName: string): Promise<T>;
|
||||
}
|
||||
|
||||
export interface OperationContextOptions {
|
||||
handlerName: string;
|
||||
|
|
|
|||
|
|
@ -1,201 +0,0 @@
|
|||
import { getLogger, type Logger } from '@stock-bot/logger';
|
||||
import type { ConnectionFactory } from './connection-factory';
|
||||
|
||||
export interface ServiceRegistration<T = any> {
|
||||
name: string;
|
||||
factory: () => T | Promise<T>;
|
||||
singleton?: boolean;
|
||||
dispose?: (instance: T) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface ServiceResolver {
|
||||
resolve<T>(name: string, options?: any): T;
|
||||
resolveAsync<T>(name: string, options?: any): Promise<T>;
|
||||
}
|
||||
|
||||
export class ServiceContainer implements ServiceResolver {
|
||||
private readonly logger: Logger;
|
||||
private readonly registrations = new Map<string, ServiceRegistration>();
|
||||
private readonly instances = new Map<string, any>();
|
||||
private readonly scopedInstances = new Map<string, any>();
|
||||
private readonly parent?: ServiceContainer;
|
||||
|
||||
constructor(name: string, parent?: ServiceContainer) {
|
||||
this.logger = getLogger(`service-container:${name}`);
|
||||
this.parent = parent;
|
||||
}
|
||||
|
||||
register<T>(registration: ServiceRegistration<T>): void {
|
||||
this.registrations.set(registration.name, registration);
|
||||
this.logger.debug('Service registered', { name: registration.name, singleton: registration.singleton });
|
||||
}
|
||||
|
||||
resolve<T>(name: string, _options?: any): T {
|
||||
// Check scoped instances first
|
||||
if (this.scopedInstances.has(name)) {
|
||||
return this.scopedInstances.get(name);
|
||||
}
|
||||
|
||||
// Check singleton instances
|
||||
if (this.instances.has(name)) {
|
||||
return this.instances.get(name);
|
||||
}
|
||||
|
||||
// Get registration from this container or parent
|
||||
const registration = this.getRegistration(name);
|
||||
if (!registration) {
|
||||
throw new Error(`Service ${name} not registered`);
|
||||
}
|
||||
|
||||
// Create instance synchronously
|
||||
const instance = registration.factory();
|
||||
|
||||
// Check if factory returned a promise
|
||||
if (instance instanceof Promise) {
|
||||
throw new Error(`Service ${name} is async. Use resolveAsync() instead.`);
|
||||
}
|
||||
|
||||
// Store based on singleton flag
|
||||
if (registration.singleton) {
|
||||
this.instances.set(name, instance);
|
||||
} else {
|
||||
this.scopedInstances.set(name, instance);
|
||||
}
|
||||
|
||||
return instance as T;
|
||||
}
|
||||
|
||||
async resolveAsync<T>(name: string, _options?: any): Promise<T> {
|
||||
// Check scoped instances first
|
||||
if (this.scopedInstances.has(name)) {
|
||||
return this.scopedInstances.get(name);
|
||||
}
|
||||
|
||||
// Check singleton instances
|
||||
if (this.instances.has(name)) {
|
||||
return this.instances.get(name);
|
||||
}
|
||||
|
||||
// Get registration from this container or parent
|
||||
const registration = this.getRegistration(name);
|
||||
if (!registration) {
|
||||
throw new Error(`Service ${name} not registered`);
|
||||
}
|
||||
|
||||
// Create instance
|
||||
const instance = await Promise.resolve(registration.factory());
|
||||
|
||||
// Store based on singleton flag
|
||||
if (registration.singleton) {
|
||||
this.instances.set(name, instance);
|
||||
} else {
|
||||
this.scopedInstances.set(name, instance);
|
||||
}
|
||||
|
||||
return instance as T;
|
||||
}
|
||||
|
||||
createScope(): ServiceContainer {
|
||||
return new ServiceContainer('scoped', this);
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
// Dispose scoped instances
|
||||
for (const [name, instance] of this.scopedInstances.entries()) {
|
||||
const registration = this.getRegistration(name);
|
||||
if (registration?.dispose) {
|
||||
await registration.dispose(instance);
|
||||
}
|
||||
}
|
||||
this.scopedInstances.clear();
|
||||
|
||||
// Only dispose singletons if this is the root container
|
||||
if (!this.parent) {
|
||||
for (const [name, instance] of this.instances.entries()) {
|
||||
const registration = this.registrations.get(name);
|
||||
if (registration?.dispose) {
|
||||
await registration.dispose(instance);
|
||||
}
|
||||
}
|
||||
this.instances.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private getRegistration(name: string): ServiceRegistration | undefined {
|
||||
return this.registrations.get(name) || this.parent?.getRegistration(name);
|
||||
}
|
||||
}
|
||||
|
||||
// Enhanced service container factory with infrastructure services
|
||||
export function createServiceContainer(
|
||||
serviceName: string,
|
||||
connectionFactory: ConnectionFactory,
|
||||
config?: any
|
||||
): ServiceContainer {
|
||||
const container = new ServiceContainer(serviceName);
|
||||
|
||||
// Register configuration if provided
|
||||
if (config) {
|
||||
container.register({
|
||||
name: 'config',
|
||||
factory: () => config,
|
||||
singleton: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Register connection factories
|
||||
container.register({
|
||||
name: 'mongodb',
|
||||
factory: async () => {
|
||||
const pool = await connectionFactory.createMongoDB({
|
||||
name: 'default',
|
||||
config: {} as any, // Config injected by factory
|
||||
});
|
||||
return pool.client;
|
||||
},
|
||||
singleton: true,
|
||||
});
|
||||
|
||||
container.register({
|
||||
name: 'postgres',
|
||||
factory: async () => {
|
||||
const pool = await connectionFactory.createPostgreSQL({
|
||||
name: 'default',
|
||||
config: {} as any, // Config injected by factory
|
||||
});
|
||||
return pool.client;
|
||||
},
|
||||
singleton: true,
|
||||
});
|
||||
|
||||
container.register({
|
||||
name: 'cache',
|
||||
factory: async () => {
|
||||
const pool = await connectionFactory.createCache({
|
||||
name: 'default',
|
||||
config: {} as any, // Config injected by factory
|
||||
});
|
||||
return pool.client;
|
||||
},
|
||||
singleton: true,
|
||||
});
|
||||
|
||||
container.register({
|
||||
name: 'queue',
|
||||
factory: async () => {
|
||||
const pool = await connectionFactory.createQueue({
|
||||
name: 'default',
|
||||
config: {} as any, // Config injected by factory
|
||||
});
|
||||
return pool.client;
|
||||
},
|
||||
singleton: true,
|
||||
});
|
||||
|
||||
// Note: Additional services can be registered by individual applications as needed:
|
||||
// - ProxyManager: container.register({ name: 'proxyManager', factory: () => ProxyManager.getInstance() })
|
||||
// - Browser: container.register({ name: 'browser', factory: () => Browser })
|
||||
// - HttpClient: container.register({ name: 'httpClient', factory: () => createHttpClient(...) })
|
||||
|
||||
return container;
|
||||
}
|
||||
|
|
@ -90,12 +90,6 @@ export abstract class BaseHandler implements IHandler {
|
|||
await queue.add(operation, jobData, { delay });
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to schedule an operation with delay in seconds
|
||||
*/
|
||||
async scheduleIn(operation: string, payload: unknown, delaySeconds: number): Promise<void> {
|
||||
return this.scheduleOperation(operation, payload, delaySeconds * 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create execution context for operations
|
||||
|
|
@ -118,11 +112,11 @@ export abstract class BaseHandler implements IHandler {
|
|||
/**
|
||||
* Get a MongoDB collection with type safety
|
||||
*/
|
||||
protected collection<T = any>(name: string): Collection<T> {
|
||||
protected collection<T extends {} = any>(name: string): Collection<T> {
|
||||
if (!this.mongodb) {
|
||||
throw new Error('MongoDB service is not available');
|
||||
}
|
||||
return this.mongodb.collection<T>(name);
|
||||
return this.mongodb.collection(name);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -142,7 +136,7 @@ export abstract class BaseHandler implements IHandler {
|
|||
if (!this.cache) {
|
||||
return null;
|
||||
}
|
||||
return this.cache.get<T>(`${this.handlerName}:${key}`);
|
||||
return this.cache.get(`${this.handlerName}:${key}`);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
"references": [
|
||||
{ "path": "../config" },
|
||||
{ "path": "../logger" },
|
||||
{ "path": "../di" }
|
||||
{ "path": "../di" },
|
||||
{ "path": "../../utils" }
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue