From 87037e013f6e77879f31ca0c9a4fe65b25ae9d16 Mon Sep 17 00:00:00 2001 From: Boki Date: Fri, 20 Jun 2025 09:51:32 -0400 Subject: [PATCH] fixed up more type issues --- .../src/providers/exchange-sync.provider.ts | 32 +++---- .../data-service/src/providers/ib.provider.ts | 24 +++-- .../src/providers/proxy.provider.ts | 11 +-- .../data-service/src/providers/qm.provider.ts | 24 ++--- .../src/providers/webshare.provider.ts | 7 +- apps/data-service/src/types/exchange.types.ts | 40 ++++++++ apps/data-service/src/types/job-payloads.ts | 93 +++++++++++++++++++ libs/queue/src/index.ts | 4 + libs/queue/src/types.ts | 27 ++++++ 9 files changed, 210 insertions(+), 52 deletions(-) create mode 100644 apps/data-service/src/types/exchange.types.ts create mode 100644 apps/data-service/src/types/job-payloads.ts diff --git a/apps/data-service/src/providers/exchange-sync.provider.ts b/apps/data-service/src/providers/exchange-sync.provider.ts index 40eeebc..8115d30 100644 --- a/apps/data-service/src/providers/exchange-sync.provider.ts +++ b/apps/data-service/src/providers/exchange-sync.provider.ts @@ -4,7 +4,8 @@ import { getLogger } from '@stock-bot/logger'; import type { MasterExchange } from '@stock-bot/mongodb-client'; import type { HandlerConfigWithSchedule } from '@stock-bot/queue'; -import { handlerRegistry } from '@stock-bot/queue'; +import { createJobHandler, handlerRegistry } from '@stock-bot/queue'; +import type { IBExchange } from '../types/exchange.types'; const logger = getLogger('exchange-sync'); @@ -15,16 +16,16 @@ export function initializeExchangeSyncProvider() { name: 'exchange-sync', operations: { - 'sync-ib-exchanges': async _payload => { + 'sync-ib-exchanges': createJobHandler(async () => { logger.info('Syncing IB exchanges to master table'); return await syncIBExchanges(); - }, + }), - 'get-master-exchange': async (payload: { masterExchangeId: string }) => { + 'get-master-exchange': createJobHandler(async (payload: { masterExchangeId: string }) => { logger.debug('Getting master exchange details', payload); const exchange = await getMasterExchangeDetails(payload.masterExchangeId); return { exchange, ...payload }; - }, + }), }, scheduledJobs: [ @@ -60,7 +61,7 @@ async function syncIBExchanges(): Promise<{ syncedCount: number; totalExchanges: // Filter by country code US and CA const ibExchanges = await db - .collection('ibExchanges') + .collection('ibExchanges') .find({ country_code: { $in: ['US', 'CA'] }, }) @@ -99,14 +100,7 @@ async function syncIBExchanges(): Promise<{ syncedCount: number; totalExchanges: /** * Create or update master exchange record 1:1 from IB exchange */ -async function createOrUpdateMasterExchange(ibExchange: { - id?: string; - name?: string; - code?: string; - country_code?: string; - currency?: string; - _id?: unknown; -}): Promise { +async function createOrUpdateMasterExchange(ibExchange: IBExchange): Promise { const { connectMongoDB, getDatabase } = await import('@stock-bot/mongodb-client'); await connectMongoDB(); @@ -191,11 +185,7 @@ async function getMasterExchangeDetails(masterExchangeId: string): Promise { + 'fetch-session': createJobHandler(async () => { // payload contains session configuration (not used in current implementation) logger.debug('Processing session fetch request'); const { fetchSession } = await import('./ib.tasks'); return fetchSession(); - }, + }), - 'fetch-exchanges': async _payload => { + 'fetch-exchanges': createJobHandler(async () => { // payload should contain session headers logger.debug('Processing exchanges fetch request'); const { fetchSession, fetchExchanges } = await import('./ib.tasks'); @@ -30,9 +33,9 @@ export function initializeIBProvider() { return fetchExchanges(sessionHeaders); } throw new Error('Failed to get session headers'); - }, + }), - 'fetch-symbols': async _payload => { + 'fetch-symbols': createJobHandler(async () => { // payload should contain session headers logger.debug('Processing symbols fetch request'); const { fetchSession, fetchSymbols } = await import('./ib.tasks'); @@ -41,9 +44,9 @@ export function initializeIBProvider() { return fetchSymbols(sessionHeaders); } throw new Error('Failed to get session headers'); - }, + }), - 'ib-exchanges-and-symbols': async _payload => { + 'ib-exchanges-and-symbols': createJobHandler(async () => { // Legacy operation for scheduled jobs logger.info('Fetching symbol summary from IB'); const { fetchSession, fetchExchanges, fetchSymbols } = await import('./ib.tasks'); @@ -62,7 +65,8 @@ export function initializeIBProvider() { return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length }; } - }, + return null; + }), }, scheduledJobs: [ { diff --git a/apps/data-service/src/providers/proxy.provider.ts b/apps/data-service/src/providers/proxy.provider.ts index c82a76e..de931ea 100644 --- a/apps/data-service/src/providers/proxy.provider.ts +++ b/apps/data-service/src/providers/proxy.provider.ts @@ -3,8 +3,7 @@ */ import { ProxyInfo } from '@stock-bot/http'; import { getLogger } from '@stock-bot/logger'; -import type { HandlerConfigWithSchedule } from '@stock-bot/queue'; -import { handlerRegistry } from '@stock-bot/queue'; +import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue'; const logger = getLogger('proxy-provider'); @@ -16,7 +15,7 @@ export function initializeProxyProvider() { name: 'proxy', operations: { - 'fetch-from-sources': async _payload => { + 'fetch-from-sources': createJobHandler(async () => { // Fetch proxies from all configured sources logger.info('Processing fetch proxies from sources request'); const { fetchProxiesFromSources } = await import('./proxy.tasks'); @@ -59,16 +58,16 @@ export function initializeProxyProvider() { batchesCreated: batchResult.batchesCreated, mode: batchResult.mode, }; - }, + }), - 'check-proxy': async (payload: ProxyInfo) => { + 'check-proxy': createJobHandler(async (payload: ProxyInfo) => { // payload is now the raw proxy info object logger.debug('Processing proxy check request', { proxy: `${payload.host}:${payload.port}`, }); const { checkProxy } = await import('./proxy.tasks'); return checkProxy(payload); - }, + }), }, scheduledJobs: [ { diff --git a/apps/data-service/src/providers/qm.provider.ts b/apps/data-service/src/providers/qm.provider.ts index fe1bd0b..3a106c2 100644 --- a/apps/data-service/src/providers/qm.provider.ts +++ b/apps/data-service/src/providers/qm.provider.ts @@ -1,24 +1,28 @@ import { getLogger } from '@stock-bot/logger'; -import { handlerRegistry, type HandlerConfigWithSchedule } from '@stock-bot/queue'; +import { + handlerRegistry, + createJobHandler, + type HandlerConfigWithSchedule +} from '@stock-bot/queue'; import type { SymbolSpiderJob } from './qm.tasks'; const logger = getLogger('qm-provider'); -// Initialize and register the IB provider +// Initialize and register the QM provider export function initializeQMProvider() { - logger.debug('Registering IB provider with scheduled jobs...'); + logger.debug('Registering QM provider with scheduled jobs...'); const qmProviderConfig: HandlerConfigWithSchedule = { name: 'qm', operations: { - 'create-sessions': async () => { + 'create-sessions': createJobHandler(async () => { logger.debug('Creating QM sessions...'); const { createSessions } = await import('./qm.tasks'); await createSessions(); logger.debug('QM sessions created successfully'); return { success: true, message: 'QM sessions created successfully' }; - }, - 'search-symbols': async () => { + }), + 'search-symbols': createJobHandler(async () => { logger.info('Starting QM symbol search...'); const { fetchSymbols } = await import('./qm.tasks'); const symbols = await fetchSymbols(); @@ -39,8 +43,8 @@ export function initializeQMProvider() { count: 0, }; } - }, - 'spider-symbol-search': async (payload: SymbolSpiderJob) => { + }), + 'spider-symbol-search': createJobHandler(async (payload: SymbolSpiderJob) => { logger.debug('Processing spider symbol search job', { payload }); const { spiderSymbolSearch } = await import('./qm.tasks'); const result = await spiderSymbolSearch(payload); @@ -48,12 +52,10 @@ export function initializeQMProvider() { logger.debug('Spider search job completed', { success: result.success, symbolsFound: result.symbolsFound, - jobsCreated: result.jobsCreated, - payload, }); return result; - }, + }), }, scheduledJobs: [ diff --git a/apps/data-service/src/providers/webshare.provider.ts b/apps/data-service/src/providers/webshare.provider.ts index 7deef85..add2fde 100644 --- a/apps/data-service/src/providers/webshare.provider.ts +++ b/apps/data-service/src/providers/webshare.provider.ts @@ -2,8 +2,7 @@ * WebShare Provider for proxy management */ import { getLogger } from '@stock-bot/logger'; -import type { HandlerConfigWithSchedule } from '@stock-bot/queue'; -import { handlerRegistry } from '@stock-bot/queue'; +import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue'; const logger = getLogger('webshare-provider'); @@ -30,7 +29,7 @@ export function initializeWebShareProvider() { name: 'webshare', operations: { - 'fetch-proxies': async _payload => { + 'fetch-proxies': createJobHandler(async () => { logger.debug('Fetching proxies from WebShare API'); try { @@ -66,7 +65,7 @@ export function initializeWebShareProvider() { error: error instanceof Error ? error.message : 'Unknown error', }; } - }, + }), }, scheduledJobs: [ diff --git a/apps/data-service/src/types/exchange.types.ts b/apps/data-service/src/types/exchange.types.ts new file mode 100644 index 0000000..13a10b3 --- /dev/null +++ b/apps/data-service/src/types/exchange.types.ts @@ -0,0 +1,40 @@ +/** + * Type definitions for exchange data structures + */ + +export interface IBExchange { + id: string; + country_code: string; + name: string; + code?: string; + exchange_code?: string; + currency?: string; + timezone?: string; + _id?: unknown; // MongoDB ObjectId + // Add other properties as needed +} + +export interface MasterExchangeData { + id: string; + code: string; + name: string; + country: string; + currency: string; + // Add other properties as needed +} + +export interface QMSymbol { + symbol: string; + exchange?: string; + name?: string; + type?: string; + // Add other properties as needed +} + +export interface IBSymbol { + symbol: string; + exchange: string; + name?: string; + currency?: string; + // Add other properties as needed +} \ No newline at end of file diff --git a/apps/data-service/src/types/job-payloads.ts b/apps/data-service/src/types/job-payloads.ts new file mode 100644 index 0000000..af2f82c --- /dev/null +++ b/apps/data-service/src/types/job-payloads.ts @@ -0,0 +1,93 @@ +/** + * Type definitions for all job payloads across data service providers + */ + +// Common result types +export interface JobResult { + success: boolean; + message: string; +} + +export interface CountableJobResult extends JobResult { + count: number; +} + +// QM Provider Types +export interface SymbolSpiderJob { + prefix: string | null; // null = root job (A-Z) + depth: number; // 1=A, 2=AA, 3=AAA, etc. + source: string; // 'qm' + maxDepth?: number; // optional max depth limit +} + +export interface CreateSessionsResult extends JobResult { + // No additional fields needed +} + +export interface SearchSymbolsResult extends CountableJobResult { + symbols?: unknown[]; // First 10 symbols as sample +} + +export interface SpiderSymbolSearchResult extends JobResult { + symbolsFound: number; + newSymbolsAdded: number; + duplicatesSkipped: number; + errors: number; + depth: number; + prefix: string | null; +} + +// IB Provider Types +export interface FetchSessionResult extends JobResult { + sessionData?: Record; +} + +export interface FetchExchangesResult extends CountableJobResult { + exchanges?: unknown[]; +} + +export interface FetchSymbolsResult extends CountableJobResult { + symbols?: unknown[]; +} + +export interface IBExchangesAndSymbolsResult extends JobResult { + exchanges: FetchExchangesResult; + symbols: FetchSymbolsResult; +} + +// Proxy Provider Types +export interface ProxyInfo { + host: string; + port: number; + protocol: 'http' | 'https' | 'socks4' | 'socks5'; + username?: string; + password?: string; + country?: string; + city?: string; + isValid?: boolean; + lastChecked?: Date; +} + +export interface FetchProxiesFromSourcesResult extends CountableJobResult { + proxies?: ProxyInfo[]; + sources: string[]; +} + +export interface CheckProxyResult extends JobResult { + proxy: ProxyInfo; + responseTime?: number; + error?: string; +} + +// WebShare Provider Types +export interface FetchWebShareProxiesResult extends CountableJobResult { + proxies?: ProxyInfo[]; + activeProxies: number; + totalQuota: number; + remainingQuota: number; +} + +// No payload job types (for operations that don't need input) +export interface NoPayload { + // Empty interface for operations that don't need payload +} \ No newline at end of file diff --git a/libs/queue/src/index.ts b/libs/queue/src/index.ts index 8786b37..fe606e8 100644 --- a/libs/queue/src/index.ts +++ b/libs/queue/src/index.ts @@ -2,6 +2,7 @@ export { Queue, type QueueWorkerConfig } from './queue'; export { QueueManager } from './queue-manager'; export { handlerRegistry } from './handler-registry'; +export { createJobHandler } from './types'; // Batch processing export { processBatchJob, processItems } from './batch-processor'; @@ -34,8 +35,11 @@ export type { // Handler types JobHandler, + TypedJobHandler, HandlerConfig, + TypedHandlerConfig, HandlerConfigWithSchedule, + TypedHandlerConfigWithSchedule, HandlerInitializer, // Configuration types diff --git a/libs/queue/src/types.ts b/libs/queue/src/types.ts index 82104c8..2309851 100644 --- a/libs/queue/src/types.ts +++ b/libs/queue/src/types.ts @@ -104,6 +104,18 @@ export interface JobHandler { (payload: TPayload): Promise; } +// Type-safe wrapper for creating job handlers +export type TypedJobHandler = (payload: TPayload) => Promise; + +// Helper to create type-safe job handlers +export function createJobHandler( + handler: TypedJobHandler +): JobHandler { + return async (payload: unknown): Promise => { + return handler(payload as TPayload); + }; +} + export interface ScheduledJob { type: string; operation: string; @@ -119,6 +131,11 @@ export interface HandlerConfig { [operation: string]: JobHandler; } +// Type-safe handler configuration +export type TypedHandlerConfig = Record> = { + [K in keyof T]: T[K]; +}; + export interface HandlerConfigWithSchedule { name: string; operations: Record; @@ -128,6 +145,16 @@ export interface HandlerConfigWithSchedule { operationLimits?: Record; } +// Type-safe version of HandlerConfigWithSchedule +export interface TypedHandlerConfigWithSchedule = Record> { + name: string; + operations: T; + scheduledJobs?: ScheduledJob[]; + // Rate limiting + rateLimit?: RateLimitConfig; + operationLimits?: Record; +} + export interface BatchJobData { payloadKey: string; batchIndex: number;