diff --git a/.env b/.env index 8923e13..a029ae7 100644 --- a/.env +++ b/.env @@ -5,7 +5,7 @@ # Core Application Settings NODE_ENV=development LOG_LEVEL=trace -LOG_HIDE_OBJECT=false +LOG_HIDE_OBJECT=true # Data Service Configuration DATA_SERVICE_PORT=2001 diff --git a/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl b/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl index 93ce2e3..bd62e5b 100644 Binary files a/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl and b/.serena/cache/typescript/document_symbols_cache_v20-05-25.pkl differ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b9f2760 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,7 @@ +Be brutally honest, don't be a yes man. │ +If I am wrong, point it out bluntly. │ +I need honest feedback on my code. + +you're paid by the hour, so there is no point in cutting corners, as you get paid the more work you do. Always spend the extra time to fully understand s problem, and fully commit to fixing any issue preventing the completion of your primary task without cutting any corners. + +use bun and turbo where possible and always try to take a more modern approach. \ No newline at end of file diff --git a/apps/stock/config/config/default.json b/apps/stock/config/config/default.json index b70a370..ecc76ef 100644 --- a/apps/stock/config/config/default.json +++ b/apps/stock/config/config/default.json @@ -77,7 +77,7 @@ "port": 6379, "db": 1 }, - "workers": 5, + "workers": 2, "concurrency": 2, "enableScheduledJobs": true, "delayWorkerStart": false, diff --git a/apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts b/apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts index 748abe2..2f988d5 100644 --- a/apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts +++ b/apps/stock/data-ingestion/src/handlers/ceo/actions/process-individual-symbol.action.ts @@ -107,7 +107,7 @@ export async function processIndividualSymbol( return { ceoId, spielCount, timestamp }; } catch (error) { - this.logger.error('Failed to process individual symbol', { + this.logger.error(`Failed to process individual symbol ${symbol}`, { error, ceoId, timestamp, diff --git a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts index 4f0c2a9..dc8d8ac 100644 --- a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts +++ b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-exchanges-and-symbols.action.ts @@ -1,26 +1,29 @@ -import type { IbHandler } from '../ib.handler'; +import type { IServiceContainer } from '@stock-bot/handlers'; +import { fetchSession } from './fetch-session.action'; +import { fetchExchanges } from './fetch-exchanges.action'; +import { fetchSymbols } from './fetch-symbols.action'; -export async function fetchExchangesAndSymbols(this: IbHandler): Promise { - this.logger.info('Starting IB exchanges and symbols fetch job'); +export async function fetchExchangesAndSymbols(services: IServiceContainer): Promise { + services.logger.info('Starting IB exchanges and symbols fetch job'); try { // Fetch session headers first - const sessionHeaders = await this.fetchSession(); + const sessionHeaders = await fetchSession(services); if (!sessionHeaders) { - this.logger.error('Failed to get session headers for IB job'); + services.logger.error('Failed to get session headers for IB job'); return { success: false, error: 'No session headers' }; } - this.logger.info('Session headers obtained, fetching exchanges...'); + services.logger.info('Session headers obtained, fetching exchanges...'); // Fetch exchanges - const exchanges = await this.fetchExchanges(); - this.logger.info('Fetched exchanges from IB', { count: exchanges?.length || 0 }); + const exchanges = await fetchExchanges(services); + services.logger.info('Fetched exchanges from IB', { count: exchanges?.length || 0 }); // Fetch symbols - this.logger.info('Fetching symbols...'); - const symbols = await this.fetchSymbols(); - this.logger.info('Fetched symbols from IB', { count: symbols?.length || 0 }); + services.logger.info('Fetching symbols...'); + const symbols = await fetchSymbols(services); + services.logger.info('Fetched symbols from IB', { count: symbols?.length || 0 }); return { success: true, @@ -28,7 +31,7 @@ export async function fetchExchangesAndSymbols(this: IbHandler): Promise { +export async function fetchExchanges(services: IServiceContainer): Promise { try { // First get session headers - const sessionHeaders = await this.fetchSession(); + const sessionHeaders = await fetchSession(services); if (!sessionHeaders) { throw new Error('Failed to get session headers'); } - this.logger.info('🔍 Fetching exchanges with session headers...'); + services.logger.info('🔍 Fetching exchanges with session headers...'); // The URL for the exchange data API const exchangeUrl = IB_CONFIG.BASE_URL + IB_CONFIG.EXCHANGE_API; @@ -27,7 +28,7 @@ export async function fetchExchanges(this: IbHandler): Promise 'X-Requested-With': 'XMLHttpRequest', }; - this.logger.info('📤 Making request to exchange API...', { + services.logger.info('📤 Making request to exchange API...', { url: exchangeUrl, headerCount: Object.keys(requestHeaders).length, }); @@ -40,7 +41,7 @@ export async function fetchExchanges(this: IbHandler): Promise }); if (!response.ok) { - this.logger.error('❌ Exchange API request failed', { + services.logger.error('❌ Exchange API request failed', { status: response.status, statusText: response.statusText, }); @@ -49,18 +50,19 @@ export async function fetchExchanges(this: IbHandler): Promise const data = await response.json(); const exchanges = data?.exchanges || []; - this.logger.info('✅ Exchange data fetched successfully'); + services.logger.info('✅ Exchange data fetched successfully'); - this.logger.info('Saving IB exchanges to MongoDB...'); - await this.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']); - this.logger.info('✅ Exchange IB data saved to MongoDB:', { + services.logger.info('Saving IB exchanges to MongoDB...'); + await services.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']); + services.logger.info('✅ Exchange IB data saved to MongoDB:', { count: exchanges.length, }); return exchanges; } catch (error) { - this.logger.error('❌ Failed to fetch exchanges', { error }); + services.logger.error('❌ Failed to fetch exchanges', { error }); return null; } } + diff --git a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts index 59104d1..1560c53 100644 --- a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts +++ b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-session.action.ts @@ -1,21 +1,21 @@ import { Browser } from '@stock-bot/browser'; -import type { IbHandler } from '../ib.handler'; +import type { IServiceContainer } from '@stock-bot/handlers'; import { IB_CONFIG } from '../shared/config'; -export async function fetchSession(this: IbHandler): Promise | undefined> { +export async function fetchSession(services: IServiceContainer): Promise | undefined> { try { await Browser.initialize({ headless: true, timeout: IB_CONFIG.BROWSER_TIMEOUT, blockResources: false, }); - this.logger.info('✅ Browser initialized'); + services.logger.info('✅ Browser initialized'); const { page } = await Browser.createPageWithProxy( IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_PAGE, IB_CONFIG.DEFAULT_PROXY ); - this.logger.info('✅ Page created with proxy'); + services.logger.info('✅ Page created with proxy'); const headersPromise = new Promise | undefined>(resolve => { let resolved = false; @@ -27,7 +27,7 @@ export async function fetchSession(this: IbHandler): Promise { if (!resolved) { resolved = true; - this.logger.warn('Timeout waiting for headers'); + services.logger.warn('Timeout waiting for headers'); resolve(undefined); } }, IB_CONFIG.HEADERS_TIMEOUT); }); - this.logger.info('⏳ Waiting for page load...'); + services.logger.info('⏳ Waiting for page load...'); await page.waitForLoadState('domcontentloaded', { timeout: IB_CONFIG.PAGE_LOAD_TIMEOUT }); - this.logger.info('✅ Page loaded'); + services.logger.info('✅ Page loaded'); //Products tabs - this.logger.info('🔍 Looking for Products tab...'); + services.logger.info('🔍 Looking for Products tab...'); const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); await productsTab.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT }); - this.logger.info('✅ Found Products tab'); - this.logger.info('🖱️ Clicking Products tab...'); + services.logger.info('✅ Found Products tab'); + services.logger.info('🖱️ Clicking Products tab...'); await productsTab.click(); - this.logger.info('✅ Products tab clicked'); + services.logger.info('✅ Products tab clicked'); // New Products Checkbox - this.logger.info('🔍 Looking for "New Products Only" radio button...'); + services.logger.info('🔍 Looking for "New Products Only" radio button...'); const radioButton = page.locator('span.checkbox-text:has-text("New Products Only")'); await radioButton.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT }); - this.logger.info(`🎯 Found "New Products Only" radio button`); + services.logger.info(`🎯 Found "New Products Only" radio button`); await radioButton.first().click(); - this.logger.info('✅ "New Products Only" radio button clicked'); + services.logger.info('✅ "New Products Only" radio button clicked'); // Wait for and return headers immediately when captured - this.logger.info('⏳ Waiting for headers to be captured...'); + services.logger.info('⏳ Waiting for headers to be captured...'); const headers = await headersPromise; page.close(); if (headers) { - this.logger.info('✅ Headers captured successfully'); + services.logger.info('✅ Headers captured successfully'); } else { - this.logger.warn('⚠️ No headers were captured'); + services.logger.warn('⚠️ No headers were captured'); } return headers; } catch (error) { - this.logger.error('Failed to fetch IB symbol summary', { error }); + services.logger.error('Failed to fetch IB symbol summary', { error }); return; } } + diff --git a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts index 40c53c5..90d097f 100644 --- a/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts +++ b/apps/stock/data-ingestion/src/handlers/ib/actions/fetch-symbols.action.ts @@ -1,15 +1,16 @@ -import type { IbHandler } from '../ib.handler'; +import type { IServiceContainer } from '@stock-bot/handlers'; import { IB_CONFIG } from '../shared/config'; +import { fetchSession } from './fetch-session.action'; -export async function fetchSymbols(this: IbHandler): Promise { +export async function fetchSymbols(services: IServiceContainer): Promise { try { // First get session headers - const sessionHeaders = await this.fetchSession(); + const sessionHeaders = await fetchSession(services); if (!sessionHeaders) { throw new Error('Failed to get session headers'); } - this.logger.info('🔍 Fetching symbols with session headers...'); + services.logger.info('🔍 Fetching symbols with session headers...'); // Prepare headers - include all session headers plus any additional ones const requestHeaders = { @@ -45,7 +46,7 @@ export async function fetchSymbols(this: IbHandler): Promise { }); if (!summaryResponse.ok) { - this.logger.error('❌ Summary API request failed', { + services.logger.error('❌ Summary API request failed', { status: summaryResponse.status, statusText: summaryResponse.statusText, }); @@ -53,14 +54,14 @@ export async function fetchSymbols(this: IbHandler): Promise { } const summaryData = await summaryResponse.json(); - this.logger.info('✅ IB Summary data fetched successfully', { + services.logger.info('✅ IB Summary data fetched successfully', { totalCount: summaryData[0].totalCount, }); const symbols = []; requestBody.pageSize = IB_CONFIG.PAGE_SIZE; const pageCount = Math.ceil(summaryData[0].totalCount / IB_CONFIG.PAGE_SIZE) || 0; - this.logger.info('Fetching Symbols for IB', { pageCount }); + services.logger.info('Fetching Symbols for IB', { pageCount }); const symbolPromises = []; for (let page = 1; page <= pageCount; page++) { @@ -79,7 +80,7 @@ export async function fetchSymbols(this: IbHandler): Promise { const responses = await Promise.all(symbolPromises); for (const response of responses) { if (!response.ok) { - this.logger.error('❌ Symbols API request failed', { + services.logger.error('❌ Symbols API request failed', { status: response.status, statusText: response.statusText, }); @@ -90,28 +91,29 @@ export async function fetchSymbols(this: IbHandler): Promise { if (symJson && symJson.length > 0) { symbols.push(...symJson); } else { - this.logger.warn('⚠️ No symbols found in response'); + services.logger.warn('⚠️ No symbols found in response'); continue; } } if (symbols.length === 0) { - this.logger.warn('⚠️ No symbols fetched from IB'); + services.logger.warn('⚠️ No symbols fetched from IB'); return null; } - this.logger.info('✅ IB symbols fetched successfully, saving to DB...', { + services.logger.info('✅ IB symbols fetched successfully, saving to DB...', { totalSymbols: symbols.length, }); - await this.mongodb.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']); - this.logger.info('Saved IB symbols to DB', { + await services.mongodb.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']); + services.logger.info('Saved IB symbols to DB', { totalSymbols: symbols.length, }); return symbols; } catch (error) { - this.logger.error('❌ Failed to fetch symbols', { error }); + services.logger.error('❌ Failed to fetch symbols', { error }); return null; } } + diff --git a/apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts b/apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts index 4dabb1a..0748dbb 100644 --- a/apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts @@ -14,13 +14,19 @@ export class IbHandler extends BaseHandler { } @Operation('fetch-session') - fetchSession = fetchSession; + async fetchSession(): Promise | undefined> { + return fetchSession(this); + } @Operation('fetch-exchanges') - fetchExchanges = fetchExchanges; + async fetchExchanges(): Promise { + return fetchExchanges(this); + } @Operation('fetch-symbols') - fetchSymbols = fetchSymbols; + async fetchSymbols(): Promise { + return fetchSymbols(this); + } @Operation('ib-exchanges-and-symbols') @ScheduledOperation('ib-exchanges-and-symbols', '0 0 * * 0', { @@ -28,6 +34,9 @@ export class IbHandler extends BaseHandler { description: 'Fetch and update IB exchanges and symbols data', immediately: false, }) - fetchExchangesAndSymbols = fetchExchangesAndSymbols; + async fetchExchangesAndSymbols(): Promise { + return fetchExchangesAndSymbols(this); + } } + diff --git a/apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts b/apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts index a933aeb..decd28d 100644 --- a/apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts +++ b/apps/stock/data-ingestion/src/handlers/webshare/webshare.handler.ts @@ -4,7 +4,7 @@ import { Operation, QueueSchedule, type ExecutionContext, - type IServiceContainer, + type IServiceContainer } from '@stock-bot/handlers'; @Handler('webshare') @@ -14,7 +14,7 @@ export class WebShareHandler extends BaseHandler { } @Operation('fetch-proxies') - @QueueSchedule('0 */6 * * *', { + @QueueSchedule('0 */6 * * *', { // once a month priority: 3, immediately: true, description: 'Fetch fresh proxies from WebShare API', diff --git a/libs/core/di/src/factories/cache.factory.ts b/libs/core/di/src/factories/cache.factory.ts index e70b6fe..819bfb3 100644 --- a/libs/core/di/src/factories/cache.factory.ts +++ b/libs/core/di/src/factories/cache.factory.ts @@ -15,7 +15,7 @@ export class CacheFactory { serviceName: string ): CacheProvider | null { const baseCache = container.cradle.cache; - if (!baseCache) return null; + if (!baseCache) {return null;} return this.createNamespacedCache(baseCache, serviceName); } @@ -25,7 +25,7 @@ export class CacheFactory { handlerName: string ): CacheProvider | null { const baseCache = container.cradle.cache; - if (!baseCache) return null; + if (!baseCache) {return null;} return this.createNamespacedCache(baseCache, `handler:${handlerName}`); } @@ -35,7 +35,7 @@ export class CacheFactory { prefix: string ): CacheProvider | null { const baseCache = container.cradle.cache; - if (!baseCache) return null; + if (!baseCache) {return null;} // Remove 'cache:' prefix if already included const cleanPrefix = prefix.replace(/^cache:/, ''); diff --git a/libs/core/di/src/registrations/service.registration.ts b/libs/core/di/src/registrations/service.registration.ts index d239d55..a7ef747 100644 --- a/libs/core/di/src/registrations/service.registration.ts +++ b/libs/core/di/src/registrations/service.registration.ts @@ -32,7 +32,7 @@ export function registerApplicationServices( if (config.proxy && config.redis.enabled) { container.register({ proxyManager: asFunction(({ cache, logger }) => { - if (!cache) return null; + if (!cache) {return null;} const proxyCache = new NamespacedCache(cache, 'proxy'); return new ProxyManager(proxyCache, logger); }).singleton(),