import { Browser } from '@stock-bot/browser'; import { getLogger } from '@stock-bot/logger'; import { getMongoDBClient } from '@stock-bot/mongodb-client'; // Shared instances (module-scoped, not global) let isInitialized = false; // Track if resources are initialized let logger: ReturnType; // let cache: CacheProvider; export async function initializeIBResources(): Promise { // Skip if already initialized if (isInitialized) { return; } logger = getLogger('proxy-tasks'); // cache = createCache({ // keyPrefix: 'proxy:', // ttl: PROXY_CONFIG.CACHE_TTL, // enableMetrics: true, // }); // httpClient = new HttpClient({ timeout: 15000 }, logger); // if (waitForCache) { // // logger.info('Initializing proxy cache...'); // // await cache.waitForReady(10000); // // logger.info('Proxy cache initialized successfully'); // logger.info('Proxy tasks initialized'); // } else { // logger.info('Proxy tasks initialized (fallback mode)'); // } isInitialized = true; } export async function fetchSession(): Promise | undefined> { try { await Browser.initialize({ headless: true, timeout: 10000, blockResources: false }); logger.info('✅ Browser initialized'); const { page } = await Browser.createPageWithProxy( 'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/', 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' ); logger.info('✅ Page created with proxy'); const headersPromise = new Promise | undefined>(resolve => { let resolved = false; page.onNetworkEvent(event => { if (event.url.includes('/webrest/search/product-types/summary')) { if (event.type === 'request') { try { resolve(event.headers); } catch (e) { resolve(undefined); console.log('📊 Raw Summary Response:', (e as Error).message); } } } }); // Timeout fallback setTimeout(() => { if (!resolved) { resolved = true; logger.warn('Timeout waiting for headers'); resolve(undefined); } }, 30000); }); logger.info('⏳ Waiting for page load...'); await page.waitForLoadState('domcontentloaded', { timeout: 20000 }); logger.info('✅ Page loaded'); //Products tabs logger.info('🔍 Looking for Products tab...'); const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); await productsTab.waitFor({ timeout: 5000 }); logger.info('✅ Found Products tab'); logger.info('🖱️ Clicking Products tab...'); await productsTab.click(); logger.info('✅ Products tab clicked'); // New Products Checkbox logger.info('🔍 Looking for "New Products Only" radio button...'); const radioButton = page.locator('span.checkbox-text:has-text("New Products Only")'); await radioButton.waitFor({ timeout: 5000 }); logger.info(`🎯 Found "New Products Only" radio button`); await radioButton.first().click(); logger.info('✅ "New Products Only" radio button clicked'); // Wait for and return headers immediately when captured logger.info('⏳ Waiting for headers to be captured...'); const headers = await headersPromise; page.close(); if (headers) { logger.info('✅ Headers captured successfully'); } else { logger.warn('⚠️ No headers were captured'); } return headers; } catch (error) { logger.error('Failed to fetch IB symbol summary', { error }); return; } } export async function fetchExchanges(sessionHeaders: Record): Promise { try { logger.info('🔍 Fetching exchanges with session headers...'); // The URL for the exchange data API const exchangeUrl = 'https://www.interactivebrokers.com/webrest/exchanges'; // Configure the proxy const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'; // Prepare headers - include all session headers plus any additional ones const requestHeaders = { ...sessionHeaders, Accept: 'application/json, text/plain, */*', 'Accept-Language': 'en-US,en;q=0.9', 'Cache-Control': 'no-cache', Pragma: 'no-cache', 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', 'X-Requested-With': 'XMLHttpRequest', }; logger.info('📤 Making request to exchange API...', { url: exchangeUrl, headerCount: Object.keys(requestHeaders).length, }); // Use fetch with proxy configuration const response = await fetch(exchangeUrl, { method: 'GET', headers: requestHeaders, proxy: proxyUrl, }); if (!response.ok) { logger.error('❌ Exchange API request failed', { status: response.status, statusText: response.statusText, }); return null; } const data = await response.json(); const exchanges = data?.exchanges || []; logger.info('✅ Exchange data fetched successfully'); logger.info('Saving IB exchanges to MongoDB...'); const client = getMongoDBClient(); await client.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']); logger.info('✅ Exchange IB data saved to MongoDB:', { count: exchanges.length, }); return exchanges; } catch (error) { logger.error('❌ Failed to fetch exchanges', { error }); return null; } } // Fetch symbols from IB using the session headers export async function fetchSymbols(sessionHeaders: Record): Promise { try { logger.info('🔍 Fetching symbols with session headers...'); // Configure the proxy const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'; // Prepare headers - include all session headers plus any additional ones const requestHeaders = { ...sessionHeaders, Accept: 'application/json, text/plain, */*', 'Accept-Language': 'en-US,en;q=0.9', 'Cache-Control': 'no-cache', Pragma: 'no-cache', 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', 'X-Requested-With': 'XMLHttpRequest', }; const requestBody = { domain: 'com', newProduct: 'all', pageNumber: 1, pageSize: 100, productCountry: ['CA', 'US'], productSymbol: '', productType: ['STK'], sortDirection: 'asc', sortField: 'symbol', }; // Get Summary const summaryResponse = await fetch( 'https://www.interactivebrokers.com/webrest/search/product-types/summary', { method: 'POST', headers: requestHeaders, proxy: proxyUrl, body: JSON.stringify(requestBody), } ); if (!summaryResponse.ok) { logger.error('❌ Summary API request failed', { status: summaryResponse.status, statusText: summaryResponse.statusText, }); return null; } const summaryData = await summaryResponse.json(); logger.info('✅ IB Summary data fetched successfully', { totalCount: summaryData[0].totalCount, }); const symbols = []; requestBody.pageSize = 500; const pageCount = Math.ceil(summaryData[0].totalCount / 500) || 0; logger.info('Fetching Symbols for IB', { pageCount }); const symbolPromises = []; for (let page = 1; page <= pageCount; page++) { requestBody.pageNumber = page; // Fetch symbols for the current page const symbolsResponse = fetch( 'https://www.interactivebrokers.com/webrest/search/products-by-filters', { method: 'POST', headers: requestHeaders, proxy: proxyUrl, body: JSON.stringify(requestBody), } ); symbolPromises.push(symbolsResponse); } const responses = await Promise.all(symbolPromises); for (const response of responses) { if (!response.ok) { logger.error('❌ Symbols API request failed', { status: response.status, statusText: response.statusText, }); return null; } const data = await response.json(); const symJson = data?.products || []; if (symJson && symJson.length > 0) { symbols.push(...symJson); } else { logger.warn('⚠️ No symbols found in response'); continue; } } if (symbols.length === 0) { logger.warn('⚠️ No symbols fetched from IB'); return null; } logger.info('✅ IB symbols fetched successfully, saving to DB...', { totalSymbols: symbols.length, }); const client = getMongoDBClient(); await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']); logger.info('Saved IB symbols to DB', { totalSymbols: symbols.length, }); // logger.info('📤 Making request to exchange API...', { // url: exchangeUrl, // headerCount: Object.keys(requestHeaders).length, // }); // // Use fetch with proxy configuration // const response = await fetch(exchangeUrl, { // method: 'GET', // headers: requestHeaders, // proxy: proxyUrl, // }); // if (!response.ok) { // logger.error('❌ Exchange API request failed', { // status: response.status, // statusText: response.statusText, // }); // return null; // } } catch (error) { logger.error('❌ Failed to fetch symbols', { error }); return null; } } export const ibTasks = { fetchSymbols, fetchSession, fetchExchanges, };