diff --git a/.env b/.env index 5674a15..d9f981e 100644 --- a/.env +++ b/.env @@ -13,6 +13,8 @@ DATA_SERVICE_PORT=2001 WORKER_COUNT=4 WORKER_CONCURRENCY=20 +WEBSHARE_API_KEY=y8ay534rcbybdkk3evnzmt640xxfhy7252ce2t98 + # =========================================== # DATABASE CONFIGURATIONS # =========================================== diff --git a/apps/data-service/src/index.ts b/apps/data-service/src/index.ts index baa1037..b476e76 100644 --- a/apps/data-service/src/index.ts +++ b/apps/data-service/src/index.ts @@ -2,10 +2,12 @@ * Data Service - Combined live and historical data ingestion with queue-based architecture */ import { Hono } from 'hono'; +import { Browser } from '@stock-bot/browser'; import { loadEnvVariables } from '@stock-bot/config'; import { getLogger } from '@stock-bot/logger'; import { Shutdown } from '@stock-bot/shutdown'; -import { initializeProxyCache } from './providers/proxy.tasks'; +import { initializeIBResources } from './providers/ib.tasks'; +import { initializeProxyResources } from './providers/proxy.tasks'; import { queueManager } from './services/queue.service'; import { initializeBatchCache } from './utils/batch-helpers'; import { healthRoutes, marketDataRoutes, proxyRoutes, queueRoutes, testRoutes } from './routes'; @@ -33,6 +35,11 @@ async function initializeServices() { logger.info('Initializing data service...'); try { + // Initialize browser resources + logger.info('Starting browser resources initialization...'); + await Browser.initialize(); + logger.info('Browser resources initialized'); + // Initialize batch cache FIRST - before queue service logger.info('Starting batch cache initialization...'); await initializeBatchCache(); @@ -40,7 +47,12 @@ async function initializeServices() { // Initialize proxy cache - before queue service logger.info('Starting proxy cache initialization...'); - await initializeProxyCache(); + await initializeProxyResources(true); // Wait for cache during startup + logger.info('Proxy cache initialized'); + + // Initialize proxy cache - before queue service + logger.info('Starting proxy cache initialization...'); + await initializeIBResources(true); // Wait for cache during startup logger.info('Proxy cache initialized'); // Initialize queue service (Redis connections should be ready now) diff --git a/apps/data-service/src/providers/ib.provider.ts b/apps/data-service/src/providers/ib.provider.ts new file mode 100644 index 0000000..1ed4aaf --- /dev/null +++ b/apps/data-service/src/providers/ib.provider.ts @@ -0,0 +1,32 @@ +import { getLogger } from '@stock-bot/logger'; +import { ProviderConfig } from '../services/provider-registry.service'; + +const logger = getLogger('ib-provider'); + +export const ibProvider: ProviderConfig = { + name: 'ib', + operations: { + 'ib-symbol-summary': async () => { + const { ibTasks } = await import('./ib.tasks'); + logger.info('Fetching symbol summary from IB'); + const total = await ibTasks.fetchSymbolSummary(); + logger.info('Fetched symbol summary from IB', { + count: total, + }); + return total; + }, + }, + + scheduledJobs: [ + { + type: 'ib-symbol-summary', + operation: 'ib-symbol-summary', + payload: {}, + // should remove and just run at the same time so app restarts dont keeping adding same jobs + cronPattern: '*/2 * * * *', + priority: 5, + immediately: true, // Don't run immediately during startup to avoid conflicts + description: 'Fetch and validate proxy list from sources', + }, + ], +}; diff --git a/apps/data-service/src/providers/ib.tasks.ts b/apps/data-service/src/providers/ib.tasks.ts new file mode 100644 index 0000000..d495455 --- /dev/null +++ b/apps/data-service/src/providers/ib.tasks.ts @@ -0,0 +1,152 @@ +import { Browser } from '@stock-bot/browser'; +import { getLogger } from '@stock-bot/logger'; + +// Shared instances (module-scoped, not global) +let isInitialized = false; // Track if resources are initialized +let logger: ReturnType; +// let cache: CacheProvider; + +export async function initializeIBResources(waitForCache = false): Promise { + // Skip if already initialized + if (isInitialized) { + return; + } + + logger = getLogger('proxy-tasks'); + // cache = createCache({ + // keyPrefix: 'proxy:', + // ttl: PROXY_CONFIG.CACHE_TTL, + // enableMetrics: true, + // }); + + // httpClient = new HttpClient({ timeout: 15000 }, logger); + + // if (waitForCache) { + // // logger.info('Initializing proxy cache...'); + // // await cache.waitForReady(10000); + // // logger.info('Proxy cache initialized successfully'); + // logger.info('Proxy tasks initialized'); + // } else { + // logger.info('Proxy tasks initialized (fallback mode)'); + // } + isInitialized = true; +} + +export async function fetchSymbolSummary(): Promise { + try { + await Browser.initialize({ headless: true, timeout: 10000, blockResources: false }); + logger.info('โœ… Browser initialized'); + + const { page, contextId } = await Browser.createPageWithProxy( + 'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + logger.info('โœ… Page created with proxy'); + let summaryData: any = null; // Initialize summaryData to store API response + let eventCount = 0; + page.onNetworkEvent(event => { + if (event.url.includes('/webrest/search/product-types/summary')) { + console.log(`๐ŸŽฏ Found summary API call: ${event.type} ${event.url}`); + + if (event.type === 'response' && event.responseData) { + console.log(`๐Ÿ“Š Summary API Response Data: ${event.responseData}`); + try { + summaryData = JSON.parse(event.responseData) as any; + const totalCount = summaryData[0].totalCount; + console.log('๐Ÿ“Š Summary API Response:', JSON.stringify(summaryData, null, 2)); + console.log(`๐Ÿ”ข Total symbols found: ${totalCount || 'Unknown'}`); + } catch (e) { + console.log('๐Ÿ“Š Raw Summary Response:', event.responseData); + } + } + } + eventCount++; + logger.info(`๐Ÿ“ก Event ${eventCount}: ${event.type} ${event.url}`); + }); + + logger.info('โณ Waiting for page load...'); + await page.waitForLoadState('domcontentloaded', { timeout: 20000 }); + logger.info('โœ… Page loaded'); + + // RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button + logger.info('๐Ÿ” Looking for Products tab...'); + + // Wait for the page to fully load + await page.waitForTimeout(20000); + + // First, click on the Products tab + const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); + await productsTab.waitFor({ timeout: 20000 }); + logger.info('โœ… Found Products tab'); + + logger.info('๐Ÿ–ฑ๏ธ Clicking Products tab...'); + await productsTab.click(); + logger.info('โœ… Products tab clicked'); + + // Wait for the tab content to load + await page.waitForTimeout(5000); + + // Click on the Asset Classes accordion to expand it + logger.info('๐Ÿ” Looking for Asset Classes accordion...'); + const assetClassesAccordion = page.locator( + '#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")' + ); + await assetClassesAccordion.waitFor({ timeout: 10000 }); + logger.info('โœ… Found Asset Classes accordion'); + + logger.info('๐Ÿ–ฑ๏ธ Clicking Asset Classes accordion...'); + await assetClassesAccordion.click(); + logger.info('โœ… Asset Classes accordion clicked'); + + // Wait for the accordion content to expand + await page.waitForTimeout(2000); + + logger.info('๐Ÿ” Looking for Stocks checkbox...'); + + // Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks" + const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")'); + await stocksSpan.waitFor({ timeout: 10000 }); + logger.info('โœ… Found Stocks span'); + + // Find the checkbox by looking in the same parent container + const parentContainer = stocksSpan.locator('..'); + const checkbox = parentContainer.locator('input[type="checkbox"]'); + + if ((await checkbox.count()) > 0) { + logger.info('๐Ÿ“‹ Clicking Stocks checkbox...'); + await checkbox.first().check(); + logger.info('โœ… Stocks checkbox checked'); + } else { + logger.info('โš ๏ธ Could not find checkbox near Stocks text'); + } + + // Wait a moment for any UI updates + await page.waitForTimeout(1000); + + // Find and click the nearest Apply button + logger.info('๐Ÿ” Looking for Apply button...'); + const applyButton = page.locator( + 'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]' + ); + + if ((await applyButton.count()) > 0) { + logger.info('๐ŸŽฏ Clicking Apply button...'); + await applyButton.first().click(); + logger.info('โœ… Apply button clicked'); + + // Wait for any network requests triggered by the Apply button + await page.waitForTimeout(2000); + } else { + logger.info('โš ๏ธ Could not find Apply button'); + } + + return 0; + } catch (error) { + logger.error('Failed to fetch IB symbol summary', { error }); + return 0; + } +} +// Optional: Export a convenience object that groups related tasks +export const ibTasks = { + fetchSymbolSummary, +}; diff --git a/apps/data-service/src/providers/proxy.tasks.ts b/apps/data-service/src/providers/proxy.tasks.ts index 9e45804..a1294a5 100644 --- a/apps/data-service/src/providers/proxy.tasks.ts +++ b/apps/data-service/src/providers/proxy.tasks.ts @@ -1,4 +1,3 @@ -import pLimit from 'p-limit'; import { createCache, type CacheProvider } from '@stock-bot/cache'; import { HttpClient, ProxyInfo } from '@stock-bot/http'; import { getLogger } from '@stock-bot/logger'; @@ -22,7 +21,6 @@ const PROXY_CONFIG = { CHECK_TIMEOUT: 7000, CHECK_IP: '99.246.102.205', CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955', - CONCURRENCY_LIMIT: 100, PROXY_SOURCES: [ { id: 'prxchk', @@ -154,10 +152,10 @@ const PROXY_CONFIG = { }; // Shared instances (module-scoped, not global) +let isInitialized = false; // Track if resources are initialized let logger: ReturnType; let cache: CacheProvider; let httpClient: HttpClient; -let concurrencyLimit: ReturnType; let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({ id: source.id, total: 0, @@ -167,6 +165,37 @@ let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({ url: source.url, })); +/** + * Initialize proxy resources (cache and shared dependencies) + * This should be called before any proxy operations + * @param waitForCache - Whether to wait for cache readiness (default: false for fallback mode) + */ +export async function initializeProxyResources(waitForCache = false): Promise { + // Skip if already initialized + if (isInitialized) { + return; + } + + logger = getLogger('proxy-tasks'); + cache = createCache({ + keyPrefix: 'proxy:', + ttl: PROXY_CONFIG.CACHE_TTL, + enableMetrics: true, + }); + + httpClient = new HttpClient({ timeout: 10000 }, logger); + + if (waitForCache) { + logger.info('Initializing proxy cache...'); + await cache.waitForReady(10000); + logger.info('Proxy cache initialized successfully'); + logger.info('Proxy tasks initialized'); + } else { + logger.info('Proxy tasks initialized (fallback mode)'); + } + isInitialized = true; +} + // make a function that takes in source id and a boolean success and updates the proxyStats array async function updateProxyStats(sourceId: string, success: boolean) { const source = proxyStats.find(s => s.id === sourceId); @@ -278,50 +307,8 @@ async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean): Promise } } -/** - * Initialize proxy cache for use during application startup - * This should be called before any proxy operations - */ -export async function initializeProxyCache(): Promise { - logger = getLogger('proxy-tasks'); - cache = createCache({ - keyPrefix: 'proxy:', - ttl: PROXY_CONFIG.CACHE_TTL, - enableMetrics: true, - }); - - logger.info('Initializing proxy cache...'); - await cache.waitForReady(10000); - logger.info('Proxy cache initialized successfully'); - - // Initialize other shared resources that don't require cache - httpClient = new HttpClient({ timeout: 10000 }, logger); - concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT); - - logger.info('Proxy tasks initialized'); -} - -async function initializeSharedResources() { - if (!logger) { - // If not initialized at startup, initialize with fallback mode - logger = getLogger('proxy-tasks'); - cache = createCache({ - keyPrefix: 'proxy:', - ttl: PROXY_CONFIG.CACHE_TTL, - enableMetrics: true, - }); - - httpClient = new HttpClient({ timeout: 10000 }, logger); - concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT); - - logger.info('Proxy tasks initialized (fallback mode)'); - } -} - // Individual task functions export async function queueProxyFetch(): Promise { - await initializeSharedResources(); - const { queueManager } = await import('../services/queue.service'); const job = await queueManager.addJob({ type: 'proxy-fetch', @@ -337,8 +324,6 @@ export async function queueProxyFetch(): Promise { } export async function queueProxyCheck(proxies: ProxyInfo[]): Promise { - await initializeSharedResources(); - const { queueManager } = await import('../services/queue.service'); const job = await queueManager.addJob({ type: 'proxy-check', @@ -354,35 +339,15 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise { } export async function fetchProxiesFromSources(): Promise { - await initializeSharedResources(); await resetProxyStats(); - - // Ensure concurrencyLimit is available before using it - if (!concurrencyLimit) { - logger.error('concurrencyLimit not initialized, using sequential processing'); - const result = []; - for (const source of PROXY_CONFIG.PROXY_SOURCES) { - const proxies = await fetchProxiesFromSource(source); - result.push(...proxies); - } - let allProxies: ProxyInfo[] = result; - allProxies = removeDuplicateProxies(allProxies); - return allProxies; - } - - const sources = PROXY_CONFIG.PROXY_SOURCES.map(source => - concurrencyLimit(() => fetchProxiesFromSource(source)) - ); - const result = await Promise.all(sources); - let allProxies: ProxyInfo[] = result.flat(); + const fetchPromises = PROXY_CONFIG.PROXY_SOURCES.map(source => fetchProxiesFromSource(source)); + const results = await Promise.all(fetchPromises); + let allProxies: ProxyInfo[] = results.flat(); allProxies = removeDuplicateProxies(allProxies); - // await checkProxies(allProxies); return allProxies; } export async function fetchProxiesFromSource(source: ProxySource): Promise { - await initializeSharedResources(); - const allProxies: ProxyInfo[] = []; try { @@ -436,8 +401,6 @@ export async function fetchProxiesFromSource(source: ProxySource): Promise { - await initializeSharedResources(); - let success = false; logger.debug(`Checking Proxy:`, { protocol: proxy.protocol, @@ -504,6 +467,76 @@ export async function checkProxy(proxy: ProxyInfo): Promise { } } +/** + * Get a random active proxy from the cache + * @param protocol - Optional protocol filter ('http' | 'https' | 'socks4' | 'socks5') + * @param minSuccessRate - Minimum success rate percentage (default: 50) + * @returns A random working proxy or null if none found + */ +export async function getRandomActiveProxy( + protocol?: 'http' | 'https' | 'socks4' | 'socks5', + minSuccessRate: number = 50 +): Promise { + try { + // Get all active proxy keys from cache + const pattern = protocol + ? `${PROXY_CONFIG.CACHE_KEY}:${protocol}://*` + : `${PROXY_CONFIG.CACHE_KEY}:*`; + + const keys = await cache.keys(pattern); + + if (keys.length === 0) { + logger.debug('No active proxies found in cache', { pattern }); + return null; + } + + // Shuffle the keys for randomness + const shuffledKeys = keys.sort(() => Math.random() - 0.5); + + // Find a working proxy that meets the criteria + for (const key of shuffledKeys) { + try { + const proxyData: ProxyInfo | null = await cache.get(key); + + if ( + proxyData && + proxyData.isWorking && + (!proxyData.successRate || proxyData.successRate >= minSuccessRate) + ) { + logger.debug('Random active proxy selected', { + proxy: `${proxyData.host}:${proxyData.port}`, + protocol: proxyData.protocol, + successRate: proxyData.successRate?.toFixed(1) + '%', + avgResponseTime: proxyData.averageResponseTime + ? `${proxyData.averageResponseTime.toFixed(0)}ms` + : 'N/A', + }); + + return proxyData; + } + } catch (error) { + logger.debug('Error reading proxy from cache', { key, error: (error as Error).message }); + continue; + } + } + + logger.debug('No working proxies found meeting criteria', { + protocol, + minSuccessRate, + keysChecked: shuffledKeys.length, + }); + + return null; + } catch (error) { + logger.error('Error getting random active proxy', { + error: error instanceof Error ? error.message : String(error), + protocol, + minSuccessRate, + }); + return null; + } +} + // Utility functions function cleanProxyUrl(url: string): string { return url diff --git a/apps/data-service/src/providers/quotemedia.provider.ts b/apps/data-service/src/providers/qm.provider.ts similarity index 85% rename from apps/data-service/src/providers/quotemedia.provider.ts rename to apps/data-service/src/providers/qm.provider.ts index 05c6874..8ca8ccd 100644 --- a/apps/data-service/src/providers/quotemedia.provider.ts +++ b/apps/data-service/src/providers/qm.provider.ts @@ -1,15 +1,15 @@ import { getLogger } from '@stock-bot/logger'; import { ProviderConfig } from '../services/provider-registry.service'; -const logger = getLogger('quotemedia-provider'); +const logger = getLogger('qm-provider'); -export const quotemediaProvider: ProviderConfig = { - name: 'quotemedia', +export const qmProvider: ProviderConfig = { + name: 'qm', operations: { 'live-data': async (payload: { symbol: string; fields?: string[] }) => { - logger.info('Fetching live data from QuoteMedia', { symbol: payload.symbol }); + logger.info('Fetching live data from qm', { symbol: payload.symbol }); - // Simulate QuoteMedia API call + // Simulate qm API call const mockData = { symbol: payload.symbol, price: Math.random() * 1000 + 100, @@ -17,7 +17,7 @@ export const quotemediaProvider: ProviderConfig = { change: (Math.random() - 0.5) * 20, changePercent: (Math.random() - 0.5) * 5, timestamp: new Date().toISOString(), - source: 'quotemedia', + source: 'qm', fields: payload.fields || ['price', 'volume', 'change'], }; @@ -34,7 +34,7 @@ export const quotemediaProvider: ProviderConfig = { interval?: string; fields?: string[]; }) => { - logger.info('Fetching historical data from QuoteMedia', { + logger.info('Fetching historical data from qm', { symbol: payload.symbol, from: payload.from, to: payload.to, @@ -56,7 +56,7 @@ export const quotemediaProvider: ProviderConfig = { low: Math.random() * 1000 + 100, close: Math.random() * 1000 + 100, volume: Math.floor(Math.random() * 1000000), - source: 'quotemedia', + source: 'qm', }); } @@ -67,12 +67,12 @@ export const quotemediaProvider: ProviderConfig = { symbol: payload.symbol, interval: payload.interval || '1d', data, - source: 'quotemedia', + source: 'qm', totalRecords: data.length, }; }, 'batch-quotes': async (payload: { symbols: string[]; fields?: string[] }) => { - logger.info('Fetching batch quotes from QuoteMedia', { + logger.info('Fetching batch quotes from qm', { symbols: payload.symbols, count: payload.symbols.length, }); @@ -83,7 +83,7 @@ export const quotemediaProvider: ProviderConfig = { volume: Math.floor(Math.random() * 1000000), change: (Math.random() - 0.5) * 20, timestamp: new Date().toISOString(), - source: 'quotemedia', + source: 'qm', })); // Simulate network delay @@ -91,13 +91,13 @@ export const quotemediaProvider: ProviderConfig = { return { quotes, - source: 'quotemedia', + source: 'qm', timestamp: new Date().toISOString(), totalSymbols: payload.symbols.length, }; }, 'company-profile': async (payload: { symbol: string }) => { - logger.info('Fetching company profile from QuoteMedia', { symbol: payload.symbol }); + logger.info('Fetching company profile from qm', { symbol: payload.symbol }); // Simulate company profile data const profile = { @@ -109,7 +109,7 @@ export const quotemediaProvider: ProviderConfig = { marketCap: Math.floor(Math.random() * 1000000000000), employees: Math.floor(Math.random() * 100000), website: `https://www.${payload.symbol.toLowerCase()}.com`, - source: 'quotemedia', + source: 'qm', }; await new Promise(resolve => setTimeout(resolve, 150 + Math.random() * 100)); @@ -117,7 +117,7 @@ export const quotemediaProvider: ProviderConfig = { return profile; }, 'options-chain': async (payload: { symbol: string; expiration?: string }) => { - logger.info('Fetching options chain from QuoteMedia', { + logger.info('Fetching options chain from qm', { symbol: payload.symbol, expiration: payload.expiration, }); @@ -148,14 +148,14 @@ export const quotemediaProvider: ProviderConfig = { new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0], calls, puts, - source: 'quotemedia', + source: 'qm', }; }, }, scheduledJobs: [ // { - // type: 'quotemedia-premium-refresh', + // type: 'qm-premium-refresh', // operation: 'batch-quotes', // payload: { symbols: ['AAPL', 'GOOGL', 'MSFT'] }, // cronPattern: '*/2 * * * *', // Every 2 minutes @@ -163,7 +163,7 @@ export const quotemediaProvider: ProviderConfig = { // description: 'Refresh premium quotes with detailed market data' // }, // { - // type: 'quotemedia-options-update', + // type: 'qm-options-update', // operation: 'options-chain', // payload: { symbol: 'SPY' }, // cronPattern: '*/10 * * * *', // Every 10 minutes @@ -171,7 +171,7 @@ export const quotemediaProvider: ProviderConfig = { // description: 'Update options chain data for SPY ETF' // }, // { - // type: 'quotemedia-profiles', + // type: 'qm-profiles', // operation: 'company-profile', // payload: { symbol: 'AAPL' }, // cronPattern: '0 9 * * 1-5', // Weekdays at 9 AM diff --git a/apps/data-service/src/services/queue.service.ts b/apps/data-service/src/services/queue.service.ts index d7bdbee..7f95dc5 100644 --- a/apps/data-service/src/services/queue.service.ts +++ b/apps/data-service/src/services/queue.service.ts @@ -154,8 +154,8 @@ export class QueueService { // Define providers to register const providers = [ { module: '../providers/proxy.provider', export: 'proxyProvider' }, - { module: '../providers/quotemedia.provider', export: 'quotemediaProvider' }, - { module: '../providers/yahoo.provider', export: 'yahooProvider' }, + { module: '../providers/ib.provider', export: 'ibProvider' }, + // { module: '../providers/yahoo.provider', export: 'yahooProvider' }, ]; // Import and register all providers diff --git a/apps/data-service/tsconfig.json b/apps/data-service/tsconfig.json index 1a3f5ab..55a3676 100644 --- a/apps/data-service/tsconfig.json +++ b/apps/data-service/tsconfig.json @@ -23,6 +23,8 @@ { "path": "../../libs/questdb-client" }, { "path": "../../libs/mongodb-client" }, { "path": "../../libs/event-bus" }, - { "path": "../../libs/shutdown" } + { "path": "../../libs/shutdown" }, + { "path": "../../libs/utils" }, + { "path": "../../libs/browser" } ] } diff --git a/apps/data-service/turbo.json b/apps/data-service/turbo.json index e7c9b7b..2c969b3 100644 --- a/apps/data-service/turbo.json +++ b/apps/data-service/turbo.json @@ -10,7 +10,8 @@ "@stock-bot/logger#build", "@stock-bot/mongodb-client#build", "@stock-bot/questdb-client#build", - "@stock-bot/shutdown#build" + "@stock-bot/shutdown#build", + "@stock-bot/browser#build" ], "outputs": ["dist/**"], "inputs": [ diff --git a/bun.lock b/bun.lock index a6ca4e3..b64d112 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,7 @@ "dependencies": { "bullmq": "^5.53.2", "ioredis": "^5.6.1", + "playwright": "^1.53.0", }, "devDependencies": { "@eslint/js": "^9.28.0", @@ -243,9 +244,11 @@ "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", "socks-proxy-agent": "^8.0.5", + "user-agents": "^1.1.567", }, "devDependencies": { "@types/node": "^20.11.0", + "@types/user-agents": "^1.0.4", "@typescript-eslint/eslint-plugin": "^6.19.0", "@typescript-eslint/parser": "^6.19.0", "bun-types": "^1.2.15", @@ -930,6 +933,8 @@ "@types/supertest": ["@types/supertest@6.0.3", "", { "dependencies": { "@types/methods": "^1.1.4", "@types/superagent": "^8.1.0" } }, "sha512-8WzXq62EXFhJ7QsH3Ocb/iKQ/Ty9ZVWnVzoTKc9tyyFRRF3a74Tk2+TLFgaFFw364Ere+npzHKEJ6ga2LzIL7w=="], + "@types/user-agents": ["@types/user-agents@1.0.4", "", {}, "sha512-AjeFc4oX5WPPflgKfRWWJfkEk7Wu82fnj1rROPsiqFt6yElpdGFg8Srtm/4PU4rA9UiDUZlruGPgcwTMQlwq4w=="], + "@types/webidl-conversions": ["@types/webidl-conversions@7.0.3", "", {}, "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA=="], "@types/whatwg-url": ["@types/whatwg-url@11.0.5", "", { "dependencies": { "@types/webidl-conversions": "*" } }, "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ=="], @@ -1386,7 +1391,7 @@ "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], - "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], @@ -1656,6 +1661,8 @@ "lodash.camelcase": ["lodash.camelcase@4.3.0", "", {}, "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="], + "lodash.clonedeep": ["lodash.clonedeep@4.5.0", "", {}, "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ=="], + "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], "lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="], @@ -1904,6 +1911,10 @@ "pkg-dir": ["pkg-dir@4.2.0", "", { "dependencies": { "find-up": "^4.0.0" } }, "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ=="], + "playwright": ["playwright@1.53.0", "", { "dependencies": { "playwright-core": "1.53.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-ghGNnIEYZC4E+YtclRn4/p6oYbdPiASELBIYkBXfaTVKreQUYbMUYQDwS12a8F0/HtIjr/CkGjtwABeFPGcS4Q=="], + + "playwright-core": ["playwright-core@1.53.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-mGLg8m0pm4+mmtB7M89Xw/GSqoNC+twivl8ITteqvAndachozYe2ZA7srU6uleV1vEdAHYqjq+SV8SNxRRFYBw=="], + "possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="], "postcss": ["postcss@8.5.4", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-QSa9EBe+uwlGTFmHsPKokv3B/oEMQZxfqW0QqNCyhpa6mB1afzulwn8hihglqAb2pOw+BJgNlmXQ8la2VeHB7w=="], @@ -2240,6 +2251,8 @@ "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], + "user-agents": ["user-agents@1.1.567", "", { "dependencies": { "lodash.clonedeep": "^4.5.0" } }, "sha512-K5HqPZNWYbgd5sBUnvR7Aj2qt1jPCIAHaFbjF7uVyLD6nuMVGoW+eIrmQiqSFt/u2cZUXXI44rz6Y742KN/45Q=="], + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], @@ -2596,6 +2609,8 @@ "restore-cursor/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + "rollup/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "sass/immutable": ["immutable@5.1.2", "", {}, "sha512-qHKXW1q6liAk1Oys6umoaZbDRqjcjgSrbnrifHsfsttza7zcvRAsL7mMV6xWcyhwQy7Xj5v4hhbr6b+iDYwlmQ=="], "slice-ansi/ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="], @@ -2618,6 +2633,8 @@ "tar/mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="], + "vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "wrap-ansi/ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="], "wrap-ansi/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], @@ -2888,6 +2905,8 @@ "karma-coverage/istanbul-lib-instrument/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "karma/chokidar/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "karma/chokidar/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "karma/chokidar/readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], diff --git a/libs/browser/package.json b/libs/browser/package.json new file mode 100644 index 0000000..38cc84a --- /dev/null +++ b/libs/browser/package.json @@ -0,0 +1,24 @@ +{ + "name": "@stock-bot/browser", + "version": "1.0.0", + "description": "High-performance browser automation library with proxy support", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "scripts": { + "build": "tsc", + "test": "bun test", + "dev": "tsc --watch" + }, + "dependencies": { + "playwright": "^1.53.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + }, + "peerDependencies": { + "@stock-bot/logger": "workspace:*", + "@stock-bot/http": "workspace:*" + } +} diff --git a/libs/browser/src/browser-pool.ts b/libs/browser/src/browser-pool.ts new file mode 100644 index 0000000..e69de29 diff --git a/libs/browser/src/browser.ts b/libs/browser/src/browser.ts new file mode 100644 index 0000000..d6f472f --- /dev/null +++ b/libs/browser/src/browser.ts @@ -0,0 +1,361 @@ +import { BrowserContext, chromium, Page, Browser as PlaywrightBrowser } from 'playwright'; +import { getLogger } from '@stock-bot/logger'; +import type { BrowserOptions, NetworkEvent, NetworkEventHandler } from './types'; + +class BrowserSingleton { + private browser?: PlaywrightBrowser; + private contexts: Map = new Map(); + private logger = getLogger('browser'); + private options: BrowserOptions; + private initialized = false; + + constructor() { + this.options = { + headless: true, + timeout: 30000, + blockResources: false, + enableNetworkLogging: false, + }; + } + + async initialize(options: BrowserOptions = {}): Promise { + if (this.initialized) { + return; + } + + // Merge options + this.options = { + ...this.options, + ...options, + }; + + this.logger.info('Initializing browser...'); + + try { + this.browser = await chromium.launch({ + headless: this.options.headless, + timeout: this.options.timeout, + args: [ + // Security and sandbox + '--no-sandbox', + // '--disable-setuid-sandbox', + // '--disable-dev-shm-usage', + // '--disable-web-security', + // '--disable-features=VizDisplayCompositor', + // '--disable-blink-features=AutomationControlled', + + // // Performance optimizations + // '--disable-gpu', + // '--disable-gpu-sandbox', + // '--disable-software-rasterizer', + // '--disable-background-timer-throttling', + // '--disable-renderer-backgrounding', + // '--disable-backgrounding-occluded-windows', + // '--disable-field-trial-config', + // '--disable-back-forward-cache', + // '--disable-hang-monitor', + // '--disable-ipc-flooding-protection', + + // // Extensions and plugins + // '--disable-extensions', + // '--disable-plugins', + // '--disable-component-extensions-with-background-pages', + // '--disable-component-update', + // '--disable-plugins-discovery', + // '--disable-bundled-ppapi-flash', + + // // Features we don't need + // '--disable-default-apps', + // '--disable-sync', + // '--disable-translate', + // '--disable-client-side-phishing-detection', + // '--disable-domain-reliability', + // '--disable-features=TranslateUI', + // '--disable-features=Translate', + // '--disable-breakpad', + // '--disable-preconnect', + // '--disable-print-preview', + // '--disable-password-generation', + // '--disable-password-manager-reauthentication', + // '--disable-save-password-bubble', + // '--disable-single-click-autofill', + // '--disable-autofill', + // '--disable-autofill-keyboard-accessory-view', + // '--disable-full-form-autofill-ios', + + // // Audio/Video/Media + // '--mute-audio', + // '--disable-audio-output', + // '--autoplay-policy=user-gesture-required', + // '--disable-background-media-playback', + + // // Networking + // '--disable-background-networking', + // '--disable-sync', + // '--aggressive-cache-discard', + // '--disable-default-apps', + + // // UI/UX optimizations + // '--no-first-run', + // '--disable-infobars', + // '--disable-notifications', + // '--disable-desktop-notifications', + // '--disable-prompt-on-repost', + // '--disable-logging', + // '--disable-file-system', + // '--hide-scrollbars', + + // // Memory optimizations + // '--memory-pressure-off', + // '--max_old_space_size=4096', + // '--js-flags="--max-old-space-size=4096"', + // '--media-cache-size=1', + // '--disk-cache-size=1', + + // // Process management + // '--use-mock-keychain', + // '--password-store=basic', + // '--enable-automation', + // '--no-pings', + // '--no-service-autorun', + // '--metrics-recording-only', + // '--safebrowsing-disable-auto-update', + + // // Disable unnecessary features for headless mode + // '--disable-speech-api', + // '--disable-gesture-typing', + // '--disable-voice-input', + // '--disable-wake-on-wifi', + // '--disable-webgl', + // '--disable-webgl2', + // '--disable-3d-apis', + // '--disable-accelerated-2d-canvas', + // '--disable-accelerated-jpeg-decoding', + // '--disable-accelerated-mjpeg-decode', + // '--disable-accelerated-video-decode', + // '--disable-canvas-aa', + // '--disable-2d-canvas-clip-aa', + // '--disable-gl-drawing-for-tests', + ], + }); + + this.initialized = true; + this.logger.info('Browser initialized successfully'); + } catch (error) { + this.logger.error('Failed to initialize browser', { error }); + throw error; + } + } + + async createPageWithProxy( + url: string, + proxy?: string + ): Promise<{ + page: Page & { + onNetworkEvent: (handler: NetworkEventHandler) => void; + offNetworkEvent: (handler: NetworkEventHandler) => void; + clearNetworkListeners: () => void; + }; + contextId: string; + }> { + if (!this.browser) { + throw new Error('Browser not initialized. Call Browser.initialize() first.'); + } + + const contextId = `ctx-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + const contextOptions: Record = { + ignoreHTTPSErrors: true, + bypassCSP: true, + }; + + if (proxy) { + const [protocol, rest] = proxy.split('://'); + const [auth, hostPort] = rest.includes('@') ? rest.split('@') : [null, rest]; + const [host, port] = hostPort.split(':'); + + contextOptions.proxy = { + server: `${protocol}://${host}:${port}`, + username: auth?.split(':')[0] || '', + password: auth?.split(':')[1] || '', + }; + } + + const context = await this.browser.newContext(contextOptions); + + // Block resources for performance + if (this.options.blockResources) { + await context.route('**/*.{png,jpg,jpeg,gif,svg,ico,woff,woff2,ttf,css}', route => { + route.abort(); + }); + } + + this.contexts.set(contextId, context); + + const page = await context.newPage(); + page.setDefaultTimeout(this.options.timeout || 30000); + page.setDefaultNavigationTimeout(this.options.timeout || 30000); + + // Create network event handlers for this page + const networkEventHandlers: Set = new Set(); + + // Add network monitoring methods to the page + const enhancedPage = page as Page & { + onNetworkEvent: (handler: NetworkEventHandler) => void; + offNetworkEvent: (handler: NetworkEventHandler) => void; + clearNetworkListeners: () => void; + }; + + enhancedPage.onNetworkEvent = (handler: NetworkEventHandler) => { + networkEventHandlers.add(handler); + + // Set up network monitoring on first handler + if (networkEventHandlers.size === 1) { + this.setupNetworkMonitoring(page, networkEventHandlers); + } + }; + + enhancedPage.offNetworkEvent = (handler: NetworkEventHandler) => { + networkEventHandlers.delete(handler); + }; + + enhancedPage.clearNetworkListeners = () => { + networkEventHandlers.clear(); + }; + + if (url) { + await page.goto(url, { + waitUntil: 'domcontentloaded', + timeout: this.options.timeout, + }); + } + + return { page: enhancedPage, contextId }; + } + + private setupNetworkMonitoring(page: Page, handlers: Set): void { + // Listen to requests + page.on('request', async request => { + const event: NetworkEvent = { + url: request.url(), + method: request.method(), + type: 'request', + timestamp: Date.now(), + headers: request.headers(), + }; + + // Capture request data for POST/PUT/PATCH requests + if (['POST', 'PUT', 'PATCH'].includes(request.method())) { + try { + const postData = request.postData(); + if (postData) { + event.requestData = postData; + } + } catch { + // Some requests might not have accessible post data + } + } + + this.emitNetworkEvent(event, handlers); + }); + + // Listen to responses + page.on('response', async response => { + const event: NetworkEvent = { + url: response.url(), + method: response.request().method(), + status: response.status(), + type: 'response', + timestamp: Date.now(), + headers: response.headers(), + }; + + // Capture response data for GET/POST requests with JSON content + const contentType = response.headers()['content-type'] || ''; + if (contentType.includes('application/json') || contentType.includes('text/')) { + try { + const responseData = await response.text(); + event.responseData = responseData; + } catch { + // Response might be too large or not accessible + } + } + + this.emitNetworkEvent(event, handlers); + }); + + // Listen to failed requests + page.on('requestfailed', request => { + const event: NetworkEvent = { + url: request.url(), + method: request.method(), + type: 'failed', + timestamp: Date.now(), + headers: request.headers(), + }; + + // Try to capture request data for failed requests too + if (['POST', 'PUT', 'PATCH'].includes(request.method())) { + try { + const postData = request.postData(); + if (postData) { + event.requestData = postData; + } + } catch { + // Ignore errors when accessing post data + } + } + + this.emitNetworkEvent(event, handlers); + }); + } + + private emitNetworkEvent(event: NetworkEvent, handlers: Set): void { + for (const handler of handlers) { + try { + handler(event); + } catch (error) { + this.logger.error('Network event handler error', { error }); + } + } + } + + async evaluate(page: Page, fn: () => T): Promise { + return page.evaluate(fn); + } + + async closeContext(contextId: string): Promise { + const context = this.contexts.get(contextId); + if (context) { + await context.close(); + this.contexts.delete(contextId); + } + } + + async close(): Promise { + // Close all contexts + for (const [, context] of this.contexts) { + await context.close(); + } + this.contexts.clear(); + + // Close browser + if (this.browser) { + await this.browser.close(); + this.browser = undefined; + } + + this.initialized = false; + this.logger.info('Browser closed'); + } + + get isInitialized(): boolean { + return this.initialized; + } +} + +// Export singleton instance +export const Browser = new BrowserSingleton(); + +// Also export the class for typing if needed +export { BrowserSingleton as BrowserClass }; diff --git a/libs/browser/src/fast-browser.ts b/libs/browser/src/fast-browser.ts new file mode 100644 index 0000000..e69de29 diff --git a/libs/browser/src/index.ts b/libs/browser/src/index.ts new file mode 100644 index 0000000..96cb4ab --- /dev/null +++ b/libs/browser/src/index.ts @@ -0,0 +1,3 @@ +export { Browser } from './browser'; +export { BrowserTabManager } from './tab-manager'; +export type { BrowserOptions, ScrapingResult } from './types'; diff --git a/libs/browser/src/tab-manager.ts b/libs/browser/src/tab-manager.ts new file mode 100644 index 0000000..293de1a --- /dev/null +++ b/libs/browser/src/tab-manager.ts @@ -0,0 +1,103 @@ +import { Page } from 'playwright'; +import { getLogger } from '@stock-bot/logger'; +import { Browser } from './browser'; +import type { ScrapingResult } from './types'; + +interface TabInfo { + page: Page; + contextId: string; +} + +export class BrowserTabManager { + private tabs: Map = new Map(); + private logger = getLogger('browser-tab-manager'); + + async createTab(url?: string): Promise<{ page: Page; tabId: string }> { + const tabId = `tab-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + const { page, contextId } = await Browser.createPageWithProxy(url || 'about:blank'); + + this.tabs.set(tabId, { page, contextId }); + this.logger.debug('Tab created', { tabId, url }); + + return { page, tabId }; + } + + async createTabWithProxy( + url: string, + proxy: string + ): Promise<{ page: Page; tabId: string; contextId: string }> { + const tabId = `tab-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + const { page, contextId } = await Browser.createPageWithProxy(url, proxy); + + this.tabs.set(tabId, { page, contextId }); + this.logger.debug('Tab with proxy created', { tabId, url, proxy }); + + return { page, tabId, contextId }; + } + + async scrapeUrlsWithProxies( + urlProxyPairs: Array<{ url: string; proxy: string }>, + extractor: (page: Page) => Promise, + options: { concurrency?: number } = {} + ): Promise[]> { + const { concurrency = 3 } = options; + const results: ScrapingResult[] = []; + + for (let i = 0; i < urlProxyPairs.length; i += concurrency) { + const batch = urlProxyPairs.slice(i, i + concurrency); + + const batchPromises = batch.map(async ({ url, proxy }) => { + let tabId: string | undefined; + + try { + const result = await this.createTabWithProxy(url, proxy); + tabId = result.tabId; + + const data = await extractor(result.page); + + return { + data, + url, + success: true, + } as ScrapingResult; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + return { + data: null as T, + url, + success: false, + error: errorMessage, + } as ScrapingResult; + } finally { + if (tabId) { + await this.closeTab(tabId); + } + } + }); + + const batchResults = await Promise.all(batchPromises); + results.push(...batchResults); + } + + return results; + } + + async closeTab(tabId: string): Promise { + const tab = this.tabs.get(tabId); + if (tab) { + await tab.page.close(); + await Browser.closeContext(tab.contextId); + this.tabs.delete(tabId); + this.logger.debug('Tab closed', { tabId }); + } + } + + getTabCount(): number { + return this.tabs.size; + } + + getAllTabIds(): string[] { + return Array.from(this.tabs.keys()); + } +} diff --git a/libs/browser/src/types.ts b/libs/browser/src/types.ts new file mode 100644 index 0000000..84e41ee --- /dev/null +++ b/libs/browser/src/types.ts @@ -0,0 +1,30 @@ +export interface BrowserOptions { + proxy?: string; + headless?: boolean; + timeout?: number; + blockResources?: boolean; + enableNetworkLogging?: boolean; +} + +// Keep the old name for backward compatibility +export type FastBrowserOptions = BrowserOptions; + +export interface ScrapingResult { + data: T; + url: string; + success: boolean; + error?: string; +} + +export interface NetworkEvent { + url: string; + method: string; + status?: number; + type: 'request' | 'response' | 'failed'; + timestamp: number; + requestData?: string; + responseData?: string; + headers?: Record; +} + +export type NetworkEventHandler = (event: NetworkEvent) => void; diff --git a/libs/browser/src/utils.ts b/libs/browser/src/utils.ts new file mode 100644 index 0000000..e69de29 diff --git a/libs/browser/tsconfig.json b/libs/browser/tsconfig.json new file mode 100644 index 0000000..350f503 --- /dev/null +++ b/libs/browser/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.lib.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"], + "references": [{ "path": "../../libs/logger" }] +} diff --git a/libs/cache/src/redis-cache.ts b/libs/cache/src/redis-cache.ts index a42a21c..02756e2 100644 --- a/libs/cache/src/redis-cache.ts +++ b/libs/cache/src/redis-cache.ts @@ -289,6 +289,19 @@ export class RedisCache implements CacheProvider { ); } + async keys(pattern: string): Promise { + return this.safeExecute( + async () => { + const fullPattern = `${this.keyPrefix}${pattern}`; + const keys = await this.redis.keys(fullPattern); + // Remove the prefix from returned keys to match the interface expectation + return keys.map(key => key.replace(this.keyPrefix, '')); + }, + [], + 'keys' + ); + } + async health(): Promise { try { const pong = await this.redis.ping(); diff --git a/libs/cache/src/types.ts b/libs/cache/src/types.ts index 9ad35aa..f19c8cb 100644 --- a/libs/cache/src/types.ts +++ b/libs/cache/src/types.ts @@ -16,6 +16,7 @@ export interface CacheProvider { del(key: string): Promise; exists(key: string): Promise; clear(): Promise; + keys(pattern: string): Promise; getStats(): CacheStats; health(): Promise; diff --git a/libs/http/package.json b/libs/http/package.json index 0e32950..08dfbd3 100644 --- a/libs/http/package.json +++ b/libs/http/package.json @@ -20,15 +20,17 @@ "axios": "^1.9.0", "http-proxy-agent": "^7.0.2", "https-proxy-agent": "^7.0.6", - "socks-proxy-agent": "^8.0.5" + "socks-proxy-agent": "^8.0.5", + "user-agents": "^1.1.567" }, "devDependencies": { "@types/node": "^20.11.0", - "typescript": "^5.3.0", - "eslint": "^8.56.0", + "@types/user-agents": "^1.0.4", "@typescript-eslint/eslint-plugin": "^6.19.0", "@typescript-eslint/parser": "^6.19.0", - "bun-types": "^1.2.15" + "bun-types": "^1.2.15", + "eslint": "^8.56.0", + "typescript": "^5.3.0" }, "exports": { ".": { diff --git a/libs/http/src/adapters/axios-adapter.ts b/libs/http/src/adapters/axios-adapter.ts index 477ab04..cb98a5c 100644 --- a/libs/http/src/adapters/axios-adapter.ts +++ b/libs/http/src/adapters/axios-adapter.ts @@ -11,15 +11,17 @@ export class AxiosAdapter implements RequestAdapter { canHandle(config: RequestConfig): boolean { // Axios handles SOCKS proxies return Boolean( - config.proxy && (config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5') + config.proxy && + typeof config.proxy !== 'string' && + (config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5') ); } async request(config: RequestConfig, signal: AbortSignal): Promise> { const { url, method = 'GET', headers, data, proxy } = config; - if (!proxy) { - throw new Error('Axios adapter requires proxy configuration'); + if (!proxy || typeof proxy === 'string') { + throw new Error('Axios adapter requires ProxyInfo configuration'); } // Create proxy configuration using ProxyManager diff --git a/libs/http/src/adapters/fetch-adapter.ts b/libs/http/src/adapters/fetch-adapter.ts index 238a8ac..2a172c9 100644 --- a/libs/http/src/adapters/fetch-adapter.ts +++ b/libs/http/src/adapters/fetch-adapter.ts @@ -9,6 +9,9 @@ import type { RequestAdapter } from './types'; export class FetchAdapter implements RequestAdapter { canHandle(config: RequestConfig): boolean { // Fetch handles non-proxy requests and HTTP/HTTPS proxies + if (typeof config.proxy === 'string') { + return config.proxy.startsWith('http'); + } return !config.proxy || config.proxy.protocol === 'http' || config.proxy.protocol === 'https'; } @@ -31,7 +34,11 @@ export class FetchAdapter implements RequestAdapter { } // Add proxy if needed (using Bun's built-in proxy support) - if (proxy) { + if (typeof proxy === 'string') { + // If proxy is a URL string, use it directly + (fetchOptions as any).proxy = proxy; + } else if (proxy) { + // If proxy is a ProxyInfo object, create a proxy URL (fetchOptions as any).proxy = ProxyManager.createProxyUrl(proxy); } const response = await fetch(url, fetchOptions); diff --git a/libs/http/src/client.ts b/libs/http/src/client.ts index 5302f28..8086c57 100644 --- a/libs/http/src/client.ts +++ b/libs/http/src/client.ts @@ -2,6 +2,7 @@ import type { Logger } from '@stock-bot/logger'; import { AdapterFactory } from './adapters/index'; import type { HttpClientConfig, HttpResponse, RequestConfig } from './types'; import { HttpError } from './types'; +import { getRandomUserAgent } from './user-agent'; export class HttpClient { private readonly config: HttpClientConfig; @@ -165,9 +166,17 @@ export class HttpClient { * Merge configs with defaults */ private mergeConfig(config: RequestConfig): RequestConfig { + // Merge headers with automatic User-Agent assignment + const mergedHeaders = { ...this.config.headers, ...config.headers }; + + // Add random User-Agent if not specified + if (!mergedHeaders['User-Agent'] && !mergedHeaders['user-agent']) { + mergedHeaders['User-Agent'] = getRandomUserAgent(); + } + return { ...config, - headers: { ...this.config.headers, ...config.headers }, + headers: mergedHeaders, timeout: config.timeout ?? this.config.timeout, }; } diff --git a/libs/http/src/index.ts b/libs/http/src/index.ts index a70ad6e..ad1daa1 100644 --- a/libs/http/src/index.ts +++ b/libs/http/src/index.ts @@ -1,8 +1,9 @@ // Re-export all types and classes -export * from './types'; +export * from './adapters/index'; export * from './client'; export * from './proxy-manager'; -export * from './adapters/index'; +export * from './types'; +export * from './user-agent'; // Default export export { HttpClient as default } from './client'; diff --git a/libs/http/src/types.ts b/libs/http/src/types.ts index 0340bec..330e05d 100644 --- a/libs/http/src/types.ts +++ b/libs/http/src/types.ts @@ -32,7 +32,7 @@ export interface RequestConfig { headers?: Record; data?: any; // Changed from 'body' to 'data' for consistency timeout?: number; - proxy?: ProxyInfo; + proxy?: ProxyInfo | string; // Proxy can be a ProxyInfo object or a URL string } export interface HttpResponse { diff --git a/libs/http/src/user-agent.ts b/libs/http/src/user-agent.ts new file mode 100644 index 0000000..1b25dd1 --- /dev/null +++ b/libs/http/src/user-agent.ts @@ -0,0 +1,6 @@ +import UserAgent from 'user-agents'; + +export function getRandomUserAgent(): string { + const userAgent = new UserAgent(); + return userAgent.toString(); +} diff --git a/libs/proxy/package.json b/libs/proxy/package.json new file mode 100644 index 0000000..bbef6d0 --- /dev/null +++ b/libs/proxy/package.json @@ -0,0 +1,18 @@ +{ + "name": "@stock-bot/proxy", + "version": "1.0.0", + "description": "Simple proxy management library", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "type": "module", + "scripts": { + "build": "tsc", + "test": "bun test", + "dev": "tsc --watch" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + }, + "peerDependencies": {} +} diff --git a/libs/proxy/src/index.ts b/libs/proxy/src/index.ts new file mode 100644 index 0000000..829a7ed --- /dev/null +++ b/libs/proxy/src/index.ts @@ -0,0 +1,97 @@ +// Simple proxy list manager +let proxies: string[] = []; +let currentIndex = 0; + +const DEFAULT_PROXY_URL = + 'https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all'; + +/** + * Fetch proxy list from URL and store in module + */ +export async function refreshProxies(fetchUrl: string = DEFAULT_PROXY_URL): Promise { + try { + const response = await fetch(fetchUrl); + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.text(); + const newProxies = data + .trim() + .split('\n') + .map(line => line.trim()) + .filter(line => line && line.includes(':')) + .map(line => { + // Convert host:port to http://host:port format + return line.startsWith('http') ? line : `http://${line}`; + }); + + proxies = newProxies; + currentIndex = 0; + + return proxies; + } catch (error) { + throw new Error(`Failed to fetch proxies: ${error}`); + } +} + +/** + * Get next proxy URL in round-robin fashion + */ +export function getProxyURL(): string | null { + if (proxies.length === 0) { + return null; + } + + const proxy = proxies[currentIndex]; + currentIndex = (currentIndex + 1) % proxies.length; + + return proxy; +} + +/** + * Get multiple proxy URLs + */ +export function getProxyURLs(count: number): string[] { + const urls: string[] = []; + for (let i = 0; i < count; i++) { + const url = getProxyURL(); + if (url) { + urls.push(url); + } + } + return urls; +} + +/** + * Get random proxy URL + */ +export function getRandomProxyURL(): string | null { + if (proxies.length === 0) { + return null; + } + + const randomIndex = Math.floor(Math.random() * proxies.length); + return proxies[randomIndex]; +} + +/** + * Get current proxy count + */ +export function getProxyCount(): number { + return proxies.length; +} + +/** + * Get all proxies + */ +export function getAllProxies(): string[] { + return [...proxies]; +} + +/** + * Initialize proxy manager with initial fetch + */ +export async function initializeProxies(fetchUrl?: string): Promise { + await refreshProxies(fetchUrl); +} diff --git a/libs/proxy/src/proxy-manager.ts b/libs/proxy/src/proxy-manager.ts new file mode 100644 index 0000000..e69de29 diff --git a/libs/proxy/src/types.ts b/libs/proxy/src/types.ts new file mode 100644 index 0000000..a4f94ed --- /dev/null +++ b/libs/proxy/src/types.ts @@ -0,0 +1,22 @@ +export interface ProxyInfo { + host: string; + port: number; + protocol: 'http' | 'https' | 'socks4' | 'socks5'; + username?: string; + password?: string; + country?: string; + isActive?: boolean; +} + +export interface ProxyManagerOptions { + fetchUrl?: string; + refreshIntervalMs?: number; + maxRetries?: number; + timeout?: number; +} + +export interface ProxyResponse { + proxies: ProxyInfo[]; + totalCount: number; + activeCount: number; +} diff --git a/libs/proxy/tsconfig.json b/libs/proxy/tsconfig.json new file mode 100644 index 0000000..ffc624d --- /dev/null +++ b/libs/proxy/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.lib.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "skipLibCheck": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"] +} diff --git a/libs/utils/src/common.ts b/libs/utils/src/common.ts new file mode 100644 index 0000000..26a47df --- /dev/null +++ b/libs/utils/src/common.ts @@ -0,0 +1,7 @@ +export function createProxyUrl(proxy: any): string { + const { protocol, host, port, username, password } = proxy; + if (username && password) { + return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`; + } + return `${protocol}://${host}:${port}`; +} diff --git a/libs/utils/src/index.ts b/libs/utils/src/index.ts index e1875f3..213f278 100644 --- a/libs/utils/src/index.ts +++ b/libs/utils/src/index.ts @@ -1,2 +1,3 @@ -export * from './dateUtils'; export * from './calculations/index'; +export * from './common'; +export * from './dateUtils'; diff --git a/package.json b/package.json index b4af578..e69d8e0 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,8 @@ }, "dependencies": { "bullmq": "^5.53.2", - "ioredis": "^5.6.1" + "ioredis": "^5.6.1", + "playwright": "^1.53.0" }, "trustedDependencies": [ "@tailwindcss/oxide", diff --git a/test-browser-simple.ts b/test-browser-simple.ts new file mode 100644 index 0000000..318f3f9 --- /dev/null +++ b/test-browser-simple.ts @@ -0,0 +1,166 @@ +/** + * Simple Browser and Network Monitoring Test + */ +import { Browser } from '@stock-bot/browser'; + +async function testBasicBrowser() { + console.log('๐Ÿš€ Testing basic browser functionality...'); + + try { + // Initialize browser + await Browser.initialize({ + headless: true, + timeout: 15000, + blockResources: false, + enableNetworkLogging: true, + }); + + console.log('โœ… Browser initialized'); + + // Test 1: Simple page without proxy + console.log('๐Ÿ“„ Testing simple page without proxy...'); + const { page, contextId } = await Browser.createPageWithProxy( + 'https://httpbin.org/json' + ); + + let capturedData = null; + let eventCount = 0; + + page.onNetworkEvent(event => { + eventCount++; + console.log(`๐Ÿ“ก Event ${eventCount}: ${event.type} - ${event.method} ${event.url}`); + + if (event.type === 'response' && event.url.includes('httpbin.org/json')) { + console.log(` ๐Ÿ“Š Status: ${event.status}`); + if (event.responseData) { + capturedData = event.responseData; + console.log(` ๐Ÿ“ Response: ${event.responseData}`); + } + } + }); + + await page.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 2000)); + + console.log(`โœ… Test completed. Events captured: ${eventCount}`); + if (capturedData) { + console.log('โœ… Successfully captured response data'); + } + + await Browser.closeContext(contextId); + return true; + + } catch (error) { + console.error('โŒ Basic test failed:', error); + return false; + } finally { + await Browser.close(); + } +} + +async function testProxyConnection() { + console.log('\n๐Ÿ”„ Testing proxy connection...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 10000, + blockResources: false, + }); + + // Test different proxy formats + const proxyConfigs = [ + null, // No proxy + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', + ]; + + for (const proxy of proxyConfigs) { + console.log(`\n๐ŸŒ Testing with proxy: ${proxy || 'No proxy'}`); + + try { + const { page, contextId } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip', + proxy + ); + + page.onNetworkEvent(event => { + if (event.type === 'response' && event.url.includes('httpbin.org/ip')) { + console.log(` ๐Ÿ“ IP Response: ${event.responseData}`); + } + }); + + await page.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 1500)); + await Browser.closeContext(contextId); + console.log(' โœ… Success'); + + } catch (error) { + console.log(` โŒ Failed: ${error.message}`); + } + } + + } catch (error) { + console.error('โŒ Proxy test setup failed:', error); + } finally { + await Browser.close(); + } +} + +async function testIBWithWorkaround() { + console.log('\n๐Ÿฆ Testing IB endpoint with workaround...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 20000, + blockResources: true, // Block resources for performance + }); + + // Try without proxy first + console.log('๐ŸŒ Attempting IB without proxy...'); + try { + const { page, contextId } = await Browser.createPageWithProxy( + 'https://www.interactivebrokers.com' + ); + + let responseCount = 0; + page.onNetworkEvent(event => { + if (event.type === 'response') { + responseCount++; + console.log(` ๐Ÿ“ฅ Response ${responseCount}: ${event.status} ${event.url}`); + } + }); + + await page.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 3000)); + console.log(`โœ… IB main page loaded. Responses: ${responseCount}`); + await Browser.closeContext(contextId); + + } catch (error) { + console.log(`โŒ IB without proxy failed: ${error.message}`); + } + + } catch (error) { + console.error('โŒ IB test failed:', error); + } finally { + await Browser.close(); + } +} + +// Run tests +async function runAllTests() { + console.log('๐Ÿงช Starting Browser Network Monitoring Tests\n'); + + const basicResult = await testBasicBrowser(); + await testProxyConnection(); + await testIBWithWorkaround(); + + console.log(`\n๐Ÿ Basic functionality: ${basicResult ? 'โœ… PASS' : 'โŒ FAIL'}`); + console.log('โœ… All tests completed!'); +} + +if (import.meta.main) { + runAllTests().catch(console.error); +} + +export { testBasicBrowser, testProxyConnection, testIBWithWorkaround }; diff --git a/test-browser.ts b/test-browser.ts new file mode 100644 index 0000000..11040f5 --- /dev/null +++ b/test-browser.ts @@ -0,0 +1,62 @@ +import { Browser, BrowserTabManager } from './libs/browser/src'; + +async function testSimplifiedBrowser() { + console.log('Testing simplified browser library...'); + + try { + console.log('Initializing browser...'); + await Browser.initialize({ + headless: true, + blockResources: true, + timeout: 10000, + }); + + // Test single page with proxy support + console.log('Testing page creation...'); + const { page, contextId } = await Browser.createPageWithProxy( + 'https://httpbin.org/json', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + const content = await Browser.evaluate(page, () => document.body.textContent); + console.log('Page content:', content?.substring(0, 100) + '...'); + + // Test tab manager (no longer needs browser instance) + console.log('Testing tab manager...'); + const tabManager = new BrowserTabManager(); + + // Test multiple URL scraping with different proxies + const urlProxyPairs = [ + { url: 'https://httpbin.org/uuid', proxy: '' }, // No proxy + { url: 'https://httpbin.org/ip', proxy: '' }, // No proxy + ]; + + const results = await tabManager.scrapeUrlsWithProxies( + urlProxyPairs, + async page => { + const text = await page.textContent('body'); + return { content: text?.substring(0, 50) }; + }, + { concurrency: 2 } + ); + + console.log('Scraping results:'); + results.forEach((result, index) => { + console.log(` ${index + 1}. ${result.url}: ${result.success ? 'SUCCESS' : 'FAILED'}`); + if (result.data) { + console.log(` Data: ${result.data.content}...`); + } + }); + + // Clean up + await page.close(); + await Browser.closeContext(contextId); + await Browser.close(); + + console.log('โœ… Simplified browser test completed successfully!'); + } catch (error) { + console.error('โŒ Browser test failed:', error); + } +} + +testSimplifiedBrowser(); diff --git a/test-ib-no-proxy.ts b/test-ib-no-proxy.ts new file mode 100644 index 0000000..e6fe62f --- /dev/null +++ b/test-ib-no-proxy.ts @@ -0,0 +1,139 @@ +import { Browser } from '@stock-bot/browser'; + +async function testWithoutProxy() { + console.log('๐Ÿ”ฌ Testing WITHOUT proxy...'); + + try { + await Browser.initialize({ headless: true, timeout: 15000, blockResources: false }); + console.log('โœ… Browser initialized'); + + const { page, contextId } = await Browser.createPageWithProxy( + 'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/' + // No proxy parameter + ); + console.log('โœ… Page created without proxy'); + + let eventCount = 0; + let summaryData: SummaryResponse | null = null; + + page.onNetworkEvent(event => { + eventCount++; + + // Capture the summary API response + if (event.url.includes('/webrest/search/product-types/summary')) { + console.log(`๐ŸŽฏ Found summary API call: ${event.type} ${event.url}`); + + if (event.type === 'response' && event.responseData) { + console.log(`๐Ÿ“Š Summary API Response Data: ${event.responseData}`); + try { + summaryData = JSON.parse(event.responseData) as any; + const totalCount = summaryData[0].totalCount; + console.log('๐Ÿ“Š Summary API Response:', JSON.stringify(summaryData, null, 2)); + console.log(`๐Ÿ”ข Total symbols found: ${totalCount || 'Unknown'}`); + } catch (e) { + console.log('๐Ÿ“Š Raw Summary Response:', event.responseData); + } + } + } + + // Uncomment to see all network events + // console.log(`๐Ÿ“ก Event ${eventCount}: ${event.type} ${event.url}`); + }); + + console.log('โณ Waiting for page load...'); + await page.waitForLoadState('domcontentloaded', { timeout: 15000 }); + console.log('โœ… Page loaded'); + + // Complete interaction flow + try { + console.log('๐Ÿ” Looking for Products tab...'); + await page.waitForTimeout(3000); + + const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); + await productsTab.waitFor({ timeout: 10000 }); + console.log('โœ… Found Products tab'); + + console.log('๐Ÿ–ฑ๏ธ Clicking Products tab...'); + await productsTab.click(); + console.log('โœ… Products tab clicked'); + + await page.waitForTimeout(2000); + + console.log('๐Ÿ” Looking for Asset Classes accordion...'); + const assetClassesAccordion = page.locator( + '#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")' + ); + await assetClassesAccordion.waitFor({ timeout: 10000 }); + console.log('โœ… Found Asset Classes accordion'); + + console.log('๐Ÿ–ฑ๏ธ Clicking Asset Classes accordion...'); + await assetClassesAccordion.click(); + console.log('โœ… Asset Classes accordion clicked'); + + await page.waitForTimeout(2000); + + console.log('๐Ÿ” Looking for Stocks checkbox...'); + const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")'); + await stocksSpan.waitFor({ timeout: 10000 }); + console.log('โœ… Found Stocks span'); + + const parentContainer = stocksSpan.locator('..'); + const checkbox = parentContainer.locator('input[type="checkbox"]'); + + if ((await checkbox.count()) > 0) { + console.log('๐Ÿ“‹ Clicking Stocks checkbox...'); + await checkbox.first().check(); + console.log('โœ… Stocks checkbox checked'); + } else { + console.log('โš ๏ธ Could not find checkbox near Stocks text'); + } + + await page.waitForTimeout(1000); + + console.log('๐Ÿ” Looking for Apply button...'); + const applyButton = page.locator( + 'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]' + ); + + if ((await applyButton.count()) > 0) { + console.log('๐ŸŽฏ Clicking Apply button...'); + await applyButton.first().click(); + console.log('โœ… Apply button clicked'); + await page.waitForTimeout(3000); + } else { + console.log('โš ๏ธ Could not find Apply button'); + } + } catch (interactionError) { + const errorMessage = + interactionError instanceof Error ? interactionError.message : String(interactionError); + console.error('โŒ Page interaction failed:', errorMessage); + } + + await new Promise(resolve => setTimeout(resolve, 2000)); + console.log(`๐Ÿ“Š Total events captured: ${eventCount}`); + + // Show final results + if (summaryData) { + console.log('โœ… SUCCESS: Captured summary data!'); + console.log(`๐Ÿ”ข Final total count: ${summaryData?.data?.totalCount || 'Unknown'}`); + console.log(`๐Ÿ“‹ Data keys: ${Object.keys(summaryData).join(', ')}`); + } else { + console.log('โŒ No summary data captured'); + } + + await Browser.closeContext(contextId); + await Browser.close(); + + console.log('โœ… Test completed successfully'); + return true; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error('โŒ Error:', errorMessage); + await Browser.close(); + return false; + } +} + +testWithoutProxy().then(success => { + console.log(`๐Ÿ Final result: ${success ? 'SUCCESS' : 'FAILED'}`); +}); diff --git a/test-ib-working.ts b/test-ib-working.ts new file mode 100644 index 0000000..de3d109 --- /dev/null +++ b/test-ib-working.ts @@ -0,0 +1,160 @@ +/** + * Working Interactive Brokers test with verified network monitoring + */ +import { Browser } from '@stock-bot/browser'; + +async function testIBWithWorking() { + console.log('๐Ÿฆ Testing IB with working network monitoring and fixed proxy auth...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 20000, + blockResources: false, // Don't block resources initially + }); + + // Test 1: Try a simple proxy detection service first + console.log('๐ŸŒ Testing proxy connectivity...'); + const { page: proxyPage, contextId: proxyCtx } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + let proxyEvents = 0; + let myIP = null; + proxyPage.onNetworkEvent(event => { + proxyEvents++; + if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { + try { + const data = JSON.parse(event.responseData); + myIP = data.origin; + console.log(` ๐Ÿ“ Proxy IP: ${myIP}`); + } catch (e) { + console.log(` ๐Ÿ“Š Raw response: ${event.responseData}`); + } + } + }); + + await proxyPage.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 2000)); + await Browser.closeContext(proxyCtx); + + console.log(`๐Ÿ“Š Proxy test events: ${proxyEvents}`); + + // Test 2: Try IB API endpoint with fixed proxy auth + console.log('๐ŸŽฏ Testing IB API endpoint...'); + const { page: apiPage, contextId: apiCtx } = await Browser.createPageWithProxy( + 'https://www.interactivebrokers.com/webrest/search/product-types/summary', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + let apiEvents = 0; + let summaryData = null; + apiPage.onNetworkEvent(event => { + apiEvents++; + console.log(` ๐Ÿ“ก API Event: ${event.type} ${event.method} ${event.url}`); + + if (event.type === 'response' && event.url.includes('summary')) { + console.log(` ๐ŸŽฏ Found summary response! Status: ${event.status}`); + if (event.responseData) { + summaryData = event.responseData; + try { + const data = JSON.parse(event.responseData); + console.log(` ๐Ÿ“Š Summary data: ${JSON.stringify(data, null, 2)}`); + } catch (e) { + console.log(` ๐Ÿ“Š Raw summary: ${event.responseData.substring(0, 200)}...`); + } + } + } + }); + + await apiPage.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 3000)); + await Browser.closeContext(apiCtx); + + return { + proxyEvents, + apiEvents, + summaryData, + proxyIP: myIP, + success: apiEvents > 0 || summaryData !== null, + }; + } catch (error) { + console.error('โŒ IB test failed:', error); + return { + proxyEvents: 0, + apiEvents: 0, + summaryData: null, + proxyIP: null, + success: false, + error: error.message, + }; + } finally { + await Browser.close(); + } +} + +async function testWithProxyFallback() { + console.log('\n๐Ÿ”„ Testing with proxy fallback strategy...'); + + const proxiesToTest = [ + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', // Your proxy + ]; + + for (const proxy of proxiesToTest) { + console.log(`\n๐ŸŒ Testing with: ${proxy || 'No proxy'}`); + + try { + await Browser.initialize({ + headless: true, + timeout: 15000, + blockResources: false, + }); + + const { page, contextId } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip', + proxy + ); + + let ipResponse = null; + page.onNetworkEvent(event => { + if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { + ipResponse = event.responseData; + console.log(` ๐Ÿ“ IP: ${JSON.parse(event.responseData).origin}`); + } + }); + + await page.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 1000)); + await Browser.closeContext(contextId); + } catch (error) { + console.log(` โŒ Failed: ${error.message}`); + } finally { + await Browser.close(); + } + } +} + +async function runIBTests() { + console.log('๐Ÿš€ Interactive Brokers Network Monitoring Tests with Fixed Proxy Auth\n'); + + const result = await testIBWithWorking(); + await testWithProxyFallback(); + + console.log('\n๐Ÿ Final Results:'); + console.log(` ๐ŸŒ Proxy events: ${result.proxyEvents || 0}`); + console.log(` ๐Ÿ“ Proxy IP: ${result.proxyIP || 'Not captured'}`); + console.log(` ๐ŸŽฏ API events: ${result.apiEvents || 0}`); + console.log(` ๐Ÿ“Š Summary data: ${result.summaryData ? 'Captured' : 'Not captured'}`); + console.log(` โœ… Overall success: ${result.success}`); + + if (result.error) { + console.log(` โŒ Error: ${result.error}`); + } +} + +if (import.meta.main) { + runIBTests().catch(console.error); +} + +export { testIBWithWorking, testWithProxyFallback }; diff --git a/test-ib.ts b/test-ib.ts new file mode 100644 index 0000000..c9fed34 --- /dev/null +++ b/test-ib.ts @@ -0,0 +1,194 @@ +/** + * Test Interactive Brokers functionality with network monitoring + */ +import { Browser } from '@stock-bot/browser'; +import { getRandomProxyURL } from '@stock-bot/proxy'; + +async function testIBSymbolSummary() { + console.log('๐Ÿš€ Testing Interactive Brokers Symbol Summary with Network Monitoring...'); + + try { + // Initialize browser + await Browser.initialize({ + headless: true, + timeout: 30000, + blockResources: true, + enableNetworkLogging: true, + }); + + console.log('โœ… Browser initialized'); + + // Get a random proxy + + // Create page with proxy + const { page, contextId } = await Browser.createPageWithProxy( + 'https://www.interactivebrokers.com/webrest/search/product-types/summary', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + console.log('๐Ÿ“„ Page created with proxy'); + + // Set up network monitoring + let summaryResponse: any = null; + let requestCount = 0; + let responseCount = 0; + + page.onNetworkEvent(event => { + console.log(`๐Ÿ“ก Network Event: ${event.type} - ${event.method} ${event.url}`); + + if (event.type === 'request') { + requestCount++; + console.log(` ๐Ÿ“ค Request #${requestCount}: ${event.method} ${event.url}`); + + // Log request data for POST requests + if (event.requestData) { + console.log(` ๐Ÿ“ Request Data: ${event.requestData.substring(0, 200)}...`); + } + } + + if (event.type === 'response') { + responseCount++; + console.log(` ๐Ÿ“ฅ Response #${responseCount}: ${event.status} ${event.url}`); + + // Capture the summary response + if (event.url.includes('summary')) { + console.log(` ๐ŸŽฏ Found summary response!`); + summaryResponse = event.responseData; + + if (event.responseData) { + try { + const data = JSON.parse(event.responseData); + console.log(` ๐Ÿ“Š Summary Data: ${JSON.stringify(data, null, 2)}`); + } catch (e) { + console.log(` ๐Ÿ“Š Raw Response: ${event.responseData.substring(0, 500)}...`); + } + } + } + } + + if (event.type === 'failed') { + console.log(` โŒ Failed Request: ${event.url}`); + } + }); + + console.log('๐Ÿ” Network monitoring set up, waiting for page to load...'); + + // Wait for page to load and capture network activity + await page.waitForLoadState('domcontentloaded'); + console.log('โœ… Page loaded'); + + // Wait a bit more for any additional network requests + await new Promise(resolve => setTimeout(resolve, 3000)); + + console.log(`๐Ÿ“Š Network Summary:`); + console.log(` ๐Ÿ“ค Total Requests: ${requestCount}`); + console.log(` ๐Ÿ“ฅ Total Responses: ${responseCount}`); + + if (summaryResponse) { + console.log('โœ… Successfully captured summary response'); + try { + const parsed = JSON.parse(summaryResponse); + console.log(`๐Ÿ”ข Total symbols found: ${parsed?.data?.totalCount || 'Unknown'}`); + return parsed?.data?.totalCount || 0; + } catch (e) { + console.log('โš ๏ธ Could not parse response as JSON'); + return 1; // Indicate success but unknown count + } + } else { + console.log('โŒ No summary response captured'); + return 0; + } + + } catch (error) { + console.error('โŒ Test failed:', error); + + // Log more details about the error + if (error instanceof Error) { + console.error('Error details:', { + message: error.message, + stack: error.stack, + name: error.name + }); + } + + return -1; + } finally { + try { + await Browser.close(); + console.log('๐Ÿ”’ Browser closed'); + } catch (closeError) { + console.error('Error closing browser:', closeError); + } + } +} + +async function testWithDifferentProxy() { + console.log('\n๐Ÿ”„ Testing with different proxy configuration...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 15000, + blockResources: false, // Don't block resources for this test + }); + + // Test without proxy first + console.log('๐ŸŒ Testing without proxy...'); + const { page: pageNoProxy, contextId: contextNoProxy } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip' + ); + + pageNoProxy.onNetworkEvent(event => { + if (event.type === 'response' && event.url.includes('httpbin.org/ip')) { + console.log('๐Ÿ“ No proxy IP response:', event.responseData); + } + }); + + await pageNoProxy.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 1000)); + await Browser.closeContext(contextNoProxy); + + // Test with proxy + console.log('๐ŸŒ Testing with proxy...'); + const { page: pageWithProxy, contextId: contextWithProxy } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + pageWithProxy.onNetworkEvent(event => { + if (event.type === 'response' && event.url.includes('httpbin.org/ip')) { + console.log('๐Ÿ”„ Proxy IP response:', event.responseData); + } + }); + + await pageWithProxy.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 1000)); + await Browser.closeContext(contextWithProxy); + + } catch (error) { + console.error('โŒ Proxy test failed:', error); + } finally { + await Browser.close(); + } +} + +// Run the tests +async function runTests() { + console.log('๐Ÿงช Starting IB Network Monitoring Tests\n'); + + // Test 1: Main IB functionality + const result = await testIBSymbolSummary(); + console.log(`\n๐Ÿ Test Result: ${result}`); + + // Test 2: Proxy verification + await testWithDifferentProxy(); + + console.log('\nโœ… All tests completed!'); +} + +// Run if this file is executed directly +if (import.meta.main) { + runTests().catch(console.error); +} + +export { testIBSymbolSummary, testWithDifferentProxy }; diff --git a/test-network-debug.ts b/test-network-debug.ts new file mode 100644 index 0000000..d227bd5 --- /dev/null +++ b/test-network-debug.ts @@ -0,0 +1,135 @@ +/** + * Debug network monitoring setup + */ +import { Browser } from '@stock-bot/browser'; + +async function debugNetworkSetup() { + console.log('๐Ÿ› Debugging Network Monitoring Setup...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 10000, + blockResources: false, // Ensure we don't block requests + }); + + // Create page but don't navigate yet + const { page, contextId } = await Browser.createPageWithProxy( + '', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + let eventCount = 0; + console.log('๐Ÿ“ก Setting up network event listener...'); + + page.onNetworkEvent(event => { + eventCount++; + console.log(`๐Ÿ”” Event ${eventCount}: ${event.type} ${event.method} ${event.url}`); + console.log(` Headers: ${Object.keys(event.headers || {}).length} headers`); + + if (event.responseData) { + console.log(` Data: ${event.responseData.substring(0, 100)}...`); + } + }); + + console.log('๐ŸŒ Navigating to httpbin.org/headers...'); + await page.goto('https://httpbin.org/headers'); + + console.log('โณ Waiting for page load...'); + await page.waitForLoadState('domcontentloaded'); + + console.log('โณ Waiting additional time for network events...'); + await new Promise(resolve => setTimeout(resolve, 3000)); + + console.log(`๐Ÿ“Š Total events captured: ${eventCount}`); + + // Try to evaluate page content to see if it loaded + const title = await page.title(); + console.log(`๐Ÿ“„ Page title: "${title}"`); + + const bodyText = await page.locator('body').textContent(); + if (bodyText) { + console.log(`๐Ÿ“ Page content (first 200 chars): ${bodyText.substring(0, 200)}...`); + } + + await Browser.closeContext(contextId); + return eventCount > 0; + } catch (error) { + console.error('โŒ Debug test failed:', error); + return false; + } finally { + await Browser.close(); + } +} + +async function testManualNetworkCall() { + console.log('\n๐Ÿ”ง Testing with manual fetch call...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 10000, + blockResources: false, + }); + + const { page, contextId } = await Browser.createPageWithProxy( + 'https://www.interactivebrokers.com/webrest/search/product-types/summary', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + let eventCount = 0; + page.onNetworkEvent(event => { + eventCount++; + console.log(`๐Ÿ“ก Manual test event ${eventCount}: ${event.type} ${event.method} ${event.url}`); + if (event.responseData && event.url.includes('httpbin')) { + console.log(` ๐Ÿ“Š Response: ${event.responseData}`); + } + }); + + // Navigate to a simple page first + await page.goto('data:text/html,

Test Page

'); + await page.waitForLoadState('domcontentloaded'); + + console.log('๐Ÿš€ Making manual fetch call...'); + // Make a fetch request from the page context + const result = await page.evaluate(async () => { + try { + const response = await fetch('https://httpbin.org/json'); + const data = await response.json(); + return { success: true, data }; + } catch (error) { + return { success: false, error: error.message }; + } + }); + + console.log('๐Ÿ“‹ Fetch result:', result); + + await new Promise(resolve => setTimeout(resolve, 2000)); + console.log(`๐Ÿ“Š Events from manual fetch: ${eventCount}`); + + await Browser.closeContext(contextId); + return eventCount > 0; + } catch (error) { + console.error('โŒ Manual test failed:', error); + return false; + } finally { + await Browser.close(); + } +} + +async function runDebugTests() { + console.log('๐Ÿš€ Network Monitoring Debug Tests\n'); + + const setupResult = await debugNetworkSetup(); + const manualResult = await testManualNetworkCall(); + + console.log(`\n๐Ÿ Results:`); + console.log(` ๐Ÿ”ง Setup test: ${setupResult ? 'โœ… EVENTS CAPTURED' : 'โŒ NO EVENTS'}`); + console.log(` ๐Ÿ“ก Manual test: ${manualResult ? 'โœ… EVENTS CAPTURED' : 'โŒ NO EVENTS'}`); +} + +if (import.meta.main) { + runDebugTests().catch(console.error); +} + +export { debugNetworkSetup, testManualNetworkCall }; diff --git a/test-network-monitoring.ts b/test-network-monitoring.ts new file mode 100644 index 0000000..60a3bba --- /dev/null +++ b/test-network-monitoring.ts @@ -0,0 +1,137 @@ +/** + * Simple test to verify network monitoring is working + */ +import { Browser } from '@stock-bot/browser'; + +async function testNetworkMonitoring() { + console.log('๐Ÿงช Testing Network Monitoring with httpbin.org...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 15000, + blockResources: false, // Don't block resources so we can see requests + }); + + console.log('โœ… Browser initialized'); + + // Test with a simple API that returns JSON + const { page, contextId } = await Browser.createPageWithProxy( + 'https://httpbin.org/json' + ); + + let capturedRequests = 0; + let capturedResponses = 0; + let jsonResponse = null; + + page.onNetworkEvent(event => { + console.log(`๐Ÿ“ก ${event.type.toUpperCase()}: ${event.method} ${event.url}`); + + if (event.type === 'request') { + capturedRequests++; + } + + if (event.type === 'response') { + capturedResponses++; + console.log(` Status: ${event.status}`); + + if (event.url.includes('httpbin.org/json') && event.responseData) { + jsonResponse = event.responseData; + console.log(` ๐Ÿ“Š JSON Response: ${event.responseData}`); + } + } + }); + + await page.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 2000)); + + console.log(`\n๐Ÿ“Š Summary:`); + console.log(` ๐Ÿ“ค Requests captured: ${capturedRequests}`); + console.log(` ๐Ÿ“ฅ Responses captured: ${capturedResponses}`); + console.log(` ๐Ÿ“ JSON data captured: ${jsonResponse ? 'Yes' : 'No'}`); + + await Browser.closeContext(contextId); + return true; + + } catch (error) { + console.error('โŒ Test failed:', error); + return false; + } finally { + await Browser.close(); + } +} + +async function testWithProxy() { + console.log('\n๐ŸŒ Testing with proxy to see IP change...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 10000, + blockResources: false, + }); + + // Test IP without proxy + console.log('๐Ÿ“ Getting IP without proxy...'); + const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip' + ); + + let ipWithoutProxy = null; + page1.onNetworkEvent(event => { + if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { + ipWithoutProxy = JSON.parse(event.responseData).origin; + console.log(` ๐Ÿ”น Your IP: ${ipWithoutProxy}`); + } + }); + + await page1.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 1000)); + await Browser.closeContext(ctx1); + + // Test IP with proxy + console.log('๐Ÿ”„ Getting IP with proxy...'); + const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + let ipWithProxy = null; + page2.onNetworkEvent(event => { + if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { + ipWithProxy = JSON.parse(event.responseData).origin; + console.log(` ๐Ÿ”ธ Proxy IP: ${ipWithProxy}`); + } + }); + + await page2.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 1000)); + await Browser.closeContext(ctx2); + + if (ipWithoutProxy && ipWithProxy && ipWithoutProxy !== ipWithProxy) { + console.log('โœ… Proxy is working - IPs are different!'); + } else { + console.log('โš ๏ธ Proxy may not be working - IPs are the same or not captured'); + } + + } catch (error) { + console.error('โŒ Proxy test failed:', error); + } finally { + await Browser.close(); + } +} + +async function runTests() { + console.log('๐Ÿš€ Network Monitoring Verification Tests\n'); + + const basicResult = await testNetworkMonitoring(); + await testWithProxy(); + + console.log(`\n๐Ÿ Network monitoring: ${basicResult ? 'โœ… WORKING' : 'โŒ FAILED'}`); +} + +if (import.meta.main) { + runTests().catch(console.error); +} + +export { testNetworkMonitoring, testWithProxy }; diff --git a/test-network.ts b/test-network.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-proxy-auth.ts b/test-proxy-auth.ts new file mode 100644 index 0000000..fc45846 --- /dev/null +++ b/test-proxy-auth.ts @@ -0,0 +1,156 @@ +/** + * Test Playwright proxy authentication specifically + */ +import { Browser } from '@stock-bot/browser'; + +async function testPlaywrightProxyAuth() { + console.log('๐Ÿ” Testing Playwright Proxy Authentication...'); + + try { + await Browser.initialize({ + headless: true, + timeout: 15000, + blockResources: false, + }); + + console.log('โœ… Browser initialized'); + + // Test 1: Without proxy + console.log('\n๐Ÿ“ Test 1: Without proxy'); + const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip' + ); + + let events1 = 0; + let ip1 = null; + page1.onNetworkEvent(event => { + events1++; + console.log(` ๐Ÿ“ก Event: ${event.type} ${event.url}`); + if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { + ip1 = JSON.parse(event.responseData).origin; + console.log(` ๐ŸŒ Your IP: ${ip1}`); + } + }); + + await page1.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 2000)); + await Browser.closeContext(ctx1); + console.log(` Events captured: ${events1}`); + + // Test 2: With proxy using new authentication method + console.log('\n๐Ÿ”’ Test 2: With proxy (new auth method)'); + const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + let events2 = 0; + let ip2 = null; + page2.onNetworkEvent(event => { + events2++; + console.log(` ๐Ÿ“ก Event: ${event.type} ${event.url}`); + if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { + ip2 = JSON.parse(event.responseData).origin; + console.log(` ๐Ÿ”„ Proxy IP: ${ip2}`); + } + }); + + await page2.waitForLoadState('domcontentloaded'); + await new Promise(resolve => setTimeout(resolve, 2000)); + await Browser.closeContext(ctx2); + console.log(` Events captured: ${events2}`); + + // Results + console.log('\n๐Ÿ“Š Results:'); + console.log(` ๐ŸŒ Direct IP: ${ip1 || 'Not captured'}`); + console.log(` ๐Ÿ”„ Proxy IP: ${ip2 || 'Not captured'}`); + console.log(` ๐Ÿ“ก Direct events: ${events1}`); + console.log(` ๐Ÿ“ก Proxy events: ${events2}`); + + if (ip1 && ip2 && ip1 !== ip2) { + console.log('โœ… Proxy authentication is working - different IPs detected!'); + return true; + } else if (events1 > 0 || events2 > 0) { + console.log('โš ๏ธ Network monitoring working, but proxy may not be changing IP'); + return true; + } else { + console.log('โŒ No network events captured'); + return false; + } + + } catch (error) { + console.error('โŒ Test failed:', error); + return false; + } finally { + await Browser.close(); + } +} + +async function testManualPageEvaluation() { + console.log('\n๐Ÿงช Test 3: Manual page evaluation (without network monitoring)'); + + try { + await Browser.initialize({ + headless: true, + timeout: 10000, + blockResources: false, + }); + + const { page, contextId } = await Browser.createPageWithProxy( + 'https://httpbin.org/ip', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + + // Try to get the page content directly + const title = await page.title(); + console.log(` ๐Ÿ“„ Page title: "${title}"`); + + // Try to evaluate some JavaScript + const result = await page.evaluate(() => { + return { + url: window.location.href, + userAgent: navigator.userAgent.substring(0, 50), + readyState: document.readyState, + }; + }); + + console.log(` ๐Ÿ” Page info:`, result); + + // Try to get page content + const bodyText = await page.locator('body').textContent(); + if (bodyText) { + console.log(` ๐Ÿ“ Body content (first 200 chars): ${bodyText.substring(0, 200)}...`); + + // Check if it looks like an IP response + if (bodyText.includes('origin')) { + console.log(' โœ… Looks like httpbin.org response!'); + } + } + + await Browser.closeContext(contextId); + return true; + + } catch (error) { + console.error(' โŒ Manual evaluation failed:', error); + return false; + } finally { + await Browser.close(); + } +} + +async function runProxyTests() { + console.log('๐Ÿš€ Playwright Proxy Authentication Tests\n'); + + const authResult = await testPlaywrightProxyAuth(); + const manualResult = await testManualPageEvaluation(); + + console.log(`\n๐Ÿ Final Results:`); + console.log(` ๐Ÿ” Proxy auth test: ${authResult ? 'โœ… PASS' : 'โŒ FAIL'}`); + console.log(` ๐Ÿงช Manual eval test: ${manualResult ? 'โœ… PASS' : 'โŒ FAIL'}`); +} + +if (import.meta.main) { + runProxyTests().catch(console.error); +} + +export { testPlaywrightProxyAuth, testManualPageEvaluation }; diff --git a/test-proxy.ts b/test-proxy.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-simple-proxy.ts b/test-simple-proxy.ts new file mode 100644 index 0000000..8a9eafc --- /dev/null +++ b/test-simple-proxy.ts @@ -0,0 +1,151 @@ +import { Browser } from '@stock-bot/browser'; + +async function simpleProxyTest() { + console.log('๐Ÿ”ฌ Simple Proxy Test...'); + + try { + await Browser.initialize({ headless: true, timeout: 10000, blockResources: false }); + console.log('โœ… Browser initialized'); + + const { page, contextId } = await Browser.createPageWithProxy( + 'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/', + 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' + ); + console.log('โœ… Page created with proxy'); + let summaryData: any = null; + let eventCount = 0; + page.onNetworkEvent(event => { + // Capture the summary API response + if (event.url.includes('/webrest/search/product-types/summary')) { + console.log(`๐ŸŽฏ Found summary API call: ${event.type} ${event.url}`); + + if (event.type === 'response' && event.responseData) { + console.log(`๐Ÿ“Š Summary API Response Data: ${event.responseData}`); + try { + summaryData = JSON.parse(event.responseData) as any; + const totalCount = summaryData[0].totalCount; + console.log('๐Ÿ“Š Summary API Response:', JSON.stringify(summaryData, null, 2)); + console.log(`๐Ÿ”ข Total symbols found: ${totalCount || 'Unknown'}`); + } catch (e) { + console.log('๐Ÿ“Š Raw Summary Response:', event.responseData); + } + } + } + eventCount++; + console.log(`๐Ÿ“ก Event ${eventCount}: ${event.type} ${event.url}`); + }); + + console.log('โณ Waiting for page load...'); + await page.waitForLoadState('domcontentloaded', { timeout: 8000 }); + console.log('โœ… Page loaded'); + + // RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button + try { + console.log('๐Ÿ” Looking for Products tab...'); + + // Wait for the page to fully load + await page.waitForTimeout(3000); + + // First, click on the Products tab + const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); + await productsTab.waitFor({ timeout: 10000 }); + console.log('โœ… Found Products tab'); + + console.log('๐Ÿ–ฑ๏ธ Clicking Products tab...'); + await productsTab.click(); + console.log('โœ… Products tab clicked'); + + // Wait for the tab content to load + await page.waitForTimeout(2000); + + // Click on the Asset Classes accordion to expand it + console.log('๐Ÿ” Looking for Asset Classes accordion...'); + const assetClassesAccordion = page.locator( + '#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")' + ); + await assetClassesAccordion.waitFor({ timeout: 10000 }); + console.log('โœ… Found Asset Classes accordion'); + + console.log('๐Ÿ–ฑ๏ธ Clicking Asset Classes accordion...'); + await assetClassesAccordion.click(); + console.log('โœ… Asset Classes accordion clicked'); + + // Wait for the accordion content to expand + await page.waitForTimeout(2000); + + console.log('๐Ÿ” Looking for Stocks checkbox...'); + + // Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks" + const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")'); + await stocksSpan.waitFor({ timeout: 10000 }); + console.log('โœ… Found Stocks span'); + + // Find the checkbox by looking in the same parent container + const parentContainer = stocksSpan.locator('..'); + const checkbox = parentContainer.locator('input[type="checkbox"]'); + + if ((await checkbox.count()) > 0) { + console.log('๐Ÿ“‹ Clicking Stocks checkbox...'); + await checkbox.first().check(); + console.log('โœ… Stocks checkbox checked'); + } else { + console.log('โš ๏ธ Could not find checkbox near Stocks text'); + } + + // Wait a moment for any UI updates + await page.waitForTimeout(1000); + + // Find and click the nearest Apply button + console.log('๐Ÿ” Looking for Apply button...'); + const applyButton = page.locator( + 'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]' + ); + + if ((await applyButton.count()) > 0) { + console.log('๐ŸŽฏ Clicking Apply button...'); + await applyButton.first().click(); + console.log('โœ… Apply button clicked'); + + // Wait for any network requests triggered by the Apply button + await page.waitForTimeout(2000); + } else { + console.log('โš ๏ธ Could not find Apply button'); + } + } catch (interactionError) { + const errorMessage = + interactionError instanceof Error ? interactionError.message : String(interactionError); + console.error('โŒ Page interaction failed:', errorMessage); + + // Get debug info about the page + try { + const title = await page.title(); + console.log(`๐Ÿ“„ Current page title: "${title}"`); + + const stocksElements = await page.locator('*:has-text("Stocks")').count(); + console.log(`๐Ÿ” Found ${stocksElements} elements containing "Stocks"`); + + const applyButtons = await page + .locator('button:has-text("Apply"), input[value*="Apply"]') + .count(); + console.log(`๐Ÿ” Found ${applyButtons} Apply buttons`); + } catch (debugError) { + const debugMessage = debugError instanceof Error ? debugError.message : String(debugError); + console.log('โŒ Could not get debug info:', debugMessage); + } + } + + await new Promise(resolve => setTimeout(resolve, 2000)); + console.log(`๐Ÿ“Š Total events: ${eventCount}`); + + await Browser.closeContext(contextId); + await Browser.close(); + + console.log('โœ… Test completed'); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error('โŒ Error:', errorMessage); + await Browser.close(); + } +} + +simpleProxyTest(); diff --git a/test-simple.ts b/test-simple.ts new file mode 100644 index 0000000..a9de60f --- /dev/null +++ b/test-simple.ts @@ -0,0 +1 @@ +console.log("Testing browser import..."); import { Browser } from "@stock-bot/browser"; console.log("Browser imported successfully:", typeof Browser); Browser.initialize().then(() => console.log("Browser initialized")).catch(e => console.error("Error:", e)); diff --git a/test-user-agent.js b/test-user-agent.js new file mode 100644 index 0000000..e69de29