refactor of data-service
This commit is contained in:
parent
3097686849
commit
54314a0cde
49 changed files with 2394 additions and 112 deletions
2
.env
2
.env
|
|
@ -13,6 +13,8 @@ DATA_SERVICE_PORT=2001
|
|||
WORKER_COUNT=4
|
||||
WORKER_CONCURRENCY=20
|
||||
|
||||
WEBSHARE_API_KEY=y8ay534rcbybdkk3evnzmt640xxfhy7252ce2t98
|
||||
|
||||
# ===========================================
|
||||
# DATABASE CONFIGURATIONS
|
||||
# ===========================================
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@
|
|||
* Data Service - Combined live and historical data ingestion with queue-based architecture
|
||||
*/
|
||||
import { Hono } from 'hono';
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
import { loadEnvVariables } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { Shutdown } from '@stock-bot/shutdown';
|
||||
import { initializeProxyCache } from './providers/proxy.tasks';
|
||||
import { initializeIBResources } from './providers/ib.tasks';
|
||||
import { initializeProxyResources } from './providers/proxy.tasks';
|
||||
import { queueManager } from './services/queue.service';
|
||||
import { initializeBatchCache } from './utils/batch-helpers';
|
||||
import { healthRoutes, marketDataRoutes, proxyRoutes, queueRoutes, testRoutes } from './routes';
|
||||
|
|
@ -33,6 +35,11 @@ async function initializeServices() {
|
|||
logger.info('Initializing data service...');
|
||||
|
||||
try {
|
||||
// Initialize browser resources
|
||||
logger.info('Starting browser resources initialization...');
|
||||
await Browser.initialize();
|
||||
logger.info('Browser resources initialized');
|
||||
|
||||
// Initialize batch cache FIRST - before queue service
|
||||
logger.info('Starting batch cache initialization...');
|
||||
await initializeBatchCache();
|
||||
|
|
@ -40,7 +47,12 @@ async function initializeServices() {
|
|||
|
||||
// Initialize proxy cache - before queue service
|
||||
logger.info('Starting proxy cache initialization...');
|
||||
await initializeProxyCache();
|
||||
await initializeProxyResources(true); // Wait for cache during startup
|
||||
logger.info('Proxy cache initialized');
|
||||
|
||||
// Initialize proxy cache - before queue service
|
||||
logger.info('Starting proxy cache initialization...');
|
||||
await initializeIBResources(true); // Wait for cache during startup
|
||||
logger.info('Proxy cache initialized');
|
||||
|
||||
// Initialize queue service (Redis connections should be ready now)
|
||||
|
|
|
|||
32
apps/data-service/src/providers/ib.provider.ts
Normal file
32
apps/data-service/src/providers/ib.provider.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
|
||||
const logger = getLogger('ib-provider');
|
||||
|
||||
export const ibProvider: ProviderConfig = {
|
||||
name: 'ib',
|
||||
operations: {
|
||||
'ib-symbol-summary': async () => {
|
||||
const { ibTasks } = await import('./ib.tasks');
|
||||
logger.info('Fetching symbol summary from IB');
|
||||
const total = await ibTasks.fetchSymbolSummary();
|
||||
logger.info('Fetched symbol summary from IB', {
|
||||
count: total,
|
||||
});
|
||||
return total;
|
||||
},
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'ib-symbol-summary',
|
||||
operation: 'ib-symbol-summary',
|
||||
payload: {},
|
||||
// should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
cronPattern: '*/2 * * * *',
|
||||
priority: 5,
|
||||
immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
description: 'Fetch and validate proxy list from sources',
|
||||
},
|
||||
],
|
||||
};
|
||||
152
apps/data-service/src/providers/ib.tasks.ts
Normal file
152
apps/data-service/src/providers/ib.tasks.ts
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
|
||||
// Shared instances (module-scoped, not global)
|
||||
let isInitialized = false; // Track if resources are initialized
|
||||
let logger: ReturnType<typeof getLogger>;
|
||||
// let cache: CacheProvider;
|
||||
|
||||
export async function initializeIBResources(waitForCache = false): Promise<void> {
|
||||
// Skip if already initialized
|
||||
if (isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger = getLogger('proxy-tasks');
|
||||
// cache = createCache({
|
||||
// keyPrefix: 'proxy:',
|
||||
// ttl: PROXY_CONFIG.CACHE_TTL,
|
||||
// enableMetrics: true,
|
||||
// });
|
||||
|
||||
// httpClient = new HttpClient({ timeout: 15000 }, logger);
|
||||
|
||||
// if (waitForCache) {
|
||||
// // logger.info('Initializing proxy cache...');
|
||||
// // await cache.waitForReady(10000);
|
||||
// // logger.info('Proxy cache initialized successfully');
|
||||
// logger.info('Proxy tasks initialized');
|
||||
// } else {
|
||||
// logger.info('Proxy tasks initialized (fallback mode)');
|
||||
// }
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
export async function fetchSymbolSummary(): Promise<number> {
|
||||
try {
|
||||
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
|
||||
logger.info('✅ Browser initialized');
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
logger.info('✅ Page created with proxy');
|
||||
let summaryData: any = null; // Initialize summaryData to store API response
|
||||
let eventCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.responseData) {
|
||||
console.log(`📊 Summary API Response Data: ${event.responseData}`);
|
||||
try {
|
||||
summaryData = JSON.parse(event.responseData) as any;
|
||||
const totalCount = summaryData[0].totalCount;
|
||||
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
|
||||
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
|
||||
} catch (e) {
|
||||
console.log('📊 Raw Summary Response:', event.responseData);
|
||||
}
|
||||
}
|
||||
}
|
||||
eventCount++;
|
||||
logger.info(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
|
||||
});
|
||||
|
||||
logger.info('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 20000 });
|
||||
logger.info('✅ Page loaded');
|
||||
|
||||
// RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button
|
||||
logger.info('🔍 Looking for Products tab...');
|
||||
|
||||
// Wait for the page to fully load
|
||||
await page.waitForTimeout(20000);
|
||||
|
||||
// First, click on the Products tab
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: 20000 });
|
||||
logger.info('✅ Found Products tab');
|
||||
|
||||
logger.info('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
logger.info('✅ Products tab clicked');
|
||||
|
||||
// Wait for the tab content to load
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
// Click on the Asset Classes accordion to expand it
|
||||
logger.info('🔍 Looking for Asset Classes accordion...');
|
||||
const assetClassesAccordion = page.locator(
|
||||
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
|
||||
);
|
||||
await assetClassesAccordion.waitFor({ timeout: 10000 });
|
||||
logger.info('✅ Found Asset Classes accordion');
|
||||
|
||||
logger.info('🖱️ Clicking Asset Classes accordion...');
|
||||
await assetClassesAccordion.click();
|
||||
logger.info('✅ Asset Classes accordion clicked');
|
||||
|
||||
// Wait for the accordion content to expand
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
logger.info('🔍 Looking for Stocks checkbox...');
|
||||
|
||||
// Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks"
|
||||
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
|
||||
await stocksSpan.waitFor({ timeout: 10000 });
|
||||
logger.info('✅ Found Stocks span');
|
||||
|
||||
// Find the checkbox by looking in the same parent container
|
||||
const parentContainer = stocksSpan.locator('..');
|
||||
const checkbox = parentContainer.locator('input[type="checkbox"]');
|
||||
|
||||
if ((await checkbox.count()) > 0) {
|
||||
logger.info('📋 Clicking Stocks checkbox...');
|
||||
await checkbox.first().check();
|
||||
logger.info('✅ Stocks checkbox checked');
|
||||
} else {
|
||||
logger.info('⚠️ Could not find checkbox near Stocks text');
|
||||
}
|
||||
|
||||
// Wait a moment for any UI updates
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Find and click the nearest Apply button
|
||||
logger.info('🔍 Looking for Apply button...');
|
||||
const applyButton = page.locator(
|
||||
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
|
||||
);
|
||||
|
||||
if ((await applyButton.count()) > 0) {
|
||||
logger.info('🎯 Clicking Apply button...');
|
||||
await applyButton.first().click();
|
||||
logger.info('✅ Apply button clicked');
|
||||
|
||||
// Wait for any network requests triggered by the Apply button
|
||||
await page.waitForTimeout(2000);
|
||||
} else {
|
||||
logger.info('⚠️ Could not find Apply button');
|
||||
}
|
||||
|
||||
return 0;
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch IB symbol summary', { error });
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
// Optional: Export a convenience object that groups related tasks
|
||||
export const ibTasks = {
|
||||
fetchSymbolSummary,
|
||||
};
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
import pLimit from 'p-limit';
|
||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
|
|
@ -22,7 +21,6 @@ const PROXY_CONFIG = {
|
|||
CHECK_TIMEOUT: 7000,
|
||||
CHECK_IP: '99.246.102.205',
|
||||
CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
|
||||
CONCURRENCY_LIMIT: 100,
|
||||
PROXY_SOURCES: [
|
||||
{
|
||||
id: 'prxchk',
|
||||
|
|
@ -154,10 +152,10 @@ const PROXY_CONFIG = {
|
|||
};
|
||||
|
||||
// Shared instances (module-scoped, not global)
|
||||
let isInitialized = false; // Track if resources are initialized
|
||||
let logger: ReturnType<typeof getLogger>;
|
||||
let cache: CacheProvider;
|
||||
let httpClient: HttpClient;
|
||||
let concurrencyLimit: ReturnType<typeof pLimit>;
|
||||
let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
|
||||
id: source.id,
|
||||
total: 0,
|
||||
|
|
@ -167,6 +165,37 @@ let proxyStats: ProxySource[] = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
|
|||
url: source.url,
|
||||
}));
|
||||
|
||||
/**
|
||||
* Initialize proxy resources (cache and shared dependencies)
|
||||
* This should be called before any proxy operations
|
||||
* @param waitForCache - Whether to wait for cache readiness (default: false for fallback mode)
|
||||
*/
|
||||
export async function initializeProxyResources(waitForCache = false): Promise<void> {
|
||||
// Skip if already initialized
|
||||
if (isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger = getLogger('proxy-tasks');
|
||||
cache = createCache({
|
||||
keyPrefix: 'proxy:',
|
||||
ttl: PROXY_CONFIG.CACHE_TTL,
|
||||
enableMetrics: true,
|
||||
});
|
||||
|
||||
httpClient = new HttpClient({ timeout: 10000 }, logger);
|
||||
|
||||
if (waitForCache) {
|
||||
logger.info('Initializing proxy cache...');
|
||||
await cache.waitForReady(10000);
|
||||
logger.info('Proxy cache initialized successfully');
|
||||
logger.info('Proxy tasks initialized');
|
||||
} else {
|
||||
logger.info('Proxy tasks initialized (fallback mode)');
|
||||
}
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
// make a function that takes in source id and a boolean success and updates the proxyStats array
|
||||
async function updateProxyStats(sourceId: string, success: boolean) {
|
||||
const source = proxyStats.find(s => s.id === sourceId);
|
||||
|
|
@ -278,50 +307,8 @@ async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean): Promise
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize proxy cache for use during application startup
|
||||
* This should be called before any proxy operations
|
||||
*/
|
||||
export async function initializeProxyCache(): Promise<void> {
|
||||
logger = getLogger('proxy-tasks');
|
||||
cache = createCache({
|
||||
keyPrefix: 'proxy:',
|
||||
ttl: PROXY_CONFIG.CACHE_TTL,
|
||||
enableMetrics: true,
|
||||
});
|
||||
|
||||
logger.info('Initializing proxy cache...');
|
||||
await cache.waitForReady(10000);
|
||||
logger.info('Proxy cache initialized successfully');
|
||||
|
||||
// Initialize other shared resources that don't require cache
|
||||
httpClient = new HttpClient({ timeout: 10000 }, logger);
|
||||
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
|
||||
|
||||
logger.info('Proxy tasks initialized');
|
||||
}
|
||||
|
||||
async function initializeSharedResources() {
|
||||
if (!logger) {
|
||||
// If not initialized at startup, initialize with fallback mode
|
||||
logger = getLogger('proxy-tasks');
|
||||
cache = createCache({
|
||||
keyPrefix: 'proxy:',
|
||||
ttl: PROXY_CONFIG.CACHE_TTL,
|
||||
enableMetrics: true,
|
||||
});
|
||||
|
||||
httpClient = new HttpClient({ timeout: 10000 }, logger);
|
||||
concurrencyLimit = pLimit(PROXY_CONFIG.CONCURRENCY_LIMIT);
|
||||
|
||||
logger.info('Proxy tasks initialized (fallback mode)');
|
||||
}
|
||||
}
|
||||
|
||||
// Individual task functions
|
||||
export async function queueProxyFetch(): Promise<string> {
|
||||
await initializeSharedResources();
|
||||
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
const job = await queueManager.addJob({
|
||||
type: 'proxy-fetch',
|
||||
|
|
@ -337,8 +324,6 @@ export async function queueProxyFetch(): Promise<string> {
|
|||
}
|
||||
|
||||
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
|
||||
await initializeSharedResources();
|
||||
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
const job = await queueManager.addJob({
|
||||
type: 'proxy-check',
|
||||
|
|
@ -354,35 +339,15 @@ export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
|
|||
}
|
||||
|
||||
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
|
||||
await initializeSharedResources();
|
||||
await resetProxyStats();
|
||||
|
||||
// Ensure concurrencyLimit is available before using it
|
||||
if (!concurrencyLimit) {
|
||||
logger.error('concurrencyLimit not initialized, using sequential processing');
|
||||
const result = [];
|
||||
for (const source of PROXY_CONFIG.PROXY_SOURCES) {
|
||||
const proxies = await fetchProxiesFromSource(source);
|
||||
result.push(...proxies);
|
||||
}
|
||||
let allProxies: ProxyInfo[] = result;
|
||||
allProxies = removeDuplicateProxies(allProxies);
|
||||
return allProxies;
|
||||
}
|
||||
|
||||
const sources = PROXY_CONFIG.PROXY_SOURCES.map(source =>
|
||||
concurrencyLimit(() => fetchProxiesFromSource(source))
|
||||
);
|
||||
const result = await Promise.all(sources);
|
||||
let allProxies: ProxyInfo[] = result.flat();
|
||||
const fetchPromises = PROXY_CONFIG.PROXY_SOURCES.map(source => fetchProxiesFromSource(source));
|
||||
const results = await Promise.all(fetchPromises);
|
||||
let allProxies: ProxyInfo[] = results.flat();
|
||||
allProxies = removeDuplicateProxies(allProxies);
|
||||
// await checkProxies(allProxies);
|
||||
return allProxies;
|
||||
}
|
||||
|
||||
export async function fetchProxiesFromSource(source: ProxySource): Promise<ProxyInfo[]> {
|
||||
await initializeSharedResources();
|
||||
|
||||
const allProxies: ProxyInfo[] = [];
|
||||
|
||||
try {
|
||||
|
|
@ -436,8 +401,6 @@ export async function fetchProxiesFromSource(source: ProxySource): Promise<Proxy
|
|||
* Check if a proxy is working
|
||||
*/
|
||||
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||
await initializeSharedResources();
|
||||
|
||||
let success = false;
|
||||
logger.debug(`Checking Proxy:`, {
|
||||
protocol: proxy.protocol,
|
||||
|
|
@ -504,6 +467,76 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a random active proxy from the cache
|
||||
* @param protocol - Optional protocol filter ('http' | 'https' | 'socks4' | 'socks5')
|
||||
* @param minSuccessRate - Minimum success rate percentage (default: 50)
|
||||
* @returns A random working proxy or null if none found
|
||||
*/
|
||||
export async function getRandomActiveProxy(
|
||||
protocol?: 'http' | 'https' | 'socks4' | 'socks5',
|
||||
minSuccessRate: number = 50
|
||||
): Promise<ProxyInfo | null> {
|
||||
try {
|
||||
// Get all active proxy keys from cache
|
||||
const pattern = protocol
|
||||
? `${PROXY_CONFIG.CACHE_KEY}:${protocol}://*`
|
||||
: `${PROXY_CONFIG.CACHE_KEY}:*`;
|
||||
|
||||
const keys = await cache.keys(pattern);
|
||||
|
||||
if (keys.length === 0) {
|
||||
logger.debug('No active proxies found in cache', { pattern });
|
||||
return null;
|
||||
}
|
||||
|
||||
// Shuffle the keys for randomness
|
||||
const shuffledKeys = keys.sort(() => Math.random() - 0.5);
|
||||
|
||||
// Find a working proxy that meets the criteria
|
||||
for (const key of shuffledKeys) {
|
||||
try {
|
||||
const proxyData: ProxyInfo | null = await cache.get(key);
|
||||
|
||||
if (
|
||||
proxyData &&
|
||||
proxyData.isWorking &&
|
||||
(!proxyData.successRate || proxyData.successRate >= minSuccessRate)
|
||||
) {
|
||||
logger.debug('Random active proxy selected', {
|
||||
proxy: `${proxyData.host}:${proxyData.port}`,
|
||||
protocol: proxyData.protocol,
|
||||
successRate: proxyData.successRate?.toFixed(1) + '%',
|
||||
avgResponseTime: proxyData.averageResponseTime
|
||||
? `${proxyData.averageResponseTime.toFixed(0)}ms`
|
||||
: 'N/A',
|
||||
});
|
||||
|
||||
return proxyData;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.debug('Error reading proxy from cache', { key, error: (error as Error).message });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug('No working proxies found meeting criteria', {
|
||||
protocol,
|
||||
minSuccessRate,
|
||||
keysChecked: shuffledKeys.length,
|
||||
});
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
logger.error('Error getting random active proxy', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
protocol,
|
||||
minSuccessRate,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Utility functions
|
||||
function cleanProxyUrl(url: string): string {
|
||||
return url
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
|
||||
const logger = getLogger('quotemedia-provider');
|
||||
const logger = getLogger('qm-provider');
|
||||
|
||||
export const quotemediaProvider: ProviderConfig = {
|
||||
name: 'quotemedia',
|
||||
export const qmProvider: ProviderConfig = {
|
||||
name: 'qm',
|
||||
operations: {
|
||||
'live-data': async (payload: { symbol: string; fields?: string[] }) => {
|
||||
logger.info('Fetching live data from QuoteMedia', { symbol: payload.symbol });
|
||||
logger.info('Fetching live data from qm', { symbol: payload.symbol });
|
||||
|
||||
// Simulate QuoteMedia API call
|
||||
// Simulate qm API call
|
||||
const mockData = {
|
||||
symbol: payload.symbol,
|
||||
price: Math.random() * 1000 + 100,
|
||||
|
|
@ -17,7 +17,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
change: (Math.random() - 0.5) * 20,
|
||||
changePercent: (Math.random() - 0.5) * 5,
|
||||
timestamp: new Date().toISOString(),
|
||||
source: 'quotemedia',
|
||||
source: 'qm',
|
||||
fields: payload.fields || ['price', 'volume', 'change'],
|
||||
};
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
interval?: string;
|
||||
fields?: string[];
|
||||
}) => {
|
||||
logger.info('Fetching historical data from QuoteMedia', {
|
||||
logger.info('Fetching historical data from qm', {
|
||||
symbol: payload.symbol,
|
||||
from: payload.from,
|
||||
to: payload.to,
|
||||
|
|
@ -56,7 +56,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
low: Math.random() * 1000 + 100,
|
||||
close: Math.random() * 1000 + 100,
|
||||
volume: Math.floor(Math.random() * 1000000),
|
||||
source: 'quotemedia',
|
||||
source: 'qm',
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -67,12 +67,12 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
symbol: payload.symbol,
|
||||
interval: payload.interval || '1d',
|
||||
data,
|
||||
source: 'quotemedia',
|
||||
source: 'qm',
|
||||
totalRecords: data.length,
|
||||
};
|
||||
},
|
||||
'batch-quotes': async (payload: { symbols: string[]; fields?: string[] }) => {
|
||||
logger.info('Fetching batch quotes from QuoteMedia', {
|
||||
logger.info('Fetching batch quotes from qm', {
|
||||
symbols: payload.symbols,
|
||||
count: payload.symbols.length,
|
||||
});
|
||||
|
|
@ -83,7 +83,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
volume: Math.floor(Math.random() * 1000000),
|
||||
change: (Math.random() - 0.5) * 20,
|
||||
timestamp: new Date().toISOString(),
|
||||
source: 'quotemedia',
|
||||
source: 'qm',
|
||||
}));
|
||||
|
||||
// Simulate network delay
|
||||
|
|
@ -91,13 +91,13 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
|
||||
return {
|
||||
quotes,
|
||||
source: 'quotemedia',
|
||||
source: 'qm',
|
||||
timestamp: new Date().toISOString(),
|
||||
totalSymbols: payload.symbols.length,
|
||||
};
|
||||
},
|
||||
'company-profile': async (payload: { symbol: string }) => {
|
||||
logger.info('Fetching company profile from QuoteMedia', { symbol: payload.symbol });
|
||||
logger.info('Fetching company profile from qm', { symbol: payload.symbol });
|
||||
|
||||
// Simulate company profile data
|
||||
const profile = {
|
||||
|
|
@ -109,7 +109,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
marketCap: Math.floor(Math.random() * 1000000000000),
|
||||
employees: Math.floor(Math.random() * 100000),
|
||||
website: `https://www.${payload.symbol.toLowerCase()}.com`,
|
||||
source: 'quotemedia',
|
||||
source: 'qm',
|
||||
};
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 150 + Math.random() * 100));
|
||||
|
|
@ -117,7 +117,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
return profile;
|
||||
},
|
||||
'options-chain': async (payload: { symbol: string; expiration?: string }) => {
|
||||
logger.info('Fetching options chain from QuoteMedia', {
|
||||
logger.info('Fetching options chain from qm', {
|
||||
symbol: payload.symbol,
|
||||
expiration: payload.expiration,
|
||||
});
|
||||
|
|
@ -148,14 +148,14 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0],
|
||||
calls,
|
||||
puts,
|
||||
source: 'quotemedia',
|
||||
source: 'qm',
|
||||
};
|
||||
},
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
// {
|
||||
// type: 'quotemedia-premium-refresh',
|
||||
// type: 'qm-premium-refresh',
|
||||
// operation: 'batch-quotes',
|
||||
// payload: { symbols: ['AAPL', 'GOOGL', 'MSFT'] },
|
||||
// cronPattern: '*/2 * * * *', // Every 2 minutes
|
||||
|
|
@ -163,7 +163,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
// description: 'Refresh premium quotes with detailed market data'
|
||||
// },
|
||||
// {
|
||||
// type: 'quotemedia-options-update',
|
||||
// type: 'qm-options-update',
|
||||
// operation: 'options-chain',
|
||||
// payload: { symbol: 'SPY' },
|
||||
// cronPattern: '*/10 * * * *', // Every 10 minutes
|
||||
|
|
@ -171,7 +171,7 @@ export const quotemediaProvider: ProviderConfig = {
|
|||
// description: 'Update options chain data for SPY ETF'
|
||||
// },
|
||||
// {
|
||||
// type: 'quotemedia-profiles',
|
||||
// type: 'qm-profiles',
|
||||
// operation: 'company-profile',
|
||||
// payload: { symbol: 'AAPL' },
|
||||
// cronPattern: '0 9 * * 1-5', // Weekdays at 9 AM
|
||||
|
|
@ -154,8 +154,8 @@ export class QueueService {
|
|||
// Define providers to register
|
||||
const providers = [
|
||||
{ module: '../providers/proxy.provider', export: 'proxyProvider' },
|
||||
{ module: '../providers/quotemedia.provider', export: 'quotemediaProvider' },
|
||||
{ module: '../providers/yahoo.provider', export: 'yahooProvider' },
|
||||
{ module: '../providers/ib.provider', export: 'ibProvider' },
|
||||
// { module: '../providers/yahoo.provider', export: 'yahooProvider' },
|
||||
];
|
||||
|
||||
// Import and register all providers
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@
|
|||
{ "path": "../../libs/questdb-client" },
|
||||
{ "path": "../../libs/mongodb-client" },
|
||||
{ "path": "../../libs/event-bus" },
|
||||
{ "path": "../../libs/shutdown" }
|
||||
{ "path": "../../libs/shutdown" },
|
||||
{ "path": "../../libs/utils" },
|
||||
{ "path": "../../libs/browser" }
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@
|
|||
"@stock-bot/logger#build",
|
||||
"@stock-bot/mongodb-client#build",
|
||||
"@stock-bot/questdb-client#build",
|
||||
"@stock-bot/shutdown#build"
|
||||
"@stock-bot/shutdown#build",
|
||||
"@stock-bot/browser#build"
|
||||
],
|
||||
"outputs": ["dist/**"],
|
||||
"inputs": [
|
||||
|
|
|
|||
21
bun.lock
21
bun.lock
|
|
@ -6,6 +6,7 @@
|
|||
"dependencies": {
|
||||
"bullmq": "^5.53.2",
|
||||
"ioredis": "^5.6.1",
|
||||
"playwright": "^1.53.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.28.0",
|
||||
|
|
@ -243,9 +244,11 @@
|
|||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"user-agents": "^1.1.567",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.0",
|
||||
"@types/user-agents": "^1.0.4",
|
||||
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
||||
"@typescript-eslint/parser": "^6.19.0",
|
||||
"bun-types": "^1.2.15",
|
||||
|
|
@ -930,6 +933,8 @@
|
|||
|
||||
"@types/supertest": ["@types/supertest@6.0.3", "", { "dependencies": { "@types/methods": "^1.1.4", "@types/superagent": "^8.1.0" } }, "sha512-8WzXq62EXFhJ7QsH3Ocb/iKQ/Ty9ZVWnVzoTKc9tyyFRRF3a74Tk2+TLFgaFFw364Ere+npzHKEJ6ga2LzIL7w=="],
|
||||
|
||||
"@types/user-agents": ["@types/user-agents@1.0.4", "", {}, "sha512-AjeFc4oX5WPPflgKfRWWJfkEk7Wu82fnj1rROPsiqFt6yElpdGFg8Srtm/4PU4rA9UiDUZlruGPgcwTMQlwq4w=="],
|
||||
|
||||
"@types/webidl-conversions": ["@types/webidl-conversions@7.0.3", "", {}, "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA=="],
|
||||
|
||||
"@types/whatwg-url": ["@types/whatwg-url@11.0.5", "", { "dependencies": { "@types/webidl-conversions": "*" } }, "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ=="],
|
||||
|
|
@ -1386,7 +1391,7 @@
|
|||
|
||||
"fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
"fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="],
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
|
||||
|
|
@ -1656,6 +1661,8 @@
|
|||
|
||||
"lodash.camelcase": ["lodash.camelcase@4.3.0", "", {}, "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="],
|
||||
|
||||
"lodash.clonedeep": ["lodash.clonedeep@4.5.0", "", {}, "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ=="],
|
||||
|
||||
"lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="],
|
||||
|
||||
"lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="],
|
||||
|
|
@ -1904,6 +1911,10 @@
|
|||
|
||||
"pkg-dir": ["pkg-dir@4.2.0", "", { "dependencies": { "find-up": "^4.0.0" } }, "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ=="],
|
||||
|
||||
"playwright": ["playwright@1.53.0", "", { "dependencies": { "playwright-core": "1.53.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-ghGNnIEYZC4E+YtclRn4/p6oYbdPiASELBIYkBXfaTVKreQUYbMUYQDwS12a8F0/HtIjr/CkGjtwABeFPGcS4Q=="],
|
||||
|
||||
"playwright-core": ["playwright-core@1.53.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-mGLg8m0pm4+mmtB7M89Xw/GSqoNC+twivl8ITteqvAndachozYe2ZA7srU6uleV1vEdAHYqjq+SV8SNxRRFYBw=="],
|
||||
|
||||
"possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="],
|
||||
|
||||
"postcss": ["postcss@8.5.4", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-QSa9EBe+uwlGTFmHsPKokv3B/oEMQZxfqW0QqNCyhpa6mB1afzulwn8hihglqAb2pOw+BJgNlmXQ8la2VeHB7w=="],
|
||||
|
|
@ -2240,6 +2251,8 @@
|
|||
|
||||
"uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="],
|
||||
|
||||
"user-agents": ["user-agents@1.1.567", "", { "dependencies": { "lodash.clonedeep": "^4.5.0" } }, "sha512-K5HqPZNWYbgd5sBUnvR7Aj2qt1jPCIAHaFbjF7uVyLD6nuMVGoW+eIrmQiqSFt/u2cZUXXI44rz6Y742KN/45Q=="],
|
||||
|
||||
"util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="],
|
||||
|
||||
"utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="],
|
||||
|
|
@ -2596,6 +2609,8 @@
|
|||
|
||||
"restore-cursor/signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
|
||||
|
||||
"rollup/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"sass/immutable": ["immutable@5.1.2", "", {}, "sha512-qHKXW1q6liAk1Oys6umoaZbDRqjcjgSrbnrifHsfsttza7zcvRAsL7mMV6xWcyhwQy7Xj5v4hhbr6b+iDYwlmQ=="],
|
||||
|
||||
"slice-ansi/ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="],
|
||||
|
|
@ -2618,6 +2633,8 @@
|
|||
|
||||
"tar/mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="],
|
||||
|
||||
"vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"wrap-ansi/ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="],
|
||||
|
||||
"wrap-ansi/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="],
|
||||
|
|
@ -2888,6 +2905,8 @@
|
|||
|
||||
"karma-coverage/istanbul-lib-instrument/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
|
||||
|
||||
"karma/chokidar/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"karma/chokidar/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="],
|
||||
|
||||
"karma/chokidar/readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="],
|
||||
|
|
|
|||
24
libs/browser/package.json
Normal file
24
libs/browser/package.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"name": "@stock-bot/browser",
|
||||
"version": "1.0.0",
|
||||
"description": "High-performance browser automation library with proxy support",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "bun test",
|
||||
"dev": "tsc --watch"
|
||||
},
|
||||
"dependencies": {
|
||||
"playwright": "^1.53.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.0.0",
|
||||
"typescript": "^5.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@stock-bot/logger": "workspace:*",
|
||||
"@stock-bot/http": "workspace:*"
|
||||
}
|
||||
}
|
||||
0
libs/browser/src/browser-pool.ts
Normal file
0
libs/browser/src/browser-pool.ts
Normal file
361
libs/browser/src/browser.ts
Normal file
361
libs/browser/src/browser.ts
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
import { BrowserContext, chromium, Page, Browser as PlaywrightBrowser } from 'playwright';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { BrowserOptions, NetworkEvent, NetworkEventHandler } from './types';
|
||||
|
||||
class BrowserSingleton {
|
||||
private browser?: PlaywrightBrowser;
|
||||
private contexts: Map<string, BrowserContext> = new Map();
|
||||
private logger = getLogger('browser');
|
||||
private options: BrowserOptions;
|
||||
private initialized = false;
|
||||
|
||||
constructor() {
|
||||
this.options = {
|
||||
headless: true,
|
||||
timeout: 30000,
|
||||
blockResources: false,
|
||||
enableNetworkLogging: false,
|
||||
};
|
||||
}
|
||||
|
||||
async initialize(options: BrowserOptions = {}): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Merge options
|
||||
this.options = {
|
||||
...this.options,
|
||||
...options,
|
||||
};
|
||||
|
||||
this.logger.info('Initializing browser...');
|
||||
|
||||
try {
|
||||
this.browser = await chromium.launch({
|
||||
headless: this.options.headless,
|
||||
timeout: this.options.timeout,
|
||||
args: [
|
||||
// Security and sandbox
|
||||
'--no-sandbox',
|
||||
// '--disable-setuid-sandbox',
|
||||
// '--disable-dev-shm-usage',
|
||||
// '--disable-web-security',
|
||||
// '--disable-features=VizDisplayCompositor',
|
||||
// '--disable-blink-features=AutomationControlled',
|
||||
|
||||
// // Performance optimizations
|
||||
// '--disable-gpu',
|
||||
// '--disable-gpu-sandbox',
|
||||
// '--disable-software-rasterizer',
|
||||
// '--disable-background-timer-throttling',
|
||||
// '--disable-renderer-backgrounding',
|
||||
// '--disable-backgrounding-occluded-windows',
|
||||
// '--disable-field-trial-config',
|
||||
// '--disable-back-forward-cache',
|
||||
// '--disable-hang-monitor',
|
||||
// '--disable-ipc-flooding-protection',
|
||||
|
||||
// // Extensions and plugins
|
||||
// '--disable-extensions',
|
||||
// '--disable-plugins',
|
||||
// '--disable-component-extensions-with-background-pages',
|
||||
// '--disable-component-update',
|
||||
// '--disable-plugins-discovery',
|
||||
// '--disable-bundled-ppapi-flash',
|
||||
|
||||
// // Features we don't need
|
||||
// '--disable-default-apps',
|
||||
// '--disable-sync',
|
||||
// '--disable-translate',
|
||||
// '--disable-client-side-phishing-detection',
|
||||
// '--disable-domain-reliability',
|
||||
// '--disable-features=TranslateUI',
|
||||
// '--disable-features=Translate',
|
||||
// '--disable-breakpad',
|
||||
// '--disable-preconnect',
|
||||
// '--disable-print-preview',
|
||||
// '--disable-password-generation',
|
||||
// '--disable-password-manager-reauthentication',
|
||||
// '--disable-save-password-bubble',
|
||||
// '--disable-single-click-autofill',
|
||||
// '--disable-autofill',
|
||||
// '--disable-autofill-keyboard-accessory-view',
|
||||
// '--disable-full-form-autofill-ios',
|
||||
|
||||
// // Audio/Video/Media
|
||||
// '--mute-audio',
|
||||
// '--disable-audio-output',
|
||||
// '--autoplay-policy=user-gesture-required',
|
||||
// '--disable-background-media-playback',
|
||||
|
||||
// // Networking
|
||||
// '--disable-background-networking',
|
||||
// '--disable-sync',
|
||||
// '--aggressive-cache-discard',
|
||||
// '--disable-default-apps',
|
||||
|
||||
// // UI/UX optimizations
|
||||
// '--no-first-run',
|
||||
// '--disable-infobars',
|
||||
// '--disable-notifications',
|
||||
// '--disable-desktop-notifications',
|
||||
// '--disable-prompt-on-repost',
|
||||
// '--disable-logging',
|
||||
// '--disable-file-system',
|
||||
// '--hide-scrollbars',
|
||||
|
||||
// // Memory optimizations
|
||||
// '--memory-pressure-off',
|
||||
// '--max_old_space_size=4096',
|
||||
// '--js-flags="--max-old-space-size=4096"',
|
||||
// '--media-cache-size=1',
|
||||
// '--disk-cache-size=1',
|
||||
|
||||
// // Process management
|
||||
// '--use-mock-keychain',
|
||||
// '--password-store=basic',
|
||||
// '--enable-automation',
|
||||
// '--no-pings',
|
||||
// '--no-service-autorun',
|
||||
// '--metrics-recording-only',
|
||||
// '--safebrowsing-disable-auto-update',
|
||||
|
||||
// // Disable unnecessary features for headless mode
|
||||
// '--disable-speech-api',
|
||||
// '--disable-gesture-typing',
|
||||
// '--disable-voice-input',
|
||||
// '--disable-wake-on-wifi',
|
||||
// '--disable-webgl',
|
||||
// '--disable-webgl2',
|
||||
// '--disable-3d-apis',
|
||||
// '--disable-accelerated-2d-canvas',
|
||||
// '--disable-accelerated-jpeg-decoding',
|
||||
// '--disable-accelerated-mjpeg-decode',
|
||||
// '--disable-accelerated-video-decode',
|
||||
// '--disable-canvas-aa',
|
||||
// '--disable-2d-canvas-clip-aa',
|
||||
// '--disable-gl-drawing-for-tests',
|
||||
],
|
||||
});
|
||||
|
||||
this.initialized = true;
|
||||
this.logger.info('Browser initialized successfully');
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to initialize browser', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async createPageWithProxy(
|
||||
url: string,
|
||||
proxy?: string
|
||||
): Promise<{
|
||||
page: Page & {
|
||||
onNetworkEvent: (handler: NetworkEventHandler) => void;
|
||||
offNetworkEvent: (handler: NetworkEventHandler) => void;
|
||||
clearNetworkListeners: () => void;
|
||||
};
|
||||
contextId: string;
|
||||
}> {
|
||||
if (!this.browser) {
|
||||
throw new Error('Browser not initialized. Call Browser.initialize() first.');
|
||||
}
|
||||
|
||||
const contextId = `ctx-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
|
||||
const contextOptions: Record<string, unknown> = {
|
||||
ignoreHTTPSErrors: true,
|
||||
bypassCSP: true,
|
||||
};
|
||||
|
||||
if (proxy) {
|
||||
const [protocol, rest] = proxy.split('://');
|
||||
const [auth, hostPort] = rest.includes('@') ? rest.split('@') : [null, rest];
|
||||
const [host, port] = hostPort.split(':');
|
||||
|
||||
contextOptions.proxy = {
|
||||
server: `${protocol}://${host}:${port}`,
|
||||
username: auth?.split(':')[0] || '',
|
||||
password: auth?.split(':')[1] || '',
|
||||
};
|
||||
}
|
||||
|
||||
const context = await this.browser.newContext(contextOptions);
|
||||
|
||||
// Block resources for performance
|
||||
if (this.options.blockResources) {
|
||||
await context.route('**/*.{png,jpg,jpeg,gif,svg,ico,woff,woff2,ttf,css}', route => {
|
||||
route.abort();
|
||||
});
|
||||
}
|
||||
|
||||
this.contexts.set(contextId, context);
|
||||
|
||||
const page = await context.newPage();
|
||||
page.setDefaultTimeout(this.options.timeout || 30000);
|
||||
page.setDefaultNavigationTimeout(this.options.timeout || 30000);
|
||||
|
||||
// Create network event handlers for this page
|
||||
const networkEventHandlers: Set<NetworkEventHandler> = new Set();
|
||||
|
||||
// Add network monitoring methods to the page
|
||||
const enhancedPage = page as Page & {
|
||||
onNetworkEvent: (handler: NetworkEventHandler) => void;
|
||||
offNetworkEvent: (handler: NetworkEventHandler) => void;
|
||||
clearNetworkListeners: () => void;
|
||||
};
|
||||
|
||||
enhancedPage.onNetworkEvent = (handler: NetworkEventHandler) => {
|
||||
networkEventHandlers.add(handler);
|
||||
|
||||
// Set up network monitoring on first handler
|
||||
if (networkEventHandlers.size === 1) {
|
||||
this.setupNetworkMonitoring(page, networkEventHandlers);
|
||||
}
|
||||
};
|
||||
|
||||
enhancedPage.offNetworkEvent = (handler: NetworkEventHandler) => {
|
||||
networkEventHandlers.delete(handler);
|
||||
};
|
||||
|
||||
enhancedPage.clearNetworkListeners = () => {
|
||||
networkEventHandlers.clear();
|
||||
};
|
||||
|
||||
if (url) {
|
||||
await page.goto(url, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: this.options.timeout,
|
||||
});
|
||||
}
|
||||
|
||||
return { page: enhancedPage, contextId };
|
||||
}
|
||||
|
||||
private setupNetworkMonitoring(page: Page, handlers: Set<NetworkEventHandler>): void {
|
||||
// Listen to requests
|
||||
page.on('request', async request => {
|
||||
const event: NetworkEvent = {
|
||||
url: request.url(),
|
||||
method: request.method(),
|
||||
type: 'request',
|
||||
timestamp: Date.now(),
|
||||
headers: request.headers(),
|
||||
};
|
||||
|
||||
// Capture request data for POST/PUT/PATCH requests
|
||||
if (['POST', 'PUT', 'PATCH'].includes(request.method())) {
|
||||
try {
|
||||
const postData = request.postData();
|
||||
if (postData) {
|
||||
event.requestData = postData;
|
||||
}
|
||||
} catch {
|
||||
// Some requests might not have accessible post data
|
||||
}
|
||||
}
|
||||
|
||||
this.emitNetworkEvent(event, handlers);
|
||||
});
|
||||
|
||||
// Listen to responses
|
||||
page.on('response', async response => {
|
||||
const event: NetworkEvent = {
|
||||
url: response.url(),
|
||||
method: response.request().method(),
|
||||
status: response.status(),
|
||||
type: 'response',
|
||||
timestamp: Date.now(),
|
||||
headers: response.headers(),
|
||||
};
|
||||
|
||||
// Capture response data for GET/POST requests with JSON content
|
||||
const contentType = response.headers()['content-type'] || '';
|
||||
if (contentType.includes('application/json') || contentType.includes('text/')) {
|
||||
try {
|
||||
const responseData = await response.text();
|
||||
event.responseData = responseData;
|
||||
} catch {
|
||||
// Response might be too large or not accessible
|
||||
}
|
||||
}
|
||||
|
||||
this.emitNetworkEvent(event, handlers);
|
||||
});
|
||||
|
||||
// Listen to failed requests
|
||||
page.on('requestfailed', request => {
|
||||
const event: NetworkEvent = {
|
||||
url: request.url(),
|
||||
method: request.method(),
|
||||
type: 'failed',
|
||||
timestamp: Date.now(),
|
||||
headers: request.headers(),
|
||||
};
|
||||
|
||||
// Try to capture request data for failed requests too
|
||||
if (['POST', 'PUT', 'PATCH'].includes(request.method())) {
|
||||
try {
|
||||
const postData = request.postData();
|
||||
if (postData) {
|
||||
event.requestData = postData;
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors when accessing post data
|
||||
}
|
||||
}
|
||||
|
||||
this.emitNetworkEvent(event, handlers);
|
||||
});
|
||||
}
|
||||
|
||||
private emitNetworkEvent(event: NetworkEvent, handlers: Set<NetworkEventHandler>): void {
|
||||
for (const handler of handlers) {
|
||||
try {
|
||||
handler(event);
|
||||
} catch (error) {
|
||||
this.logger.error('Network event handler error', { error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async evaluate<T>(page: Page, fn: () => T): Promise<T> {
|
||||
return page.evaluate(fn);
|
||||
}
|
||||
|
||||
async closeContext(contextId: string): Promise<void> {
|
||||
const context = this.contexts.get(contextId);
|
||||
if (context) {
|
||||
await context.close();
|
||||
this.contexts.delete(contextId);
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
// Close all contexts
|
||||
for (const [, context] of this.contexts) {
|
||||
await context.close();
|
||||
}
|
||||
this.contexts.clear();
|
||||
|
||||
// Close browser
|
||||
if (this.browser) {
|
||||
await this.browser.close();
|
||||
this.browser = undefined;
|
||||
}
|
||||
|
||||
this.initialized = false;
|
||||
this.logger.info('Browser closed');
|
||||
}
|
||||
|
||||
get isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const Browser = new BrowserSingleton();
|
||||
|
||||
// Also export the class for typing if needed
|
||||
export { BrowserSingleton as BrowserClass };
|
||||
0
libs/browser/src/fast-browser.ts
Normal file
0
libs/browser/src/fast-browser.ts
Normal file
3
libs/browser/src/index.ts
Normal file
3
libs/browser/src/index.ts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
export { Browser } from './browser';
|
||||
export { BrowserTabManager } from './tab-manager';
|
||||
export type { BrowserOptions, ScrapingResult } from './types';
|
||||
103
libs/browser/src/tab-manager.ts
Normal file
103
libs/browser/src/tab-manager.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { Page } from 'playwright';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { Browser } from './browser';
|
||||
import type { ScrapingResult } from './types';
|
||||
|
||||
interface TabInfo {
|
||||
page: Page;
|
||||
contextId: string;
|
||||
}
|
||||
|
||||
export class BrowserTabManager {
|
||||
private tabs: Map<string, TabInfo> = new Map();
|
||||
private logger = getLogger('browser-tab-manager');
|
||||
|
||||
async createTab(url?: string): Promise<{ page: Page; tabId: string }> {
|
||||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
const { page, contextId } = await Browser.createPageWithProxy(url || 'about:blank');
|
||||
|
||||
this.tabs.set(tabId, { page, contextId });
|
||||
this.logger.debug('Tab created', { tabId, url });
|
||||
|
||||
return { page, tabId };
|
||||
}
|
||||
|
||||
async createTabWithProxy(
|
||||
url: string,
|
||||
proxy: string
|
||||
): Promise<{ page: Page; tabId: string; contextId: string }> {
|
||||
const tabId = `tab-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
const { page, contextId } = await Browser.createPageWithProxy(url, proxy);
|
||||
|
||||
this.tabs.set(tabId, { page, contextId });
|
||||
this.logger.debug('Tab with proxy created', { tabId, url, proxy });
|
||||
|
||||
return { page, tabId, contextId };
|
||||
}
|
||||
|
||||
async scrapeUrlsWithProxies<T>(
|
||||
urlProxyPairs: Array<{ url: string; proxy: string }>,
|
||||
extractor: (page: Page) => Promise<T>,
|
||||
options: { concurrency?: number } = {}
|
||||
): Promise<ScrapingResult<T>[]> {
|
||||
const { concurrency = 3 } = options;
|
||||
const results: ScrapingResult<T>[] = [];
|
||||
|
||||
for (let i = 0; i < urlProxyPairs.length; i += concurrency) {
|
||||
const batch = urlProxyPairs.slice(i, i + concurrency);
|
||||
|
||||
const batchPromises = batch.map(async ({ url, proxy }) => {
|
||||
let tabId: string | undefined;
|
||||
|
||||
try {
|
||||
const result = await this.createTabWithProxy(url, proxy);
|
||||
tabId = result.tabId;
|
||||
|
||||
const data = await extractor(result.page);
|
||||
|
||||
return {
|
||||
data,
|
||||
url,
|
||||
success: true,
|
||||
} as ScrapingResult<T>;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
return {
|
||||
data: null as T,
|
||||
url,
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
} as ScrapingResult<T>;
|
||||
} finally {
|
||||
if (tabId) {
|
||||
await this.closeTab(tabId);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
results.push(...batchResults);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async closeTab(tabId: string): Promise<void> {
|
||||
const tab = this.tabs.get(tabId);
|
||||
if (tab) {
|
||||
await tab.page.close();
|
||||
await Browser.closeContext(tab.contextId);
|
||||
this.tabs.delete(tabId);
|
||||
this.logger.debug('Tab closed', { tabId });
|
||||
}
|
||||
}
|
||||
|
||||
getTabCount(): number {
|
||||
return this.tabs.size;
|
||||
}
|
||||
|
||||
getAllTabIds(): string[] {
|
||||
return Array.from(this.tabs.keys());
|
||||
}
|
||||
}
|
||||
30
libs/browser/src/types.ts
Normal file
30
libs/browser/src/types.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
export interface BrowserOptions {
|
||||
proxy?: string;
|
||||
headless?: boolean;
|
||||
timeout?: number;
|
||||
blockResources?: boolean;
|
||||
enableNetworkLogging?: boolean;
|
||||
}
|
||||
|
||||
// Keep the old name for backward compatibility
|
||||
export type FastBrowserOptions = BrowserOptions;
|
||||
|
||||
export interface ScrapingResult<T = unknown> {
|
||||
data: T;
|
||||
url: string;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface NetworkEvent {
|
||||
url: string;
|
||||
method: string;
|
||||
status?: number;
|
||||
type: 'request' | 'response' | 'failed';
|
||||
timestamp: number;
|
||||
requestData?: string;
|
||||
responseData?: string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export type NetworkEventHandler = (event: NetworkEvent) => void;
|
||||
0
libs/browser/src/utils.ts
Normal file
0
libs/browser/src/utils.ts
Normal file
10
libs/browser/tsconfig.json
Normal file
10
libs/browser/tsconfig.json
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"extends": "../../tsconfig.lib.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": "src"
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"],
|
||||
"references": [{ "path": "../../libs/logger" }]
|
||||
}
|
||||
13
libs/cache/src/redis-cache.ts
vendored
13
libs/cache/src/redis-cache.ts
vendored
|
|
@ -289,6 +289,19 @@ export class RedisCache implements CacheProvider {
|
|||
);
|
||||
}
|
||||
|
||||
async keys(pattern: string): Promise<string[]> {
|
||||
return this.safeExecute(
|
||||
async () => {
|
||||
const fullPattern = `${this.keyPrefix}${pattern}`;
|
||||
const keys = await this.redis.keys(fullPattern);
|
||||
// Remove the prefix from returned keys to match the interface expectation
|
||||
return keys.map(key => key.replace(this.keyPrefix, ''));
|
||||
},
|
||||
[],
|
||||
'keys'
|
||||
);
|
||||
}
|
||||
|
||||
async health(): Promise<boolean> {
|
||||
try {
|
||||
const pong = await this.redis.ping();
|
||||
|
|
|
|||
1
libs/cache/src/types.ts
vendored
1
libs/cache/src/types.ts
vendored
|
|
@ -16,6 +16,7 @@ export interface CacheProvider {
|
|||
del(key: string): Promise<void>;
|
||||
exists(key: string): Promise<boolean>;
|
||||
clear(): Promise<void>;
|
||||
keys(pattern: string): Promise<string[]>;
|
||||
getStats(): CacheStats;
|
||||
health(): Promise<boolean>;
|
||||
|
||||
|
|
|
|||
|
|
@ -20,15 +20,17 @@
|
|||
"axios": "^1.9.0",
|
||||
"http-proxy-agent": "^7.0.2",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"socks-proxy-agent": "^8.0.5"
|
||||
"socks-proxy-agent": "^8.0.5",
|
||||
"user-agents": "^1.1.567"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.0",
|
||||
"typescript": "^5.3.0",
|
||||
"eslint": "^8.56.0",
|
||||
"@types/user-agents": "^1.0.4",
|
||||
"@typescript-eslint/eslint-plugin": "^6.19.0",
|
||||
"@typescript-eslint/parser": "^6.19.0",
|
||||
"bun-types": "^1.2.15"
|
||||
"bun-types": "^1.2.15",
|
||||
"eslint": "^8.56.0",
|
||||
"typescript": "^5.3.0"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
|
|
|
|||
|
|
@ -11,15 +11,17 @@ export class AxiosAdapter implements RequestAdapter {
|
|||
canHandle(config: RequestConfig): boolean {
|
||||
// Axios handles SOCKS proxies
|
||||
return Boolean(
|
||||
config.proxy && (config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5')
|
||||
config.proxy &&
|
||||
typeof config.proxy !== 'string' &&
|
||||
(config.proxy.protocol === 'socks4' || config.proxy.protocol === 'socks5')
|
||||
);
|
||||
}
|
||||
|
||||
async request<T = any>(config: RequestConfig, signal: AbortSignal): Promise<HttpResponse<T>> {
|
||||
const { url, method = 'GET', headers, data, proxy } = config;
|
||||
|
||||
if (!proxy) {
|
||||
throw new Error('Axios adapter requires proxy configuration');
|
||||
if (!proxy || typeof proxy === 'string') {
|
||||
throw new Error('Axios adapter requires ProxyInfo configuration');
|
||||
}
|
||||
|
||||
// Create proxy configuration using ProxyManager
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@ import type { RequestAdapter } from './types';
|
|||
export class FetchAdapter implements RequestAdapter {
|
||||
canHandle(config: RequestConfig): boolean {
|
||||
// Fetch handles non-proxy requests and HTTP/HTTPS proxies
|
||||
if (typeof config.proxy === 'string') {
|
||||
return config.proxy.startsWith('http');
|
||||
}
|
||||
return !config.proxy || config.proxy.protocol === 'http' || config.proxy.protocol === 'https';
|
||||
}
|
||||
|
||||
|
|
@ -31,7 +34,11 @@ export class FetchAdapter implements RequestAdapter {
|
|||
}
|
||||
|
||||
// Add proxy if needed (using Bun's built-in proxy support)
|
||||
if (proxy) {
|
||||
if (typeof proxy === 'string') {
|
||||
// If proxy is a URL string, use it directly
|
||||
(fetchOptions as any).proxy = proxy;
|
||||
} else if (proxy) {
|
||||
// If proxy is a ProxyInfo object, create a proxy URL
|
||||
(fetchOptions as any).proxy = ProxyManager.createProxyUrl(proxy);
|
||||
}
|
||||
const response = await fetch(url, fetchOptions);
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import type { Logger } from '@stock-bot/logger';
|
|||
import { AdapterFactory } from './adapters/index';
|
||||
import type { HttpClientConfig, HttpResponse, RequestConfig } from './types';
|
||||
import { HttpError } from './types';
|
||||
import { getRandomUserAgent } from './user-agent';
|
||||
|
||||
export class HttpClient {
|
||||
private readonly config: HttpClientConfig;
|
||||
|
|
@ -165,9 +166,17 @@ export class HttpClient {
|
|||
* Merge configs with defaults
|
||||
*/
|
||||
private mergeConfig(config: RequestConfig): RequestConfig {
|
||||
// Merge headers with automatic User-Agent assignment
|
||||
const mergedHeaders = { ...this.config.headers, ...config.headers };
|
||||
|
||||
// Add random User-Agent if not specified
|
||||
if (!mergedHeaders['User-Agent'] && !mergedHeaders['user-agent']) {
|
||||
mergedHeaders['User-Agent'] = getRandomUserAgent();
|
||||
}
|
||||
|
||||
return {
|
||||
...config,
|
||||
headers: { ...this.config.headers, ...config.headers },
|
||||
headers: mergedHeaders,
|
||||
timeout: config.timeout ?? this.config.timeout,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
// Re-export all types and classes
|
||||
export * from './types';
|
||||
export * from './adapters/index';
|
||||
export * from './client';
|
||||
export * from './proxy-manager';
|
||||
export * from './adapters/index';
|
||||
export * from './types';
|
||||
export * from './user-agent';
|
||||
|
||||
// Default export
|
||||
export { HttpClient as default } from './client';
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ export interface RequestConfig {
|
|||
headers?: Record<string, string>;
|
||||
data?: any; // Changed from 'body' to 'data' for consistency
|
||||
timeout?: number;
|
||||
proxy?: ProxyInfo;
|
||||
proxy?: ProxyInfo | string; // Proxy can be a ProxyInfo object or a URL string
|
||||
}
|
||||
|
||||
export interface HttpResponse<T = any> {
|
||||
|
|
|
|||
6
libs/http/src/user-agent.ts
Normal file
6
libs/http/src/user-agent.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import UserAgent from 'user-agents';
|
||||
|
||||
export function getRandomUserAgent(): string {
|
||||
const userAgent = new UserAgent();
|
||||
return userAgent.toString();
|
||||
}
|
||||
18
libs/proxy/package.json
Normal file
18
libs/proxy/package.json
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"name": "@stock-bot/proxy",
|
||||
"version": "1.0.0",
|
||||
"description": "Simple proxy management library",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "bun test",
|
||||
"dev": "tsc --watch"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.0.0",
|
||||
"typescript": "^5.0.0"
|
||||
},
|
||||
"peerDependencies": {}
|
||||
}
|
||||
97
libs/proxy/src/index.ts
Normal file
97
libs/proxy/src/index.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
// Simple proxy list manager
|
||||
let proxies: string[] = [];
|
||||
let currentIndex = 0;
|
||||
|
||||
const DEFAULT_PROXY_URL =
|
||||
'https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all';
|
||||
|
||||
/**
|
||||
* Fetch proxy list from URL and store in module
|
||||
*/
|
||||
export async function refreshProxies(fetchUrl: string = DEFAULT_PROXY_URL): Promise<string[]> {
|
||||
try {
|
||||
const response = await fetch(fetchUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.text();
|
||||
const newProxies = data
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line && line.includes(':'))
|
||||
.map(line => {
|
||||
// Convert host:port to http://host:port format
|
||||
return line.startsWith('http') ? line : `http://${line}`;
|
||||
});
|
||||
|
||||
proxies = newProxies;
|
||||
currentIndex = 0;
|
||||
|
||||
return proxies;
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to fetch proxies: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get next proxy URL in round-robin fashion
|
||||
*/
|
||||
export function getProxyURL(): string | null {
|
||||
if (proxies.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const proxy = proxies[currentIndex];
|
||||
currentIndex = (currentIndex + 1) % proxies.length;
|
||||
|
||||
return proxy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get multiple proxy URLs
|
||||
*/
|
||||
export function getProxyURLs(count: number): string[] {
|
||||
const urls: string[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const url = getProxyURL();
|
||||
if (url) {
|
||||
urls.push(url);
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get random proxy URL
|
||||
*/
|
||||
export function getRandomProxyURL(): string | null {
|
||||
if (proxies.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const randomIndex = Math.floor(Math.random() * proxies.length);
|
||||
return proxies[randomIndex];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current proxy count
|
||||
*/
|
||||
export function getProxyCount(): number {
|
||||
return proxies.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all proxies
|
||||
*/
|
||||
export function getAllProxies(): string[] {
|
||||
return [...proxies];
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize proxy manager with initial fetch
|
||||
*/
|
||||
export async function initializeProxies(fetchUrl?: string): Promise<void> {
|
||||
await refreshProxies(fetchUrl);
|
||||
}
|
||||
0
libs/proxy/src/proxy-manager.ts
Normal file
0
libs/proxy/src/proxy-manager.ts
Normal file
22
libs/proxy/src/types.ts
Normal file
22
libs/proxy/src/types.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
export interface ProxyInfo {
|
||||
host: string;
|
||||
port: number;
|
||||
protocol: 'http' | 'https' | 'socks4' | 'socks5';
|
||||
username?: string;
|
||||
password?: string;
|
||||
country?: string;
|
||||
isActive?: boolean;
|
||||
}
|
||||
|
||||
export interface ProxyManagerOptions {
|
||||
fetchUrl?: string;
|
||||
refreshIntervalMs?: number;
|
||||
maxRetries?: number;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export interface ProxyResponse {
|
||||
proxies: ProxyInfo[];
|
||||
totalCount: number;
|
||||
activeCount: number;
|
||||
}
|
||||
10
libs/proxy/tsconfig.json
Normal file
10
libs/proxy/tsconfig.json
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"extends": "../../tsconfig.lib.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"]
|
||||
}
|
||||
7
libs/utils/src/common.ts
Normal file
7
libs/utils/src/common.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export function createProxyUrl(proxy: any): string {
|
||||
const { protocol, host, port, username, password } = proxy;
|
||||
if (username && password) {
|
||||
return `${protocol}://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}`;
|
||||
}
|
||||
return `${protocol}://${host}:${port}`;
|
||||
}
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
export * from './dateUtils';
|
||||
export * from './calculations/index';
|
||||
export * from './common';
|
||||
export * from './dateUtils';
|
||||
|
|
|
|||
|
|
@ -108,7 +108,8 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"bullmq": "^5.53.2",
|
||||
"ioredis": "^5.6.1"
|
||||
"ioredis": "^5.6.1",
|
||||
"playwright": "^1.53.0"
|
||||
},
|
||||
"trustedDependencies": [
|
||||
"@tailwindcss/oxide",
|
||||
|
|
|
|||
166
test-browser-simple.ts
Normal file
166
test-browser-simple.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
/**
|
||||
* Simple Browser and Network Monitoring Test
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testBasicBrowser() {
|
||||
console.log('🚀 Testing basic browser functionality...');
|
||||
|
||||
try {
|
||||
// Initialize browser
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false,
|
||||
enableNetworkLogging: true,
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Test 1: Simple page without proxy
|
||||
console.log('📄 Testing simple page without proxy...');
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/json'
|
||||
);
|
||||
|
||||
let capturedData = null;
|
||||
let eventCount = 0;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
console.log(`📡 Event ${eventCount}: ${event.type} - ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/json')) {
|
||||
console.log(` 📊 Status: ${event.status}`);
|
||||
if (event.responseData) {
|
||||
capturedData = event.responseData;
|
||||
console.log(` 📝 Response: ${event.responseData}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
|
||||
console.log(`✅ Test completed. Events captured: ${eventCount}`);
|
||||
if (capturedData) {
|
||||
console.log('✅ Successfully captured response data');
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return true;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Basic test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testProxyConnection() {
|
||||
console.log('\n🔄 Testing proxy connection...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
// Test different proxy formats
|
||||
const proxyConfigs = [
|
||||
null, // No proxy
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80',
|
||||
];
|
||||
|
||||
for (const proxy of proxyConfigs) {
|
||||
console.log(`\n🌐 Testing with proxy: ${proxy || 'No proxy'}`);
|
||||
|
||||
try {
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
proxy
|
||||
);
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
|
||||
console.log(` 📍 IP Response: ${event.responseData}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
await Browser.closeContext(contextId);
|
||||
console.log(' ✅ Success');
|
||||
|
||||
} catch (error) {
|
||||
console.log(` ❌ Failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Proxy test setup failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testIBWithWorkaround() {
|
||||
console.log('\n🏦 Testing IB endpoint with workaround...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 20000,
|
||||
blockResources: true, // Block resources for performance
|
||||
});
|
||||
|
||||
// Try without proxy first
|
||||
console.log('🌐 Attempting IB without proxy...');
|
||||
try {
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com'
|
||||
);
|
||||
|
||||
let responseCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.type === 'response') {
|
||||
responseCount++;
|
||||
console.log(` 📥 Response ${responseCount}: ${event.status} ${event.url}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
console.log(`✅ IB main page loaded. Responses: ${responseCount}`);
|
||||
await Browser.closeContext(contextId);
|
||||
|
||||
} catch (error) {
|
||||
console.log(`❌ IB without proxy failed: ${error.message}`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ IB test failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run tests
|
||||
async function runAllTests() {
|
||||
console.log('🧪 Starting Browser Network Monitoring Tests\n');
|
||||
|
||||
const basicResult = await testBasicBrowser();
|
||||
await testProxyConnection();
|
||||
await testIBWithWorkaround();
|
||||
|
||||
console.log(`\n🏁 Basic functionality: ${basicResult ? '✅ PASS' : '❌ FAIL'}`);
|
||||
console.log('✅ All tests completed!');
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runAllTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testBasicBrowser, testProxyConnection, testIBWithWorkaround };
|
||||
62
test-browser.ts
Normal file
62
test-browser.ts
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import { Browser, BrowserTabManager } from './libs/browser/src';
|
||||
|
||||
async function testSimplifiedBrowser() {
|
||||
console.log('Testing simplified browser library...');
|
||||
|
||||
try {
|
||||
console.log('Initializing browser...');
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
blockResources: true,
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
// Test single page with proxy support
|
||||
console.log('Testing page creation...');
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/json',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
const content = await Browser.evaluate(page, () => document.body.textContent);
|
||||
console.log('Page content:', content?.substring(0, 100) + '...');
|
||||
|
||||
// Test tab manager (no longer needs browser instance)
|
||||
console.log('Testing tab manager...');
|
||||
const tabManager = new BrowserTabManager();
|
||||
|
||||
// Test multiple URL scraping with different proxies
|
||||
const urlProxyPairs = [
|
||||
{ url: 'https://httpbin.org/uuid', proxy: '' }, // No proxy
|
||||
{ url: 'https://httpbin.org/ip', proxy: '' }, // No proxy
|
||||
];
|
||||
|
||||
const results = await tabManager.scrapeUrlsWithProxies(
|
||||
urlProxyPairs,
|
||||
async page => {
|
||||
const text = await page.textContent('body');
|
||||
return { content: text?.substring(0, 50) };
|
||||
},
|
||||
{ concurrency: 2 }
|
||||
);
|
||||
|
||||
console.log('Scraping results:');
|
||||
results.forEach((result, index) => {
|
||||
console.log(` ${index + 1}. ${result.url}: ${result.success ? 'SUCCESS' : 'FAILED'}`);
|
||||
if (result.data) {
|
||||
console.log(` Data: ${result.data.content}...`);
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up
|
||||
await page.close();
|
||||
await Browser.closeContext(contextId);
|
||||
await Browser.close();
|
||||
|
||||
console.log('✅ Simplified browser test completed successfully!');
|
||||
} catch (error) {
|
||||
console.error('❌ Browser test failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
testSimplifiedBrowser();
|
||||
139
test-ib-no-proxy.ts
Normal file
139
test-ib-no-proxy.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testWithoutProxy() {
|
||||
console.log('🔬 Testing WITHOUT proxy...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({ headless: true, timeout: 15000, blockResources: false });
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/'
|
||||
// No proxy parameter
|
||||
);
|
||||
console.log('✅ Page created without proxy');
|
||||
|
||||
let eventCount = 0;
|
||||
let summaryData: SummaryResponse | null = null;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
|
||||
// Capture the summary API response
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.responseData) {
|
||||
console.log(`📊 Summary API Response Data: ${event.responseData}`);
|
||||
try {
|
||||
summaryData = JSON.parse(event.responseData) as any;
|
||||
const totalCount = summaryData[0].totalCount;
|
||||
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
|
||||
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
|
||||
} catch (e) {
|
||||
console.log('📊 Raw Summary Response:', event.responseData);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Uncomment to see all network events
|
||||
// console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
|
||||
});
|
||||
|
||||
console.log('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 15000 });
|
||||
console.log('✅ Page loaded');
|
||||
|
||||
// Complete interaction flow
|
||||
try {
|
||||
console.log('🔍 Looking for Products tab...');
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Products tab');
|
||||
|
||||
console.log('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
console.log('✅ Products tab clicked');
|
||||
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
console.log('🔍 Looking for Asset Classes accordion...');
|
||||
const assetClassesAccordion = page.locator(
|
||||
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
|
||||
);
|
||||
await assetClassesAccordion.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Asset Classes accordion');
|
||||
|
||||
console.log('🖱️ Clicking Asset Classes accordion...');
|
||||
await assetClassesAccordion.click();
|
||||
console.log('✅ Asset Classes accordion clicked');
|
||||
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
console.log('🔍 Looking for Stocks checkbox...');
|
||||
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
|
||||
await stocksSpan.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Stocks span');
|
||||
|
||||
const parentContainer = stocksSpan.locator('..');
|
||||
const checkbox = parentContainer.locator('input[type="checkbox"]');
|
||||
|
||||
if ((await checkbox.count()) > 0) {
|
||||
console.log('📋 Clicking Stocks checkbox...');
|
||||
await checkbox.first().check();
|
||||
console.log('✅ Stocks checkbox checked');
|
||||
} else {
|
||||
console.log('⚠️ Could not find checkbox near Stocks text');
|
||||
}
|
||||
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
console.log('🔍 Looking for Apply button...');
|
||||
const applyButton = page.locator(
|
||||
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
|
||||
);
|
||||
|
||||
if ((await applyButton.count()) > 0) {
|
||||
console.log('🎯 Clicking Apply button...');
|
||||
await applyButton.first().click();
|
||||
console.log('✅ Apply button clicked');
|
||||
await page.waitForTimeout(3000);
|
||||
} else {
|
||||
console.log('⚠️ Could not find Apply button');
|
||||
}
|
||||
} catch (interactionError) {
|
||||
const errorMessage =
|
||||
interactionError instanceof Error ? interactionError.message : String(interactionError);
|
||||
console.error('❌ Page interaction failed:', errorMessage);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
console.log(`📊 Total events captured: ${eventCount}`);
|
||||
|
||||
// Show final results
|
||||
if (summaryData) {
|
||||
console.log('✅ SUCCESS: Captured summary data!');
|
||||
console.log(`🔢 Final total count: ${summaryData?.data?.totalCount || 'Unknown'}`);
|
||||
console.log(`📋 Data keys: ${Object.keys(summaryData).join(', ')}`);
|
||||
} else {
|
||||
console.log('❌ No summary data captured');
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
await Browser.close();
|
||||
|
||||
console.log('✅ Test completed successfully');
|
||||
return true;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error('❌ Error:', errorMessage);
|
||||
await Browser.close();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
testWithoutProxy().then(success => {
|
||||
console.log(`🏁 Final result: ${success ? 'SUCCESS' : 'FAILED'}`);
|
||||
});
|
||||
160
test-ib-working.ts
Normal file
160
test-ib-working.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
/**
|
||||
* Working Interactive Brokers test with verified network monitoring
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testIBWithWorking() {
|
||||
console.log('🏦 Testing IB with working network monitoring and fixed proxy auth...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 20000,
|
||||
blockResources: false, // Don't block resources initially
|
||||
});
|
||||
|
||||
// Test 1: Try a simple proxy detection service first
|
||||
console.log('🌐 Testing proxy connectivity...');
|
||||
const { page: proxyPage, contextId: proxyCtx } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let proxyEvents = 0;
|
||||
let myIP = null;
|
||||
proxyPage.onNetworkEvent(event => {
|
||||
proxyEvents++;
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
try {
|
||||
const data = JSON.parse(event.responseData);
|
||||
myIP = data.origin;
|
||||
console.log(` 📍 Proxy IP: ${myIP}`);
|
||||
} catch (e) {
|
||||
console.log(` 📊 Raw response: ${event.responseData}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await proxyPage.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
await Browser.closeContext(proxyCtx);
|
||||
|
||||
console.log(`📊 Proxy test events: ${proxyEvents}`);
|
||||
|
||||
// Test 2: Try IB API endpoint with fixed proxy auth
|
||||
console.log('🎯 Testing IB API endpoint...');
|
||||
const { page: apiPage, contextId: apiCtx } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let apiEvents = 0;
|
||||
let summaryData = null;
|
||||
apiPage.onNetworkEvent(event => {
|
||||
apiEvents++;
|
||||
console.log(` 📡 API Event: ${event.type} ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.url.includes('summary')) {
|
||||
console.log(` 🎯 Found summary response! Status: ${event.status}`);
|
||||
if (event.responseData) {
|
||||
summaryData = event.responseData;
|
||||
try {
|
||||
const data = JSON.parse(event.responseData);
|
||||
console.log(` 📊 Summary data: ${JSON.stringify(data, null, 2)}`);
|
||||
} catch (e) {
|
||||
console.log(` 📊 Raw summary: ${event.responseData.substring(0, 200)}...`);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await apiPage.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
await Browser.closeContext(apiCtx);
|
||||
|
||||
return {
|
||||
proxyEvents,
|
||||
apiEvents,
|
||||
summaryData,
|
||||
proxyIP: myIP,
|
||||
success: apiEvents > 0 || summaryData !== null,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('❌ IB test failed:', error);
|
||||
return {
|
||||
proxyEvents: 0,
|
||||
apiEvents: 0,
|
||||
summaryData: null,
|
||||
proxyIP: null,
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testWithProxyFallback() {
|
||||
console.log('\n🔄 Testing with proxy fallback strategy...');
|
||||
|
||||
const proxiesToTest = [
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', // Your proxy
|
||||
];
|
||||
|
||||
for (const proxy of proxiesToTest) {
|
||||
console.log(`\n🌐 Testing with: ${proxy || 'No proxy'}`);
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
proxy
|
||||
);
|
||||
|
||||
let ipResponse = null;
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ipResponse = event.responseData;
|
||||
console.log(` 📍 IP: ${JSON.parse(event.responseData).origin}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(contextId);
|
||||
} catch (error) {
|
||||
console.log(` ❌ Failed: ${error.message}`);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runIBTests() {
|
||||
console.log('🚀 Interactive Brokers Network Monitoring Tests with Fixed Proxy Auth\n');
|
||||
|
||||
const result = await testIBWithWorking();
|
||||
await testWithProxyFallback();
|
||||
|
||||
console.log('\n🏁 Final Results:');
|
||||
console.log(` 🌐 Proxy events: ${result.proxyEvents || 0}`);
|
||||
console.log(` 📍 Proxy IP: ${result.proxyIP || 'Not captured'}`);
|
||||
console.log(` 🎯 API events: ${result.apiEvents || 0}`);
|
||||
console.log(` 📊 Summary data: ${result.summaryData ? 'Captured' : 'Not captured'}`);
|
||||
console.log(` ✅ Overall success: ${result.success}`);
|
||||
|
||||
if (result.error) {
|
||||
console.log(` ❌ Error: ${result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runIBTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testIBWithWorking, testWithProxyFallback };
|
||||
194
test-ib.ts
Normal file
194
test-ib.ts
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
/**
|
||||
* Test Interactive Brokers functionality with network monitoring
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
import { getRandomProxyURL } from '@stock-bot/proxy';
|
||||
|
||||
async function testIBSymbolSummary() {
|
||||
console.log('🚀 Testing Interactive Brokers Symbol Summary with Network Monitoring...');
|
||||
|
||||
try {
|
||||
// Initialize browser
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 30000,
|
||||
blockResources: true,
|
||||
enableNetworkLogging: true,
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Get a random proxy
|
||||
|
||||
// Create page with proxy
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
console.log('📄 Page created with proxy');
|
||||
|
||||
// Set up network monitoring
|
||||
let summaryResponse: any = null;
|
||||
let requestCount = 0;
|
||||
let responseCount = 0;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
console.log(`📡 Network Event: ${event.type} - ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'request') {
|
||||
requestCount++;
|
||||
console.log(` 📤 Request #${requestCount}: ${event.method} ${event.url}`);
|
||||
|
||||
// Log request data for POST requests
|
||||
if (event.requestData) {
|
||||
console.log(` 📝 Request Data: ${event.requestData.substring(0, 200)}...`);
|
||||
}
|
||||
}
|
||||
|
||||
if (event.type === 'response') {
|
||||
responseCount++;
|
||||
console.log(` 📥 Response #${responseCount}: ${event.status} ${event.url}`);
|
||||
|
||||
// Capture the summary response
|
||||
if (event.url.includes('summary')) {
|
||||
console.log(` 🎯 Found summary response!`);
|
||||
summaryResponse = event.responseData;
|
||||
|
||||
if (event.responseData) {
|
||||
try {
|
||||
const data = JSON.parse(event.responseData);
|
||||
console.log(` 📊 Summary Data: ${JSON.stringify(data, null, 2)}`);
|
||||
} catch (e) {
|
||||
console.log(` 📊 Raw Response: ${event.responseData.substring(0, 500)}...`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (event.type === 'failed') {
|
||||
console.log(` ❌ Failed Request: ${event.url}`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log('🔍 Network monitoring set up, waiting for page to load...');
|
||||
|
||||
// Wait for page to load and capture network activity
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
console.log('✅ Page loaded');
|
||||
|
||||
// Wait a bit more for any additional network requests
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
|
||||
console.log(`📊 Network Summary:`);
|
||||
console.log(` 📤 Total Requests: ${requestCount}`);
|
||||
console.log(` 📥 Total Responses: ${responseCount}`);
|
||||
|
||||
if (summaryResponse) {
|
||||
console.log('✅ Successfully captured summary response');
|
||||
try {
|
||||
const parsed = JSON.parse(summaryResponse);
|
||||
console.log(`🔢 Total symbols found: ${parsed?.data?.totalCount || 'Unknown'}`);
|
||||
return parsed?.data?.totalCount || 0;
|
||||
} catch (e) {
|
||||
console.log('⚠️ Could not parse response as JSON');
|
||||
return 1; // Indicate success but unknown count
|
||||
}
|
||||
} else {
|
||||
console.log('❌ No summary response captured');
|
||||
return 0;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Test failed:', error);
|
||||
|
||||
// Log more details about the error
|
||||
if (error instanceof Error) {
|
||||
console.error('Error details:', {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
name: error.name
|
||||
});
|
||||
}
|
||||
|
||||
return -1;
|
||||
} finally {
|
||||
try {
|
||||
await Browser.close();
|
||||
console.log('🔒 Browser closed');
|
||||
} catch (closeError) {
|
||||
console.error('Error closing browser:', closeError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function testWithDifferentProxy() {
|
||||
console.log('\n🔄 Testing with different proxy configuration...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false, // Don't block resources for this test
|
||||
});
|
||||
|
||||
// Test without proxy first
|
||||
console.log('🌐 Testing without proxy...');
|
||||
const { page: pageNoProxy, contextId: contextNoProxy } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip'
|
||||
);
|
||||
|
||||
pageNoProxy.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
|
||||
console.log('📍 No proxy IP response:', event.responseData);
|
||||
}
|
||||
});
|
||||
|
||||
await pageNoProxy.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(contextNoProxy);
|
||||
|
||||
// Test with proxy
|
||||
console.log('🌐 Testing with proxy...');
|
||||
const { page: pageWithProxy, contextId: contextWithProxy } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
pageWithProxy.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
|
||||
console.log('🔄 Proxy IP response:', event.responseData);
|
||||
}
|
||||
});
|
||||
|
||||
await pageWithProxy.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(contextWithProxy);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Proxy test failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run the tests
|
||||
async function runTests() {
|
||||
console.log('🧪 Starting IB Network Monitoring Tests\n');
|
||||
|
||||
// Test 1: Main IB functionality
|
||||
const result = await testIBSymbolSummary();
|
||||
console.log(`\n🏁 Test Result: ${result}`);
|
||||
|
||||
// Test 2: Proxy verification
|
||||
await testWithDifferentProxy();
|
||||
|
||||
console.log('\n✅ All tests completed!');
|
||||
}
|
||||
|
||||
// Run if this file is executed directly
|
||||
if (import.meta.main) {
|
||||
runTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testIBSymbolSummary, testWithDifferentProxy };
|
||||
135
test-network-debug.ts
Normal file
135
test-network-debug.ts
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
/**
|
||||
* Debug network monitoring setup
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function debugNetworkSetup() {
|
||||
console.log('🐛 Debugging Network Monitoring Setup...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false, // Ensure we don't block requests
|
||||
});
|
||||
|
||||
// Create page but don't navigate yet
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let eventCount = 0;
|
||||
console.log('📡 Setting up network event listener...');
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
console.log(`🔔 Event ${eventCount}: ${event.type} ${event.method} ${event.url}`);
|
||||
console.log(` Headers: ${Object.keys(event.headers || {}).length} headers`);
|
||||
|
||||
if (event.responseData) {
|
||||
console.log(` Data: ${event.responseData.substring(0, 100)}...`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log('🌐 Navigating to httpbin.org/headers...');
|
||||
await page.goto('https://httpbin.org/headers');
|
||||
|
||||
console.log('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
|
||||
console.log('⏳ Waiting additional time for network events...');
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
|
||||
console.log(`📊 Total events captured: ${eventCount}`);
|
||||
|
||||
// Try to evaluate page content to see if it loaded
|
||||
const title = await page.title();
|
||||
console.log(`📄 Page title: "${title}"`);
|
||||
|
||||
const bodyText = await page.locator('body').textContent();
|
||||
if (bodyText) {
|
||||
console.log(`📝 Page content (first 200 chars): ${bodyText.substring(0, 200)}...`);
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return eventCount > 0;
|
||||
} catch (error) {
|
||||
console.error('❌ Debug test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testManualNetworkCall() {
|
||||
console.log('\n🔧 Testing with manual fetch call...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let eventCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
console.log(`📡 Manual test event ${eventCount}: ${event.type} ${event.method} ${event.url}`);
|
||||
if (event.responseData && event.url.includes('httpbin')) {
|
||||
console.log(` 📊 Response: ${event.responseData}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Navigate to a simple page first
|
||||
await page.goto('data:text/html,<html><body><h1>Test Page</h1></body></html>');
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
|
||||
console.log('🚀 Making manual fetch call...');
|
||||
// Make a fetch request from the page context
|
||||
const result = await page.evaluate(async () => {
|
||||
try {
|
||||
const response = await fetch('https://httpbin.org/json');
|
||||
const data = await response.json();
|
||||
return { success: true, data };
|
||||
} catch (error) {
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
console.log('📋 Fetch result:', result);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
console.log(`📊 Events from manual fetch: ${eventCount}`);
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return eventCount > 0;
|
||||
} catch (error) {
|
||||
console.error('❌ Manual test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runDebugTests() {
|
||||
console.log('🚀 Network Monitoring Debug Tests\n');
|
||||
|
||||
const setupResult = await debugNetworkSetup();
|
||||
const manualResult = await testManualNetworkCall();
|
||||
|
||||
console.log(`\n🏁 Results:`);
|
||||
console.log(` 🔧 Setup test: ${setupResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`);
|
||||
console.log(` 📡 Manual test: ${manualResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runDebugTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { debugNetworkSetup, testManualNetworkCall };
|
||||
137
test-network-monitoring.ts
Normal file
137
test-network-monitoring.ts
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
/**
|
||||
* Simple test to verify network monitoring is working
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testNetworkMonitoring() {
|
||||
console.log('🧪 Testing Network Monitoring with httpbin.org...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false, // Don't block resources so we can see requests
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Test with a simple API that returns JSON
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/json'
|
||||
);
|
||||
|
||||
let capturedRequests = 0;
|
||||
let capturedResponses = 0;
|
||||
let jsonResponse = null;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
console.log(`📡 ${event.type.toUpperCase()}: ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'request') {
|
||||
capturedRequests++;
|
||||
}
|
||||
|
||||
if (event.type === 'response') {
|
||||
capturedResponses++;
|
||||
console.log(` Status: ${event.status}`);
|
||||
|
||||
if (event.url.includes('httpbin.org/json') && event.responseData) {
|
||||
jsonResponse = event.responseData;
|
||||
console.log(` 📊 JSON Response: ${event.responseData}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
|
||||
console.log(`\n📊 Summary:`);
|
||||
console.log(` 📤 Requests captured: ${capturedRequests}`);
|
||||
console.log(` 📥 Responses captured: ${capturedResponses}`);
|
||||
console.log(` 📝 JSON data captured: ${jsonResponse ? 'Yes' : 'No'}`);
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return true;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testWithProxy() {
|
||||
console.log('\n🌐 Testing with proxy to see IP change...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
// Test IP without proxy
|
||||
console.log('📍 Getting IP without proxy...');
|
||||
const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip'
|
||||
);
|
||||
|
||||
let ipWithoutProxy = null;
|
||||
page1.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ipWithoutProxy = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🔹 Your IP: ${ipWithoutProxy}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page1.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(ctx1);
|
||||
|
||||
// Test IP with proxy
|
||||
console.log('🔄 Getting IP with proxy...');
|
||||
const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let ipWithProxy = null;
|
||||
page2.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ipWithProxy = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🔸 Proxy IP: ${ipWithProxy}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page2.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(ctx2);
|
||||
|
||||
if (ipWithoutProxy && ipWithProxy && ipWithoutProxy !== ipWithProxy) {
|
||||
console.log('✅ Proxy is working - IPs are different!');
|
||||
} else {
|
||||
console.log('⚠️ Proxy may not be working - IPs are the same or not captured');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Proxy test failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('🚀 Network Monitoring Verification Tests\n');
|
||||
|
||||
const basicResult = await testNetworkMonitoring();
|
||||
await testWithProxy();
|
||||
|
||||
console.log(`\n🏁 Network monitoring: ${basicResult ? '✅ WORKING' : '❌ FAILED'}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testNetworkMonitoring, testWithProxy };
|
||||
0
test-network.ts
Normal file
0
test-network.ts
Normal file
156
test-proxy-auth.ts
Normal file
156
test-proxy-auth.ts
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
/**
|
||||
* Test Playwright proxy authentication specifically
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testPlaywrightProxyAuth() {
|
||||
console.log('🔐 Testing Playwright Proxy Authentication...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Test 1: Without proxy
|
||||
console.log('\n📍 Test 1: Without proxy');
|
||||
const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip'
|
||||
);
|
||||
|
||||
let events1 = 0;
|
||||
let ip1 = null;
|
||||
page1.onNetworkEvent(event => {
|
||||
events1++;
|
||||
console.log(` 📡 Event: ${event.type} ${event.url}`);
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ip1 = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🌐 Your IP: ${ip1}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page1.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
await Browser.closeContext(ctx1);
|
||||
console.log(` Events captured: ${events1}`);
|
||||
|
||||
// Test 2: With proxy using new authentication method
|
||||
console.log('\n🔒 Test 2: With proxy (new auth method)');
|
||||
const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let events2 = 0;
|
||||
let ip2 = null;
|
||||
page2.onNetworkEvent(event => {
|
||||
events2++;
|
||||
console.log(` 📡 Event: ${event.type} ${event.url}`);
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ip2 = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🔄 Proxy IP: ${ip2}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page2.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
await Browser.closeContext(ctx2);
|
||||
console.log(` Events captured: ${events2}`);
|
||||
|
||||
// Results
|
||||
console.log('\n📊 Results:');
|
||||
console.log(` 🌐 Direct IP: ${ip1 || 'Not captured'}`);
|
||||
console.log(` 🔄 Proxy IP: ${ip2 || 'Not captured'}`);
|
||||
console.log(` 📡 Direct events: ${events1}`);
|
||||
console.log(` 📡 Proxy events: ${events2}`);
|
||||
|
||||
if (ip1 && ip2 && ip1 !== ip2) {
|
||||
console.log('✅ Proxy authentication is working - different IPs detected!');
|
||||
return true;
|
||||
} else if (events1 > 0 || events2 > 0) {
|
||||
console.log('⚠️ Network monitoring working, but proxy may not be changing IP');
|
||||
return true;
|
||||
} else {
|
||||
console.log('❌ No network events captured');
|
||||
return false;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testManualPageEvaluation() {
|
||||
console.log('\n🧪 Test 3: Manual page evaluation (without network monitoring)');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
// Try to get the page content directly
|
||||
const title = await page.title();
|
||||
console.log(` 📄 Page title: "${title}"`);
|
||||
|
||||
// Try to evaluate some JavaScript
|
||||
const result = await page.evaluate(() => {
|
||||
return {
|
||||
url: window.location.href,
|
||||
userAgent: navigator.userAgent.substring(0, 50),
|
||||
readyState: document.readyState,
|
||||
};
|
||||
});
|
||||
|
||||
console.log(` 🔍 Page info:`, result);
|
||||
|
||||
// Try to get page content
|
||||
const bodyText = await page.locator('body').textContent();
|
||||
if (bodyText) {
|
||||
console.log(` 📝 Body content (first 200 chars): ${bodyText.substring(0, 200)}...`);
|
||||
|
||||
// Check if it looks like an IP response
|
||||
if (bodyText.includes('origin')) {
|
||||
console.log(' ✅ Looks like httpbin.org response!');
|
||||
}
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return true;
|
||||
|
||||
} catch (error) {
|
||||
console.error(' ❌ Manual evaluation failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runProxyTests() {
|
||||
console.log('🚀 Playwright Proxy Authentication Tests\n');
|
||||
|
||||
const authResult = await testPlaywrightProxyAuth();
|
||||
const manualResult = await testManualPageEvaluation();
|
||||
|
||||
console.log(`\n🏁 Final Results:`);
|
||||
console.log(` 🔐 Proxy auth test: ${authResult ? '✅ PASS' : '❌ FAIL'}`);
|
||||
console.log(` 🧪 Manual eval test: ${manualResult ? '✅ PASS' : '❌ FAIL'}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runProxyTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testPlaywrightProxyAuth, testManualPageEvaluation };
|
||||
0
test-proxy.ts
Normal file
0
test-proxy.ts
Normal file
151
test-simple-proxy.ts
Normal file
151
test-simple-proxy.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function simpleProxyTest() {
|
||||
console.log('🔬 Simple Proxy Test...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
console.log('✅ Page created with proxy');
|
||||
let summaryData: any = null;
|
||||
let eventCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
// Capture the summary API response
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.responseData) {
|
||||
console.log(`📊 Summary API Response Data: ${event.responseData}`);
|
||||
try {
|
||||
summaryData = JSON.parse(event.responseData) as any;
|
||||
const totalCount = summaryData[0].totalCount;
|
||||
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
|
||||
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
|
||||
} catch (e) {
|
||||
console.log('📊 Raw Summary Response:', event.responseData);
|
||||
}
|
||||
}
|
||||
}
|
||||
eventCount++;
|
||||
console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
|
||||
});
|
||||
|
||||
console.log('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 8000 });
|
||||
console.log('✅ Page loaded');
|
||||
|
||||
// RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button
|
||||
try {
|
||||
console.log('🔍 Looking for Products tab...');
|
||||
|
||||
// Wait for the page to fully load
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
// First, click on the Products tab
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Products tab');
|
||||
|
||||
console.log('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
console.log('✅ Products tab clicked');
|
||||
|
||||
// Wait for the tab content to load
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
// Click on the Asset Classes accordion to expand it
|
||||
console.log('🔍 Looking for Asset Classes accordion...');
|
||||
const assetClassesAccordion = page.locator(
|
||||
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
|
||||
);
|
||||
await assetClassesAccordion.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Asset Classes accordion');
|
||||
|
||||
console.log('🖱️ Clicking Asset Classes accordion...');
|
||||
await assetClassesAccordion.click();
|
||||
console.log('✅ Asset Classes accordion clicked');
|
||||
|
||||
// Wait for the accordion content to expand
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
console.log('🔍 Looking for Stocks checkbox...');
|
||||
|
||||
// Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks"
|
||||
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
|
||||
await stocksSpan.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Stocks span');
|
||||
|
||||
// Find the checkbox by looking in the same parent container
|
||||
const parentContainer = stocksSpan.locator('..');
|
||||
const checkbox = parentContainer.locator('input[type="checkbox"]');
|
||||
|
||||
if ((await checkbox.count()) > 0) {
|
||||
console.log('📋 Clicking Stocks checkbox...');
|
||||
await checkbox.first().check();
|
||||
console.log('✅ Stocks checkbox checked');
|
||||
} else {
|
||||
console.log('⚠️ Could not find checkbox near Stocks text');
|
||||
}
|
||||
|
||||
// Wait a moment for any UI updates
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Find and click the nearest Apply button
|
||||
console.log('🔍 Looking for Apply button...');
|
||||
const applyButton = page.locator(
|
||||
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
|
||||
);
|
||||
|
||||
if ((await applyButton.count()) > 0) {
|
||||
console.log('🎯 Clicking Apply button...');
|
||||
await applyButton.first().click();
|
||||
console.log('✅ Apply button clicked');
|
||||
|
||||
// Wait for any network requests triggered by the Apply button
|
||||
await page.waitForTimeout(2000);
|
||||
} else {
|
||||
console.log('⚠️ Could not find Apply button');
|
||||
}
|
||||
} catch (interactionError) {
|
||||
const errorMessage =
|
||||
interactionError instanceof Error ? interactionError.message : String(interactionError);
|
||||
console.error('❌ Page interaction failed:', errorMessage);
|
||||
|
||||
// Get debug info about the page
|
||||
try {
|
||||
const title = await page.title();
|
||||
console.log(`📄 Current page title: "${title}"`);
|
||||
|
||||
const stocksElements = await page.locator('*:has-text("Stocks")').count();
|
||||
console.log(`🔍 Found ${stocksElements} elements containing "Stocks"`);
|
||||
|
||||
const applyButtons = await page
|
||||
.locator('button:has-text("Apply"), input[value*="Apply"]')
|
||||
.count();
|
||||
console.log(`🔍 Found ${applyButtons} Apply buttons`);
|
||||
} catch (debugError) {
|
||||
const debugMessage = debugError instanceof Error ? debugError.message : String(debugError);
|
||||
console.log('❌ Could not get debug info:', debugMessage);
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
console.log(`📊 Total events: ${eventCount}`);
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
await Browser.close();
|
||||
|
||||
console.log('✅ Test completed');
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error('❌ Error:', errorMessage);
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
simpleProxyTest();
|
||||
1
test-simple.ts
Normal file
1
test-simple.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
console.log("Testing browser import..."); import { Browser } from "@stock-bot/browser"; console.log("Browser imported successfully:", typeof Browser); Browser.initialize().then(() => console.log("Browser initialized")).catch(e => console.error("Error:", e));
|
||||
0
test-user-agent.js
Normal file
0
test-user-agent.js
Normal file
Loading…
Add table
Add a link
Reference in a new issue