diff --git a/.env b/.env index c141355..fba692b 100644 --- a/.env +++ b/.env @@ -42,10 +42,10 @@ QUESTDB_PASSWORD=quest # MongoDB Configuration MONGODB_HOST=localhost MONGODB_PORT=27017 -MONGODB_DB=stockbot -MONGODB_USER= -MONGODB_PASSWORD= -MONGODB_URI=mongodb://localhost:27017/stockbot +MONGODB_DATABASE=stock +MONGODB_USERNAME=trading_admin +MONGODB_PASSWORD=trading_mongo_dev +MONGODB_URI=mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin # =========================================== # DATA PROVIDER CONFIGURATIONS diff --git a/apps/data-service/src/index.ts b/apps/data-service/src/index.ts index 83ffb59..8700374 100644 --- a/apps/data-service/src/index.ts +++ b/apps/data-service/src/index.ts @@ -5,6 +5,7 @@ import { Hono } from 'hono'; import { Browser } from '@stock-bot/browser'; import { loadEnvVariables } from '@stock-bot/config'; import { getLogger, shutdownLoggers } from '@stock-bot/logger'; +import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client'; import { Shutdown } from '@stock-bot/shutdown'; import { initializeIBResources } from './providers/ib.tasks'; import { initializeProxyResources } from './providers/proxy.tasks'; @@ -18,7 +19,7 @@ loadEnvVariables(); const app = new Hono(); const logger = getLogger('data-service'); const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002'); -let server: any = null; +let server: ReturnType | null = null; // Initialize shutdown manager with 15 second timeout const shutdown = Shutdown.getInstance({ timeout: 15000 }); @@ -35,6 +36,11 @@ async function initializeServices() { logger.info('Initializing data service...'); try { + // Initialize MongoDB client first + logger.info('Starting MongoDB client initialization...'); + await connectMongoDB(); + logger.info('MongoDB client initialized'); + // Initialize browser resources logger.info('Starting browser resources initialization...'); await Browser.initialize(); @@ -122,6 +128,18 @@ shutdown.onShutdown(async () => { } }); +// Add MongoDB shutdown handler +shutdown.onShutdown(async () => { + logger.info('Shutting down MongoDB client...'); + try { + await disconnectMongoDB(); + logger.info('MongoDB client shut down successfully'); + } catch (error) { + logger.error('Error shutting down MongoDB client', { error }); + // Don't throw here to allow other shutdown handlers to complete + } +}); + // Add logger shutdown handler (should be last) shutdown.onShutdown(async () => { try { diff --git a/apps/data-service/src/providers/ib.provider.ts b/apps/data-service/src/providers/ib.provider.ts index d97ac0c..3b1edd2 100644 --- a/apps/data-service/src/providers/ib.provider.ts +++ b/apps/data-service/src/providers/ib.provider.ts @@ -6,31 +6,36 @@ const logger = getLogger('ib-provider'); export const ibProvider: ProviderConfig = { name: 'ib', operations: { - 'ib-basics': async () => { + 'ib-exchanges-and-symbols': async () => { const { ibTasks } = await import('./ib.tasks'); logger.info('Fetching symbol summary from IB'); const sessionHeaders = await ibTasks.fetchSession(); - logger.info('Fetched symbol summary from IB', { - sessionHeaders, - }); + logger.info('Fetched symbol summary from IB'); - // Get Exchanges - logger.info('Fetching exchanges from IB'); - const exchanges = await ibTasks.fetchExchanges(sessionHeaders); - logger.info('Fetched exchanges from IB', { exchanges }); - // return total; + if (sessionHeaders) { + logger.info('Fetching exchanges from IB'); + const exchanges = await ibTasks.fetchExchanges(sessionHeaders); + logger.info('Fetched exchanges from IB', { count: exchanges.lenght }); + + // do the same as above but for symbols + logger.info('Fetching symbols from IB'); + const symbols = await ibTasks.fetchSymbols(sessionHeaders); + logger.info('Fetched symbols from IB', { symbols }); + + return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length }; + } }, }, scheduledJobs: [ { - type: 'ib-basics', - operation: 'ib-basics', + type: 'ib-exchanges-and-symbols', + operation: 'ib-exchanges-and-symbols', payload: {}, // should remove and just run at the same time so app restarts dont keeping adding same jobs - cronPattern: '*/2 * * * *', + cronPattern: '0 0 * * 0', priority: 5, - immediately: true, // Don't run immediately during startup to avoid conflicts + // immediately: true, // Don't run immediately during startup to avoid conflicts description: 'Fetch and validate proxy list from sources', }, ], diff --git a/apps/data-service/src/providers/ib.tasks.ts b/apps/data-service/src/providers/ib.tasks.ts index ee0f1fb..5a4a2a8 100644 --- a/apps/data-service/src/providers/ib.tasks.ts +++ b/apps/data-service/src/providers/ib.tasks.ts @@ -1,12 +1,13 @@ import { Browser } from '@stock-bot/browser'; import { getLogger } from '@stock-bot/logger'; +import { getMongoDBClient } from '@stock-bot/mongodb-client'; // Shared instances (module-scoped, not global) let isInitialized = false; // Track if resources are initialized let logger: ReturnType; // let cache: CacheProvider; -export async function initializeIBResources(waitForCache = false): Promise { +export async function initializeIBResources(): Promise { // Skip if already initialized if (isInitialized) { return; @@ -93,7 +94,7 @@ export async function fetchSession(): Promise | undefined // Wait for and return headers immediately when captured logger.info('⏳ Waiting for headers to be captured...'); const headers = await headersPromise; - + page.close(); if (headers) { logger.info('✅ Headers captured successfully'); } else { @@ -151,19 +152,156 @@ export async function fetchExchanges(sessionHeaders: Record): Pr } const data = await response.json(); + const exchanges = data?.exchanges || []; + logger.info('✅ Exchange data fetched successfully'); - logger.info('✅ Exchange data fetched successfully', { - dataKeys: Object.keys(data || {}), - dataSize: JSON.stringify(data).length, + logger.info('Saving IB exchanges to MongoDB...'); + const client = getMongoDBClient(); + await client.batchUpsert('ib_exchanges', exchanges, ['id', 'country_code']); + logger.info('✅ Exchange IB data saved to MongoDB:', { + count: exchanges.length, }); - return data; + return exchanges; } catch (error) { logger.error('❌ Failed to fetch exchanges', { error }); return null; } } + +// Fetch symbols from IB using the session headers +export async function fetchSymbols(sessionHeaders: Record): Promise { + try { + logger.info('🔍 Fetching symbols with session headers...'); + // Configure the proxy + const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'; + // Prepare headers - include all session headers plus any additional ones + const requestHeaders = { + ...sessionHeaders, + Accept: 'application/json, text/plain, */*', + 'Accept-Language': 'en-US,en;q=0.9', + 'Cache-Control': 'no-cache', + Pragma: 'no-cache', + 'Sec-Fetch-Dest': 'empty', + 'Sec-Fetch-Mode': 'cors', + 'Sec-Fetch-Site': 'same-origin', + 'X-Requested-With': 'XMLHttpRequest', + }; + + const requestBody = { + domain: 'com', + newProduct: 'all', + pageNumber: 1, + pageSize: 100, + productCountry: ['CA', 'US'], + productSymbol: '', + productType: ['STK'], + sortDirection: 'asc', + sortField: 'symbol', + }; + + // Get Summary + const summaryResponse = await fetch( + 'https://www.interactivebrokers.com/webrest/search/product-types/summary', + { + method: 'POST', + headers: requestHeaders, + proxy: proxyUrl, + body: JSON.stringify(requestBody), + } + ); + + if (!summaryResponse.ok) { + logger.error('❌ Summary API request failed', { + status: summaryResponse.status, + statusText: summaryResponse.statusText, + }); + return null; + } + + const summaryData = await summaryResponse.json(); + logger.info('✅ IB Summary data fetched successfully', { + totalCount: summaryData[0].totalCount, + }); + + const symbols = []; + requestBody.pageSize = 500; + const pageCount = Math.ceil(summaryData[0].totalCount / 500) || 0; + logger.info('Fetching Symbols for IB', { pageCount }); + const symbolPromises = []; + for (let page = 1; page <= pageCount; page++) { + requestBody.pageNumber = page; + + // Fetch symbols for the current page + const symbolsResponse = fetch( + 'https://www.interactivebrokers.com/webrest/search/products-by-filters', + { + method: 'POST', + headers: requestHeaders, + proxy: proxyUrl, + body: JSON.stringify(requestBody), + } + ); + symbolPromises.push(symbolsResponse); + } + const responses = await Promise.all(symbolPromises); + for (const response of responses) { + if (!response.ok) { + logger.error('❌ Symbols API request failed', { + status: response.status, + statusText: response.statusText, + }); + return null; + } + const data = await response.json(); + const symJson = data?.products || []; + if (symJson && symJson.length > 0) { + symbols.push(...symJson); + } else { + logger.warn('⚠️ No symbols found in response'); + continue; + } + } + if (symbols.length === 0) { + logger.warn('⚠️ No symbols fetched from IB'); + return null; + } + + logger.info('✅ IB symbols fetched successfully, saving to DB...', { + totalSymbols: symbols.length, + }); + const client = getMongoDBClient(); + await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']); + logger.info('Saved IB symbols to DB', { + totalSymbols: symbols.length, + }); + // logger.info('📤 Making request to exchange API...', { + // url: exchangeUrl, + // headerCount: Object.keys(requestHeaders).length, + // }); + + // // Use fetch with proxy configuration + // const response = await fetch(exchangeUrl, { + // method: 'GET', + // headers: requestHeaders, + // proxy: proxyUrl, + // }); + + // if (!response.ok) { + // logger.error('❌ Exchange API request failed', { + // status: response.status, + // statusText: response.statusText, + // }); + // return null; + // } + } catch (error) { + logger.error('❌ Failed to fetch symbols', { error }); + return null; + } +} + export const ibTasks = { + fetchSymbols, fetchSession, fetchExchanges, }; diff --git a/database/mongodb/init/01-init-collections.js b/database/mongodb/init/01-init-collections.js index 4923594..8be438c 100644 --- a/database/mongodb/init/01-init-collections.js +++ b/database/mongodb/init/01-init-collections.js @@ -2,231 +2,231 @@ // This script creates collections and indexes for sentiment and document storage // Switch to the trading_documents database -db = db.getSiblingDB('trading_documents'); +db = db.getSiblingDB('stock'); -// Create collections with validation schemas +// // Create collections with validation schemas // Sentiment Analysis Collection -db.createCollection('sentiment_analysis', { - validator: { - $jsonSchema: { - bsonType: 'object', - required: ['symbol', 'source', 'timestamp', 'sentiment_score'], - properties: { - symbol: { - bsonType: 'string', - description: 'Stock symbol (e.g., AAPL, GOOGL)' - }, - source: { - bsonType: 'string', - description: 'Data source (news, social, earnings_call, etc.)' - }, - timestamp: { - bsonType: 'date', - description: 'When the sentiment was recorded' - }, - sentiment_score: { - bsonType: 'double', - minimum: -1.0, - maximum: 1.0, - description: 'Sentiment score between -1 (negative) and 1 (positive)' - }, - confidence: { - bsonType: 'double', - minimum: 0.0, - maximum: 1.0, - description: 'Confidence level of the sentiment analysis' - }, - text_snippet: { - bsonType: 'string', - description: 'Original text that was analyzed' - }, - metadata: { - bsonType: 'object', - description: 'Additional metadata about the sentiment source' - } - } - } - } -}); +// db.createCollection('sentiment_analysis', { +// validator: { +// $jsonSchema: { +// bsonType: 'object', +// required: ['symbol', 'source', 'timestamp', 'sentiment_score'], +// properties: { +// symbol: { +// bsonType: 'string', +// description: 'Stock symbol (e.g., AAPL, GOOGL)' +// }, +// source: { +// bsonType: 'string', +// description: 'Data source (news, social, earnings_call, etc.)' +// }, +// timestamp: { +// bsonType: 'date', +// description: 'When the sentiment was recorded' +// }, +// sentiment_score: { +// bsonType: 'double', +// minimum: -1.0, +// maximum: 1.0, +// description: 'Sentiment score between -1 (negative) and 1 (positive)' +// }, +// confidence: { +// bsonType: 'double', +// minimum: 0.0, +// maximum: 1.0, +// description: 'Confidence level of the sentiment analysis' +// }, +// text_snippet: { +// bsonType: 'string', +// description: 'Original text that was analyzed' +// }, +// metadata: { +// bsonType: 'object', +// description: 'Additional metadata about the sentiment source' +// } +// } +// } +// } +// }); -// Raw Documents Collection (for news articles, social media posts, etc.) -db.createCollection('raw_documents', { - validator: { - $jsonSchema: { - bsonType: 'object', - required: ['source', 'document_type', 'timestamp', 'content'], - properties: { - source: { - bsonType: 'string', - description: 'Document source (news_api, twitter, reddit, etc.)' - }, - document_type: { - bsonType: 'string', - enum: ['news_article', 'social_post', 'earnings_transcript', 'research_report', 'press_release'], - description: 'Type of document' - }, - timestamp: { - bsonType: 'date', - description: 'When the document was created/published' - }, - symbols: { - bsonType: 'array', - items: { - bsonType: 'string' - }, - description: 'Array of stock symbols mentioned in the document' - }, - title: { - bsonType: 'string', - description: 'Document title or headline' - }, - content: { - bsonType: 'string', - description: 'Full document content' - }, - url: { - bsonType: 'string', - description: 'Original URL of the document' - }, - author: { - bsonType: 'string', - description: 'Document author or source account' - }, - processed: { - bsonType: 'bool', - description: 'Whether this document has been processed for sentiment' - }, - metadata: { - bsonType: 'object', - description: 'Additional document metadata' - } - } - } - } -}); +// // Raw Documents Collection (for news articles, social media posts, etc.) +// db.createCollection('raw_documents', { +// validator: { +// $jsonSchema: { +// bsonType: 'object', +// required: ['source', 'document_type', 'timestamp', 'content'], +// properties: { +// source: { +// bsonType: 'string', +// description: 'Document source (news_api, twitter, reddit, etc.)' +// }, +// document_type: { +// bsonType: 'string', +// enum: ['news_article', 'social_post', 'earnings_transcript', 'research_report', 'press_release'], +// description: 'Type of document' +// }, +// timestamp: { +// bsonType: 'date', +// description: 'When the document was created/published' +// }, +// symbols: { +// bsonType: 'array', +// items: { +// bsonType: 'string' +// }, +// description: 'Array of stock symbols mentioned in the document' +// }, +// title: { +// bsonType: 'string', +// description: 'Document title or headline' +// }, +// content: { +// bsonType: 'string', +// description: 'Full document content' +// }, +// url: { +// bsonType: 'string', +// description: 'Original URL of the document' +// }, +// author: { +// bsonType: 'string', +// description: 'Document author or source account' +// }, +// processed: { +// bsonType: 'bool', +// description: 'Whether this document has been processed for sentiment' +// }, +// metadata: { +// bsonType: 'object', +// description: 'Additional document metadata' +// } +// } +// } +// } +// }); -// Market Events Collection (for significant market events and their impact) -db.createCollection('market_events', { - validator: { - $jsonSchema: { - bsonType: 'object', - required: ['event_type', 'timestamp', 'description'], - properties: { - event_type: { - bsonType: 'string', - enum: ['earnings', 'merger', 'acquisition', 'ipo', 'dividend', 'split', 'regulatory', 'economic_indicator'], - description: 'Type of market event' - }, - timestamp: { - bsonType: 'date', - description: 'When the event occurred or was announced' - }, - symbols: { - bsonType: 'array', - items: { - bsonType: 'string' - }, - description: 'Stock symbols affected by this event' - }, - description: { - bsonType: 'string', - description: 'Event description' - }, - impact_score: { - bsonType: 'double', - minimum: -5.0, - maximum: 5.0, - description: 'Expected market impact score' - }, - source_documents: { - bsonType: 'array', - items: { - bsonType: 'objectId' - }, - description: 'References to raw_documents that reported this event' - } - } - } - } -}); +// // Market Events Collection (for significant market events and their impact) +// db.createCollection('market_events', { +// validator: { +// $jsonSchema: { +// bsonType: 'object', +// required: ['event_type', 'timestamp', 'description'], +// properties: { +// event_type: { +// bsonType: 'string', +// enum: ['earnings', 'merger', 'acquisition', 'ipo', 'dividend', 'split', 'regulatory', 'economic_indicator'], +// description: 'Type of market event' +// }, +// timestamp: { +// bsonType: 'date', +// description: 'When the event occurred or was announced' +// }, +// symbols: { +// bsonType: 'array', +// items: { +// bsonType: 'string' +// }, +// description: 'Stock symbols affected by this event' +// }, +// description: { +// bsonType: 'string', +// description: 'Event description' +// }, +// impact_score: { +// bsonType: 'double', +// minimum: -5.0, +// maximum: 5.0, +// description: 'Expected market impact score' +// }, +// source_documents: { +// bsonType: 'array', +// items: { +// bsonType: 'objectId' +// }, +// description: 'References to raw_documents that reported this event' +// } +// } +// } +// } +// }); -// Create indexes for efficient querying +// // Create indexes for efficient querying -// Sentiment Analysis indexes -db.sentiment_analysis.createIndex({ symbol: 1, timestamp: -1 }); -db.sentiment_analysis.createIndex({ source: 1, timestamp: -1 }); -db.sentiment_analysis.createIndex({ timestamp: -1 }); -db.sentiment_analysis.createIndex({ symbol: 1, source: 1, timestamp: -1 }); +// // Sentiment Analysis indexes +// db.sentiment_analysis.createIndex({ symbol: 1, timestamp: -1 }); +// db.sentiment_analysis.createIndex({ source: 1, timestamp: -1 }); +// db.sentiment_analysis.createIndex({ timestamp: -1 }); +// db.sentiment_analysis.createIndex({ symbol: 1, source: 1, timestamp: -1 }); -// Raw Documents indexes -db.raw_documents.createIndex({ symbols: 1, timestamp: -1 }); -db.raw_documents.createIndex({ source: 1, timestamp: -1 }); -db.raw_documents.createIndex({ document_type: 1, timestamp: -1 }); -db.raw_documents.createIndex({ processed: 1, timestamp: -1 }); -db.raw_documents.createIndex({ timestamp: -1 }); +// // Raw Documents indexes +// db.raw_documents.createIndex({ symbols: 1, timestamp: -1 }); +// db.raw_documents.createIndex({ source: 1, timestamp: -1 }); +// db.raw_documents.createIndex({ document_type: 1, timestamp: -1 }); +// db.raw_documents.createIndex({ processed: 1, timestamp: -1 }); +// db.raw_documents.createIndex({ timestamp: -1 }); -// Market Events indexes -db.market_events.createIndex({ symbols: 1, timestamp: -1 }); -db.market_events.createIndex({ event_type: 1, timestamp: -1 }); -db.market_events.createIndex({ timestamp: -1 }); +// // Market Events indexes +// db.market_events.createIndex({ symbols: 1, timestamp: -1 }); +// db.market_events.createIndex({ event_type: 1, timestamp: -1 }); +// db.market_events.createIndex({ timestamp: -1 }); -// Insert some sample data for testing +// // Insert some sample data for testing -// Sample sentiment data -db.sentiment_analysis.insertMany([ - { - symbol: 'AAPL', - source: 'news_analysis', - timestamp: new Date(), - sentiment_score: 0.75, - confidence: 0.89, - text_snippet: 'Apple reports strong quarterly earnings...', - metadata: { - article_id: 'news_001', - provider: 'financial_news_api' - } - }, - { - symbol: 'GOOGL', - source: 'social_media', - timestamp: new Date(), - sentiment_score: -0.25, - confidence: 0.67, - text_snippet: 'Concerns about Google AI regulation...', - metadata: { - platform: 'twitter', - engagement_score: 450 - } - } -]); +// // Sample sentiment data +// db.sentiment_analysis.insertMany([ +// { +// symbol: 'AAPL', +// source: 'news_analysis', +// timestamp: new Date(), +// sentiment_score: 0.75, +// confidence: 0.89, +// text_snippet: 'Apple reports strong quarterly earnings...', +// metadata: { +// article_id: 'news_001', +// provider: 'financial_news_api' +// } +// }, +// { +// symbol: 'GOOGL', +// source: 'social_media', +// timestamp: new Date(), +// sentiment_score: -0.25, +// confidence: 0.67, +// text_snippet: 'Concerns about Google AI regulation...', +// metadata: { +// platform: 'twitter', +// engagement_score: 450 +// } +// } +// ]); -// Sample raw document -db.raw_documents.insertOne({ - source: 'financial_news_api', - document_type: 'news_article', - timestamp: new Date(), - symbols: ['AAPL', 'MSFT'], - title: 'Tech Giants Show Strong Q4 Performance', - content: 'Apple and Microsoft both reported better than expected earnings for Q4...', - url: 'https://example.com/tech-earnings-q4', - author: 'Financial Reporter', - processed: true, - metadata: { - word_count: 850, - readability_score: 0.75 - } -}); +// // Sample raw document +// db.raw_documents.insertOne({ +// source: 'financial_news_api', +// document_type: 'news_article', +// timestamp: new Date(), +// symbols: ['AAPL', 'MSFT'], +// title: 'Tech Giants Show Strong Q4 Performance', +// content: 'Apple and Microsoft both reported better than expected earnings for Q4...', +// url: 'https://example.com/tech-earnings-q4', +// author: 'Financial Reporter', +// processed: true, +// metadata: { +// word_count: 850, +// readability_score: 0.75 +// } +// }); -// Sample market event -db.market_events.insertOne({ - event_type: 'earnings', - timestamp: new Date(), - symbols: ['AAPL'], - description: 'Apple Q4 2024 Earnings Report', - impact_score: 2.5, - source_documents: [] -}); +// // Sample market event +// db.market_events.insertOne({ +// event_type: 'earnings', +// timestamp: new Date(), +// symbols: ['AAPL'], +// description: 'Apple Q4 2024 Earnings Report', +// impact_score: 2.5, +// source_documents: [] +// }); print('MongoDB initialization completed successfully!'); print('Created collections: sentiment_analysis, raw_documents, market_events'); diff --git a/test-network.ts b/database/postgres/providers/01-ib-simple.sql similarity index 100% rename from test-network.ts rename to database/postgres/providers/01-ib-simple.sql diff --git a/test-proxy.ts b/database/postgres/scripts/ib.ts similarity index 100% rename from test-proxy.ts rename to database/postgres/scripts/ib.ts diff --git a/test-user-agent.js b/database/postgres/scripts/populate-ib-exchanges-simple.ts similarity index 100% rename from test-user-agent.js rename to database/postgres/scripts/populate-ib-exchanges-simple.ts diff --git a/database/postgres/scripts/populate-ib-exchanges.ts b/database/postgres/scripts/populate-ib-exchanges.ts new file mode 100644 index 0000000..e69de29 diff --git a/database/postgres/scripts/setup-ib-fast.ts b/database/postgres/scripts/setup-ib-fast.ts new file mode 100644 index 0000000..e69de29 diff --git a/database/postgres/scripts/setup-ib-schema-simple.ts b/database/postgres/scripts/setup-ib-schema-simple.ts new file mode 100644 index 0000000..e69de29 diff --git a/database/postgres/scripts/setup-ib-schema.ts b/database/postgres/scripts/setup-ib-schema.ts new file mode 100644 index 0000000..e69de29 diff --git a/database/postgres/scripts/setup.ts b/database/postgres/scripts/setup.ts new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml index 3ecf0f5..395aeb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,7 +82,7 @@ services: # Dragonfly - Redis replacement for caching and events environment: MONGO_INITDB_ROOT_USERNAME: trading_admin MONGO_INITDB_ROOT_PASSWORD: trading_mongo_dev - MONGO_INITDB_DATABASE: trading_documents + MONGO_INITDB_DATABASE: stock ports: - "27017:27017" volumes: diff --git a/libs/config/src/mongodb.ts b/libs/config/src/mongodb.ts index bc50c86..9e552c3 100644 --- a/libs/config/src/mongodb.ts +++ b/libs/config/src/mongodb.ts @@ -13,7 +13,7 @@ export const mongodbConfig = cleanEnv(process.env, { // MongoDB Connection MONGODB_HOST: str('localhost', 'MongoDB host'), MONGODB_PORT: port(27017, 'MongoDB port'), - MONGODB_DATABASE: str('trading_documents', 'MongoDB database name'), + MONGODB_DATABASE: str('stock', 'MongoDB database name'), // Authentication MONGODB_USERNAME: str('trading_admin', 'MongoDB username'), diff --git a/libs/mongodb-client/src/aggregation.ts b/libs/mongodb-client/src/aggregation.ts deleted file mode 100644 index a767d46..0000000 --- a/libs/mongodb-client/src/aggregation.ts +++ /dev/null @@ -1,249 +0,0 @@ -import type { Document } from 'mongodb'; -import type { MongoDBClient } from './client'; -import type { CollectionNames } from './types'; - -/** - * MongoDB Aggregation Builder - * - * Provides a fluent interface for building MongoDB aggregation pipelines - */ -export class MongoDBAggregationBuilder { - private pipeline: any[] = []; - private readonly client: MongoDBClient; - private collection: CollectionNames | null = null; - - constructor(client: MongoDBClient) { - this.client = client; - } - - /** - * Set the collection to aggregate on - */ - from(collection: CollectionNames): this { - this.collection = collection; - return this; - } - - /** - * Add a match stage - */ - match(filter: any): this { - this.pipeline.push({ $match: filter }); - return this; - } - - /** - * Add a group stage - */ - group(groupBy: any): this { - this.pipeline.push({ $group: groupBy }); - return this; - } - - /** - * Add a sort stage - */ - sort(sortBy: any): this { - this.pipeline.push({ $sort: sortBy }); - return this; - } - - /** - * Add a limit stage - */ - limit(count: number): this { - this.pipeline.push({ $limit: count }); - return this; - } - - /** - * Add a skip stage - */ - skip(count: number): this { - this.pipeline.push({ $skip: count }); - return this; - } - - /** - * Add a project stage - */ - project(projection: any): this { - this.pipeline.push({ $project: projection }); - return this; - } - - /** - * Add an unwind stage - */ - unwind(field: string, options?: any): this { - this.pipeline.push({ - $unwind: options ? { path: field, ...options } : field, - }); - return this; - } - - /** - * Add a lookup stage (join) - */ - lookup(from: string, localField: string, foreignField: string, as: string): this { - this.pipeline.push({ - $lookup: { - from, - localField, - foreignField, - as, - }, - }); - return this; - } - - /** - * Add a custom stage - */ - addStage(stage: any): this { - this.pipeline.push(stage); - return this; - } - /** - * Execute the aggregation pipeline - */ - async execute(): Promise { - if (!this.collection) { - throw new Error('Collection not specified. Use .from() to set the collection.'); - } - - const collection = this.client.getCollection(this.collection); - return await collection.aggregate(this.pipeline).toArray(); - } - - /** - * Get the pipeline array - */ - getPipeline(): any[] { - return [...this.pipeline]; - } - - /** - * Reset the pipeline - */ - reset(): this { - this.pipeline = []; - this.collection = null; - return this; - } - - // Convenience methods for common aggregations - - /** - * Sentiment analysis aggregation - */ - sentimentAnalysis(symbol?: string, timeframe?: { start: Date; end: Date }): this { - this.from('sentiment_data'); - - const matchConditions: any = {}; - if (symbol) { - matchConditions.symbol = symbol; - } - if (timeframe) { - matchConditions.timestamp = { - $gte: timeframe.start, - $lte: timeframe.end, - }; - } - - if (Object.keys(matchConditions).length > 0) { - this.match(matchConditions); - } - - return this.group({ - _id: { - symbol: '$symbol', - sentiment: '$sentiment_label', - }, - count: { $sum: 1 }, - avgScore: { $avg: '$sentiment_score' }, - avgConfidence: { $avg: '$confidence' }, - }); - } - - /** - * News article aggregation by publication - */ - newsByPublication(symbols?: string[]): this { - this.from('news_articles'); - - if (symbols && symbols.length > 0) { - this.match({ symbols: { $in: symbols } }); - } - - return this.group({ - _id: '$publication', - articleCount: { $sum: 1 }, - symbols: { $addToSet: '$symbols' }, - avgSentiment: { $avg: '$sentiment_score' }, - latestArticle: { $max: '$published_date' }, - }); - } - - /** - * SEC filings by company - */ - secFilingsByCompany(filingTypes?: string[]): this { - this.from('sec_filings'); - - if (filingTypes && filingTypes.length > 0) { - this.match({ filing_type: { $in: filingTypes } }); - } - - return this.group({ - _id: { - cik: '$cik', - company: '$company_name', - }, - filingCount: { $sum: 1 }, - filingTypes: { $addToSet: '$filing_type' }, - latestFiling: { $max: '$filing_date' }, - symbols: { $addToSet: '$symbols' }, - }); - } - - /** - * Document processing status summary - */ - processingStatusSummary(collection: CollectionNames): this { - this.from(collection); - - return this.group({ - _id: '$processing_status', - count: { $sum: 1 }, - avgSizeBytes: { $avg: '$size_bytes' }, - oldestDocument: { $min: '$created_at' }, - newestDocument: { $max: '$created_at' }, - }); - } - - /** - * Time-based aggregation (daily/hourly counts) - */ - timeBasedCounts( - collection: CollectionNames, - dateField: string = 'created_at', - interval: 'hour' | 'day' | 'week' | 'month' = 'day' - ): this { - this.from(collection); - - const dateFormat = { - hour: { $dateToString: { format: '%Y-%m-%d %H:00:00', date: `$${dateField}` } }, - day: { $dateToString: { format: '%Y-%m-%d', date: `$${dateField}` } }, - week: { $dateToString: { format: '%Y-W%V', date: `$${dateField}` } }, - month: { $dateToString: { format: '%Y-%m', date: `$${dateField}` } }, - }; - - return this.group({ - _id: dateFormat[interval], - count: { $sum: 1 }, - firstDocument: { $min: `$${dateField}` }, - lastDocument: { $max: `$${dateField}` }, - }).sort({ _id: 1 }); - } -} diff --git a/libs/mongodb-client/src/client.ts b/libs/mongodb-client/src/client.ts index d184d42..52d9d26 100644 --- a/libs/mongodb-client/src/client.ts +++ b/libs/mongodb-client/src/client.ts @@ -1,110 +1,68 @@ -import { - Collection, - Db, - Document, - MongoClient, - MongoClientOptions, - OptionalUnlessRequiredId, - WithId, -} from 'mongodb'; -import * as yup from 'yup'; +import { Collection, Db, MongoClient, OptionalUnlessRequiredId } from 'mongodb'; import { mongodbConfig } from '@stock-bot/config'; import { getLogger } from '@stock-bot/logger'; -import { MongoDBHealthMonitor } from './health'; -import { schemaMap } from './schemas'; -import type { - AnalystReport, - CollectionNames, - DocumentBase, - EarningsTranscript, - MongoDBClientConfig, - MongoDBConnectionOptions, - NewsArticle, - RawDocument, - SecFiling, - SentimentData, -} from './types'; +import type { DocumentBase } from './types'; /** - * MongoDB Client for Stock Bot + * Simplified MongoDB Client for Stock Bot Data Service * - * Provides type-safe access to MongoDB collections with built-in - * health monitoring, connection pooling, and schema validation. + * A singleton MongoDB client focused solely on batch upsert operations + * with minimal configuration and no health monitoring complexity. */ export class MongoDBClient { + private static instance: MongoDBClient | null = null; private client: MongoClient | null = null; private db: Db | null = null; - private readonly config: MongoDBClientConfig; - private readonly options: MongoDBConnectionOptions; - private readonly logger: ReturnType; - private readonly healthMonitor: MongoDBHealthMonitor; + private readonly logger = getLogger('mongodb-client-simple'); private isConnected = false; - constructor(config?: Partial, options?: MongoDBConnectionOptions) { - this.config = this.buildConfig(config); - this.options = { - retryAttempts: 3, - retryDelay: 1000, - healthCheckInterval: 30000, - ...options, - }; + private constructor() {} - this.logger = getLogger('mongodb-client'); - this.healthMonitor = new MongoDBHealthMonitor(this); + /** + * Get singleton instance + */ + static getInstance(): MongoDBClient { + if (!MongoDBClient.instance) { + MongoDBClient.instance = new MongoDBClient(); + } + return MongoDBClient.instance; } /** - * Connect to MongoDB + * Connect to MongoDB with simple configuration */ async connect(): Promise { if (this.isConnected && this.client) { return; } - const uri = this.buildConnectionUri(); - const clientOptions = this.buildClientOptions(); + try { + const uri = this.buildConnectionUri(); + this.logger.info('Connecting to MongoDB...'); - let lastError: Error | null = null; + this.client = new MongoClient(uri, { + maxPoolSize: 10, + minPoolSize: 1, + connectTimeoutMS: 10000, + socketTimeoutMS: 30000, + serverSelectionTimeoutMS: 5000, + }); - for (let attempt = 1; attempt <= this.options.retryAttempts!; attempt++) { - try { - this.logger.info( - `Connecting to MongoDB (attempt ${attempt}/${this.options.retryAttempts})...` - ); + await this.client.connect(); + await this.client.db(mongodbConfig.MONGODB_DATABASE).admin().ping(); - this.client = new MongoClient(uri, clientOptions); - await this.client.connect(); + this.db = this.client.db(mongodbConfig.MONGODB_DATABASE); + this.isConnected = true; - // Test the connection - await this.client.db(this.config.database).admin().ping(); - - this.db = this.client.db(this.config.database); - this.isConnected = true; - - this.logger.info('Successfully connected to MongoDB'); - - // Start health monitoring - this.healthMonitor.start(); - - return; - } catch (error) { - lastError = error as Error; - this.logger.error(`MongoDB connection attempt ${attempt} failed:`, error); - - if (this.client) { - await this.client.close(); - this.client = null; - } - - if (attempt < this.options.retryAttempts!) { - await this.delay(this.options.retryDelay! * attempt); - } + this.logger.info('Successfully connected to MongoDB'); + } catch (error) { + this.logger.error('MongoDB connection failed:', error); + if (this.client) { + await this.client.close(); + this.client = null; } + throw error; } - - throw new Error( - `Failed to connect to MongoDB after ${this.options.retryAttempts} attempts: ${lastError?.message}` - ); } /** @@ -116,7 +74,6 @@ export class MongoDBClient { } try { - this.healthMonitor.stop(); await this.client.close(); this.isConnected = false; this.client = null; @@ -128,10 +85,138 @@ export class MongoDBClient { } } + /** + * Batch upsert documents for high-performance operations + * Supports single or multiple unique keys for matching + */ + async batchUpsert( + collectionName: string, + documents: Array< + Omit & Partial> + >, + uniqueKeys: string | string[], + options: { + chunkSize?: number; + } = {} + ): Promise<{ insertedCount: number; updatedCount: number; errors: unknown[] }> { + if (!this.db) { + throw new Error('MongoDB client not connected'); + } + + if (documents.length === 0) { + return { insertedCount: 0, updatedCount: 0, errors: [] }; + } + + // Normalize uniqueKeys to array + const keyFields = Array.isArray(uniqueKeys) ? uniqueKeys : [uniqueKeys]; + + if (keyFields.length === 0) { + throw new Error('At least one unique key must be provided'); + } + + const { chunkSize = 10000 } = options; + const collection = this.db.collection(collectionName); + const operationId = Math.random().toString(36).substring(7); + + let totalInserted = 0; + let totalUpdated = 0; + const errors: unknown[] = []; + + this.logger.info(`Starting batch upsert operation [${operationId}]`, { + collection: collectionName, + totalDocuments: documents.length, + uniqueKeys: keyFields, + chunkSize + }); + + // Process documents in chunks to avoid memory issues + for (let i = 0; i < documents.length; i += chunkSize) { + const chunk = documents.slice(i, i + chunkSize); + + try { + const startTime = Date.now(); + + // Prepare bulk operations + const bulkOps = chunk.map(doc => { + const now = new Date(); + const docWithTimestamps = { + ...doc, + created_at: doc.created_at || now, + updated_at: now, + }; + + // Create filter using multiple unique keys + const filter: Record = {}; + keyFields.forEach(key => { + const value = (doc as Record)[key]; + if (value === undefined || value === null) { + throw new Error(`Document missing required unique key: ${key}`); + } + filter[key] = value; + }); + + // Remove created_at from $set to avoid conflict with $setOnInsert + const { created_at, ...updateFields } = docWithTimestamps; + + return { + updateOne: { + filter, + update: { + $set: updateFields, + $setOnInsert: { created_at }, + }, + upsert: true, + }, + }; + }); + + // Execute bulk operation with type assertion to handle complex MongoDB types + const result = await collection.bulkWrite(bulkOps as never, { ordered: false }); + + const executionTime = Date.now() - startTime; + const inserted = result.upsertedCount; + const updated = result.modifiedCount; + + totalInserted += inserted; + totalUpdated += updated; + + this.logger.debug(`Batch upsert chunk processed [${operationId}]`, { + chunkNumber: Math.floor(i / chunkSize) + 1, + chunkSize: chunk.length, + inserted, + updated, + executionTime, + collection: collectionName, + }); + } catch (error) { + this.logger.error(`Batch upsert failed on chunk [${operationId}]`, { + error, + collection: collectionName, + chunkNumber: Math.floor(i / chunkSize) + 1, + chunkStart: i, + chunkSize: chunk.length, + uniqueKeys: keyFields, + }); + errors.push(error); + } + } + + this.logger.info(`Batch upsert completed [${operationId}]`, { + collection: collectionName, + totalRecords: documents.length, + inserted: totalInserted, + updated: totalUpdated, + errors: errors.length, + uniqueKeys: keyFields, + }); + + return { insertedCount: totalInserted, updatedCount: totalUpdated, errors }; + } + /** * Get a typed collection */ - getCollection(name: CollectionNames): Collection { + getCollection(name: string): Collection { if (!this.db) { throw new Error('MongoDB client not connected'); } @@ -139,162 +224,26 @@ export class MongoDBClient { } /** - * Insert a document with validation + * Simple insert operation */ async insertOne( - collectionName: CollectionNames, + collectionName: string, document: Omit & Partial> ): Promise { const collection = this.getCollection(collectionName); - // Add timestamps const now = new Date(); const docWithTimestamps = { ...document, created_at: document.created_at || now, updated_at: now, - } as T; // Validate document if schema exists - if (collectionName in schemaMap) { - try { - (schemaMap as any)[collectionName].validateSync(docWithTimestamps); - } catch (error) { - if (error instanceof yup.ValidationError) { - this.logger.error(`Document validation failed for ${collectionName}:`, error.errors); - throw new Error(`Document validation failed: ${error.errors?.map(e => e).join(', ')}`); - } - throw error; - } - } + } as T; + const result = await collection.insertOne(docWithTimestamps as OptionalUnlessRequiredId); return { ...docWithTimestamps, _id: result.insertedId } as T; } - /** - * Update a document with validation - */ - async updateOne( - collectionName: CollectionNames, - filter: any, - update: Partial - ): Promise { - const collection = this.getCollection(collectionName); - - // Add updated timestamp - const updateWithTimestamp = { - ...update, - updated_at: new Date(), - }; - - const result = await collection.updateOne(filter, { $set: updateWithTimestamp }); - return result.modifiedCount > 0; - } - /** - * Find documents with optional validation - */ - async find( - collectionName: CollectionNames, - filter: any = {}, - options: any = {} - ): Promise { - const collection = this.getCollection(collectionName); - return (await collection.find(filter, options).toArray()) as T[]; - } - - /** - * Find one document - */ - async findOne( - collectionName: CollectionNames, - filter: any - ): Promise { - const collection = this.getCollection(collectionName); - return (await collection.findOne(filter)) as T | null; - } - - /** - * Aggregate with type safety - */ - async aggregate( - collectionName: CollectionNames, - pipeline: any[] - ): Promise { - const collection = this.getCollection(collectionName); - return await collection.aggregate(pipeline).toArray(); - } - - /** - * Count documents - */ - async countDocuments(collectionName: CollectionNames, filter: any = {}): Promise { - const collection = this.getCollection(collectionName); - return await collection.countDocuments(filter); - } - - /** - * Create indexes for better performance - */ - async createIndexes(): Promise { - if (!this.db) { - throw new Error('MongoDB client not connected'); - } - - try { - // Sentiment data indexes - await this.db - .collection('sentiment_data') - .createIndexes([ - { key: { symbol: 1, timestamp: -1 } }, - { key: { sentiment_label: 1 } }, - { key: { source_type: 1 } }, - { key: { created_at: -1 } }, - ]); - - // News articles indexes - await this.db - .collection('news_articles') - .createIndexes([ - { key: { symbols: 1, published_date: -1 } }, - { key: { publication: 1 } }, - { key: { categories: 1 } }, - { key: { created_at: -1 } }, - ]); - - // SEC filings indexes - await this.db - .collection('sec_filings') - .createIndexes([ - { key: { symbols: 1, filing_date: -1 } }, - { key: { filing_type: 1 } }, - { key: { cik: 1 } }, - { key: { created_at: -1 } }, - ]); // Raw documents indexes - await this.db.collection('raw_documents').createIndex({ content_hash: 1 }, { unique: true }); - await this.db - .collection('raw_documents') - .createIndexes([ - { key: { processing_status: 1 } }, - { key: { document_type: 1 } }, - { key: { created_at: -1 } }, - ]); - - this.logger.info('MongoDB indexes created successfully'); - } catch (error) { - this.logger.error('Error creating MongoDB indexes:', error); - throw error; - } - } - - /** - * Get database statistics - */ - async getStats(): Promise { - if (!this.db) { - throw new Error('MongoDB client not connected'); - } - return await this.db.stats(); - } - /** * Check if client is connected */ @@ -302,13 +251,6 @@ export class MongoDBClient { return this.isConnected && !!this.client; } - /** - * Get the underlying MongoDB client - */ - get mongoClient(): MongoClient | null { - return this.client; - } - /** * Get the database instance */ @@ -316,81 +258,24 @@ export class MongoDBClient { return this.db; } - private buildConfig(config?: Partial): MongoDBClientConfig { - return { - host: config?.host || mongodbConfig.MONGODB_HOST, - port: config?.port || mongodbConfig.MONGODB_PORT, - database: config?.database || mongodbConfig.MONGODB_DATABASE, - username: config?.username || mongodbConfig.MONGODB_USERNAME, - password: config?.password || mongodbConfig.MONGODB_PASSWORD, - authSource: config?.authSource || mongodbConfig.MONGODB_AUTH_SOURCE, - uri: config?.uri || mongodbConfig.MONGODB_URI, - poolSettings: { - maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE, - minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE, - maxIdleTime: mongodbConfig.MONGODB_MAX_IDLE_TIME, - ...config?.poolSettings, - }, - timeouts: { - connectTimeout: mongodbConfig.MONGODB_CONNECT_TIMEOUT, - socketTimeout: mongodbConfig.MONGODB_SOCKET_TIMEOUT, - serverSelectionTimeout: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT, - ...config?.timeouts, - }, - tls: { - enabled: mongodbConfig.MONGODB_TLS, - insecure: mongodbConfig.MONGODB_TLS_INSECURE, - caFile: mongodbConfig.MONGODB_TLS_CA_FILE, - ...config?.tls, - }, - options: { - retryWrites: mongodbConfig.MONGODB_RETRY_WRITES, - journal: mongodbConfig.MONGODB_JOURNAL, - readPreference: mongodbConfig.MONGODB_READ_PREFERENCE as any, - writeConcern: mongodbConfig.MONGODB_WRITE_CONCERN, - ...config?.options, - }, - }; - } - private buildConnectionUri(): string { - if (this.config.uri) { - return this.config.uri; + if (mongodbConfig.MONGODB_URI) { + return mongodbConfig.MONGODB_URI; } - const { host, port, username, password, database, authSource } = this.config; + const { + MONGODB_HOST: host, + MONGODB_PORT: port, + MONGODB_USERNAME: username, + MONGODB_PASSWORD: password, + MONGODB_DATABASE: database, + MONGODB_AUTH_SOURCE: authSource, + } = mongodbConfig; + + // Build URI components const auth = username && password ? `${username}:${password}@` : ''; - const authDb = authSource ? `?authSource=${authSource}` : ''; + const authParam = authSource && username ? `?authSource=${authSource}` : ''; - return `mongodb://${auth}${host}:${port}/${database}${authDb}`; - } - - private buildClientOptions(): MongoClientOptions { - return { - maxPoolSize: this.config.poolSettings?.maxPoolSize, - minPoolSize: this.config.poolSettings?.minPoolSize, - maxIdleTimeMS: this.config.poolSettings?.maxIdleTime, - connectTimeoutMS: this.config.timeouts?.connectTimeout, - socketTimeoutMS: this.config.timeouts?.socketTimeout, - serverSelectionTimeoutMS: this.config.timeouts?.serverSelectionTimeout, - retryWrites: this.config.options?.retryWrites, - journal: this.config.options?.journal, - readPreference: this.config.options?.readPreference, - writeConcern: this.config.options?.writeConcern - ? { - w: - this.config.options.writeConcern === 'majority' - ? ('majority' as const) - : parseInt(this.config.options.writeConcern, 10) || 1, - } - : undefined, - tls: this.config.tls?.enabled, - tlsInsecure: this.config.tls?.insecure, - tlsCAFile: this.config.tls?.caFile, - }; - } - - private delay(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); + return `mongodb://${auth}${host}:${port}/${database}${authParam}`; } } diff --git a/libs/mongodb-client/src/factory.ts b/libs/mongodb-client/src/factory.ts index 78cad95..4ba7455 100644 --- a/libs/mongodb-client/src/factory.ts +++ b/libs/mongodb-client/src/factory.ts @@ -1,56 +1,19 @@ -import { mongodbConfig } from '@stock-bot/config'; import { MongoDBClient } from './client'; -import type { MongoDBClientConfig, MongoDBConnectionOptions } from './types'; /** - * Factory function to create a MongoDB client instance - */ -export function createMongoDBClient( - config?: Partial, - options?: MongoDBConnectionOptions -): MongoDBClient { - return new MongoDBClient(config, options); -} - -/** - * Create a MongoDB client with default configuration - */ -export function createDefaultMongoDBClient(): MongoDBClient { - const config: Partial = { - host: mongodbConfig.MONGODB_HOST, - port: mongodbConfig.MONGODB_PORT, - database: mongodbConfig.MONGODB_DATABASE, - username: mongodbConfig.MONGODB_USERNAME, - password: mongodbConfig.MONGODB_PASSWORD, - uri: mongodbConfig.MONGODB_URI, - }; - - return new MongoDBClient(config); -} - -/** - * Singleton MongoDB client instance - */ -let defaultClient: MongoDBClient | null = null; - -/** - * Get or create the default MongoDB client instance + * Get the singleton MongoDB client instance */ export function getMongoDBClient(): MongoDBClient { - if (!defaultClient) { - defaultClient = createDefaultMongoDBClient(); - } - return defaultClient; + return MongoDBClient.getInstance(); } /** - * Connect to MongoDB using the default client + * Connect to MongoDB using the singleton client */ export async function connectMongoDB(): Promise { const client = getMongoDBClient(); if (!client.connected) { await client.connect(); - await client.createIndexes(); } return client; } @@ -59,8 +22,8 @@ export async function connectMongoDB(): Promise { * Disconnect from MongoDB */ export async function disconnectMongoDB(): Promise { - if (defaultClient) { - await defaultClient.disconnect(); - defaultClient = null; + const client = getMongoDBClient(); + if (client.connected) { + await client.disconnect(); } } diff --git a/libs/mongodb-client/src/health.ts b/libs/mongodb-client/src/health.ts deleted file mode 100644 index b997e8c..0000000 --- a/libs/mongodb-client/src/health.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { getLogger } from '@stock-bot/logger'; -import type { MongoDBClient } from './client'; -import type { MongoDBHealthCheck, MongoDBHealthStatus, MongoDBMetrics } from './types'; - -/** - * MongoDB Health Monitor - * - * Monitors MongoDB connection health and provides metrics - */ -export class MongoDBHealthMonitor { - private readonly client: MongoDBClient; - private readonly logger: ReturnType; - private healthCheckInterval: NodeJS.Timeout | null = null; - private metrics: MongoDBMetrics; - private lastHealthCheck: MongoDBHealthCheck | null = null; - - constructor(client: MongoDBClient) { - this.client = client; - this.logger = getLogger('mongodb-health-monitor'); - this.metrics = { - operationsPerSecond: 0, - averageLatency: 0, - errorRate: 0, - connectionPoolUtilization: 0, - documentsProcessed: 0, - }; - } - - /** - * Start health monitoring - */ - start(intervalMs: number = 30000): void { - if (this.healthCheckInterval) { - this.stop(); - } - - this.logger.info(`Starting MongoDB health monitoring (interval: ${intervalMs}ms)`); - - this.healthCheckInterval = setInterval(async () => { - try { - await this.performHealthCheck(); - } catch (error) { - this.logger.error('Health check failed:', error); - } - }, intervalMs); - - // Perform initial health check - this.performHealthCheck().catch(error => { - this.logger.error('Initial health check failed:', error); - }); - } - - /** - * Stop health monitoring - */ - stop(): void { - if (this.healthCheckInterval) { - clearInterval(this.healthCheckInterval); - this.healthCheckInterval = null; - this.logger.info('Stopped MongoDB health monitoring'); - } - } - - /** - * Get current health status - */ - async getHealth(): Promise { - if (!this.lastHealthCheck) { - await this.performHealthCheck(); - } - return this.lastHealthCheck!; - } - - /** - * Get current metrics - */ - getMetrics(): MongoDBMetrics { - return { ...this.metrics }; - } - - /** - * Perform a health check - */ - private async performHealthCheck(): Promise { - const startTime = Date.now(); - const errors: string[] = []; - let status: MongoDBHealthStatus = 'healthy'; - - try { - if (!this.client.connected) { - errors.push('MongoDB client not connected'); - status = 'unhealthy'; - } else { - // Test basic connectivity - const mongoClient = this.client.mongoClient; - const db = this.client.database; - - if (!mongoClient || !db) { - errors.push('MongoDB client or database not available'); - status = 'unhealthy'; - } else { - // Ping the database - await db.admin().ping(); - - // Get server status for metrics - try { - const serverStatus = await db.admin().serverStatus(); - this.updateMetricsFromServerStatus(serverStatus); - - // Check connection pool status - const poolStats = this.getConnectionPoolStats(serverStatus); - - if (poolStats.utilization > 0.9) { - errors.push('High connection pool utilization'); - status = status === 'healthy' ? 'degraded' : status; - } - - // Check for high latency - const latency = Date.now() - startTime; - if (latency > 1000) { - errors.push(`High latency: ${latency}ms`); - status = status === 'healthy' ? 'degraded' : status; - } - } catch (statusError) { - errors.push(`Failed to get server status: ${(statusError as Error).message}`); - status = 'degraded'; - } - } - } - } catch (error) { - errors.push(`Health check failed: ${(error as Error).message}`); - status = 'unhealthy'; - } - - const latency = Date.now() - startTime; - - // Get connection stats - const connectionStats = this.getConnectionStats(); - - this.lastHealthCheck = { - status, - timestamp: new Date(), - latency, - connections: connectionStats, - errors: errors.length > 0 ? errors : undefined, - }; - - // Log health status changes - if (status !== 'healthy') { - this.logger.warn(`MongoDB health status: ${status}`, { errors, latency }); - } else { - this.logger.debug(`MongoDB health check passed (${latency}ms)`); - } - } - - /** - * Update metrics from MongoDB server status - */ - private updateMetricsFromServerStatus(serverStatus: any): void { - try { - const opcounters = serverStatus.opcounters || {}; - const connections = serverStatus.connections || {}; - const dur = serverStatus.dur || {}; - - // Calculate operations per second (approximate) - const totalOps = Object.values(opcounters).reduce( - (sum: number, count: any) => sum + (count || 0), - 0 - ); - this.metrics.operationsPerSecond = totalOps; - - // Connection pool utilization - if (connections.current && connections.available) { - const total = connections.current + connections.available; - this.metrics.connectionPoolUtilization = connections.current / total; - } - - // Average latency (from durability stats if available) - if (dur.timeMS) { - this.metrics.averageLatency = dur.timeMS.dt || 0; - } - } catch (error) { - this.logger.debug('Error parsing server status for metrics:', error as any); - } - } - - /** - * Get connection pool statistics - */ - private getConnectionPoolStats(serverStatus: any): { - utilization: number; - active: number; - available: number; - } { - const connections = serverStatus.connections || {}; - const active = connections.current || 0; - const available = connections.available || 0; - const total = active + available; - - return { - utilization: total > 0 ? active / total : 0, - active, - available, - }; - } - - /** - * Get connection statistics - */ - private getConnectionStats(): { active: number; available: number; total: number } { - // This would ideally come from the MongoDB driver's connection pool - // For now, we'll return estimated values - return { - active: 1, - available: 9, - total: 10, - }; - } - - /** - * Update error rate metric - */ - updateErrorRate(errorCount: number, totalOperations: number): void { - this.metrics.errorRate = totalOperations > 0 ? errorCount / totalOperations : 0; - } - - /** - * Update documents processed metric - */ - updateDocumentsProcessed(count: number): void { - this.metrics.documentsProcessed += count; - } -} diff --git a/libs/mongodb-client/src/index.ts b/libs/mongodb-client/src/index.ts index 2ecea22..a46552d 100644 --- a/libs/mongodb-client/src/index.ts +++ b/libs/mongodb-client/src/index.ts @@ -1,40 +1,22 @@ /** - * MongoDB Client Library for Stock Bot + * Simplified MongoDB Client Library for Stock Bot Data Service * - * Provides type-safe MongoDB access for document storage, sentiment data, - * and raw content processing. + * Provides a singleton MongoDB client focused on batch upsert operations + * for high-performance data ingestion. */ export { MongoDBClient } from './client'; -export { MongoDBHealthMonitor } from './health'; -export { MongoDBTransactionManager } from './transactions'; -export { MongoDBAggregationBuilder } from './aggregation'; // Types export type { - MongoDBClientConfig, - MongoDBConnectionOptions, - MongoDBHealthStatus, - MongoDBMetrics, - CollectionNames, - DocumentBase, - SentimentData, - RawDocument, - NewsArticle, - SecFiling, - EarningsTranscript, AnalystReport, + DocumentBase, + EarningsTranscript, + NewsArticle, + RawDocument, + SecFiling, + SentimentData, } from './types'; -// Schemas -export { - sentimentDataSchema, - rawDocumentSchema, - newsArticleSchema, - secFilingSchema, - earningsTranscriptSchema, - analystReportSchema, -} from './schemas'; - -// Utils -export { createMongoDBClient } from './factory'; +// Factory functions +export { connectMongoDB, disconnectMongoDB, getMongoDBClient } from './factory'; diff --git a/libs/mongodb-client/src/schemas.ts b/libs/mongodb-client/src/schemas.ts deleted file mode 100644 index 85da534..0000000 --- a/libs/mongodb-client/src/schemas.ts +++ /dev/null @@ -1,146 +0,0 @@ -import * as yup from 'yup'; - -/** - * Yup Schemas for MongoDB Document Validation - */ - -// Base schema for all documents -export const documentBaseSchema = yup.object({ - _id: yup.mixed().optional(), - created_at: yup.date().required(), - updated_at: yup.date().required(), - source: yup.string().required(), - metadata: yup.object().optional(), -}); - -// Sentiment Data Schema -export const sentimentDataSchema = documentBaseSchema.shape({ - symbol: yup.string().min(1).max(10).required(), - sentiment_score: yup.number().min(-1).max(1).required(), - sentiment_label: yup.string().oneOf(['positive', 'negative', 'neutral']).required(), - confidence: yup.number().min(0).max(1).required(), - text: yup.string().min(1).required(), - source_type: yup.string().oneOf(['reddit', 'twitter', 'news', 'forums']).required(), - source_id: yup.string().required(), - timestamp: yup.date().required(), - processed_at: yup.date().required(), - language: yup.string().default('en'), - keywords: yup.array(yup.string()).required(), - entities: yup - .array( - yup.object({ - name: yup.string().required(), - type: yup.string().required(), - confidence: yup.number().min(0).max(1).required(), - }) - ) - .required(), -}); - -// Raw Document Schema -export const rawDocumentSchema = documentBaseSchema.shape({ - document_type: yup.string().oneOf(['html', 'pdf', 'text', 'json', 'xml']).required(), - content: yup.string().required(), - content_hash: yup.string().required(), - url: yup.string().url().optional(), - title: yup.string().optional(), - author: yup.string().optional(), - published_date: yup.date().optional(), - extracted_text: yup.string().optional(), - processing_status: yup.string().oneOf(['pending', 'processed', 'failed']).required(), - size_bytes: yup.number().positive().required(), - language: yup.string().optional(), -}); - -// News Article Schema -export const newsArticleSchema = documentBaseSchema.shape({ - headline: yup.string().min(1).required(), - content: yup.string().min(1).required(), - summary: yup.string().optional(), - author: yup.string().required(), - publication: yup.string().required(), - published_date: yup.date().required(), - url: yup.string().url().required(), - symbols: yup.array(yup.string()).required(), - categories: yup.array(yup.string()).required(), - sentiment_score: yup.number().min(-1).max(1).optional(), - relevance_score: yup.number().min(0).max(1).optional(), - image_url: yup.string().url().optional(), - tags: yup.array(yup.string()).required(), -}); - -// SEC Filing Schema -export const secFilingSchema = documentBaseSchema.shape({ - cik: yup.string().required(), - accession_number: yup.string().required(), - filing_type: yup.string().required(), - company_name: yup.string().required(), - symbols: yup.array(yup.string()).required(), - filing_date: yup.date().required(), - period_end_date: yup.date().required(), - url: yup.string().url().required(), - content: yup.string().required(), - extracted_data: yup.object().optional(), - financial_statements: yup - .array( - yup.object({ - statement_type: yup.string().required(), - data: yup.object().required(), - }) - ) - .optional(), - processing_status: yup.string().oneOf(['pending', 'processed', 'failed']).required(), -}); - -// Earnings Transcript Schema -export const earningsTranscriptSchema = documentBaseSchema.shape({ - symbol: yup.string().min(1).max(10).required(), - company_name: yup.string().required(), - quarter: yup.string().required(), - year: yup.number().min(2000).max(3000).required(), - call_date: yup.date().required(), - transcript: yup.string().required(), - participants: yup - .array( - yup.object({ - name: yup.string().required(), - title: yup.string().required(), - type: yup.string().oneOf(['executive', 'analyst']).required(), - }) - ) - .required(), - key_topics: yup.array(yup.string()).required(), - sentiment_analysis: yup - .object({ - overall_sentiment: yup.number().min(-1).max(1).required(), - topic_sentiments: yup.object().required(), - }) - .optional(), - financial_highlights: yup.object().optional(), -}); - -// Analyst Report Schema -export const analystReportSchema = documentBaseSchema.shape({ - symbol: yup.string().min(1).max(10).required(), - analyst_firm: yup.string().required(), - analyst_name: yup.string().required(), - report_title: yup.string().required(), - report_date: yup.date().required(), - rating: yup.string().oneOf(['buy', 'hold', 'sell', 'strong_buy', 'strong_sell']).required(), - price_target: yup.number().positive().optional(), - previous_rating: yup.string().optional(), - content: yup.string().required(), - summary: yup.string().required(), - key_points: yup.array(yup.string()).required(), - financial_projections: yup.object().optional(), -}); - -// Schema mapping for collections -export const schemaMap = { - sentiment_data: sentimentDataSchema, - raw_documents: rawDocumentSchema, - news_articles: newsArticleSchema, - sec_filings: secFilingSchema, - earnings_transcripts: earningsTranscriptSchema, - analyst_reports: analystReportSchema, -} as const; diff --git a/libs/mongodb-client/src/transactions.ts b/libs/mongodb-client/src/transactions.ts deleted file mode 100644 index 166b489..0000000 --- a/libs/mongodb-client/src/transactions.ts +++ /dev/null @@ -1,238 +0,0 @@ -import type { OptionalUnlessRequiredId, WithId } from 'mongodb'; -import { getLogger } from '@stock-bot/logger'; -import type { MongoDBClient } from './client'; -import type { CollectionNames, DocumentBase } from './types'; - -/** - * MongoDB Transaction Manager - * - * Provides transaction support for multi-document operations - */ -export class MongoDBTransactionManager { - private readonly client: MongoDBClient; - private readonly logger: ReturnType; - - constructor(client: MongoDBClient) { - this.client = client; - this.logger = getLogger('mongodb-transaction-manager'); - } - - /** - * Execute operations within a transaction - */ - async withTransaction( - operations: (session: any) => Promise, - options?: { - readPreference?: string; - readConcern?: string; - writeConcern?: any; - maxCommitTimeMS?: number; - } - ): Promise { - const mongoClient = this.client.mongoClient; - if (!mongoClient) { - throw new Error('MongoDB client not connected'); - } - - const session = mongoClient.startSession(); - - try { - this.logger.debug('Starting MongoDB transaction'); - - const result = await session.withTransaction( - async () => { - return await operations(session); - }, - { - readPreference: options?.readPreference as any, - readConcern: { level: options?.readConcern || 'majority' } as any, - writeConcern: options?.writeConcern || { w: 'majority' }, - maxCommitTimeMS: options?.maxCommitTimeMS || 10000, - } - ); - - this.logger.debug('MongoDB transaction completed successfully'); - return result; - } catch (error) { - this.logger.error('MongoDB transaction failed:', error); - throw error; - } finally { - await session.endSession(); - } - } - - /** - * Batch insert documents across collections within a transaction - */ - async batchInsert( - operations: Array<{ - collection: CollectionNames; - documents: DocumentBase[]; - }>, - options?: { ordered?: boolean; bypassDocumentValidation?: boolean } - ): Promise { - await this.withTransaction(async session => { - for (const operation of operations) { - const collection = this.client.getCollection(operation.collection); - - // Add timestamps to all documents - const now = new Date(); - const documentsWithTimestamps = operation.documents.map(doc => ({ - ...doc, - created_at: doc.created_at || now, - updated_at: now, - })); - - await collection.insertMany(documentsWithTimestamps, { - session, - ordered: options?.ordered ?? true, - bypassDocumentValidation: options?.bypassDocumentValidation ?? false, - }); - - this.logger.debug( - `Inserted ${documentsWithTimestamps.length} documents into ${operation.collection}` - ); - } - }); - } - - /** - * Batch update documents across collections within a transaction - */ - async batchUpdate( - operations: Array<{ - collection: CollectionNames; - filter: any; - update: any; - options?: any; - }> - ): Promise { - await this.withTransaction(async session => { - const results = []; - - for (const operation of operations) { - const collection = this.client.getCollection(operation.collection); - - // Add updated timestamp - const updateWithTimestamp = { - ...operation.update, - $set: { - ...operation.update.$set, - updated_at: new Date(), - }, - }; - - const result = await collection.updateMany(operation.filter, updateWithTimestamp, { - session, - ...operation.options, - }); - - results.push(result); - this.logger.debug(`Updated ${result.modifiedCount} documents in ${operation.collection}`); - } - - return results; - }); - } - - /** - * Move documents between collections within a transaction - */ - async moveDocuments( - fromCollection: CollectionNames, - toCollection: CollectionNames, - filter: any, - transform?: (doc: T) => T - ): Promise { - return await this.withTransaction(async session => { - const sourceCollection = this.client.getCollection(fromCollection); - const targetCollection = this.client.getCollection(toCollection); - - // Find documents to move - const documents = await sourceCollection.find(filter, { session }).toArray(); - - if (documents.length === 0) { - return 0; - } // Transform documents if needed - const documentsToInsert = transform - ? documents.map((doc: WithId) => transform(doc as T)) - : documents; - - // Add updated timestamp - const now = new Date(); - documentsToInsert.forEach(doc => { - doc.updated_at = now; - }); // Insert into target collection - await targetCollection.insertMany(documentsToInsert as OptionalUnlessRequiredId[], { - session, - }); - - // Remove from source collection - const deleteResult = await sourceCollection.deleteMany(filter, { session }); - - this.logger.info( - `Moved ${documents.length} documents from ${fromCollection} to ${toCollection}` - ); - - return deleteResult.deletedCount || 0; - }); - } - - /** - * Archive old documents within a transaction - */ - async archiveDocuments( - sourceCollection: CollectionNames, - archiveCollection: CollectionNames, - cutoffDate: Date, - batchSize: number = 1000 - ): Promise { - let totalArchived = 0; - - while (true) { - const batchArchived = await this.withTransaction(async session => { - const collection = this.client.getCollection(sourceCollection); - const archiveCol = this.client.getCollection(archiveCollection); - - // Find old documents - const documents = await collection - .find({ created_at: { $lt: cutoffDate } }, { limit: batchSize, session }) - .toArray(); - - if (documents.length === 0) { - return 0; - } - - // Add archive metadata - const now = new Date(); - const documentsToArchive = documents.map(doc => ({ - ...doc, - archived_at: now, - archived_from: sourceCollection, - })); - - // Insert into archive collection - await archiveCol.insertMany(documentsToArchive, { session }); - - // Remove from source collection - const ids = documents.map(doc => doc._id); - const deleteResult = await collection.deleteMany({ _id: { $in: ids } }, { session }); - - return deleteResult.deletedCount || 0; - }); - - totalArchived += batchArchived; - - if (batchArchived === 0) { - break; - } - - this.logger.debug(`Archived batch of ${batchArchived} documents`); - } - - this.logger.info( - `Archived ${totalArchived} documents from ${sourceCollection} to ${archiveCollection}` - ); - return totalArchived; - } -} diff --git a/libs/mongodb-client/src/types.ts b/libs/mongodb-client/src/types.ts index 05143fb..cb54811 100644 --- a/libs/mongodb-client/src/types.ts +++ b/libs/mongodb-client/src/types.ts @@ -1,5 +1,4 @@ import type { ObjectId } from 'mongodb'; -import * as yup from 'yup'; /** * MongoDB Client Configuration @@ -69,20 +68,6 @@ export interface MongoDBMetrics { documentsProcessed: number; } -/** - * Collection Names - */ -export type CollectionNames = - | 'sentiment_data' - | 'raw_documents' - | 'news_articles' - | 'sec_filings' - | 'earnings_transcripts' - | 'analyst_reports' - | 'social_media_posts' - | 'market_events' - | 'economic_indicators'; - /** * Base Document Interface */ diff --git a/scripts/populate-ib-exchanges.ts b/scripts/populate-ib-exchanges.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-browser-simple.ts b/test-browser-simple.ts deleted file mode 100644 index 318f3f9..0000000 --- a/test-browser-simple.ts +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Simple Browser and Network Monitoring Test - */ -import { Browser } from '@stock-bot/browser'; - -async function testBasicBrowser() { - console.log('🚀 Testing basic browser functionality...'); - - try { - // Initialize browser - await Browser.initialize({ - headless: true, - timeout: 15000, - blockResources: false, - enableNetworkLogging: true, - }); - - console.log('✅ Browser initialized'); - - // Test 1: Simple page without proxy - console.log('📄 Testing simple page without proxy...'); - const { page, contextId } = await Browser.createPageWithProxy( - 'https://httpbin.org/json' - ); - - let capturedData = null; - let eventCount = 0; - - page.onNetworkEvent(event => { - eventCount++; - console.log(`📡 Event ${eventCount}: ${event.type} - ${event.method} ${event.url}`); - - if (event.type === 'response' && event.url.includes('httpbin.org/json')) { - console.log(` 📊 Status: ${event.status}`); - if (event.responseData) { - capturedData = event.responseData; - console.log(` 📝 Response: ${event.responseData}`); - } - } - }); - - await page.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 2000)); - - console.log(`✅ Test completed. Events captured: ${eventCount}`); - if (capturedData) { - console.log('✅ Successfully captured response data'); - } - - await Browser.closeContext(contextId); - return true; - - } catch (error) { - console.error('❌ Basic test failed:', error); - return false; - } finally { - await Browser.close(); - } -} - -async function testProxyConnection() { - console.log('\n🔄 Testing proxy connection...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 10000, - blockResources: false, - }); - - // Test different proxy formats - const proxyConfigs = [ - null, // No proxy - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', - ]; - - for (const proxy of proxyConfigs) { - console.log(`\n🌐 Testing with proxy: ${proxy || 'No proxy'}`); - - try { - const { page, contextId } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip', - proxy - ); - - page.onNetworkEvent(event => { - if (event.type === 'response' && event.url.includes('httpbin.org/ip')) { - console.log(` 📍 IP Response: ${event.responseData}`); - } - }); - - await page.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 1500)); - await Browser.closeContext(contextId); - console.log(' ✅ Success'); - - } catch (error) { - console.log(` ❌ Failed: ${error.message}`); - } - } - - } catch (error) { - console.error('❌ Proxy test setup failed:', error); - } finally { - await Browser.close(); - } -} - -async function testIBWithWorkaround() { - console.log('\n🏦 Testing IB endpoint with workaround...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 20000, - blockResources: true, // Block resources for performance - }); - - // Try without proxy first - console.log('🌐 Attempting IB without proxy...'); - try { - const { page, contextId } = await Browser.createPageWithProxy( - 'https://www.interactivebrokers.com' - ); - - let responseCount = 0; - page.onNetworkEvent(event => { - if (event.type === 'response') { - responseCount++; - console.log(` 📥 Response ${responseCount}: ${event.status} ${event.url}`); - } - }); - - await page.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 3000)); - console.log(`✅ IB main page loaded. Responses: ${responseCount}`); - await Browser.closeContext(contextId); - - } catch (error) { - console.log(`❌ IB without proxy failed: ${error.message}`); - } - - } catch (error) { - console.error('❌ IB test failed:', error); - } finally { - await Browser.close(); - } -} - -// Run tests -async function runAllTests() { - console.log('🧪 Starting Browser Network Monitoring Tests\n'); - - const basicResult = await testBasicBrowser(); - await testProxyConnection(); - await testIBWithWorkaround(); - - console.log(`\n🏁 Basic functionality: ${basicResult ? '✅ PASS' : '❌ FAIL'}`); - console.log('✅ All tests completed!'); -} - -if (import.meta.main) { - runAllTests().catch(console.error); -} - -export { testBasicBrowser, testProxyConnection, testIBWithWorkaround }; diff --git a/test-browser.ts b/test-browser.ts deleted file mode 100644 index 11040f5..0000000 --- a/test-browser.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { Browser, BrowserTabManager } from './libs/browser/src'; - -async function testSimplifiedBrowser() { - console.log('Testing simplified browser library...'); - - try { - console.log('Initializing browser...'); - await Browser.initialize({ - headless: true, - blockResources: true, - timeout: 10000, - }); - - // Test single page with proxy support - console.log('Testing page creation...'); - const { page, contextId } = await Browser.createPageWithProxy( - 'https://httpbin.org/json', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - const content = await Browser.evaluate(page, () => document.body.textContent); - console.log('Page content:', content?.substring(0, 100) + '...'); - - // Test tab manager (no longer needs browser instance) - console.log('Testing tab manager...'); - const tabManager = new BrowserTabManager(); - - // Test multiple URL scraping with different proxies - const urlProxyPairs = [ - { url: 'https://httpbin.org/uuid', proxy: '' }, // No proxy - { url: 'https://httpbin.org/ip', proxy: '' }, // No proxy - ]; - - const results = await tabManager.scrapeUrlsWithProxies( - urlProxyPairs, - async page => { - const text = await page.textContent('body'); - return { content: text?.substring(0, 50) }; - }, - { concurrency: 2 } - ); - - console.log('Scraping results:'); - results.forEach((result, index) => { - console.log(` ${index + 1}. ${result.url}: ${result.success ? 'SUCCESS' : 'FAILED'}`); - if (result.data) { - console.log(` Data: ${result.data.content}...`); - } - }); - - // Clean up - await page.close(); - await Browser.closeContext(contextId); - await Browser.close(); - - console.log('✅ Simplified browser test completed successfully!'); - } catch (error) { - console.error('❌ Browser test failed:', error); - } -} - -testSimplifiedBrowser(); diff --git a/test-ib-no-proxy.ts b/test-ib-no-proxy.ts deleted file mode 100644 index e6fe62f..0000000 --- a/test-ib-no-proxy.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { Browser } from '@stock-bot/browser'; - -async function testWithoutProxy() { - console.log('🔬 Testing WITHOUT proxy...'); - - try { - await Browser.initialize({ headless: true, timeout: 15000, blockResources: false }); - console.log('✅ Browser initialized'); - - const { page, contextId } = await Browser.createPageWithProxy( - 'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/' - // No proxy parameter - ); - console.log('✅ Page created without proxy'); - - let eventCount = 0; - let summaryData: SummaryResponse | null = null; - - page.onNetworkEvent(event => { - eventCount++; - - // Capture the summary API response - if (event.url.includes('/webrest/search/product-types/summary')) { - console.log(`🎯 Found summary API call: ${event.type} ${event.url}`); - - if (event.type === 'response' && event.responseData) { - console.log(`📊 Summary API Response Data: ${event.responseData}`); - try { - summaryData = JSON.parse(event.responseData) as any; - const totalCount = summaryData[0].totalCount; - console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2)); - console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`); - } catch (e) { - console.log('📊 Raw Summary Response:', event.responseData); - } - } - } - - // Uncomment to see all network events - // console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`); - }); - - console.log('⏳ Waiting for page load...'); - await page.waitForLoadState('domcontentloaded', { timeout: 15000 }); - console.log('✅ Page loaded'); - - // Complete interaction flow - try { - console.log('🔍 Looking for Products tab...'); - await page.waitForTimeout(3000); - - const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); - await productsTab.waitFor({ timeout: 10000 }); - console.log('✅ Found Products tab'); - - console.log('🖱️ Clicking Products tab...'); - await productsTab.click(); - console.log('✅ Products tab clicked'); - - await page.waitForTimeout(2000); - - console.log('🔍 Looking for Asset Classes accordion...'); - const assetClassesAccordion = page.locator( - '#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")' - ); - await assetClassesAccordion.waitFor({ timeout: 10000 }); - console.log('✅ Found Asset Classes accordion'); - - console.log('🖱️ Clicking Asset Classes accordion...'); - await assetClassesAccordion.click(); - console.log('✅ Asset Classes accordion clicked'); - - await page.waitForTimeout(2000); - - console.log('🔍 Looking for Stocks checkbox...'); - const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")'); - await stocksSpan.waitFor({ timeout: 10000 }); - console.log('✅ Found Stocks span'); - - const parentContainer = stocksSpan.locator('..'); - const checkbox = parentContainer.locator('input[type="checkbox"]'); - - if ((await checkbox.count()) > 0) { - console.log('📋 Clicking Stocks checkbox...'); - await checkbox.first().check(); - console.log('✅ Stocks checkbox checked'); - } else { - console.log('⚠️ Could not find checkbox near Stocks text'); - } - - await page.waitForTimeout(1000); - - console.log('🔍 Looking for Apply button...'); - const applyButton = page.locator( - 'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]' - ); - - if ((await applyButton.count()) > 0) { - console.log('🎯 Clicking Apply button...'); - await applyButton.first().click(); - console.log('✅ Apply button clicked'); - await page.waitForTimeout(3000); - } else { - console.log('⚠️ Could not find Apply button'); - } - } catch (interactionError) { - const errorMessage = - interactionError instanceof Error ? interactionError.message : String(interactionError); - console.error('❌ Page interaction failed:', errorMessage); - } - - await new Promise(resolve => setTimeout(resolve, 2000)); - console.log(`📊 Total events captured: ${eventCount}`); - - // Show final results - if (summaryData) { - console.log('✅ SUCCESS: Captured summary data!'); - console.log(`🔢 Final total count: ${summaryData?.data?.totalCount || 'Unknown'}`); - console.log(`📋 Data keys: ${Object.keys(summaryData).join(', ')}`); - } else { - console.log('❌ No summary data captured'); - } - - await Browser.closeContext(contextId); - await Browser.close(); - - console.log('✅ Test completed successfully'); - return true; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('❌ Error:', errorMessage); - await Browser.close(); - return false; - } -} - -testWithoutProxy().then(success => { - console.log(`🏁 Final result: ${success ? 'SUCCESS' : 'FAILED'}`); -}); diff --git a/test-ib-working.ts b/test-ib-working.ts deleted file mode 100644 index de3d109..0000000 --- a/test-ib-working.ts +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Working Interactive Brokers test with verified network monitoring - */ -import { Browser } from '@stock-bot/browser'; - -async function testIBWithWorking() { - console.log('🏦 Testing IB with working network monitoring and fixed proxy auth...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 20000, - blockResources: false, // Don't block resources initially - }); - - // Test 1: Try a simple proxy detection service first - console.log('🌐 Testing proxy connectivity...'); - const { page: proxyPage, contextId: proxyCtx } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - let proxyEvents = 0; - let myIP = null; - proxyPage.onNetworkEvent(event => { - proxyEvents++; - if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { - try { - const data = JSON.parse(event.responseData); - myIP = data.origin; - console.log(` 📍 Proxy IP: ${myIP}`); - } catch (e) { - console.log(` 📊 Raw response: ${event.responseData}`); - } - } - }); - - await proxyPage.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 2000)); - await Browser.closeContext(proxyCtx); - - console.log(`📊 Proxy test events: ${proxyEvents}`); - - // Test 2: Try IB API endpoint with fixed proxy auth - console.log('🎯 Testing IB API endpoint...'); - const { page: apiPage, contextId: apiCtx } = await Browser.createPageWithProxy( - 'https://www.interactivebrokers.com/webrest/search/product-types/summary', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - let apiEvents = 0; - let summaryData = null; - apiPage.onNetworkEvent(event => { - apiEvents++; - console.log(` 📡 API Event: ${event.type} ${event.method} ${event.url}`); - - if (event.type === 'response' && event.url.includes('summary')) { - console.log(` 🎯 Found summary response! Status: ${event.status}`); - if (event.responseData) { - summaryData = event.responseData; - try { - const data = JSON.parse(event.responseData); - console.log(` 📊 Summary data: ${JSON.stringify(data, null, 2)}`); - } catch (e) { - console.log(` 📊 Raw summary: ${event.responseData.substring(0, 200)}...`); - } - } - } - }); - - await apiPage.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 3000)); - await Browser.closeContext(apiCtx); - - return { - proxyEvents, - apiEvents, - summaryData, - proxyIP: myIP, - success: apiEvents > 0 || summaryData !== null, - }; - } catch (error) { - console.error('❌ IB test failed:', error); - return { - proxyEvents: 0, - apiEvents: 0, - summaryData: null, - proxyIP: null, - success: false, - error: error.message, - }; - } finally { - await Browser.close(); - } -} - -async function testWithProxyFallback() { - console.log('\n🔄 Testing with proxy fallback strategy...'); - - const proxiesToTest = [ - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', // Your proxy - ]; - - for (const proxy of proxiesToTest) { - console.log(`\n🌐 Testing with: ${proxy || 'No proxy'}`); - - try { - await Browser.initialize({ - headless: true, - timeout: 15000, - blockResources: false, - }); - - const { page, contextId } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip', - proxy - ); - - let ipResponse = null; - page.onNetworkEvent(event => { - if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { - ipResponse = event.responseData; - console.log(` 📍 IP: ${JSON.parse(event.responseData).origin}`); - } - }); - - await page.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 1000)); - await Browser.closeContext(contextId); - } catch (error) { - console.log(` ❌ Failed: ${error.message}`); - } finally { - await Browser.close(); - } - } -} - -async function runIBTests() { - console.log('🚀 Interactive Brokers Network Monitoring Tests with Fixed Proxy Auth\n'); - - const result = await testIBWithWorking(); - await testWithProxyFallback(); - - console.log('\n🏁 Final Results:'); - console.log(` 🌐 Proxy events: ${result.proxyEvents || 0}`); - console.log(` 📍 Proxy IP: ${result.proxyIP || 'Not captured'}`); - console.log(` 🎯 API events: ${result.apiEvents || 0}`); - console.log(` 📊 Summary data: ${result.summaryData ? 'Captured' : 'Not captured'}`); - console.log(` ✅ Overall success: ${result.success}`); - - if (result.error) { - console.log(` ❌ Error: ${result.error}`); - } -} - -if (import.meta.main) { - runIBTests().catch(console.error); -} - -export { testIBWithWorking, testWithProxyFallback }; diff --git a/test-ib.ts b/test-ib.ts deleted file mode 100644 index c9fed34..0000000 --- a/test-ib.ts +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Test Interactive Brokers functionality with network monitoring - */ -import { Browser } from '@stock-bot/browser'; -import { getRandomProxyURL } from '@stock-bot/proxy'; - -async function testIBSymbolSummary() { - console.log('🚀 Testing Interactive Brokers Symbol Summary with Network Monitoring...'); - - try { - // Initialize browser - await Browser.initialize({ - headless: true, - timeout: 30000, - blockResources: true, - enableNetworkLogging: true, - }); - - console.log('✅ Browser initialized'); - - // Get a random proxy - - // Create page with proxy - const { page, contextId } = await Browser.createPageWithProxy( - 'https://www.interactivebrokers.com/webrest/search/product-types/summary', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - console.log('📄 Page created with proxy'); - - // Set up network monitoring - let summaryResponse: any = null; - let requestCount = 0; - let responseCount = 0; - - page.onNetworkEvent(event => { - console.log(`📡 Network Event: ${event.type} - ${event.method} ${event.url}`); - - if (event.type === 'request') { - requestCount++; - console.log(` 📤 Request #${requestCount}: ${event.method} ${event.url}`); - - // Log request data for POST requests - if (event.requestData) { - console.log(` 📝 Request Data: ${event.requestData.substring(0, 200)}...`); - } - } - - if (event.type === 'response') { - responseCount++; - console.log(` 📥 Response #${responseCount}: ${event.status} ${event.url}`); - - // Capture the summary response - if (event.url.includes('summary')) { - console.log(` 🎯 Found summary response!`); - summaryResponse = event.responseData; - - if (event.responseData) { - try { - const data = JSON.parse(event.responseData); - console.log(` 📊 Summary Data: ${JSON.stringify(data, null, 2)}`); - } catch (e) { - console.log(` 📊 Raw Response: ${event.responseData.substring(0, 500)}...`); - } - } - } - } - - if (event.type === 'failed') { - console.log(` ❌ Failed Request: ${event.url}`); - } - }); - - console.log('🔍 Network monitoring set up, waiting for page to load...'); - - // Wait for page to load and capture network activity - await page.waitForLoadState('domcontentloaded'); - console.log('✅ Page loaded'); - - // Wait a bit more for any additional network requests - await new Promise(resolve => setTimeout(resolve, 3000)); - - console.log(`📊 Network Summary:`); - console.log(` 📤 Total Requests: ${requestCount}`); - console.log(` 📥 Total Responses: ${responseCount}`); - - if (summaryResponse) { - console.log('✅ Successfully captured summary response'); - try { - const parsed = JSON.parse(summaryResponse); - console.log(`🔢 Total symbols found: ${parsed?.data?.totalCount || 'Unknown'}`); - return parsed?.data?.totalCount || 0; - } catch (e) { - console.log('⚠️ Could not parse response as JSON'); - return 1; // Indicate success but unknown count - } - } else { - console.log('❌ No summary response captured'); - return 0; - } - - } catch (error) { - console.error('❌ Test failed:', error); - - // Log more details about the error - if (error instanceof Error) { - console.error('Error details:', { - message: error.message, - stack: error.stack, - name: error.name - }); - } - - return -1; - } finally { - try { - await Browser.close(); - console.log('🔒 Browser closed'); - } catch (closeError) { - console.error('Error closing browser:', closeError); - } - } -} - -async function testWithDifferentProxy() { - console.log('\n🔄 Testing with different proxy configuration...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 15000, - blockResources: false, // Don't block resources for this test - }); - - // Test without proxy first - console.log('🌐 Testing without proxy...'); - const { page: pageNoProxy, contextId: contextNoProxy } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip' - ); - - pageNoProxy.onNetworkEvent(event => { - if (event.type === 'response' && event.url.includes('httpbin.org/ip')) { - console.log('📍 No proxy IP response:', event.responseData); - } - }); - - await pageNoProxy.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 1000)); - await Browser.closeContext(contextNoProxy); - - // Test with proxy - console.log('🌐 Testing with proxy...'); - const { page: pageWithProxy, contextId: contextWithProxy } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - pageWithProxy.onNetworkEvent(event => { - if (event.type === 'response' && event.url.includes('httpbin.org/ip')) { - console.log('🔄 Proxy IP response:', event.responseData); - } - }); - - await pageWithProxy.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 1000)); - await Browser.closeContext(contextWithProxy); - - } catch (error) { - console.error('❌ Proxy test failed:', error); - } finally { - await Browser.close(); - } -} - -// Run the tests -async function runTests() { - console.log('🧪 Starting IB Network Monitoring Tests\n'); - - // Test 1: Main IB functionality - const result = await testIBSymbolSummary(); - console.log(`\n🏁 Test Result: ${result}`); - - // Test 2: Proxy verification - await testWithDifferentProxy(); - - console.log('\n✅ All tests completed!'); -} - -// Run if this file is executed directly -if (import.meta.main) { - runTests().catch(console.error); -} - -export { testIBSymbolSummary, testWithDifferentProxy }; diff --git a/test-large-scale-performance.ts b/test-large-scale-performance.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-mongodb-batch.ts b/test-mongodb-batch.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-mongodb-simplified.ts b/test-mongodb-simplified.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-network-debug.ts b/test-network-debug.ts deleted file mode 100644 index d227bd5..0000000 --- a/test-network-debug.ts +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Debug network monitoring setup - */ -import { Browser } from '@stock-bot/browser'; - -async function debugNetworkSetup() { - console.log('🐛 Debugging Network Monitoring Setup...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 10000, - blockResources: false, // Ensure we don't block requests - }); - - // Create page but don't navigate yet - const { page, contextId } = await Browser.createPageWithProxy( - '', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - let eventCount = 0; - console.log('📡 Setting up network event listener...'); - - page.onNetworkEvent(event => { - eventCount++; - console.log(`🔔 Event ${eventCount}: ${event.type} ${event.method} ${event.url}`); - console.log(` Headers: ${Object.keys(event.headers || {}).length} headers`); - - if (event.responseData) { - console.log(` Data: ${event.responseData.substring(0, 100)}...`); - } - }); - - console.log('🌐 Navigating to httpbin.org/headers...'); - await page.goto('https://httpbin.org/headers'); - - console.log('⏳ Waiting for page load...'); - await page.waitForLoadState('domcontentloaded'); - - console.log('⏳ Waiting additional time for network events...'); - await new Promise(resolve => setTimeout(resolve, 3000)); - - console.log(`📊 Total events captured: ${eventCount}`); - - // Try to evaluate page content to see if it loaded - const title = await page.title(); - console.log(`📄 Page title: "${title}"`); - - const bodyText = await page.locator('body').textContent(); - if (bodyText) { - console.log(`📝 Page content (first 200 chars): ${bodyText.substring(0, 200)}...`); - } - - await Browser.closeContext(contextId); - return eventCount > 0; - } catch (error) { - console.error('❌ Debug test failed:', error); - return false; - } finally { - await Browser.close(); - } -} - -async function testManualNetworkCall() { - console.log('\n🔧 Testing with manual fetch call...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 10000, - blockResources: false, - }); - - const { page, contextId } = await Browser.createPageWithProxy( - 'https://www.interactivebrokers.com/webrest/search/product-types/summary', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - let eventCount = 0; - page.onNetworkEvent(event => { - eventCount++; - console.log(`📡 Manual test event ${eventCount}: ${event.type} ${event.method} ${event.url}`); - if (event.responseData && event.url.includes('httpbin')) { - console.log(` 📊 Response: ${event.responseData}`); - } - }); - - // Navigate to a simple page first - await page.goto('data:text/html,

Test Page

'); - await page.waitForLoadState('domcontentloaded'); - - console.log('🚀 Making manual fetch call...'); - // Make a fetch request from the page context - const result = await page.evaluate(async () => { - try { - const response = await fetch('https://httpbin.org/json'); - const data = await response.json(); - return { success: true, data }; - } catch (error) { - return { success: false, error: error.message }; - } - }); - - console.log('📋 Fetch result:', result); - - await new Promise(resolve => setTimeout(resolve, 2000)); - console.log(`📊 Events from manual fetch: ${eventCount}`); - - await Browser.closeContext(contextId); - return eventCount > 0; - } catch (error) { - console.error('❌ Manual test failed:', error); - return false; - } finally { - await Browser.close(); - } -} - -async function runDebugTests() { - console.log('🚀 Network Monitoring Debug Tests\n'); - - const setupResult = await debugNetworkSetup(); - const manualResult = await testManualNetworkCall(); - - console.log(`\n🏁 Results:`); - console.log(` 🔧 Setup test: ${setupResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`); - console.log(` 📡 Manual test: ${manualResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`); -} - -if (import.meta.main) { - runDebugTests().catch(console.error); -} - -export { debugNetworkSetup, testManualNetworkCall }; diff --git a/test-network-monitoring.ts b/test-network-monitoring.ts deleted file mode 100644 index 60a3bba..0000000 --- a/test-network-monitoring.ts +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Simple test to verify network monitoring is working - */ -import { Browser } from '@stock-bot/browser'; - -async function testNetworkMonitoring() { - console.log('🧪 Testing Network Monitoring with httpbin.org...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 15000, - blockResources: false, // Don't block resources so we can see requests - }); - - console.log('✅ Browser initialized'); - - // Test with a simple API that returns JSON - const { page, contextId } = await Browser.createPageWithProxy( - 'https://httpbin.org/json' - ); - - let capturedRequests = 0; - let capturedResponses = 0; - let jsonResponse = null; - - page.onNetworkEvent(event => { - console.log(`📡 ${event.type.toUpperCase()}: ${event.method} ${event.url}`); - - if (event.type === 'request') { - capturedRequests++; - } - - if (event.type === 'response') { - capturedResponses++; - console.log(` Status: ${event.status}`); - - if (event.url.includes('httpbin.org/json') && event.responseData) { - jsonResponse = event.responseData; - console.log(` 📊 JSON Response: ${event.responseData}`); - } - } - }); - - await page.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 2000)); - - console.log(`\n📊 Summary:`); - console.log(` 📤 Requests captured: ${capturedRequests}`); - console.log(` 📥 Responses captured: ${capturedResponses}`); - console.log(` 📝 JSON data captured: ${jsonResponse ? 'Yes' : 'No'}`); - - await Browser.closeContext(contextId); - return true; - - } catch (error) { - console.error('❌ Test failed:', error); - return false; - } finally { - await Browser.close(); - } -} - -async function testWithProxy() { - console.log('\n🌐 Testing with proxy to see IP change...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 10000, - blockResources: false, - }); - - // Test IP without proxy - console.log('📍 Getting IP without proxy...'); - const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip' - ); - - let ipWithoutProxy = null; - page1.onNetworkEvent(event => { - if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { - ipWithoutProxy = JSON.parse(event.responseData).origin; - console.log(` 🔹 Your IP: ${ipWithoutProxy}`); - } - }); - - await page1.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 1000)); - await Browser.closeContext(ctx1); - - // Test IP with proxy - console.log('🔄 Getting IP with proxy...'); - const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - let ipWithProxy = null; - page2.onNetworkEvent(event => { - if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { - ipWithProxy = JSON.parse(event.responseData).origin; - console.log(` 🔸 Proxy IP: ${ipWithProxy}`); - } - }); - - await page2.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 1000)); - await Browser.closeContext(ctx2); - - if (ipWithoutProxy && ipWithProxy && ipWithoutProxy !== ipWithProxy) { - console.log('✅ Proxy is working - IPs are different!'); - } else { - console.log('⚠️ Proxy may not be working - IPs are the same or not captured'); - } - - } catch (error) { - console.error('❌ Proxy test failed:', error); - } finally { - await Browser.close(); - } -} - -async function runTests() { - console.log('🚀 Network Monitoring Verification Tests\n'); - - const basicResult = await testNetworkMonitoring(); - await testWithProxy(); - - console.log(`\n🏁 Network monitoring: ${basicResult ? '✅ WORKING' : '❌ FAILED'}`); -} - -if (import.meta.main) { - runTests().catch(console.error); -} - -export { testNetworkMonitoring, testWithProxy }; diff --git a/test-proxy-auth.ts b/test-proxy-auth.ts deleted file mode 100644 index fc45846..0000000 --- a/test-proxy-auth.ts +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Test Playwright proxy authentication specifically - */ -import { Browser } from '@stock-bot/browser'; - -async function testPlaywrightProxyAuth() { - console.log('🔐 Testing Playwright Proxy Authentication...'); - - try { - await Browser.initialize({ - headless: true, - timeout: 15000, - blockResources: false, - }); - - console.log('✅ Browser initialized'); - - // Test 1: Without proxy - console.log('\n📍 Test 1: Without proxy'); - const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip' - ); - - let events1 = 0; - let ip1 = null; - page1.onNetworkEvent(event => { - events1++; - console.log(` 📡 Event: ${event.type} ${event.url}`); - if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { - ip1 = JSON.parse(event.responseData).origin; - console.log(` 🌐 Your IP: ${ip1}`); - } - }); - - await page1.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 2000)); - await Browser.closeContext(ctx1); - console.log(` Events captured: ${events1}`); - - // Test 2: With proxy using new authentication method - console.log('\n🔒 Test 2: With proxy (new auth method)'); - const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - let events2 = 0; - let ip2 = null; - page2.onNetworkEvent(event => { - events2++; - console.log(` 📡 Event: ${event.type} ${event.url}`); - if (event.type === 'response' && event.url.includes('/ip') && event.responseData) { - ip2 = JSON.parse(event.responseData).origin; - console.log(` 🔄 Proxy IP: ${ip2}`); - } - }); - - await page2.waitForLoadState('domcontentloaded'); - await new Promise(resolve => setTimeout(resolve, 2000)); - await Browser.closeContext(ctx2); - console.log(` Events captured: ${events2}`); - - // Results - console.log('\n📊 Results:'); - console.log(` 🌐 Direct IP: ${ip1 || 'Not captured'}`); - console.log(` 🔄 Proxy IP: ${ip2 || 'Not captured'}`); - console.log(` 📡 Direct events: ${events1}`); - console.log(` 📡 Proxy events: ${events2}`); - - if (ip1 && ip2 && ip1 !== ip2) { - console.log('✅ Proxy authentication is working - different IPs detected!'); - return true; - } else if (events1 > 0 || events2 > 0) { - console.log('⚠️ Network monitoring working, but proxy may not be changing IP'); - return true; - } else { - console.log('❌ No network events captured'); - return false; - } - - } catch (error) { - console.error('❌ Test failed:', error); - return false; - } finally { - await Browser.close(); - } -} - -async function testManualPageEvaluation() { - console.log('\n🧪 Test 3: Manual page evaluation (without network monitoring)'); - - try { - await Browser.initialize({ - headless: true, - timeout: 10000, - blockResources: false, - }); - - const { page, contextId } = await Browser.createPageWithProxy( - 'https://httpbin.org/ip', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - - // Try to get the page content directly - const title = await page.title(); - console.log(` 📄 Page title: "${title}"`); - - // Try to evaluate some JavaScript - const result = await page.evaluate(() => { - return { - url: window.location.href, - userAgent: navigator.userAgent.substring(0, 50), - readyState: document.readyState, - }; - }); - - console.log(` 🔍 Page info:`, result); - - // Try to get page content - const bodyText = await page.locator('body').textContent(); - if (bodyText) { - console.log(` 📝 Body content (first 200 chars): ${bodyText.substring(0, 200)}...`); - - // Check if it looks like an IP response - if (bodyText.includes('origin')) { - console.log(' ✅ Looks like httpbin.org response!'); - } - } - - await Browser.closeContext(contextId); - return true; - - } catch (error) { - console.error(' ❌ Manual evaluation failed:', error); - return false; - } finally { - await Browser.close(); - } -} - -async function runProxyTests() { - console.log('🚀 Playwright Proxy Authentication Tests\n'); - - const authResult = await testPlaywrightProxyAuth(); - const manualResult = await testManualPageEvaluation(); - - console.log(`\n🏁 Final Results:`); - console.log(` 🔐 Proxy auth test: ${authResult ? '✅ PASS' : '❌ FAIL'}`); - console.log(` 🧪 Manual eval test: ${manualResult ? '✅ PASS' : '❌ FAIL'}`); -} - -if (import.meta.main) { - runProxyTests().catch(console.error); -} - -export { testPlaywrightProxyAuth, testManualPageEvaluation }; diff --git a/test-query-performance.ts b/test-query-performance.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-shutdown-simple.ts b/test-shutdown-simple.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-shutdown.ts b/test-shutdown.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-signals.ts b/test-signals.ts new file mode 100644 index 0000000..e69de29 diff --git a/test-simple-proxy.ts b/test-simple-proxy.ts deleted file mode 100644 index 7d7ec27..0000000 --- a/test-simple-proxy.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { Browser } from '@stock-bot/browser'; - -async function simpleProxyTest() { - console.log('🔬 Simple Proxy Test...'); - - try { - await Browser.initialize({ headless: true, timeout: 10000, blockResources: false }); - console.log('✅ Browser initialized'); - - await Browser.initialize({ headless: true, timeout: 10000, blockResources: false }); - console.log('✅ Browser initialized'); - - const { page, contextId } = await Browser.createPageWithProxy( - 'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/', - 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80' - ); - console.log('✅ Page created with proxy'); - let summaryData: any = null; // Initialize summaryData to store API response - let eventCount = 0; - page.onNetworkEvent(event => { - if (event.url.includes('/webrest/search/product-types/summary')) { - console.log(`🎯 Found summary API call: ${event.type} ${event.url}`); - - if (event.type === 'response' && event.responseData) { - console.log(`📊 Summary API Response Data: ${event.responseData}`); - try { - summaryData = JSON.parse(event.responseData) as any; - const totalCount = summaryData[0].totalCount; - console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2)); - console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`); - } catch (e) { - console.log('📊 Raw Summary Response:', event.responseData); - } - } - } - eventCount++; - console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`); - }); - - console.log('⏳ Waiting for page load...'); - await page.waitForLoadState('domcontentloaded', { timeout: 20000 }); - console.log('✅ Page loaded'); - - // RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button - console.log('🔍 Looking for Products tab...'); - - // Wait for the page to fully load - await page.waitForTimeout(20000); - - // First, click on the Products tab - const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]'); - await productsTab.waitFor({ timeout: 20000 }); - console.log('✅ Found Products tab'); - - console.log('🖱️ Clicking Products tab...'); - await productsTab.click(); - console.log('✅ Products tab clicked'); - - // Wait for the tab content to load - await page.waitForTimeout(5000); - - // Click on the Asset Classes accordion to expand it - console.log('🔍 Looking for Asset Classes accordion...'); - const assetClassesAccordion = page.locator( - '#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")' - ); - await assetClassesAccordion.waitFor({ timeout: 10000 }); - console.log('✅ Found Asset Classes accordion'); - - console.log('🖱️ Clicking Asset Classes accordion...'); - await assetClassesAccordion.click(); - console.log('✅ Asset Classes accordion clicked'); - - // Wait for the accordion content to expand - await page.waitForTimeout(2000); - - console.log('🔍 Looking for Stocks checkbox...'); - - // Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks" - const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")'); - await stocksSpan.waitFor({ timeout: 10000 }); - console.log('✅ Found Stocks span'); - - // Find the checkbox by looking in the same parent container - const parentContainer = stocksSpan.locator('..'); - const checkbox = parentContainer.locator('input[type="checkbox"]'); - - if ((await checkbox.count()) > 0) { - console.log('📋 Clicking Stocks checkbox...'); - await checkbox.first().check(); - console.log('✅ Stocks checkbox checked'); - } else { - console.log('⚠️ Could not find checkbox near Stocks text'); - } - - // Wait a moment for any UI updates - await page.waitForTimeout(1000); - - // Find and click the nearest Apply button - console.log('🔍 Looking for Apply button...'); - const applyButton = page.locator( - 'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]' - ); - - if ((await applyButton.count()) > 0) { - console.log('🎯 Clicking Apply button...'); - await applyButton.first().click(); - console.log('✅ Apply button clicked'); - - // Wait for any network requests triggered by the Apply button - await page.waitForTimeout(2000); - } else { - console.log('⚠️ Could not find Apply button'); - } - } catch (interactionError) { - const errorMessage = - interactionError instanceof Error ? interactionError.message : String(interactionError); - console.error('❌ Page interaction failed:', errorMessage); - - // Get debug info about the page - try { - const title = await page.title(); - console.log(`📄 Current page title: "${title}"`); - - const stocksElements = await page.locator('*:has-text("Stocks")').count(); - console.log(`🔍 Found ${stocksElements} elements containing "Stocks"`); - - const applyButtons = await page - .locator('button:has-text("Apply"), input[value*="Apply"]') - .count(); - console.log(`🔍 Found ${applyButtons} Apply buttons`); - } catch (debugError) { - const debugMessage = debugError instanceof Error ? debugError.message : String(debugError); - console.log('❌ Could not get debug info:', debugMessage); - } - } - - await new Promise(resolve => setTimeout(resolve, 2000)); - console.log(`📊 Total events: ${eventCount}`); - - await Browser.closeContext(contextId); - await Browser.close(); - - console.log('✅ Test completed'); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('❌ Error:', errorMessage); - await Browser.close(); - } -} - -simpleProxyTest(); diff --git a/test-simple.ts b/test-simple.ts deleted file mode 100644 index a9de60f..0000000 --- a/test-simple.ts +++ /dev/null @@ -1 +0,0 @@ -console.log("Testing browser import..."); import { Browser } from "@stock-bot/browser"; console.log("Browser imported successfully:", typeof Browser); Browser.initialize().then(() => console.log("Browser initialized")).catch(e => console.error("Error:", e));