boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

work on ib and cleanup

Browse source
This commit is contained in:
Boki 2025-06-14 09:17:48 -04:00
parent a20a11c1aa
commit d686a72591
41 changed files with 601 additions and 2793 deletions
Download patch file Download diff file Expand all files Collapse all files

8
.env
View file

@ -42,10 +42,10 @@ QUESTDB_PASSWORD=quest
# MongoDB Configuration # MongoDB Configuration
MONGODB_HOST=localhost MONGODB_HOST=localhost
MONGODB_PORT=27017 MONGODB_PORT=27017
MONGODB_DB=stockbot MONGODB_DATABASE=stock
MONGODB_USER= MONGODB_USERNAME=trading_admin
MONGODB_PASSWORD= MONGODB_PASSWORD=trading_mongo_dev
MONGODB_URI=mongodb://localhost:27017/stockbot MONGODB_URI=mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin
# =========================================== # ===========================================
# DATA PROVIDER CONFIGURATIONS # DATA PROVIDER CONFIGURATIONS

20
apps/data-service/src/index.ts
View file

@ -5,6 +5,7 @@ import { Hono } from 'hono';
import { Browser } from '@stock-bot/browser'; import { Browser } from '@stock-bot/browser';
import { loadEnvVariables } from '@stock-bot/config'; import { loadEnvVariables } from '@stock-bot/config';
import { getLogger, shutdownLoggers } from '@stock-bot/logger'; import { getLogger, shutdownLoggers } from '@stock-bot/logger';
import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client';
import { Shutdown } from '@stock-bot/shutdown'; import { Shutdown } from '@stock-bot/shutdown';
import { initializeIBResources } from './providers/ib.tasks'; import { initializeIBResources } from './providers/ib.tasks';
import { initializeProxyResources } from './providers/proxy.tasks'; import { initializeProxyResources } from './providers/proxy.tasks';
@ -18,7 +19,7 @@ loadEnvVariables();
const app = new Hono(); const app = new Hono();
const logger = getLogger('data-service'); const logger = getLogger('data-service');
const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002'); const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002');
let server: any = null; let server: ReturnType<typeof Bun.serve> | null = null;
// Initialize shutdown manager with 15 second timeout // Initialize shutdown manager with 15 second timeout
const shutdown = Shutdown.getInstance({ timeout: 15000 }); const shutdown = Shutdown.getInstance({ timeout: 15000 });
@ -35,6 +36,11 @@ async function initializeServices() {
logger.info('Initializing data service...'); logger.info('Initializing data service...');
try { try {
// Initialize MongoDB client first
logger.info('Starting MongoDB client initialization...');
await connectMongoDB();
logger.info('MongoDB client initialized');
// Initialize browser resources // Initialize browser resources
logger.info('Starting browser resources initialization...'); logger.info('Starting browser resources initialization...');
await Browser.initialize(); await Browser.initialize();
@ -122,6 +128,18 @@ shutdown.onShutdown(async () => {
} }
}); });
// Add MongoDB shutdown handler
shutdown.onShutdown(async () => {
logger.info('Shutting down MongoDB client...');
try {
await disconnectMongoDB();
logger.info('MongoDB client shut down successfully');
} catch (error) {
logger.error('Error shutting down MongoDB client', { error });
// Don't throw here to allow other shutdown handlers to complete
}
});
// Add logger shutdown handler (should be last) // Add logger shutdown handler (should be last)
shutdown.onShutdown(async () => { shutdown.onShutdown(async () => {
try { try {

31
apps/data-service/src/providers/ib.provider.ts
View file

@ -6,31 +6,36 @@ const logger = getLogger('ib-provider');
export const ibProvider: ProviderConfig = { export const ibProvider: ProviderConfig = {
name: 'ib', name: 'ib',
operations: { operations: {
'ib-basics': async () => { 'ib-exchanges-and-symbols': async () => {
const { ibTasks } = await import('./ib.tasks'); const { ibTasks } = await import('./ib.tasks');
logger.info('Fetching symbol summary from IB'); logger.info('Fetching symbol summary from IB');
const sessionHeaders = await ibTasks.fetchSession(); const sessionHeaders = await ibTasks.fetchSession();
logger.info('Fetched symbol summary from IB', { logger.info('Fetched symbol summary from IB');
sessionHeaders,
});
// Get Exchanges if (sessionHeaders) {
logger.info('Fetching exchanges from IB'); logger.info('Fetching exchanges from IB');
const exchanges = await ibTasks.fetchExchanges(sessionHeaders); const exchanges = await ibTasks.fetchExchanges(sessionHeaders);
logger.info('Fetched exchanges from IB', { exchanges }); logger.info('Fetched exchanges from IB', { count: exchanges.lenght });
// return total;
// do the same as above but for symbols
logger.info('Fetching symbols from IB');
const symbols = await ibTasks.fetchSymbols(sessionHeaders);
logger.info('Fetched symbols from IB', { symbols });
return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length };
}
}, },
}, },
scheduledJobs: [ scheduledJobs: [
{ {
type: 'ib-basics', type: 'ib-exchanges-and-symbols',
operation: 'ib-basics', operation: 'ib-exchanges-and-symbols',
payload: {}, payload: {},
// should remove and just run at the same time so app restarts dont keeping adding same jobs // should remove and just run at the same time so app restarts dont keeping adding same jobs
cronPattern: '*/2 * * * *', cronPattern: '0 0 * * 0',
priority: 5, priority: 5,
immediately: true, // Don't run immediately during startup to avoid conflicts // immediately: true, // Don't run immediately during startup to avoid conflicts
description: 'Fetch and validate proxy list from sources', description: 'Fetch and validate proxy list from sources',
}, },
], ],

150
apps/data-service/src/providers/ib.tasks.ts
View file

@ -1,12 +1,13 @@
import { Browser } from '@stock-bot/browser'; import { Browser } from '@stock-bot/browser';
import { getLogger } from '@stock-bot/logger'; import { getLogger } from '@stock-bot/logger';
import { getMongoDBClient } from '@stock-bot/mongodb-client';
// Shared instances (module-scoped, not global) // Shared instances (module-scoped, not global)
let isInitialized = false; // Track if resources are initialized let isInitialized = false; // Track if resources are initialized
let logger: ReturnType<typeof getLogger>; let logger: ReturnType<typeof getLogger>;
// let cache: CacheProvider; // let cache: CacheProvider;
export async function initializeIBResources(waitForCache = false): Promise<void> { export async function initializeIBResources(): Promise<void> {
// Skip if already initialized // Skip if already initialized
if (isInitialized) { if (isInitialized) {
return; return;
@ -93,7 +94,7 @@ export async function fetchSession(): Promise<Record<string, string> | undefined
// Wait for and return headers immediately when captured // Wait for and return headers immediately when captured
logger.info('⏳ Waiting for headers to be captured...'); logger.info('⏳ Waiting for headers to be captured...');
const headers = await headersPromise; const headers = await headersPromise;
page.close();
if (headers) { if (headers) {
logger.info('✅ Headers captured successfully'); logger.info('✅ Headers captured successfully');
} else { } else {
@ -151,19 +152,156 @@ export async function fetchExchanges(sessionHeaders: Record<string, string>): Pr
} }
const data = await response.json(); const data = await response.json();
const exchanges = data?.exchanges || [];
logger.info('✅ Exchange data fetched successfully');
logger.info('✅ Exchange data fetched successfully', { logger.info('Saving IB exchanges to MongoDB...');
dataKeys: Object.keys(data || {}), const client = getMongoDBClient();
dataSize: JSON.stringify(data).length, await client.batchUpsert('ib_exchanges', exchanges, ['id', 'country_code']);
logger.info('✅ Exchange IB data saved to MongoDB:', {
count: exchanges.length,
}); });
return data; return exchanges;
} catch (error) { } catch (error) {
logger.error('❌ Failed to fetch exchanges', { error }); logger.error('❌ Failed to fetch exchanges', { error });
return null; return null;
} }
} }
// Fetch symbols from IB using the session headers
export async function fetchSymbols(sessionHeaders: Record<string, string>): Promise<any> {
try {
logger.info('🔍 Fetching symbols with session headers...');
// Configure the proxy
const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80';
// Prepare headers - include all session headers plus any additional ones
const requestHeaders = {
...sessionHeaders,
Accept: 'application/json, text/plain, */*',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'no-cache',
Pragma: 'no-cache',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'X-Requested-With': 'XMLHttpRequest',
};
const requestBody = {
domain: 'com',
newProduct: 'all',
pageNumber: 1,
pageSize: 100,
productCountry: ['CA', 'US'],
productSymbol: '',
productType: ['STK'],
sortDirection: 'asc',
sortField: 'symbol',
};
// Get Summary
const summaryResponse = await fetch(
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
{
method: 'POST',
headers: requestHeaders,
proxy: proxyUrl,
body: JSON.stringify(requestBody),
}
);
if (!summaryResponse.ok) {
logger.error('❌ Summary API request failed', {
status: summaryResponse.status,
statusText: summaryResponse.statusText,
});
return null;
}
const summaryData = await summaryResponse.json();
logger.info('✅ IB Summary data fetched successfully', {
totalCount: summaryData[0].totalCount,
});
const symbols = [];
requestBody.pageSize = 500;
const pageCount = Math.ceil(summaryData[0].totalCount / 500) || 0;
logger.info('Fetching Symbols for IB', { pageCount });
const symbolPromises = [];
for (let page = 1; page <= pageCount; page++) {
requestBody.pageNumber = page;
// Fetch symbols for the current page
const symbolsResponse = fetch(
'https://www.interactivebrokers.com/webrest/search/products-by-filters',
{
method: 'POST',
headers: requestHeaders,
proxy: proxyUrl,
body: JSON.stringify(requestBody),
}
);
symbolPromises.push(symbolsResponse);
}
const responses = await Promise.all(symbolPromises);
for (const response of responses) {
if (!response.ok) {
logger.error('❌ Symbols API request failed', {
status: response.status,
statusText: response.statusText,
});
return null;
}
const data = await response.json();
const symJson = data?.products || [];
if (symJson && symJson.length > 0) {
symbols.push(...symJson);
} else {
logger.warn('⚠️ No symbols found in response');
continue;
}
}
if (symbols.length === 0) {
logger.warn('⚠️ No symbols fetched from IB');
return null;
}
logger.info('✅ IB symbols fetched successfully, saving to DB...', {
totalSymbols: symbols.length,
});
const client = getMongoDBClient();
await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
logger.info('Saved IB symbols to DB', {
totalSymbols: symbols.length,
});
// logger.info('📤 Making request to exchange API...', {
// url: exchangeUrl,
// headerCount: Object.keys(requestHeaders).length,
// });
// // Use fetch with proxy configuration
// const response = await fetch(exchangeUrl, {
// method: 'GET',
// headers: requestHeaders,
// proxy: proxyUrl,
// });
// if (!response.ok) {
// logger.error('❌ Exchange API request failed', {
// status: response.status,
// statusText: response.statusText,
// });
// return null;
// }
} catch (error) {
logger.error('❌ Failed to fetch symbols', { error });
return null;
}
}
export const ibTasks = { export const ibTasks = {
fetchSymbols,
fetchSession, fetchSession,
fetchExchanges, fetchExchanges,
}; };

424
database/mongodb/init/01-init-collections.js
View file

@ -2,231 +2,231 @@
// This script creates collections and indexes for sentiment and document storage // This script creates collections and indexes for sentiment and document storage
// Switch to the trading_documents database // Switch to the trading_documents database
db = db.getSiblingDB('trading_documents'); db = db.getSiblingDB('stock');
// Create collections with validation schemas // // Create collections with validation schemas
// Sentiment Analysis Collection // Sentiment Analysis Collection
db.createCollection('sentiment_analysis', { // db.createCollection('sentiment_analysis', {
validator: { // validator: {
$jsonSchema: { // $jsonSchema: {
bsonType: 'object', // bsonType: 'object',
required: ['symbol', 'source', 'timestamp', 'sentiment_score'], // required: ['symbol', 'source', 'timestamp', 'sentiment_score'],
properties: { // properties: {
symbol: { // symbol: {
bsonType: 'string', // bsonType: 'string',
description: 'Stock symbol (e.g., AAPL, GOOGL)' // description: 'Stock symbol (e.g., AAPL, GOOGL)'
}, // },
source: { // source: {
bsonType: 'string', // bsonType: 'string',
description: 'Data source (news, social, earnings_call, etc.)' // description: 'Data source (news, social, earnings_call, etc.)'
}, // },
timestamp: { // timestamp: {
bsonType: 'date', // bsonType: 'date',
description: 'When the sentiment was recorded' // description: 'When the sentiment was recorded'
}, // },
sentiment_score: { // sentiment_score: {
bsonType: 'double', // bsonType: 'double',
minimum: -1.0, // minimum: -1.0,
maximum: 1.0, // maximum: 1.0,
description: 'Sentiment score between -1 (negative) and 1 (positive)' // description: 'Sentiment score between -1 (negative) and 1 (positive)'
}, // },
confidence: { // confidence: {
bsonType: 'double', // bsonType: 'double',
minimum: 0.0, // minimum: 0.0,
maximum: 1.0, // maximum: 1.0,
description: 'Confidence level of the sentiment analysis' // description: 'Confidence level of the sentiment analysis'
}, // },
text_snippet: { // text_snippet: {
bsonType: 'string', // bsonType: 'string',
description: 'Original text that was analyzed' // description: 'Original text that was analyzed'
}, // },
metadata: { // metadata: {
bsonType: 'object', // bsonType: 'object',
description: 'Additional metadata about the sentiment source' // description: 'Additional metadata about the sentiment source'
} // }
} // }
} // }
} // }
}); // });
// Raw Documents Collection (for news articles, social media posts, etc.) // // Raw Documents Collection (for news articles, social media posts, etc.)
db.createCollection('raw_documents', { // db.createCollection('raw_documents', {
validator: { // validator: {
$jsonSchema: { // $jsonSchema: {
bsonType: 'object', // bsonType: 'object',
required: ['source', 'document_type', 'timestamp', 'content'], // required: ['source', 'document_type', 'timestamp', 'content'],
properties: { // properties: {
source: { // source: {
bsonType: 'string', // bsonType: 'string',
description: 'Document source (news_api, twitter, reddit, etc.)' // description: 'Document source (news_api, twitter, reddit, etc.)'
}, // },
document_type: { // document_type: {
bsonType: 'string', // bsonType: 'string',
enum: ['news_article', 'social_post', 'earnings_transcript', 'research_report', 'press_release'], // enum: ['news_article', 'social_post', 'earnings_transcript', 'research_report', 'press_release'],
description: 'Type of document' // description: 'Type of document'
}, // },
timestamp: { // timestamp: {
bsonType: 'date', // bsonType: 'date',
description: 'When the document was created/published' // description: 'When the document was created/published'
}, // },
symbols: { // symbols: {
bsonType: 'array', // bsonType: 'array',
items: { // items: {
bsonType: 'string' // bsonType: 'string'
}, // },
description: 'Array of stock symbols mentioned in the document' // description: 'Array of stock symbols mentioned in the document'
}, // },
title: { // title: {
bsonType: 'string', // bsonType: 'string',
description: 'Document title or headline' // description: 'Document title or headline'
}, // },
content: { // content: {
bsonType: 'string', // bsonType: 'string',
description: 'Full document content' // description: 'Full document content'
}, // },
url: { // url: {
bsonType: 'string', // bsonType: 'string',
description: 'Original URL of the document' // description: 'Original URL of the document'
}, // },
author: { // author: {
bsonType: 'string', // bsonType: 'string',
description: 'Document author or source account' // description: 'Document author or source account'
}, // },
processed: { // processed: {
bsonType: 'bool', // bsonType: 'bool',
description: 'Whether this document has been processed for sentiment' // description: 'Whether this document has been processed for sentiment'
}, // },
metadata: { // metadata: {
bsonType: 'object', // bsonType: 'object',
description: 'Additional document metadata' // description: 'Additional document metadata'
} // }
} // }
} // }
} // }
}); // });
// Market Events Collection (for significant market events and their impact) // // Market Events Collection (for significant market events and their impact)
db.createCollection('market_events', { // db.createCollection('market_events', {
validator: { // validator: {
$jsonSchema: { // $jsonSchema: {
bsonType: 'object', // bsonType: 'object',
required: ['event_type', 'timestamp', 'description'], // required: ['event_type', 'timestamp', 'description'],
properties: { // properties: {
event_type: { // event_type: {
bsonType: 'string', // bsonType: 'string',
enum: ['earnings', 'merger', 'acquisition', 'ipo', 'dividend', 'split', 'regulatory', 'economic_indicator'], // enum: ['earnings', 'merger', 'acquisition', 'ipo', 'dividend', 'split', 'regulatory', 'economic_indicator'],
description: 'Type of market event' // description: 'Type of market event'
}, // },
timestamp: { // timestamp: {
bsonType: 'date', // bsonType: 'date',
description: 'When the event occurred or was announced' // description: 'When the event occurred or was announced'
}, // },
symbols: { // symbols: {
bsonType: 'array', // bsonType: 'array',
items: { // items: {
bsonType: 'string' // bsonType: 'string'
}, // },
description: 'Stock symbols affected by this event' // description: 'Stock symbols affected by this event'
}, // },
description: { // description: {
bsonType: 'string', // bsonType: 'string',
description: 'Event description' // description: 'Event description'
}, // },
impact_score: { // impact_score: {
bsonType: 'double', // bsonType: 'double',
minimum: -5.0, // minimum: -5.0,
maximum: 5.0, // maximum: 5.0,
description: 'Expected market impact score' // description: 'Expected market impact score'
}, // },
source_documents: { // source_documents: {
bsonType: 'array', // bsonType: 'array',
items: { // items: {
bsonType: 'objectId' // bsonType: 'objectId'
}, // },
description: 'References to raw_documents that reported this event' // description: 'References to raw_documents that reported this event'
} // }
} // }
} // }
} // }
}); // });
// Create indexes for efficient querying // // Create indexes for efficient querying
// Sentiment Analysis indexes // // Sentiment Analysis indexes
db.sentiment_analysis.createIndex({ symbol: 1, timestamp: -1 }); // db.sentiment_analysis.createIndex({ symbol: 1, timestamp: -1 });
db.sentiment_analysis.createIndex({ source: 1, timestamp: -1 }); // db.sentiment_analysis.createIndex({ source: 1, timestamp: -1 });
db.sentiment_analysis.createIndex({ timestamp: -1 }); // db.sentiment_analysis.createIndex({ timestamp: -1 });
db.sentiment_analysis.createIndex({ symbol: 1, source: 1, timestamp: -1 }); // db.sentiment_analysis.createIndex({ symbol: 1, source: 1, timestamp: -1 });
// Raw Documents indexes // // Raw Documents indexes
db.raw_documents.createIndex({ symbols: 1, timestamp: -1 }); // db.raw_documents.createIndex({ symbols: 1, timestamp: -1 });
db.raw_documents.createIndex({ source: 1, timestamp: -1 }); // db.raw_documents.createIndex({ source: 1, timestamp: -1 });
db.raw_documents.createIndex({ document_type: 1, timestamp: -1 }); // db.raw_documents.createIndex({ document_type: 1, timestamp: -1 });
db.raw_documents.createIndex({ processed: 1, timestamp: -1 }); // db.raw_documents.createIndex({ processed: 1, timestamp: -1 });
db.raw_documents.createIndex({ timestamp: -1 }); // db.raw_documents.createIndex({ timestamp: -1 });
// Market Events indexes // // Market Events indexes
db.market_events.createIndex({ symbols: 1, timestamp: -1 }); // db.market_events.createIndex({ symbols: 1, timestamp: -1 });
db.market_events.createIndex({ event_type: 1, timestamp: -1 }); // db.market_events.createIndex({ event_type: 1, timestamp: -1 });
db.market_events.createIndex({ timestamp: -1 }); // db.market_events.createIndex({ timestamp: -1 });
// Insert some sample data for testing // // Insert some sample data for testing
// Sample sentiment data // // Sample sentiment data
db.sentiment_analysis.insertMany([ // db.sentiment_analysis.insertMany([
{ // {
symbol: 'AAPL', // symbol: 'AAPL',
source: 'news_analysis', // source: 'news_analysis',
timestamp: new Date(), // timestamp: new Date(),
sentiment_score: 0.75, // sentiment_score: 0.75,
confidence: 0.89, // confidence: 0.89,
text_snippet: 'Apple reports strong quarterly earnings...', // text_snippet: 'Apple reports strong quarterly earnings...',
metadata: { // metadata: {
article_id: 'news_001', // article_id: 'news_001',
provider: 'financial_news_api' // provider: 'financial_news_api'
} // }
}, // },
{ // {
symbol: 'GOOGL', // symbol: 'GOOGL',
source: 'social_media', // source: 'social_media',
timestamp: new Date(), // timestamp: new Date(),
sentiment_score: -0.25, // sentiment_score: -0.25,
confidence: 0.67, // confidence: 0.67,
text_snippet: 'Concerns about Google AI regulation...', // text_snippet: 'Concerns about Google AI regulation...',
metadata: { // metadata: {
platform: 'twitter', // platform: 'twitter',
engagement_score: 450 // engagement_score: 450
} // }
} // }
]); // ]);
// Sample raw document // // Sample raw document
db.raw_documents.insertOne({ // db.raw_documents.insertOne({
source: 'financial_news_api', // source: 'financial_news_api',
document_type: 'news_article', // document_type: 'news_article',
timestamp: new Date(), // timestamp: new Date(),
symbols: ['AAPL', 'MSFT'], // symbols: ['AAPL', 'MSFT'],
title: 'Tech Giants Show Strong Q4 Performance', // title: 'Tech Giants Show Strong Q4 Performance',
content: 'Apple and Microsoft both reported better than expected earnings for Q4...', // content: 'Apple and Microsoft both reported better than expected earnings for Q4...',
url: 'https://example.com/tech-earnings-q4', // url: 'https://example.com/tech-earnings-q4',
author: 'Financial Reporter', // author: 'Financial Reporter',
processed: true, // processed: true,
metadata: { // metadata: {
word_count: 850, // word_count: 850,
readability_score: 0.75 // readability_score: 0.75
} // }
}); // });
// Sample market event // // Sample market event
db.market_events.insertOne({ // db.market_events.insertOne({
event_type: 'earnings', // event_type: 'earnings',
timestamp: new Date(), // timestamp: new Date(),
symbols: ['AAPL'], // symbols: ['AAPL'],
description: 'Apple Q4 2024 Earnings Report', // description: 'Apple Q4 2024 Earnings Report',
impact_score: 2.5, // impact_score: 2.5,
source_documents: [] // source_documents: []
}); // });
print('MongoDB initialization completed successfully!'); print('MongoDB initialization completed successfully!');
print('Created collections: sentiment_analysis, raw_documents, market_events'); print('Created collections: sentiment_analysis, raw_documents, market_events');

0
test-network.ts → database/postgres/providers/01-ib-simple.sql
View file

0
test-proxy.ts → database/postgres/scripts/ib.ts
View file

0
test-user-agent.js → database/postgres/scripts/populate-ib-exchanges-simple.ts
View file

0
database/postgres/scripts/populate-ib-exchanges.ts Normal file
View file

0
database/postgres/scripts/setup-ib-fast.ts Normal file
View file

0
database/postgres/scripts/setup-ib-schema-simple.ts Normal file
View file

0
database/postgres/scripts/setup-ib-schema.ts Normal file
View file

0
database/postgres/scripts/setup.ts Normal file
View file

2
docker-compose.yml
View file

@ -82,7 +82,7 @@ services: # Dragonfly - Redis replacement for caching and events
environment: environment:
MONGO_INITDB_ROOT_USERNAME: trading_admin MONGO_INITDB_ROOT_USERNAME: trading_admin
MONGO_INITDB_ROOT_PASSWORD: trading_mongo_dev MONGO_INITDB_ROOT_PASSWORD: trading_mongo_dev
MONGO_INITDB_DATABASE: trading_documents MONGO_INITDB_DATABASE: stock
ports: ports:
- "27017:27017" - "27017:27017"
volumes: volumes:

2
libs/config/src/mongodb.ts
View file

@ -13,7 +13,7 @@ export const mongodbConfig = cleanEnv(process.env, {
// MongoDB Connection // MongoDB Connection
MONGODB_HOST: str('localhost', 'MongoDB host'), MONGODB_HOST: str('localhost', 'MongoDB host'),
MONGODB_PORT: port(27017, 'MongoDB port'), MONGODB_PORT: port(27017, 'MongoDB port'),
MONGODB_DATABASE: str('trading_documents', 'MongoDB database name'), MONGODB_DATABASE: str('stock', 'MongoDB database name'),
// Authentication // Authentication
MONGODB_USERNAME: str('trading_admin', 'MongoDB username'), MONGODB_USERNAME: str('trading_admin', 'MongoDB username'),

249
libs/mongodb-client/src/aggregation.ts
View file

@ -1,249 +0,0 @@
import type { Document } from 'mongodb';
import type { MongoDBClient } from './client';
import type { CollectionNames } from './types';
/**
* MongoDB Aggregation Builder
*
* Provides a fluent interface for building MongoDB aggregation pipelines
*/
export class MongoDBAggregationBuilder {
private pipeline: any[] = [];
private readonly client: MongoDBClient;
private collection: CollectionNames | null = null;
constructor(client: MongoDBClient) {
this.client = client;
}
/**
* Set the collection to aggregate on
*/
from(collection: CollectionNames): this {
this.collection = collection;
return this;
}
/**
* Add a match stage
*/
match(filter: any): this {
this.pipeline.push({ $match: filter });
return this;
}
/**
* Add a group stage
*/
group(groupBy: any): this {
this.pipeline.push({ $group: groupBy });
return this;
}
/**
* Add a sort stage
*/
sort(sortBy: any): this {
this.pipeline.push({ $sort: sortBy });
return this;
}
/**
* Add a limit stage
*/
limit(count: number): this {
this.pipeline.push({ $limit: count });
return this;
}
/**
* Add a skip stage
*/
skip(count: number): this {
this.pipeline.push({ $skip: count });
return this;
}
/**
* Add a project stage
*/
project(projection: any): this {
this.pipeline.push({ $project: projection });
return this;
}
/**
* Add an unwind stage
*/
unwind(field: string, options?: any): this {
this.pipeline.push({
$unwind: options ? { path: field, ...options } : field,
});
return this;
}
/**
* Add a lookup stage (join)
*/
lookup(from: string, localField: string, foreignField: string, as: string): this {
this.pipeline.push({
$lookup: {
from,
localField,
foreignField,
as,
},
});
return this;
}
/**
* Add a custom stage
*/
addStage(stage: any): this {
this.pipeline.push(stage);
return this;
}
/**
* Execute the aggregation pipeline
*/
async execute<T extends Document = Document>(): Promise<T[]> {
if (!this.collection) {
throw new Error('Collection not specified. Use .from() to set the collection.');
}
const collection = this.client.getCollection(this.collection);
return await collection.aggregate<T>(this.pipeline).toArray();
}
/**
* Get the pipeline array
*/
getPipeline(): any[] {
return [...this.pipeline];
}
/**
* Reset the pipeline
*/
reset(): this {
this.pipeline = [];
this.collection = null;
return this;
}
// Convenience methods for common aggregations
/**
* Sentiment analysis aggregation
*/
sentimentAnalysis(symbol?: string, timeframe?: { start: Date; end: Date }): this {
this.from('sentiment_data');
const matchConditions: any = {};
if (symbol) {
matchConditions.symbol = symbol;
}
if (timeframe) {
matchConditions.timestamp = {
$gte: timeframe.start,
$lte: timeframe.end,
};
}
if (Object.keys(matchConditions).length > 0) {
this.match(matchConditions);
}
return this.group({
_id: {
symbol: '$symbol',
sentiment: '$sentiment_label',
},
count: { $sum: 1 },
avgScore: { $avg: '$sentiment_score' },
avgConfidence: { $avg: '$confidence' },
});
}
/**
* News article aggregation by publication
*/
newsByPublication(symbols?: string[]): this {
this.from('news_articles');
if (symbols && symbols.length > 0) {
this.match({ symbols: { $in: symbols } });
}
return this.group({
_id: '$publication',
articleCount: { $sum: 1 },
symbols: { $addToSet: '$symbols' },
avgSentiment: { $avg: '$sentiment_score' },
latestArticle: { $max: '$published_date' },
});
}
/**
* SEC filings by company
*/
secFilingsByCompany(filingTypes?: string[]): this {
this.from('sec_filings');
if (filingTypes && filingTypes.length > 0) {
this.match({ filing_type: { $in: filingTypes } });
}
return this.group({
_id: {
cik: '$cik',
company: '$company_name',
},
filingCount: { $sum: 1 },
filingTypes: { $addToSet: '$filing_type' },
latestFiling: { $max: '$filing_date' },
symbols: { $addToSet: '$symbols' },
});
}
/**
* Document processing status summary
*/
processingStatusSummary(collection: CollectionNames): this {
this.from(collection);
return this.group({
_id: '$processing_status',
count: { $sum: 1 },
avgSizeBytes: { $avg: '$size_bytes' },
oldestDocument: { $min: '$created_at' },
newestDocument: { $max: '$created_at' },
});
}
/**
* Time-based aggregation (daily/hourly counts)
*/
timeBasedCounts(
collection: CollectionNames,
dateField: string = 'created_at',
interval: 'hour' | 'day' | 'week' | 'month' = 'day'
): this {
this.from(collection);
const dateFormat = {
hour: { $dateToString: { format: '%Y-%m-%d %H:00:00', date: `$${dateField}` } },
day: { $dateToString: { format: '%Y-%m-%d', date: `$${dateField}` } },
week: { $dateToString: { format: '%Y-W%V', date: `$${dateField}` } },
month: { $dateToString: { format: '%Y-%m', date: `$${dateField}` } },
};
return this.group({
_id: dateFormat[interval],
count: { $sum: 1 },
firstDocument: { $min: `$${dateField}` },
lastDocument: { $max: `$${dateField}` },
}).sort({ _id: 1 });
}
}

485
libs/mongodb-client/src/client.ts
View file

@ -1,110 +1,68 @@
import { import { Collection, Db, MongoClient, OptionalUnlessRequiredId } from 'mongodb';
Collection,
Db,
Document,
MongoClient,
MongoClientOptions,
OptionalUnlessRequiredId,
WithId,
} from 'mongodb';
import * as yup from 'yup';
import { mongodbConfig } from '@stock-bot/config'; import { mongodbConfig } from '@stock-bot/config';
import { getLogger } from '@stock-bot/logger'; import { getLogger } from '@stock-bot/logger';
import { MongoDBHealthMonitor } from './health'; import type { DocumentBase } from './types';
import { schemaMap } from './schemas';
import type {
AnalystReport,
CollectionNames,
DocumentBase,
EarningsTranscript,
MongoDBClientConfig,
MongoDBConnectionOptions,
NewsArticle,
RawDocument,
SecFiling,
SentimentData,
} from './types';
/** /**
* MongoDB Client for Stock Bot * Simplified MongoDB Client for Stock Bot Data Service
* *
* Provides type-safe access to MongoDB collections with built-in * A singleton MongoDB client focused solely on batch upsert operations
* health monitoring, connection pooling, and schema validation. * with minimal configuration and no health monitoring complexity.
*/ */
export class MongoDBClient { export class MongoDBClient {
private static instance: MongoDBClient | null = null;
private client: MongoClient | null = null; private client: MongoClient | null = null;
private db: Db | null = null; private db: Db | null = null;
private readonly config: MongoDBClientConfig; private readonly logger = getLogger('mongodb-client-simple');
private readonly options: MongoDBConnectionOptions;
private readonly logger: ReturnType<typeof getLogger>;
private readonly healthMonitor: MongoDBHealthMonitor;
private isConnected = false; private isConnected = false;
constructor(config?: Partial<MongoDBClientConfig>, options?: MongoDBConnectionOptions) { private constructor() {}
this.config = this.buildConfig(config);
this.options = {
retryAttempts: 3,
retryDelay: 1000,
healthCheckInterval: 30000,
...options,
};
this.logger = getLogger('mongodb-client'); /**
this.healthMonitor = new MongoDBHealthMonitor(this); * Get singleton instance
*/
static getInstance(): MongoDBClient {
if (!MongoDBClient.instance) {
MongoDBClient.instance = new MongoDBClient();
}
return MongoDBClient.instance;
} }
/** /**
* Connect to MongoDB * Connect to MongoDB with simple configuration
*/ */
async connect(): Promise<void> { async connect(): Promise<void> {
if (this.isConnected && this.client) { if (this.isConnected && this.client) {
return; return;
} }
const uri = this.buildConnectionUri(); try {
const clientOptions = this.buildClientOptions(); const uri = this.buildConnectionUri();
this.logger.info('Connecting to MongoDB...');
let lastError: Error | null = null; this.client = new MongoClient(uri, {
maxPoolSize: 10,
minPoolSize: 1,
connectTimeoutMS: 10000,
socketTimeoutMS: 30000,
serverSelectionTimeoutMS: 5000,
});
for (let attempt = 1; attempt <= this.options.retryAttempts!; attempt++) { await this.client.connect();
try { await this.client.db(mongodbConfig.MONGODB_DATABASE).admin().ping();
this.logger.info(
`Connecting to MongoDB (attempt ${attempt}/${this.options.retryAttempts})...`
);
this.client = new MongoClient(uri, clientOptions); this.db = this.client.db(mongodbConfig.MONGODB_DATABASE);
await this.client.connect(); this.isConnected = true;
// Test the connection this.logger.info('Successfully connected to MongoDB');
await this.client.db(this.config.database).admin().ping(); } catch (error) {
this.logger.error('MongoDB connection failed:', error);
this.db = this.client.db(this.config.database); if (this.client) {
this.isConnected = true; await this.client.close();
this.client = null;
this.logger.info('Successfully connected to MongoDB');
// Start health monitoring
this.healthMonitor.start();
return;
} catch (error) {
lastError = error as Error;
this.logger.error(`MongoDB connection attempt ${attempt} failed:`, error);
if (this.client) {
await this.client.close();
this.client = null;
}
if (attempt < this.options.retryAttempts!) {
await this.delay(this.options.retryDelay! * attempt);
}
} }
throw error;
} }
throw new Error(
`Failed to connect to MongoDB after ${this.options.retryAttempts} attempts: ${lastError?.message}`
);
} }
/** /**
@ -116,7 +74,6 @@ export class MongoDBClient {
} }
try { try {
this.healthMonitor.stop();
await this.client.close(); await this.client.close();
this.isConnected = false; this.isConnected = false;
this.client = null; this.client = null;
@ -128,10 +85,138 @@ export class MongoDBClient {
} }
} }
/**
* Batch upsert documents for high-performance operations
* Supports single or multiple unique keys for matching
*/
async batchUpsert<T extends DocumentBase>(
collectionName: string,
documents: Array<
Omit<T, '_id' | 'created_at' | 'updated_at'> & Partial<Pick<T, 'created_at' | 'updated_at'>>
>,
uniqueKeys: string | string[],
options: {
chunkSize?: number;
} = {}
): Promise<{ insertedCount: number; updatedCount: number; errors: unknown[] }> {
if (!this.db) {
throw new Error('MongoDB client not connected');
}
if (documents.length === 0) {
return { insertedCount: 0, updatedCount: 0, errors: [] };
}
// Normalize uniqueKeys to array
const keyFields = Array.isArray(uniqueKeys) ? uniqueKeys : [uniqueKeys];
if (keyFields.length === 0) {
throw new Error('At least one unique key must be provided');
}
const { chunkSize = 10000 } = options;
const collection = this.db.collection<T>(collectionName);
const operationId = Math.random().toString(36).substring(7);
let totalInserted = 0;
let totalUpdated = 0;
const errors: unknown[] = [];
this.logger.info(`Starting batch upsert operation [${operationId}]`, {
collection: collectionName,
totalDocuments: documents.length,
uniqueKeys: keyFields,
chunkSize
});
// Process documents in chunks to avoid memory issues
for (let i = 0; i < documents.length; i += chunkSize) {
const chunk = documents.slice(i, i + chunkSize);
try {
const startTime = Date.now();
// Prepare bulk operations
const bulkOps = chunk.map(doc => {
const now = new Date();
const docWithTimestamps = {
...doc,
created_at: doc.created_at || now,
updated_at: now,
};
// Create filter using multiple unique keys
const filter: Record<string, unknown> = {};
keyFields.forEach(key => {
const value = (doc as Record<string, unknown>)[key];
if (value === undefined || value === null) {
throw new Error(`Document missing required unique key: ${key}`);
}
filter[key] = value;
});
// Remove created_at from $set to avoid conflict with $setOnInsert
const { created_at, ...updateFields } = docWithTimestamps;
return {
updateOne: {
filter,
update: {
$set: updateFields,
$setOnInsert: { created_at },
},
upsert: true,
},
};
});
// Execute bulk operation with type assertion to handle complex MongoDB types
const result = await collection.bulkWrite(bulkOps as never, { ordered: false });
const executionTime = Date.now() - startTime;
const inserted = result.upsertedCount;
const updated = result.modifiedCount;
totalInserted += inserted;
totalUpdated += updated;
this.logger.debug(`Batch upsert chunk processed [${operationId}]`, {
chunkNumber: Math.floor(i / chunkSize) + 1,
chunkSize: chunk.length,
inserted,
updated,
executionTime,
collection: collectionName,
});
} catch (error) {
this.logger.error(`Batch upsert failed on chunk [${operationId}]`, {
error,
collection: collectionName,
chunkNumber: Math.floor(i / chunkSize) + 1,
chunkStart: i,
chunkSize: chunk.length,
uniqueKeys: keyFields,
});
errors.push(error);
}
}
this.logger.info(`Batch upsert completed [${operationId}]`, {
collection: collectionName,
totalRecords: documents.length,
inserted: totalInserted,
updated: totalUpdated,
errors: errors.length,
uniqueKeys: keyFields,
});
return { insertedCount: totalInserted, updatedCount: totalUpdated, errors };
}
/** /**
* Get a typed collection * Get a typed collection
*/ */
getCollection<T extends DocumentBase>(name: CollectionNames): Collection<T> { getCollection<T extends DocumentBase>(name: string): Collection<T> {
if (!this.db) { if (!this.db) {
throw new Error('MongoDB client not connected'); throw new Error('MongoDB client not connected');
} }
@ -139,162 +224,26 @@ export class MongoDBClient {
} }
/** /**
* Insert a document with validation * Simple insert operation
*/ */
async insertOne<T extends DocumentBase>( async insertOne<T extends DocumentBase>(
collectionName: CollectionNames, collectionName: string,
document: Omit<T, '_id' | 'created_at' | 'updated_at'> & document: Omit<T, '_id' | 'created_at' | 'updated_at'> &
Partial<Pick<T, 'created_at' | 'updated_at'>> Partial<Pick<T, 'created_at' | 'updated_at'>>
): Promise<T> { ): Promise<T> {
const collection = this.getCollection<T>(collectionName); const collection = this.getCollection<T>(collectionName);
// Add timestamps
const now = new Date(); const now = new Date();
const docWithTimestamps = { const docWithTimestamps = {
...document, ...document,
created_at: document.created_at || now, created_at: document.created_at || now,
updated_at: now, updated_at: now,
} as T; // Validate document if schema exists } as T;
if (collectionName in schemaMap) {
try {
(schemaMap as any)[collectionName].validateSync(docWithTimestamps);
} catch (error) {
if (error instanceof yup.ValidationError) {
this.logger.error(`Document validation failed for ${collectionName}:`, error.errors);
throw new Error(`Document validation failed: ${error.errors?.map(e => e).join(', ')}`);
}
throw error;
}
}
const result = await collection.insertOne(docWithTimestamps as OptionalUnlessRequiredId<T>); const result = await collection.insertOne(docWithTimestamps as OptionalUnlessRequiredId<T>);
return { ...docWithTimestamps, _id: result.insertedId } as T; return { ...docWithTimestamps, _id: result.insertedId } as T;
} }
/**
* Update a document with validation
*/
async updateOne<T extends DocumentBase>(
collectionName: CollectionNames,
filter: any,
update: Partial<T>
): Promise<boolean> {
const collection = this.getCollection<T>(collectionName);
// Add updated timestamp
const updateWithTimestamp = {
...update,
updated_at: new Date(),
};
const result = await collection.updateOne(filter, { $set: updateWithTimestamp });
return result.modifiedCount > 0;
}
/**
* Find documents with optional validation
*/
async find<T extends DocumentBase>(
collectionName: CollectionNames,
filter: any = {},
options: any = {}
): Promise<T[]> {
const collection = this.getCollection<T>(collectionName);
return (await collection.find(filter, options).toArray()) as T[];
}
/**
* Find one document
*/
async findOne<T extends DocumentBase>(
collectionName: CollectionNames,
filter: any
): Promise<T | null> {
const collection = this.getCollection<T>(collectionName);
return (await collection.findOne(filter)) as T | null;
}
/**
* Aggregate with type safety
*/
async aggregate<T extends DocumentBase>(
collectionName: CollectionNames,
pipeline: any[]
): Promise<T[]> {
const collection = this.getCollection<T>(collectionName);
return await collection.aggregate<T>(pipeline).toArray();
}
/**
* Count documents
*/
async countDocuments(collectionName: CollectionNames, filter: any = {}): Promise<number> {
const collection = this.getCollection(collectionName);
return await collection.countDocuments(filter);
}
/**
* Create indexes for better performance
*/
async createIndexes(): Promise<void> {
if (!this.db) {
throw new Error('MongoDB client not connected');
}
try {
// Sentiment data indexes
await this.db
.collection('sentiment_data')
.createIndexes([
{ key: { symbol: 1, timestamp: -1 } },
{ key: { sentiment_label: 1 } },
{ key: { source_type: 1 } },
{ key: { created_at: -1 } },
]);
// News articles indexes
await this.db
.collection('news_articles')
.createIndexes([
{ key: { symbols: 1, published_date: -1 } },
{ key: { publication: 1 } },
{ key: { categories: 1 } },
{ key: { created_at: -1 } },
]);
// SEC filings indexes
await this.db
.collection('sec_filings')
.createIndexes([
{ key: { symbols: 1, filing_date: -1 } },
{ key: { filing_type: 1 } },
{ key: { cik: 1 } },
{ key: { created_at: -1 } },
]); // Raw documents indexes
await this.db.collection('raw_documents').createIndex({ content_hash: 1 }, { unique: true });
await this.db
.collection('raw_documents')
.createIndexes([
{ key: { processing_status: 1 } },
{ key: { document_type: 1 } },
{ key: { created_at: -1 } },
]);
this.logger.info('MongoDB indexes created successfully');
} catch (error) {
this.logger.error('Error creating MongoDB indexes:', error);
throw error;
}
}
/**
* Get database statistics
*/
async getStats(): Promise<any> {
if (!this.db) {
throw new Error('MongoDB client not connected');
}
return await this.db.stats();
}
/** /**
* Check if client is connected * Check if client is connected
*/ */
@ -302,13 +251,6 @@ export class MongoDBClient {
return this.isConnected && !!this.client; return this.isConnected && !!this.client;
} }
/**
* Get the underlying MongoDB client
*/
get mongoClient(): MongoClient | null {
return this.client;
}
/** /**
* Get the database instance * Get the database instance
*/ */
@ -316,81 +258,24 @@ export class MongoDBClient {
return this.db; return this.db;
} }
private buildConfig(config?: Partial<MongoDBClientConfig>): MongoDBClientConfig {
return {
host: config?.host || mongodbConfig.MONGODB_HOST,
port: config?.port || mongodbConfig.MONGODB_PORT,
database: config?.database || mongodbConfig.MONGODB_DATABASE,
username: config?.username || mongodbConfig.MONGODB_USERNAME,
password: config?.password || mongodbConfig.MONGODB_PASSWORD,
authSource: config?.authSource || mongodbConfig.MONGODB_AUTH_SOURCE,
uri: config?.uri || mongodbConfig.MONGODB_URI,
poolSettings: {
maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE,
maxIdleTime: mongodbConfig.MONGODB_MAX_IDLE_TIME,
...config?.poolSettings,
},
timeouts: {
connectTimeout: mongodbConfig.MONGODB_CONNECT_TIMEOUT,
socketTimeout: mongodbConfig.MONGODB_SOCKET_TIMEOUT,
serverSelectionTimeout: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT,
...config?.timeouts,
},
tls: {
enabled: mongodbConfig.MONGODB_TLS,
insecure: mongodbConfig.MONGODB_TLS_INSECURE,
caFile: mongodbConfig.MONGODB_TLS_CA_FILE,
...config?.tls,
},
options: {
retryWrites: mongodbConfig.MONGODB_RETRY_WRITES,
journal: mongodbConfig.MONGODB_JOURNAL,
readPreference: mongodbConfig.MONGODB_READ_PREFERENCE as any,
writeConcern: mongodbConfig.MONGODB_WRITE_CONCERN,
...config?.options,
},
};
}
private buildConnectionUri(): string { private buildConnectionUri(): string {
if (this.config.uri) { if (mongodbConfig.MONGODB_URI) {
return this.config.uri; return mongodbConfig.MONGODB_URI;
} }
const { host, port, username, password, database, authSource } = this.config; const {
MONGODB_HOST: host,
MONGODB_PORT: port,
MONGODB_USERNAME: username,
MONGODB_PASSWORD: password,
MONGODB_DATABASE: database,
MONGODB_AUTH_SOURCE: authSource,
} = mongodbConfig;
// Build URI components
const auth = username && password ? `${username}:${password}@` : ''; const auth = username && password ? `${username}:${password}@` : '';
const authDb = authSource ? `?authSource=${authSource}` : ''; const authParam = authSource && username ? `?authSource=${authSource}` : '';
return `mongodb://${auth}${host}:${port}/${database}${authDb}`; return `mongodb://${auth}${host}:${port}/${database}${authParam}`;
}
private buildClientOptions(): MongoClientOptions {
return {
maxPoolSize: this.config.poolSettings?.maxPoolSize,
minPoolSize: this.config.poolSettings?.minPoolSize,
maxIdleTimeMS: this.config.poolSettings?.maxIdleTime,
connectTimeoutMS: this.config.timeouts?.connectTimeout,
socketTimeoutMS: this.config.timeouts?.socketTimeout,
serverSelectionTimeoutMS: this.config.timeouts?.serverSelectionTimeout,
retryWrites: this.config.options?.retryWrites,
journal: this.config.options?.journal,
readPreference: this.config.options?.readPreference,
writeConcern: this.config.options?.writeConcern
? {
w:
this.config.options.writeConcern === 'majority'
? ('majority' as const)
: parseInt(this.config.options.writeConcern, 10) || 1,
}
: undefined,
tls: this.config.tls?.enabled,
tlsInsecure: this.config.tls?.insecure,
tlsCAFile: this.config.tls?.caFile,
};
}
private delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
} }
} }

49
libs/mongodb-client/src/factory.ts
View file

@ -1,56 +1,19 @@
import { mongodbConfig } from '@stock-bot/config';
import { MongoDBClient } from './client'; import { MongoDBClient } from './client';
import type { MongoDBClientConfig, MongoDBConnectionOptions } from './types';
/** /**
* Factory function to create a MongoDB client instance * Get the singleton MongoDB client instance
*/
export function createMongoDBClient(
config?: Partial<MongoDBClientConfig>,
options?: MongoDBConnectionOptions
): MongoDBClient {
return new MongoDBClient(config, options);
}
/**
* Create a MongoDB client with default configuration
*/
export function createDefaultMongoDBClient(): MongoDBClient {
const config: Partial<MongoDBClientConfig> = {
host: mongodbConfig.MONGODB_HOST,
port: mongodbConfig.MONGODB_PORT,
database: mongodbConfig.MONGODB_DATABASE,
username: mongodbConfig.MONGODB_USERNAME,
password: mongodbConfig.MONGODB_PASSWORD,
uri: mongodbConfig.MONGODB_URI,
};
return new MongoDBClient(config);
}
/**
* Singleton MongoDB client instance
*/
let defaultClient: MongoDBClient | null = null;
/**
* Get or create the default MongoDB client instance
*/ */
export function getMongoDBClient(): MongoDBClient { export function getMongoDBClient(): MongoDBClient {
if (!defaultClient) { return MongoDBClient.getInstance();
defaultClient = createDefaultMongoDBClient();
}
return defaultClient;
} }
/** /**
* Connect to MongoDB using the default client * Connect to MongoDB using the singleton client
*/ */
export async function connectMongoDB(): Promise<MongoDBClient> { export async function connectMongoDB(): Promise<MongoDBClient> {
const client = getMongoDBClient(); const client = getMongoDBClient();
if (!client.connected) { if (!client.connected) {
await client.connect(); await client.connect();
await client.createIndexes();
} }
return client; return client;
} }
@ -59,8 +22,8 @@ export async function connectMongoDB(): Promise<MongoDBClient> {
* Disconnect from MongoDB * Disconnect from MongoDB
*/ */
export async function disconnectMongoDB(): Promise<void> { export async function disconnectMongoDB(): Promise<void> {
if (defaultClient) { const client = getMongoDBClient();
await defaultClient.disconnect(); if (client.connected) {
defaultClient = null; await client.disconnect();
} }
} }

233
libs/mongodb-client/src/health.ts
View file

@ -1,233 +0,0 @@
import { getLogger } from '@stock-bot/logger';
import type { MongoDBClient } from './client';
import type { MongoDBHealthCheck, MongoDBHealthStatus, MongoDBMetrics } from './types';
/**
* MongoDB Health Monitor
*
* Monitors MongoDB connection health and provides metrics
*/
export class MongoDBHealthMonitor {
private readonly client: MongoDBClient;
private readonly logger: ReturnType<typeof getLogger>;
private healthCheckInterval: NodeJS.Timeout | null = null;
private metrics: MongoDBMetrics;
private lastHealthCheck: MongoDBHealthCheck | null = null;
constructor(client: MongoDBClient) {
this.client = client;
this.logger = getLogger('mongodb-health-monitor');
this.metrics = {
operationsPerSecond: 0,
averageLatency: 0,
errorRate: 0,
connectionPoolUtilization: 0,
documentsProcessed: 0,
};
}
/**
* Start health monitoring
*/
start(intervalMs: number = 30000): void {
if (this.healthCheckInterval) {
this.stop();
}
this.logger.info(`Starting MongoDB health monitoring (interval: ${intervalMs}ms)`);
this.healthCheckInterval = setInterval(async () => {
try {
await this.performHealthCheck();
} catch (error) {
this.logger.error('Health check failed:', error);
}
}, intervalMs);
// Perform initial health check
this.performHealthCheck().catch(error => {
this.logger.error('Initial health check failed:', error);
});
}
/**
* Stop health monitoring
*/
stop(): void {
if (this.healthCheckInterval) {
clearInterval(this.healthCheckInterval);
this.healthCheckInterval = null;
this.logger.info('Stopped MongoDB health monitoring');
}
}
/**
* Get current health status
*/
async getHealth(): Promise<MongoDBHealthCheck> {
if (!this.lastHealthCheck) {
await this.performHealthCheck();
}
return this.lastHealthCheck!;
}
/**
* Get current metrics
*/
getMetrics(): MongoDBMetrics {
return { ...this.metrics };
}
/**
* Perform a health check
*/
private async performHealthCheck(): Promise<void> {
const startTime = Date.now();
const errors: string[] = [];
let status: MongoDBHealthStatus = 'healthy';
try {
if (!this.client.connected) {
errors.push('MongoDB client not connected');
status = 'unhealthy';
} else {
// Test basic connectivity
const mongoClient = this.client.mongoClient;
const db = this.client.database;
if (!mongoClient || !db) {
errors.push('MongoDB client or database not available');
status = 'unhealthy';
} else {
// Ping the database
await db.admin().ping();
// Get server status for metrics
try {
const serverStatus = await db.admin().serverStatus();
this.updateMetricsFromServerStatus(serverStatus);
// Check connection pool status
const poolStats = this.getConnectionPoolStats(serverStatus);
if (poolStats.utilization > 0.9) {
errors.push('High connection pool utilization');
status = status === 'healthy' ? 'degraded' : status;
}
// Check for high latency
const latency = Date.now() - startTime;
if (latency > 1000) {
errors.push(`High latency: ${latency}ms`);
status = status === 'healthy' ? 'degraded' : status;
}
} catch (statusError) {
errors.push(`Failed to get server status: ${(statusError as Error).message}`);
status = 'degraded';
}
}
}
} catch (error) {
errors.push(`Health check failed: ${(error as Error).message}`);
status = 'unhealthy';
}
const latency = Date.now() - startTime;
// Get connection stats
const connectionStats = this.getConnectionStats();
this.lastHealthCheck = {
status,
timestamp: new Date(),
latency,
connections: connectionStats,
errors: errors.length > 0 ? errors : undefined,
};
// Log health status changes
if (status !== 'healthy') {
this.logger.warn(`MongoDB health status: ${status}`, { errors, latency });
} else {
this.logger.debug(`MongoDB health check passed (${latency}ms)`);
}
}
/**
* Update metrics from MongoDB server status
*/
private updateMetricsFromServerStatus(serverStatus: any): void {
try {
const opcounters = serverStatus.opcounters || {};
const connections = serverStatus.connections || {};
const dur = serverStatus.dur || {};
// Calculate operations per second (approximate)
const totalOps = Object.values(opcounters).reduce(
(sum: number, count: any) => sum + (count || 0),
0
);
this.metrics.operationsPerSecond = totalOps;
// Connection pool utilization
if (connections.current && connections.available) {
const total = connections.current + connections.available;
this.metrics.connectionPoolUtilization = connections.current / total;
}
// Average latency (from durability stats if available)
if (dur.timeMS) {
this.metrics.averageLatency = dur.timeMS.dt || 0;
}
} catch (error) {
this.logger.debug('Error parsing server status for metrics:', error as any);
}
}
/**
* Get connection pool statistics
*/
private getConnectionPoolStats(serverStatus: any): {
utilization: number;
active: number;
available: number;
} {
const connections = serverStatus.connections || {};
const active = connections.current || 0;
const available = connections.available || 0;
const total = active + available;
return {
utilization: total > 0 ? active / total : 0,
active,
available,
};
}
/**
* Get connection statistics
*/
private getConnectionStats(): { active: number; available: number; total: number } {
// This would ideally come from the MongoDB driver's connection pool
// For now, we'll return estimated values
return {
active: 1,
available: 9,
total: 10,
};
}
/**
* Update error rate metric
*/
updateErrorRate(errorCount: number, totalOperations: number): void {
this.metrics.errorRate = totalOperations > 0 ? errorCount / totalOperations : 0;
}
/**
* Update documents processed metric
*/
updateDocumentsProcessed(count: number): void {
this.metrics.documentsProcessed += count;
}
}

40
libs/mongodb-client/src/index.ts
View file

@ -1,40 +1,22 @@
/** /**
* MongoDB Client Library for Stock Bot * Simplified MongoDB Client Library for Stock Bot Data Service
* *
* Provides type-safe MongoDB access for document storage, sentiment data, * Provides a singleton MongoDB client focused on batch upsert operations
* and raw content processing. * for high-performance data ingestion.
*/ */
export { MongoDBClient } from './client'; export { MongoDBClient } from './client';
export { MongoDBHealthMonitor } from './health';
export { MongoDBTransactionManager } from './transactions';
export { MongoDBAggregationBuilder } from './aggregation';
// Types // Types
export type { export type {
MongoDBClientConfig,
MongoDBConnectionOptions,
MongoDBHealthStatus,
MongoDBMetrics,
CollectionNames,
DocumentBase,
SentimentData,
RawDocument,
NewsArticle,
SecFiling,
EarningsTranscript,
AnalystReport, AnalystReport,
DocumentBase,
EarningsTranscript,
NewsArticle,
RawDocument,
SecFiling,
SentimentData,
} from './types'; } from './types';
// Schemas // Factory functions
export { export { connectMongoDB, disconnectMongoDB, getMongoDBClient } from './factory';
sentimentDataSchema,
rawDocumentSchema,
newsArticleSchema,
secFilingSchema,
earningsTranscriptSchema,
analystReportSchema,
} from './schemas';
// Utils
export { createMongoDBClient } from './factory';

146
libs/mongodb-client/src/schemas.ts
View file

@ -1,146 +0,0 @@
import * as yup from 'yup';
/**
* Yup Schemas for MongoDB Document Validation
*/
// Base schema for all documents
export const documentBaseSchema = yup.object({
_id: yup.mixed().optional(),
created_at: yup.date().required(),
updated_at: yup.date().required(),
source: yup.string().required(),
metadata: yup.object().optional(),
});
// Sentiment Data Schema
export const sentimentDataSchema = documentBaseSchema.shape({
symbol: yup.string().min(1).max(10).required(),
sentiment_score: yup.number().min(-1).max(1).required(),
sentiment_label: yup.string().oneOf(['positive', 'negative', 'neutral']).required(),
confidence: yup.number().min(0).max(1).required(),
text: yup.string().min(1).required(),
source_type: yup.string().oneOf(['reddit', 'twitter', 'news', 'forums']).required(),
source_id: yup.string().required(),
timestamp: yup.date().required(),
processed_at: yup.date().required(),
language: yup.string().default('en'),
keywords: yup.array(yup.string()).required(),
entities: yup
.array(
yup.object({
name: yup.string().required(),
type: yup.string().required(),
confidence: yup.number().min(0).max(1).required(),
})
)
.required(),
});
// Raw Document Schema
export const rawDocumentSchema = documentBaseSchema.shape({
document_type: yup.string().oneOf(['html', 'pdf', 'text', 'json', 'xml']).required(),
content: yup.string().required(),
content_hash: yup.string().required(),
url: yup.string().url().optional(),
title: yup.string().optional(),
author: yup.string().optional(),
published_date: yup.date().optional(),
extracted_text: yup.string().optional(),
processing_status: yup.string().oneOf(['pending', 'processed', 'failed']).required(),
size_bytes: yup.number().positive().required(),
language: yup.string().optional(),
});
// News Article Schema
export const newsArticleSchema = documentBaseSchema.shape({
headline: yup.string().min(1).required(),
content: yup.string().min(1).required(),
summary: yup.string().optional(),
author: yup.string().required(),
publication: yup.string().required(),
published_date: yup.date().required(),
url: yup.string().url().required(),
symbols: yup.array(yup.string()).required(),
categories: yup.array(yup.string()).required(),
sentiment_score: yup.number().min(-1).max(1).optional(),
relevance_score: yup.number().min(0).max(1).optional(),
image_url: yup.string().url().optional(),
tags: yup.array(yup.string()).required(),
});
// SEC Filing Schema
export const secFilingSchema = documentBaseSchema.shape({
cik: yup.string().required(),
accession_number: yup.string().required(),
filing_type: yup.string().required(),
company_name: yup.string().required(),
symbols: yup.array(yup.string()).required(),
filing_date: yup.date().required(),
period_end_date: yup.date().required(),
url: yup.string().url().required(),
content: yup.string().required(),
extracted_data: yup.object().optional(),
financial_statements: yup
.array(
yup.object({
statement_type: yup.string().required(),
data: yup.object().required(),
})
)
.optional(),
processing_status: yup.string().oneOf(['pending', 'processed', 'failed']).required(),
});
// Earnings Transcript Schema
export const earningsTranscriptSchema = documentBaseSchema.shape({
symbol: yup.string().min(1).max(10).required(),
company_name: yup.string().required(),
quarter: yup.string().required(),
year: yup.number().min(2000).max(3000).required(),
call_date: yup.date().required(),
transcript: yup.string().required(),
participants: yup
.array(
yup.object({
name: yup.string().required(),
title: yup.string().required(),
type: yup.string().oneOf(['executive', 'analyst']).required(),
})
)
.required(),
key_topics: yup.array(yup.string()).required(),
sentiment_analysis: yup
.object({
overall_sentiment: yup.number().min(-1).max(1).required(),
topic_sentiments: yup.object().required(),
})
.optional(),
financial_highlights: yup.object().optional(),
});
// Analyst Report Schema
export const analystReportSchema = documentBaseSchema.shape({
symbol: yup.string().min(1).max(10).required(),
analyst_firm: yup.string().required(),
analyst_name: yup.string().required(),
report_title: yup.string().required(),
report_date: yup.date().required(),
rating: yup.string().oneOf(['buy', 'hold', 'sell', 'strong_buy', 'strong_sell']).required(),
price_target: yup.number().positive().optional(),
previous_rating: yup.string().optional(),
content: yup.string().required(),
summary: yup.string().required(),
key_points: yup.array(yup.string()).required(),
financial_projections: yup.object().optional(),
});
// Schema mapping for collections
export const schemaMap = {
sentiment_data: sentimentDataSchema,
raw_documents: rawDocumentSchema,
news_articles: newsArticleSchema,
sec_filings: secFilingSchema,
earnings_transcripts: earningsTranscriptSchema,
analyst_reports: analystReportSchema,
} as const;

238
libs/mongodb-client/src/transactions.ts
View file

@ -1,238 +0,0 @@
import type { OptionalUnlessRequiredId, WithId } from 'mongodb';
import { getLogger } from '@stock-bot/logger';
import type { MongoDBClient } from './client';
import type { CollectionNames, DocumentBase } from './types';
/**
* MongoDB Transaction Manager
*
* Provides transaction support for multi-document operations
*/
export class MongoDBTransactionManager {
private readonly client: MongoDBClient;
private readonly logger: ReturnType<typeof getLogger>;
constructor(client: MongoDBClient) {
this.client = client;
this.logger = getLogger('mongodb-transaction-manager');
}
/**
* Execute operations within a transaction
*/
async withTransaction<T>(
operations: (session: any) => Promise<T>,
options?: {
readPreference?: string;
readConcern?: string;
writeConcern?: any;
maxCommitTimeMS?: number;
}
): Promise<T> {
const mongoClient = this.client.mongoClient;
if (!mongoClient) {
throw new Error('MongoDB client not connected');
}
const session = mongoClient.startSession();
try {
this.logger.debug('Starting MongoDB transaction');
const result = await session.withTransaction(
async () => {
return await operations(session);
},
{
readPreference: options?.readPreference as any,
readConcern: { level: options?.readConcern || 'majority' } as any,
writeConcern: options?.writeConcern || { w: 'majority' },
maxCommitTimeMS: options?.maxCommitTimeMS || 10000,
}
);
this.logger.debug('MongoDB transaction completed successfully');
return result;
} catch (error) {
this.logger.error('MongoDB transaction failed:', error);
throw error;
} finally {
await session.endSession();
}
}
/**
* Batch insert documents across collections within a transaction
*/
async batchInsert(
operations: Array<{
collection: CollectionNames;
documents: DocumentBase[];
}>,
options?: { ordered?: boolean; bypassDocumentValidation?: boolean }
): Promise<void> {
await this.withTransaction(async session => {
for (const operation of operations) {
const collection = this.client.getCollection(operation.collection);
// Add timestamps to all documents
const now = new Date();
const documentsWithTimestamps = operation.documents.map(doc => ({
...doc,
created_at: doc.created_at || now,
updated_at: now,
}));
await collection.insertMany(documentsWithTimestamps, {
session,
ordered: options?.ordered ?? true,
bypassDocumentValidation: options?.bypassDocumentValidation ?? false,
});
this.logger.debug(
`Inserted ${documentsWithTimestamps.length} documents into ${operation.collection}`
);
}
});
}
/**
* Batch update documents across collections within a transaction
*/
async batchUpdate(
operations: Array<{
collection: CollectionNames;
filter: any;
update: any;
options?: any;
}>
): Promise<void> {
await this.withTransaction(async session => {
const results = [];
for (const operation of operations) {
const collection = this.client.getCollection(operation.collection);
// Add updated timestamp
const updateWithTimestamp = {
...operation.update,
$set: {
...operation.update.$set,
updated_at: new Date(),
},
};
const result = await collection.updateMany(operation.filter, updateWithTimestamp, {
session,
...operation.options,
});
results.push(result);
this.logger.debug(`Updated ${result.modifiedCount} documents in ${operation.collection}`);
}
return results;
});
}
/**
* Move documents between collections within a transaction
*/
async moveDocuments<T extends DocumentBase>(
fromCollection: CollectionNames,
toCollection: CollectionNames,
filter: any,
transform?: (doc: T) => T
): Promise<number> {
return await this.withTransaction(async session => {
const sourceCollection = this.client.getCollection<T>(fromCollection);
const targetCollection = this.client.getCollection<T>(toCollection);
// Find documents to move
const documents = await sourceCollection.find(filter, { session }).toArray();
if (documents.length === 0) {
return 0;
} // Transform documents if needed
const documentsToInsert = transform
? documents.map((doc: WithId<T>) => transform(doc as T))
: documents;
// Add updated timestamp
const now = new Date();
documentsToInsert.forEach(doc => {
doc.updated_at = now;
}); // Insert into target collection
await targetCollection.insertMany(documentsToInsert as OptionalUnlessRequiredId<T>[], {
session,
});
// Remove from source collection
const deleteResult = await sourceCollection.deleteMany(filter, { session });
this.logger.info(
`Moved ${documents.length} documents from ${fromCollection} to ${toCollection}`
);
return deleteResult.deletedCount || 0;
});
}
/**
* Archive old documents within a transaction
*/
async archiveDocuments(
sourceCollection: CollectionNames,
archiveCollection: CollectionNames,
cutoffDate: Date,
batchSize: number = 1000
): Promise<number> {
let totalArchived = 0;
while (true) {
const batchArchived = await this.withTransaction(async session => {
const collection = this.client.getCollection(sourceCollection);
const archiveCol = this.client.getCollection(archiveCollection);
// Find old documents
const documents = await collection
.find({ created_at: { $lt: cutoffDate } }, { limit: batchSize, session })
.toArray();
if (documents.length === 0) {
return 0;
}
// Add archive metadata
const now = new Date();
const documentsToArchive = documents.map(doc => ({
...doc,
archived_at: now,
archived_from: sourceCollection,
}));
// Insert into archive collection
await archiveCol.insertMany(documentsToArchive, { session });
// Remove from source collection
const ids = documents.map(doc => doc._id);
const deleteResult = await collection.deleteMany({ _id: { $in: ids } }, { session });
return deleteResult.deletedCount || 0;
});
totalArchived += batchArchived;
if (batchArchived === 0) {
break;
}
this.logger.debug(`Archived batch of ${batchArchived} documents`);
}
this.logger.info(
`Archived ${totalArchived} documents from ${sourceCollection} to ${archiveCollection}`
);
return totalArchived;
}
}

15
libs/mongodb-client/src/types.ts
View file

@ -1,5 +1,4 @@
import type { ObjectId } from 'mongodb'; import type { ObjectId } from 'mongodb';
import * as yup from 'yup';
/** /**
* MongoDB Client Configuration * MongoDB Client Configuration
@ -69,20 +68,6 @@ export interface MongoDBMetrics {
documentsProcessed: number; documentsProcessed: number;
} }
/**
* Collection Names
*/
export type CollectionNames =
| 'sentiment_data'
| 'raw_documents'
| 'news_articles'
| 'sec_filings'
| 'earnings_transcripts'
| 'analyst_reports'
| 'social_media_posts'
| 'market_events'
| 'economic_indicators';
/** /**
* Base Document Interface * Base Document Interface
*/ */

0
scripts/populate-ib-exchanges.ts Normal file
View file

166
test-browser-simple.ts
View file

@ -1,166 +0,0 @@
/**
* Simple Browser and Network Monitoring Test
*/
import { Browser } from '@stock-bot/browser';
async function testBasicBrowser() {
console.log('🚀 Testing basic browser functionality...');
try {
// Initialize browser
await Browser.initialize({
headless: true,
timeout: 15000,
blockResources: false,
enableNetworkLogging: true,
});
console.log('✅ Browser initialized');
// Test 1: Simple page without proxy
console.log('📄 Testing simple page without proxy...');
const { page, contextId } = await Browser.createPageWithProxy(
'https://httpbin.org/json'
);
let capturedData = null;
let eventCount = 0;
page.onNetworkEvent(event => {
eventCount++;
console.log(`📡 Event ${eventCount}: ${event.type} - ${event.method} ${event.url}`);
if (event.type === 'response' && event.url.includes('httpbin.org/json')) {
console.log(` 📊 Status: ${event.status}`);
if (event.responseData) {
capturedData = event.responseData;
console.log(` 📝 Response: ${event.responseData}`);
}
}
});
await page.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 2000));
console.log(`✅ Test completed. Events captured: ${eventCount}`);
if (capturedData) {
console.log('✅ Successfully captured response data');
}
await Browser.closeContext(contextId);
return true;
} catch (error) {
console.error('❌ Basic test failed:', error);
return false;
} finally {
await Browser.close();
}
}
async function testProxyConnection() {
console.log('\n🔄 Testing proxy connection...');
try {
await Browser.initialize({
headless: true,
timeout: 10000,
blockResources: false,
});
// Test different proxy formats
const proxyConfigs = [
null, // No proxy
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80',
];
for (const proxy of proxyConfigs) {
console.log(`\n🌐 Testing with proxy: ${proxy || 'No proxy'}`);
try {
const { page, contextId } = await Browser.createPageWithProxy(
'https://httpbin.org/ip',
proxy
);
page.onNetworkEvent(event => {
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
console.log(` 📍 IP Response: ${event.responseData}`);
}
});
await page.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 1500));
await Browser.closeContext(contextId);
console.log(' ✅ Success');
} catch (error) {
console.log(` ❌ Failed: ${error.message}`);
}
}
} catch (error) {
console.error('❌ Proxy test setup failed:', error);
} finally {
await Browser.close();
}
}
async function testIBWithWorkaround() {
console.log('\n🏦 Testing IB endpoint with workaround...');
try {
await Browser.initialize({
headless: true,
timeout: 20000,
blockResources: true, // Block resources for performance
});
// Try without proxy first
console.log('🌐 Attempting IB without proxy...');
try {
const { page, contextId } = await Browser.createPageWithProxy(
'https://www.interactivebrokers.com'
);
let responseCount = 0;
page.onNetworkEvent(event => {
if (event.type === 'response') {
responseCount++;
console.log(` 📥 Response ${responseCount}: ${event.status} ${event.url}`);
}
});
await page.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 3000));
console.log(`✅ IB main page loaded. Responses: ${responseCount}`);
await Browser.closeContext(contextId);
} catch (error) {
console.log(`❌ IB without proxy failed: ${error.message}`);
}
} catch (error) {
console.error('❌ IB test failed:', error);
} finally {
await Browser.close();
}
}
// Run tests
async function runAllTests() {
console.log('🧪 Starting Browser Network Monitoring Tests\n');
const basicResult = await testBasicBrowser();
await testProxyConnection();
await testIBWithWorkaround();
console.log(`\n🏁 Basic functionality: ${basicResult ? '✅ PASS' : '❌ FAIL'}`);
console.log('✅ All tests completed!');
}
if (import.meta.main) {
runAllTests().catch(console.error);
}
export { testBasicBrowser, testProxyConnection, testIBWithWorkaround };

62
test-browser.ts
View file

@ -1,62 +0,0 @@
import { Browser, BrowserTabManager } from './libs/browser/src';
async function testSimplifiedBrowser() {
console.log('Testing simplified browser library...');
try {
console.log('Initializing browser...');
await Browser.initialize({
headless: true,
blockResources: true,
timeout: 10000,
});
// Test single page with proxy support
console.log('Testing page creation...');
const { page, contextId } = await Browser.createPageWithProxy(
'https://httpbin.org/json',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
const content = await Browser.evaluate(page, () => document.body.textContent);
console.log('Page content:', content?.substring(0, 100) + '...');
// Test tab manager (no longer needs browser instance)
console.log('Testing tab manager...');
const tabManager = new BrowserTabManager();
// Test multiple URL scraping with different proxies
const urlProxyPairs = [
{ url: 'https://httpbin.org/uuid', proxy: '' }, // No proxy
{ url: 'https://httpbin.org/ip', proxy: '' }, // No proxy
];
const results = await tabManager.scrapeUrlsWithProxies(
urlProxyPairs,
async page => {
const text = await page.textContent('body');
return { content: text?.substring(0, 50) };
},
{ concurrency: 2 }
);
console.log('Scraping results:');
results.forEach((result, index) => {
console.log(` ${index + 1}. ${result.url}: ${result.success ? 'SUCCESS' : 'FAILED'}`);
if (result.data) {
console.log(` Data: ${result.data.content}...`);
}
});
// Clean up
await page.close();
await Browser.closeContext(contextId);
await Browser.close();
console.log('✅ Simplified browser test completed successfully!');
} catch (error) {
console.error('❌ Browser test failed:', error);
}
}
testSimplifiedBrowser();

139
test-ib-no-proxy.ts
View file

@ -1,139 +0,0 @@
import { Browser } from '@stock-bot/browser';
async function testWithoutProxy() {
console.log('🔬 Testing WITHOUT proxy...');
try {
await Browser.initialize({ headless: true, timeout: 15000, blockResources: false });
console.log('✅ Browser initialized');
const { page, contextId } = await Browser.createPageWithProxy(
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/'
// No proxy parameter
);
console.log('✅ Page created without proxy');
let eventCount = 0;
let summaryData: SummaryResponse | null = null;
page.onNetworkEvent(event => {
eventCount++;
// Capture the summary API response
if (event.url.includes('/webrest/search/product-types/summary')) {
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
if (event.type === 'response' && event.responseData) {
console.log(`📊 Summary API Response Data: ${event.responseData}`);
try {
summaryData = JSON.parse(event.responseData) as any;
const totalCount = summaryData[0].totalCount;
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
} catch (e) {
console.log('📊 Raw Summary Response:', event.responseData);
}
}
}
// Uncomment to see all network events
// console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
});
console.log('⏳ Waiting for page load...');
await page.waitForLoadState('domcontentloaded', { timeout: 15000 });
console.log('✅ Page loaded');
// Complete interaction flow
try {
console.log('🔍 Looking for Products tab...');
await page.waitForTimeout(3000);
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
await productsTab.waitFor({ timeout: 10000 });
console.log('✅ Found Products tab');
console.log('🖱️ Clicking Products tab...');
await productsTab.click();
console.log('✅ Products tab clicked');
await page.waitForTimeout(2000);
console.log('🔍 Looking for Asset Classes accordion...');
const assetClassesAccordion = page.locator(
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
);
await assetClassesAccordion.waitFor({ timeout: 10000 });
console.log('✅ Found Asset Classes accordion');
console.log('🖱️ Clicking Asset Classes accordion...');
await assetClassesAccordion.click();
console.log('✅ Asset Classes accordion clicked');
await page.waitForTimeout(2000);
console.log('🔍 Looking for Stocks checkbox...');
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
await stocksSpan.waitFor({ timeout: 10000 });
console.log('✅ Found Stocks span');
const parentContainer = stocksSpan.locator('..');
const checkbox = parentContainer.locator('input[type="checkbox"]');
if ((await checkbox.count()) > 0) {
console.log('📋 Clicking Stocks checkbox...');
await checkbox.first().check();
console.log('✅ Stocks checkbox checked');
} else {
console.log('⚠️ Could not find checkbox near Stocks text');
}
await page.waitForTimeout(1000);
console.log('🔍 Looking for Apply button...');
const applyButton = page.locator(
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
);
if ((await applyButton.count()) > 0) {
console.log('🎯 Clicking Apply button...');
await applyButton.first().click();
console.log('✅ Apply button clicked');
await page.waitForTimeout(3000);
} else {
console.log('⚠️ Could not find Apply button');
}
} catch (interactionError) {
const errorMessage =
interactionError instanceof Error ? interactionError.message : String(interactionError);
console.error('❌ Page interaction failed:', errorMessage);
}
await new Promise(resolve => setTimeout(resolve, 2000));
console.log(`📊 Total events captured: ${eventCount}`);
// Show final results
if (summaryData) {
console.log('✅ SUCCESS: Captured summary data!');
console.log(`🔢 Final total count: ${summaryData?.data?.totalCount || 'Unknown'}`);
console.log(`📋 Data keys: ${Object.keys(summaryData).join(', ')}`);
} else {
console.log('❌ No summary data captured');
}
await Browser.closeContext(contextId);
await Browser.close();
console.log('✅ Test completed successfully');
return true;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error('❌ Error:', errorMessage);
await Browser.close();
return false;
}
}
testWithoutProxy().then(success => {
console.log(`🏁 Final result: ${success ? 'SUCCESS' : 'FAILED'}`);
});

160
test-ib-working.ts
View file

@ -1,160 +0,0 @@
/**
* Working Interactive Brokers test with verified network monitoring
*/
import { Browser } from '@stock-bot/browser';
async function testIBWithWorking() {
console.log('🏦 Testing IB with working network monitoring and fixed proxy auth...');
try {
await Browser.initialize({
headless: true,
timeout: 20000,
blockResources: false, // Don't block resources initially
});
// Test 1: Try a simple proxy detection service first
console.log('🌐 Testing proxy connectivity...');
const { page: proxyPage, contextId: proxyCtx } = await Browser.createPageWithProxy(
'https://httpbin.org/ip',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
let proxyEvents = 0;
let myIP = null;
proxyPage.onNetworkEvent(event => {
proxyEvents++;
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
try {
const data = JSON.parse(event.responseData);
myIP = data.origin;
console.log(` 📍 Proxy IP: ${myIP}`);
} catch (e) {
console.log(` 📊 Raw response: ${event.responseData}`);
}
}
});
await proxyPage.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 2000));
await Browser.closeContext(proxyCtx);
console.log(`📊 Proxy test events: ${proxyEvents}`);
// Test 2: Try IB API endpoint with fixed proxy auth
console.log('🎯 Testing IB API endpoint...');
const { page: apiPage, contextId: apiCtx } = await Browser.createPageWithProxy(
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
let apiEvents = 0;
let summaryData = null;
apiPage.onNetworkEvent(event => {
apiEvents++;
console.log(` 📡 API Event: ${event.type} ${event.method} ${event.url}`);
if (event.type === 'response' && event.url.includes('summary')) {
console.log(` 🎯 Found summary response! Status: ${event.status}`);
if (event.responseData) {
summaryData = event.responseData;
try {
const data = JSON.parse(event.responseData);
console.log(` 📊 Summary data: ${JSON.stringify(data, null, 2)}`);
} catch (e) {
console.log(` 📊 Raw summary: ${event.responseData.substring(0, 200)}...`);
}
}
}
});
await apiPage.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 3000));
await Browser.closeContext(apiCtx);
return {
proxyEvents,
apiEvents,
summaryData,
proxyIP: myIP,
success: apiEvents > 0 || summaryData !== null,
};
} catch (error) {
console.error('❌ IB test failed:', error);
return {
proxyEvents: 0,
apiEvents: 0,
summaryData: null,
proxyIP: null,
success: false,
error: error.message,
};
} finally {
await Browser.close();
}
}
async function testWithProxyFallback() {
console.log('\n🔄 Testing with proxy fallback strategy...');
const proxiesToTest = [
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', // Your proxy
];
for (const proxy of proxiesToTest) {
console.log(`\n🌐 Testing with: ${proxy || 'No proxy'}`);
try {
await Browser.initialize({
headless: true,
timeout: 15000,
blockResources: false,
});
const { page, contextId } = await Browser.createPageWithProxy(
'https://httpbin.org/ip',
proxy
);
let ipResponse = null;
page.onNetworkEvent(event => {
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
ipResponse = event.responseData;
console.log(` 📍 IP: ${JSON.parse(event.responseData).origin}`);
}
});
await page.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 1000));
await Browser.closeContext(contextId);
} catch (error) {
console.log(` ❌ Failed: ${error.message}`);
} finally {
await Browser.close();
}
}
}
async function runIBTests() {
console.log('🚀 Interactive Brokers Network Monitoring Tests with Fixed Proxy Auth\n');
const result = await testIBWithWorking();
await testWithProxyFallback();
console.log('\n🏁 Final Results:');
console.log(` 🌐 Proxy events: ${result.proxyEvents || 0}`);
console.log(` 📍 Proxy IP: ${result.proxyIP || 'Not captured'}`);
console.log(` 🎯 API events: ${result.apiEvents || 0}`);
console.log(` 📊 Summary data: ${result.summaryData ? 'Captured' : 'Not captured'}`);
console.log(` ✅ Overall success: ${result.success}`);
if (result.error) {
console.log(` ❌ Error: ${result.error}`);
}
}
if (import.meta.main) {
runIBTests().catch(console.error);
}
export { testIBWithWorking, testWithProxyFallback };

194
test-ib.ts
View file

@ -1,194 +0,0 @@
/**
* Test Interactive Brokers functionality with network monitoring
*/
import { Browser } from '@stock-bot/browser';
import { getRandomProxyURL } from '@stock-bot/proxy';
async function testIBSymbolSummary() {
console.log('🚀 Testing Interactive Brokers Symbol Summary with Network Monitoring...');
try {
// Initialize browser
await Browser.initialize({
headless: true,
timeout: 30000,
blockResources: true,
enableNetworkLogging: true,
});
console.log('✅ Browser initialized');
// Get a random proxy
// Create page with proxy
const { page, contextId } = await Browser.createPageWithProxy(
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
console.log('📄 Page created with proxy');
// Set up network monitoring
let summaryResponse: any = null;
let requestCount = 0;
let responseCount = 0;
page.onNetworkEvent(event => {
console.log(`📡 Network Event: ${event.type} - ${event.method} ${event.url}`);
if (event.type === 'request') {
requestCount++;
console.log(` 📤 Request #${requestCount}: ${event.method} ${event.url}`);
// Log request data for POST requests
if (event.requestData) {
console.log(` 📝 Request Data: ${event.requestData.substring(0, 200)}...`);
}
}
if (event.type === 'response') {
responseCount++;
console.log(` 📥 Response #${responseCount}: ${event.status} ${event.url}`);
// Capture the summary response
if (event.url.includes('summary')) {
console.log(` 🎯 Found summary response!`);
summaryResponse = event.responseData;
if (event.responseData) {
try {
const data = JSON.parse(event.responseData);
console.log(` 📊 Summary Data: ${JSON.stringify(data, null, 2)}`);
} catch (e) {
console.log(` 📊 Raw Response: ${event.responseData.substring(0, 500)}...`);
}
}
}
}
if (event.type === 'failed') {
console.log(` ❌ Failed Request: ${event.url}`);
}
});
console.log('🔍 Network monitoring set up, waiting for page to load...');
// Wait for page to load and capture network activity
await page.waitForLoadState('domcontentloaded');
console.log('✅ Page loaded');
// Wait a bit more for any additional network requests
await new Promise(resolve => setTimeout(resolve, 3000));
console.log(`📊 Network Summary:`);
console.log(` 📤 Total Requests: ${requestCount}`);
console.log(` 📥 Total Responses: ${responseCount}`);
if (summaryResponse) {
console.log('✅ Successfully captured summary response');
try {
const parsed = JSON.parse(summaryResponse);
console.log(`🔢 Total symbols found: ${parsed?.data?.totalCount || 'Unknown'}`);
return parsed?.data?.totalCount || 0;
} catch (e) {
console.log('⚠️ Could not parse response as JSON');
return 1; // Indicate success but unknown count
}
} else {
console.log('❌ No summary response captured');
return 0;
}
} catch (error) {
console.error('❌ Test failed:', error);
// Log more details about the error
if (error instanceof Error) {
console.error('Error details:', {
message: error.message,
stack: error.stack,
name: error.name
});
}
return -1;
} finally {
try {
await Browser.close();
console.log('🔒 Browser closed');
} catch (closeError) {
console.error('Error closing browser:', closeError);
}
}
}
async function testWithDifferentProxy() {
console.log('\n🔄 Testing with different proxy configuration...');
try {
await Browser.initialize({
headless: true,
timeout: 15000,
blockResources: false, // Don't block resources for this test
});
// Test without proxy first
console.log('🌐 Testing without proxy...');
const { page: pageNoProxy, contextId: contextNoProxy } = await Browser.createPageWithProxy(
'https://httpbin.org/ip'
);
pageNoProxy.onNetworkEvent(event => {
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
console.log('📍 No proxy IP response:', event.responseData);
}
});
await pageNoProxy.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 1000));
await Browser.closeContext(contextNoProxy);
// Test with proxy
console.log('🌐 Testing with proxy...');
const { page: pageWithProxy, contextId: contextWithProxy } = await Browser.createPageWithProxy(
'https://httpbin.org/ip',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
pageWithProxy.onNetworkEvent(event => {
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
console.log('🔄 Proxy IP response:', event.responseData);
}
});
await pageWithProxy.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 1000));
await Browser.closeContext(contextWithProxy);
} catch (error) {
console.error('❌ Proxy test failed:', error);
} finally {
await Browser.close();
}
}
// Run the tests
async function runTests() {
console.log('🧪 Starting IB Network Monitoring Tests\n');
// Test 1: Main IB functionality
const result = await testIBSymbolSummary();
console.log(`\n🏁 Test Result: ${result}`);
// Test 2: Proxy verification
await testWithDifferentProxy();
console.log('\n✅ All tests completed!');
}
// Run if this file is executed directly
if (import.meta.main) {
runTests().catch(console.error);
}
export { testIBSymbolSummary, testWithDifferentProxy };

0
test-large-scale-performance.ts Normal file
View file

0
test-mongodb-batch.ts Normal file
View file

0
test-mongodb-simplified.ts Normal file
View file

135
test-network-debug.ts
View file

@ -1,135 +0,0 @@
/**
* Debug network monitoring setup
*/
import { Browser } from '@stock-bot/browser';
async function debugNetworkSetup() {
console.log('🐛 Debugging Network Monitoring Setup...');
try {
await Browser.initialize({
headless: true,
timeout: 10000,
blockResources: false, // Ensure we don't block requests
});
// Create page but don't navigate yet
const { page, contextId } = await Browser.createPageWithProxy(
'',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
let eventCount = 0;
console.log('📡 Setting up network event listener...');
page.onNetworkEvent(event => {
eventCount++;
console.log(`🔔 Event ${eventCount}: ${event.type} ${event.method} ${event.url}`);
console.log(` Headers: ${Object.keys(event.headers || {}).length} headers`);
if (event.responseData) {
console.log(` Data: ${event.responseData.substring(0, 100)}...`);
}
});
console.log('🌐 Navigating to httpbin.org/headers...');
await page.goto('https://httpbin.org/headers');
console.log('⏳ Waiting for page load...');
await page.waitForLoadState('domcontentloaded');
console.log('⏳ Waiting additional time for network events...');
await new Promise(resolve => setTimeout(resolve, 3000));
console.log(`📊 Total events captured: ${eventCount}`);
// Try to evaluate page content to see if it loaded
const title = await page.title();
console.log(`📄 Page title: "${title}"`);
const bodyText = await page.locator('body').textContent();
if (bodyText) {
console.log(`📝 Page content (first 200 chars): ${bodyText.substring(0, 200)}...`);
}
await Browser.closeContext(contextId);
return eventCount > 0;
} catch (error) {
console.error('❌ Debug test failed:', error);
return false;
} finally {
await Browser.close();
}
}
async function testManualNetworkCall() {
console.log('\n🔧 Testing with manual fetch call...');
try {
await Browser.initialize({
headless: true,
timeout: 10000,
blockResources: false,
});
const { page, contextId } = await Browser.createPageWithProxy(
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
let eventCount = 0;
page.onNetworkEvent(event => {
eventCount++;
console.log(`📡 Manual test event ${eventCount}: ${event.type} ${event.method} ${event.url}`);
if (event.responseData && event.url.includes('httpbin')) {
console.log(` 📊 Response: ${event.responseData}`);
}
});
// Navigate to a simple page first
await page.goto('data:text/html,<html><body><h1>Test Page</h1></body></html>');
await page.waitForLoadState('domcontentloaded');
console.log('🚀 Making manual fetch call...');
// Make a fetch request from the page context
const result = await page.evaluate(async () => {
try {
const response = await fetch('https://httpbin.org/json');
const data = await response.json();
return { success: true, data };
} catch (error) {
return { success: false, error: error.message };
}
});
console.log('📋 Fetch result:', result);
await new Promise(resolve => setTimeout(resolve, 2000));
console.log(`📊 Events from manual fetch: ${eventCount}`);
await Browser.closeContext(contextId);
return eventCount > 0;
} catch (error) {
console.error('❌ Manual test failed:', error);
return false;
} finally {
await Browser.close();
}
}
async function runDebugTests() {
console.log('🚀 Network Monitoring Debug Tests\n');
const setupResult = await debugNetworkSetup();
const manualResult = await testManualNetworkCall();
console.log(`\n🏁 Results:`);
console.log(` 🔧 Setup test: ${setupResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`);
console.log(` 📡 Manual test: ${manualResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`);
}
if (import.meta.main) {
runDebugTests().catch(console.error);
}
export { debugNetworkSetup, testManualNetworkCall };

137
test-network-monitoring.ts
View file

@ -1,137 +0,0 @@
/**
* Simple test to verify network monitoring is working
*/
import { Browser } from '@stock-bot/browser';
async function testNetworkMonitoring() {
console.log('🧪 Testing Network Monitoring with httpbin.org...');
try {
await Browser.initialize({
headless: true,
timeout: 15000,
blockResources: false, // Don't block resources so we can see requests
});
console.log('✅ Browser initialized');
// Test with a simple API that returns JSON
const { page, contextId } = await Browser.createPageWithProxy(
'https://httpbin.org/json'
);
let capturedRequests = 0;
let capturedResponses = 0;
let jsonResponse = null;
page.onNetworkEvent(event => {
console.log(`📡 ${event.type.toUpperCase()}: ${event.method} ${event.url}`);
if (event.type === 'request') {
capturedRequests++;
}
if (event.type === 'response') {
capturedResponses++;
console.log(` Status: ${event.status}`);
if (event.url.includes('httpbin.org/json') && event.responseData) {
jsonResponse = event.responseData;
console.log(` 📊 JSON Response: ${event.responseData}`);
}
}
});
await page.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 2000));
console.log(`\n📊 Summary:`);
console.log(` 📤 Requests captured: ${capturedRequests}`);
console.log(` 📥 Responses captured: ${capturedResponses}`);
console.log(` 📝 JSON data captured: ${jsonResponse ? 'Yes' : 'No'}`);
await Browser.closeContext(contextId);
return true;
} catch (error) {
console.error('❌ Test failed:', error);
return false;
} finally {
await Browser.close();
}
}
async function testWithProxy() {
console.log('\n🌐 Testing with proxy to see IP change...');
try {
await Browser.initialize({
headless: true,
timeout: 10000,
blockResources: false,
});
// Test IP without proxy
console.log('📍 Getting IP without proxy...');
const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy(
'https://httpbin.org/ip'
);
let ipWithoutProxy = null;
page1.onNetworkEvent(event => {
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
ipWithoutProxy = JSON.parse(event.responseData).origin;
console.log(` 🔹 Your IP: ${ipWithoutProxy}`);
}
});
await page1.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 1000));
await Browser.closeContext(ctx1);
// Test IP with proxy
console.log('🔄 Getting IP with proxy...');
const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy(
'https://httpbin.org/ip',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
let ipWithProxy = null;
page2.onNetworkEvent(event => {
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
ipWithProxy = JSON.parse(event.responseData).origin;
console.log(` 🔸 Proxy IP: ${ipWithProxy}`);
}
});
await page2.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 1000));
await Browser.closeContext(ctx2);
if (ipWithoutProxy && ipWithProxy && ipWithoutProxy !== ipWithProxy) {
console.log('✅ Proxy is working - IPs are different!');
} else {
console.log('⚠️ Proxy may not be working - IPs are the same or not captured');
}
} catch (error) {
console.error('❌ Proxy test failed:', error);
} finally {
await Browser.close();
}
}
async function runTests() {
console.log('🚀 Network Monitoring Verification Tests\n');
const basicResult = await testNetworkMonitoring();
await testWithProxy();
console.log(`\n🏁 Network monitoring: ${basicResult ? '✅ WORKING' : '❌ FAILED'}`);
}
if (import.meta.main) {
runTests().catch(console.error);
}
export { testNetworkMonitoring, testWithProxy };

156
test-proxy-auth.ts
View file

@ -1,156 +0,0 @@
/**
* Test Playwright proxy authentication specifically
*/
import { Browser } from '@stock-bot/browser';
async function testPlaywrightProxyAuth() {
console.log('🔐 Testing Playwright Proxy Authentication...');
try {
await Browser.initialize({
headless: true,
timeout: 15000,
blockResources: false,
});
console.log('✅ Browser initialized');
// Test 1: Without proxy
console.log('\n📍 Test 1: Without proxy');
const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy(
'https://httpbin.org/ip'
);
let events1 = 0;
let ip1 = null;
page1.onNetworkEvent(event => {
events1++;
console.log(` 📡 Event: ${event.type} ${event.url}`);
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
ip1 = JSON.parse(event.responseData).origin;
console.log(` 🌐 Your IP: ${ip1}`);
}
});
await page1.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 2000));
await Browser.closeContext(ctx1);
console.log(` Events captured: ${events1}`);
// Test 2: With proxy using new authentication method
console.log('\n🔒 Test 2: With proxy (new auth method)');
const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy(
'https://httpbin.org/ip',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
let events2 = 0;
let ip2 = null;
page2.onNetworkEvent(event => {
events2++;
console.log(` 📡 Event: ${event.type} ${event.url}`);
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
ip2 = JSON.parse(event.responseData).origin;
console.log(` 🔄 Proxy IP: ${ip2}`);
}
});
await page2.waitForLoadState('domcontentloaded');
await new Promise(resolve => setTimeout(resolve, 2000));
await Browser.closeContext(ctx2);
console.log(` Events captured: ${events2}`);
// Results
console.log('\n📊 Results:');
console.log(` 🌐 Direct IP: ${ip1 || 'Not captured'}`);
console.log(` 🔄 Proxy IP: ${ip2 || 'Not captured'}`);
console.log(` 📡 Direct events: ${events1}`);
console.log(` 📡 Proxy events: ${events2}`);
if (ip1 && ip2 && ip1 !== ip2) {
console.log('✅ Proxy authentication is working - different IPs detected!');
return true;
} else if (events1 > 0 || events2 > 0) {
console.log('⚠️ Network monitoring working, but proxy may not be changing IP');
return true;
} else {
console.log('❌ No network events captured');
return false;
}
} catch (error) {
console.error('❌ Test failed:', error);
return false;
} finally {
await Browser.close();
}
}
async function testManualPageEvaluation() {
console.log('\n🧪 Test 3: Manual page evaluation (without network monitoring)');
try {
await Browser.initialize({
headless: true,
timeout: 10000,
blockResources: false,
});
const { page, contextId } = await Browser.createPageWithProxy(
'https://httpbin.org/ip',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
// Try to get the page content directly
const title = await page.title();
console.log(` 📄 Page title: "${title}"`);
// Try to evaluate some JavaScript
const result = await page.evaluate(() => {
return {
url: window.location.href,
userAgent: navigator.userAgent.substring(0, 50),
readyState: document.readyState,
};
});
console.log(` 🔍 Page info:`, result);
// Try to get page content
const bodyText = await page.locator('body').textContent();
if (bodyText) {
console.log(` 📝 Body content (first 200 chars): ${bodyText.substring(0, 200)}...`);
// Check if it looks like an IP response
if (bodyText.includes('origin')) {
console.log(' ✅ Looks like httpbin.org response!');
}
}
await Browser.closeContext(contextId);
return true;
} catch (error) {
console.error(' ❌ Manual evaluation failed:', error);
return false;
} finally {
await Browser.close();
}
}
async function runProxyTests() {
console.log('🚀 Playwright Proxy Authentication Tests\n');
const authResult = await testPlaywrightProxyAuth();
const manualResult = await testManualPageEvaluation();
console.log(`\n🏁 Final Results:`);
console.log(` 🔐 Proxy auth test: ${authResult ? '✅ PASS' : '❌ FAIL'}`);
console.log(` 🧪 Manual eval test: ${manualResult ? '✅ PASS' : '❌ FAIL'}`);
}
if (import.meta.main) {
runProxyTests().catch(console.error);
}
export { testPlaywrightProxyAuth, testManualPageEvaluation };

0
test-query-performance.ts Normal file
View file

0
test-shutdown-simple.ts Normal file
View file

0
test-shutdown.ts Normal file
View file

0
test-signals.ts Normal file
View file

152
test-simple-proxy.ts
View file

@ -1,152 +0,0 @@
import { Browser } from '@stock-bot/browser';
async function simpleProxyTest() {
console.log('🔬 Simple Proxy Test...');
try {
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
console.log('✅ Browser initialized');
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
console.log('✅ Browser initialized');
const { page, contextId } = await Browser.createPageWithProxy(
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/',
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
);
console.log('✅ Page created with proxy');
let summaryData: any = null; // Initialize summaryData to store API response
let eventCount = 0;
page.onNetworkEvent(event => {
if (event.url.includes('/webrest/search/product-types/summary')) {
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
if (event.type === 'response' && event.responseData) {
console.log(`📊 Summary API Response Data: ${event.responseData}`);
try {
summaryData = JSON.parse(event.responseData) as any;
const totalCount = summaryData[0].totalCount;
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
} catch (e) {
console.log('📊 Raw Summary Response:', event.responseData);
}
}
}
eventCount++;
console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
});
console.log('⏳ Waiting for page load...');
await page.waitForLoadState('domcontentloaded', { timeout: 20000 });
console.log('✅ Page loaded');
// RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button
console.log('🔍 Looking for Products tab...');
// Wait for the page to fully load
await page.waitForTimeout(20000);
// First, click on the Products tab
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
await productsTab.waitFor({ timeout: 20000 });
console.log('✅ Found Products tab');
console.log('🖱️ Clicking Products tab...');
await productsTab.click();
console.log('✅ Products tab clicked');
// Wait for the tab content to load
await page.waitForTimeout(5000);
// Click on the Asset Classes accordion to expand it
console.log('🔍 Looking for Asset Classes accordion...');
const assetClassesAccordion = page.locator(
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
);
await assetClassesAccordion.waitFor({ timeout: 10000 });
console.log('✅ Found Asset Classes accordion');
console.log('🖱️ Clicking Asset Classes accordion...');
await assetClassesAccordion.click();
console.log('✅ Asset Classes accordion clicked');
// Wait for the accordion content to expand
await page.waitForTimeout(2000);
console.log('🔍 Looking for Stocks checkbox...');
// Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks"
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
await stocksSpan.waitFor({ timeout: 10000 });
console.log('✅ Found Stocks span');
// Find the checkbox by looking in the same parent container
const parentContainer = stocksSpan.locator('..');
const checkbox = parentContainer.locator('input[type="checkbox"]');
if ((await checkbox.count()) > 0) {
console.log('📋 Clicking Stocks checkbox...');
await checkbox.first().check();
console.log('✅ Stocks checkbox checked');
} else {
console.log('⚠️ Could not find checkbox near Stocks text');
}
// Wait a moment for any UI updates
await page.waitForTimeout(1000);
// Find and click the nearest Apply button
console.log('🔍 Looking for Apply button...');
const applyButton = page.locator(
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
);
if ((await applyButton.count()) > 0) {
console.log('🎯 Clicking Apply button...');
await applyButton.first().click();
console.log('✅ Apply button clicked');
// Wait for any network requests triggered by the Apply button
await page.waitForTimeout(2000);
} else {
console.log('⚠️ Could not find Apply button');
}
} catch (interactionError) {
const errorMessage =
interactionError instanceof Error ? interactionError.message : String(interactionError);
console.error('❌ Page interaction failed:', errorMessage);
// Get debug info about the page
try {
const title = await page.title();
console.log(`📄 Current page title: "${title}"`);
const stocksElements = await page.locator('*:has-text("Stocks")').count();
console.log(`🔍 Found ${stocksElements} elements containing "Stocks"`);
const applyButtons = await page
.locator('button:has-text("Apply"), input[value*="Apply"]')
.count();
console.log(`🔍 Found ${applyButtons} Apply buttons`);
} catch (debugError) {
const debugMessage = debugError instanceof Error ? debugError.message : String(debugError);
console.log('❌ Could not get debug info:', debugMessage);
}
}
await new Promise(resolve => setTimeout(resolve, 2000));
console.log(`📊 Total events: ${eventCount}`);
await Browser.closeContext(contextId);
await Browser.close();
console.log('✅ Test completed');
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error('❌ Error:', errorMessage);
await Browser.close();
}
}
simpleProxyTest();

1
test-simple.ts
View file

@ -1 +0,0 @@
console.log("Testing browser import..."); import { Browser } from "@stock-bot/browser"; console.log("Browser imported successfully:", typeof Browser); Browser.initialize().then(() => console.log("Browser initialized")).catch(e => console.error("Error:", e));