work on ib and cleanup
This commit is contained in:
parent
a20a11c1aa
commit
d686a72591
41 changed files with 601 additions and 2793 deletions
8
.env
8
.env
|
|
@ -42,10 +42,10 @@ QUESTDB_PASSWORD=quest
|
|||
# MongoDB Configuration
|
||||
MONGODB_HOST=localhost
|
||||
MONGODB_PORT=27017
|
||||
MONGODB_DB=stockbot
|
||||
MONGODB_USER=
|
||||
MONGODB_PASSWORD=
|
||||
MONGODB_URI=mongodb://localhost:27017/stockbot
|
||||
MONGODB_DATABASE=stock
|
||||
MONGODB_USERNAME=trading_admin
|
||||
MONGODB_PASSWORD=trading_mongo_dev
|
||||
MONGODB_URI=mongodb://trading_admin:trading_mongo_dev@localhost:27017/stock?authSource=admin
|
||||
|
||||
# ===========================================
|
||||
# DATA PROVIDER CONFIGURATIONS
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { Hono } from 'hono';
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
import { loadEnvVariables } from '@stock-bot/config';
|
||||
import { getLogger, shutdownLoggers } from '@stock-bot/logger';
|
||||
import { connectMongoDB, disconnectMongoDB } from '@stock-bot/mongodb-client';
|
||||
import { Shutdown } from '@stock-bot/shutdown';
|
||||
import { initializeIBResources } from './providers/ib.tasks';
|
||||
import { initializeProxyResources } from './providers/proxy.tasks';
|
||||
|
|
@ -18,7 +19,7 @@ loadEnvVariables();
|
|||
const app = new Hono();
|
||||
const logger = getLogger('data-service');
|
||||
const PORT = parseInt(process.env.DATA_SERVICE_PORT || '3002');
|
||||
let server: any = null;
|
||||
let server: ReturnType<typeof Bun.serve> | null = null;
|
||||
|
||||
// Initialize shutdown manager with 15 second timeout
|
||||
const shutdown = Shutdown.getInstance({ timeout: 15000 });
|
||||
|
|
@ -35,6 +36,11 @@ async function initializeServices() {
|
|||
logger.info('Initializing data service...');
|
||||
|
||||
try {
|
||||
// Initialize MongoDB client first
|
||||
logger.info('Starting MongoDB client initialization...');
|
||||
await connectMongoDB();
|
||||
logger.info('MongoDB client initialized');
|
||||
|
||||
// Initialize browser resources
|
||||
logger.info('Starting browser resources initialization...');
|
||||
await Browser.initialize();
|
||||
|
|
@ -122,6 +128,18 @@ shutdown.onShutdown(async () => {
|
|||
}
|
||||
});
|
||||
|
||||
// Add MongoDB shutdown handler
|
||||
shutdown.onShutdown(async () => {
|
||||
logger.info('Shutting down MongoDB client...');
|
||||
try {
|
||||
await disconnectMongoDB();
|
||||
logger.info('MongoDB client shut down successfully');
|
||||
} catch (error) {
|
||||
logger.error('Error shutting down MongoDB client', { error });
|
||||
// Don't throw here to allow other shutdown handlers to complete
|
||||
}
|
||||
});
|
||||
|
||||
// Add logger shutdown handler (should be last)
|
||||
shutdown.onShutdown(async () => {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -6,31 +6,36 @@ const logger = getLogger('ib-provider');
|
|||
export const ibProvider: ProviderConfig = {
|
||||
name: 'ib',
|
||||
operations: {
|
||||
'ib-basics': async () => {
|
||||
'ib-exchanges-and-symbols': async () => {
|
||||
const { ibTasks } = await import('./ib.tasks');
|
||||
logger.info('Fetching symbol summary from IB');
|
||||
const sessionHeaders = await ibTasks.fetchSession();
|
||||
logger.info('Fetched symbol summary from IB', {
|
||||
sessionHeaders,
|
||||
});
|
||||
logger.info('Fetched symbol summary from IB');
|
||||
|
||||
// Get Exchanges
|
||||
logger.info('Fetching exchanges from IB');
|
||||
const exchanges = await ibTasks.fetchExchanges(sessionHeaders);
|
||||
logger.info('Fetched exchanges from IB', { exchanges });
|
||||
// return total;
|
||||
if (sessionHeaders) {
|
||||
logger.info('Fetching exchanges from IB');
|
||||
const exchanges = await ibTasks.fetchExchanges(sessionHeaders);
|
||||
logger.info('Fetched exchanges from IB', { count: exchanges.lenght });
|
||||
|
||||
// do the same as above but for symbols
|
||||
logger.info('Fetching symbols from IB');
|
||||
const symbols = await ibTasks.fetchSymbols(sessionHeaders);
|
||||
logger.info('Fetched symbols from IB', { symbols });
|
||||
|
||||
return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length };
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'ib-basics',
|
||||
operation: 'ib-basics',
|
||||
type: 'ib-exchanges-and-symbols',
|
||||
operation: 'ib-exchanges-and-symbols',
|
||||
payload: {},
|
||||
// should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
cronPattern: '*/2 * * * *',
|
||||
cronPattern: '0 0 * * 0',
|
||||
priority: 5,
|
||||
immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
description: 'Fetch and validate proxy list from sources',
|
||||
},
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { getMongoDBClient } from '@stock-bot/mongodb-client';
|
||||
|
||||
// Shared instances (module-scoped, not global)
|
||||
let isInitialized = false; // Track if resources are initialized
|
||||
let logger: ReturnType<typeof getLogger>;
|
||||
// let cache: CacheProvider;
|
||||
|
||||
export async function initializeIBResources(waitForCache = false): Promise<void> {
|
||||
export async function initializeIBResources(): Promise<void> {
|
||||
// Skip if already initialized
|
||||
if (isInitialized) {
|
||||
return;
|
||||
|
|
@ -93,7 +94,7 @@ export async function fetchSession(): Promise<Record<string, string> | undefined
|
|||
// Wait for and return headers immediately when captured
|
||||
logger.info('⏳ Waiting for headers to be captured...');
|
||||
const headers = await headersPromise;
|
||||
|
||||
page.close();
|
||||
if (headers) {
|
||||
logger.info('✅ Headers captured successfully');
|
||||
} else {
|
||||
|
|
@ -151,19 +152,156 @@ export async function fetchExchanges(sessionHeaders: Record<string, string>): Pr
|
|||
}
|
||||
|
||||
const data = await response.json();
|
||||
const exchanges = data?.exchanges || [];
|
||||
logger.info('✅ Exchange data fetched successfully');
|
||||
|
||||
logger.info('✅ Exchange data fetched successfully', {
|
||||
dataKeys: Object.keys(data || {}),
|
||||
dataSize: JSON.stringify(data).length,
|
||||
logger.info('Saving IB exchanges to MongoDB...');
|
||||
const client = getMongoDBClient();
|
||||
await client.batchUpsert('ib_exchanges', exchanges, ['id', 'country_code']);
|
||||
logger.info('✅ Exchange IB data saved to MongoDB:', {
|
||||
count: exchanges.length,
|
||||
});
|
||||
|
||||
return data;
|
||||
return exchanges;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch symbols from IB using the session headers
|
||||
export async function fetchSymbols(sessionHeaders: Record<string, string>): Promise<any> {
|
||||
try {
|
||||
logger.info('🔍 Fetching symbols with session headers...');
|
||||
// Configure the proxy
|
||||
const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80';
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
const requestBody = {
|
||||
domain: 'com',
|
||||
newProduct: 'all',
|
||||
pageNumber: 1,
|
||||
pageSize: 100,
|
||||
productCountry: ['CA', 'US'],
|
||||
productSymbol: '',
|
||||
productType: ['STK'],
|
||||
sortDirection: 'asc',
|
||||
sortField: 'symbol',
|
||||
};
|
||||
|
||||
// Get Summary
|
||||
const summaryResponse = await fetch(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: proxyUrl,
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
);
|
||||
|
||||
if (!summaryResponse.ok) {
|
||||
logger.error('❌ Summary API request failed', {
|
||||
status: summaryResponse.status,
|
||||
statusText: summaryResponse.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const summaryData = await summaryResponse.json();
|
||||
logger.info('✅ IB Summary data fetched successfully', {
|
||||
totalCount: summaryData[0].totalCount,
|
||||
});
|
||||
|
||||
const symbols = [];
|
||||
requestBody.pageSize = 500;
|
||||
const pageCount = Math.ceil(summaryData[0].totalCount / 500) || 0;
|
||||
logger.info('Fetching Symbols for IB', { pageCount });
|
||||
const symbolPromises = [];
|
||||
for (let page = 1; page <= pageCount; page++) {
|
||||
requestBody.pageNumber = page;
|
||||
|
||||
// Fetch symbols for the current page
|
||||
const symbolsResponse = fetch(
|
||||
'https://www.interactivebrokers.com/webrest/search/products-by-filters',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: proxyUrl,
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
);
|
||||
symbolPromises.push(symbolsResponse);
|
||||
}
|
||||
const responses = await Promise.all(symbolPromises);
|
||||
for (const response of responses) {
|
||||
if (!response.ok) {
|
||||
logger.error('❌ Symbols API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
const data = await response.json();
|
||||
const symJson = data?.products || [];
|
||||
if (symJson && symJson.length > 0) {
|
||||
symbols.push(...symJson);
|
||||
} else {
|
||||
logger.warn('⚠️ No symbols found in response');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (symbols.length === 0) {
|
||||
logger.warn('⚠️ No symbols fetched from IB');
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.info('✅ IB symbols fetched successfully, saving to DB...', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
const client = getMongoDBClient();
|
||||
await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
|
||||
logger.info('Saved IB symbols to DB', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
// logger.info('📤 Making request to exchange API...', {
|
||||
// url: exchangeUrl,
|
||||
// headerCount: Object.keys(requestHeaders).length,
|
||||
// });
|
||||
|
||||
// // Use fetch with proxy configuration
|
||||
// const response = await fetch(exchangeUrl, {
|
||||
// method: 'GET',
|
||||
// headers: requestHeaders,
|
||||
// proxy: proxyUrl,
|
||||
// });
|
||||
|
||||
// if (!response.ok) {
|
||||
// logger.error('❌ Exchange API request failed', {
|
||||
// status: response.status,
|
||||
// statusText: response.statusText,
|
||||
// });
|
||||
// return null;
|
||||
// }
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch symbols', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const ibTasks = {
|
||||
fetchSymbols,
|
||||
fetchSession,
|
||||
fetchExchanges,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,231 +2,231 @@
|
|||
// This script creates collections and indexes for sentiment and document storage
|
||||
|
||||
// Switch to the trading_documents database
|
||||
db = db.getSiblingDB('trading_documents');
|
||||
db = db.getSiblingDB('stock');
|
||||
|
||||
// Create collections with validation schemas
|
||||
// // Create collections with validation schemas
|
||||
|
||||
// Sentiment Analysis Collection
|
||||
db.createCollection('sentiment_analysis', {
|
||||
validator: {
|
||||
$jsonSchema: {
|
||||
bsonType: 'object',
|
||||
required: ['symbol', 'source', 'timestamp', 'sentiment_score'],
|
||||
properties: {
|
||||
symbol: {
|
||||
bsonType: 'string',
|
||||
description: 'Stock symbol (e.g., AAPL, GOOGL)'
|
||||
},
|
||||
source: {
|
||||
bsonType: 'string',
|
||||
description: 'Data source (news, social, earnings_call, etc.)'
|
||||
},
|
||||
timestamp: {
|
||||
bsonType: 'date',
|
||||
description: 'When the sentiment was recorded'
|
||||
},
|
||||
sentiment_score: {
|
||||
bsonType: 'double',
|
||||
minimum: -1.0,
|
||||
maximum: 1.0,
|
||||
description: 'Sentiment score between -1 (negative) and 1 (positive)'
|
||||
},
|
||||
confidence: {
|
||||
bsonType: 'double',
|
||||
minimum: 0.0,
|
||||
maximum: 1.0,
|
||||
description: 'Confidence level of the sentiment analysis'
|
||||
},
|
||||
text_snippet: {
|
||||
bsonType: 'string',
|
||||
description: 'Original text that was analyzed'
|
||||
},
|
||||
metadata: {
|
||||
bsonType: 'object',
|
||||
description: 'Additional metadata about the sentiment source'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
// db.createCollection('sentiment_analysis', {
|
||||
// validator: {
|
||||
// $jsonSchema: {
|
||||
// bsonType: 'object',
|
||||
// required: ['symbol', 'source', 'timestamp', 'sentiment_score'],
|
||||
// properties: {
|
||||
// symbol: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Stock symbol (e.g., AAPL, GOOGL)'
|
||||
// },
|
||||
// source: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Data source (news, social, earnings_call, etc.)'
|
||||
// },
|
||||
// timestamp: {
|
||||
// bsonType: 'date',
|
||||
// description: 'When the sentiment was recorded'
|
||||
// },
|
||||
// sentiment_score: {
|
||||
// bsonType: 'double',
|
||||
// minimum: -1.0,
|
||||
// maximum: 1.0,
|
||||
// description: 'Sentiment score between -1 (negative) and 1 (positive)'
|
||||
// },
|
||||
// confidence: {
|
||||
// bsonType: 'double',
|
||||
// minimum: 0.0,
|
||||
// maximum: 1.0,
|
||||
// description: 'Confidence level of the sentiment analysis'
|
||||
// },
|
||||
// text_snippet: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Original text that was analyzed'
|
||||
// },
|
||||
// metadata: {
|
||||
// bsonType: 'object',
|
||||
// description: 'Additional metadata about the sentiment source'
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
|
||||
// Raw Documents Collection (for news articles, social media posts, etc.)
|
||||
db.createCollection('raw_documents', {
|
||||
validator: {
|
||||
$jsonSchema: {
|
||||
bsonType: 'object',
|
||||
required: ['source', 'document_type', 'timestamp', 'content'],
|
||||
properties: {
|
||||
source: {
|
||||
bsonType: 'string',
|
||||
description: 'Document source (news_api, twitter, reddit, etc.)'
|
||||
},
|
||||
document_type: {
|
||||
bsonType: 'string',
|
||||
enum: ['news_article', 'social_post', 'earnings_transcript', 'research_report', 'press_release'],
|
||||
description: 'Type of document'
|
||||
},
|
||||
timestamp: {
|
||||
bsonType: 'date',
|
||||
description: 'When the document was created/published'
|
||||
},
|
||||
symbols: {
|
||||
bsonType: 'array',
|
||||
items: {
|
||||
bsonType: 'string'
|
||||
},
|
||||
description: 'Array of stock symbols mentioned in the document'
|
||||
},
|
||||
title: {
|
||||
bsonType: 'string',
|
||||
description: 'Document title or headline'
|
||||
},
|
||||
content: {
|
||||
bsonType: 'string',
|
||||
description: 'Full document content'
|
||||
},
|
||||
url: {
|
||||
bsonType: 'string',
|
||||
description: 'Original URL of the document'
|
||||
},
|
||||
author: {
|
||||
bsonType: 'string',
|
||||
description: 'Document author or source account'
|
||||
},
|
||||
processed: {
|
||||
bsonType: 'bool',
|
||||
description: 'Whether this document has been processed for sentiment'
|
||||
},
|
||||
metadata: {
|
||||
bsonType: 'object',
|
||||
description: 'Additional document metadata'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
// // Raw Documents Collection (for news articles, social media posts, etc.)
|
||||
// db.createCollection('raw_documents', {
|
||||
// validator: {
|
||||
// $jsonSchema: {
|
||||
// bsonType: 'object',
|
||||
// required: ['source', 'document_type', 'timestamp', 'content'],
|
||||
// properties: {
|
||||
// source: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Document source (news_api, twitter, reddit, etc.)'
|
||||
// },
|
||||
// document_type: {
|
||||
// bsonType: 'string',
|
||||
// enum: ['news_article', 'social_post', 'earnings_transcript', 'research_report', 'press_release'],
|
||||
// description: 'Type of document'
|
||||
// },
|
||||
// timestamp: {
|
||||
// bsonType: 'date',
|
||||
// description: 'When the document was created/published'
|
||||
// },
|
||||
// symbols: {
|
||||
// bsonType: 'array',
|
||||
// items: {
|
||||
// bsonType: 'string'
|
||||
// },
|
||||
// description: 'Array of stock symbols mentioned in the document'
|
||||
// },
|
||||
// title: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Document title or headline'
|
||||
// },
|
||||
// content: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Full document content'
|
||||
// },
|
||||
// url: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Original URL of the document'
|
||||
// },
|
||||
// author: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Document author or source account'
|
||||
// },
|
||||
// processed: {
|
||||
// bsonType: 'bool',
|
||||
// description: 'Whether this document has been processed for sentiment'
|
||||
// },
|
||||
// metadata: {
|
||||
// bsonType: 'object',
|
||||
// description: 'Additional document metadata'
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
|
||||
// Market Events Collection (for significant market events and their impact)
|
||||
db.createCollection('market_events', {
|
||||
validator: {
|
||||
$jsonSchema: {
|
||||
bsonType: 'object',
|
||||
required: ['event_type', 'timestamp', 'description'],
|
||||
properties: {
|
||||
event_type: {
|
||||
bsonType: 'string',
|
||||
enum: ['earnings', 'merger', 'acquisition', 'ipo', 'dividend', 'split', 'regulatory', 'economic_indicator'],
|
||||
description: 'Type of market event'
|
||||
},
|
||||
timestamp: {
|
||||
bsonType: 'date',
|
||||
description: 'When the event occurred or was announced'
|
||||
},
|
||||
symbols: {
|
||||
bsonType: 'array',
|
||||
items: {
|
||||
bsonType: 'string'
|
||||
},
|
||||
description: 'Stock symbols affected by this event'
|
||||
},
|
||||
description: {
|
||||
bsonType: 'string',
|
||||
description: 'Event description'
|
||||
},
|
||||
impact_score: {
|
||||
bsonType: 'double',
|
||||
minimum: -5.0,
|
||||
maximum: 5.0,
|
||||
description: 'Expected market impact score'
|
||||
},
|
||||
source_documents: {
|
||||
bsonType: 'array',
|
||||
items: {
|
||||
bsonType: 'objectId'
|
||||
},
|
||||
description: 'References to raw_documents that reported this event'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
// // Market Events Collection (for significant market events and their impact)
|
||||
// db.createCollection('market_events', {
|
||||
// validator: {
|
||||
// $jsonSchema: {
|
||||
// bsonType: 'object',
|
||||
// required: ['event_type', 'timestamp', 'description'],
|
||||
// properties: {
|
||||
// event_type: {
|
||||
// bsonType: 'string',
|
||||
// enum: ['earnings', 'merger', 'acquisition', 'ipo', 'dividend', 'split', 'regulatory', 'economic_indicator'],
|
||||
// description: 'Type of market event'
|
||||
// },
|
||||
// timestamp: {
|
||||
// bsonType: 'date',
|
||||
// description: 'When the event occurred or was announced'
|
||||
// },
|
||||
// symbols: {
|
||||
// bsonType: 'array',
|
||||
// items: {
|
||||
// bsonType: 'string'
|
||||
// },
|
||||
// description: 'Stock symbols affected by this event'
|
||||
// },
|
||||
// description: {
|
||||
// bsonType: 'string',
|
||||
// description: 'Event description'
|
||||
// },
|
||||
// impact_score: {
|
||||
// bsonType: 'double',
|
||||
// minimum: -5.0,
|
||||
// maximum: 5.0,
|
||||
// description: 'Expected market impact score'
|
||||
// },
|
||||
// source_documents: {
|
||||
// bsonType: 'array',
|
||||
// items: {
|
||||
// bsonType: 'objectId'
|
||||
// },
|
||||
// description: 'References to raw_documents that reported this event'
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
|
||||
// Create indexes for efficient querying
|
||||
// // Create indexes for efficient querying
|
||||
|
||||
// Sentiment Analysis indexes
|
||||
db.sentiment_analysis.createIndex({ symbol: 1, timestamp: -1 });
|
||||
db.sentiment_analysis.createIndex({ source: 1, timestamp: -1 });
|
||||
db.sentiment_analysis.createIndex({ timestamp: -1 });
|
||||
db.sentiment_analysis.createIndex({ symbol: 1, source: 1, timestamp: -1 });
|
||||
// // Sentiment Analysis indexes
|
||||
// db.sentiment_analysis.createIndex({ symbol: 1, timestamp: -1 });
|
||||
// db.sentiment_analysis.createIndex({ source: 1, timestamp: -1 });
|
||||
// db.sentiment_analysis.createIndex({ timestamp: -1 });
|
||||
// db.sentiment_analysis.createIndex({ symbol: 1, source: 1, timestamp: -1 });
|
||||
|
||||
// Raw Documents indexes
|
||||
db.raw_documents.createIndex({ symbols: 1, timestamp: -1 });
|
||||
db.raw_documents.createIndex({ source: 1, timestamp: -1 });
|
||||
db.raw_documents.createIndex({ document_type: 1, timestamp: -1 });
|
||||
db.raw_documents.createIndex({ processed: 1, timestamp: -1 });
|
||||
db.raw_documents.createIndex({ timestamp: -1 });
|
||||
// // Raw Documents indexes
|
||||
// db.raw_documents.createIndex({ symbols: 1, timestamp: -1 });
|
||||
// db.raw_documents.createIndex({ source: 1, timestamp: -1 });
|
||||
// db.raw_documents.createIndex({ document_type: 1, timestamp: -1 });
|
||||
// db.raw_documents.createIndex({ processed: 1, timestamp: -1 });
|
||||
// db.raw_documents.createIndex({ timestamp: -1 });
|
||||
|
||||
// Market Events indexes
|
||||
db.market_events.createIndex({ symbols: 1, timestamp: -1 });
|
||||
db.market_events.createIndex({ event_type: 1, timestamp: -1 });
|
||||
db.market_events.createIndex({ timestamp: -1 });
|
||||
// // Market Events indexes
|
||||
// db.market_events.createIndex({ symbols: 1, timestamp: -1 });
|
||||
// db.market_events.createIndex({ event_type: 1, timestamp: -1 });
|
||||
// db.market_events.createIndex({ timestamp: -1 });
|
||||
|
||||
// Insert some sample data for testing
|
||||
// // Insert some sample data for testing
|
||||
|
||||
// Sample sentiment data
|
||||
db.sentiment_analysis.insertMany([
|
||||
{
|
||||
symbol: 'AAPL',
|
||||
source: 'news_analysis',
|
||||
timestamp: new Date(),
|
||||
sentiment_score: 0.75,
|
||||
confidence: 0.89,
|
||||
text_snippet: 'Apple reports strong quarterly earnings...',
|
||||
metadata: {
|
||||
article_id: 'news_001',
|
||||
provider: 'financial_news_api'
|
||||
}
|
||||
},
|
||||
{
|
||||
symbol: 'GOOGL',
|
||||
source: 'social_media',
|
||||
timestamp: new Date(),
|
||||
sentiment_score: -0.25,
|
||||
confidence: 0.67,
|
||||
text_snippet: 'Concerns about Google AI regulation...',
|
||||
metadata: {
|
||||
platform: 'twitter',
|
||||
engagement_score: 450
|
||||
}
|
||||
}
|
||||
]);
|
||||
// // Sample sentiment data
|
||||
// db.sentiment_analysis.insertMany([
|
||||
// {
|
||||
// symbol: 'AAPL',
|
||||
// source: 'news_analysis',
|
||||
// timestamp: new Date(),
|
||||
// sentiment_score: 0.75,
|
||||
// confidence: 0.89,
|
||||
// text_snippet: 'Apple reports strong quarterly earnings...',
|
||||
// metadata: {
|
||||
// article_id: 'news_001',
|
||||
// provider: 'financial_news_api'
|
||||
// }
|
||||
// },
|
||||
// {
|
||||
// symbol: 'GOOGL',
|
||||
// source: 'social_media',
|
||||
// timestamp: new Date(),
|
||||
// sentiment_score: -0.25,
|
||||
// confidence: 0.67,
|
||||
// text_snippet: 'Concerns about Google AI regulation...',
|
||||
// metadata: {
|
||||
// platform: 'twitter',
|
||||
// engagement_score: 450
|
||||
// }
|
||||
// }
|
||||
// ]);
|
||||
|
||||
// Sample raw document
|
||||
db.raw_documents.insertOne({
|
||||
source: 'financial_news_api',
|
||||
document_type: 'news_article',
|
||||
timestamp: new Date(),
|
||||
symbols: ['AAPL', 'MSFT'],
|
||||
title: 'Tech Giants Show Strong Q4 Performance',
|
||||
content: 'Apple and Microsoft both reported better than expected earnings for Q4...',
|
||||
url: 'https://example.com/tech-earnings-q4',
|
||||
author: 'Financial Reporter',
|
||||
processed: true,
|
||||
metadata: {
|
||||
word_count: 850,
|
||||
readability_score: 0.75
|
||||
}
|
||||
});
|
||||
// // Sample raw document
|
||||
// db.raw_documents.insertOne({
|
||||
// source: 'financial_news_api',
|
||||
// document_type: 'news_article',
|
||||
// timestamp: new Date(),
|
||||
// symbols: ['AAPL', 'MSFT'],
|
||||
// title: 'Tech Giants Show Strong Q4 Performance',
|
||||
// content: 'Apple and Microsoft both reported better than expected earnings for Q4...',
|
||||
// url: 'https://example.com/tech-earnings-q4',
|
||||
// author: 'Financial Reporter',
|
||||
// processed: true,
|
||||
// metadata: {
|
||||
// word_count: 850,
|
||||
// readability_score: 0.75
|
||||
// }
|
||||
// });
|
||||
|
||||
// Sample market event
|
||||
db.market_events.insertOne({
|
||||
event_type: 'earnings',
|
||||
timestamp: new Date(),
|
||||
symbols: ['AAPL'],
|
||||
description: 'Apple Q4 2024 Earnings Report',
|
||||
impact_score: 2.5,
|
||||
source_documents: []
|
||||
});
|
||||
// // Sample market event
|
||||
// db.market_events.insertOne({
|
||||
// event_type: 'earnings',
|
||||
// timestamp: new Date(),
|
||||
// symbols: ['AAPL'],
|
||||
// description: 'Apple Q4 2024 Earnings Report',
|
||||
// impact_score: 2.5,
|
||||
// source_documents: []
|
||||
// });
|
||||
|
||||
print('MongoDB initialization completed successfully!');
|
||||
print('Created collections: sentiment_analysis, raw_documents, market_events');
|
||||
|
|
|
|||
0
database/postgres/scripts/populate-ib-exchanges.ts
Normal file
0
database/postgres/scripts/populate-ib-exchanges.ts
Normal file
0
database/postgres/scripts/setup-ib-fast.ts
Normal file
0
database/postgres/scripts/setup-ib-fast.ts
Normal file
0
database/postgres/scripts/setup-ib-schema-simple.ts
Normal file
0
database/postgres/scripts/setup-ib-schema-simple.ts
Normal file
0
database/postgres/scripts/setup-ib-schema.ts
Normal file
0
database/postgres/scripts/setup-ib-schema.ts
Normal file
0
database/postgres/scripts/setup.ts
Normal file
0
database/postgres/scripts/setup.ts
Normal file
|
|
@ -82,7 +82,7 @@ services: # Dragonfly - Redis replacement for caching and events
|
|||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: trading_admin
|
||||
MONGO_INITDB_ROOT_PASSWORD: trading_mongo_dev
|
||||
MONGO_INITDB_DATABASE: trading_documents
|
||||
MONGO_INITDB_DATABASE: stock
|
||||
ports:
|
||||
- "27017:27017"
|
||||
volumes:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ export const mongodbConfig = cleanEnv(process.env, {
|
|||
// MongoDB Connection
|
||||
MONGODB_HOST: str('localhost', 'MongoDB host'),
|
||||
MONGODB_PORT: port(27017, 'MongoDB port'),
|
||||
MONGODB_DATABASE: str('trading_documents', 'MongoDB database name'),
|
||||
MONGODB_DATABASE: str('stock', 'MongoDB database name'),
|
||||
|
||||
// Authentication
|
||||
MONGODB_USERNAME: str('trading_admin', 'MongoDB username'),
|
||||
|
|
|
|||
|
|
@ -1,249 +0,0 @@
|
|||
import type { Document } from 'mongodb';
|
||||
import type { MongoDBClient } from './client';
|
||||
import type { CollectionNames } from './types';
|
||||
|
||||
/**
|
||||
* MongoDB Aggregation Builder
|
||||
*
|
||||
* Provides a fluent interface for building MongoDB aggregation pipelines
|
||||
*/
|
||||
export class MongoDBAggregationBuilder {
|
||||
private pipeline: any[] = [];
|
||||
private readonly client: MongoDBClient;
|
||||
private collection: CollectionNames | null = null;
|
||||
|
||||
constructor(client: MongoDBClient) {
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the collection to aggregate on
|
||||
*/
|
||||
from(collection: CollectionNames): this {
|
||||
this.collection = collection;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a match stage
|
||||
*/
|
||||
match(filter: any): this {
|
||||
this.pipeline.push({ $match: filter });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a group stage
|
||||
*/
|
||||
group(groupBy: any): this {
|
||||
this.pipeline.push({ $group: groupBy });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a sort stage
|
||||
*/
|
||||
sort(sortBy: any): this {
|
||||
this.pipeline.push({ $sort: sortBy });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a limit stage
|
||||
*/
|
||||
limit(count: number): this {
|
||||
this.pipeline.push({ $limit: count });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a skip stage
|
||||
*/
|
||||
skip(count: number): this {
|
||||
this.pipeline.push({ $skip: count });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a project stage
|
||||
*/
|
||||
project(projection: any): this {
|
||||
this.pipeline.push({ $project: projection });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an unwind stage
|
||||
*/
|
||||
unwind(field: string, options?: any): this {
|
||||
this.pipeline.push({
|
||||
$unwind: options ? { path: field, ...options } : field,
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a lookup stage (join)
|
||||
*/
|
||||
lookup(from: string, localField: string, foreignField: string, as: string): this {
|
||||
this.pipeline.push({
|
||||
$lookup: {
|
||||
from,
|
||||
localField,
|
||||
foreignField,
|
||||
as,
|
||||
},
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a custom stage
|
||||
*/
|
||||
addStage(stage: any): this {
|
||||
this.pipeline.push(stage);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Execute the aggregation pipeline
|
||||
*/
|
||||
async execute<T extends Document = Document>(): Promise<T[]> {
|
||||
if (!this.collection) {
|
||||
throw new Error('Collection not specified. Use .from() to set the collection.');
|
||||
}
|
||||
|
||||
const collection = this.client.getCollection(this.collection);
|
||||
return await collection.aggregate<T>(this.pipeline).toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the pipeline array
|
||||
*/
|
||||
getPipeline(): any[] {
|
||||
return [...this.pipeline];
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the pipeline
|
||||
*/
|
||||
reset(): this {
|
||||
this.pipeline = [];
|
||||
this.collection = null;
|
||||
return this;
|
||||
}
|
||||
|
||||
// Convenience methods for common aggregations
|
||||
|
||||
/**
|
||||
* Sentiment analysis aggregation
|
||||
*/
|
||||
sentimentAnalysis(symbol?: string, timeframe?: { start: Date; end: Date }): this {
|
||||
this.from('sentiment_data');
|
||||
|
||||
const matchConditions: any = {};
|
||||
if (symbol) {
|
||||
matchConditions.symbol = symbol;
|
||||
}
|
||||
if (timeframe) {
|
||||
matchConditions.timestamp = {
|
||||
$gte: timeframe.start,
|
||||
$lte: timeframe.end,
|
||||
};
|
||||
}
|
||||
|
||||
if (Object.keys(matchConditions).length > 0) {
|
||||
this.match(matchConditions);
|
||||
}
|
||||
|
||||
return this.group({
|
||||
_id: {
|
||||
symbol: '$symbol',
|
||||
sentiment: '$sentiment_label',
|
||||
},
|
||||
count: { $sum: 1 },
|
||||
avgScore: { $avg: '$sentiment_score' },
|
||||
avgConfidence: { $avg: '$confidence' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* News article aggregation by publication
|
||||
*/
|
||||
newsByPublication(symbols?: string[]): this {
|
||||
this.from('news_articles');
|
||||
|
||||
if (symbols && symbols.length > 0) {
|
||||
this.match({ symbols: { $in: symbols } });
|
||||
}
|
||||
|
||||
return this.group({
|
||||
_id: '$publication',
|
||||
articleCount: { $sum: 1 },
|
||||
symbols: { $addToSet: '$symbols' },
|
||||
avgSentiment: { $avg: '$sentiment_score' },
|
||||
latestArticle: { $max: '$published_date' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* SEC filings by company
|
||||
*/
|
||||
secFilingsByCompany(filingTypes?: string[]): this {
|
||||
this.from('sec_filings');
|
||||
|
||||
if (filingTypes && filingTypes.length > 0) {
|
||||
this.match({ filing_type: { $in: filingTypes } });
|
||||
}
|
||||
|
||||
return this.group({
|
||||
_id: {
|
||||
cik: '$cik',
|
||||
company: '$company_name',
|
||||
},
|
||||
filingCount: { $sum: 1 },
|
||||
filingTypes: { $addToSet: '$filing_type' },
|
||||
latestFiling: { $max: '$filing_date' },
|
||||
symbols: { $addToSet: '$symbols' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Document processing status summary
|
||||
*/
|
||||
processingStatusSummary(collection: CollectionNames): this {
|
||||
this.from(collection);
|
||||
|
||||
return this.group({
|
||||
_id: '$processing_status',
|
||||
count: { $sum: 1 },
|
||||
avgSizeBytes: { $avg: '$size_bytes' },
|
||||
oldestDocument: { $min: '$created_at' },
|
||||
newestDocument: { $max: '$created_at' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Time-based aggregation (daily/hourly counts)
|
||||
*/
|
||||
timeBasedCounts(
|
||||
collection: CollectionNames,
|
||||
dateField: string = 'created_at',
|
||||
interval: 'hour' | 'day' | 'week' | 'month' = 'day'
|
||||
): this {
|
||||
this.from(collection);
|
||||
|
||||
const dateFormat = {
|
||||
hour: { $dateToString: { format: '%Y-%m-%d %H:00:00', date: `$${dateField}` } },
|
||||
day: { $dateToString: { format: '%Y-%m-%d', date: `$${dateField}` } },
|
||||
week: { $dateToString: { format: '%Y-W%V', date: `$${dateField}` } },
|
||||
month: { $dateToString: { format: '%Y-%m', date: `$${dateField}` } },
|
||||
};
|
||||
|
||||
return this.group({
|
||||
_id: dateFormat[interval],
|
||||
count: { $sum: 1 },
|
||||
firstDocument: { $min: `$${dateField}` },
|
||||
lastDocument: { $max: `$${dateField}` },
|
||||
}).sort({ _id: 1 });
|
||||
}
|
||||
}
|
||||
|
|
@ -1,110 +1,68 @@
|
|||
import {
|
||||
Collection,
|
||||
Db,
|
||||
Document,
|
||||
MongoClient,
|
||||
MongoClientOptions,
|
||||
OptionalUnlessRequiredId,
|
||||
WithId,
|
||||
} from 'mongodb';
|
||||
import * as yup from 'yup';
|
||||
import { Collection, Db, MongoClient, OptionalUnlessRequiredId } from 'mongodb';
|
||||
import { mongodbConfig } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { MongoDBHealthMonitor } from './health';
|
||||
import { schemaMap } from './schemas';
|
||||
import type {
|
||||
AnalystReport,
|
||||
CollectionNames,
|
||||
DocumentBase,
|
||||
EarningsTranscript,
|
||||
MongoDBClientConfig,
|
||||
MongoDBConnectionOptions,
|
||||
NewsArticle,
|
||||
RawDocument,
|
||||
SecFiling,
|
||||
SentimentData,
|
||||
} from './types';
|
||||
import type { DocumentBase } from './types';
|
||||
|
||||
/**
|
||||
* MongoDB Client for Stock Bot
|
||||
* Simplified MongoDB Client for Stock Bot Data Service
|
||||
*
|
||||
* Provides type-safe access to MongoDB collections with built-in
|
||||
* health monitoring, connection pooling, and schema validation.
|
||||
* A singleton MongoDB client focused solely on batch upsert operations
|
||||
* with minimal configuration and no health monitoring complexity.
|
||||
*/
|
||||
export class MongoDBClient {
|
||||
private static instance: MongoDBClient | null = null;
|
||||
private client: MongoClient | null = null;
|
||||
private db: Db | null = null;
|
||||
private readonly config: MongoDBClientConfig;
|
||||
private readonly options: MongoDBConnectionOptions;
|
||||
private readonly logger: ReturnType<typeof getLogger>;
|
||||
private readonly healthMonitor: MongoDBHealthMonitor;
|
||||
private readonly logger = getLogger('mongodb-client-simple');
|
||||
private isConnected = false;
|
||||
|
||||
constructor(config?: Partial<MongoDBClientConfig>, options?: MongoDBConnectionOptions) {
|
||||
this.config = this.buildConfig(config);
|
||||
this.options = {
|
||||
retryAttempts: 3,
|
||||
retryDelay: 1000,
|
||||
healthCheckInterval: 30000,
|
||||
...options,
|
||||
};
|
||||
private constructor() {}
|
||||
|
||||
this.logger = getLogger('mongodb-client');
|
||||
this.healthMonitor = new MongoDBHealthMonitor(this);
|
||||
/**
|
||||
* Get singleton instance
|
||||
*/
|
||||
static getInstance(): MongoDBClient {
|
||||
if (!MongoDBClient.instance) {
|
||||
MongoDBClient.instance = new MongoDBClient();
|
||||
}
|
||||
return MongoDBClient.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to MongoDB
|
||||
* Connect to MongoDB with simple configuration
|
||||
*/
|
||||
async connect(): Promise<void> {
|
||||
if (this.isConnected && this.client) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uri = this.buildConnectionUri();
|
||||
const clientOptions = this.buildClientOptions();
|
||||
try {
|
||||
const uri = this.buildConnectionUri();
|
||||
this.logger.info('Connecting to MongoDB...');
|
||||
|
||||
let lastError: Error | null = null;
|
||||
this.client = new MongoClient(uri, {
|
||||
maxPoolSize: 10,
|
||||
minPoolSize: 1,
|
||||
connectTimeoutMS: 10000,
|
||||
socketTimeoutMS: 30000,
|
||||
serverSelectionTimeoutMS: 5000,
|
||||
});
|
||||
|
||||
for (let attempt = 1; attempt <= this.options.retryAttempts!; attempt++) {
|
||||
try {
|
||||
this.logger.info(
|
||||
`Connecting to MongoDB (attempt ${attempt}/${this.options.retryAttempts})...`
|
||||
);
|
||||
await this.client.connect();
|
||||
await this.client.db(mongodbConfig.MONGODB_DATABASE).admin().ping();
|
||||
|
||||
this.client = new MongoClient(uri, clientOptions);
|
||||
await this.client.connect();
|
||||
this.db = this.client.db(mongodbConfig.MONGODB_DATABASE);
|
||||
this.isConnected = true;
|
||||
|
||||
// Test the connection
|
||||
await this.client.db(this.config.database).admin().ping();
|
||||
|
||||
this.db = this.client.db(this.config.database);
|
||||
this.isConnected = true;
|
||||
|
||||
this.logger.info('Successfully connected to MongoDB');
|
||||
|
||||
// Start health monitoring
|
||||
this.healthMonitor.start();
|
||||
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
this.logger.error(`MongoDB connection attempt ${attempt} failed:`, error);
|
||||
|
||||
if (this.client) {
|
||||
await this.client.close();
|
||||
this.client = null;
|
||||
}
|
||||
|
||||
if (attempt < this.options.retryAttempts!) {
|
||||
await this.delay(this.options.retryDelay! * attempt);
|
||||
}
|
||||
this.logger.info('Successfully connected to MongoDB');
|
||||
} catch (error) {
|
||||
this.logger.error('MongoDB connection failed:', error);
|
||||
if (this.client) {
|
||||
await this.client.close();
|
||||
this.client = null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to connect to MongoDB after ${this.options.retryAttempts} attempts: ${lastError?.message}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -116,7 +74,6 @@ export class MongoDBClient {
|
|||
}
|
||||
|
||||
try {
|
||||
this.healthMonitor.stop();
|
||||
await this.client.close();
|
||||
this.isConnected = false;
|
||||
this.client = null;
|
||||
|
|
@ -128,10 +85,138 @@ export class MongoDBClient {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch upsert documents for high-performance operations
|
||||
* Supports single or multiple unique keys for matching
|
||||
*/
|
||||
async batchUpsert<T extends DocumentBase>(
|
||||
collectionName: string,
|
||||
documents: Array<
|
||||
Omit<T, '_id' | 'created_at' | 'updated_at'> & Partial<Pick<T, 'created_at' | 'updated_at'>>
|
||||
>,
|
||||
uniqueKeys: string | string[],
|
||||
options: {
|
||||
chunkSize?: number;
|
||||
} = {}
|
||||
): Promise<{ insertedCount: number; updatedCount: number; errors: unknown[] }> {
|
||||
if (!this.db) {
|
||||
throw new Error('MongoDB client not connected');
|
||||
}
|
||||
|
||||
if (documents.length === 0) {
|
||||
return { insertedCount: 0, updatedCount: 0, errors: [] };
|
||||
}
|
||||
|
||||
// Normalize uniqueKeys to array
|
||||
const keyFields = Array.isArray(uniqueKeys) ? uniqueKeys : [uniqueKeys];
|
||||
|
||||
if (keyFields.length === 0) {
|
||||
throw new Error('At least one unique key must be provided');
|
||||
}
|
||||
|
||||
const { chunkSize = 10000 } = options;
|
||||
const collection = this.db.collection<T>(collectionName);
|
||||
const operationId = Math.random().toString(36).substring(7);
|
||||
|
||||
let totalInserted = 0;
|
||||
let totalUpdated = 0;
|
||||
const errors: unknown[] = [];
|
||||
|
||||
this.logger.info(`Starting batch upsert operation [${operationId}]`, {
|
||||
collection: collectionName,
|
||||
totalDocuments: documents.length,
|
||||
uniqueKeys: keyFields,
|
||||
chunkSize
|
||||
});
|
||||
|
||||
// Process documents in chunks to avoid memory issues
|
||||
for (let i = 0; i < documents.length; i += chunkSize) {
|
||||
const chunk = documents.slice(i, i + chunkSize);
|
||||
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
|
||||
// Prepare bulk operations
|
||||
const bulkOps = chunk.map(doc => {
|
||||
const now = new Date();
|
||||
const docWithTimestamps = {
|
||||
...doc,
|
||||
created_at: doc.created_at || now,
|
||||
updated_at: now,
|
||||
};
|
||||
|
||||
// Create filter using multiple unique keys
|
||||
const filter: Record<string, unknown> = {};
|
||||
keyFields.forEach(key => {
|
||||
const value = (doc as Record<string, unknown>)[key];
|
||||
if (value === undefined || value === null) {
|
||||
throw new Error(`Document missing required unique key: ${key}`);
|
||||
}
|
||||
filter[key] = value;
|
||||
});
|
||||
|
||||
// Remove created_at from $set to avoid conflict with $setOnInsert
|
||||
const { created_at, ...updateFields } = docWithTimestamps;
|
||||
|
||||
return {
|
||||
updateOne: {
|
||||
filter,
|
||||
update: {
|
||||
$set: updateFields,
|
||||
$setOnInsert: { created_at },
|
||||
},
|
||||
upsert: true,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
// Execute bulk operation with type assertion to handle complex MongoDB types
|
||||
const result = await collection.bulkWrite(bulkOps as never, { ordered: false });
|
||||
|
||||
const executionTime = Date.now() - startTime;
|
||||
const inserted = result.upsertedCount;
|
||||
const updated = result.modifiedCount;
|
||||
|
||||
totalInserted += inserted;
|
||||
totalUpdated += updated;
|
||||
|
||||
this.logger.debug(`Batch upsert chunk processed [${operationId}]`, {
|
||||
chunkNumber: Math.floor(i / chunkSize) + 1,
|
||||
chunkSize: chunk.length,
|
||||
inserted,
|
||||
updated,
|
||||
executionTime,
|
||||
collection: collectionName,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(`Batch upsert failed on chunk [${operationId}]`, {
|
||||
error,
|
||||
collection: collectionName,
|
||||
chunkNumber: Math.floor(i / chunkSize) + 1,
|
||||
chunkStart: i,
|
||||
chunkSize: chunk.length,
|
||||
uniqueKeys: keyFields,
|
||||
});
|
||||
errors.push(error);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info(`Batch upsert completed [${operationId}]`, {
|
||||
collection: collectionName,
|
||||
totalRecords: documents.length,
|
||||
inserted: totalInserted,
|
||||
updated: totalUpdated,
|
||||
errors: errors.length,
|
||||
uniqueKeys: keyFields,
|
||||
});
|
||||
|
||||
return { insertedCount: totalInserted, updatedCount: totalUpdated, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a typed collection
|
||||
*/
|
||||
getCollection<T extends DocumentBase>(name: CollectionNames): Collection<T> {
|
||||
getCollection<T extends DocumentBase>(name: string): Collection<T> {
|
||||
if (!this.db) {
|
||||
throw new Error('MongoDB client not connected');
|
||||
}
|
||||
|
|
@ -139,162 +224,26 @@ export class MongoDBClient {
|
|||
}
|
||||
|
||||
/**
|
||||
* Insert a document with validation
|
||||
* Simple insert operation
|
||||
*/
|
||||
async insertOne<T extends DocumentBase>(
|
||||
collectionName: CollectionNames,
|
||||
collectionName: string,
|
||||
document: Omit<T, '_id' | 'created_at' | 'updated_at'> &
|
||||
Partial<Pick<T, 'created_at' | 'updated_at'>>
|
||||
): Promise<T> {
|
||||
const collection = this.getCollection<T>(collectionName);
|
||||
|
||||
// Add timestamps
|
||||
const now = new Date();
|
||||
const docWithTimestamps = {
|
||||
...document,
|
||||
created_at: document.created_at || now,
|
||||
updated_at: now,
|
||||
} as T; // Validate document if schema exists
|
||||
if (collectionName in schemaMap) {
|
||||
try {
|
||||
(schemaMap as any)[collectionName].validateSync(docWithTimestamps);
|
||||
} catch (error) {
|
||||
if (error instanceof yup.ValidationError) {
|
||||
this.logger.error(`Document validation failed for ${collectionName}:`, error.errors);
|
||||
throw new Error(`Document validation failed: ${error.errors?.map(e => e).join(', ')}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} as T;
|
||||
|
||||
const result = await collection.insertOne(docWithTimestamps as OptionalUnlessRequiredId<T>);
|
||||
return { ...docWithTimestamps, _id: result.insertedId } as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a document with validation
|
||||
*/
|
||||
async updateOne<T extends DocumentBase>(
|
||||
collectionName: CollectionNames,
|
||||
filter: any,
|
||||
update: Partial<T>
|
||||
): Promise<boolean> {
|
||||
const collection = this.getCollection<T>(collectionName);
|
||||
|
||||
// Add updated timestamp
|
||||
const updateWithTimestamp = {
|
||||
...update,
|
||||
updated_at: new Date(),
|
||||
};
|
||||
|
||||
const result = await collection.updateOne(filter, { $set: updateWithTimestamp });
|
||||
return result.modifiedCount > 0;
|
||||
}
|
||||
/**
|
||||
* Find documents with optional validation
|
||||
*/
|
||||
async find<T extends DocumentBase>(
|
||||
collectionName: CollectionNames,
|
||||
filter: any = {},
|
||||
options: any = {}
|
||||
): Promise<T[]> {
|
||||
const collection = this.getCollection<T>(collectionName);
|
||||
return (await collection.find(filter, options).toArray()) as T[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Find one document
|
||||
*/
|
||||
async findOne<T extends DocumentBase>(
|
||||
collectionName: CollectionNames,
|
||||
filter: any
|
||||
): Promise<T | null> {
|
||||
const collection = this.getCollection<T>(collectionName);
|
||||
return (await collection.findOne(filter)) as T | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate with type safety
|
||||
*/
|
||||
async aggregate<T extends DocumentBase>(
|
||||
collectionName: CollectionNames,
|
||||
pipeline: any[]
|
||||
): Promise<T[]> {
|
||||
const collection = this.getCollection<T>(collectionName);
|
||||
return await collection.aggregate<T>(pipeline).toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Count documents
|
||||
*/
|
||||
async countDocuments(collectionName: CollectionNames, filter: any = {}): Promise<number> {
|
||||
const collection = this.getCollection(collectionName);
|
||||
return await collection.countDocuments(filter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create indexes for better performance
|
||||
*/
|
||||
async createIndexes(): Promise<void> {
|
||||
if (!this.db) {
|
||||
throw new Error('MongoDB client not connected');
|
||||
}
|
||||
|
||||
try {
|
||||
// Sentiment data indexes
|
||||
await this.db
|
||||
.collection('sentiment_data')
|
||||
.createIndexes([
|
||||
{ key: { symbol: 1, timestamp: -1 } },
|
||||
{ key: { sentiment_label: 1 } },
|
||||
{ key: { source_type: 1 } },
|
||||
{ key: { created_at: -1 } },
|
||||
]);
|
||||
|
||||
// News articles indexes
|
||||
await this.db
|
||||
.collection('news_articles')
|
||||
.createIndexes([
|
||||
{ key: { symbols: 1, published_date: -1 } },
|
||||
{ key: { publication: 1 } },
|
||||
{ key: { categories: 1 } },
|
||||
{ key: { created_at: -1 } },
|
||||
]);
|
||||
|
||||
// SEC filings indexes
|
||||
await this.db
|
||||
.collection('sec_filings')
|
||||
.createIndexes([
|
||||
{ key: { symbols: 1, filing_date: -1 } },
|
||||
{ key: { filing_type: 1 } },
|
||||
{ key: { cik: 1 } },
|
||||
{ key: { created_at: -1 } },
|
||||
]); // Raw documents indexes
|
||||
await this.db.collection('raw_documents').createIndex({ content_hash: 1 }, { unique: true });
|
||||
await this.db
|
||||
.collection('raw_documents')
|
||||
.createIndexes([
|
||||
{ key: { processing_status: 1 } },
|
||||
{ key: { document_type: 1 } },
|
||||
{ key: { created_at: -1 } },
|
||||
]);
|
||||
|
||||
this.logger.info('MongoDB indexes created successfully');
|
||||
} catch (error) {
|
||||
this.logger.error('Error creating MongoDB indexes:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get database statistics
|
||||
*/
|
||||
async getStats(): Promise<any> {
|
||||
if (!this.db) {
|
||||
throw new Error('MongoDB client not connected');
|
||||
}
|
||||
return await this.db.stats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if client is connected
|
||||
*/
|
||||
|
|
@ -302,13 +251,6 @@ export class MongoDBClient {
|
|||
return this.isConnected && !!this.client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the underlying MongoDB client
|
||||
*/
|
||||
get mongoClient(): MongoClient | null {
|
||||
return this.client;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the database instance
|
||||
*/
|
||||
|
|
@ -316,81 +258,24 @@ export class MongoDBClient {
|
|||
return this.db;
|
||||
}
|
||||
|
||||
private buildConfig(config?: Partial<MongoDBClientConfig>): MongoDBClientConfig {
|
||||
return {
|
||||
host: config?.host || mongodbConfig.MONGODB_HOST,
|
||||
port: config?.port || mongodbConfig.MONGODB_PORT,
|
||||
database: config?.database || mongodbConfig.MONGODB_DATABASE,
|
||||
username: config?.username || mongodbConfig.MONGODB_USERNAME,
|
||||
password: config?.password || mongodbConfig.MONGODB_PASSWORD,
|
||||
authSource: config?.authSource || mongodbConfig.MONGODB_AUTH_SOURCE,
|
||||
uri: config?.uri || mongodbConfig.MONGODB_URI,
|
||||
poolSettings: {
|
||||
maxPoolSize: mongodbConfig.MONGODB_MAX_POOL_SIZE,
|
||||
minPoolSize: mongodbConfig.MONGODB_MIN_POOL_SIZE,
|
||||
maxIdleTime: mongodbConfig.MONGODB_MAX_IDLE_TIME,
|
||||
...config?.poolSettings,
|
||||
},
|
||||
timeouts: {
|
||||
connectTimeout: mongodbConfig.MONGODB_CONNECT_TIMEOUT,
|
||||
socketTimeout: mongodbConfig.MONGODB_SOCKET_TIMEOUT,
|
||||
serverSelectionTimeout: mongodbConfig.MONGODB_SERVER_SELECTION_TIMEOUT,
|
||||
...config?.timeouts,
|
||||
},
|
||||
tls: {
|
||||
enabled: mongodbConfig.MONGODB_TLS,
|
||||
insecure: mongodbConfig.MONGODB_TLS_INSECURE,
|
||||
caFile: mongodbConfig.MONGODB_TLS_CA_FILE,
|
||||
...config?.tls,
|
||||
},
|
||||
options: {
|
||||
retryWrites: mongodbConfig.MONGODB_RETRY_WRITES,
|
||||
journal: mongodbConfig.MONGODB_JOURNAL,
|
||||
readPreference: mongodbConfig.MONGODB_READ_PREFERENCE as any,
|
||||
writeConcern: mongodbConfig.MONGODB_WRITE_CONCERN,
|
||||
...config?.options,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private buildConnectionUri(): string {
|
||||
if (this.config.uri) {
|
||||
return this.config.uri;
|
||||
if (mongodbConfig.MONGODB_URI) {
|
||||
return mongodbConfig.MONGODB_URI;
|
||||
}
|
||||
|
||||
const { host, port, username, password, database, authSource } = this.config;
|
||||
const {
|
||||
MONGODB_HOST: host,
|
||||
MONGODB_PORT: port,
|
||||
MONGODB_USERNAME: username,
|
||||
MONGODB_PASSWORD: password,
|
||||
MONGODB_DATABASE: database,
|
||||
MONGODB_AUTH_SOURCE: authSource,
|
||||
} = mongodbConfig;
|
||||
|
||||
// Build URI components
|
||||
const auth = username && password ? `${username}:${password}@` : '';
|
||||
const authDb = authSource ? `?authSource=${authSource}` : '';
|
||||
const authParam = authSource && username ? `?authSource=${authSource}` : '';
|
||||
|
||||
return `mongodb://${auth}${host}:${port}/${database}${authDb}`;
|
||||
}
|
||||
|
||||
private buildClientOptions(): MongoClientOptions {
|
||||
return {
|
||||
maxPoolSize: this.config.poolSettings?.maxPoolSize,
|
||||
minPoolSize: this.config.poolSettings?.minPoolSize,
|
||||
maxIdleTimeMS: this.config.poolSettings?.maxIdleTime,
|
||||
connectTimeoutMS: this.config.timeouts?.connectTimeout,
|
||||
socketTimeoutMS: this.config.timeouts?.socketTimeout,
|
||||
serverSelectionTimeoutMS: this.config.timeouts?.serverSelectionTimeout,
|
||||
retryWrites: this.config.options?.retryWrites,
|
||||
journal: this.config.options?.journal,
|
||||
readPreference: this.config.options?.readPreference,
|
||||
writeConcern: this.config.options?.writeConcern
|
||||
? {
|
||||
w:
|
||||
this.config.options.writeConcern === 'majority'
|
||||
? ('majority' as const)
|
||||
: parseInt(this.config.options.writeConcern, 10) || 1,
|
||||
}
|
||||
: undefined,
|
||||
tls: this.config.tls?.enabled,
|
||||
tlsInsecure: this.config.tls?.insecure,
|
||||
tlsCAFile: this.config.tls?.caFile,
|
||||
};
|
||||
}
|
||||
|
||||
private delay(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
return `mongodb://${auth}${host}:${port}/${database}${authParam}`;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,56 +1,19 @@
|
|||
import { mongodbConfig } from '@stock-bot/config';
|
||||
import { MongoDBClient } from './client';
|
||||
import type { MongoDBClientConfig, MongoDBConnectionOptions } from './types';
|
||||
|
||||
/**
|
||||
* Factory function to create a MongoDB client instance
|
||||
*/
|
||||
export function createMongoDBClient(
|
||||
config?: Partial<MongoDBClientConfig>,
|
||||
options?: MongoDBConnectionOptions
|
||||
): MongoDBClient {
|
||||
return new MongoDBClient(config, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a MongoDB client with default configuration
|
||||
*/
|
||||
export function createDefaultMongoDBClient(): MongoDBClient {
|
||||
const config: Partial<MongoDBClientConfig> = {
|
||||
host: mongodbConfig.MONGODB_HOST,
|
||||
port: mongodbConfig.MONGODB_PORT,
|
||||
database: mongodbConfig.MONGODB_DATABASE,
|
||||
username: mongodbConfig.MONGODB_USERNAME,
|
||||
password: mongodbConfig.MONGODB_PASSWORD,
|
||||
uri: mongodbConfig.MONGODB_URI,
|
||||
};
|
||||
|
||||
return new MongoDBClient(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Singleton MongoDB client instance
|
||||
*/
|
||||
let defaultClient: MongoDBClient | null = null;
|
||||
|
||||
/**
|
||||
* Get or create the default MongoDB client instance
|
||||
* Get the singleton MongoDB client instance
|
||||
*/
|
||||
export function getMongoDBClient(): MongoDBClient {
|
||||
if (!defaultClient) {
|
||||
defaultClient = createDefaultMongoDBClient();
|
||||
}
|
||||
return defaultClient;
|
||||
return MongoDBClient.getInstance();
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to MongoDB using the default client
|
||||
* Connect to MongoDB using the singleton client
|
||||
*/
|
||||
export async function connectMongoDB(): Promise<MongoDBClient> {
|
||||
const client = getMongoDBClient();
|
||||
if (!client.connected) {
|
||||
await client.connect();
|
||||
await client.createIndexes();
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
|
@ -59,8 +22,8 @@ export async function connectMongoDB(): Promise<MongoDBClient> {
|
|||
* Disconnect from MongoDB
|
||||
*/
|
||||
export async function disconnectMongoDB(): Promise<void> {
|
||||
if (defaultClient) {
|
||||
await defaultClient.disconnect();
|
||||
defaultClient = null;
|
||||
const client = getMongoDBClient();
|
||||
if (client.connected) {
|
||||
await client.disconnect();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,233 +0,0 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { MongoDBClient } from './client';
|
||||
import type { MongoDBHealthCheck, MongoDBHealthStatus, MongoDBMetrics } from './types';
|
||||
|
||||
/**
|
||||
* MongoDB Health Monitor
|
||||
*
|
||||
* Monitors MongoDB connection health and provides metrics
|
||||
*/
|
||||
export class MongoDBHealthMonitor {
|
||||
private readonly client: MongoDBClient;
|
||||
private readonly logger: ReturnType<typeof getLogger>;
|
||||
private healthCheckInterval: NodeJS.Timeout | null = null;
|
||||
private metrics: MongoDBMetrics;
|
||||
private lastHealthCheck: MongoDBHealthCheck | null = null;
|
||||
|
||||
constructor(client: MongoDBClient) {
|
||||
this.client = client;
|
||||
this.logger = getLogger('mongodb-health-monitor');
|
||||
this.metrics = {
|
||||
operationsPerSecond: 0,
|
||||
averageLatency: 0,
|
||||
errorRate: 0,
|
||||
connectionPoolUtilization: 0,
|
||||
documentsProcessed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start health monitoring
|
||||
*/
|
||||
start(intervalMs: number = 30000): void {
|
||||
if (this.healthCheckInterval) {
|
||||
this.stop();
|
||||
}
|
||||
|
||||
this.logger.info(`Starting MongoDB health monitoring (interval: ${intervalMs}ms)`);
|
||||
|
||||
this.healthCheckInterval = setInterval(async () => {
|
||||
try {
|
||||
await this.performHealthCheck();
|
||||
} catch (error) {
|
||||
this.logger.error('Health check failed:', error);
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
// Perform initial health check
|
||||
this.performHealthCheck().catch(error => {
|
||||
this.logger.error('Initial health check failed:', error);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop health monitoring
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.healthCheckInterval) {
|
||||
clearInterval(this.healthCheckInterval);
|
||||
this.healthCheckInterval = null;
|
||||
this.logger.info('Stopped MongoDB health monitoring');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current health status
|
||||
*/
|
||||
async getHealth(): Promise<MongoDBHealthCheck> {
|
||||
if (!this.lastHealthCheck) {
|
||||
await this.performHealthCheck();
|
||||
}
|
||||
return this.lastHealthCheck!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current metrics
|
||||
*/
|
||||
getMetrics(): MongoDBMetrics {
|
||||
return { ...this.metrics };
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a health check
|
||||
*/
|
||||
private async performHealthCheck(): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
const errors: string[] = [];
|
||||
let status: MongoDBHealthStatus = 'healthy';
|
||||
|
||||
try {
|
||||
if (!this.client.connected) {
|
||||
errors.push('MongoDB client not connected');
|
||||
status = 'unhealthy';
|
||||
} else {
|
||||
// Test basic connectivity
|
||||
const mongoClient = this.client.mongoClient;
|
||||
const db = this.client.database;
|
||||
|
||||
if (!mongoClient || !db) {
|
||||
errors.push('MongoDB client or database not available');
|
||||
status = 'unhealthy';
|
||||
} else {
|
||||
// Ping the database
|
||||
await db.admin().ping();
|
||||
|
||||
// Get server status for metrics
|
||||
try {
|
||||
const serverStatus = await db.admin().serverStatus();
|
||||
this.updateMetricsFromServerStatus(serverStatus);
|
||||
|
||||
// Check connection pool status
|
||||
const poolStats = this.getConnectionPoolStats(serverStatus);
|
||||
|
||||
if (poolStats.utilization > 0.9) {
|
||||
errors.push('High connection pool utilization');
|
||||
status = status === 'healthy' ? 'degraded' : status;
|
||||
}
|
||||
|
||||
// Check for high latency
|
||||
const latency = Date.now() - startTime;
|
||||
if (latency > 1000) {
|
||||
errors.push(`High latency: ${latency}ms`);
|
||||
status = status === 'healthy' ? 'degraded' : status;
|
||||
}
|
||||
} catch (statusError) {
|
||||
errors.push(`Failed to get server status: ${(statusError as Error).message}`);
|
||||
status = 'degraded';
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push(`Health check failed: ${(error as Error).message}`);
|
||||
status = 'unhealthy';
|
||||
}
|
||||
|
||||
const latency = Date.now() - startTime;
|
||||
|
||||
// Get connection stats
|
||||
const connectionStats = this.getConnectionStats();
|
||||
|
||||
this.lastHealthCheck = {
|
||||
status,
|
||||
timestamp: new Date(),
|
||||
latency,
|
||||
connections: connectionStats,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
};
|
||||
|
||||
// Log health status changes
|
||||
if (status !== 'healthy') {
|
||||
this.logger.warn(`MongoDB health status: ${status}`, { errors, latency });
|
||||
} else {
|
||||
this.logger.debug(`MongoDB health check passed (${latency}ms)`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update metrics from MongoDB server status
|
||||
*/
|
||||
private updateMetricsFromServerStatus(serverStatus: any): void {
|
||||
try {
|
||||
const opcounters = serverStatus.opcounters || {};
|
||||
const connections = serverStatus.connections || {};
|
||||
const dur = serverStatus.dur || {};
|
||||
|
||||
// Calculate operations per second (approximate)
|
||||
const totalOps = Object.values(opcounters).reduce(
|
||||
(sum: number, count: any) => sum + (count || 0),
|
||||
0
|
||||
);
|
||||
this.metrics.operationsPerSecond = totalOps;
|
||||
|
||||
// Connection pool utilization
|
||||
if (connections.current && connections.available) {
|
||||
const total = connections.current + connections.available;
|
||||
this.metrics.connectionPoolUtilization = connections.current / total;
|
||||
}
|
||||
|
||||
// Average latency (from durability stats if available)
|
||||
if (dur.timeMS) {
|
||||
this.metrics.averageLatency = dur.timeMS.dt || 0;
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.debug('Error parsing server status for metrics:', error as any);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get connection pool statistics
|
||||
*/
|
||||
private getConnectionPoolStats(serverStatus: any): {
|
||||
utilization: number;
|
||||
active: number;
|
||||
available: number;
|
||||
} {
|
||||
const connections = serverStatus.connections || {};
|
||||
const active = connections.current || 0;
|
||||
const available = connections.available || 0;
|
||||
const total = active + available;
|
||||
|
||||
return {
|
||||
utilization: total > 0 ? active / total : 0,
|
||||
active,
|
||||
available,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get connection statistics
|
||||
*/
|
||||
private getConnectionStats(): { active: number; available: number; total: number } {
|
||||
// This would ideally come from the MongoDB driver's connection pool
|
||||
// For now, we'll return estimated values
|
||||
return {
|
||||
active: 1,
|
||||
available: 9,
|
||||
total: 10,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update error rate metric
|
||||
*/
|
||||
updateErrorRate(errorCount: number, totalOperations: number): void {
|
||||
this.metrics.errorRate = totalOperations > 0 ? errorCount / totalOperations : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update documents processed metric
|
||||
*/
|
||||
updateDocumentsProcessed(count: number): void {
|
||||
this.metrics.documentsProcessed += count;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,40 +1,22 @@
|
|||
/**
|
||||
* MongoDB Client Library for Stock Bot
|
||||
* Simplified MongoDB Client Library for Stock Bot Data Service
|
||||
*
|
||||
* Provides type-safe MongoDB access for document storage, sentiment data,
|
||||
* and raw content processing.
|
||||
* Provides a singleton MongoDB client focused on batch upsert operations
|
||||
* for high-performance data ingestion.
|
||||
*/
|
||||
|
||||
export { MongoDBClient } from './client';
|
||||
export { MongoDBHealthMonitor } from './health';
|
||||
export { MongoDBTransactionManager } from './transactions';
|
||||
export { MongoDBAggregationBuilder } from './aggregation';
|
||||
|
||||
// Types
|
||||
export type {
|
||||
MongoDBClientConfig,
|
||||
MongoDBConnectionOptions,
|
||||
MongoDBHealthStatus,
|
||||
MongoDBMetrics,
|
||||
CollectionNames,
|
||||
DocumentBase,
|
||||
SentimentData,
|
||||
RawDocument,
|
||||
NewsArticle,
|
||||
SecFiling,
|
||||
EarningsTranscript,
|
||||
AnalystReport,
|
||||
DocumentBase,
|
||||
EarningsTranscript,
|
||||
NewsArticle,
|
||||
RawDocument,
|
||||
SecFiling,
|
||||
SentimentData,
|
||||
} from './types';
|
||||
|
||||
// Schemas
|
||||
export {
|
||||
sentimentDataSchema,
|
||||
rawDocumentSchema,
|
||||
newsArticleSchema,
|
||||
secFilingSchema,
|
||||
earningsTranscriptSchema,
|
||||
analystReportSchema,
|
||||
} from './schemas';
|
||||
|
||||
// Utils
|
||||
export { createMongoDBClient } from './factory';
|
||||
// Factory functions
|
||||
export { connectMongoDB, disconnectMongoDB, getMongoDBClient } from './factory';
|
||||
|
|
|
|||
|
|
@ -1,146 +0,0 @@
|
|||
import * as yup from 'yup';
|
||||
|
||||
/**
|
||||
* Yup Schemas for MongoDB Document Validation
|
||||
*/
|
||||
|
||||
// Base schema for all documents
|
||||
export const documentBaseSchema = yup.object({
|
||||
_id: yup.mixed().optional(),
|
||||
created_at: yup.date().required(),
|
||||
updated_at: yup.date().required(),
|
||||
source: yup.string().required(),
|
||||
metadata: yup.object().optional(),
|
||||
});
|
||||
|
||||
// Sentiment Data Schema
|
||||
export const sentimentDataSchema = documentBaseSchema.shape({
|
||||
symbol: yup.string().min(1).max(10).required(),
|
||||
sentiment_score: yup.number().min(-1).max(1).required(),
|
||||
sentiment_label: yup.string().oneOf(['positive', 'negative', 'neutral']).required(),
|
||||
confidence: yup.number().min(0).max(1).required(),
|
||||
text: yup.string().min(1).required(),
|
||||
source_type: yup.string().oneOf(['reddit', 'twitter', 'news', 'forums']).required(),
|
||||
source_id: yup.string().required(),
|
||||
timestamp: yup.date().required(),
|
||||
processed_at: yup.date().required(),
|
||||
language: yup.string().default('en'),
|
||||
keywords: yup.array(yup.string()).required(),
|
||||
entities: yup
|
||||
.array(
|
||||
yup.object({
|
||||
name: yup.string().required(),
|
||||
type: yup.string().required(),
|
||||
confidence: yup.number().min(0).max(1).required(),
|
||||
})
|
||||
)
|
||||
.required(),
|
||||
});
|
||||
|
||||
// Raw Document Schema
|
||||
export const rawDocumentSchema = documentBaseSchema.shape({
|
||||
document_type: yup.string().oneOf(['html', 'pdf', 'text', 'json', 'xml']).required(),
|
||||
content: yup.string().required(),
|
||||
content_hash: yup.string().required(),
|
||||
url: yup.string().url().optional(),
|
||||
title: yup.string().optional(),
|
||||
author: yup.string().optional(),
|
||||
published_date: yup.date().optional(),
|
||||
extracted_text: yup.string().optional(),
|
||||
processing_status: yup.string().oneOf(['pending', 'processed', 'failed']).required(),
|
||||
size_bytes: yup.number().positive().required(),
|
||||
language: yup.string().optional(),
|
||||
});
|
||||
|
||||
// News Article Schema
|
||||
export const newsArticleSchema = documentBaseSchema.shape({
|
||||
headline: yup.string().min(1).required(),
|
||||
content: yup.string().min(1).required(),
|
||||
summary: yup.string().optional(),
|
||||
author: yup.string().required(),
|
||||
publication: yup.string().required(),
|
||||
published_date: yup.date().required(),
|
||||
url: yup.string().url().required(),
|
||||
symbols: yup.array(yup.string()).required(),
|
||||
categories: yup.array(yup.string()).required(),
|
||||
sentiment_score: yup.number().min(-1).max(1).optional(),
|
||||
relevance_score: yup.number().min(0).max(1).optional(),
|
||||
image_url: yup.string().url().optional(),
|
||||
tags: yup.array(yup.string()).required(),
|
||||
});
|
||||
|
||||
// SEC Filing Schema
|
||||
export const secFilingSchema = documentBaseSchema.shape({
|
||||
cik: yup.string().required(),
|
||||
accession_number: yup.string().required(),
|
||||
filing_type: yup.string().required(),
|
||||
company_name: yup.string().required(),
|
||||
symbols: yup.array(yup.string()).required(),
|
||||
filing_date: yup.date().required(),
|
||||
period_end_date: yup.date().required(),
|
||||
url: yup.string().url().required(),
|
||||
content: yup.string().required(),
|
||||
extracted_data: yup.object().optional(),
|
||||
financial_statements: yup
|
||||
.array(
|
||||
yup.object({
|
||||
statement_type: yup.string().required(),
|
||||
data: yup.object().required(),
|
||||
})
|
||||
)
|
||||
.optional(),
|
||||
processing_status: yup.string().oneOf(['pending', 'processed', 'failed']).required(),
|
||||
});
|
||||
|
||||
// Earnings Transcript Schema
|
||||
export const earningsTranscriptSchema = documentBaseSchema.shape({
|
||||
symbol: yup.string().min(1).max(10).required(),
|
||||
company_name: yup.string().required(),
|
||||
quarter: yup.string().required(),
|
||||
year: yup.number().min(2000).max(3000).required(),
|
||||
call_date: yup.date().required(),
|
||||
transcript: yup.string().required(),
|
||||
participants: yup
|
||||
.array(
|
||||
yup.object({
|
||||
name: yup.string().required(),
|
||||
title: yup.string().required(),
|
||||
type: yup.string().oneOf(['executive', 'analyst']).required(),
|
||||
})
|
||||
)
|
||||
.required(),
|
||||
key_topics: yup.array(yup.string()).required(),
|
||||
sentiment_analysis: yup
|
||||
.object({
|
||||
overall_sentiment: yup.number().min(-1).max(1).required(),
|
||||
topic_sentiments: yup.object().required(),
|
||||
})
|
||||
.optional(),
|
||||
financial_highlights: yup.object().optional(),
|
||||
});
|
||||
|
||||
// Analyst Report Schema
|
||||
export const analystReportSchema = documentBaseSchema.shape({
|
||||
symbol: yup.string().min(1).max(10).required(),
|
||||
analyst_firm: yup.string().required(),
|
||||
analyst_name: yup.string().required(),
|
||||
report_title: yup.string().required(),
|
||||
report_date: yup.date().required(),
|
||||
rating: yup.string().oneOf(['buy', 'hold', 'sell', 'strong_buy', 'strong_sell']).required(),
|
||||
price_target: yup.number().positive().optional(),
|
||||
previous_rating: yup.string().optional(),
|
||||
content: yup.string().required(),
|
||||
summary: yup.string().required(),
|
||||
key_points: yup.array(yup.string()).required(),
|
||||
financial_projections: yup.object().optional(),
|
||||
});
|
||||
|
||||
// Schema mapping for collections
|
||||
export const schemaMap = {
|
||||
sentiment_data: sentimentDataSchema,
|
||||
raw_documents: rawDocumentSchema,
|
||||
news_articles: newsArticleSchema,
|
||||
sec_filings: secFilingSchema,
|
||||
earnings_transcripts: earningsTranscriptSchema,
|
||||
analyst_reports: analystReportSchema,
|
||||
} as const;
|
||||
|
|
@ -1,238 +0,0 @@
|
|||
import type { OptionalUnlessRequiredId, WithId } from 'mongodb';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { MongoDBClient } from './client';
|
||||
import type { CollectionNames, DocumentBase } from './types';
|
||||
|
||||
/**
|
||||
* MongoDB Transaction Manager
|
||||
*
|
||||
* Provides transaction support for multi-document operations
|
||||
*/
|
||||
export class MongoDBTransactionManager {
|
||||
private readonly client: MongoDBClient;
|
||||
private readonly logger: ReturnType<typeof getLogger>;
|
||||
|
||||
constructor(client: MongoDBClient) {
|
||||
this.client = client;
|
||||
this.logger = getLogger('mongodb-transaction-manager');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute operations within a transaction
|
||||
*/
|
||||
async withTransaction<T>(
|
||||
operations: (session: any) => Promise<T>,
|
||||
options?: {
|
||||
readPreference?: string;
|
||||
readConcern?: string;
|
||||
writeConcern?: any;
|
||||
maxCommitTimeMS?: number;
|
||||
}
|
||||
): Promise<T> {
|
||||
const mongoClient = this.client.mongoClient;
|
||||
if (!mongoClient) {
|
||||
throw new Error('MongoDB client not connected');
|
||||
}
|
||||
|
||||
const session = mongoClient.startSession();
|
||||
|
||||
try {
|
||||
this.logger.debug('Starting MongoDB transaction');
|
||||
|
||||
const result = await session.withTransaction(
|
||||
async () => {
|
||||
return await operations(session);
|
||||
},
|
||||
{
|
||||
readPreference: options?.readPreference as any,
|
||||
readConcern: { level: options?.readConcern || 'majority' } as any,
|
||||
writeConcern: options?.writeConcern || { w: 'majority' },
|
||||
maxCommitTimeMS: options?.maxCommitTimeMS || 10000,
|
||||
}
|
||||
);
|
||||
|
||||
this.logger.debug('MongoDB transaction completed successfully');
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error('MongoDB transaction failed:', error);
|
||||
throw error;
|
||||
} finally {
|
||||
await session.endSession();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch insert documents across collections within a transaction
|
||||
*/
|
||||
async batchInsert(
|
||||
operations: Array<{
|
||||
collection: CollectionNames;
|
||||
documents: DocumentBase[];
|
||||
}>,
|
||||
options?: { ordered?: boolean; bypassDocumentValidation?: boolean }
|
||||
): Promise<void> {
|
||||
await this.withTransaction(async session => {
|
||||
for (const operation of operations) {
|
||||
const collection = this.client.getCollection(operation.collection);
|
||||
|
||||
// Add timestamps to all documents
|
||||
const now = new Date();
|
||||
const documentsWithTimestamps = operation.documents.map(doc => ({
|
||||
...doc,
|
||||
created_at: doc.created_at || now,
|
||||
updated_at: now,
|
||||
}));
|
||||
|
||||
await collection.insertMany(documentsWithTimestamps, {
|
||||
session,
|
||||
ordered: options?.ordered ?? true,
|
||||
bypassDocumentValidation: options?.bypassDocumentValidation ?? false,
|
||||
});
|
||||
|
||||
this.logger.debug(
|
||||
`Inserted ${documentsWithTimestamps.length} documents into ${operation.collection}`
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch update documents across collections within a transaction
|
||||
*/
|
||||
async batchUpdate(
|
||||
operations: Array<{
|
||||
collection: CollectionNames;
|
||||
filter: any;
|
||||
update: any;
|
||||
options?: any;
|
||||
}>
|
||||
): Promise<void> {
|
||||
await this.withTransaction(async session => {
|
||||
const results = [];
|
||||
|
||||
for (const operation of operations) {
|
||||
const collection = this.client.getCollection(operation.collection);
|
||||
|
||||
// Add updated timestamp
|
||||
const updateWithTimestamp = {
|
||||
...operation.update,
|
||||
$set: {
|
||||
...operation.update.$set,
|
||||
updated_at: new Date(),
|
||||
},
|
||||
};
|
||||
|
||||
const result = await collection.updateMany(operation.filter, updateWithTimestamp, {
|
||||
session,
|
||||
...operation.options,
|
||||
});
|
||||
|
||||
results.push(result);
|
||||
this.logger.debug(`Updated ${result.modifiedCount} documents in ${operation.collection}`);
|
||||
}
|
||||
|
||||
return results;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Move documents between collections within a transaction
|
||||
*/
|
||||
async moveDocuments<T extends DocumentBase>(
|
||||
fromCollection: CollectionNames,
|
||||
toCollection: CollectionNames,
|
||||
filter: any,
|
||||
transform?: (doc: T) => T
|
||||
): Promise<number> {
|
||||
return await this.withTransaction(async session => {
|
||||
const sourceCollection = this.client.getCollection<T>(fromCollection);
|
||||
const targetCollection = this.client.getCollection<T>(toCollection);
|
||||
|
||||
// Find documents to move
|
||||
const documents = await sourceCollection.find(filter, { session }).toArray();
|
||||
|
||||
if (documents.length === 0) {
|
||||
return 0;
|
||||
} // Transform documents if needed
|
||||
const documentsToInsert = transform
|
||||
? documents.map((doc: WithId<T>) => transform(doc as T))
|
||||
: documents;
|
||||
|
||||
// Add updated timestamp
|
||||
const now = new Date();
|
||||
documentsToInsert.forEach(doc => {
|
||||
doc.updated_at = now;
|
||||
}); // Insert into target collection
|
||||
await targetCollection.insertMany(documentsToInsert as OptionalUnlessRequiredId<T>[], {
|
||||
session,
|
||||
});
|
||||
|
||||
// Remove from source collection
|
||||
const deleteResult = await sourceCollection.deleteMany(filter, { session });
|
||||
|
||||
this.logger.info(
|
||||
`Moved ${documents.length} documents from ${fromCollection} to ${toCollection}`
|
||||
);
|
||||
|
||||
return deleteResult.deletedCount || 0;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Archive old documents within a transaction
|
||||
*/
|
||||
async archiveDocuments(
|
||||
sourceCollection: CollectionNames,
|
||||
archiveCollection: CollectionNames,
|
||||
cutoffDate: Date,
|
||||
batchSize: number = 1000
|
||||
): Promise<number> {
|
||||
let totalArchived = 0;
|
||||
|
||||
while (true) {
|
||||
const batchArchived = await this.withTransaction(async session => {
|
||||
const collection = this.client.getCollection(sourceCollection);
|
||||
const archiveCol = this.client.getCollection(archiveCollection);
|
||||
|
||||
// Find old documents
|
||||
const documents = await collection
|
||||
.find({ created_at: { $lt: cutoffDate } }, { limit: batchSize, session })
|
||||
.toArray();
|
||||
|
||||
if (documents.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Add archive metadata
|
||||
const now = new Date();
|
||||
const documentsToArchive = documents.map(doc => ({
|
||||
...doc,
|
||||
archived_at: now,
|
||||
archived_from: sourceCollection,
|
||||
}));
|
||||
|
||||
// Insert into archive collection
|
||||
await archiveCol.insertMany(documentsToArchive, { session });
|
||||
|
||||
// Remove from source collection
|
||||
const ids = documents.map(doc => doc._id);
|
||||
const deleteResult = await collection.deleteMany({ _id: { $in: ids } }, { session });
|
||||
|
||||
return deleteResult.deletedCount || 0;
|
||||
});
|
||||
|
||||
totalArchived += batchArchived;
|
||||
|
||||
if (batchArchived === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
this.logger.debug(`Archived batch of ${batchArchived} documents`);
|
||||
}
|
||||
|
||||
this.logger.info(
|
||||
`Archived ${totalArchived} documents from ${sourceCollection} to ${archiveCollection}`
|
||||
);
|
||||
return totalArchived;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import type { ObjectId } from 'mongodb';
|
||||
import * as yup from 'yup';
|
||||
|
||||
/**
|
||||
* MongoDB Client Configuration
|
||||
|
|
@ -69,20 +68,6 @@ export interface MongoDBMetrics {
|
|||
documentsProcessed: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collection Names
|
||||
*/
|
||||
export type CollectionNames =
|
||||
| 'sentiment_data'
|
||||
| 'raw_documents'
|
||||
| 'news_articles'
|
||||
| 'sec_filings'
|
||||
| 'earnings_transcripts'
|
||||
| 'analyst_reports'
|
||||
| 'social_media_posts'
|
||||
| 'market_events'
|
||||
| 'economic_indicators';
|
||||
|
||||
/**
|
||||
* Base Document Interface
|
||||
*/
|
||||
|
|
|
|||
0
scripts/populate-ib-exchanges.ts
Normal file
0
scripts/populate-ib-exchanges.ts
Normal file
|
|
@ -1,166 +0,0 @@
|
|||
/**
|
||||
* Simple Browser and Network Monitoring Test
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testBasicBrowser() {
|
||||
console.log('🚀 Testing basic browser functionality...');
|
||||
|
||||
try {
|
||||
// Initialize browser
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false,
|
||||
enableNetworkLogging: true,
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Test 1: Simple page without proxy
|
||||
console.log('📄 Testing simple page without proxy...');
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/json'
|
||||
);
|
||||
|
||||
let capturedData = null;
|
||||
let eventCount = 0;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
console.log(`📡 Event ${eventCount}: ${event.type} - ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/json')) {
|
||||
console.log(` 📊 Status: ${event.status}`);
|
||||
if (event.responseData) {
|
||||
capturedData = event.responseData;
|
||||
console.log(` 📝 Response: ${event.responseData}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
|
||||
console.log(`✅ Test completed. Events captured: ${eventCount}`);
|
||||
if (capturedData) {
|
||||
console.log('✅ Successfully captured response data');
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return true;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Basic test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testProxyConnection() {
|
||||
console.log('\n🔄 Testing proxy connection...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
// Test different proxy formats
|
||||
const proxyConfigs = [
|
||||
null, // No proxy
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80',
|
||||
];
|
||||
|
||||
for (const proxy of proxyConfigs) {
|
||||
console.log(`\n🌐 Testing with proxy: ${proxy || 'No proxy'}`);
|
||||
|
||||
try {
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
proxy
|
||||
);
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
|
||||
console.log(` 📍 IP Response: ${event.responseData}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
await Browser.closeContext(contextId);
|
||||
console.log(' ✅ Success');
|
||||
|
||||
} catch (error) {
|
||||
console.log(` ❌ Failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Proxy test setup failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testIBWithWorkaround() {
|
||||
console.log('\n🏦 Testing IB endpoint with workaround...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 20000,
|
||||
blockResources: true, // Block resources for performance
|
||||
});
|
||||
|
||||
// Try without proxy first
|
||||
console.log('🌐 Attempting IB without proxy...');
|
||||
try {
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com'
|
||||
);
|
||||
|
||||
let responseCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.type === 'response') {
|
||||
responseCount++;
|
||||
console.log(` 📥 Response ${responseCount}: ${event.status} ${event.url}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
console.log(`✅ IB main page loaded. Responses: ${responseCount}`);
|
||||
await Browser.closeContext(contextId);
|
||||
|
||||
} catch (error) {
|
||||
console.log(`❌ IB without proxy failed: ${error.message}`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ IB test failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run tests
|
||||
async function runAllTests() {
|
||||
console.log('🧪 Starting Browser Network Monitoring Tests\n');
|
||||
|
||||
const basicResult = await testBasicBrowser();
|
||||
await testProxyConnection();
|
||||
await testIBWithWorkaround();
|
||||
|
||||
console.log(`\n🏁 Basic functionality: ${basicResult ? '✅ PASS' : '❌ FAIL'}`);
|
||||
console.log('✅ All tests completed!');
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runAllTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testBasicBrowser, testProxyConnection, testIBWithWorkaround };
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
import { Browser, BrowserTabManager } from './libs/browser/src';
|
||||
|
||||
async function testSimplifiedBrowser() {
|
||||
console.log('Testing simplified browser library...');
|
||||
|
||||
try {
|
||||
console.log('Initializing browser...');
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
blockResources: true,
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
// Test single page with proxy support
|
||||
console.log('Testing page creation...');
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/json',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
const content = await Browser.evaluate(page, () => document.body.textContent);
|
||||
console.log('Page content:', content?.substring(0, 100) + '...');
|
||||
|
||||
// Test tab manager (no longer needs browser instance)
|
||||
console.log('Testing tab manager...');
|
||||
const tabManager = new BrowserTabManager();
|
||||
|
||||
// Test multiple URL scraping with different proxies
|
||||
const urlProxyPairs = [
|
||||
{ url: 'https://httpbin.org/uuid', proxy: '' }, // No proxy
|
||||
{ url: 'https://httpbin.org/ip', proxy: '' }, // No proxy
|
||||
];
|
||||
|
||||
const results = await tabManager.scrapeUrlsWithProxies(
|
||||
urlProxyPairs,
|
||||
async page => {
|
||||
const text = await page.textContent('body');
|
||||
return { content: text?.substring(0, 50) };
|
||||
},
|
||||
{ concurrency: 2 }
|
||||
);
|
||||
|
||||
console.log('Scraping results:');
|
||||
results.forEach((result, index) => {
|
||||
console.log(` ${index + 1}. ${result.url}: ${result.success ? 'SUCCESS' : 'FAILED'}`);
|
||||
if (result.data) {
|
||||
console.log(` Data: ${result.data.content}...`);
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up
|
||||
await page.close();
|
||||
await Browser.closeContext(contextId);
|
||||
await Browser.close();
|
||||
|
||||
console.log('✅ Simplified browser test completed successfully!');
|
||||
} catch (error) {
|
||||
console.error('❌ Browser test failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
testSimplifiedBrowser();
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testWithoutProxy() {
|
||||
console.log('🔬 Testing WITHOUT proxy...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({ headless: true, timeout: 15000, blockResources: false });
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/'
|
||||
// No proxy parameter
|
||||
);
|
||||
console.log('✅ Page created without proxy');
|
||||
|
||||
let eventCount = 0;
|
||||
let summaryData: SummaryResponse | null = null;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
|
||||
// Capture the summary API response
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.responseData) {
|
||||
console.log(`📊 Summary API Response Data: ${event.responseData}`);
|
||||
try {
|
||||
summaryData = JSON.parse(event.responseData) as any;
|
||||
const totalCount = summaryData[0].totalCount;
|
||||
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
|
||||
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
|
||||
} catch (e) {
|
||||
console.log('📊 Raw Summary Response:', event.responseData);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Uncomment to see all network events
|
||||
// console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
|
||||
});
|
||||
|
||||
console.log('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 15000 });
|
||||
console.log('✅ Page loaded');
|
||||
|
||||
// Complete interaction flow
|
||||
try {
|
||||
console.log('🔍 Looking for Products tab...');
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Products tab');
|
||||
|
||||
console.log('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
console.log('✅ Products tab clicked');
|
||||
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
console.log('🔍 Looking for Asset Classes accordion...');
|
||||
const assetClassesAccordion = page.locator(
|
||||
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
|
||||
);
|
||||
await assetClassesAccordion.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Asset Classes accordion');
|
||||
|
||||
console.log('🖱️ Clicking Asset Classes accordion...');
|
||||
await assetClassesAccordion.click();
|
||||
console.log('✅ Asset Classes accordion clicked');
|
||||
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
console.log('🔍 Looking for Stocks checkbox...');
|
||||
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
|
||||
await stocksSpan.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Stocks span');
|
||||
|
||||
const parentContainer = stocksSpan.locator('..');
|
||||
const checkbox = parentContainer.locator('input[type="checkbox"]');
|
||||
|
||||
if ((await checkbox.count()) > 0) {
|
||||
console.log('📋 Clicking Stocks checkbox...');
|
||||
await checkbox.first().check();
|
||||
console.log('✅ Stocks checkbox checked');
|
||||
} else {
|
||||
console.log('⚠️ Could not find checkbox near Stocks text');
|
||||
}
|
||||
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
console.log('🔍 Looking for Apply button...');
|
||||
const applyButton = page.locator(
|
||||
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
|
||||
);
|
||||
|
||||
if ((await applyButton.count()) > 0) {
|
||||
console.log('🎯 Clicking Apply button...');
|
||||
await applyButton.first().click();
|
||||
console.log('✅ Apply button clicked');
|
||||
await page.waitForTimeout(3000);
|
||||
} else {
|
||||
console.log('⚠️ Could not find Apply button');
|
||||
}
|
||||
} catch (interactionError) {
|
||||
const errorMessage =
|
||||
interactionError instanceof Error ? interactionError.message : String(interactionError);
|
||||
console.error('❌ Page interaction failed:', errorMessage);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
console.log(`📊 Total events captured: ${eventCount}`);
|
||||
|
||||
// Show final results
|
||||
if (summaryData) {
|
||||
console.log('✅ SUCCESS: Captured summary data!');
|
||||
console.log(`🔢 Final total count: ${summaryData?.data?.totalCount || 'Unknown'}`);
|
||||
console.log(`📋 Data keys: ${Object.keys(summaryData).join(', ')}`);
|
||||
} else {
|
||||
console.log('❌ No summary data captured');
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
await Browser.close();
|
||||
|
||||
console.log('✅ Test completed successfully');
|
||||
return true;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error('❌ Error:', errorMessage);
|
||||
await Browser.close();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
testWithoutProxy().then(success => {
|
||||
console.log(`🏁 Final result: ${success ? 'SUCCESS' : 'FAILED'}`);
|
||||
});
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
/**
|
||||
* Working Interactive Brokers test with verified network monitoring
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testIBWithWorking() {
|
||||
console.log('🏦 Testing IB with working network monitoring and fixed proxy auth...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 20000,
|
||||
blockResources: false, // Don't block resources initially
|
||||
});
|
||||
|
||||
// Test 1: Try a simple proxy detection service first
|
||||
console.log('🌐 Testing proxy connectivity...');
|
||||
const { page: proxyPage, contextId: proxyCtx } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let proxyEvents = 0;
|
||||
let myIP = null;
|
||||
proxyPage.onNetworkEvent(event => {
|
||||
proxyEvents++;
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
try {
|
||||
const data = JSON.parse(event.responseData);
|
||||
myIP = data.origin;
|
||||
console.log(` 📍 Proxy IP: ${myIP}`);
|
||||
} catch (e) {
|
||||
console.log(` 📊 Raw response: ${event.responseData}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await proxyPage.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
await Browser.closeContext(proxyCtx);
|
||||
|
||||
console.log(`📊 Proxy test events: ${proxyEvents}`);
|
||||
|
||||
// Test 2: Try IB API endpoint with fixed proxy auth
|
||||
console.log('🎯 Testing IB API endpoint...');
|
||||
const { page: apiPage, contextId: apiCtx } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let apiEvents = 0;
|
||||
let summaryData = null;
|
||||
apiPage.onNetworkEvent(event => {
|
||||
apiEvents++;
|
||||
console.log(` 📡 API Event: ${event.type} ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.url.includes('summary')) {
|
||||
console.log(` 🎯 Found summary response! Status: ${event.status}`);
|
||||
if (event.responseData) {
|
||||
summaryData = event.responseData;
|
||||
try {
|
||||
const data = JSON.parse(event.responseData);
|
||||
console.log(` 📊 Summary data: ${JSON.stringify(data, null, 2)}`);
|
||||
} catch (e) {
|
||||
console.log(` 📊 Raw summary: ${event.responseData.substring(0, 200)}...`);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await apiPage.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
await Browser.closeContext(apiCtx);
|
||||
|
||||
return {
|
||||
proxyEvents,
|
||||
apiEvents,
|
||||
summaryData,
|
||||
proxyIP: myIP,
|
||||
success: apiEvents > 0 || summaryData !== null,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('❌ IB test failed:', error);
|
||||
return {
|
||||
proxyEvents: 0,
|
||||
apiEvents: 0,
|
||||
summaryData: null,
|
||||
proxyIP: null,
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testWithProxyFallback() {
|
||||
console.log('\n🔄 Testing with proxy fallback strategy...');
|
||||
|
||||
const proxiesToTest = [
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80', // Your proxy
|
||||
];
|
||||
|
||||
for (const proxy of proxiesToTest) {
|
||||
console.log(`\n🌐 Testing with: ${proxy || 'No proxy'}`);
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
proxy
|
||||
);
|
||||
|
||||
let ipResponse = null;
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ipResponse = event.responseData;
|
||||
console.log(` 📍 IP: ${JSON.parse(event.responseData).origin}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(contextId);
|
||||
} catch (error) {
|
||||
console.log(` ❌ Failed: ${error.message}`);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runIBTests() {
|
||||
console.log('🚀 Interactive Brokers Network Monitoring Tests with Fixed Proxy Auth\n');
|
||||
|
||||
const result = await testIBWithWorking();
|
||||
await testWithProxyFallback();
|
||||
|
||||
console.log('\n🏁 Final Results:');
|
||||
console.log(` 🌐 Proxy events: ${result.proxyEvents || 0}`);
|
||||
console.log(` 📍 Proxy IP: ${result.proxyIP || 'Not captured'}`);
|
||||
console.log(` 🎯 API events: ${result.apiEvents || 0}`);
|
||||
console.log(` 📊 Summary data: ${result.summaryData ? 'Captured' : 'Not captured'}`);
|
||||
console.log(` ✅ Overall success: ${result.success}`);
|
||||
|
||||
if (result.error) {
|
||||
console.log(` ❌ Error: ${result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runIBTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testIBWithWorking, testWithProxyFallback };
|
||||
194
test-ib.ts
194
test-ib.ts
|
|
@ -1,194 +0,0 @@
|
|||
/**
|
||||
* Test Interactive Brokers functionality with network monitoring
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
import { getRandomProxyURL } from '@stock-bot/proxy';
|
||||
|
||||
async function testIBSymbolSummary() {
|
||||
console.log('🚀 Testing Interactive Brokers Symbol Summary with Network Monitoring...');
|
||||
|
||||
try {
|
||||
// Initialize browser
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 30000,
|
||||
blockResources: true,
|
||||
enableNetworkLogging: true,
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Get a random proxy
|
||||
|
||||
// Create page with proxy
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
console.log('📄 Page created with proxy');
|
||||
|
||||
// Set up network monitoring
|
||||
let summaryResponse: any = null;
|
||||
let requestCount = 0;
|
||||
let responseCount = 0;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
console.log(`📡 Network Event: ${event.type} - ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'request') {
|
||||
requestCount++;
|
||||
console.log(` 📤 Request #${requestCount}: ${event.method} ${event.url}`);
|
||||
|
||||
// Log request data for POST requests
|
||||
if (event.requestData) {
|
||||
console.log(` 📝 Request Data: ${event.requestData.substring(0, 200)}...`);
|
||||
}
|
||||
}
|
||||
|
||||
if (event.type === 'response') {
|
||||
responseCount++;
|
||||
console.log(` 📥 Response #${responseCount}: ${event.status} ${event.url}`);
|
||||
|
||||
// Capture the summary response
|
||||
if (event.url.includes('summary')) {
|
||||
console.log(` 🎯 Found summary response!`);
|
||||
summaryResponse = event.responseData;
|
||||
|
||||
if (event.responseData) {
|
||||
try {
|
||||
const data = JSON.parse(event.responseData);
|
||||
console.log(` 📊 Summary Data: ${JSON.stringify(data, null, 2)}`);
|
||||
} catch (e) {
|
||||
console.log(` 📊 Raw Response: ${event.responseData.substring(0, 500)}...`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (event.type === 'failed') {
|
||||
console.log(` ❌ Failed Request: ${event.url}`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log('🔍 Network monitoring set up, waiting for page to load...');
|
||||
|
||||
// Wait for page to load and capture network activity
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
console.log('✅ Page loaded');
|
||||
|
||||
// Wait a bit more for any additional network requests
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
|
||||
console.log(`📊 Network Summary:`);
|
||||
console.log(` 📤 Total Requests: ${requestCount}`);
|
||||
console.log(` 📥 Total Responses: ${responseCount}`);
|
||||
|
||||
if (summaryResponse) {
|
||||
console.log('✅ Successfully captured summary response');
|
||||
try {
|
||||
const parsed = JSON.parse(summaryResponse);
|
||||
console.log(`🔢 Total symbols found: ${parsed?.data?.totalCount || 'Unknown'}`);
|
||||
return parsed?.data?.totalCount || 0;
|
||||
} catch (e) {
|
||||
console.log('⚠️ Could not parse response as JSON');
|
||||
return 1; // Indicate success but unknown count
|
||||
}
|
||||
} else {
|
||||
console.log('❌ No summary response captured');
|
||||
return 0;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Test failed:', error);
|
||||
|
||||
// Log more details about the error
|
||||
if (error instanceof Error) {
|
||||
console.error('Error details:', {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
name: error.name
|
||||
});
|
||||
}
|
||||
|
||||
return -1;
|
||||
} finally {
|
||||
try {
|
||||
await Browser.close();
|
||||
console.log('🔒 Browser closed');
|
||||
} catch (closeError) {
|
||||
console.error('Error closing browser:', closeError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function testWithDifferentProxy() {
|
||||
console.log('\n🔄 Testing with different proxy configuration...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false, // Don't block resources for this test
|
||||
});
|
||||
|
||||
// Test without proxy first
|
||||
console.log('🌐 Testing without proxy...');
|
||||
const { page: pageNoProxy, contextId: contextNoProxy } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip'
|
||||
);
|
||||
|
||||
pageNoProxy.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
|
||||
console.log('📍 No proxy IP response:', event.responseData);
|
||||
}
|
||||
});
|
||||
|
||||
await pageNoProxy.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(contextNoProxy);
|
||||
|
||||
// Test with proxy
|
||||
console.log('🌐 Testing with proxy...');
|
||||
const { page: pageWithProxy, contextId: contextWithProxy } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
pageWithProxy.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('httpbin.org/ip')) {
|
||||
console.log('🔄 Proxy IP response:', event.responseData);
|
||||
}
|
||||
});
|
||||
|
||||
await pageWithProxy.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(contextWithProxy);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Proxy test failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Run the tests
|
||||
async function runTests() {
|
||||
console.log('🧪 Starting IB Network Monitoring Tests\n');
|
||||
|
||||
// Test 1: Main IB functionality
|
||||
const result = await testIBSymbolSummary();
|
||||
console.log(`\n🏁 Test Result: ${result}`);
|
||||
|
||||
// Test 2: Proxy verification
|
||||
await testWithDifferentProxy();
|
||||
|
||||
console.log('\n✅ All tests completed!');
|
||||
}
|
||||
|
||||
// Run if this file is executed directly
|
||||
if (import.meta.main) {
|
||||
runTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testIBSymbolSummary, testWithDifferentProxy };
|
||||
0
test-large-scale-performance.ts
Normal file
0
test-large-scale-performance.ts
Normal file
0
test-mongodb-batch.ts
Normal file
0
test-mongodb-batch.ts
Normal file
0
test-mongodb-simplified.ts
Normal file
0
test-mongodb-simplified.ts
Normal file
|
|
@ -1,135 +0,0 @@
|
|||
/**
|
||||
* Debug network monitoring setup
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function debugNetworkSetup() {
|
||||
console.log('🐛 Debugging Network Monitoring Setup...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false, // Ensure we don't block requests
|
||||
});
|
||||
|
||||
// Create page but don't navigate yet
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let eventCount = 0;
|
||||
console.log('📡 Setting up network event listener...');
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
console.log(`🔔 Event ${eventCount}: ${event.type} ${event.method} ${event.url}`);
|
||||
console.log(` Headers: ${Object.keys(event.headers || {}).length} headers`);
|
||||
|
||||
if (event.responseData) {
|
||||
console.log(` Data: ${event.responseData.substring(0, 100)}...`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log('🌐 Navigating to httpbin.org/headers...');
|
||||
await page.goto('https://httpbin.org/headers');
|
||||
|
||||
console.log('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
|
||||
console.log('⏳ Waiting additional time for network events...');
|
||||
await new Promise(resolve => setTimeout(resolve, 3000));
|
||||
|
||||
console.log(`📊 Total events captured: ${eventCount}`);
|
||||
|
||||
// Try to evaluate page content to see if it loaded
|
||||
const title = await page.title();
|
||||
console.log(`📄 Page title: "${title}"`);
|
||||
|
||||
const bodyText = await page.locator('body').textContent();
|
||||
if (bodyText) {
|
||||
console.log(`📝 Page content (first 200 chars): ${bodyText.substring(0, 200)}...`);
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return eventCount > 0;
|
||||
} catch (error) {
|
||||
console.error('❌ Debug test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testManualNetworkCall() {
|
||||
console.log('\n🔧 Testing with manual fetch call...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let eventCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
eventCount++;
|
||||
console.log(`📡 Manual test event ${eventCount}: ${event.type} ${event.method} ${event.url}`);
|
||||
if (event.responseData && event.url.includes('httpbin')) {
|
||||
console.log(` 📊 Response: ${event.responseData}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Navigate to a simple page first
|
||||
await page.goto('data:text/html,<html><body><h1>Test Page</h1></body></html>');
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
|
||||
console.log('🚀 Making manual fetch call...');
|
||||
// Make a fetch request from the page context
|
||||
const result = await page.evaluate(async () => {
|
||||
try {
|
||||
const response = await fetch('https://httpbin.org/json');
|
||||
const data = await response.json();
|
||||
return { success: true, data };
|
||||
} catch (error) {
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
});
|
||||
|
||||
console.log('📋 Fetch result:', result);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
console.log(`📊 Events from manual fetch: ${eventCount}`);
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return eventCount > 0;
|
||||
} catch (error) {
|
||||
console.error('❌ Manual test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runDebugTests() {
|
||||
console.log('🚀 Network Monitoring Debug Tests\n');
|
||||
|
||||
const setupResult = await debugNetworkSetup();
|
||||
const manualResult = await testManualNetworkCall();
|
||||
|
||||
console.log(`\n🏁 Results:`);
|
||||
console.log(` 🔧 Setup test: ${setupResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`);
|
||||
console.log(` 📡 Manual test: ${manualResult ? '✅ EVENTS CAPTURED' : '❌ NO EVENTS'}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runDebugTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { debugNetworkSetup, testManualNetworkCall };
|
||||
|
|
@ -1,137 +0,0 @@
|
|||
/**
|
||||
* Simple test to verify network monitoring is working
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testNetworkMonitoring() {
|
||||
console.log('🧪 Testing Network Monitoring with httpbin.org...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false, // Don't block resources so we can see requests
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Test with a simple API that returns JSON
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/json'
|
||||
);
|
||||
|
||||
let capturedRequests = 0;
|
||||
let capturedResponses = 0;
|
||||
let jsonResponse = null;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
console.log(`📡 ${event.type.toUpperCase()}: ${event.method} ${event.url}`);
|
||||
|
||||
if (event.type === 'request') {
|
||||
capturedRequests++;
|
||||
}
|
||||
|
||||
if (event.type === 'response') {
|
||||
capturedResponses++;
|
||||
console.log(` Status: ${event.status}`);
|
||||
|
||||
if (event.url.includes('httpbin.org/json') && event.responseData) {
|
||||
jsonResponse = event.responseData;
|
||||
console.log(` 📊 JSON Response: ${event.responseData}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await page.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
|
||||
console.log(`\n📊 Summary:`);
|
||||
console.log(` 📤 Requests captured: ${capturedRequests}`);
|
||||
console.log(` 📥 Responses captured: ${capturedResponses}`);
|
||||
console.log(` 📝 JSON data captured: ${jsonResponse ? 'Yes' : 'No'}`);
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return true;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testWithProxy() {
|
||||
console.log('\n🌐 Testing with proxy to see IP change...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
// Test IP without proxy
|
||||
console.log('📍 Getting IP without proxy...');
|
||||
const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip'
|
||||
);
|
||||
|
||||
let ipWithoutProxy = null;
|
||||
page1.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ipWithoutProxy = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🔹 Your IP: ${ipWithoutProxy}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page1.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(ctx1);
|
||||
|
||||
// Test IP with proxy
|
||||
console.log('🔄 Getting IP with proxy...');
|
||||
const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let ipWithProxy = null;
|
||||
page2.onNetworkEvent(event => {
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ipWithProxy = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🔸 Proxy IP: ${ipWithProxy}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page2.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
await Browser.closeContext(ctx2);
|
||||
|
||||
if (ipWithoutProxy && ipWithProxy && ipWithoutProxy !== ipWithProxy) {
|
||||
console.log('✅ Proxy is working - IPs are different!');
|
||||
} else {
|
||||
console.log('⚠️ Proxy may not be working - IPs are the same or not captured');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Proxy test failed:', error);
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('🚀 Network Monitoring Verification Tests\n');
|
||||
|
||||
const basicResult = await testNetworkMonitoring();
|
||||
await testWithProxy();
|
||||
|
||||
console.log(`\n🏁 Network monitoring: ${basicResult ? '✅ WORKING' : '❌ FAILED'}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testNetworkMonitoring, testWithProxy };
|
||||
|
|
@ -1,156 +0,0 @@
|
|||
/**
|
||||
* Test Playwright proxy authentication specifically
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function testPlaywrightProxyAuth() {
|
||||
console.log('🔐 Testing Playwright Proxy Authentication...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 15000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
// Test 1: Without proxy
|
||||
console.log('\n📍 Test 1: Without proxy');
|
||||
const { page: page1, contextId: ctx1 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip'
|
||||
);
|
||||
|
||||
let events1 = 0;
|
||||
let ip1 = null;
|
||||
page1.onNetworkEvent(event => {
|
||||
events1++;
|
||||
console.log(` 📡 Event: ${event.type} ${event.url}`);
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ip1 = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🌐 Your IP: ${ip1}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page1.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
await Browser.closeContext(ctx1);
|
||||
console.log(` Events captured: ${events1}`);
|
||||
|
||||
// Test 2: With proxy using new authentication method
|
||||
console.log('\n🔒 Test 2: With proxy (new auth method)');
|
||||
const { page: page2, contextId: ctx2 } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
let events2 = 0;
|
||||
let ip2 = null;
|
||||
page2.onNetworkEvent(event => {
|
||||
events2++;
|
||||
console.log(` 📡 Event: ${event.type} ${event.url}`);
|
||||
if (event.type === 'response' && event.url.includes('/ip') && event.responseData) {
|
||||
ip2 = JSON.parse(event.responseData).origin;
|
||||
console.log(` 🔄 Proxy IP: ${ip2}`);
|
||||
}
|
||||
});
|
||||
|
||||
await page2.waitForLoadState('domcontentloaded');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
await Browser.closeContext(ctx2);
|
||||
console.log(` Events captured: ${events2}`);
|
||||
|
||||
// Results
|
||||
console.log('\n📊 Results:');
|
||||
console.log(` 🌐 Direct IP: ${ip1 || 'Not captured'}`);
|
||||
console.log(` 🔄 Proxy IP: ${ip2 || 'Not captured'}`);
|
||||
console.log(` 📡 Direct events: ${events1}`);
|
||||
console.log(` 📡 Proxy events: ${events2}`);
|
||||
|
||||
if (ip1 && ip2 && ip1 !== ip2) {
|
||||
console.log('✅ Proxy authentication is working - different IPs detected!');
|
||||
return true;
|
||||
} else if (events1 > 0 || events2 > 0) {
|
||||
console.log('⚠️ Network monitoring working, but proxy may not be changing IP');
|
||||
return true;
|
||||
} else {
|
||||
console.log('❌ No network events captured');
|
||||
return false;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Test failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function testManualPageEvaluation() {
|
||||
console.log('\n🧪 Test 3: Manual page evaluation (without network monitoring)');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: 10000,
|
||||
blockResources: false,
|
||||
});
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://httpbin.org/ip',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
|
||||
// Try to get the page content directly
|
||||
const title = await page.title();
|
||||
console.log(` 📄 Page title: "${title}"`);
|
||||
|
||||
// Try to evaluate some JavaScript
|
||||
const result = await page.evaluate(() => {
|
||||
return {
|
||||
url: window.location.href,
|
||||
userAgent: navigator.userAgent.substring(0, 50),
|
||||
readyState: document.readyState,
|
||||
};
|
||||
});
|
||||
|
||||
console.log(` 🔍 Page info:`, result);
|
||||
|
||||
// Try to get page content
|
||||
const bodyText = await page.locator('body').textContent();
|
||||
if (bodyText) {
|
||||
console.log(` 📝 Body content (first 200 chars): ${bodyText.substring(0, 200)}...`);
|
||||
|
||||
// Check if it looks like an IP response
|
||||
if (bodyText.includes('origin')) {
|
||||
console.log(' ✅ Looks like httpbin.org response!');
|
||||
}
|
||||
}
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
return true;
|
||||
|
||||
} catch (error) {
|
||||
console.error(' ❌ Manual evaluation failed:', error);
|
||||
return false;
|
||||
} finally {
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runProxyTests() {
|
||||
console.log('🚀 Playwright Proxy Authentication Tests\n');
|
||||
|
||||
const authResult = await testPlaywrightProxyAuth();
|
||||
const manualResult = await testManualPageEvaluation();
|
||||
|
||||
console.log(`\n🏁 Final Results:`);
|
||||
console.log(` 🔐 Proxy auth test: ${authResult ? '✅ PASS' : '❌ FAIL'}`);
|
||||
console.log(` 🧪 Manual eval test: ${manualResult ? '✅ PASS' : '❌ FAIL'}`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
runProxyTests().catch(console.error);
|
||||
}
|
||||
|
||||
export { testPlaywrightProxyAuth, testManualPageEvaluation };
|
||||
0
test-query-performance.ts
Normal file
0
test-query-performance.ts
Normal file
0
test-shutdown-simple.ts
Normal file
0
test-shutdown-simple.ts
Normal file
0
test-shutdown.ts
Normal file
0
test-shutdown.ts
Normal file
0
test-signals.ts
Normal file
0
test-signals.ts
Normal file
|
|
@ -1,152 +0,0 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
|
||||
async function simpleProxyTest() {
|
||||
console.log('🔬 Simple Proxy Test...');
|
||||
|
||||
try {
|
||||
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
|
||||
console.log('✅ Browser initialized');
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
console.log('✅ Page created with proxy');
|
||||
let summaryData: any = null; // Initialize summaryData to store API response
|
||||
let eventCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.responseData) {
|
||||
console.log(`📊 Summary API Response Data: ${event.responseData}`);
|
||||
try {
|
||||
summaryData = JSON.parse(event.responseData) as any;
|
||||
const totalCount = summaryData[0].totalCount;
|
||||
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
|
||||
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
|
||||
} catch (e) {
|
||||
console.log('📊 Raw Summary Response:', event.responseData);
|
||||
}
|
||||
}
|
||||
}
|
||||
eventCount++;
|
||||
console.log(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
|
||||
});
|
||||
|
||||
console.log('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 20000 });
|
||||
console.log('✅ Page loaded');
|
||||
|
||||
// RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button
|
||||
console.log('🔍 Looking for Products tab...');
|
||||
|
||||
// Wait for the page to fully load
|
||||
await page.waitForTimeout(20000);
|
||||
|
||||
// First, click on the Products tab
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: 20000 });
|
||||
console.log('✅ Found Products tab');
|
||||
|
||||
console.log('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
console.log('✅ Products tab clicked');
|
||||
|
||||
// Wait for the tab content to load
|
||||
await page.waitForTimeout(5000);
|
||||
|
||||
// Click on the Asset Classes accordion to expand it
|
||||
console.log('🔍 Looking for Asset Classes accordion...');
|
||||
const assetClassesAccordion = page.locator(
|
||||
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
|
||||
);
|
||||
await assetClassesAccordion.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Asset Classes accordion');
|
||||
|
||||
console.log('🖱️ Clicking Asset Classes accordion...');
|
||||
await assetClassesAccordion.click();
|
||||
console.log('✅ Asset Classes accordion clicked');
|
||||
|
||||
// Wait for the accordion content to expand
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
console.log('🔍 Looking for Stocks checkbox...');
|
||||
|
||||
// Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks"
|
||||
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
|
||||
await stocksSpan.waitFor({ timeout: 10000 });
|
||||
console.log('✅ Found Stocks span');
|
||||
|
||||
// Find the checkbox by looking in the same parent container
|
||||
const parentContainer = stocksSpan.locator('..');
|
||||
const checkbox = parentContainer.locator('input[type="checkbox"]');
|
||||
|
||||
if ((await checkbox.count()) > 0) {
|
||||
console.log('📋 Clicking Stocks checkbox...');
|
||||
await checkbox.first().check();
|
||||
console.log('✅ Stocks checkbox checked');
|
||||
} else {
|
||||
console.log('⚠️ Could not find checkbox near Stocks text');
|
||||
}
|
||||
|
||||
// Wait a moment for any UI updates
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Find and click the nearest Apply button
|
||||
console.log('🔍 Looking for Apply button...');
|
||||
const applyButton = page.locator(
|
||||
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
|
||||
);
|
||||
|
||||
if ((await applyButton.count()) > 0) {
|
||||
console.log('🎯 Clicking Apply button...');
|
||||
await applyButton.first().click();
|
||||
console.log('✅ Apply button clicked');
|
||||
|
||||
// Wait for any network requests triggered by the Apply button
|
||||
await page.waitForTimeout(2000);
|
||||
} else {
|
||||
console.log('⚠️ Could not find Apply button');
|
||||
}
|
||||
} catch (interactionError) {
|
||||
const errorMessage =
|
||||
interactionError instanceof Error ? interactionError.message : String(interactionError);
|
||||
console.error('❌ Page interaction failed:', errorMessage);
|
||||
|
||||
// Get debug info about the page
|
||||
try {
|
||||
const title = await page.title();
|
||||
console.log(`📄 Current page title: "${title}"`);
|
||||
|
||||
const stocksElements = await page.locator('*:has-text("Stocks")').count();
|
||||
console.log(`🔍 Found ${stocksElements} elements containing "Stocks"`);
|
||||
|
||||
const applyButtons = await page
|
||||
.locator('button:has-text("Apply"), input[value*="Apply"]')
|
||||
.count();
|
||||
console.log(`🔍 Found ${applyButtons} Apply buttons`);
|
||||
} catch (debugError) {
|
||||
const debugMessage = debugError instanceof Error ? debugError.message : String(debugError);
|
||||
console.log('❌ Could not get debug info:', debugMessage);
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
console.log(`📊 Total events: ${eventCount}`);
|
||||
|
||||
await Browser.closeContext(contextId);
|
||||
await Browser.close();
|
||||
|
||||
console.log('✅ Test completed');
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.error('❌ Error:', errorMessage);
|
||||
await Browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
simpleProxyTest();
|
||||
|
|
@ -1 +0,0 @@
|
|||
console.log("Testing browser import..."); import { Browser } from "@stock-bot/browser"; console.log("Browser imported successfully:", typeof Browser); Browser.initialize().then(() => console.log("Browser initialized")).catch(e => console.error("Error:", e));
|
||||
Loading…
Add table
Add a link
Reference in a new issue