work on postgress / will prob remove and work on ib exchanges and symbols
This commit is contained in:
parent
cce5126cb7
commit
a20a11c1aa
16 changed files with 1441 additions and 95 deletions
|
|
@ -4,7 +4,7 @@
|
|||
import { Hono } from 'hono';
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
import { loadEnvVariables } from '@stock-bot/config';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { getLogger, shutdownLoggers } from '@stock-bot/logger';
|
||||
import { Shutdown } from '@stock-bot/shutdown';
|
||||
import { initializeIBResources } from './providers/ib.tasks';
|
||||
import { initializeProxyResources } from './providers/proxy.tasks';
|
||||
|
|
@ -99,7 +99,38 @@ shutdown.onShutdown(async () => {
|
|||
logger.info('Queue manager shut down successfully');
|
||||
} catch (error) {
|
||||
logger.error('Error shutting down queue manager', { error });
|
||||
throw error; // Re-throw to mark shutdown as failed
|
||||
// Don't re-throw to allow other shutdown handlers to complete
|
||||
// The shutdown library tracks failures internally
|
||||
}
|
||||
});
|
||||
|
||||
// Add Browser shutdown handler
|
||||
shutdown.onShutdown(async () => {
|
||||
logger.info('Shutting down browser resources...');
|
||||
try {
|
||||
await Browser.close();
|
||||
logger.info('Browser resources shut down successfully');
|
||||
} catch (error) {
|
||||
// Browser might already be closed by running tasks, this is expected
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
if (errorMessage.includes('Target page, context or browser has been closed')) {
|
||||
logger.info('Browser was already closed by running tasks');
|
||||
} else {
|
||||
logger.error('Error shutting down browser resources', { error });
|
||||
}
|
||||
// Don't throw here as browser shutdown shouldn't block app shutdown
|
||||
}
|
||||
});
|
||||
|
||||
// Add logger shutdown handler (should be last)
|
||||
shutdown.onShutdown(async () => {
|
||||
try {
|
||||
await shutdownLoggers();
|
||||
// Use process.stdout since loggers are being shut down
|
||||
process.stdout.write('All loggers flushed and shut down successfully\n');
|
||||
} catch (error) {
|
||||
process.stderr.write(`Error shutting down loggers: ${error}\n`);
|
||||
// Don't throw here as this is the final cleanup
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -6,21 +6,26 @@ const logger = getLogger('ib-provider');
|
|||
export const ibProvider: ProviderConfig = {
|
||||
name: 'ib',
|
||||
operations: {
|
||||
'ib-symbol-summary': async () => {
|
||||
'ib-basics': async () => {
|
||||
const { ibTasks } = await import('./ib.tasks');
|
||||
logger.info('Fetching symbol summary from IB');
|
||||
const total = await ibTasks.fetchSymbolSummary();
|
||||
const sessionHeaders = await ibTasks.fetchSession();
|
||||
logger.info('Fetched symbol summary from IB', {
|
||||
count: total,
|
||||
sessionHeaders,
|
||||
});
|
||||
return total;
|
||||
|
||||
// Get Exchanges
|
||||
logger.info('Fetching exchanges from IB');
|
||||
const exchanges = await ibTasks.fetchExchanges(sessionHeaders);
|
||||
logger.info('Fetched exchanges from IB', { exchanges });
|
||||
// return total;
|
||||
},
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'ib-symbol-summary',
|
||||
operation: 'ib-symbol-summary',
|
||||
type: 'ib-basics',
|
||||
operation: 'ib-basics',
|
||||
payload: {},
|
||||
// should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
cronPattern: '*/2 * * * *',
|
||||
|
|
|
|||
|
|
@ -32,121 +32,138 @@ export async function initializeIBResources(waitForCache = false): Promise<void>
|
|||
isInitialized = true;
|
||||
}
|
||||
|
||||
export async function fetchSymbolSummary(): Promise<number> {
|
||||
export async function fetchSession(): Promise<Record<string, string> | undefined> {
|
||||
try {
|
||||
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
|
||||
logger.info('✅ Browser initialized');
|
||||
|
||||
const { page, contextId } = await Browser.createPageWithProxy(
|
||||
const { page } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
logger.info('✅ Page created with proxy');
|
||||
let summaryData: any = null; // Initialize summaryData to store API response
|
||||
let eventCount = 0;
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
console.log(`🎯 Found summary API call: ${event.type} ${event.url}`);
|
||||
|
||||
if (event.type === 'response' && event.responseData) {
|
||||
console.log(`📊 Summary API Response Data: ${event.responseData}`);
|
||||
try {
|
||||
summaryData = JSON.parse(event.responseData) as any;
|
||||
const totalCount = summaryData[0].totalCount;
|
||||
console.log('📊 Summary API Response:', JSON.stringify(summaryData, null, 2));
|
||||
console.log(`🔢 Total symbols found: ${totalCount || 'Unknown'}`);
|
||||
} catch (e) {
|
||||
console.log('📊 Raw Summary Response:', event.responseData);
|
||||
const headersPromise = new Promise<Record<string, string> | undefined>(resolve => {
|
||||
let resolved = false;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
if (event.type === 'request') {
|
||||
try {
|
||||
resolve(event.headers);
|
||||
} catch (e) {
|
||||
resolve(undefined);
|
||||
console.log('📊 Raw Summary Response:', (e as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
eventCount++;
|
||||
logger.info(`📡 Event ${eventCount}: ${event.type} ${event.url}`);
|
||||
});
|
||||
|
||||
// Timeout fallback
|
||||
setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
logger.warn('Timeout waiting for headers');
|
||||
resolve(undefined);
|
||||
}
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
logger.info('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 20000 });
|
||||
logger.info('✅ Page loaded');
|
||||
|
||||
// RIGHT HERE - Interact with the page to find Stocks checkbox and Apply button
|
||||
//Products tabs
|
||||
logger.info('🔍 Looking for Products tab...');
|
||||
|
||||
// Wait for the page to fully load
|
||||
await page.waitForTimeout(20000);
|
||||
|
||||
// First, click on the Products tab
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: 20000 });
|
||||
await productsTab.waitFor({ timeout: 5000 });
|
||||
logger.info('✅ Found Products tab');
|
||||
|
||||
logger.info('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
logger.info('✅ Products tab clicked');
|
||||
|
||||
// Wait for the tab content to load
|
||||
await page.waitForTimeout(5000);
|
||||
// New Products Checkbox
|
||||
logger.info('🔍 Looking for "New Products Only" radio button...');
|
||||
const radioButton = page.locator('span.checkbox-text:has-text("New Products Only")');
|
||||
await radioButton.waitFor({ timeout: 5000 });
|
||||
logger.info(`🎯 Found "New Products Only" radio button`);
|
||||
await radioButton.first().click();
|
||||
logger.info('✅ "New Products Only" radio button clicked');
|
||||
|
||||
// Click on the Asset Classes accordion to expand it
|
||||
logger.info('🔍 Looking for Asset Classes accordion...');
|
||||
const assetClassesAccordion = page.locator(
|
||||
'#products .accordion-item #acc-products .accordion_btn:has-text("Asset Classes")'
|
||||
);
|
||||
await assetClassesAccordion.waitFor({ timeout: 10000 });
|
||||
logger.info('✅ Found Asset Classes accordion');
|
||||
// Wait for and return headers immediately when captured
|
||||
logger.info('⏳ Waiting for headers to be captured...');
|
||||
const headers = await headersPromise;
|
||||
|
||||
logger.info('🖱️ Clicking Asset Classes accordion...');
|
||||
await assetClassesAccordion.click();
|
||||
logger.info('✅ Asset Classes accordion clicked');
|
||||
|
||||
// Wait for the accordion content to expand
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
logger.info('🔍 Looking for Stocks checkbox...');
|
||||
|
||||
// Find the span with class "fs-7 checkbox-text" and inner text containing "Stocks"
|
||||
const stocksSpan = page.locator('span.fs-7.checkbox-text:has-text("Stocks")');
|
||||
await stocksSpan.waitFor({ timeout: 10000 });
|
||||
logger.info('✅ Found Stocks span');
|
||||
|
||||
// Find the checkbox by looking in the same parent container
|
||||
const parentContainer = stocksSpan.locator('..');
|
||||
const checkbox = parentContainer.locator('input[type="checkbox"]');
|
||||
|
||||
if ((await checkbox.count()) > 0) {
|
||||
logger.info('📋 Clicking Stocks checkbox...');
|
||||
await checkbox.first().check();
|
||||
logger.info('✅ Stocks checkbox checked');
|
||||
if (headers) {
|
||||
logger.info('✅ Headers captured successfully');
|
||||
} else {
|
||||
logger.info('⚠️ Could not find checkbox near Stocks text');
|
||||
logger.warn('⚠️ No headers were captured');
|
||||
}
|
||||
|
||||
// Wait a moment for any UI updates
|
||||
await page.waitForTimeout(1000);
|
||||
|
||||
// Find and click the nearest Apply button
|
||||
logger.info('🔍 Looking for Apply button...');
|
||||
const applyButton = page.locator(
|
||||
'button:has-text("Apply"), input[type="submit"][value*="Apply"], input[type="button"][value*="Apply"]'
|
||||
);
|
||||
|
||||
if ((await applyButton.count()) > 0) {
|
||||
logger.info('🎯 Clicking Apply button...');
|
||||
await applyButton.first().click();
|
||||
logger.info('✅ Apply button clicked');
|
||||
|
||||
// Wait for any network requests triggered by the Apply button
|
||||
await page.waitForTimeout(2000);
|
||||
} else {
|
||||
logger.info('⚠️ Could not find Apply button');
|
||||
}
|
||||
|
||||
return 0;
|
||||
return headers;
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch IB symbol summary', { error });
|
||||
return 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchExchanges(sessionHeaders: Record<string, string>): Promise<any> {
|
||||
try {
|
||||
logger.info('🔍 Fetching exchanges with session headers...');
|
||||
|
||||
// The URL for the exchange data API
|
||||
const exchangeUrl = 'https://www.interactivebrokers.com/webrest/exchanges';
|
||||
|
||||
// Configure the proxy
|
||||
const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80';
|
||||
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
logger.info('📤 Making request to exchange API...', {
|
||||
url: exchangeUrl,
|
||||
headerCount: Object.keys(requestHeaders).length,
|
||||
});
|
||||
|
||||
// Use fetch with proxy configuration
|
||||
const response = await fetch(exchangeUrl, {
|
||||
method: 'GET',
|
||||
headers: requestHeaders,
|
||||
proxy: proxyUrl,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error('❌ Exchange API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
logger.info('✅ Exchange data fetched successfully', {
|
||||
dataKeys: Object.keys(data || {}),
|
||||
dataSize: JSON.stringify(data).length,
|
||||
});
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// Optional: Export a convenience object that groups related tasks
|
||||
export const ibTasks = {
|
||||
fetchSymbolSummary,
|
||||
fetchSession,
|
||||
fetchExchanges,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue