reorganized providers to handlers and changed folder structure for maintaiablity
This commit is contained in:
parent
1bb2380a28
commit
59ab0940ae
11 changed files with 21 additions and 21 deletions
309
apps/data-service/src/handlers/ib/ib.operations.ts
Normal file
309
apps/data-service/src/handlers/ib/ib.operations.ts
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { getMongoDBClient } from '@stock-bot/mongodb-client';
|
||||
|
||||
// Shared instances (module-scoped, not global)
|
||||
let isInitialized = false; // Track if resources are initialized
|
||||
let logger: ReturnType<typeof getLogger>;
|
||||
// let cache: CacheProvider;
|
||||
|
||||
export async function initializeIBResources(): Promise<void> {
|
||||
// Skip if already initialized
|
||||
if (isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger = getLogger('proxy-tasks');
|
||||
// cache = createCache({
|
||||
// keyPrefix: 'proxy:',
|
||||
// ttl: PROXY_CONFIG.CACHE_TTL,
|
||||
// enableMetrics: true,
|
||||
// });
|
||||
|
||||
// httpClient = new HttpClient({ timeout: 15000 }, logger);
|
||||
|
||||
// if (waitForCache) {
|
||||
// // logger.info('Initializing proxy cache...');
|
||||
// // await cache.waitForReady(10000);
|
||||
// // logger.info('Proxy cache initialized successfully');
|
||||
// logger.info('Proxy tasks initialized');
|
||||
// } else {
|
||||
// logger.info('Proxy tasks initialized (fallback mode)');
|
||||
// }
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
export async function fetchSession(): Promise<Record<string, string> | undefined> {
|
||||
try {
|
||||
await Browser.initialize({ headless: true, timeout: 10000, blockResources: false });
|
||||
logger.info('✅ Browser initialized');
|
||||
|
||||
const { page } = await Browser.createPageWithProxy(
|
||||
'https://www.interactivebrokers.com/en/trading/products-exchanges.php#/',
|
||||
'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80'
|
||||
);
|
||||
logger.info('✅ Page created with proxy');
|
||||
|
||||
const headersPromise = new Promise<Record<string, string> | undefined>(resolve => {
|
||||
let resolved = false;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
if (event.type === 'request') {
|
||||
try {
|
||||
resolve(event.headers);
|
||||
} catch (e) {
|
||||
resolve(undefined);
|
||||
logger.debug('Raw Summary Response error', { error: (e as Error).message });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Timeout fallback
|
||||
setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
logger.warn('Timeout waiting for headers');
|
||||
resolve(undefined);
|
||||
}
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
logger.info('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 20000 });
|
||||
logger.info('✅ Page loaded');
|
||||
|
||||
//Products tabs
|
||||
logger.info('🔍 Looking for Products tab...');
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: 5000 });
|
||||
logger.info('✅ Found Products tab');
|
||||
logger.info('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
logger.info('✅ Products tab clicked');
|
||||
|
||||
// New Products Checkbox
|
||||
logger.info('🔍 Looking for "New Products Only" radio button...');
|
||||
const radioButton = page.locator('span.checkbox-text:has-text("New Products Only")');
|
||||
await radioButton.waitFor({ timeout: 5000 });
|
||||
logger.info(`🎯 Found "New Products Only" radio button`);
|
||||
await radioButton.first().click();
|
||||
logger.info('✅ "New Products Only" radio button clicked');
|
||||
|
||||
// Wait for and return headers immediately when captured
|
||||
logger.info('⏳ Waiting for headers to be captured...');
|
||||
const headers = await headersPromise;
|
||||
page.close();
|
||||
if (headers) {
|
||||
logger.info('✅ Headers captured successfully');
|
||||
} else {
|
||||
logger.warn('⚠️ No headers were captured');
|
||||
}
|
||||
|
||||
return headers;
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch IB symbol summary', { error });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchExchanges(sessionHeaders: Record<string, string>): Promise<unknown[] | null> {
|
||||
try {
|
||||
logger.info('🔍 Fetching exchanges with session headers...');
|
||||
|
||||
// The URL for the exchange data API
|
||||
const exchangeUrl = 'https://www.interactivebrokers.com/webrest/exchanges';
|
||||
|
||||
// Configure the proxy
|
||||
const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80';
|
||||
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
logger.info('📤 Making request to exchange API...', {
|
||||
url: exchangeUrl,
|
||||
headerCount: Object.keys(requestHeaders).length,
|
||||
});
|
||||
|
||||
// Use fetch with proxy configuration
|
||||
const response = await fetch(exchangeUrl, {
|
||||
method: 'GET',
|
||||
headers: requestHeaders,
|
||||
proxy: proxyUrl,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error('❌ Exchange API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const exchanges = data?.exchanges || [];
|
||||
logger.info('✅ Exchange data fetched successfully');
|
||||
|
||||
logger.info('Saving IB exchanges to MongoDB...');
|
||||
const client = getMongoDBClient();
|
||||
await client.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']);
|
||||
logger.info('✅ Exchange IB data saved to MongoDB:', {
|
||||
count: exchanges.length,
|
||||
});
|
||||
|
||||
return exchanges;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch symbols from IB using the session headers
|
||||
export async function fetchSymbols(sessionHeaders: Record<string, string>): Promise<unknown[] | null> {
|
||||
try {
|
||||
logger.info('🔍 Fetching symbols with session headers...');
|
||||
// Configure the proxy
|
||||
const proxyUrl = 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80';
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
const requestBody = {
|
||||
domain: 'com',
|
||||
newProduct: 'all',
|
||||
pageNumber: 1,
|
||||
pageSize: 100,
|
||||
productCountry: ['CA', 'US'],
|
||||
productSymbol: '',
|
||||
productType: ['STK'],
|
||||
sortDirection: 'asc',
|
||||
sortField: 'symbol',
|
||||
};
|
||||
|
||||
// Get Summary
|
||||
const summaryResponse = await fetch(
|
||||
'https://www.interactivebrokers.com/webrest/search/product-types/summary',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: proxyUrl,
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
);
|
||||
|
||||
if (!summaryResponse.ok) {
|
||||
logger.error('❌ Summary API request failed', {
|
||||
status: summaryResponse.status,
|
||||
statusText: summaryResponse.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const summaryData = await summaryResponse.json();
|
||||
logger.info('✅ IB Summary data fetched successfully', {
|
||||
totalCount: summaryData[0].totalCount,
|
||||
});
|
||||
|
||||
const symbols = [];
|
||||
requestBody.pageSize = 500;
|
||||
const pageCount = Math.ceil(summaryData[0].totalCount / 500) || 0;
|
||||
logger.info('Fetching Symbols for IB', { pageCount });
|
||||
const symbolPromises = [];
|
||||
for (let page = 1; page <= pageCount; page++) {
|
||||
requestBody.pageNumber = page;
|
||||
|
||||
// Fetch symbols for the current page
|
||||
const symbolsResponse = fetch(
|
||||
'https://www.interactivebrokers.com/webrest/search/products-by-filters',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: proxyUrl,
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
);
|
||||
symbolPromises.push(symbolsResponse);
|
||||
}
|
||||
const responses = await Promise.all(symbolPromises);
|
||||
for (const response of responses) {
|
||||
if (!response.ok) {
|
||||
logger.error('❌ Symbols API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
const data = await response.json();
|
||||
const symJson = data?.products || [];
|
||||
if (symJson && symJson.length > 0) {
|
||||
symbols.push(...symJson);
|
||||
} else {
|
||||
logger.warn('⚠️ No symbols found in response');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (symbols.length === 0) {
|
||||
logger.warn('⚠️ No symbols fetched from IB');
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.info('✅ IB symbols fetched successfully, saving to DB...', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
const client = getMongoDBClient();
|
||||
await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
|
||||
logger.info('Saved IB symbols to DB', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
|
||||
return symbols;
|
||||
// logger.info('📤 Making request to exchange API...', {
|
||||
// url: exchangeUrl,
|
||||
// headerCount: Object.keys(requestHeaders).length,
|
||||
// });
|
||||
|
||||
// // Use fetch with proxy configuration
|
||||
// const response = await fetch(exchangeUrl, {
|
||||
// method: 'GET',
|
||||
// headers: requestHeaders,
|
||||
// proxy: proxyUrl,
|
||||
// });
|
||||
|
||||
// if (!response.ok) {
|
||||
// logger.error('❌ Exchange API request failed', {
|
||||
// status: response.status,
|
||||
// statusText: response.statusText,
|
||||
// });
|
||||
// return null;
|
||||
// }
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch symbols', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const ibTasks = {
|
||||
fetchSymbols,
|
||||
fetchSession,
|
||||
fetchExchanges,
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue