refactored monorepo for more projects
This commit is contained in:
parent
4632c174dc
commit
9492f1b15e
180 changed files with 1438 additions and 424 deletions
|
|
@ -0,0 +1,38 @@
|
|||
import type { IbHandler } from '../ib.handler';
|
||||
|
||||
export async function fetchExchangesAndSymbols(this: IbHandler): Promise<unknown> {
|
||||
this.logger.info('Starting IB exchanges and symbols fetch job');
|
||||
|
||||
try {
|
||||
// Fetch session headers first
|
||||
const sessionHeaders = await this.fetchSession();
|
||||
if (!sessionHeaders) {
|
||||
this.logger.error('Failed to get session headers for IB job');
|
||||
return { success: false, error: 'No session headers' };
|
||||
}
|
||||
|
||||
this.logger.info('Session headers obtained, fetching exchanges...');
|
||||
|
||||
// Fetch exchanges
|
||||
const exchanges = await this.fetchExchanges();
|
||||
this.logger.info('Fetched exchanges from IB', { count: exchanges?.length || 0 });
|
||||
|
||||
// Fetch symbols
|
||||
this.logger.info('Fetching symbols...');
|
||||
const symbols = await this.fetchSymbols();
|
||||
this.logger.info('Fetched symbols from IB', { count: symbols?.length || 0 });
|
||||
|
||||
return {
|
||||
success: true,
|
||||
exchangesCount: exchanges?.length || 0,
|
||||
symbolsCount: symbols?.length || 0,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to fetch IB exchanges and symbols', { error });
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import type { IbHandler } from '../ib.handler';
|
||||
import { IB_CONFIG } from '../shared/config';
|
||||
|
||||
export async function fetchExchanges(this: IbHandler): Promise<unknown[] | null> {
|
||||
try {
|
||||
// First get session headers
|
||||
const sessionHeaders = await this.fetchSession();
|
||||
if (!sessionHeaders) {
|
||||
throw new Error('Failed to get session headers');
|
||||
}
|
||||
|
||||
this.logger.info('🔍 Fetching exchanges with session headers...');
|
||||
|
||||
// The URL for the exchange data API
|
||||
const exchangeUrl = IB_CONFIG.BASE_URL + IB_CONFIG.EXCHANGE_API;
|
||||
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
this.logger.info('📤 Making request to exchange API...', {
|
||||
url: exchangeUrl,
|
||||
headerCount: Object.keys(requestHeaders).length,
|
||||
});
|
||||
|
||||
// Use fetch with proxy configuration
|
||||
const response = await fetch(exchangeUrl, {
|
||||
method: 'GET',
|
||||
headers: requestHeaders,
|
||||
proxy: IB_CONFIG.DEFAULT_PROXY,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
this.logger.error('❌ Exchange API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const exchanges = data?.exchanges || [];
|
||||
this.logger.info('✅ Exchange data fetched successfully');
|
||||
|
||||
this.logger.info('Saving IB exchanges to MongoDB...');
|
||||
await this.mongodb.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']);
|
||||
this.logger.info('✅ Exchange IB data saved to MongoDB:', {
|
||||
count: exchanges.length,
|
||||
});
|
||||
|
||||
return exchanges;
|
||||
} catch (error) {
|
||||
this.logger.error('❌ Failed to fetch exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
import { Browser } from '@stock-bot/browser';
|
||||
import type { IbHandler } from '../ib.handler';
|
||||
import { IB_CONFIG } from '../shared/config';
|
||||
|
||||
export async function fetchSession(this: IbHandler): Promise<Record<string, string> | undefined> {
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: IB_CONFIG.BROWSER_TIMEOUT,
|
||||
blockResources: false,
|
||||
});
|
||||
this.logger.info('✅ Browser initialized');
|
||||
|
||||
const { page } = await Browser.createPageWithProxy(
|
||||
IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_PAGE,
|
||||
IB_CONFIG.DEFAULT_PROXY
|
||||
);
|
||||
this.logger.info('✅ Page created with proxy');
|
||||
|
||||
const headersPromise = new Promise<Record<string, string> | undefined>(resolve => {
|
||||
let resolved = false;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
if (event.type === 'request') {
|
||||
try {
|
||||
resolve(event.headers);
|
||||
} catch (e) {
|
||||
resolve(undefined);
|
||||
this.logger.debug('Raw Summary Response error', { error: (e as Error).message });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Timeout fallback
|
||||
setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
this.logger.warn('Timeout waiting for headers');
|
||||
resolve(undefined);
|
||||
}
|
||||
}, IB_CONFIG.HEADERS_TIMEOUT);
|
||||
});
|
||||
|
||||
this.logger.info('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: IB_CONFIG.PAGE_LOAD_TIMEOUT });
|
||||
this.logger.info('✅ Page loaded');
|
||||
|
||||
//Products tabs
|
||||
this.logger.info('🔍 Looking for Products tab...');
|
||||
const productsTab = page.locator('#productSearchTab[role="tab"][href="#products"]');
|
||||
await productsTab.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT });
|
||||
this.logger.info('✅ Found Products tab');
|
||||
this.logger.info('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
this.logger.info('✅ Products tab clicked');
|
||||
|
||||
// New Products Checkbox
|
||||
this.logger.info('🔍 Looking for "New Products Only" radio button...');
|
||||
const radioButton = page.locator('span.checkbox-text:has-text("New Products Only")');
|
||||
await radioButton.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT });
|
||||
this.logger.info(`🎯 Found "New Products Only" radio button`);
|
||||
await radioButton.first().click();
|
||||
this.logger.info('✅ "New Products Only" radio button clicked');
|
||||
|
||||
// Wait for and return headers immediately when captured
|
||||
this.logger.info('⏳ Waiting for headers to be captured...');
|
||||
const headers = await headersPromise;
|
||||
page.close();
|
||||
if (headers) {
|
||||
this.logger.info('✅ Headers captured successfully');
|
||||
} else {
|
||||
this.logger.warn('⚠️ No headers were captured');
|
||||
}
|
||||
|
||||
return headers;
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to fetch IB symbol summary', { error });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
import type { IbHandler } from '../ib.handler';
|
||||
import { IB_CONFIG } from '../shared/config';
|
||||
|
||||
export async function fetchSymbols(this: IbHandler): Promise<unknown[] | null> {
|
||||
try {
|
||||
// First get session headers
|
||||
const sessionHeaders = await this.fetchSession();
|
||||
if (!sessionHeaders) {
|
||||
throw new Error('Failed to get session headers');
|
||||
}
|
||||
|
||||
this.logger.info('🔍 Fetching symbols with session headers...');
|
||||
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
const requestBody = {
|
||||
domain: 'com',
|
||||
newProduct: 'all',
|
||||
pageNumber: 1,
|
||||
pageSize: 100,
|
||||
productCountry: IB_CONFIG.PRODUCT_COUNTRIES,
|
||||
productSymbol: '',
|
||||
productType: IB_CONFIG.PRODUCT_TYPES,
|
||||
sortDirection: 'asc',
|
||||
sortField: 'symbol',
|
||||
};
|
||||
|
||||
// Get Summary
|
||||
const summaryResponse = await fetch(IB_CONFIG.BASE_URL + IB_CONFIG.SUMMARY_API, {
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: IB_CONFIG.DEFAULT_PROXY,
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
|
||||
if (!summaryResponse.ok) {
|
||||
this.logger.error('❌ Summary API request failed', {
|
||||
status: summaryResponse.status,
|
||||
statusText: summaryResponse.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const summaryData = await summaryResponse.json();
|
||||
this.logger.info('✅ IB Summary data fetched successfully', {
|
||||
totalCount: summaryData[0].totalCount,
|
||||
});
|
||||
|
||||
const symbols = [];
|
||||
requestBody.pageSize = IB_CONFIG.PAGE_SIZE;
|
||||
const pageCount = Math.ceil(summaryData[0].totalCount / IB_CONFIG.PAGE_SIZE) || 0;
|
||||
this.logger.info('Fetching Symbols for IB', { pageCount });
|
||||
|
||||
const symbolPromises = [];
|
||||
for (let page = 1; page <= pageCount; page++) {
|
||||
requestBody.pageNumber = page;
|
||||
|
||||
// Fetch symbols for the current page
|
||||
const symbolsResponse = fetch(IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_API, {
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: IB_CONFIG.DEFAULT_PROXY,
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
symbolPromises.push(symbolsResponse);
|
||||
}
|
||||
|
||||
const responses = await Promise.all(symbolPromises);
|
||||
for (const response of responses) {
|
||||
if (!response.ok) {
|
||||
this.logger.error('❌ Symbols API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
const data = await response.json();
|
||||
const symJson = data?.products || [];
|
||||
if (symJson && symJson.length > 0) {
|
||||
symbols.push(...symJson);
|
||||
} else {
|
||||
this.logger.warn('⚠️ No symbols found in response');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (symbols.length === 0) {
|
||||
this.logger.warn('⚠️ No symbols fetched from IB');
|
||||
return null;
|
||||
}
|
||||
|
||||
this.logger.info('✅ IB symbols fetched successfully, saving to DB...', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
await this.mongodb.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
|
||||
this.logger.info('Saved IB symbols to DB', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
|
||||
return symbols;
|
||||
} catch (error) {
|
||||
this.logger.error('❌ Failed to fetch symbols', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
export { fetchSession } from './fetch-session.action';
|
||||
export { fetchExchanges } from './fetch-exchanges.action';
|
||||
export { fetchSymbols } from './fetch-symbols.action';
|
||||
export { fetchExchangesAndSymbols } from './fetch-exchanges-and-symbols.action';
|
||||
|
||||
33
apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts
Normal file
33
apps/stock/data-ingestion/src/handlers/ib/ib.handler.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import {
|
||||
BaseHandler,
|
||||
Handler,
|
||||
Operation,
|
||||
ScheduledOperation,
|
||||
type IServiceContainer,
|
||||
} from '@stock-bot/handlers';
|
||||
import { fetchExchanges, fetchExchangesAndSymbols, fetchSession, fetchSymbols } from './actions';
|
||||
|
||||
@Handler('ib')
|
||||
export class IbHandler extends BaseHandler {
|
||||
constructor(services: IServiceContainer) {
|
||||
super(services);
|
||||
}
|
||||
|
||||
@Operation('fetch-session')
|
||||
fetchSession = fetchSession;
|
||||
|
||||
@Operation('fetch-exchanges')
|
||||
fetchExchanges = fetchExchanges;
|
||||
|
||||
@Operation('fetch-symbols')
|
||||
fetchSymbols = fetchSymbols;
|
||||
|
||||
@Operation('ib-exchanges-and-symbols')
|
||||
@ScheduledOperation('ib-exchanges-and-symbols', '0 0 * * 0', {
|
||||
priority: 5,
|
||||
description: 'Fetch and update IB exchanges and symbols data',
|
||||
immediately: false,
|
||||
})
|
||||
fetchExchangesAndSymbols = fetchExchangesAndSymbols;
|
||||
}
|
||||
|
||||
24
apps/stock/data-ingestion/src/handlers/ib/shared/config.ts
Normal file
24
apps/stock/data-ingestion/src/handlers/ib/shared/config.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Interactive Brokers Configuration Constants
|
||||
*/
|
||||
|
||||
export const IB_CONFIG = {
|
||||
BASE_URL: 'https://www.interactivebrokers.com',
|
||||
PRODUCTS_PAGE: '/en/trading/products-exchanges.php#/',
|
||||
EXCHANGE_API: '/webrest/exchanges',
|
||||
SUMMARY_API: '/webrest/search/product-types/summary',
|
||||
PRODUCTS_API: '/webrest/search/products-by-filters',
|
||||
|
||||
// Browser configuration
|
||||
BROWSER_TIMEOUT: 10000,
|
||||
PAGE_LOAD_TIMEOUT: 20000,
|
||||
ELEMENT_TIMEOUT: 5000,
|
||||
HEADERS_TIMEOUT: 30000,
|
||||
|
||||
// API configuration
|
||||
DEFAULT_PROXY: 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80',
|
||||
PAGE_SIZE: 500,
|
||||
PRODUCT_COUNTRIES: ['CA', 'US'],
|
||||
PRODUCT_TYPES: ['STK'],
|
||||
};
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue