renaming services to more suitable names
This commit is contained in:
parent
3ae9de8376
commit
be6afef832
69 changed files with 41 additions and 2956 deletions
89
apps/data-ingestion/src/handlers/ib/ib.handler.ts
Normal file
89
apps/data-ingestion/src/handlers/ib/ib.handler.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
/**
|
||||
* Interactive Brokers Provider for new queue system
|
||||
*/
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import {
|
||||
createJobHandler,
|
||||
handlerRegistry,
|
||||
type HandlerConfigWithSchedule,
|
||||
} from '@stock-bot/queue';
|
||||
|
||||
const logger = getLogger('ib-provider');
|
||||
|
||||
// Initialize and register the IB provider
|
||||
export function initializeIBProvider() {
|
||||
logger.debug('Registering IB provider with scheduled jobs...');
|
||||
|
||||
const ibProviderConfig: HandlerConfigWithSchedule = {
|
||||
name: 'ib',
|
||||
operations: {
|
||||
'fetch-session': createJobHandler(async () => {
|
||||
// payload contains session configuration (not used in current implementation)
|
||||
logger.debug('Processing session fetch request');
|
||||
const { fetchSession } = await import('./operations/session.operations');
|
||||
return fetchSession();
|
||||
}),
|
||||
|
||||
'fetch-exchanges': createJobHandler(async () => {
|
||||
// payload should contain session headers
|
||||
logger.debug('Processing exchanges fetch request');
|
||||
const { fetchSession } = await import('./operations/session.operations');
|
||||
const { fetchExchanges } = await import('./operations/exchanges.operations');
|
||||
const sessionHeaders = await fetchSession();
|
||||
if (sessionHeaders) {
|
||||
return fetchExchanges(sessionHeaders);
|
||||
}
|
||||
throw new Error('Failed to get session headers');
|
||||
}),
|
||||
|
||||
'fetch-symbols': createJobHandler(async () => {
|
||||
// payload should contain session headers
|
||||
logger.debug('Processing symbols fetch request');
|
||||
const { fetchSession } = await import('./operations/session.operations');
|
||||
const { fetchSymbols } = await import('./operations/symbols.operations');
|
||||
const sessionHeaders = await fetchSession();
|
||||
if (sessionHeaders) {
|
||||
return fetchSymbols(sessionHeaders);
|
||||
}
|
||||
throw new Error('Failed to get session headers');
|
||||
}),
|
||||
|
||||
'ib-exchanges-and-symbols': createJobHandler(async () => {
|
||||
// Legacy operation for scheduled jobs
|
||||
logger.info('Fetching symbol summary from IB');
|
||||
const { fetchSession } = await import('./operations/session.operations');
|
||||
const { fetchExchanges } = await import('./operations/exchanges.operations');
|
||||
const { fetchSymbols } = await import('./operations/symbols.operations');
|
||||
|
||||
const sessionHeaders = await fetchSession();
|
||||
logger.info('Fetched symbol summary from IB');
|
||||
|
||||
if (sessionHeaders) {
|
||||
logger.debug('Fetching exchanges from IB');
|
||||
const exchanges = await fetchExchanges(sessionHeaders);
|
||||
logger.info('Fetched exchanges from IB', { count: exchanges?.length });
|
||||
|
||||
logger.debug('Fetching symbols from IB');
|
||||
const symbols = await fetchSymbols(sessionHeaders);
|
||||
logger.info('Fetched symbols from IB', { symbols });
|
||||
|
||||
return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length };
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
},
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'ib-exchanges-and-symbols',
|
||||
operation: 'ib-exchanges-and-symbols',
|
||||
cronPattern: '0 0 * * 0', // Every Sunday at midnight
|
||||
priority: 5,
|
||||
description: 'Fetch and update IB exchanges and symbols data',
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
handlerRegistry.registerWithSchedule(ibProviderConfig);
|
||||
logger.debug('IB provider registered successfully with scheduled jobs');
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* IB Exchanges Operations - Fetching exchange data from IB API
|
||||
*/
|
||||
import { getMongoDBClient } from '@stock-bot/mongodb-client';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { IB_CONFIG } from '../shared/config';
|
||||
|
||||
export async function fetchExchanges(sessionHeaders: Record<string, string>): Promise<unknown[] | null> {
|
||||
const ctx = OperationContext.create('ib', 'exchanges');
|
||||
|
||||
try {
|
||||
ctx.logger.info('🔍 Fetching exchanges with session headers...');
|
||||
|
||||
// The URL for the exchange data API
|
||||
const exchangeUrl = IB_CONFIG.BASE_URL + IB_CONFIG.EXCHANGE_API;
|
||||
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
ctx.logger.info('📤 Making request to exchange API...', {
|
||||
url: exchangeUrl,
|
||||
headerCount: Object.keys(requestHeaders).length,
|
||||
});
|
||||
|
||||
// Use fetch with proxy configuration
|
||||
const response = await fetch(exchangeUrl, {
|
||||
method: 'GET',
|
||||
headers: requestHeaders,
|
||||
proxy: IB_CONFIG.DEFAULT_PROXY,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
ctx.logger.error('❌ Exchange API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const exchanges = data?.exchanges || [];
|
||||
ctx.logger.info('✅ Exchange data fetched successfully');
|
||||
|
||||
ctx.logger.info('Saving IB exchanges to MongoDB...');
|
||||
const client = getMongoDBClient();
|
||||
await client.batchUpsert('ibExchanges', exchanges, ['id', 'country_code']);
|
||||
ctx.logger.info('✅ Exchange IB data saved to MongoDB:', {
|
||||
count: exchanges.length,
|
||||
});
|
||||
|
||||
return exchanges;
|
||||
} catch (error) {
|
||||
ctx.logger.error('❌ Failed to fetch exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
/**
|
||||
* IB Session Operations - Browser automation for session headers
|
||||
*/
|
||||
import { Browser } from '@stock-bot/browser';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { IB_CONFIG } from '../shared/config';
|
||||
|
||||
export async function fetchSession(): Promise<Record<string, string> | undefined> {
|
||||
const ctx = OperationContext.create('ib', 'session');
|
||||
|
||||
try {
|
||||
await Browser.initialize({
|
||||
headless: true,
|
||||
timeout: IB_CONFIG.BROWSER_TIMEOUT,
|
||||
blockResources: false
|
||||
});
|
||||
ctx.logger.info('✅ Browser initialized');
|
||||
|
||||
const { page } = await Browser.createPageWithProxy(
|
||||
IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_PAGE,
|
||||
IB_CONFIG.DEFAULT_PROXY
|
||||
);
|
||||
ctx.logger.info('✅ Page created with proxy');
|
||||
|
||||
const headersPromise = new Promise<Record<string, string> | undefined>(resolve => {
|
||||
let resolved = false;
|
||||
|
||||
page.onNetworkEvent(event => {
|
||||
if (event.url.includes('/webrest/search/product-types/summary')) {
|
||||
if (event.type === 'request') {
|
||||
try {
|
||||
resolve(event.headers);
|
||||
} catch (e) {
|
||||
resolve(undefined);
|
||||
ctx.logger.debug('Raw Summary Response error', { error: (e as Error).message });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Timeout fallback
|
||||
setTimeout(() => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
ctx.logger.warn('Timeout waiting for headers');
|
||||
resolve(undefined);
|
||||
}
|
||||
}, IB_CONFIG.HEADERS_TIMEOUT);
|
||||
});
|
||||
|
||||
ctx.logger.info('⏳ Waiting for page load...');
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: IB_CONFIG.PAGE_LOAD_TIMEOUT });
|
||||
ctx.logger.info('✅ Page loaded');
|
||||
|
||||
//Products tabs
|
||||
ctx.logger.info('🔍 Looking for Products tab...');
|
||||
const productsTab = page.locator('#productSearchTab[role=\"tab\"][href=\"#products\"]');
|
||||
await productsTab.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT });
|
||||
ctx.logger.info('✅ Found Products tab');
|
||||
ctx.logger.info('🖱️ Clicking Products tab...');
|
||||
await productsTab.click();
|
||||
ctx.logger.info('✅ Products tab clicked');
|
||||
|
||||
// New Products Checkbox
|
||||
ctx.logger.info('🔍 Looking for \"New Products Only\" radio button...');
|
||||
const radioButton = page.locator('span.checkbox-text:has-text(\"New Products Only\")');
|
||||
await radioButton.waitFor({ timeout: IB_CONFIG.ELEMENT_TIMEOUT });
|
||||
ctx.logger.info(`🎯 Found \"New Products Only\" radio button`);
|
||||
await radioButton.first().click();
|
||||
ctx.logger.info('✅ \"New Products Only\" radio button clicked');
|
||||
|
||||
// Wait for and return headers immediately when captured
|
||||
ctx.logger.info('⏳ Waiting for headers to be captured...');
|
||||
const headers = await headersPromise;
|
||||
page.close();
|
||||
if (headers) {
|
||||
ctx.logger.info('✅ Headers captured successfully');
|
||||
} else {
|
||||
ctx.logger.warn('⚠️ No headers were captured');
|
||||
}
|
||||
|
||||
return headers;
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to fetch IB symbol summary', { error });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
/**
|
||||
* IB Symbols Operations - Fetching symbol data from IB API
|
||||
*/
|
||||
import { getMongoDBClient } from '@stock-bot/mongodb-client';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { IB_CONFIG } from '../shared/config';
|
||||
|
||||
// Fetch symbols from IB using the session headers
|
||||
export async function fetchSymbols(sessionHeaders: Record<string, string>): Promise<unknown[] | null> {
|
||||
const ctx = OperationContext.create('ib', 'symbols');
|
||||
|
||||
try {
|
||||
ctx.logger.info('🔍 Fetching symbols with session headers...');
|
||||
|
||||
// Prepare headers - include all session headers plus any additional ones
|
||||
const requestHeaders = {
|
||||
...sessionHeaders,
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Cache-Control': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
};
|
||||
|
||||
const requestBody = {
|
||||
domain: 'com',
|
||||
newProduct: 'all',
|
||||
pageNumber: 1,
|
||||
pageSize: 100,
|
||||
productCountry: IB_CONFIG.PRODUCT_COUNTRIES,
|
||||
productSymbol: '',
|
||||
productType: IB_CONFIG.PRODUCT_TYPES,
|
||||
sortDirection: 'asc',
|
||||
sortField: 'symbol',
|
||||
};
|
||||
|
||||
// Get Summary
|
||||
const summaryResponse = await fetch(
|
||||
IB_CONFIG.BASE_URL + IB_CONFIG.SUMMARY_API,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: IB_CONFIG.DEFAULT_PROXY,
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
);
|
||||
|
||||
if (!summaryResponse.ok) {
|
||||
ctx.logger.error('❌ Summary API request failed', {
|
||||
status: summaryResponse.status,
|
||||
statusText: summaryResponse.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const summaryData = await summaryResponse.json();
|
||||
ctx.logger.info('✅ IB Summary data fetched successfully', {
|
||||
totalCount: summaryData[0].totalCount,
|
||||
});
|
||||
|
||||
const symbols = [];
|
||||
requestBody.pageSize = IB_CONFIG.PAGE_SIZE;
|
||||
const pageCount = Math.ceil(summaryData[0].totalCount / IB_CONFIG.PAGE_SIZE) || 0;
|
||||
ctx.logger.info('Fetching Symbols for IB', { pageCount });
|
||||
|
||||
const symbolPromises = [];
|
||||
for (let page = 1; page <= pageCount; page++) {
|
||||
requestBody.pageNumber = page;
|
||||
|
||||
// Fetch symbols for the current page
|
||||
const symbolsResponse = fetch(
|
||||
IB_CONFIG.BASE_URL + IB_CONFIG.PRODUCTS_API,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: requestHeaders,
|
||||
proxy: IB_CONFIG.DEFAULT_PROXY,
|
||||
body: JSON.stringify(requestBody),
|
||||
}
|
||||
);
|
||||
symbolPromises.push(symbolsResponse);
|
||||
}
|
||||
|
||||
const responses = await Promise.all(symbolPromises);
|
||||
for (const response of responses) {
|
||||
if (!response.ok) {
|
||||
ctx.logger.error('❌ Symbols API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
const data = await response.json();
|
||||
const symJson = data?.products || [];
|
||||
if (symJson && symJson.length > 0) {
|
||||
symbols.push(...symJson);
|
||||
} else {
|
||||
ctx.logger.warn('⚠️ No symbols found in response');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (symbols.length === 0) {
|
||||
ctx.logger.warn('⚠️ No symbols fetched from IB');
|
||||
return null;
|
||||
}
|
||||
|
||||
ctx.logger.info('✅ IB symbols fetched successfully, saving to DB...', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
const client = getMongoDBClient();
|
||||
await client.batchUpsert('ib_symbols', symbols, ['symbol', 'exchangeId']);
|
||||
ctx.logger.info('Saved IB symbols to DB', {
|
||||
totalSymbols: symbols.length,
|
||||
});
|
||||
|
||||
return symbols;
|
||||
} catch (error) {
|
||||
ctx.logger.error('❌ Failed to fetch symbols', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
23
apps/data-ingestion/src/handlers/ib/shared/config.ts
Normal file
23
apps/data-ingestion/src/handlers/ib/shared/config.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
/**
|
||||
* Interactive Brokers Configuration Constants
|
||||
*/
|
||||
|
||||
export const IB_CONFIG = {
|
||||
BASE_URL: 'https://www.interactivebrokers.com',
|
||||
PRODUCTS_PAGE: '/en/trading/products-exchanges.php#/',
|
||||
EXCHANGE_API: '/webrest/exchanges',
|
||||
SUMMARY_API: '/webrest/search/product-types/summary',
|
||||
PRODUCTS_API: '/webrest/search/products-by-filters',
|
||||
|
||||
// Browser configuration
|
||||
BROWSER_TIMEOUT: 10000,
|
||||
PAGE_LOAD_TIMEOUT: 20000,
|
||||
ELEMENT_TIMEOUT: 5000,
|
||||
HEADERS_TIMEOUT: 30000,
|
||||
|
||||
// API configuration
|
||||
DEFAULT_PROXY: 'http://doimvbnb-US-rotate:w5fpiwrb9895@p.webshare.io:80',
|
||||
PAGE_SIZE: 500,
|
||||
PRODUCT_COUNTRIES: ['CA', 'US'],
|
||||
PRODUCT_TYPES: ['STK'],
|
||||
};
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
/**
|
||||
* Proxy Check Operations - Checking proxy functionality
|
||||
*/
|
||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
import { ProxyStatsManager } from '../shared/proxy-manager';
|
||||
|
||||
// Shared HTTP client
|
||||
let httpClient: HttpClient;
|
||||
|
||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
||||
if (!httpClient) {
|
||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
||||
}
|
||||
return httpClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a proxy is working
|
||||
*/
|
||||
export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
||||
const ctx = OperationContext.create('proxy', 'check');
|
||||
|
||||
let success = false;
|
||||
ctx.logger.debug(`Checking Proxy:`, {
|
||||
protocol: proxy.protocol,
|
||||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
});
|
||||
|
||||
try {
|
||||
// Test the proxy
|
||||
const client = getHttpClient(ctx);
|
||||
const response = await client.get(PROXY_CONFIG.CHECK_URL, {
|
||||
proxy,
|
||||
timeout: PROXY_CONFIG.CHECK_TIMEOUT,
|
||||
});
|
||||
|
||||
const isWorking = response.status >= 200 && response.status < 300;
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking,
|
||||
lastChecked: new Date(),
|
||||
responseTime: response.responseTime,
|
||||
};
|
||||
|
||||
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
|
||||
success = true;
|
||||
await updateProxyInCache(result, true, ctx);
|
||||
} else {
|
||||
await updateProxyInCache(result, false, ctx);
|
||||
}
|
||||
|
||||
if (proxy.source) {
|
||||
updateProxyStats(proxy.source, success, ctx);
|
||||
}
|
||||
|
||||
ctx.logger.debug('Proxy check completed', {
|
||||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
isWorking,
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const result: ProxyInfo = {
|
||||
...proxy,
|
||||
isWorking: false,
|
||||
error: errorMessage,
|
||||
lastChecked: new Date(),
|
||||
};
|
||||
|
||||
// Update cache for failed proxy (increment total, don't update TTL)
|
||||
await updateProxyInCache(result, false, ctx);
|
||||
|
||||
if (proxy.source) {
|
||||
updateProxyStats(proxy.source, success, ctx);
|
||||
}
|
||||
|
||||
ctx.logger.debug('Proxy check failed', {
|
||||
host: proxy.host,
|
||||
port: proxy.port,
|
||||
error: errorMessage,
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update proxy data in cache with working/total stats and average response time
|
||||
*/
|
||||
async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean, ctx: OperationContext): Promise<void> {
|
||||
const cacheKey = `${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||
|
||||
try {
|
||||
const existing: ProxyInfo | null = await ctx.cache.get(cacheKey);
|
||||
|
||||
// For failed proxies, only update if they already exist
|
||||
if (!isWorking && !existing) {
|
||||
ctx.logger.debug('Proxy not in cache, skipping failed update', {
|
||||
proxy: `${proxy.host}:${proxy.port}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate new average response time if we have a response time
|
||||
let newAverageResponseTime = existing?.averageResponseTime;
|
||||
if (proxy.responseTime !== undefined) {
|
||||
const existingAvg = existing?.averageResponseTime || 0;
|
||||
const existingTotal = existing?.total || 0;
|
||||
|
||||
// Calculate weighted average: (existing_avg * existing_count + new_response) / (existing_count + 1)
|
||||
newAverageResponseTime =
|
||||
existingTotal > 0
|
||||
? (existingAvg * existingTotal + proxy.responseTime) / (existingTotal + 1)
|
||||
: proxy.responseTime;
|
||||
}
|
||||
|
||||
// Build updated proxy data
|
||||
const updated = {
|
||||
...existing,
|
||||
...proxy, // Keep latest proxy info
|
||||
total: (existing?.total || 0) + 1,
|
||||
working: isWorking ? (existing?.working || 0) + 1 : existing?.working || 0,
|
||||
isWorking,
|
||||
lastChecked: new Date(),
|
||||
// Add firstSeen only for new entries
|
||||
...(existing ? {} : { firstSeen: new Date() }),
|
||||
// Update average response time if we calculated a new one
|
||||
...(newAverageResponseTime !== undefined
|
||||
? { averageResponseTime: newAverageResponseTime }
|
||||
: {}),
|
||||
};
|
||||
|
||||
// Calculate success rate
|
||||
updated.successRate = updated.total > 0 ? (updated.working / updated.total) * 100 : 0;
|
||||
|
||||
// Save to cache: reset TTL for working proxies, keep existing TTL for failed ones
|
||||
const cacheOptions = isWorking ? { ttl: PROXY_CONFIG.CACHE_TTL } : undefined;
|
||||
await ctx.cache.set(cacheKey, updated, cacheOptions);
|
||||
|
||||
ctx.logger.debug(`Updated ${isWorking ? 'working' : 'failed'} proxy in cache`, {
|
||||
proxy: `${proxy.host}:${proxy.port}`,
|
||||
working: updated.working,
|
||||
total: updated.total,
|
||||
successRate: updated.successRate.toFixed(1) + '%',
|
||||
avgResponseTime: updated.averageResponseTime
|
||||
? `${updated.averageResponseTime.toFixed(0)}ms`
|
||||
: 'N/A',
|
||||
});
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to update proxy in cache', {
|
||||
proxy: `${proxy.host}:${proxy.port}`,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function updateProxyStats(sourceId: string, success: boolean, ctx: OperationContext) {
|
||||
const statsManager = ProxyStatsManager.getInstance();
|
||||
const source = statsManager.updateSourceStats(sourceId, success);
|
||||
|
||||
if (!source) {
|
||||
ctx.logger.warn(`Unknown proxy source: ${sourceId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cache the updated stats
|
||||
ctx.cache.set(`${PROXY_CONFIG.CACHE_STATS_KEY}:${source.id}`, source, { ttl: PROXY_CONFIG.CACHE_TTL })
|
||||
.catch(error => ctx.logger.debug('Failed to cache proxy stats', { error }));
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
/**
|
||||
* Proxy Fetch Operations - Fetching proxies from sources
|
||||
*/
|
||||
import { HttpClient, ProxyInfo } from '@stock-bot/http';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
import { ProxyStatsManager } from '../shared/proxy-manager';
|
||||
import type { ProxySource } from '../shared/types';
|
||||
|
||||
// Shared HTTP client
|
||||
let httpClient: HttpClient;
|
||||
|
||||
function getHttpClient(ctx: OperationContext): HttpClient {
|
||||
if (!httpClient) {
|
||||
httpClient = new HttpClient({ timeout: 10000 }, ctx.logger);
|
||||
}
|
||||
return httpClient;
|
||||
}
|
||||
|
||||
export async function fetchProxiesFromSources(): Promise<ProxyInfo[]> {
|
||||
const ctx = OperationContext.create('proxy', 'fetch-sources');
|
||||
|
||||
const statsManager = ProxyStatsManager.getInstance();
|
||||
statsManager.resetStats();
|
||||
|
||||
const fetchPromises = PROXY_CONFIG.PROXY_SOURCES.map(source => fetchProxiesFromSource(source, ctx));
|
||||
const results = await Promise.all(fetchPromises);
|
||||
let allProxies: ProxyInfo[] = results.flat();
|
||||
allProxies = removeDuplicateProxies(allProxies);
|
||||
|
||||
ctx.logger.info('Fetched proxies from all sources', { total: allProxies.length });
|
||||
return allProxies;
|
||||
}
|
||||
|
||||
export async function fetchProxiesFromSource(source: ProxySource, ctx?: OperationContext): Promise<ProxyInfo[]> {
|
||||
if (!ctx) {
|
||||
ctx = OperationContext.create('proxy', 'fetch-source');
|
||||
}
|
||||
|
||||
const allProxies: ProxyInfo[] = [];
|
||||
|
||||
try {
|
||||
ctx.logger.info(`Fetching proxies from ${source.url}`);
|
||||
|
||||
const client = getHttpClient(ctx);
|
||||
const response = await client.get(source.url, {
|
||||
timeout: 10000,
|
||||
});
|
||||
|
||||
if (response.status !== 200) {
|
||||
ctx.logger.warn(`Failed to fetch from ${source.url}: ${response.status}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const text = response.data;
|
||||
const lines = text.split('\n').filter((line: string) => line.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
let trimmed = line.trim();
|
||||
trimmed = cleanProxyUrl(trimmed);
|
||||
if (!trimmed || trimmed.startsWith('#')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse formats like \"host:port\" or \"host:port:user:pass\"
|
||||
const parts = trimmed.split(':');
|
||||
if (parts.length >= 2) {
|
||||
const proxy: ProxyInfo = {
|
||||
source: source.id,
|
||||
protocol: source.protocol as 'http' | 'https' | 'socks4' | 'socks5',
|
||||
host: parts[0],
|
||||
port: parseInt(parts[1]),
|
||||
};
|
||||
|
||||
if (!isNaN(proxy.port) && proxy.host) {
|
||||
allProxies.push(proxy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ctx.logger.info(`Parsed ${allProxies.length} proxies from ${source.url}`);
|
||||
} catch (error) {
|
||||
ctx.logger.error(`Error fetching proxies from ${source.url}`, error);
|
||||
return [];
|
||||
}
|
||||
|
||||
return allProxies;
|
||||
}
|
||||
|
||||
// Utility functions
|
||||
function cleanProxyUrl(url: string): string {
|
||||
return url
|
||||
.replace(/^https?:\/\//, '')
|
||||
.replace(/^0+/, '')
|
||||
.replace(/:0+(\d)/g, ':$1');
|
||||
}
|
||||
|
||||
function removeDuplicateProxies(proxies: ProxyInfo[]): ProxyInfo[] {
|
||||
const seen = new Set<string>();
|
||||
const unique: ProxyInfo[] = [];
|
||||
|
||||
for (const proxy of proxies) {
|
||||
const key = `${proxy.protocol}://${proxy.host}:${proxy.port}`;
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
unique.push(proxy);
|
||||
}
|
||||
}
|
||||
|
||||
return unique;
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Proxy Query Operations - Getting active proxies from cache
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { PROXY_CONFIG } from '../shared/config';
|
||||
|
||||
/**
|
||||
* Get a random active proxy from the cache
|
||||
* @param protocol - Optional protocol filter ('http' | 'https' | 'socks4' | 'socks5')
|
||||
* @param minSuccessRate - Minimum success rate percentage (default: 50)
|
||||
* @returns A random working proxy or null if none found
|
||||
*/
|
||||
export async function getRandomActiveProxy(
|
||||
protocol?: 'http' | 'https' | 'socks4' | 'socks5',
|
||||
minSuccessRate: number = 50
|
||||
): Promise<ProxyInfo | null> {
|
||||
const ctx = OperationContext.create('proxy', 'get-random');
|
||||
|
||||
try {
|
||||
// Get all active proxy keys from cache
|
||||
const pattern = protocol
|
||||
? `${PROXY_CONFIG.CACHE_KEY}:${protocol}://*`
|
||||
: `${PROXY_CONFIG.CACHE_KEY}:*`;
|
||||
|
||||
const keys = await ctx.cache.keys(pattern);
|
||||
|
||||
if (keys.length === 0) {
|
||||
ctx.logger.debug('No active proxies found in cache', { pattern });
|
||||
return null;
|
||||
}
|
||||
|
||||
// Shuffle the keys for randomness
|
||||
const shuffledKeys = keys.sort(() => Math.random() - 0.5);
|
||||
|
||||
// Find a working proxy that meets the criteria
|
||||
for (const key of shuffledKeys) {
|
||||
try {
|
||||
const proxyData: ProxyInfo | null = await ctx.cache.get(key);
|
||||
|
||||
if (
|
||||
proxyData &&
|
||||
proxyData.isWorking &&
|
||||
(!proxyData.successRate || proxyData.successRate >= minSuccessRate)
|
||||
) {
|
||||
ctx.logger.debug('Random active proxy selected', {
|
||||
proxy: `${proxyData.host}:${proxyData.port}`,
|
||||
protocol: proxyData.protocol,
|
||||
successRate: proxyData.successRate?.toFixed(1) + '%',
|
||||
avgResponseTime: proxyData.averageResponseTime
|
||||
? `${proxyData.averageResponseTime.toFixed(0)}ms`
|
||||
: 'N/A',
|
||||
});
|
||||
|
||||
return proxyData;
|
||||
}
|
||||
} catch (error) {
|
||||
ctx.logger.debug('Error reading proxy from cache', { key, error: (error as Error).message });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
ctx.logger.debug('No working proxies found meeting criteria', {
|
||||
protocol,
|
||||
minSuccessRate,
|
||||
keysChecked: shuffledKeys.length,
|
||||
});
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
ctx.logger.error('Error getting random active proxy', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
protocol,
|
||||
minSuccessRate,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* Proxy Queue Operations - Queueing proxy operations
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
export async function queueProxyFetch(): Promise<string> {
|
||||
const ctx = OperationContext.create('proxy', 'queue-fetch');
|
||||
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('proxy');
|
||||
const job = await queue.add('proxy-fetch', {
|
||||
handler: 'proxy',
|
||||
operation: 'fetch-and-check',
|
||||
payload: {},
|
||||
priority: 5,
|
||||
});
|
||||
|
||||
const jobId = job.id || 'unknown';
|
||||
ctx.logger.info('Proxy fetch job queued', { jobId });
|
||||
return jobId;
|
||||
}
|
||||
|
||||
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
|
||||
const ctx = OperationContext.create('proxy', 'queue-check');
|
||||
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('proxy');
|
||||
const job = await queue.add('proxy-check', {
|
||||
handler: 'proxy',
|
||||
operation: 'check-specific',
|
||||
payload: { proxies },
|
||||
priority: 3,
|
||||
});
|
||||
|
||||
const jobId = job.id || 'unknown';
|
||||
ctx.logger.info('Proxy check job queued', { jobId, count: proxies.length });
|
||||
return jobId;
|
||||
}
|
||||
86
apps/data-ingestion/src/handlers/proxy/proxy.handler.ts
Normal file
86
apps/data-ingestion/src/handlers/proxy/proxy.handler.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
/**
|
||||
* Proxy Provider for new queue system
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { handlerRegistry, createJobHandler, type HandlerConfigWithSchedule } from '@stock-bot/queue';
|
||||
|
||||
const handlerLogger = getLogger('proxy-handler');
|
||||
|
||||
// Initialize and register the Proxy provider
|
||||
export function initializeProxyProvider() {
|
||||
handlerLogger.debug('Registering proxy provider with scheduled jobs...');
|
||||
|
||||
const proxyProviderConfig: HandlerConfigWithSchedule = {
|
||||
name: 'proxy',
|
||||
|
||||
operations: {
|
||||
'fetch-from-sources': createJobHandler(async () => {
|
||||
// Fetch proxies from all configured sources
|
||||
handlerLogger.info('Processing fetch proxies from sources request');
|
||||
const { fetchProxiesFromSources } = await import('./operations/fetch.operations');
|
||||
const { processItems } = await import('@stock-bot/queue');
|
||||
|
||||
// Fetch all proxies from sources
|
||||
const proxies = await fetchProxiesFromSources();
|
||||
handlerLogger.info('Fetched proxies from sources', { count: proxies.length });
|
||||
|
||||
if (proxies.length === 0) {
|
||||
handlerLogger.warn('No proxies fetched from sources');
|
||||
return { processed: 0, successful: 0 };
|
||||
}
|
||||
|
||||
// Batch process the proxies through check-proxy operation
|
||||
const batchResult = await processItems(proxies, 'proxy', {
|
||||
handler: 'proxy',
|
||||
operation: 'check-proxy',
|
||||
totalDelayHours: 0.083, // 5 minutes (5/60 hours)
|
||||
batchSize: 50, // Process 50 proxies per batch
|
||||
priority: 3,
|
||||
useBatching: true,
|
||||
retries: 1,
|
||||
ttl: 30000, // 30 second timeout per proxy check
|
||||
removeOnComplete: 5,
|
||||
removeOnFail: 3,
|
||||
});
|
||||
|
||||
handlerLogger.info('Batch proxy validation completed', {
|
||||
totalProxies: proxies.length,
|
||||
jobsCreated: batchResult.jobsCreated,
|
||||
mode: batchResult.mode,
|
||||
batchesCreated: batchResult.batchesCreated,
|
||||
duration: `${batchResult.duration}ms`,
|
||||
});
|
||||
|
||||
return {
|
||||
processed: proxies.length,
|
||||
jobsCreated: batchResult.jobsCreated,
|
||||
batchesCreated: batchResult.batchesCreated,
|
||||
mode: batchResult.mode,
|
||||
};
|
||||
}),
|
||||
|
||||
'check-proxy': createJobHandler(async (payload: ProxyInfo) => {
|
||||
// payload is now the raw proxy info object
|
||||
handlerLogger.debug('Processing proxy check request', {
|
||||
proxy: `${payload.host}:${payload.port}`,
|
||||
});
|
||||
const { checkProxy } = await import('./operations/check.operations');
|
||||
return checkProxy(payload);
|
||||
}),
|
||||
},
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'proxy-fetch-and-check',
|
||||
operation: 'fetch-from-sources',
|
||||
cronPattern: '0 0 * * 0', // Every week at midnight on Sunday
|
||||
priority: 0,
|
||||
description: 'Fetch and validate proxy list from sources',
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
handlerRegistry.registerWithSchedule(proxyProviderConfig);
|
||||
handlerLogger.debug('Proxy provider registered successfully with scheduled jobs');
|
||||
}
|
||||
140
apps/data-ingestion/src/handlers/proxy/shared/config.ts
Normal file
140
apps/data-ingestion/src/handlers/proxy/shared/config.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
/**
|
||||
* Proxy Configuration Constants
|
||||
*/
|
||||
|
||||
export const PROXY_CONFIG = {
|
||||
CACHE_KEY: 'active',
|
||||
CACHE_STATS_KEY: 'stats',
|
||||
CACHE_TTL: 86400, // 24 hours
|
||||
CHECK_TIMEOUT: 7000,
|
||||
CHECK_IP: '99.246.102.205',
|
||||
CHECK_URL: 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955',
|
||||
PROXY_SOURCES: [
|
||||
{
|
||||
id: 'prxchk',
|
||||
url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'casals',
|
||||
url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'sunny9577',
|
||||
url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'themiralay',
|
||||
url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'casa-ls',
|
||||
url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'databay',
|
||||
url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'speedx',
|
||||
url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'monosans',
|
||||
url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'murong',
|
||||
url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'vakhov-fresh',
|
||||
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'kangproxy',
|
||||
url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'gfpcom',
|
||||
url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'dpangestuw',
|
||||
url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'gitrecon',
|
||||
url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'vakhov-master',
|
||||
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'breaking-tech',
|
||||
url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'ercindedeoglu',
|
||||
url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
{
|
||||
id: 'tuanminpay',
|
||||
url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',
|
||||
protocol: 'http',
|
||||
},
|
||||
|
||||
{
|
||||
id: 'r00tee-https',
|
||||
url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',
|
||||
protocol: 'https',
|
||||
},
|
||||
{
|
||||
id: 'ercindedeoglu-https',
|
||||
url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',
|
||||
protocol: 'https',
|
||||
},
|
||||
{
|
||||
id: 'vakhov-fresh-https',
|
||||
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt',
|
||||
protocol: 'https',
|
||||
},
|
||||
{
|
||||
id: 'databay-https',
|
||||
url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',
|
||||
protocol: 'https',
|
||||
},
|
||||
{
|
||||
id: 'kangproxy-https',
|
||||
url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',
|
||||
protocol: 'https',
|
||||
},
|
||||
{
|
||||
id: 'zloi-user-https',
|
||||
url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',
|
||||
protocol: 'https',
|
||||
},
|
||||
{
|
||||
id: 'gfpcom-https',
|
||||
url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',
|
||||
protocol: 'https',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Proxy Stats Manager - Singleton for managing proxy statistics
|
||||
*/
|
||||
import type { ProxySource } from './types';
|
||||
import { PROXY_CONFIG } from './config';
|
||||
|
||||
export class ProxyStatsManager {
|
||||
private static instance: ProxyStatsManager | null = null;
|
||||
private proxyStats: ProxySource[] = [];
|
||||
|
||||
private constructor() {
|
||||
this.resetStats();
|
||||
}
|
||||
|
||||
static getInstance(): ProxyStatsManager {
|
||||
if (!ProxyStatsManager.instance) {
|
||||
ProxyStatsManager.instance = new ProxyStatsManager();
|
||||
}
|
||||
return ProxyStatsManager.instance;
|
||||
}
|
||||
|
||||
resetStats(): void {
|
||||
this.proxyStats = PROXY_CONFIG.PROXY_SOURCES.map(source => ({
|
||||
id: source.id,
|
||||
total: 0,
|
||||
working: 0,
|
||||
lastChecked: new Date(),
|
||||
protocol: source.protocol,
|
||||
url: source.url,
|
||||
}));
|
||||
}
|
||||
|
||||
getStats(): ProxySource[] {
|
||||
return [...this.proxyStats];
|
||||
}
|
||||
|
||||
updateSourceStats(sourceId: string, success: boolean): ProxySource | undefined {
|
||||
const source = this.proxyStats.find(s => s.id === sourceId);
|
||||
if (source) {
|
||||
if (typeof source.working !== 'number') {
|
||||
source.working = 0;
|
||||
}
|
||||
if (typeof source.total !== 'number') {
|
||||
source.total = 0;
|
||||
}
|
||||
source.total += 1;
|
||||
if (success) {
|
||||
source.working += 1;
|
||||
}
|
||||
source.percentWorking = (source.working / source.total) * 100;
|
||||
source.lastChecked = new Date();
|
||||
return source;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
13
apps/data-ingestion/src/handlers/proxy/shared/types.ts
Normal file
13
apps/data-ingestion/src/handlers/proxy/shared/types.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/**
|
||||
* Proxy Shared Types
|
||||
*/
|
||||
|
||||
export interface ProxySource {
|
||||
id: string;
|
||||
url: string;
|
||||
protocol: string;
|
||||
working?: number; // Optional, used for stats
|
||||
total?: number; // Optional, used for stats
|
||||
percentWorking?: number; // Optional, used for stats
|
||||
lastChecked?: Date; // Optional, used for stats
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* QM Exchanges Operations - Exchange fetching functionality
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { initializeQMResources } from './session.operations';
|
||||
|
||||
export async function fetchExchanges(): Promise<unknown[] | null> {
|
||||
const ctx = OperationContext.create('qm', 'exchanges');
|
||||
|
||||
try {
|
||||
// Ensure resources are initialized
|
||||
const { QMSessionManager } = await import('../shared/session-manager');
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
ctx.logger.info('QM exchanges fetch - not implemented yet');
|
||||
|
||||
// Cache the "not implemented" status
|
||||
await ctx.cache.set('fetch-status', {
|
||||
implemented: false,
|
||||
message: 'QM exchanges fetching not yet implemented',
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
|
||||
// TODO: Implement QM exchanges fetching logic
|
||||
// This could involve:
|
||||
// 1. Querying existing exchanges from MongoDB
|
||||
// 2. Making API calls to discover new exchanges
|
||||
// 3. Processing and storing exchange metadata
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to fetch QM exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
/**
|
||||
* QM Session Operations - Session creation and management
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
import { isShutdownSignalReceived } from '@stock-bot/shutdown';
|
||||
import { getRandomProxy } from '@stock-bot/utils';
|
||||
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG, getQmHeaders } from '../shared/config';
|
||||
import type { QMSession } from '../shared/types';
|
||||
|
||||
export async function createSessions(): Promise<void> {
|
||||
const ctx = OperationContext.create('qm', 'session');
|
||||
|
||||
try {
|
||||
ctx.logger.info('Creating QM sessions...');
|
||||
|
||||
// Get session manager instance
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
|
||||
// Check if already initialized
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
// Clean up failed sessions first
|
||||
const removedCount = sessionManager.cleanupFailedSessions();
|
||||
if (removedCount > 0) {
|
||||
ctx.logger.info(`Cleaned up ${removedCount} failed sessions`);
|
||||
}
|
||||
|
||||
// Cache session creation stats
|
||||
const initialStats = sessionManager.getStats();
|
||||
await ctx.cache.set('pre-creation-stats', initialStats, { ttl: 300 });
|
||||
|
||||
// Create sessions for each session ID that needs them
|
||||
for (const [sessionKey, sessionId] of Object.entries(QM_SESSION_IDS)) {
|
||||
if (sessionManager.isAtCapacity(sessionId)) {
|
||||
ctx.logger.debug(`Session ID ${sessionKey} is at capacity, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
while (sessionManager.needsMoreSessions(sessionId)) {
|
||||
if (isShutdownSignalReceived()) {
|
||||
ctx.logger.info('Shutting down, skipping session creation');
|
||||
return;
|
||||
}
|
||||
|
||||
await createSingleSession(sessionId, sessionKey, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
// Cache final stats and session count
|
||||
const finalStats = sessionManager.getStats();
|
||||
const totalSessions = sessionManager.getSessionCount();
|
||||
|
||||
await ctx.cache.set('post-creation-stats', finalStats, { ttl: 3600 });
|
||||
await ctx.cache.set('session-count', totalSessions, { ttl: 900 });
|
||||
await ctx.cache.set('last-session-creation', new Date().toISOString());
|
||||
|
||||
ctx.logger.info('QM session creation completed', {
|
||||
totalSessions,
|
||||
sessionStats: finalStats
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to create QM sessions', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function createSingleSession(
|
||||
sessionId: string,
|
||||
sessionKey: string,
|
||||
ctx: OperationContext
|
||||
): Promise<void> {
|
||||
ctx.logger.debug(`Creating new session for ${sessionKey}`, { sessionId });
|
||||
|
||||
const proxyInfo = await getRandomProxy();
|
||||
if (!proxyInfo) {
|
||||
ctx.logger.error('No proxy available for QM session creation');
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert ProxyInfo to string format
|
||||
const auth = proxyInfo.username && proxyInfo.password ?
|
||||
`${proxyInfo.username}:${proxyInfo.password}@` : '';
|
||||
const proxy = `${proxyInfo.protocol}://${auth}${proxyInfo.host}:${proxyInfo.port}`;
|
||||
|
||||
const newSession: QMSession = {
|
||||
proxy: proxy,
|
||||
headers: getQmHeaders(),
|
||||
successfulCalls: 0,
|
||||
failedCalls: 0,
|
||||
lastUsed: new Date(),
|
||||
};
|
||||
|
||||
try {
|
||||
const sessionResponse = await fetch(
|
||||
`${QM_CONFIG.BASE_URL}${QM_CONFIG.AUTH_PATH}/${sessionId}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: newSession.headers,
|
||||
signal: AbortSignal.timeout(SESSION_CONFIG.SESSION_TIMEOUT),
|
||||
}
|
||||
);
|
||||
|
||||
ctx.logger.debug('Session response received', {
|
||||
status: sessionResponse.status,
|
||||
sessionKey,
|
||||
});
|
||||
|
||||
if (!sessionResponse.ok) {
|
||||
ctx.logger.error('Failed to create QM session', {
|
||||
sessionKey,
|
||||
sessionId,
|
||||
status: sessionResponse.status,
|
||||
statusText: sessionResponse.statusText,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionData = await sessionResponse.json();
|
||||
|
||||
// Add token to headers
|
||||
newSession.headers['Datatool-Token'] = sessionData.token;
|
||||
|
||||
// Add session to manager
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
sessionManager.addSession(sessionId, newSession);
|
||||
|
||||
// Cache successful session creation
|
||||
await ctx.cache.set(
|
||||
`successful-session:${sessionKey}:${Date.now()}`,
|
||||
{ sessionId, proxy, tokenExists: !!sessionData.token },
|
||||
{ ttl: 300 }
|
||||
);
|
||||
|
||||
ctx.logger.info('QM session created successfully', {
|
||||
sessionKey,
|
||||
sessionId,
|
||||
proxy: newSession.proxy,
|
||||
sessionCount: sessionManager.getSessions(sessionId).length,
|
||||
hasToken: !!sessionData.token
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
if (error.name === 'TimeoutError') {
|
||||
ctx.logger.warn('QM session creation timed out', { sessionKey, sessionId });
|
||||
} else {
|
||||
ctx.logger.error('Error creating QM session', { sessionKey, sessionId, error });
|
||||
}
|
||||
|
||||
// Cache failed session attempt for debugging
|
||||
await ctx.cache.set(
|
||||
`failed-session:${sessionKey}:${Date.now()}`,
|
||||
{ sessionId, proxy, error: error.message },
|
||||
{ ttl: 300 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function initializeQMResources(): Promise<void> {
|
||||
const ctx = OperationContext.create('qm', 'init');
|
||||
|
||||
// Check if already initialized
|
||||
const alreadyInitialized = await ctx.cache.get('initialized');
|
||||
if (alreadyInitialized) {
|
||||
ctx.logger.debug('QM resources already initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.logger.debug('Initializing QM resources...');
|
||||
|
||||
// Mark as initialized in cache and session manager
|
||||
await ctx.cache.set('initialized', true, { ttl: 3600 });
|
||||
await ctx.cache.set('initialization-time', new Date().toISOString());
|
||||
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
sessionManager.setInitialized(true);
|
||||
|
||||
ctx.logger.info('QM resources initialized successfully');
|
||||
}
|
||||
|
|
@ -0,0 +1,268 @@
|
|||
/**
|
||||
* QM Spider Operations - Symbol spider search functionality
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
import { QM_SESSION_IDS } from '../shared/config';
|
||||
import type { SymbolSpiderJob, SpiderResult } from '../shared/types';
|
||||
import { initializeQMResources } from './session.operations';
|
||||
import { searchQMSymbolsAPI } from './symbols.operations';
|
||||
|
||||
export async function spiderSymbolSearch(
|
||||
payload: SymbolSpiderJob
|
||||
): Promise<SpiderResult> {
|
||||
const ctx = OperationContext.create('qm', 'spider');
|
||||
|
||||
try {
|
||||
const { prefix, depth, source = 'qm', maxDepth = 4 } = payload;
|
||||
|
||||
ctx.logger.info('Starting spider search', {
|
||||
prefix: prefix || 'ROOT',
|
||||
depth,
|
||||
source,
|
||||
maxDepth
|
||||
});
|
||||
|
||||
// Check cache for recent results
|
||||
const cacheKey = `search-result:${prefix || 'ROOT'}:${depth}`;
|
||||
const cachedResult = await ctx.cache.get<SpiderResult>(cacheKey);
|
||||
if (cachedResult) {
|
||||
ctx.logger.debug('Using cached spider search result', { prefix, depth });
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
// Ensure resources are initialized
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
let result: SpiderResult;
|
||||
|
||||
// Root job: Create A-Z jobs
|
||||
if (prefix === null || prefix === undefined || prefix === '') {
|
||||
result = await createAlphabetJobs(source, maxDepth, ctx);
|
||||
} else {
|
||||
// Leaf job: Search for symbols with this prefix
|
||||
result = await searchAndSpawnJobs(prefix, depth, source, maxDepth, ctx);
|
||||
}
|
||||
|
||||
// Cache the result
|
||||
await ctx.cache.set(cacheKey, result, { ttl: 3600 });
|
||||
|
||||
// Store spider operation metrics in cache instead of PostgreSQL for now
|
||||
try {
|
||||
const statsKey = `spider-stats:${prefix || 'ROOT'}:${depth}:${Date.now()}`;
|
||||
await ctx.cache.set(statsKey, {
|
||||
handler: 'qm',
|
||||
operation: 'spider',
|
||||
prefix: prefix || 'ROOT',
|
||||
depth,
|
||||
symbolsFound: result.symbolsFound,
|
||||
jobsCreated: result.jobsCreated,
|
||||
searchTime: new Date().toISOString()
|
||||
}, { ttl: 86400 }); // Keep for 24 hours
|
||||
} catch (error) {
|
||||
ctx.logger.debug('Failed to store spider stats in cache', { error });
|
||||
}
|
||||
|
||||
ctx.logger.info('Spider search completed', {
|
||||
prefix: prefix || 'ROOT',
|
||||
depth,
|
||||
success: result.success,
|
||||
symbolsFound: result.symbolsFound,
|
||||
jobsCreated: result.jobsCreated
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error('Spider symbol search failed', { error, payload });
|
||||
const failedResult = { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
|
||||
// Cache failed result for a shorter time
|
||||
const cacheKey = `search-result:${payload.prefix || 'ROOT'}:${payload.depth}`;
|
||||
await ctx.cache.set(cacheKey, failedResult, { ttl: 300 });
|
||||
|
||||
return failedResult;
|
||||
}
|
||||
}
|
||||
|
||||
async function createAlphabetJobs(
|
||||
source: string,
|
||||
maxDepth: number,
|
||||
ctx: OperationContext
|
||||
): Promise<SpiderResult> {
|
||||
try {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
let jobsCreated = 0;
|
||||
|
||||
ctx.logger.info('Creating alphabet jobs (A-Z)');
|
||||
|
||||
// Create jobs for A-Z
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i); // A=65, B=66, etc.
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: letter,
|
||||
depth: 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: 5,
|
||||
delay: i * 100, // Stagger jobs by 100ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
// Cache alphabet job creation
|
||||
await ctx.cache.set('alphabet-jobs-created', {
|
||||
count: jobsCreated,
|
||||
timestamp: new Date().toISOString(),
|
||||
source,
|
||||
maxDepth
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(`Created ${jobsCreated} alphabet jobs (A-Z)`);
|
||||
return { success: true, symbolsFound: 0, jobsCreated };
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to create alphabet jobs', { error });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function searchAndSpawnJobs(
|
||||
prefix: string,
|
||||
depth: number,
|
||||
source: string,
|
||||
maxDepth: number,
|
||||
ctx: OperationContext
|
||||
): Promise<SpiderResult> {
|
||||
try {
|
||||
// Ensure sessions exist for symbol search
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
const lookupSession = sessionManager.getSession(QM_SESSION_IDS.LOOKUP);
|
||||
|
||||
if (!lookupSession) {
|
||||
ctx.logger.info('No lookup sessions available, creating sessions first...');
|
||||
const { createSessions } = await import('./session.operations');
|
||||
await createSessions();
|
||||
|
||||
// Wait a bit for session creation
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
// Search for symbols with this prefix
|
||||
const symbols = await searchQMSymbolsAPI(prefix);
|
||||
const symbolCount = symbols.length;
|
||||
|
||||
ctx.logger.info(`Prefix "${prefix}" returned ${symbolCount} symbols`);
|
||||
|
||||
let jobsCreated = 0;
|
||||
|
||||
// Store symbols in MongoDB
|
||||
if (ctx.mongodb && symbols.length > 0) {
|
||||
try {
|
||||
const updatedSymbols = symbols.map((symbol: Record<string, unknown>) => ({
|
||||
...symbol,
|
||||
qmSearchCode: symbol.symbol,
|
||||
symbol: (symbol.symbol as string)?.split(':')[0],
|
||||
searchPrefix: prefix,
|
||||
searchDepth: depth,
|
||||
discoveredAt: new Date()
|
||||
}));
|
||||
|
||||
await ctx.mongodb.batchUpsert('qmSymbols', updatedSymbols, ['qmSearchCode']);
|
||||
ctx.logger.debug('Stored symbols in MongoDB', { count: symbols.length });
|
||||
} catch (error) {
|
||||
ctx.logger.warn('Failed to store symbols in MongoDB', { error });
|
||||
}
|
||||
}
|
||||
|
||||
// If we have 50+ symbols and haven't reached max depth, spawn sub-jobs
|
||||
if (symbolCount >= 50 && depth < maxDepth) {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
|
||||
ctx.logger.info(`Spawning sub-jobs for prefix "${prefix}" (${symbolCount} >= 50 symbols)`);
|
||||
|
||||
// Create jobs for prefixA, prefixB, prefixC... prefixZ
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i);
|
||||
const newPrefix = prefix + letter;
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: newPrefix,
|
||||
depth: depth + 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: Math.max(1, 6 - depth), // Higher priority for deeper jobs
|
||||
delay: i * 50, // Stagger sub-jobs by 50ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
// Cache sub-job creation info
|
||||
await ctx.cache.set(`sub-jobs:${prefix}`, {
|
||||
parentPrefix: prefix,
|
||||
depth,
|
||||
symbolCount,
|
||||
jobsCreated,
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(`Created ${jobsCreated} sub-jobs for prefix "${prefix}"`);
|
||||
} else {
|
||||
// Terminal case: save symbols (already done above)
|
||||
ctx.logger.info(`Terminal case for prefix "${prefix}": ${symbolCount} symbols saved`);
|
||||
|
||||
// Cache terminal case info
|
||||
await ctx.cache.set(`terminal:${prefix}`, {
|
||||
prefix,
|
||||
depth,
|
||||
symbolCount,
|
||||
isTerminal: true,
|
||||
reason: symbolCount < 50 ? 'insufficient_symbols' : 'max_depth_reached',
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
}
|
||||
|
||||
return { success: true, symbolsFound: symbolCount, jobsCreated };
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error(`Failed to search and spawn jobs for prefix "${prefix}"`, { error, depth });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
/**
|
||||
* QM Symbols Operations - Symbol fetching and API interactions
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
import { getRandomProxy } from '@stock-bot/utils';
|
||||
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG } from '../shared/config';
|
||||
import type { SymbolSpiderJob, Exchange } from '../shared/types';
|
||||
import { initializeQMResources } from './session.operations';
|
||||
import { spiderSymbolSearch } from './spider.operations';
|
||||
|
||||
export async function fetchSymbols(): Promise<unknown[] | null> {
|
||||
const ctx = OperationContext.create('qm', 'symbols');
|
||||
|
||||
try {
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
ctx.logger.info('Starting QM spider-based symbol search...');
|
||||
|
||||
// Check if we have a recent symbol fetch
|
||||
const lastFetch = await ctx.cache.get('last-symbol-fetch');
|
||||
if (lastFetch) {
|
||||
ctx.logger.info('Recent symbol fetch found, using spider search');
|
||||
}
|
||||
|
||||
// Start the spider process with root job
|
||||
const rootJob: SymbolSpiderJob = {
|
||||
prefix: null, // Root job creates A-Z jobs
|
||||
depth: 0,
|
||||
source: 'qm',
|
||||
maxDepth: 4,
|
||||
};
|
||||
|
||||
const result = await spiderSymbolSearch(rootJob);
|
||||
|
||||
if (result.success) {
|
||||
// Cache successful fetch info
|
||||
await ctx.cache.set('last-symbol-fetch', {
|
||||
timestamp: new Date().toISOString(),
|
||||
jobsCreated: result.jobsCreated,
|
||||
success: true
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(
|
||||
`QM spider search initiated successfully. Created ${result.jobsCreated} initial jobs`
|
||||
);
|
||||
return [`Spider search initiated with ${result.jobsCreated} jobs`];
|
||||
} else {
|
||||
ctx.logger.error('Failed to initiate QM spider search');
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to start QM spider symbol search', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function searchQMSymbolsAPI(query: string): Promise<any[]> {
|
||||
const ctx = OperationContext.create('qm', 'api-search');
|
||||
|
||||
const proxyInfo = await getRandomProxy();
|
||||
if (!proxyInfo) {
|
||||
throw new Error('No proxy available for QM API call');
|
||||
}
|
||||
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
const session = sessionManager.getSession(QM_SESSION_IDS.LOOKUP);
|
||||
|
||||
if (!session) {
|
||||
throw new Error(`No active session found for QM API with ID: ${QM_SESSION_IDS.LOOKUP}`);
|
||||
}
|
||||
|
||||
try {
|
||||
ctx.logger.debug('Searching QM symbols API', { query, proxy: session.proxy });
|
||||
|
||||
// Check cache for recent API results
|
||||
const cacheKey = `api-search:${query}`;
|
||||
const cachedResult = await ctx.cache.get(cacheKey);
|
||||
if (cachedResult) {
|
||||
ctx.logger.debug('Using cached API search result', { query });
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
// QM lookup endpoint for symbol search
|
||||
const searchParams = new URLSearchParams({
|
||||
marketType: 'equity',
|
||||
pathName: '/demo/portal/company-summary.php',
|
||||
q: query,
|
||||
qmodTool: 'SmartSymbolLookup',
|
||||
searchType: 'symbol',
|
||||
showFree: 'false',
|
||||
showHisa: 'false',
|
||||
webmasterId: '500'
|
||||
});
|
||||
|
||||
const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`;
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
headers: session.headers,
|
||||
signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const symbols = await response.json();
|
||||
|
||||
// Update session stats
|
||||
session.successfulCalls++;
|
||||
session.lastUsed = new Date();
|
||||
|
||||
// Process symbols and extract exchanges
|
||||
if (ctx.mongodb && symbols.length > 0) {
|
||||
try {
|
||||
const updatedSymbols = symbols.map((symbol: Record<string, unknown>) => ({
|
||||
...symbol,
|
||||
qmSearchCode: symbol.symbol,
|
||||
symbol: (symbol.symbol as string)?.split(':')[0],
|
||||
searchQuery: query,
|
||||
fetchedAt: new Date()
|
||||
}));
|
||||
|
||||
await ctx.mongodb.batchUpsert('qmSymbols', updatedSymbols, ['qmSearchCode']);
|
||||
|
||||
// Extract and store unique exchanges
|
||||
const exchanges: Exchange[] = [];
|
||||
for (const symbol of symbols) {
|
||||
if (!exchanges.some(ex => ex.exchange === symbol.exchange)) {
|
||||
exchanges.push({
|
||||
exchange: symbol.exchange,
|
||||
exchangeCode: symbol.exchangeCode,
|
||||
exchangeShortName: symbol.exchangeShortName,
|
||||
countryCode: symbol.countryCode,
|
||||
source: 'qm',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (exchanges.length > 0) {
|
||||
await ctx.mongodb.batchUpsert('qmExchanges', exchanges, ['exchange']);
|
||||
ctx.logger.debug('Stored exchanges in MongoDB', { count: exchanges.length });
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.warn('Failed to store symbols/exchanges in MongoDB', { error });
|
||||
}
|
||||
}
|
||||
|
||||
// Cache the result
|
||||
await ctx.cache.set(cacheKey, symbols, { ttl: 1800 }); // 30 minutes
|
||||
|
||||
// Store API call stats
|
||||
await ctx.cache.set(`api-stats:${query}:${Date.now()}`, {
|
||||
query,
|
||||
symbolCount: symbols.length,
|
||||
proxy: session.proxy,
|
||||
success: true,
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(
|
||||
`QM API returned ${symbols.length} symbols for query: ${query}`,
|
||||
{ proxy: session.proxy, symbolCount: symbols.length }
|
||||
);
|
||||
|
||||
return symbols;
|
||||
|
||||
} catch (error) {
|
||||
// Update session failure stats
|
||||
session.failedCalls++;
|
||||
session.lastUsed = new Date();
|
||||
|
||||
// Cache failed API call info
|
||||
await ctx.cache.set(`api-failure:${query}:${Date.now()}`, {
|
||||
query,
|
||||
error: error.message,
|
||||
proxy: session.proxy,
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 600 });
|
||||
|
||||
ctx.logger.error(`Error searching QM symbols for query "${query}"`, {
|
||||
error: error.message,
|
||||
proxy: session.proxy
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
78
apps/data-ingestion/src/handlers/qm/qm.handler.ts
Normal file
78
apps/data-ingestion/src/handlers/qm/qm.handler.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import {
|
||||
createJobHandler,
|
||||
handlerRegistry,
|
||||
type HandlerConfigWithSchedule
|
||||
} from '@stock-bot/queue';
|
||||
import type { SymbolSpiderJob } from './shared/types';
|
||||
|
||||
const handlerLogger = getLogger('qm-handler');
|
||||
|
||||
// Initialize and register the QM provider
|
||||
export function initializeQMProvider() {
|
||||
handlerLogger.debug('Registering QM provider with scheduled jobs...');
|
||||
|
||||
const qmProviderConfig: HandlerConfigWithSchedule = {
|
||||
name: 'qm',
|
||||
operations: {
|
||||
'create-sessions': createJobHandler(async () => {
|
||||
const { createSessions } = await import('./operations/session.operations');
|
||||
await createSessions();
|
||||
return { success: true, message: 'QM sessions created successfully' };
|
||||
}),
|
||||
'search-symbols': createJobHandler(async () => {
|
||||
const { fetchSymbols } = await import('./operations/symbols.operations');
|
||||
const symbols = await fetchSymbols();
|
||||
|
||||
if (symbols && symbols.length > 0) {
|
||||
return {
|
||||
success: true,
|
||||
message: 'QM symbol search completed successfully',
|
||||
count: symbols.length,
|
||||
symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
message: 'No symbols found',
|
||||
count: 0,
|
||||
};
|
||||
}
|
||||
}),
|
||||
'spider-symbol-search': createJobHandler(async (payload: SymbolSpiderJob) => {
|
||||
const { spiderSymbolSearch } = await import('./operations/spider.operations');
|
||||
const result = await spiderSymbolSearch(payload);
|
||||
|
||||
return result;
|
||||
}),
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'session-management',
|
||||
operation: 'create-sessions',
|
||||
cronPattern: '0 */15 * * *', // Every 15 minutes
|
||||
priority: 7,
|
||||
immediately: true, // Don't run on startup to avoid blocking
|
||||
description: 'Create and maintain QM sessions',
|
||||
},
|
||||
{
|
||||
type: 'qm-maintnance',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: {
|
||||
prefix: null,
|
||||
depth: 1,
|
||||
source: 'qm',
|
||||
maxDepth: 4
|
||||
},
|
||||
cronPattern: '0 0 * * 0', // Every Sunday at midnight
|
||||
priority: 10,
|
||||
immediately: true, // Don't run on startup - this is a heavy operation
|
||||
description: 'Comprehensive symbol search using QM API',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
handlerRegistry.registerWithSchedule(qmProviderConfig);
|
||||
handlerLogger.debug('QM provider registered successfully with scheduled jobs');
|
||||
}
|
||||
420
apps/data-ingestion/src/handlers/qm/qm.operations.ts.old
Normal file
420
apps/data-ingestion/src/handlers/qm/qm.operations.ts.old
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
import { getRandomUserAgent } from '@stock-bot/http';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { getMongoDBClient } from '@stock-bot/mongodb-client';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
import { isShutdownSignalReceived } from '@stock-bot/shutdown';
|
||||
import { getRandomProxy } from '@stock-bot/utils';
|
||||
|
||||
// Shared instances (module-scoped, not global)
|
||||
let isInitialized = false; // Track if resources are initialized
|
||||
let logger: ReturnType<typeof getLogger>;
|
||||
// let cache: CacheProvider;
|
||||
|
||||
export interface QMSession {
|
||||
proxy: string;
|
||||
headers: Record<string, string>;
|
||||
successfulCalls: number;
|
||||
failedCalls: number;
|
||||
lastUsed: Date;
|
||||
}
|
||||
|
||||
export interface SymbolSpiderJob {
|
||||
prefix: string | null; // null = root job (A-Z)
|
||||
depth: number; // 1=A, 2=AA, 3=AAA, etc.
|
||||
source: string; // 'qm'
|
||||
maxDepth?: number; // optional max depth limit
|
||||
}
|
||||
|
||||
interface Exchange {
|
||||
exchange: string;
|
||||
exchangeCode: string;
|
||||
exchangeShortName: string;
|
||||
countryCode: string;
|
||||
source: string;
|
||||
}
|
||||
|
||||
function getQmHeaders(): Record<string, string> {
|
||||
return {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
Accept: '*/*',
|
||||
'Accept-Language': 'en',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
Origin: 'https://www.quotemedia.com',
|
||||
Referer: 'https://www.quotemedia.com/',
|
||||
};
|
||||
}
|
||||
|
||||
const sessionCache: Record<string, QMSession[]> = {
|
||||
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
|
||||
// cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
|
||||
// '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
|
||||
// '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
|
||||
// a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
|
||||
// a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
|
||||
// b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
|
||||
dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6: [], //fceb3c4bdd // lookup
|
||||
// '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
|
||||
// '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
|
||||
// '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
|
||||
// d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
|
||||
// d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
|
||||
// '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
|
||||
};
|
||||
|
||||
export async function initializeQMResources(): Promise<void> {
|
||||
// Skip if already initialized
|
||||
if (isInitialized) {
|
||||
return;
|
||||
}
|
||||
logger = getLogger('qm-tasks');
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
export async function createSessions(): Promise<void> {
|
||||
try {
|
||||
//for each session, check array length, if less than 5, create new session
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
logger.info('Creating QM sessions...');
|
||||
for (const [sessionId, sessionArray] of Object.entries(sessionCache)) {
|
||||
const initialCount = sessionArray.length;
|
||||
const filteredArray = sessionArray.filter(session => session.failedCalls <= 10);
|
||||
sessionCache[sessionId] = filteredArray;
|
||||
|
||||
const removedCount = initialCount - filteredArray.length;
|
||||
if (removedCount > 0) {
|
||||
logger.info(
|
||||
`Removed ${removedCount} sessions with excessive failures for ${sessionId}. Remaining: ${filteredArray.length}`
|
||||
);
|
||||
}
|
||||
|
||||
while (sessionCache[sessionId].length < 10) {
|
||||
if(isShutdownSignalReceived()) {
|
||||
logger.info('Shutting down, skipping session creation');
|
||||
break; // Exit if shutting down
|
||||
}
|
||||
logger.info(`Creating new session for ${sessionId}`);
|
||||
const proxyInfo = await getRandomProxy();
|
||||
if (!proxyInfo) {
|
||||
logger.error('No proxy available for QM session creation');
|
||||
break; // Skip session creation if no proxy is available
|
||||
}
|
||||
|
||||
// Convert ProxyInfo to string format
|
||||
const auth = proxyInfo.username && proxyInfo.password ? `${proxyInfo.username}:${proxyInfo.password}@` : '';
|
||||
const proxy = `${proxyInfo.protocol}://${auth}${proxyInfo.host}:${proxyInfo.port}`;
|
||||
const newSession: QMSession = {
|
||||
proxy: proxy, // Placeholder, should be set to a valid proxy
|
||||
headers: getQmHeaders(),
|
||||
successfulCalls: 0,
|
||||
failedCalls: 0,
|
||||
lastUsed: new Date(),
|
||||
};
|
||||
const sessionResponse = await fetch(
|
||||
`https://app.quotemedia.com/auth/g/authenticate/dataTool/v0/500/${sessionId}`,
|
||||
{
|
||||
method: 'GET',
|
||||
proxy: newSession.proxy,
|
||||
headers: newSession.headers,
|
||||
}
|
||||
);
|
||||
|
||||
logger.debug('Session response received', {
|
||||
status: sessionResponse.status,
|
||||
sessionId,
|
||||
});
|
||||
if (!sessionResponse.ok) {
|
||||
logger.error('Failed to create QM session', {
|
||||
sessionId,
|
||||
status: sessionResponse.status,
|
||||
statusText: sessionResponse.statusText,
|
||||
});
|
||||
continue; // Skip this session if creation failed
|
||||
}
|
||||
const sessionData = await sessionResponse.json();
|
||||
logger.info('QM session created successfully', {
|
||||
sessionId,
|
||||
sessionData,
|
||||
proxy: newSession.proxy,
|
||||
sessionCount: sessionCache[sessionId].length + 1,
|
||||
});
|
||||
newSession.headers['Datatool-Token'] = sessionData.token;
|
||||
sessionCache[sessionId].push(newSession);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch QM session', { error });
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Spider-based symbol search functions
|
||||
export async function spiderSymbolSearch(
|
||||
payload: SymbolSpiderJob
|
||||
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
const { prefix, depth, source = 'qm', maxDepth = 4 } = payload;
|
||||
|
||||
logger.info(`Starting spider search`, { prefix: prefix || 'ROOT', depth, source });
|
||||
|
||||
// Root job: Create A-Z jobs
|
||||
if (prefix === null || prefix === undefined || prefix === '') {
|
||||
return await createAlphabetJobs(source, maxDepth);
|
||||
}
|
||||
|
||||
// Leaf job: Search for symbols with this prefix
|
||||
return await searchAndSpawnJobs(prefix, depth, source, maxDepth);
|
||||
} catch (error) {
|
||||
logger.error('Spider symbol search failed', { error, payload });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function createAlphabetJobs(
|
||||
source: string,
|
||||
maxDepth: number
|
||||
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
||||
try {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
let jobsCreated = 0;
|
||||
|
||||
// Create jobs for A-Z
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i); // A=65, B=66, etc.
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: letter,
|
||||
depth: 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: 5,
|
||||
delay: i * 100, // Stagger jobs by 100ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
logger.info(`Created ${jobsCreated} alphabet jobs (A-Z)`);
|
||||
return { success: true, symbolsFound: 0, jobsCreated };
|
||||
} catch (error) {
|
||||
logger.error('Failed to create alphabet jobs', { error });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function searchAndSpawnJobs(
|
||||
prefix: string,
|
||||
depth: number,
|
||||
source: string,
|
||||
maxDepth: number
|
||||
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
||||
try {
|
||||
// Ensure sessions exist
|
||||
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6';
|
||||
const currentSessions = sessionCache[sessionId] || [];
|
||||
|
||||
if (currentSessions.length === 0) {
|
||||
logger.info('No sessions found, creating sessions first...');
|
||||
await createSessions();
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
// Search for symbols with this prefix
|
||||
const symbols = await searchQMSymbolsAPI(prefix);
|
||||
const symbolCount = symbols.length;
|
||||
|
||||
logger.info(`Prefix "${prefix}" returned ${symbolCount} symbols`);
|
||||
|
||||
let jobsCreated = 0;
|
||||
|
||||
// If we have 50+ symbols and haven't reached max depth, spawn sub-jobs
|
||||
if (symbolCount >= 50 && depth < maxDepth) {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
|
||||
logger.info(`Spawning sub-jobs for prefix "${prefix}" (${symbolCount} >= 50 symbols)`);
|
||||
|
||||
// Create jobs for prefixA, prefixB, prefixC... prefixZ
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i);
|
||||
const newPrefix = prefix + letter;
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: newPrefix,
|
||||
depth: depth + 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: Math.max(1, 6 - depth), // Higher priority for deeper jobs
|
||||
delay: i * 50, // Stagger sub-jobs by 50ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
logger.info(`Created ${jobsCreated} sub-jobs for prefix "${prefix}"`);
|
||||
} else {
|
||||
// Terminal case: save symbols and exchanges (already done in searchQMSymbolsAPI)
|
||||
logger.info(`Terminal case for prefix "${prefix}": ${symbolCount} symbols saved`);
|
||||
}
|
||||
|
||||
return { success: true, symbolsFound: symbolCount, jobsCreated };
|
||||
} catch (error) {
|
||||
logger.error(`Failed to search and spawn jobs for prefix "${prefix}"`, { error, depth });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// API call function to search symbols via QM
|
||||
async function searchQMSymbolsAPI(query: string): Promise<string[]> {
|
||||
const proxyInfo = await getRandomProxy();
|
||||
|
||||
if (!proxyInfo) {
|
||||
throw new Error('No proxy available for QM API call');
|
||||
}
|
||||
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6'; // Use the session ID for symbol lookup
|
||||
const session =
|
||||
sessionCache[sessionId][Math.floor(Math.random() * sessionCache[sessionId].length)]; // lookup session
|
||||
if (!session) {
|
||||
throw new Error(`No active session found for QM API with ID: ${sessionId}`);
|
||||
}
|
||||
try {
|
||||
// QM lookup endpoint for symbol search
|
||||
const apiUrl = `https://app.quotemedia.com/datatool/lookup.json?marketType=equity&pathName=%2Fdemo%2Fportal%2Fcompany-summary.php&q=${encodeURIComponent(query)}&qmodTool=SmartSymbolLookup&searchType=symbol&showFree=false&showHisa=false&webmasterId=500`;
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
headers: session.headers,
|
||||
proxy: session.proxy,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const symbols = await response.json();
|
||||
const mongoClient = getMongoDBClient();
|
||||
const updatedSymbols = symbols.map((symbol: Record<string, unknown>) => {
|
||||
return {
|
||||
...symbol,
|
||||
qmSearchCode: symbol.symbol, // Store original symbol for reference
|
||||
symbol: symbol.symbol.split(':')[0], // Extract symbol from "symbol:exchange"
|
||||
};
|
||||
});
|
||||
await mongoClient.batchUpsert('qmSymbols', updatedSymbols, ['qmSearchCode']);
|
||||
const exchanges: Exchange[] = [];
|
||||
for (const symbol of symbols) {
|
||||
if (!exchanges.some(ex => ex.exchange === symbol.exchange)) {
|
||||
exchanges.push({
|
||||
exchange: symbol.exchange,
|
||||
exchangeCode: symbol.exchangeCode,
|
||||
exchangeShortName: symbol.exchangeShortName,
|
||||
countryCode: symbol.countryCode,
|
||||
source: 'qm',
|
||||
});
|
||||
}
|
||||
}
|
||||
await mongoClient.batchUpsert('qmExchanges', exchanges, ['exchange']);
|
||||
session.successfulCalls++;
|
||||
session.lastUsed = new Date();
|
||||
|
||||
logger.info(
|
||||
`QM API returned ${symbols.length} symbols for query: ${query} with proxy ${session.proxy}`
|
||||
);
|
||||
return symbols;
|
||||
} catch (error) {
|
||||
logger.error(`Error searching QM symbols for query "${query}":`, error);
|
||||
if (session) {
|
||||
session.failedCalls++;
|
||||
session.lastUsed = new Date();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchSymbols(): Promise<unknown[] | null> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
logger.info('🔄 Starting QM spider-based symbol search...');
|
||||
|
||||
// Start the spider process with root job
|
||||
const rootJob: SymbolSpiderJob = {
|
||||
prefix: null, // Root job creates A-Z jobs
|
||||
depth: 0,
|
||||
source: 'qm',
|
||||
maxDepth: 4,
|
||||
};
|
||||
|
||||
const result = await spiderSymbolSearch(rootJob);
|
||||
|
||||
if (result.success) {
|
||||
logger.info(
|
||||
`QM spider search initiated successfully. Created ${result.jobsCreated} initial jobs`
|
||||
);
|
||||
return [`Spider search initiated with ${result.jobsCreated} jobs`];
|
||||
} else {
|
||||
logger.error('Failed to initiate QM spider search');
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to start QM spider symbol search', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchExchanges(): Promise<unknown[] | null> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
logger.info('🔄 QM exchanges fetch - not implemented yet');
|
||||
// TODO: Implement QM exchanges fetching logic
|
||||
return null;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch QM exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const qmTasks = {
|
||||
createSessions,
|
||||
fetchSymbols,
|
||||
fetchExchanges,
|
||||
spiderSymbolSearch,
|
||||
};
|
||||
56
apps/data-ingestion/src/handlers/qm/shared/config.ts
Normal file
56
apps/data-ingestion/src/handlers/qm/shared/config.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Shared configuration for QM operations
|
||||
*/
|
||||
|
||||
import { getRandomUserAgent } from '@stock-bot/http';
|
||||
|
||||
// QM Session IDs for different endpoints
|
||||
export const QM_SESSION_IDS = {
|
||||
LOOKUP: 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6', // lookup endpoint
|
||||
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
|
||||
// cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
|
||||
// '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
|
||||
// '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
|
||||
// a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
|
||||
// a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
|
||||
// b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
|
||||
// '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
|
||||
// '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
|
||||
// '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
|
||||
// d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
|
||||
// d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
|
||||
// '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
|
||||
// Add other session IDs as needed
|
||||
} as const;
|
||||
|
||||
// QM API Configuration
|
||||
export const QM_CONFIG = {
|
||||
BASE_URL: 'https://app.quotemedia.com',
|
||||
AUTH_PATH: '/auth/g/authenticate/dataTool/v0/500',
|
||||
LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json',
|
||||
ORIGIN: 'https://www.quotemedia.com',
|
||||
REFERER: 'https://www.quotemedia.com/',
|
||||
} as const;
|
||||
|
||||
// Session management settings
|
||||
export const SESSION_CONFIG = {
|
||||
MIN_SESSIONS: 5,
|
||||
MAX_SESSIONS: 10,
|
||||
MAX_FAILED_CALLS: 10,
|
||||
SESSION_TIMEOUT: 10000, // 10 seconds
|
||||
API_TIMEOUT: 15000, // 15 seconds
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Generate standard QM headers
|
||||
*/
|
||||
export function getQmHeaders(): Record<string, string> {
|
||||
return {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
Accept: '*/*',
|
||||
'Accept-Language': 'en',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
Origin: QM_CONFIG.ORIGIN,
|
||||
Referer: QM_CONFIG.REFERER,
|
||||
};
|
||||
}
|
||||
136
apps/data-ingestion/src/handlers/qm/shared/session-manager.ts
Normal file
136
apps/data-ingestion/src/handlers/qm/shared/session-manager.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
/**
|
||||
* QM Session Manager - Centralized session state management
|
||||
*/
|
||||
|
||||
import type { QMSession } from './types';
|
||||
import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
|
||||
|
||||
export class QMSessionManager {
|
||||
private static instance: QMSessionManager | null = null;
|
||||
private sessionCache: Record<string, QMSession[]> = {};
|
||||
private isInitialized = false;
|
||||
|
||||
private constructor() {
|
||||
// Initialize session cache with known session IDs
|
||||
Object.values(QM_SESSION_IDS).forEach(sessionId => {
|
||||
this.sessionCache[sessionId] = [];
|
||||
});
|
||||
}
|
||||
|
||||
static getInstance(): QMSessionManager {
|
||||
if (!QMSessionManager.instance) {
|
||||
QMSessionManager.instance = new QMSessionManager();
|
||||
}
|
||||
return QMSessionManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a random session for the given session ID
|
||||
*/
|
||||
getSession(sessionId: string): QMSession | null {
|
||||
const sessions = this.sessionCache[sessionId];
|
||||
if (!sessions || sessions.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Filter out sessions with excessive failures
|
||||
const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
if (validSessions.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return validSessions[Math.floor(Math.random() * validSessions.length)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a session to the cache
|
||||
*/
|
||||
addSession(sessionId: string, session: QMSession): void {
|
||||
if (!this.sessionCache[sessionId]) {
|
||||
this.sessionCache[sessionId] = [];
|
||||
}
|
||||
this.sessionCache[sessionId].push(session);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all sessions for a session ID
|
||||
*/
|
||||
getSessions(sessionId: string): QMSession[] {
|
||||
return this.sessionCache[sessionId] || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session count for all session IDs
|
||||
*/
|
||||
getSessionCount(): number {
|
||||
return Object.values(this.sessionCache).reduce((total, sessions) => total + sessions.length, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up failed sessions
|
||||
*/
|
||||
cleanupFailedSessions(): number {
|
||||
let removedCount = 0;
|
||||
|
||||
Object.keys(this.sessionCache).forEach(sessionId => {
|
||||
const initialCount = this.sessionCache[sessionId].length;
|
||||
this.sessionCache[sessionId] = this.sessionCache[sessionId].filter(
|
||||
session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
|
||||
);
|
||||
removedCount += initialCount - this.sessionCache[sessionId].length;
|
||||
});
|
||||
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if more sessions are needed for a session ID
|
||||
*/
|
||||
needsMoreSessions(sessionId: string): boolean {
|
||||
const sessions = this.sessionCache[sessionId] || [];
|
||||
const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
return validSessions.length < SESSION_CONFIG.MIN_SESSIONS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if session ID is at capacity
|
||||
*/
|
||||
isAtCapacity(sessionId: string): boolean {
|
||||
const sessions = this.sessionCache[sessionId] || [];
|
||||
return sessions.length >= SESSION_CONFIG.MAX_SESSIONS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
const stats: Record<string, { total: number; valid: number; failed: number }> = {};
|
||||
|
||||
Object.entries(this.sessionCache).forEach(([sessionId, sessions]) => {
|
||||
const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
const failedSessions = sessions.filter(session => session.failedCalls > SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
|
||||
stats[sessionId] = {
|
||||
total: sessions.length,
|
||||
valid: validSessions.length,
|
||||
failed: failedSessions.length
|
||||
};
|
||||
});
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark manager as initialized
|
||||
*/
|
||||
setInitialized(initialized: boolean = true): void {
|
||||
this.isInitialized = initialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if manager is initialized
|
||||
*/
|
||||
getInitialized(): boolean {
|
||||
return this.isInitialized;
|
||||
}
|
||||
}
|
||||
32
apps/data-ingestion/src/handlers/qm/shared/types.ts
Normal file
32
apps/data-ingestion/src/handlers/qm/shared/types.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* Shared types for QM operations
|
||||
*/
|
||||
|
||||
export interface QMSession {
|
||||
proxy: string;
|
||||
headers: Record<string, string>;
|
||||
successfulCalls: number;
|
||||
failedCalls: number;
|
||||
lastUsed: Date;
|
||||
}
|
||||
|
||||
export interface SymbolSpiderJob {
|
||||
prefix: string | null; // null = root job (A-Z)
|
||||
depth: number; // 1=A, 2=AA, 3=AAA, etc.
|
||||
source: string; // 'qm'
|
||||
maxDepth?: number; // optional max depth limit
|
||||
}
|
||||
|
||||
export interface Exchange {
|
||||
exchange: string;
|
||||
exchangeCode: string;
|
||||
exchangeShortName: string;
|
||||
countryCode: string;
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface SpiderResult {
|
||||
success: boolean;
|
||||
symbolsFound: number;
|
||||
jobsCreated: number;
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
/**
|
||||
* WebShare Fetch Operations - API integration
|
||||
*/
|
||||
import { type ProxyInfo } from '@stock-bot/http';
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { WEBSHARE_CONFIG } from '../shared/config';
|
||||
|
||||
/**
|
||||
* Fetch proxies from WebShare API and convert to ProxyInfo format
|
||||
*/
|
||||
export async function fetchWebShareProxies(): Promise<ProxyInfo[]> {
|
||||
const ctx = OperationContext.create('webshare', 'fetch-proxies');
|
||||
|
||||
try {
|
||||
// Get configuration from config system
|
||||
const { getConfig } = await import('@stock-bot/config');
|
||||
const config = getConfig();
|
||||
|
||||
const apiKey = config.webshare?.apiKey;
|
||||
const apiUrl = config.webshare?.apiUrl;
|
||||
|
||||
if (!apiKey || !apiUrl) {
|
||||
ctx.logger.error('Missing WebShare configuration', {
|
||||
hasApiKey: !!apiKey,
|
||||
hasApiUrl: !!apiUrl,
|
||||
});
|
||||
return [];
|
||||
}
|
||||
|
||||
ctx.logger.info('Fetching proxies from WebShare API', { apiUrl });
|
||||
|
||||
const response = await fetch(`${apiUrl}proxy/list/?mode=${WEBSHARE_CONFIG.DEFAULT_MODE}&page=${WEBSHARE_CONFIG.DEFAULT_PAGE}&page_size=${WEBSHARE_CONFIG.DEFAULT_PAGE_SIZE}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Token ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
signal: AbortSignal.timeout(WEBSHARE_CONFIG.TIMEOUT),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
ctx.logger.error('WebShare API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return [];
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.results || !Array.isArray(data.results)) {
|
||||
ctx.logger.error('Invalid response format from WebShare API', { data });
|
||||
return [];
|
||||
}
|
||||
|
||||
// Transform proxy data to ProxyInfo format
|
||||
const proxies: ProxyInfo[] = data.results.map((proxy: {
|
||||
username: string;
|
||||
password: string;
|
||||
proxy_address: string;
|
||||
port: number;
|
||||
}) => ({
|
||||
source: 'webshare',
|
||||
protocol: 'http' as const,
|
||||
host: proxy.proxy_address,
|
||||
port: proxy.port,
|
||||
username: proxy.username,
|
||||
password: proxy.password,
|
||||
isWorking: true, // WebShare provides working proxies
|
||||
firstSeen: new Date(),
|
||||
lastChecked: new Date(),
|
||||
}));
|
||||
|
||||
ctx.logger.info('Successfully fetched proxies from WebShare', {
|
||||
count: proxies.length,
|
||||
total: data.count || proxies.length,
|
||||
});
|
||||
|
||||
return proxies;
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to fetch proxies from WebShare', { error });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
10
apps/data-ingestion/src/handlers/webshare/shared/config.ts
Normal file
10
apps/data-ingestion/src/handlers/webshare/shared/config.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
/**
|
||||
* WebShare Configuration Constants
|
||||
*/
|
||||
|
||||
export const WEBSHARE_CONFIG = {
|
||||
DEFAULT_PAGE_SIZE: 100,
|
||||
DEFAULT_MODE: 'direct',
|
||||
DEFAULT_PAGE: 1,
|
||||
TIMEOUT: 10000,
|
||||
};
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/**
|
||||
* WebShare Provider for proxy management with scheduled updates
|
||||
*/
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import {
|
||||
createJobHandler,
|
||||
handlerRegistry,
|
||||
type HandlerConfigWithSchedule,
|
||||
} from '@stock-bot/queue';
|
||||
import { updateProxies } from '@stock-bot/utils';
|
||||
|
||||
const logger = getLogger('webshare-provider');
|
||||
|
||||
// Initialize and register the WebShare provider
|
||||
export function initializeWebShareProvider() {
|
||||
logger.debug('Registering WebShare provider with scheduled jobs...');
|
||||
|
||||
const webShareProviderConfig: HandlerConfigWithSchedule = {
|
||||
name: 'webshare',
|
||||
|
||||
operations: {
|
||||
'fetch-proxies': createJobHandler(async () => {
|
||||
logger.info('Fetching proxies from WebShare API');
|
||||
const { fetchWebShareProxies } = await import('./operations/fetch.operations');
|
||||
|
||||
try {
|
||||
const proxies = await fetchWebShareProxies();
|
||||
|
||||
if (proxies.length > 0) {
|
||||
// Update the centralized proxy manager
|
||||
await updateProxies(proxies);
|
||||
|
||||
logger.info('Updated proxy manager with WebShare proxies', {
|
||||
count: proxies.length,
|
||||
workingCount: proxies.filter(p => p.isWorking !== false).length,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
proxiesUpdated: proxies.length,
|
||||
workingProxies: proxies.filter(p => p.isWorking !== false).length,
|
||||
};
|
||||
} else {
|
||||
logger.warn('No proxies fetched from WebShare API');
|
||||
return {
|
||||
success: false,
|
||||
proxiesUpdated: 0,
|
||||
error: 'No proxies returned from API',
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch and update proxies', { error });
|
||||
return {
|
||||
success: false,
|
||||
proxiesUpdated: 0,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
}
|
||||
}),
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'webshare-fetch',
|
||||
operation: 'fetch-proxies',
|
||||
cronPattern: '0 */6 * * *', // Every 6 hours
|
||||
priority: 3,
|
||||
description: 'Fetch fresh proxies from WebShare API',
|
||||
immediately: true, // Run on startup
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
handlerRegistry.registerWithSchedule(webShareProviderConfig);
|
||||
logger.debug('WebShare provider registered successfully');
|
||||
}
|
||||
|
||||
export const webShareProvider = {
|
||||
initialize: initializeWebShareProvider,
|
||||
};
|
||||
|
||||
278
apps/data-ingestion/src/index.ts
Normal file
278
apps/data-ingestion/src/index.ts
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
// Framework imports
|
||||
import { initializeServiceConfig } from '@stock-bot/config';
|
||||
import { Hono } from 'hono';
|
||||
import { cors } from 'hono/cors';
|
||||
// Library imports
|
||||
import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
|
||||
import { connectMongoDB } from '@stock-bot/mongodb-client';
|
||||
import { connectPostgreSQL } from '@stock-bot/postgres-client';
|
||||
import { QueueManager, type QueueManagerConfig } from '@stock-bot/queue';
|
||||
import { Shutdown } from '@stock-bot/shutdown';
|
||||
import { ProxyManager } from '@stock-bot/utils';
|
||||
// Local imports
|
||||
import { exchangeRoutes, healthRoutes, queueRoutes } from './routes';
|
||||
|
||||
const config = initializeServiceConfig();
|
||||
console.log('Data Service Configuration:', JSON.stringify(config, null, 2));
|
||||
const serviceConfig = config.service;
|
||||
const databaseConfig = config.database;
|
||||
const queueConfig = config.queue;
|
||||
|
||||
if (config.log) {
|
||||
setLoggerConfig({
|
||||
logLevel: config.log.level,
|
||||
logConsole: true,
|
||||
logFile: false,
|
||||
environment: config.environment,
|
||||
hideObject: config.log.hideObject,
|
||||
});
|
||||
}
|
||||
|
||||
// Create logger AFTER config is set
|
||||
const logger = getLogger('data-ingestion');
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
// Add CORS middleware
|
||||
app.use(
|
||||
'*',
|
||||
cors({
|
||||
origin: '*',
|
||||
allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'],
|
||||
allowHeaders: ['Content-Type', 'Authorization'],
|
||||
credentials: false,
|
||||
})
|
||||
);
|
||||
const PORT = serviceConfig.port;
|
||||
let server: ReturnType<typeof Bun.serve> | null = null;
|
||||
// Singleton clients are managed in libraries
|
||||
let queueManager: QueueManager | null = null;
|
||||
|
||||
// Initialize shutdown manager
|
||||
const shutdown = Shutdown.getInstance({ timeout: 15000 });
|
||||
|
||||
// Mount routes
|
||||
app.route('/health', healthRoutes);
|
||||
app.route('/api/exchanges', exchangeRoutes);
|
||||
app.route('/api/queue', queueRoutes);
|
||||
|
||||
// Initialize services
|
||||
async function initializeServices() {
|
||||
logger.info('Initializing data service...');
|
||||
|
||||
try {
|
||||
// Initialize MongoDB client singleton
|
||||
logger.debug('Connecting to MongoDB...');
|
||||
const mongoConfig = databaseConfig.mongodb;
|
||||
await connectMongoDB({
|
||||
uri: mongoConfig.uri,
|
||||
database: mongoConfig.database,
|
||||
host: mongoConfig.host || 'localhost',
|
||||
port: mongoConfig.port || 27017,
|
||||
timeouts: {
|
||||
connectTimeout: 30000,
|
||||
socketTimeout: 30000,
|
||||
serverSelectionTimeout: 5000,
|
||||
},
|
||||
});
|
||||
logger.info('MongoDB connected');
|
||||
|
||||
// Initialize PostgreSQL client singleton
|
||||
logger.debug('Connecting to PostgreSQL...');
|
||||
const pgConfig = databaseConfig.postgres;
|
||||
await connectPostgreSQL({
|
||||
host: pgConfig.host,
|
||||
port: pgConfig.port,
|
||||
database: pgConfig.database,
|
||||
username: pgConfig.user,
|
||||
password: pgConfig.password,
|
||||
poolSettings: {
|
||||
min: 2,
|
||||
max: pgConfig.poolSize || 10,
|
||||
idleTimeoutMillis: pgConfig.idleTimeout || 30000,
|
||||
},
|
||||
});
|
||||
logger.info('PostgreSQL connected');
|
||||
|
||||
// Initialize queue system (with delayed worker start)
|
||||
logger.debug('Initializing queue system...');
|
||||
const queueManagerConfig: QueueManagerConfig = {
|
||||
redis: queueConfig?.redis || {
|
||||
host: 'localhost',
|
||||
port: 6379,
|
||||
db: 1,
|
||||
},
|
||||
defaultQueueOptions: {
|
||||
defaultJobOptions: queueConfig?.defaultJobOptions || {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000,
|
||||
},
|
||||
removeOnComplete: 10,
|
||||
removeOnFail: 5,
|
||||
},
|
||||
workers: 2,
|
||||
concurrency: 1,
|
||||
enableMetrics: true,
|
||||
enableDLQ: true,
|
||||
},
|
||||
enableScheduledJobs: true,
|
||||
delayWorkerStart: true, // Prevent workers from starting until all singletons are ready
|
||||
};
|
||||
|
||||
queueManager = QueueManager.getOrInitialize(queueManagerConfig);
|
||||
logger.info('Queue system initialized');
|
||||
|
||||
// Initialize proxy manager
|
||||
logger.debug('Initializing proxy manager...');
|
||||
await ProxyManager.initialize();
|
||||
logger.info('Proxy manager initialized');
|
||||
|
||||
// Initialize handlers (register handlers and scheduled jobs)
|
||||
logger.debug('Initializing data handlers...');
|
||||
const { initializeWebShareProvider } = await import('./handlers/webshare/webshare.handler');
|
||||
const { initializeIBProvider } = await import('./handlers/ib/ib.handler');
|
||||
const { initializeProxyProvider } = await import('./handlers/proxy/proxy.handler');
|
||||
const { initializeQMProvider } = await import('./handlers/qm/qm.handler');
|
||||
|
||||
initializeWebShareProvider();
|
||||
initializeIBProvider();
|
||||
initializeProxyProvider();
|
||||
initializeQMProvider();
|
||||
|
||||
logger.info('Data handlers initialized');
|
||||
|
||||
// Create scheduled jobs from registered handlers
|
||||
logger.debug('Creating scheduled jobs from registered handlers...');
|
||||
const { handlerRegistry } = await import('@stock-bot/queue');
|
||||
const allHandlers = handlerRegistry.getAllHandlers();
|
||||
|
||||
let totalScheduledJobs = 0;
|
||||
for (const [handlerName, config] of allHandlers) {
|
||||
if (config.scheduledJobs && config.scheduledJobs.length > 0) {
|
||||
const queue = queueManager.getQueue(handlerName);
|
||||
|
||||
for (const scheduledJob of config.scheduledJobs) {
|
||||
// Include handler and operation info in job data
|
||||
const jobData = {
|
||||
handler: handlerName,
|
||||
operation: scheduledJob.operation,
|
||||
payload: scheduledJob.payload || {},
|
||||
};
|
||||
|
||||
// Build job options from scheduled job config
|
||||
const jobOptions = {
|
||||
priority: scheduledJob.priority,
|
||||
delay: scheduledJob.delay,
|
||||
repeat: {
|
||||
immediately: scheduledJob.immediately,
|
||||
},
|
||||
};
|
||||
|
||||
await queue.addScheduledJob(
|
||||
scheduledJob.operation,
|
||||
jobData,
|
||||
scheduledJob.cronPattern,
|
||||
jobOptions
|
||||
);
|
||||
totalScheduledJobs++;
|
||||
logger.debug('Scheduled job created', {
|
||||
handler: handlerName,
|
||||
operation: scheduledJob.operation,
|
||||
cronPattern: scheduledJob.cronPattern,
|
||||
immediately: scheduledJob.immediately,
|
||||
priority: scheduledJob.priority,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.info('Scheduled jobs created', { totalJobs: totalScheduledJobs });
|
||||
|
||||
// Now that all singletons are initialized and jobs are scheduled, start the workers
|
||||
logger.debug('Starting queue workers...');
|
||||
queueManager.startAllWorkers();
|
||||
logger.info('Queue workers started');
|
||||
|
||||
logger.info('All services initialized successfully');
|
||||
} catch (error) {
|
||||
logger.error('Failed to initialize services', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Start server
|
||||
async function startServer() {
|
||||
await initializeServices();
|
||||
|
||||
server = Bun.serve({
|
||||
port: PORT,
|
||||
fetch: app.fetch,
|
||||
development: config.environment === 'development',
|
||||
});
|
||||
|
||||
logger.info(`Data Service started on port ${PORT}`);
|
||||
}
|
||||
|
||||
// Register shutdown handlers with priorities
|
||||
// Priority 1: Queue system (highest priority)
|
||||
shutdown.onShutdownHigh(async () => {
|
||||
logger.info('Shutting down queue system...');
|
||||
try {
|
||||
if (queueManager) {
|
||||
await queueManager.shutdown();
|
||||
}
|
||||
logger.info('Queue system shut down');
|
||||
} catch (error) {
|
||||
logger.error('Error shutting down queue system', { error });
|
||||
}
|
||||
}, 'Queue System');
|
||||
|
||||
// Priority 1: HTTP Server (high priority)
|
||||
shutdown.onShutdownHigh(async () => {
|
||||
if (server) {
|
||||
logger.info('Stopping HTTP server...');
|
||||
try {
|
||||
server.stop();
|
||||
logger.info('HTTP server stopped');
|
||||
} catch (error) {
|
||||
logger.error('Error stopping HTTP server', { error });
|
||||
}
|
||||
}
|
||||
}, 'HTTP Server');
|
||||
|
||||
// Priority 2: Database connections (medium priority)
|
||||
shutdown.onShutdownMedium(async () => {
|
||||
logger.info('Disconnecting from databases...');
|
||||
try {
|
||||
const { disconnectMongoDB } = await import('@stock-bot/mongodb-client');
|
||||
const { disconnectPostgreSQL } = await import('@stock-bot/postgres-client');
|
||||
|
||||
await disconnectMongoDB();
|
||||
await disconnectPostgreSQL();
|
||||
logger.info('Database connections closed');
|
||||
} catch (error) {
|
||||
logger.error('Error closing database connections', { error });
|
||||
}
|
||||
}, 'Databases');
|
||||
|
||||
// Priority 3: Logger shutdown (lowest priority - runs last)
|
||||
shutdown.onShutdownLow(async () => {
|
||||
try {
|
||||
logger.info('Shutting down loggers...');
|
||||
await shutdownLoggers();
|
||||
// Don't log after shutdown
|
||||
} catch {
|
||||
// Silently ignore logger shutdown errors
|
||||
}
|
||||
}, 'Loggers');
|
||||
|
||||
// Start the service
|
||||
startServer().catch(error => {
|
||||
logger.fatal('Failed to start data service', { error });
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
logger.info('Data service startup initiated');
|
||||
|
||||
// ProxyManager class and singleton instance are available via @stock-bot/utils
|
||||
22
apps/data-ingestion/src/routes/exchange.routes.ts
Normal file
22
apps/data-ingestion/src/routes/exchange.routes.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import { Hono } from 'hono';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
|
||||
const logger = getLogger('exchange-routes');
|
||||
const exchange = new Hono();
|
||||
|
||||
// Get all exchanges
|
||||
exchange.get('/', async c => {
|
||||
try {
|
||||
// TODO: Implement exchange listing from database
|
||||
return c.json({
|
||||
status: 'success',
|
||||
data: [],
|
||||
message: 'Exchange endpoints will be implemented with database integration'
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get exchanges', { error });
|
||||
return c.json({ status: 'error', message: 'Failed to get exchanges' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
export { exchange as exchangeRoutes };
|
||||
14
apps/data-ingestion/src/routes/health.routes.ts
Normal file
14
apps/data-ingestion/src/routes/health.routes.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { Hono } from 'hono';
|
||||
|
||||
const health = new Hono();
|
||||
|
||||
// Health check endpoint
|
||||
health.get('/', c => {
|
||||
return c.json({
|
||||
status: 'healthy',
|
||||
service: 'data-ingestion',
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
export { health as healthRoutes };
|
||||
6
apps/data-ingestion/src/routes/index.ts
Normal file
6
apps/data-ingestion/src/routes/index.ts
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
/**
|
||||
* Routes index - exports all route modules
|
||||
*/
|
||||
export { exchangeRoutes } from './exchange.routes';
|
||||
export { healthRoutes } from './health.routes';
|
||||
export { queueRoutes } from './queue.routes';
|
||||
121
apps/data-ingestion/src/routes/market-data.routes.ts
Normal file
121
apps/data-ingestion/src/routes/market-data.routes.ts
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
/**
|
||||
* Market data routes
|
||||
*/
|
||||
import { Hono } from 'hono';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { processItems, QueueManager } from '@stock-bot/queue';
|
||||
|
||||
const logger = getLogger('market-data-routes');
|
||||
|
||||
export const marketDataRoutes = new Hono();
|
||||
|
||||
// Market data endpoints
|
||||
marketDataRoutes.get('/api/live/:symbol', async c => {
|
||||
const symbol = c.req.param('symbol');
|
||||
logger.info('Live data request', { symbol });
|
||||
|
||||
try {
|
||||
// Queue job for live data using Yahoo provider
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('yahoo-finance');
|
||||
const job = await queue.add('live-data', {
|
||||
handler: 'yahoo-finance',
|
||||
operation: 'live-data',
|
||||
payload: { symbol },
|
||||
});
|
||||
return c.json({
|
||||
status: 'success',
|
||||
message: 'Live data job queued',
|
||||
jobId: job.id,
|
||||
symbol,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to queue live data job', { symbol, error });
|
||||
return c.json({ status: 'error', message: 'Failed to queue live data job' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
marketDataRoutes.get('/api/historical/:symbol', async c => {
|
||||
const symbol = c.req.param('symbol');
|
||||
const from = c.req.query('from');
|
||||
const to = c.req.query('to');
|
||||
|
||||
logger.info('Historical data request', { symbol, from, to });
|
||||
|
||||
try {
|
||||
const fromDate = from ? new Date(from) : new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); // 30 days ago
|
||||
const toDate = to ? new Date(to) : new Date(); // Now
|
||||
|
||||
// Queue job for historical data using Yahoo provider
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('yahoo-finance');
|
||||
const job = await queue.add('historical-data', {
|
||||
handler: 'yahoo-finance',
|
||||
operation: 'historical-data',
|
||||
payload: {
|
||||
symbol,
|
||||
from: fromDate.toISOString(),
|
||||
to: toDate.toISOString(),
|
||||
},
|
||||
});
|
||||
|
||||
return c.json({
|
||||
status: 'success',
|
||||
message: 'Historical data job queued',
|
||||
jobId: job.id,
|
||||
symbol,
|
||||
from: fromDate,
|
||||
to: toDate,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to queue historical data job', { symbol, from, to, error });
|
||||
return c.json({ status: 'error', message: 'Failed to queue historical data job' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
// Batch processing endpoint using new queue system
|
||||
marketDataRoutes.post('/api/process-symbols', async c => {
|
||||
try {
|
||||
const {
|
||||
symbols,
|
||||
provider = 'ib',
|
||||
operation = 'fetch-session',
|
||||
useBatching = true,
|
||||
totalDelayHours = 0.0083, // ~30 seconds (30/3600 hours)
|
||||
batchSize = 10,
|
||||
} = await c.req.json();
|
||||
|
||||
if (!symbols || !Array.isArray(symbols) || symbols.length === 0) {
|
||||
return c.json({ status: 'error', message: 'Invalid symbols array' }, 400);
|
||||
}
|
||||
|
||||
logger.info('Batch processing symbols', {
|
||||
count: symbols.length,
|
||||
provider,
|
||||
operation,
|
||||
useBatching,
|
||||
});
|
||||
|
||||
const result = await processItems(symbols, provider, {
|
||||
handler: provider,
|
||||
operation,
|
||||
totalDelayHours,
|
||||
useBatching,
|
||||
batchSize,
|
||||
priority: 2,
|
||||
retries: 2,
|
||||
removeOnComplete: 5,
|
||||
removeOnFail: 10,
|
||||
});
|
||||
|
||||
return c.json({
|
||||
status: 'success',
|
||||
message: 'Batch processing initiated',
|
||||
result,
|
||||
symbols: symbols.length,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to process symbols batch', { error });
|
||||
return c.json({ status: 'error', message: 'Failed to process symbols batch' }, 500);
|
||||
}
|
||||
});
|
||||
25
apps/data-ingestion/src/routes/queue.routes.ts
Normal file
25
apps/data-ingestion/src/routes/queue.routes.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import { Hono } from 'hono';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
|
||||
const logger = getLogger('queue-routes');
|
||||
const queue = new Hono();
|
||||
|
||||
// Queue status endpoint
|
||||
queue.get('/status', async c => {
|
||||
try {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const globalStats = await queueManager.getGlobalStats();
|
||||
|
||||
return c.json({
|
||||
status: 'success',
|
||||
data: globalStats,
|
||||
message: 'Queue status retrieved successfully'
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get queue status', { error });
|
||||
return c.json({ status: 'error', message: 'Failed to get queue status' }, 500);
|
||||
}
|
||||
});
|
||||
|
||||
export { queue as queueRoutes };
|
||||
40
apps/data-ingestion/src/types/exchange.types.ts
Normal file
40
apps/data-ingestion/src/types/exchange.types.ts
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
/**
|
||||
* Type definitions for exchange data structures
|
||||
*/
|
||||
|
||||
export interface IBExchange {
|
||||
id: string;
|
||||
country_code: string;
|
||||
name: string;
|
||||
code?: string;
|
||||
exchange_code?: string;
|
||||
currency?: string;
|
||||
timezone?: string;
|
||||
_id?: unknown; // MongoDB ObjectId
|
||||
// Add other properties as needed
|
||||
}
|
||||
|
||||
export interface MasterExchangeData {
|
||||
id: string;
|
||||
code: string;
|
||||
name: string;
|
||||
country: string;
|
||||
currency: string;
|
||||
// Add other properties as needed
|
||||
}
|
||||
|
||||
export interface QMSymbol {
|
||||
symbol: string;
|
||||
exchange?: string;
|
||||
name?: string;
|
||||
type?: string;
|
||||
// Add other properties as needed
|
||||
}
|
||||
|
||||
export interface IBSymbol {
|
||||
symbol: string;
|
||||
exchange: string;
|
||||
name?: string;
|
||||
currency?: string;
|
||||
// Add other properties as needed
|
||||
}
|
||||
93
apps/data-ingestion/src/types/job-payloads.ts
Normal file
93
apps/data-ingestion/src/types/job-payloads.ts
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
/**
|
||||
* Type definitions for all job payloads across data service providers
|
||||
*/
|
||||
|
||||
// Common result types
|
||||
export interface JobResult {
|
||||
success: boolean;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface CountableJobResult extends JobResult {
|
||||
count: number;
|
||||
}
|
||||
|
||||
// QM Provider Types
|
||||
export interface SymbolSpiderJob {
|
||||
prefix: string | null; // null = root job (A-Z)
|
||||
depth: number; // 1=A, 2=AA, 3=AAA, etc.
|
||||
source: string; // 'qm'
|
||||
maxDepth?: number; // optional max depth limit
|
||||
}
|
||||
|
||||
export interface CreateSessionsResult extends JobResult {
|
||||
// No additional fields needed
|
||||
}
|
||||
|
||||
export interface SearchSymbolsResult extends CountableJobResult {
|
||||
symbols?: unknown[]; // First 10 symbols as sample
|
||||
}
|
||||
|
||||
export interface SpiderSymbolSearchResult extends JobResult {
|
||||
symbolsFound: number;
|
||||
newSymbolsAdded: number;
|
||||
duplicatesSkipped: number;
|
||||
errors: number;
|
||||
depth: number;
|
||||
prefix: string | null;
|
||||
}
|
||||
|
||||
// IB Provider Types
|
||||
export interface FetchSessionResult extends JobResult {
|
||||
sessionData?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface FetchExchangesResult extends CountableJobResult {
|
||||
exchanges?: unknown[];
|
||||
}
|
||||
|
||||
export interface FetchSymbolsResult extends CountableJobResult {
|
||||
symbols?: unknown[];
|
||||
}
|
||||
|
||||
export interface IBExchangesAndSymbolsResult extends JobResult {
|
||||
exchanges: FetchExchangesResult;
|
||||
symbols: FetchSymbolsResult;
|
||||
}
|
||||
|
||||
// Proxy Provider Types
|
||||
export interface ProxyInfo {
|
||||
host: string;
|
||||
port: number;
|
||||
protocol: 'http' | 'https' | 'socks4' | 'socks5';
|
||||
username?: string;
|
||||
password?: string;
|
||||
country?: string;
|
||||
city?: string;
|
||||
isValid?: boolean;
|
||||
lastChecked?: Date;
|
||||
}
|
||||
|
||||
export interface FetchProxiesFromSourcesResult extends CountableJobResult {
|
||||
proxies?: ProxyInfo[];
|
||||
sources: string[];
|
||||
}
|
||||
|
||||
export interface CheckProxyResult extends JobResult {
|
||||
proxy: ProxyInfo;
|
||||
responseTime?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// WebShare Provider Types
|
||||
export interface FetchWebShareProxiesResult extends CountableJobResult {
|
||||
proxies?: ProxyInfo[];
|
||||
activeProxies: number;
|
||||
totalQuota: number;
|
||||
remainingQuota: number;
|
||||
}
|
||||
|
||||
// No payload job types (for operations that don't need input)
|
||||
export interface NoPayload {
|
||||
// Empty interface for operations that don't need payload
|
||||
}
|
||||
109
apps/data-ingestion/src/utils/symbol-search.util.ts
Normal file
109
apps/data-ingestion/src/utils/symbol-search.util.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import { sleep } from '@stock-bot/utils';
|
||||
|
||||
const logger = getLogger('symbol-search-util');
|
||||
|
||||
export interface SearchFunction {
|
||||
(query: string): Promise<string[]>;
|
||||
}
|
||||
|
||||
export class SymbolSearchUtil {
|
||||
private threshold: number;
|
||||
private searchFunction: SearchFunction;
|
||||
private maxDepth: number;
|
||||
private delay: number;
|
||||
|
||||
constructor(
|
||||
searchFunction: SearchFunction,
|
||||
threshold: number = 50,
|
||||
maxDepth: number = 4,
|
||||
delay: number = 100
|
||||
) {
|
||||
this.searchFunction = searchFunction;
|
||||
this.threshold = threshold;
|
||||
this.maxDepth = maxDepth;
|
||||
this.delay = delay;
|
||||
}
|
||||
|
||||
async searchAllSymbols(): Promise<string[]> {
|
||||
logger.info('Starting comprehensive symbol search...');
|
||||
const allSymbols: string[] = [];
|
||||
|
||||
// Start with single letters A-Z
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const singleLetter = String.fromCharCode(65 + i);
|
||||
|
||||
try {
|
||||
const symbols = await this.searchRecursive(singleLetter, 1);
|
||||
allSymbols.push(...symbols);
|
||||
|
||||
// Add delay between top-level searches
|
||||
if (this.delay > 0) {
|
||||
await sleep(this.delay);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed to search for "${singleLetter}":`, error);
|
||||
// Continue with next letter
|
||||
}
|
||||
}
|
||||
|
||||
// Remove duplicates
|
||||
const uniqueSymbols = [...new Set(allSymbols)];
|
||||
logger.info(`Symbol search completed. Found ${uniqueSymbols.length} unique symbols`);
|
||||
return uniqueSymbols;
|
||||
}
|
||||
|
||||
private async searchRecursive(prefix: string, depth: number): Promise<string[]> {
|
||||
try {
|
||||
const symbols = await this.searchFunction(prefix);
|
||||
|
||||
logger.debug(`Query "${prefix}" returned ${symbols.length} symbols`);
|
||||
|
||||
// If we're at max depth or results are under threshold, return the symbols
|
||||
if (depth >= this.maxDepth || symbols.length < this.threshold) {
|
||||
logger.info(`Added ${symbols.length} symbols from query: ${prefix}`);
|
||||
return symbols;
|
||||
}
|
||||
|
||||
// If we have too many results, go deeper
|
||||
logger.info(
|
||||
`Query "${prefix}" returned ${symbols.length} results (>= ${this.threshold}), going deeper...`
|
||||
);
|
||||
const allSymbols: string[] = [];
|
||||
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const nextQuery = prefix + String.fromCharCode(65 + i);
|
||||
|
||||
try {
|
||||
const deeperSymbols = await this.searchRecursive(nextQuery, depth + 1);
|
||||
allSymbols.push(...deeperSymbols);
|
||||
|
||||
// Add delay between recursive calls
|
||||
if (this.delay > 0 && depth < 3) {
|
||||
// Only delay for first few levels
|
||||
await sleep(this.delay);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed recursive search for "${nextQuery}":`, error);
|
||||
// Continue with next combination
|
||||
}
|
||||
}
|
||||
|
||||
return allSymbols;
|
||||
} catch (error) {
|
||||
logger.error(`Error in recursive search for "${prefix}":`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Static method for one-off searches
|
||||
static async search(
|
||||
searchFunction: SearchFunction,
|
||||
threshold: number = 50,
|
||||
maxDepth: number = 4,
|
||||
delay: number = 100
|
||||
): Promise<string[]> {
|
||||
const util = new SymbolSearchUtil(searchFunction, threshold, maxDepth, delay);
|
||||
return util.searchAllSymbols();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue