416 lines
14 KiB
TypeScript
416 lines
14 KiB
TypeScript
import { getRandomUserAgent } from '@stock-bot/http';
|
|
import { getLogger } from '@stock-bot/logger';
|
|
import { getMongoDBClient } from '@stock-bot/mongodb-client';
|
|
import { getProxy } from './webshare.provider';
|
|
|
|
// Shared instances (module-scoped, not global)
|
|
let isInitialized = false; // Track if resources are initialized
|
|
let logger: ReturnType<typeof getLogger>;
|
|
// let cache: CacheProvider;
|
|
|
|
export interface QMSession {
|
|
proxy: string;
|
|
headers: Record<string, string>;
|
|
successfulCalls: number;
|
|
failedCalls: number;
|
|
lastUsed: Date;
|
|
}
|
|
|
|
export interface SymbolSpiderJob {
|
|
prefix: string | null; // null = root job (A-Z)
|
|
depth: number; // 1=A, 2=AA, 3=AAA, etc.
|
|
source: string; // 'qm'
|
|
maxDepth?: number; // optional max depth limit
|
|
}
|
|
|
|
interface Exchange {
|
|
exchange: string;
|
|
exchangeCode: string;
|
|
exchangeShortName: string;
|
|
countryCode: string;
|
|
source: string;
|
|
}
|
|
|
|
function getQmHeaders(): Record<string, string> {
|
|
return {
|
|
'User-Agent': getRandomUserAgent(),
|
|
Accept: '*/*',
|
|
'Accept-Language': 'en',
|
|
'Sec-Fetch-Mode': 'cors',
|
|
Origin: 'https://www.quotemedia.com',
|
|
Referer: 'https://www.quotemedia.com/',
|
|
};
|
|
}
|
|
|
|
const sessionCache: Record<string, QMSession[]> = {
|
|
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
|
|
// cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
|
|
// '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
|
|
// '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
|
|
// a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
|
|
// a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
|
|
// b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
|
|
dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6: [], //fceb3c4bdd // lookup
|
|
// '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
|
|
// '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
|
|
// '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
|
|
// d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
|
|
// d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
|
|
// '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
|
|
};
|
|
|
|
export async function initializeQMResources(): Promise<void> {
|
|
// Skip if already initialized
|
|
if (isInitialized) {
|
|
return;
|
|
}
|
|
logger = getLogger('qm-tasks');
|
|
isInitialized = true;
|
|
}
|
|
|
|
export async function createSessions(): Promise<void> {
|
|
try {
|
|
//for each session, check array length, if less than 5, create new session
|
|
if (!isInitialized) {
|
|
await initializeQMResources();
|
|
}
|
|
logger.info('Creating QM sessions...');
|
|
for (const [sessionId, sessionArray] of Object.entries(sessionCache)) {
|
|
const initialCount = sessionArray.length;
|
|
const filteredArray = sessionArray.filter(session => session.failedCalls <= 10);
|
|
sessionCache[sessionId] = filteredArray;
|
|
|
|
const removedCount = initialCount - filteredArray.length;
|
|
if (removedCount > 0) {
|
|
logger.info(
|
|
`Removed ${removedCount} sessions with excessive failures for ${sessionId}. Remaining: ${filteredArray.length}`
|
|
);
|
|
}
|
|
|
|
while (sessionCache[sessionId].length < 50) {
|
|
logger.info(`Creating new session for ${sessionId}`);
|
|
const proxy = getProxy();
|
|
if (proxy === null) {
|
|
logger.error('No proxy available for QM session creation');
|
|
break; // Skip session creation if no proxy is available
|
|
}
|
|
const newSession: QMSession = {
|
|
proxy: proxy, // Placeholder, should be set to a valid proxy
|
|
headers: getQmHeaders(),
|
|
successfulCalls: 0,
|
|
failedCalls: 0,
|
|
lastUsed: new Date(),
|
|
};
|
|
const sessionResponse = await fetch(
|
|
`https://app.quotemedia.com/auth/g/authenticate/dataTool/v0/500/${sessionId}`,
|
|
{
|
|
method: 'GET',
|
|
proxy: newSession.proxy,
|
|
headers: newSession.headers,
|
|
}
|
|
);
|
|
|
|
logger.debug('Session response received', {
|
|
status: sessionResponse.status,
|
|
sessionId,
|
|
});
|
|
if (!sessionResponse.ok) {
|
|
logger.error('Failed to create QM session', {
|
|
sessionId,
|
|
status: sessionResponse.status,
|
|
statusText: sessionResponse.statusText,
|
|
});
|
|
continue; // Skip this session if creation failed
|
|
}
|
|
const sessionData = await sessionResponse.json();
|
|
logger.info('QM session created successfully', {
|
|
sessionId,
|
|
sessionData,
|
|
proxy: newSession.proxy,
|
|
sessionCount: sessionCache[sessionId].length + 1,
|
|
});
|
|
newSession.headers['Datatool-Token'] = sessionData.token;
|
|
sessionCache[sessionId].push(newSession);
|
|
}
|
|
}
|
|
return undefined;
|
|
} catch (error) {
|
|
logger.error('❌ Failed to fetch QM session', { error });
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
// Spider-based symbol search functions
|
|
export async function spiderSymbolSearch(
|
|
payload: SymbolSpiderJob
|
|
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
|
try {
|
|
if (!isInitialized) {
|
|
await initializeQMResources();
|
|
}
|
|
|
|
const { prefix, depth, source = 'qm', maxDepth = 4 } = payload;
|
|
|
|
logger.info(`Starting spider search`, { prefix: prefix || 'ROOT', depth, source });
|
|
|
|
// Root job: Create A-Z jobs
|
|
if (prefix === null || prefix === undefined || prefix === '') {
|
|
return await createAlphabetJobs(source, maxDepth);
|
|
}
|
|
|
|
// Leaf job: Search for symbols with this prefix
|
|
return await searchAndSpawnJobs(prefix, depth, source, maxDepth);
|
|
} catch (error) {
|
|
logger.error('Spider symbol search failed', { error, payload });
|
|
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
|
}
|
|
}
|
|
|
|
async function createAlphabetJobs(
|
|
source: string,
|
|
maxDepth: number
|
|
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
|
try {
|
|
const { queueManager } = await import('../index');
|
|
if (!queueManager) {
|
|
throw new Error('Queue manager not initialized');
|
|
}
|
|
const queue = queueManager.getQueue('qm');
|
|
let jobsCreated = 0;
|
|
|
|
// Create jobs for A-Z
|
|
for (let i = 0; i < 26; i++) {
|
|
const letter = String.fromCharCode(65 + i); // A=65, B=66, etc.
|
|
|
|
const job: SymbolSpiderJob = {
|
|
prefix: letter,
|
|
depth: 1,
|
|
source,
|
|
maxDepth,
|
|
};
|
|
|
|
await queue.add(
|
|
'spider-symbol-search',
|
|
{
|
|
handler: 'qm',
|
|
operation: 'spider-symbol-search',
|
|
payload: job,
|
|
},
|
|
{
|
|
priority: 5,
|
|
delay: i * 100, // Stagger jobs by 100ms
|
|
attempts: 3,
|
|
backoff: { type: 'exponential', delay: 2000 },
|
|
}
|
|
);
|
|
|
|
jobsCreated++;
|
|
}
|
|
|
|
logger.info(`Created ${jobsCreated} alphabet jobs (A-Z)`);
|
|
return { success: true, symbolsFound: 0, jobsCreated };
|
|
} catch (error) {
|
|
logger.error('Failed to create alphabet jobs', { error });
|
|
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
|
}
|
|
}
|
|
|
|
async function searchAndSpawnJobs(
|
|
prefix: string,
|
|
depth: number,
|
|
source: string,
|
|
maxDepth: number
|
|
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
|
try {
|
|
// Ensure sessions exist
|
|
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6';
|
|
const currentSessions = sessionCache[sessionId] || [];
|
|
|
|
if (currentSessions.length === 0) {
|
|
logger.info('No sessions found, creating sessions first...');
|
|
await createSessions();
|
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
}
|
|
|
|
// Search for symbols with this prefix
|
|
const symbols = await searchQMSymbolsAPI(prefix);
|
|
const symbolCount = symbols.length;
|
|
|
|
logger.info(`Prefix "${prefix}" returned ${symbolCount} symbols`);
|
|
|
|
let jobsCreated = 0;
|
|
|
|
// If we have 50+ symbols and haven't reached max depth, spawn sub-jobs
|
|
if (symbolCount >= 50 && depth < maxDepth) {
|
|
const { queueManager } = await import('../index');
|
|
if (!queueManager) {
|
|
throw new Error('Queue manager not initialized');
|
|
}
|
|
const queue = queueManager.getQueue('qm');
|
|
|
|
logger.info(`Spawning sub-jobs for prefix "${prefix}" (${symbolCount} >= 50 symbols)`);
|
|
|
|
// Create jobs for prefixA, prefixB, prefixC... prefixZ
|
|
for (let i = 0; i < 26; i++) {
|
|
const letter = String.fromCharCode(65 + i);
|
|
const newPrefix = prefix + letter;
|
|
|
|
const job: SymbolSpiderJob = {
|
|
prefix: newPrefix,
|
|
depth: depth + 1,
|
|
source,
|
|
maxDepth,
|
|
};
|
|
|
|
await queue.add(
|
|
'spider-symbol-search',
|
|
{
|
|
handler: 'qm',
|
|
operation: 'spider-symbol-search',
|
|
payload: job,
|
|
},
|
|
{
|
|
priority: Math.max(1, 6 - depth), // Higher priority for deeper jobs
|
|
delay: i * 50, // Stagger sub-jobs by 50ms
|
|
attempts: 3,
|
|
backoff: { type: 'exponential', delay: 2000 },
|
|
}
|
|
);
|
|
|
|
jobsCreated++;
|
|
}
|
|
|
|
logger.info(`Created ${jobsCreated} sub-jobs for prefix "${prefix}"`);
|
|
} else {
|
|
// Terminal case: save symbols and exchanges (already done in searchQMSymbolsAPI)
|
|
logger.info(`Terminal case for prefix "${prefix}": ${symbolCount} symbols saved`);
|
|
}
|
|
|
|
return { success: true, symbolsFound: symbolCount, jobsCreated };
|
|
} catch (error) {
|
|
logger.error(`Failed to search and spawn jobs for prefix "${prefix}"`, { error, depth });
|
|
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
|
}
|
|
}
|
|
|
|
// API call function to search symbols via QM
|
|
async function searchQMSymbolsAPI(query: string): Promise<string[]> {
|
|
const proxy = getProxy();
|
|
|
|
if (!proxy) {
|
|
throw new Error('No proxy available for QM API call');
|
|
}
|
|
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6'; // Use the session ID for symbol lookup
|
|
const session =
|
|
sessionCache[sessionId][Math.floor(Math.random() * sessionCache[sessionId].length)]; // lookup session
|
|
if (!session) {
|
|
throw new Error(`No active session found for QM API with ID: ${sessionId}`);
|
|
}
|
|
try {
|
|
// QM lookup endpoint for symbol search
|
|
const apiUrl = `https://app.quotemedia.com/datatool/lookup.json?marketType=equity&pathName=%2Fdemo%2Fportal%2Fcompany-summary.php&q=${encodeURIComponent(query)}&qmodTool=SmartSymbolLookup&searchType=symbol&showFree=false&showHisa=false&webmasterId=500`;
|
|
|
|
const response = await fetch(apiUrl, {
|
|
method: 'GET',
|
|
headers: session.headers,
|
|
proxy: session.proxy,
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
|
}
|
|
|
|
const symbols = await response.json();
|
|
const client = getMongoDBClient();
|
|
const updatedSymbols = symbols.map((symbol: any) => {
|
|
return {
|
|
...symbol,
|
|
qmSearchCode: symbol.symbol, // Store original symbol for reference
|
|
symbol: symbol.symbol.split(':')[0], // Extract symbol from "symbol:exchange"
|
|
};
|
|
});
|
|
await client.batchUpsert('qmSymbols', updatedSymbols, ['qmSearchCode']);
|
|
const exchanges: Exchange[] = [];
|
|
for (const symbol of symbols) {
|
|
if (!exchanges.some(ex => ex.exchange === symbol.exchange)) {
|
|
exchanges.push({
|
|
exchange: symbol.exchange,
|
|
exchangeCode: symbol.exchangeCode,
|
|
exchangeShortName: symbol.exchangeShortName,
|
|
countryCode: symbol.countryCode,
|
|
source: 'qm',
|
|
});
|
|
}
|
|
}
|
|
await client.batchUpsert('qmExchanges', exchanges, ['exchange']);
|
|
session.successfulCalls++;
|
|
session.lastUsed = new Date();
|
|
|
|
logger.info(
|
|
`QM API returned ${symbols.length} symbols for query: ${query} with proxy ${session.proxy}`
|
|
);
|
|
return symbols;
|
|
} catch (error) {
|
|
logger.error(`Error searching QM symbols for query "${query}":`, error);
|
|
if (session) {
|
|
session.failedCalls++;
|
|
session.lastUsed = new Date();
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export async function fetchSymbols(): Promise<unknown[] | null> {
|
|
try {
|
|
if (!isInitialized) {
|
|
await initializeQMResources();
|
|
}
|
|
|
|
logger.info('🔄 Starting QM spider-based symbol search...');
|
|
|
|
// Start the spider process with root job
|
|
const rootJob: SymbolSpiderJob = {
|
|
prefix: null, // Root job creates A-Z jobs
|
|
depth: 0,
|
|
source: 'qm',
|
|
maxDepth: 4,
|
|
};
|
|
|
|
const result = await spiderSymbolSearch(rootJob);
|
|
|
|
if (result.success) {
|
|
logger.info(
|
|
`QM spider search initiated successfully. Created ${result.jobsCreated} initial jobs`
|
|
);
|
|
return [`Spider search initiated with ${result.jobsCreated} jobs`];
|
|
} else {
|
|
logger.error('Failed to initiate QM spider search');
|
|
return null;
|
|
}
|
|
} catch (error) {
|
|
logger.error('❌ Failed to start QM spider symbol search', { error });
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export async function fetchExchanges(): Promise<unknown[] | null> {
|
|
try {
|
|
if (!isInitialized) {
|
|
await initializeQMResources();
|
|
}
|
|
|
|
logger.info('🔄 QM exchanges fetch - not implemented yet');
|
|
// TODO: Implement QM exchanges fetching logic
|
|
return null;
|
|
} catch (error) {
|
|
logger.error('❌ Failed to fetch QM exchanges', { error });
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export const qmTasks = {
|
|
createSessions,
|
|
fetchSymbols,
|
|
fetchExchanges,
|
|
spiderSymbolSearch,
|
|
};
|