renaming services to more suitable names
This commit is contained in:
parent
3ae9de8376
commit
be6afef832
69 changed files with 41 additions and 2956 deletions
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* QM Exchanges Operations - Exchange fetching functionality
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
|
||||
import { initializeQMResources } from './session.operations';
|
||||
|
||||
export async function fetchExchanges(): Promise<unknown[] | null> {
|
||||
const ctx = OperationContext.create('qm', 'exchanges');
|
||||
|
||||
try {
|
||||
// Ensure resources are initialized
|
||||
const { QMSessionManager } = await import('../shared/session-manager');
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
ctx.logger.info('QM exchanges fetch - not implemented yet');
|
||||
|
||||
// Cache the "not implemented" status
|
||||
await ctx.cache.set('fetch-status', {
|
||||
implemented: false,
|
||||
message: 'QM exchanges fetching not yet implemented',
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
|
||||
// TODO: Implement QM exchanges fetching logic
|
||||
// This could involve:
|
||||
// 1. Querying existing exchanges from MongoDB
|
||||
// 2. Making API calls to discover new exchanges
|
||||
// 3. Processing and storing exchange metadata
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to fetch QM exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
/**
|
||||
* QM Session Operations - Session creation and management
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
import { isShutdownSignalReceived } from '@stock-bot/shutdown';
|
||||
import { getRandomProxy } from '@stock-bot/utils';
|
||||
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG, getQmHeaders } from '../shared/config';
|
||||
import type { QMSession } from '../shared/types';
|
||||
|
||||
export async function createSessions(): Promise<void> {
|
||||
const ctx = OperationContext.create('qm', 'session');
|
||||
|
||||
try {
|
||||
ctx.logger.info('Creating QM sessions...');
|
||||
|
||||
// Get session manager instance
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
|
||||
// Check if already initialized
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
// Clean up failed sessions first
|
||||
const removedCount = sessionManager.cleanupFailedSessions();
|
||||
if (removedCount > 0) {
|
||||
ctx.logger.info(`Cleaned up ${removedCount} failed sessions`);
|
||||
}
|
||||
|
||||
// Cache session creation stats
|
||||
const initialStats = sessionManager.getStats();
|
||||
await ctx.cache.set('pre-creation-stats', initialStats, { ttl: 300 });
|
||||
|
||||
// Create sessions for each session ID that needs them
|
||||
for (const [sessionKey, sessionId] of Object.entries(QM_SESSION_IDS)) {
|
||||
if (sessionManager.isAtCapacity(sessionId)) {
|
||||
ctx.logger.debug(`Session ID ${sessionKey} is at capacity, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
while (sessionManager.needsMoreSessions(sessionId)) {
|
||||
if (isShutdownSignalReceived()) {
|
||||
ctx.logger.info('Shutting down, skipping session creation');
|
||||
return;
|
||||
}
|
||||
|
||||
await createSingleSession(sessionId, sessionKey, ctx);
|
||||
}
|
||||
}
|
||||
|
||||
// Cache final stats and session count
|
||||
const finalStats = sessionManager.getStats();
|
||||
const totalSessions = sessionManager.getSessionCount();
|
||||
|
||||
await ctx.cache.set('post-creation-stats', finalStats, { ttl: 3600 });
|
||||
await ctx.cache.set('session-count', totalSessions, { ttl: 900 });
|
||||
await ctx.cache.set('last-session-creation', new Date().toISOString());
|
||||
|
||||
ctx.logger.info('QM session creation completed', {
|
||||
totalSessions,
|
||||
sessionStats: finalStats
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to create QM sessions', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function createSingleSession(
|
||||
sessionId: string,
|
||||
sessionKey: string,
|
||||
ctx: OperationContext
|
||||
): Promise<void> {
|
||||
ctx.logger.debug(`Creating new session for ${sessionKey}`, { sessionId });
|
||||
|
||||
const proxyInfo = await getRandomProxy();
|
||||
if (!proxyInfo) {
|
||||
ctx.logger.error('No proxy available for QM session creation');
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert ProxyInfo to string format
|
||||
const auth = proxyInfo.username && proxyInfo.password ?
|
||||
`${proxyInfo.username}:${proxyInfo.password}@` : '';
|
||||
const proxy = `${proxyInfo.protocol}://${auth}${proxyInfo.host}:${proxyInfo.port}`;
|
||||
|
||||
const newSession: QMSession = {
|
||||
proxy: proxy,
|
||||
headers: getQmHeaders(),
|
||||
successfulCalls: 0,
|
||||
failedCalls: 0,
|
||||
lastUsed: new Date(),
|
||||
};
|
||||
|
||||
try {
|
||||
const sessionResponse = await fetch(
|
||||
`${QM_CONFIG.BASE_URL}${QM_CONFIG.AUTH_PATH}/${sessionId}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: newSession.headers,
|
||||
signal: AbortSignal.timeout(SESSION_CONFIG.SESSION_TIMEOUT),
|
||||
}
|
||||
);
|
||||
|
||||
ctx.logger.debug('Session response received', {
|
||||
status: sessionResponse.status,
|
||||
sessionKey,
|
||||
});
|
||||
|
||||
if (!sessionResponse.ok) {
|
||||
ctx.logger.error('Failed to create QM session', {
|
||||
sessionKey,
|
||||
sessionId,
|
||||
status: sessionResponse.status,
|
||||
statusText: sessionResponse.statusText,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionData = await sessionResponse.json();
|
||||
|
||||
// Add token to headers
|
||||
newSession.headers['Datatool-Token'] = sessionData.token;
|
||||
|
||||
// Add session to manager
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
sessionManager.addSession(sessionId, newSession);
|
||||
|
||||
// Cache successful session creation
|
||||
await ctx.cache.set(
|
||||
`successful-session:${sessionKey}:${Date.now()}`,
|
||||
{ sessionId, proxy, tokenExists: !!sessionData.token },
|
||||
{ ttl: 300 }
|
||||
);
|
||||
|
||||
ctx.logger.info('QM session created successfully', {
|
||||
sessionKey,
|
||||
sessionId,
|
||||
proxy: newSession.proxy,
|
||||
sessionCount: sessionManager.getSessions(sessionId).length,
|
||||
hasToken: !!sessionData.token
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
if (error.name === 'TimeoutError') {
|
||||
ctx.logger.warn('QM session creation timed out', { sessionKey, sessionId });
|
||||
} else {
|
||||
ctx.logger.error('Error creating QM session', { sessionKey, sessionId, error });
|
||||
}
|
||||
|
||||
// Cache failed session attempt for debugging
|
||||
await ctx.cache.set(
|
||||
`failed-session:${sessionKey}:${Date.now()}`,
|
||||
{ sessionId, proxy, error: error.message },
|
||||
{ ttl: 300 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function initializeQMResources(): Promise<void> {
|
||||
const ctx = OperationContext.create('qm', 'init');
|
||||
|
||||
// Check if already initialized
|
||||
const alreadyInitialized = await ctx.cache.get('initialized');
|
||||
if (alreadyInitialized) {
|
||||
ctx.logger.debug('QM resources already initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.logger.debug('Initializing QM resources...');
|
||||
|
||||
// Mark as initialized in cache and session manager
|
||||
await ctx.cache.set('initialized', true, { ttl: 3600 });
|
||||
await ctx.cache.set('initialization-time', new Date().toISOString());
|
||||
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
sessionManager.setInitialized(true);
|
||||
|
||||
ctx.logger.info('QM resources initialized successfully');
|
||||
}
|
||||
|
|
@ -0,0 +1,268 @@
|
|||
/**
|
||||
* QM Spider Operations - Symbol spider search functionality
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
import { QM_SESSION_IDS } from '../shared/config';
|
||||
import type { SymbolSpiderJob, SpiderResult } from '../shared/types';
|
||||
import { initializeQMResources } from './session.operations';
|
||||
import { searchQMSymbolsAPI } from './symbols.operations';
|
||||
|
||||
export async function spiderSymbolSearch(
|
||||
payload: SymbolSpiderJob
|
||||
): Promise<SpiderResult> {
|
||||
const ctx = OperationContext.create('qm', 'spider');
|
||||
|
||||
try {
|
||||
const { prefix, depth, source = 'qm', maxDepth = 4 } = payload;
|
||||
|
||||
ctx.logger.info('Starting spider search', {
|
||||
prefix: prefix || 'ROOT',
|
||||
depth,
|
||||
source,
|
||||
maxDepth
|
||||
});
|
||||
|
||||
// Check cache for recent results
|
||||
const cacheKey = `search-result:${prefix || 'ROOT'}:${depth}`;
|
||||
const cachedResult = await ctx.cache.get<SpiderResult>(cacheKey);
|
||||
if (cachedResult) {
|
||||
ctx.logger.debug('Using cached spider search result', { prefix, depth });
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
// Ensure resources are initialized
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
let result: SpiderResult;
|
||||
|
||||
// Root job: Create A-Z jobs
|
||||
if (prefix === null || prefix === undefined || prefix === '') {
|
||||
result = await createAlphabetJobs(source, maxDepth, ctx);
|
||||
} else {
|
||||
// Leaf job: Search for symbols with this prefix
|
||||
result = await searchAndSpawnJobs(prefix, depth, source, maxDepth, ctx);
|
||||
}
|
||||
|
||||
// Cache the result
|
||||
await ctx.cache.set(cacheKey, result, { ttl: 3600 });
|
||||
|
||||
// Store spider operation metrics in cache instead of PostgreSQL for now
|
||||
try {
|
||||
const statsKey = `spider-stats:${prefix || 'ROOT'}:${depth}:${Date.now()}`;
|
||||
await ctx.cache.set(statsKey, {
|
||||
handler: 'qm',
|
||||
operation: 'spider',
|
||||
prefix: prefix || 'ROOT',
|
||||
depth,
|
||||
symbolsFound: result.symbolsFound,
|
||||
jobsCreated: result.jobsCreated,
|
||||
searchTime: new Date().toISOString()
|
||||
}, { ttl: 86400 }); // Keep for 24 hours
|
||||
} catch (error) {
|
||||
ctx.logger.debug('Failed to store spider stats in cache', { error });
|
||||
}
|
||||
|
||||
ctx.logger.info('Spider search completed', {
|
||||
prefix: prefix || 'ROOT',
|
||||
depth,
|
||||
success: result.success,
|
||||
symbolsFound: result.symbolsFound,
|
||||
jobsCreated: result.jobsCreated
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error('Spider symbol search failed', { error, payload });
|
||||
const failedResult = { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
|
||||
// Cache failed result for a shorter time
|
||||
const cacheKey = `search-result:${payload.prefix || 'ROOT'}:${payload.depth}`;
|
||||
await ctx.cache.set(cacheKey, failedResult, { ttl: 300 });
|
||||
|
||||
return failedResult;
|
||||
}
|
||||
}
|
||||
|
||||
async function createAlphabetJobs(
|
||||
source: string,
|
||||
maxDepth: number,
|
||||
ctx: OperationContext
|
||||
): Promise<SpiderResult> {
|
||||
try {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
let jobsCreated = 0;
|
||||
|
||||
ctx.logger.info('Creating alphabet jobs (A-Z)');
|
||||
|
||||
// Create jobs for A-Z
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i); // A=65, B=66, etc.
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: letter,
|
||||
depth: 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: 5,
|
||||
delay: i * 100, // Stagger jobs by 100ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
// Cache alphabet job creation
|
||||
await ctx.cache.set('alphabet-jobs-created', {
|
||||
count: jobsCreated,
|
||||
timestamp: new Date().toISOString(),
|
||||
source,
|
||||
maxDepth
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(`Created ${jobsCreated} alphabet jobs (A-Z)`);
|
||||
return { success: true, symbolsFound: 0, jobsCreated };
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to create alphabet jobs', { error });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function searchAndSpawnJobs(
|
||||
prefix: string,
|
||||
depth: number,
|
||||
source: string,
|
||||
maxDepth: number,
|
||||
ctx: OperationContext
|
||||
): Promise<SpiderResult> {
|
||||
try {
|
||||
// Ensure sessions exist for symbol search
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
const lookupSession = sessionManager.getSession(QM_SESSION_IDS.LOOKUP);
|
||||
|
||||
if (!lookupSession) {
|
||||
ctx.logger.info('No lookup sessions available, creating sessions first...');
|
||||
const { createSessions } = await import('./session.operations');
|
||||
await createSessions();
|
||||
|
||||
// Wait a bit for session creation
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
// Search for symbols with this prefix
|
||||
const symbols = await searchQMSymbolsAPI(prefix);
|
||||
const symbolCount = symbols.length;
|
||||
|
||||
ctx.logger.info(`Prefix "${prefix}" returned ${symbolCount} symbols`);
|
||||
|
||||
let jobsCreated = 0;
|
||||
|
||||
// Store symbols in MongoDB
|
||||
if (ctx.mongodb && symbols.length > 0) {
|
||||
try {
|
||||
const updatedSymbols = symbols.map((symbol: Record<string, unknown>) => ({
|
||||
...symbol,
|
||||
qmSearchCode: symbol.symbol,
|
||||
symbol: (symbol.symbol as string)?.split(':')[0],
|
||||
searchPrefix: prefix,
|
||||
searchDepth: depth,
|
||||
discoveredAt: new Date()
|
||||
}));
|
||||
|
||||
await ctx.mongodb.batchUpsert('qmSymbols', updatedSymbols, ['qmSearchCode']);
|
||||
ctx.logger.debug('Stored symbols in MongoDB', { count: symbols.length });
|
||||
} catch (error) {
|
||||
ctx.logger.warn('Failed to store symbols in MongoDB', { error });
|
||||
}
|
||||
}
|
||||
|
||||
// If we have 50+ symbols and haven't reached max depth, spawn sub-jobs
|
||||
if (symbolCount >= 50 && depth < maxDepth) {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
|
||||
ctx.logger.info(`Spawning sub-jobs for prefix "${prefix}" (${symbolCount} >= 50 symbols)`);
|
||||
|
||||
// Create jobs for prefixA, prefixB, prefixC... prefixZ
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i);
|
||||
const newPrefix = prefix + letter;
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: newPrefix,
|
||||
depth: depth + 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: Math.max(1, 6 - depth), // Higher priority for deeper jobs
|
||||
delay: i * 50, // Stagger sub-jobs by 50ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
// Cache sub-job creation info
|
||||
await ctx.cache.set(`sub-jobs:${prefix}`, {
|
||||
parentPrefix: prefix,
|
||||
depth,
|
||||
symbolCount,
|
||||
jobsCreated,
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(`Created ${jobsCreated} sub-jobs for prefix "${prefix}"`);
|
||||
} else {
|
||||
// Terminal case: save symbols (already done above)
|
||||
ctx.logger.info(`Terminal case for prefix "${prefix}": ${symbolCount} symbols saved`);
|
||||
|
||||
// Cache terminal case info
|
||||
await ctx.cache.set(`terminal:${prefix}`, {
|
||||
prefix,
|
||||
depth,
|
||||
symbolCount,
|
||||
isTerminal: true,
|
||||
reason: symbolCount < 50 ? 'insufficient_symbols' : 'max_depth_reached',
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
}
|
||||
|
||||
return { success: true, symbolsFound: symbolCount, jobsCreated };
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.error(`Failed to search and spawn jobs for prefix "${prefix}"`, { error, depth });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
/**
|
||||
* QM Symbols Operations - Symbol fetching and API interactions
|
||||
*/
|
||||
|
||||
import { OperationContext } from '@stock-bot/utils';
|
||||
import { getRandomProxy } from '@stock-bot/utils';
|
||||
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
import { QM_SESSION_IDS, QM_CONFIG, SESSION_CONFIG } from '../shared/config';
|
||||
import type { SymbolSpiderJob, Exchange } from '../shared/types';
|
||||
import { initializeQMResources } from './session.operations';
|
||||
import { spiderSymbolSearch } from './spider.operations';
|
||||
|
||||
export async function fetchSymbols(): Promise<unknown[] | null> {
|
||||
const ctx = OperationContext.create('qm', 'symbols');
|
||||
|
||||
try {
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
if (!sessionManager.getInitialized()) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
ctx.logger.info('Starting QM spider-based symbol search...');
|
||||
|
||||
// Check if we have a recent symbol fetch
|
||||
const lastFetch = await ctx.cache.get('last-symbol-fetch');
|
||||
if (lastFetch) {
|
||||
ctx.logger.info('Recent symbol fetch found, using spider search');
|
||||
}
|
||||
|
||||
// Start the spider process with root job
|
||||
const rootJob: SymbolSpiderJob = {
|
||||
prefix: null, // Root job creates A-Z jobs
|
||||
depth: 0,
|
||||
source: 'qm',
|
||||
maxDepth: 4,
|
||||
};
|
||||
|
||||
const result = await spiderSymbolSearch(rootJob);
|
||||
|
||||
if (result.success) {
|
||||
// Cache successful fetch info
|
||||
await ctx.cache.set('last-symbol-fetch', {
|
||||
timestamp: new Date().toISOString(),
|
||||
jobsCreated: result.jobsCreated,
|
||||
success: true
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(
|
||||
`QM spider search initiated successfully. Created ${result.jobsCreated} initial jobs`
|
||||
);
|
||||
return [`Spider search initiated with ${result.jobsCreated} jobs`];
|
||||
} else {
|
||||
ctx.logger.error('Failed to initiate QM spider search');
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
ctx.logger.error('Failed to start QM spider symbol search', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function searchQMSymbolsAPI(query: string): Promise<any[]> {
|
||||
const ctx = OperationContext.create('qm', 'api-search');
|
||||
|
||||
const proxyInfo = await getRandomProxy();
|
||||
if (!proxyInfo) {
|
||||
throw new Error('No proxy available for QM API call');
|
||||
}
|
||||
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
const session = sessionManager.getSession(QM_SESSION_IDS.LOOKUP);
|
||||
|
||||
if (!session) {
|
||||
throw new Error(`No active session found for QM API with ID: ${QM_SESSION_IDS.LOOKUP}`);
|
||||
}
|
||||
|
||||
try {
|
||||
ctx.logger.debug('Searching QM symbols API', { query, proxy: session.proxy });
|
||||
|
||||
// Check cache for recent API results
|
||||
const cacheKey = `api-search:${query}`;
|
||||
const cachedResult = await ctx.cache.get(cacheKey);
|
||||
if (cachedResult) {
|
||||
ctx.logger.debug('Using cached API search result', { query });
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
// QM lookup endpoint for symbol search
|
||||
const searchParams = new URLSearchParams({
|
||||
marketType: 'equity',
|
||||
pathName: '/demo/portal/company-summary.php',
|
||||
q: query,
|
||||
qmodTool: 'SmartSymbolLookup',
|
||||
searchType: 'symbol',
|
||||
showFree: 'false',
|
||||
showHisa: 'false',
|
||||
webmasterId: '500'
|
||||
});
|
||||
|
||||
const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`;
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
headers: session.headers,
|
||||
signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const symbols = await response.json();
|
||||
|
||||
// Update session stats
|
||||
session.successfulCalls++;
|
||||
session.lastUsed = new Date();
|
||||
|
||||
// Process symbols and extract exchanges
|
||||
if (ctx.mongodb && symbols.length > 0) {
|
||||
try {
|
||||
const updatedSymbols = symbols.map((symbol: Record<string, unknown>) => ({
|
||||
...symbol,
|
||||
qmSearchCode: symbol.symbol,
|
||||
symbol: (symbol.symbol as string)?.split(':')[0],
|
||||
searchQuery: query,
|
||||
fetchedAt: new Date()
|
||||
}));
|
||||
|
||||
await ctx.mongodb.batchUpsert('qmSymbols', updatedSymbols, ['qmSearchCode']);
|
||||
|
||||
// Extract and store unique exchanges
|
||||
const exchanges: Exchange[] = [];
|
||||
for (const symbol of symbols) {
|
||||
if (!exchanges.some(ex => ex.exchange === symbol.exchange)) {
|
||||
exchanges.push({
|
||||
exchange: symbol.exchange,
|
||||
exchangeCode: symbol.exchangeCode,
|
||||
exchangeShortName: symbol.exchangeShortName,
|
||||
countryCode: symbol.countryCode,
|
||||
source: 'qm',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (exchanges.length > 0) {
|
||||
await ctx.mongodb.batchUpsert('qmExchanges', exchanges, ['exchange']);
|
||||
ctx.logger.debug('Stored exchanges in MongoDB', { count: exchanges.length });
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
ctx.logger.warn('Failed to store symbols/exchanges in MongoDB', { error });
|
||||
}
|
||||
}
|
||||
|
||||
// Cache the result
|
||||
await ctx.cache.set(cacheKey, symbols, { ttl: 1800 }); // 30 minutes
|
||||
|
||||
// Store API call stats
|
||||
await ctx.cache.set(`api-stats:${query}:${Date.now()}`, {
|
||||
query,
|
||||
symbolCount: symbols.length,
|
||||
proxy: session.proxy,
|
||||
success: true,
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 3600 });
|
||||
|
||||
ctx.logger.info(
|
||||
`QM API returned ${symbols.length} symbols for query: ${query}`,
|
||||
{ proxy: session.proxy, symbolCount: symbols.length }
|
||||
);
|
||||
|
||||
return symbols;
|
||||
|
||||
} catch (error) {
|
||||
// Update session failure stats
|
||||
session.failedCalls++;
|
||||
session.lastUsed = new Date();
|
||||
|
||||
// Cache failed API call info
|
||||
await ctx.cache.set(`api-failure:${query}:${Date.now()}`, {
|
||||
query,
|
||||
error: error.message,
|
||||
proxy: session.proxy,
|
||||
timestamp: new Date().toISOString()
|
||||
}, { ttl: 600 });
|
||||
|
||||
ctx.logger.error(`Error searching QM symbols for query "${query}"`, {
|
||||
error: error.message,
|
||||
proxy: session.proxy
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
78
apps/data-ingestion/src/handlers/qm/qm.handler.ts
Normal file
78
apps/data-ingestion/src/handlers/qm/qm.handler.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import {
|
||||
createJobHandler,
|
||||
handlerRegistry,
|
||||
type HandlerConfigWithSchedule
|
||||
} from '@stock-bot/queue';
|
||||
import type { SymbolSpiderJob } from './shared/types';
|
||||
|
||||
const handlerLogger = getLogger('qm-handler');
|
||||
|
||||
// Initialize and register the QM provider
|
||||
export function initializeQMProvider() {
|
||||
handlerLogger.debug('Registering QM provider with scheduled jobs...');
|
||||
|
||||
const qmProviderConfig: HandlerConfigWithSchedule = {
|
||||
name: 'qm',
|
||||
operations: {
|
||||
'create-sessions': createJobHandler(async () => {
|
||||
const { createSessions } = await import('./operations/session.operations');
|
||||
await createSessions();
|
||||
return { success: true, message: 'QM sessions created successfully' };
|
||||
}),
|
||||
'search-symbols': createJobHandler(async () => {
|
||||
const { fetchSymbols } = await import('./operations/symbols.operations');
|
||||
const symbols = await fetchSymbols();
|
||||
|
||||
if (symbols && symbols.length > 0) {
|
||||
return {
|
||||
success: true,
|
||||
message: 'QM symbol search completed successfully',
|
||||
count: symbols.length,
|
||||
symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
message: 'No symbols found',
|
||||
count: 0,
|
||||
};
|
||||
}
|
||||
}),
|
||||
'spider-symbol-search': createJobHandler(async (payload: SymbolSpiderJob) => {
|
||||
const { spiderSymbolSearch } = await import('./operations/spider.operations');
|
||||
const result = await spiderSymbolSearch(payload);
|
||||
|
||||
return result;
|
||||
}),
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'session-management',
|
||||
operation: 'create-sessions',
|
||||
cronPattern: '0 */15 * * *', // Every 15 minutes
|
||||
priority: 7,
|
||||
immediately: true, // Don't run on startup to avoid blocking
|
||||
description: 'Create and maintain QM sessions',
|
||||
},
|
||||
{
|
||||
type: 'qm-maintnance',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: {
|
||||
prefix: null,
|
||||
depth: 1,
|
||||
source: 'qm',
|
||||
maxDepth: 4
|
||||
},
|
||||
cronPattern: '0 0 * * 0', // Every Sunday at midnight
|
||||
priority: 10,
|
||||
immediately: true, // Don't run on startup - this is a heavy operation
|
||||
description: 'Comprehensive symbol search using QM API',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
handlerRegistry.registerWithSchedule(qmProviderConfig);
|
||||
handlerLogger.debug('QM provider registered successfully with scheduled jobs');
|
||||
}
|
||||
420
apps/data-ingestion/src/handlers/qm/qm.operations.ts.old
Normal file
420
apps/data-ingestion/src/handlers/qm/qm.operations.ts.old
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
import { getRandomUserAgent } from '@stock-bot/http';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { getMongoDBClient } from '@stock-bot/mongodb-client';
|
||||
import { QueueManager } from '@stock-bot/queue';
|
||||
import { isShutdownSignalReceived } from '@stock-bot/shutdown';
|
||||
import { getRandomProxy } from '@stock-bot/utils';
|
||||
|
||||
// Shared instances (module-scoped, not global)
|
||||
let isInitialized = false; // Track if resources are initialized
|
||||
let logger: ReturnType<typeof getLogger>;
|
||||
// let cache: CacheProvider;
|
||||
|
||||
export interface QMSession {
|
||||
proxy: string;
|
||||
headers: Record<string, string>;
|
||||
successfulCalls: number;
|
||||
failedCalls: number;
|
||||
lastUsed: Date;
|
||||
}
|
||||
|
||||
export interface SymbolSpiderJob {
|
||||
prefix: string | null; // null = root job (A-Z)
|
||||
depth: number; // 1=A, 2=AA, 3=AAA, etc.
|
||||
source: string; // 'qm'
|
||||
maxDepth?: number; // optional max depth limit
|
||||
}
|
||||
|
||||
interface Exchange {
|
||||
exchange: string;
|
||||
exchangeCode: string;
|
||||
exchangeShortName: string;
|
||||
countryCode: string;
|
||||
source: string;
|
||||
}
|
||||
|
||||
function getQmHeaders(): Record<string, string> {
|
||||
return {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
Accept: '*/*',
|
||||
'Accept-Language': 'en',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
Origin: 'https://www.quotemedia.com',
|
||||
Referer: 'https://www.quotemedia.com/',
|
||||
};
|
||||
}
|
||||
|
||||
const sessionCache: Record<string, QMSession[]> = {
|
||||
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
|
||||
// cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
|
||||
// '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
|
||||
// '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
|
||||
// a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
|
||||
// a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
|
||||
// b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
|
||||
dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6: [], //fceb3c4bdd // lookup
|
||||
// '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
|
||||
// '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
|
||||
// '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
|
||||
// d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
|
||||
// d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
|
||||
// '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
|
||||
};
|
||||
|
||||
export async function initializeQMResources(): Promise<void> {
|
||||
// Skip if already initialized
|
||||
if (isInitialized) {
|
||||
return;
|
||||
}
|
||||
logger = getLogger('qm-tasks');
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
export async function createSessions(): Promise<void> {
|
||||
try {
|
||||
//for each session, check array length, if less than 5, create new session
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
logger.info('Creating QM sessions...');
|
||||
for (const [sessionId, sessionArray] of Object.entries(sessionCache)) {
|
||||
const initialCount = sessionArray.length;
|
||||
const filteredArray = sessionArray.filter(session => session.failedCalls <= 10);
|
||||
sessionCache[sessionId] = filteredArray;
|
||||
|
||||
const removedCount = initialCount - filteredArray.length;
|
||||
if (removedCount > 0) {
|
||||
logger.info(
|
||||
`Removed ${removedCount} sessions with excessive failures for ${sessionId}. Remaining: ${filteredArray.length}`
|
||||
);
|
||||
}
|
||||
|
||||
while (sessionCache[sessionId].length < 10) {
|
||||
if(isShutdownSignalReceived()) {
|
||||
logger.info('Shutting down, skipping session creation');
|
||||
break; // Exit if shutting down
|
||||
}
|
||||
logger.info(`Creating new session for ${sessionId}`);
|
||||
const proxyInfo = await getRandomProxy();
|
||||
if (!proxyInfo) {
|
||||
logger.error('No proxy available for QM session creation');
|
||||
break; // Skip session creation if no proxy is available
|
||||
}
|
||||
|
||||
// Convert ProxyInfo to string format
|
||||
const auth = proxyInfo.username && proxyInfo.password ? `${proxyInfo.username}:${proxyInfo.password}@` : '';
|
||||
const proxy = `${proxyInfo.protocol}://${auth}${proxyInfo.host}:${proxyInfo.port}`;
|
||||
const newSession: QMSession = {
|
||||
proxy: proxy, // Placeholder, should be set to a valid proxy
|
||||
headers: getQmHeaders(),
|
||||
successfulCalls: 0,
|
||||
failedCalls: 0,
|
||||
lastUsed: new Date(),
|
||||
};
|
||||
const sessionResponse = await fetch(
|
||||
`https://app.quotemedia.com/auth/g/authenticate/dataTool/v0/500/${sessionId}`,
|
||||
{
|
||||
method: 'GET',
|
||||
proxy: newSession.proxy,
|
||||
headers: newSession.headers,
|
||||
}
|
||||
);
|
||||
|
||||
logger.debug('Session response received', {
|
||||
status: sessionResponse.status,
|
||||
sessionId,
|
||||
});
|
||||
if (!sessionResponse.ok) {
|
||||
logger.error('Failed to create QM session', {
|
||||
sessionId,
|
||||
status: sessionResponse.status,
|
||||
statusText: sessionResponse.statusText,
|
||||
});
|
||||
continue; // Skip this session if creation failed
|
||||
}
|
||||
const sessionData = await sessionResponse.json();
|
||||
logger.info('QM session created successfully', {
|
||||
sessionId,
|
||||
sessionData,
|
||||
proxy: newSession.proxy,
|
||||
sessionCount: sessionCache[sessionId].length + 1,
|
||||
});
|
||||
newSession.headers['Datatool-Token'] = sessionData.token;
|
||||
sessionCache[sessionId].push(newSession);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch QM session', { error });
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Spider-based symbol search functions
|
||||
export async function spiderSymbolSearch(
|
||||
payload: SymbolSpiderJob
|
||||
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
const { prefix, depth, source = 'qm', maxDepth = 4 } = payload;
|
||||
|
||||
logger.info(`Starting spider search`, { prefix: prefix || 'ROOT', depth, source });
|
||||
|
||||
// Root job: Create A-Z jobs
|
||||
if (prefix === null || prefix === undefined || prefix === '') {
|
||||
return await createAlphabetJobs(source, maxDepth);
|
||||
}
|
||||
|
||||
// Leaf job: Search for symbols with this prefix
|
||||
return await searchAndSpawnJobs(prefix, depth, source, maxDepth);
|
||||
} catch (error) {
|
||||
logger.error('Spider symbol search failed', { error, payload });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function createAlphabetJobs(
|
||||
source: string,
|
||||
maxDepth: number
|
||||
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
||||
try {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
let jobsCreated = 0;
|
||||
|
||||
// Create jobs for A-Z
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i); // A=65, B=66, etc.
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: letter,
|
||||
depth: 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: 5,
|
||||
delay: i * 100, // Stagger jobs by 100ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
logger.info(`Created ${jobsCreated} alphabet jobs (A-Z)`);
|
||||
return { success: true, symbolsFound: 0, jobsCreated };
|
||||
} catch (error) {
|
||||
logger.error('Failed to create alphabet jobs', { error });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async function searchAndSpawnJobs(
|
||||
prefix: string,
|
||||
depth: number,
|
||||
source: string,
|
||||
maxDepth: number
|
||||
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
|
||||
try {
|
||||
// Ensure sessions exist
|
||||
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6';
|
||||
const currentSessions = sessionCache[sessionId] || [];
|
||||
|
||||
if (currentSessions.length === 0) {
|
||||
logger.info('No sessions found, creating sessions first...');
|
||||
await createSessions();
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
|
||||
// Search for symbols with this prefix
|
||||
const symbols = await searchQMSymbolsAPI(prefix);
|
||||
const symbolCount = symbols.length;
|
||||
|
||||
logger.info(`Prefix "${prefix}" returned ${symbolCount} symbols`);
|
||||
|
||||
let jobsCreated = 0;
|
||||
|
||||
// If we have 50+ symbols and haven't reached max depth, spawn sub-jobs
|
||||
if (symbolCount >= 50 && depth < maxDepth) {
|
||||
const queueManager = QueueManager.getInstance();
|
||||
const queue = queueManager.getQueue('qm');
|
||||
|
||||
logger.info(`Spawning sub-jobs for prefix "${prefix}" (${symbolCount} >= 50 symbols)`);
|
||||
|
||||
// Create jobs for prefixA, prefixB, prefixC... prefixZ
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i);
|
||||
const newPrefix = prefix + letter;
|
||||
|
||||
const job: SymbolSpiderJob = {
|
||||
prefix: newPrefix,
|
||||
depth: depth + 1,
|
||||
source,
|
||||
maxDepth,
|
||||
};
|
||||
|
||||
await queue.add(
|
||||
'spider-symbol-search',
|
||||
{
|
||||
handler: 'qm',
|
||||
operation: 'spider-symbol-search',
|
||||
payload: job,
|
||||
},
|
||||
{
|
||||
priority: Math.max(1, 6 - depth), // Higher priority for deeper jobs
|
||||
delay: i * 50, // Stagger sub-jobs by 50ms
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
}
|
||||
);
|
||||
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
logger.info(`Created ${jobsCreated} sub-jobs for prefix "${prefix}"`);
|
||||
} else {
|
||||
// Terminal case: save symbols and exchanges (already done in searchQMSymbolsAPI)
|
||||
logger.info(`Terminal case for prefix "${prefix}": ${symbolCount} symbols saved`);
|
||||
}
|
||||
|
||||
return { success: true, symbolsFound: symbolCount, jobsCreated };
|
||||
} catch (error) {
|
||||
logger.error(`Failed to search and spawn jobs for prefix "${prefix}"`, { error, depth });
|
||||
return { success: false, symbolsFound: 0, jobsCreated: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// API call function to search symbols via QM
|
||||
async function searchQMSymbolsAPI(query: string): Promise<string[]> {
|
||||
const proxyInfo = await getRandomProxy();
|
||||
|
||||
if (!proxyInfo) {
|
||||
throw new Error('No proxy available for QM API call');
|
||||
}
|
||||
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6'; // Use the session ID for symbol lookup
|
||||
const session =
|
||||
sessionCache[sessionId][Math.floor(Math.random() * sessionCache[sessionId].length)]; // lookup session
|
||||
if (!session) {
|
||||
throw new Error(`No active session found for QM API with ID: ${sessionId}`);
|
||||
}
|
||||
try {
|
||||
// QM lookup endpoint for symbol search
|
||||
const apiUrl = `https://app.quotemedia.com/datatool/lookup.json?marketType=equity&pathName=%2Fdemo%2Fportal%2Fcompany-summary.php&q=${encodeURIComponent(query)}&qmodTool=SmartSymbolLookup&searchType=symbol&showFree=false&showHisa=false&webmasterId=500`;
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
headers: session.headers,
|
||||
proxy: session.proxy,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const symbols = await response.json();
|
||||
const mongoClient = getMongoDBClient();
|
||||
const updatedSymbols = symbols.map((symbol: Record<string, unknown>) => {
|
||||
return {
|
||||
...symbol,
|
||||
qmSearchCode: symbol.symbol, // Store original symbol for reference
|
||||
symbol: symbol.symbol.split(':')[0], // Extract symbol from "symbol:exchange"
|
||||
};
|
||||
});
|
||||
await mongoClient.batchUpsert('qmSymbols', updatedSymbols, ['qmSearchCode']);
|
||||
const exchanges: Exchange[] = [];
|
||||
for (const symbol of symbols) {
|
||||
if (!exchanges.some(ex => ex.exchange === symbol.exchange)) {
|
||||
exchanges.push({
|
||||
exchange: symbol.exchange,
|
||||
exchangeCode: symbol.exchangeCode,
|
||||
exchangeShortName: symbol.exchangeShortName,
|
||||
countryCode: symbol.countryCode,
|
||||
source: 'qm',
|
||||
});
|
||||
}
|
||||
}
|
||||
await mongoClient.batchUpsert('qmExchanges', exchanges, ['exchange']);
|
||||
session.successfulCalls++;
|
||||
session.lastUsed = new Date();
|
||||
|
||||
logger.info(
|
||||
`QM API returned ${symbols.length} symbols for query: ${query} with proxy ${session.proxy}`
|
||||
);
|
||||
return symbols;
|
||||
} catch (error) {
|
||||
logger.error(`Error searching QM symbols for query "${query}":`, error);
|
||||
if (session) {
|
||||
session.failedCalls++;
|
||||
session.lastUsed = new Date();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchSymbols(): Promise<unknown[] | null> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
logger.info('🔄 Starting QM spider-based symbol search...');
|
||||
|
||||
// Start the spider process with root job
|
||||
const rootJob: SymbolSpiderJob = {
|
||||
prefix: null, // Root job creates A-Z jobs
|
||||
depth: 0,
|
||||
source: 'qm',
|
||||
maxDepth: 4,
|
||||
};
|
||||
|
||||
const result = await spiderSymbolSearch(rootJob);
|
||||
|
||||
if (result.success) {
|
||||
logger.info(
|
||||
`QM spider search initiated successfully. Created ${result.jobsCreated} initial jobs`
|
||||
);
|
||||
return [`Spider search initiated with ${result.jobsCreated} jobs`];
|
||||
} else {
|
||||
logger.error('Failed to initiate QM spider search');
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to start QM spider symbol search', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchExchanges(): Promise<unknown[] | null> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
logger.info('🔄 QM exchanges fetch - not implemented yet');
|
||||
// TODO: Implement QM exchanges fetching logic
|
||||
return null;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch QM exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const qmTasks = {
|
||||
createSessions,
|
||||
fetchSymbols,
|
||||
fetchExchanges,
|
||||
spiderSymbolSearch,
|
||||
};
|
||||
56
apps/data-ingestion/src/handlers/qm/shared/config.ts
Normal file
56
apps/data-ingestion/src/handlers/qm/shared/config.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Shared configuration for QM operations
|
||||
*/
|
||||
|
||||
import { getRandomUserAgent } from '@stock-bot/http';
|
||||
|
||||
// QM Session IDs for different endpoints
|
||||
export const QM_SESSION_IDS = {
|
||||
LOOKUP: 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6', // lookup endpoint
|
||||
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
|
||||
// cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
|
||||
// '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
|
||||
// '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
|
||||
// a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
|
||||
// a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
|
||||
// b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
|
||||
// '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
|
||||
// '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
|
||||
// '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
|
||||
// d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
|
||||
// d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
|
||||
// '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
|
||||
// Add other session IDs as needed
|
||||
} as const;
|
||||
|
||||
// QM API Configuration
|
||||
export const QM_CONFIG = {
|
||||
BASE_URL: 'https://app.quotemedia.com',
|
||||
AUTH_PATH: '/auth/g/authenticate/dataTool/v0/500',
|
||||
LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json',
|
||||
ORIGIN: 'https://www.quotemedia.com',
|
||||
REFERER: 'https://www.quotemedia.com/',
|
||||
} as const;
|
||||
|
||||
// Session management settings
|
||||
export const SESSION_CONFIG = {
|
||||
MIN_SESSIONS: 5,
|
||||
MAX_SESSIONS: 10,
|
||||
MAX_FAILED_CALLS: 10,
|
||||
SESSION_TIMEOUT: 10000, // 10 seconds
|
||||
API_TIMEOUT: 15000, // 15 seconds
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Generate standard QM headers
|
||||
*/
|
||||
export function getQmHeaders(): Record<string, string> {
|
||||
return {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
Accept: '*/*',
|
||||
'Accept-Language': 'en',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
Origin: QM_CONFIG.ORIGIN,
|
||||
Referer: QM_CONFIG.REFERER,
|
||||
};
|
||||
}
|
||||
136
apps/data-ingestion/src/handlers/qm/shared/session-manager.ts
Normal file
136
apps/data-ingestion/src/handlers/qm/shared/session-manager.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
/**
|
||||
* QM Session Manager - Centralized session state management
|
||||
*/
|
||||
|
||||
import type { QMSession } from './types';
|
||||
import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
|
||||
|
||||
export class QMSessionManager {
|
||||
private static instance: QMSessionManager | null = null;
|
||||
private sessionCache: Record<string, QMSession[]> = {};
|
||||
private isInitialized = false;
|
||||
|
||||
private constructor() {
|
||||
// Initialize session cache with known session IDs
|
||||
Object.values(QM_SESSION_IDS).forEach(sessionId => {
|
||||
this.sessionCache[sessionId] = [];
|
||||
});
|
||||
}
|
||||
|
||||
static getInstance(): QMSessionManager {
|
||||
if (!QMSessionManager.instance) {
|
||||
QMSessionManager.instance = new QMSessionManager();
|
||||
}
|
||||
return QMSessionManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a random session for the given session ID
|
||||
*/
|
||||
getSession(sessionId: string): QMSession | null {
|
||||
const sessions = this.sessionCache[sessionId];
|
||||
if (!sessions || sessions.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Filter out sessions with excessive failures
|
||||
const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
if (validSessions.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return validSessions[Math.floor(Math.random() * validSessions.length)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a session to the cache
|
||||
*/
|
||||
addSession(sessionId: string, session: QMSession): void {
|
||||
if (!this.sessionCache[sessionId]) {
|
||||
this.sessionCache[sessionId] = [];
|
||||
}
|
||||
this.sessionCache[sessionId].push(session);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all sessions for a session ID
|
||||
*/
|
||||
getSessions(sessionId: string): QMSession[] {
|
||||
return this.sessionCache[sessionId] || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session count for all session IDs
|
||||
*/
|
||||
getSessionCount(): number {
|
||||
return Object.values(this.sessionCache).reduce((total, sessions) => total + sessions.length, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up failed sessions
|
||||
*/
|
||||
cleanupFailedSessions(): number {
|
||||
let removedCount = 0;
|
||||
|
||||
Object.keys(this.sessionCache).forEach(sessionId => {
|
||||
const initialCount = this.sessionCache[sessionId].length;
|
||||
this.sessionCache[sessionId] = this.sessionCache[sessionId].filter(
|
||||
session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
|
||||
);
|
||||
removedCount += initialCount - this.sessionCache[sessionId].length;
|
||||
});
|
||||
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if more sessions are needed for a session ID
|
||||
*/
|
||||
needsMoreSessions(sessionId: string): boolean {
|
||||
const sessions = this.sessionCache[sessionId] || [];
|
||||
const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
return validSessions.length < SESSION_CONFIG.MIN_SESSIONS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if session ID is at capacity
|
||||
*/
|
||||
isAtCapacity(sessionId: string): boolean {
|
||||
const sessions = this.sessionCache[sessionId] || [];
|
||||
return sessions.length >= SESSION_CONFIG.MAX_SESSIONS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
const stats: Record<string, { total: number; valid: number; failed: number }> = {};
|
||||
|
||||
Object.entries(this.sessionCache).forEach(([sessionId, sessions]) => {
|
||||
const validSessions = sessions.filter(session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
const failedSessions = sessions.filter(session => session.failedCalls > SESSION_CONFIG.MAX_FAILED_CALLS);
|
||||
|
||||
stats[sessionId] = {
|
||||
total: sessions.length,
|
||||
valid: validSessions.length,
|
||||
failed: failedSessions.length
|
||||
};
|
||||
});
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark manager as initialized
|
||||
*/
|
||||
setInitialized(initialized: boolean = true): void {
|
||||
this.isInitialized = initialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if manager is initialized
|
||||
*/
|
||||
getInitialized(): boolean {
|
||||
return this.isInitialized;
|
||||
}
|
||||
}
|
||||
32
apps/data-ingestion/src/handlers/qm/shared/types.ts
Normal file
32
apps/data-ingestion/src/handlers/qm/shared/types.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* Shared types for QM operations
|
||||
*/
|
||||
|
||||
export interface QMSession {
|
||||
proxy: string;
|
||||
headers: Record<string, string>;
|
||||
successfulCalls: number;
|
||||
failedCalls: number;
|
||||
lastUsed: Date;
|
||||
}
|
||||
|
||||
export interface SymbolSpiderJob {
|
||||
prefix: string | null; // null = root job (A-Z)
|
||||
depth: number; // 1=A, 2=AA, 3=AAA, etc.
|
||||
source: string; // 'qm'
|
||||
maxDepth?: number; // optional max depth limit
|
||||
}
|
||||
|
||||
export interface Exchange {
|
||||
exchange: string;
|
||||
exchangeCode: string;
|
||||
exchangeShortName: string;
|
||||
countryCode: string;
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface SpiderResult {
|
||||
success: boolean;
|
||||
symbolsFound: number;
|
||||
jobsCreated: number;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue