refactored monorepo for more projects
This commit is contained in:
parent
4632c174dc
commit
9492f1b15e
180 changed files with 1438 additions and 424 deletions
|
|
@ -0,0 +1,19 @@
|
|||
/**
|
||||
* QM Exchanges Operations - Simple exchange data fetching
|
||||
*/
|
||||
|
||||
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||
|
||||
export async function fetchExchanges(services: IServiceContainer): Promise<any[]> {
|
||||
// Get exchanges from MongoDB
|
||||
const exchanges = await services.mongodb.collection('qm_exchanges').find({}).toArray();
|
||||
|
||||
return exchanges;
|
||||
}
|
||||
|
||||
export async function getExchangeByCode(services: IServiceContainer, code: string): Promise<any> {
|
||||
// Get specific exchange by code
|
||||
const exchange = await services.mongodb.collection('qm_exchanges').findOne({ code });
|
||||
|
||||
return exchange;
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/**
|
||||
* QM Session Actions - Session management and creation
|
||||
*/
|
||||
|
||||
import { BaseHandler } from '@stock-bot/core/handlers';
|
||||
import { QM_SESSION_IDS, SESSION_CONFIG } from '../shared/config';
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
|
||||
/**
|
||||
* Check existing sessions and queue creation jobs for needed sessions
|
||||
*/
|
||||
export async function checkSessions(handler: BaseHandler): Promise<{
|
||||
cleaned: number;
|
||||
queued: number;
|
||||
message: string;
|
||||
}> {
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
const cleanedCount = sessionManager.cleanupFailedSessions();
|
||||
// Check which session IDs need more sessions and queue creation jobs
|
||||
let queuedCount = 0;
|
||||
for (const [sessionType, sessionId] of Object.entries(QM_SESSION_IDS)) {
|
||||
console.log(`Checking session ID: ${sessionId}`);
|
||||
if (sessionManager.needsMoreSessions(sessionId)) {
|
||||
const currentCount = sessionManager.getSessions(sessionId).length;
|
||||
const neededSessions = SESSION_CONFIG.MAX_SESSIONS - currentCount;
|
||||
for (let i = 0; i < neededSessions; i++) {
|
||||
await handler.scheduleOperation('create-session', { sessionId, sessionType });
|
||||
handler.logger.info(`Queued job to create session for ${sessionType}`);
|
||||
queuedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
cleaned: cleanedCount,
|
||||
queued: queuedCount,
|
||||
message: `Session check completed: cleaned ${cleanedCount}, queued ${queuedCount}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a single session for a specific session ID
|
||||
*/
|
||||
export async function createSingleSession(
|
||||
handler: BaseHandler,
|
||||
input: any
|
||||
): Promise<{ sessionId: string; status: string; sessionType: string }> {
|
||||
const { sessionId, sessionType } = input || {};
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
|
||||
// Get proxy from proxy service
|
||||
const proxyString = handler.proxy.getProxy();
|
||||
|
||||
// const session = {
|
||||
// proxy: proxyString || 'http://proxy:8080',
|
||||
// headers: sessionManager.getQmHeaders(),
|
||||
// successfulCalls: 0,
|
||||
// failedCalls: 0,
|
||||
// lastUsed: new Date()
|
||||
// };
|
||||
|
||||
handler.logger.info(`Creating session for ${sessionType}`);
|
||||
|
||||
// Add session to manager
|
||||
// sessionManager.addSession(sessionType, session);
|
||||
|
||||
return {
|
||||
sessionId: sessionType,
|
||||
status: 'created',
|
||||
sessionType,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/**
|
||||
* QM Spider Operations - Simple symbol discovery
|
||||
*/
|
||||
|
||||
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||
import type { SymbolSpiderJob } from '../shared/types';
|
||||
|
||||
export async function spiderSymbolSearch(
|
||||
services: IServiceContainer,
|
||||
config: SymbolSpiderJob
|
||||
): Promise<{ foundSymbols: number; depth: number }> {
|
||||
// Simple spider implementation
|
||||
// TODO: Implement actual API calls to discover symbols
|
||||
|
||||
// For now, just return mock results
|
||||
const foundSymbols = Math.floor(Math.random() * 10) + 1;
|
||||
|
||||
return {
|
||||
foundSymbols,
|
||||
depth: config.depth,
|
||||
};
|
||||
}
|
||||
|
||||
export async function queueSymbolDiscovery(
|
||||
services: IServiceContainer,
|
||||
searchTerms: string[]
|
||||
): Promise<void> {
|
||||
// Queue symbol discovery jobs
|
||||
for (const term of searchTerms) {
|
||||
// TODO: Queue actual discovery jobs
|
||||
await services.cache.set(`discovery:${term}`, { queued: true }, 3600);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
/**
|
||||
* QM Symbols Operations - Simple symbol fetching
|
||||
*/
|
||||
|
||||
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||
|
||||
export async function searchSymbols(services: IServiceContainer): Promise<any[]> {
|
||||
// Get symbols from MongoDB
|
||||
const symbols = await services.mongodb.collection('qm_symbols').find({}).limit(50).toArray();
|
||||
|
||||
return symbols;
|
||||
}
|
||||
|
||||
export async function fetchSymbolData(services: IServiceContainer, symbol: string): Promise<any> {
|
||||
// Fetch data for a specific symbol
|
||||
const symbolData = await services.mongodb.collection('qm_symbols').findOne({ symbol });
|
||||
|
||||
return symbolData;
|
||||
}
|
||||
103
apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts
Normal file
103
apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import { BaseHandler, Handler, type IServiceContainer } from '@stock-bot/handlers';
|
||||
|
||||
@Handler('qm')
|
||||
export class QMHandler extends BaseHandler {
|
||||
constructor(services: IServiceContainer) {
|
||||
super(services); // Handler name read from @Handler decorator
|
||||
}
|
||||
|
||||
// @Operation('check-sessions')
|
||||
// @QueueSchedule('0 */15 * * *', {
|
||||
// priority: 7,
|
||||
// immediately: true,
|
||||
// description: 'Check and maintain QM sessions'
|
||||
// })
|
||||
// async checkSessions(input: unknown, context: ExecutionContext): Promise<unknown> {
|
||||
// // Call the session maintenance action
|
||||
// const { checkSessions } = await import('./actions/session.action');
|
||||
// return await checkSessions(this);
|
||||
// }
|
||||
|
||||
// @Operation('create-session')
|
||||
// async createSession(input: unknown, context: ExecutionContext): Promise<unknown> {
|
||||
// // Call the individual session creation action
|
||||
// const { createSingleSession } = await import('./actions/session.action');
|
||||
// return await createSingleSession(this, input);
|
||||
// }
|
||||
|
||||
// @Operation('search-symbols')
|
||||
// async searchSymbols(_input: unknown, _context: ExecutionContext): Promise<unknown> {
|
||||
// this.logger.info('Searching QM symbols with new DI pattern...');
|
||||
// try {
|
||||
// // Check existing symbols in MongoDB
|
||||
// const symbolsCollection = this.mongodb.collection('qm_symbols');
|
||||
// const symbols = await symbolsCollection.find({}).limit(100).toArray();
|
||||
|
||||
// this.logger.info('QM symbol search completed', { count: symbols.length });
|
||||
|
||||
// if (symbols && symbols.length > 0) {
|
||||
// // Cache result for performance
|
||||
// await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800);
|
||||
|
||||
// return {
|
||||
// success: true,
|
||||
// message: 'QM symbol search completed successfully',
|
||||
// count: symbols.length,
|
||||
// symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
|
||||
// };
|
||||
// } else {
|
||||
// // No symbols found - this is expected initially
|
||||
// this.logger.info('No QM symbols found in database yet');
|
||||
// return {
|
||||
// success: true,
|
||||
// message: 'No symbols found yet - database is empty',
|
||||
// count: 0,
|
||||
// };
|
||||
// }
|
||||
|
||||
// } catch (error) {
|
||||
// this.logger.error('Failed to search QM symbols', { error });
|
||||
// throw error;
|
||||
// }
|
||||
// }
|
||||
|
||||
// @Operation('spider-symbol-search')
|
||||
// @QueueSchedule('0 0 * * 0', {
|
||||
// priority: 10,
|
||||
// immediately: false,
|
||||
// description: 'Comprehensive symbol search using QM API'
|
||||
// })
|
||||
// async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise<unknown> {
|
||||
// // Set default payload for scheduled runs
|
||||
// const jobPayload: SymbolSpiderJob = payload || {
|
||||
// prefix: null,
|
||||
// depth: 1,
|
||||
// source: 'qm',
|
||||
// maxDepth: 4
|
||||
// };
|
||||
|
||||
// this.logger.info('Starting QM spider symbol search', { payload: jobPayload });
|
||||
|
||||
// // Store spider job info in cache (temporary data)
|
||||
// const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`;
|
||||
// const spiderResult = {
|
||||
// payload: jobPayload,
|
||||
// startTime: new Date().toISOString(),
|
||||
// status: 'started',
|
||||
// jobId: spiderJobId
|
||||
// };
|
||||
|
||||
// // Store in cache with 1 hour TTL (temporary data)
|
||||
// await this.cache.set(spiderJobId, spiderResult, 3600);
|
||||
// this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 });
|
||||
|
||||
// // Schedule follow-up processing if needed
|
||||
// await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, { delay: 5000 });
|
||||
|
||||
// return {
|
||||
// success: true,
|
||||
// message: 'QM spider search initiated',
|
||||
// spiderJobId
|
||||
// };
|
||||
// }
|
||||
}
|
||||
38
apps/stock/data-ingestion/src/handlers/qm/shared/config.ts
Normal file
38
apps/stock/data-ingestion/src/handlers/qm/shared/config.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Shared configuration for QM operations
|
||||
*/
|
||||
|
||||
// QM Session IDs for different endpoints
|
||||
export const QM_SESSION_IDS = {
|
||||
LOOKUP: 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6', // lookup endpoint
|
||||
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
|
||||
// cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
|
||||
// '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
|
||||
// '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
|
||||
// a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
|
||||
// a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
|
||||
// b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
|
||||
// '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
|
||||
// '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
|
||||
// '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
|
||||
// d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
|
||||
// d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
|
||||
// '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
|
||||
// Add other session IDs as needed
|
||||
} as const;
|
||||
|
||||
// QM API Configuration
|
||||
export const QM_CONFIG = {
|
||||
BASE_URL: 'https://app.quotemedia.com',
|
||||
AUTH_PATH: '/auth/g/authenticate/dataTool/v0/500',
|
||||
LOOKUP_URL: 'https://app.quotemedia.com/datatool/lookup.json',
|
||||
} as const;
|
||||
|
||||
// Session management settings
|
||||
export const SESSION_CONFIG = {
|
||||
MIN_SESSIONS: 5,
|
||||
MAX_SESSIONS: 10,
|
||||
MAX_FAILED_CALLS: 10,
|
||||
SESSION_TIMEOUT: 10000, // 10 seconds
|
||||
API_TIMEOUT: 15000, // 15 seconds
|
||||
} as const;
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
/**
|
||||
* QM Session Manager - Centralized session state management
|
||||
*/
|
||||
|
||||
import { getRandomUserAgent } from '@stock-bot/utils';
|
||||
import { QM_SESSION_IDS, SESSION_CONFIG } from './config';
|
||||
import type { QMSession } from './types';
|
||||
|
||||
export class QMSessionManager {
|
||||
private static instance: QMSessionManager | null = null;
|
||||
private sessionCache: Record<string, QMSession[]> = {};
|
||||
private isInitialized = false;
|
||||
|
||||
private constructor() {
|
||||
// Initialize session cache with known session IDs
|
||||
Object.values(QM_SESSION_IDS).forEach(sessionId => {
|
||||
this.sessionCache[sessionId] = [];
|
||||
});
|
||||
}
|
||||
|
||||
static getInstance(): QMSessionManager {
|
||||
if (!QMSessionManager.instance) {
|
||||
QMSessionManager.instance = new QMSessionManager();
|
||||
}
|
||||
return QMSessionManager.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a random session for the given session ID
|
||||
*/
|
||||
getSession(sessionId: string): QMSession | null {
|
||||
const sessions = this.sessionCache[sessionId];
|
||||
if (!sessions || sessions.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Filter out sessions with excessive failures
|
||||
const validSessions = sessions.filter(
|
||||
session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
|
||||
);
|
||||
if (validSessions.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return validSessions[Math.floor(Math.random() * validSessions.length)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a session to the cache
|
||||
*/
|
||||
addSession(sessionId: string, session: QMSession): void {
|
||||
if (!this.sessionCache[sessionId]) {
|
||||
this.sessionCache[sessionId] = [];
|
||||
}
|
||||
this.sessionCache[sessionId].push(session);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all sessions for a session ID
|
||||
*/
|
||||
getSessions(sessionId: string): QMSession[] {
|
||||
return this.sessionCache[sessionId] || [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session count for all session IDs
|
||||
*/
|
||||
getSessionCount(): number {
|
||||
return Object.values(this.sessionCache).reduce((total, sessions) => total + sessions.length, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up failed sessions
|
||||
*/
|
||||
cleanupFailedSessions(): number {
|
||||
let removedCount = 0;
|
||||
|
||||
Object.keys(this.sessionCache).forEach(sessionId => {
|
||||
const initialCount = this.sessionCache[sessionId].length;
|
||||
this.sessionCache[sessionId] = this.sessionCache[sessionId].filter(
|
||||
session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
|
||||
);
|
||||
removedCount += initialCount - this.sessionCache[sessionId].length;
|
||||
});
|
||||
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
getQmHeaders(): Record<string, string> {
|
||||
return {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
Accept: '*/*',
|
||||
'Accept-Language': 'en',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
Origin: 'https://www.quotemedia.com',
|
||||
Referer: 'https://www.quotemedia.com/',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if more sessions are needed for a session ID
|
||||
*/
|
||||
needsMoreSessions(sessionId: string): boolean {
|
||||
const sessions = this.sessionCache[sessionId] || [];
|
||||
const validSessions = sessions.filter(
|
||||
session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
|
||||
);
|
||||
return validSessions.length < SESSION_CONFIG.MIN_SESSIONS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if session ID is at capacity
|
||||
*/
|
||||
isAtCapacity(sessionId: string): boolean {
|
||||
const sessions = this.sessionCache[sessionId] || [];
|
||||
return sessions.length >= SESSION_CONFIG.MAX_SESSIONS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
const stats: Record<string, { total: number; valid: number; failed: number }> = {};
|
||||
|
||||
Object.entries(this.sessionCache).forEach(([sessionId, sessions]) => {
|
||||
const validSessions = sessions.filter(
|
||||
session => session.failedCalls <= SESSION_CONFIG.MAX_FAILED_CALLS
|
||||
);
|
||||
const failedSessions = sessions.filter(
|
||||
session => session.failedCalls > SESSION_CONFIG.MAX_FAILED_CALLS
|
||||
);
|
||||
|
||||
stats[sessionId] = {
|
||||
total: sessions.length,
|
||||
valid: validSessions.length,
|
||||
failed: failedSessions.length,
|
||||
};
|
||||
});
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark manager as initialized
|
||||
*/
|
||||
setInitialized(initialized: boolean = true): void {
|
||||
this.isInitialized = initialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if manager is initialized
|
||||
*/
|
||||
getInitialized(): boolean {
|
||||
return this.isInitialized;
|
||||
}
|
||||
}
|
||||
32
apps/stock/data-ingestion/src/handlers/qm/shared/types.ts
Normal file
32
apps/stock/data-ingestion/src/handlers/qm/shared/types.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* Shared types for QM operations
|
||||
*/
|
||||
|
||||
export interface QMSession {
|
||||
proxy: string;
|
||||
headers: Record<string, string>;
|
||||
successfulCalls: number;
|
||||
failedCalls: number;
|
||||
lastUsed: Date;
|
||||
}
|
||||
|
||||
export interface SymbolSpiderJob {
|
||||
prefix: string | null; // null = root job (A-Z)
|
||||
depth: number; // 1=A, 2=AA, 3=AAA, etc.
|
||||
source: string; // 'qm'
|
||||
maxDepth?: number; // optional max depth limit
|
||||
}
|
||||
|
||||
export interface Exchange {
|
||||
exchange: string;
|
||||
exchangeCode: string;
|
||||
exchangeShortName: string;
|
||||
countryCode: string;
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface SpiderResult {
|
||||
success: boolean;
|
||||
symbolsFound: number;
|
||||
jobsCreated: number;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue