starting to add qm sessions and symbols
This commit is contained in:
parent
e9ff913b7e
commit
f05d26d703
13 changed files with 652 additions and 432 deletions
|
|
@ -18,10 +18,12 @@
|
|||
"@stock-bot/http": "*",
|
||||
"@stock-bot/logger": "*",
|
||||
"@stock-bot/mongodb-client": "*",
|
||||
"@stock-bot/queue": "*",
|
||||
"@stock-bot/questdb-client": "*",
|
||||
"@stock-bot/queue": "*",
|
||||
"@stock-bot/shutdown": "*",
|
||||
"@stock-bot/types": "*",
|
||||
"chromium-bidi": "^5.3.1",
|
||||
"electron": "^36.4.0",
|
||||
"hono": "^4.0.0",
|
||||
"p-limit": "^6.2.0",
|
||||
"ws": "^8.0.0"
|
||||
|
|
|
|||
|
|
@ -50,6 +50,10 @@ function createDataServiceQueueManager(): QueueManager {
|
|||
},
|
||||
providers: [
|
||||
// Import and initialize providers lazily
|
||||
async () => {
|
||||
const { initializeWebShareProvider } = await import('./providers/webshare.provider');
|
||||
return initializeWebShareProvider();
|
||||
},
|
||||
async () => {
|
||||
const { initializeIBProvider } = await import('./providers/ib.provider');
|
||||
return initializeIBProvider();
|
||||
|
|
@ -58,6 +62,10 @@ function createDataServiceQueueManager(): QueueManager {
|
|||
const { initializeProxyProvider } = await import('./providers/proxy.provider');
|
||||
return initializeProxyProvider();
|
||||
},
|
||||
async () => {
|
||||
const { initializeQMProvider } = await import('./providers/qm.provider');
|
||||
return initializeQMProvider();
|
||||
},
|
||||
async () => {
|
||||
const { initializeExchangeSyncProvider } = await import(
|
||||
'./providers/exchange-sync.provider'
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ export function initializeProxyProvider() {
|
|||
operation: 'fetch-from-sources',
|
||||
payload: {},
|
||||
cronPattern: '0 0 * * 0', // Every week at midnight on Sunday
|
||||
priority: 5,
|
||||
priority: 0,
|
||||
description: 'Fetch and validate proxy list from sources',
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,182 +1,68 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
import { providerRegistry, type ProviderConfigWithSchedule } from '@stock-bot/queue';
|
||||
|
||||
const logger = getLogger('qm-provider');
|
||||
|
||||
export const qmProvider: ProviderConfig = {
|
||||
name: 'qm',
|
||||
operations: {
|
||||
'live-data': async (payload: { symbol: string; fields?: string[] }) => {
|
||||
logger.info('Fetching live data from qm', { symbol: payload.symbol });
|
||||
// Initialize and register the IB provider
|
||||
export function initializeQMProvider() {
|
||||
logger.info('Registering IB provider with scheduled jobs...');
|
||||
|
||||
// Simulate qm API call
|
||||
const mockData = {
|
||||
symbol: payload.symbol,
|
||||
price: Math.random() * 1000 + 100,
|
||||
volume: Math.floor(Math.random() * 1000000),
|
||||
change: (Math.random() - 0.5) * 20,
|
||||
changePercent: (Math.random() - 0.5) * 5,
|
||||
timestamp: new Date().toISOString(),
|
||||
source: 'qm',
|
||||
fields: payload.fields || ['price', 'volume', 'change'],
|
||||
};
|
||||
const qmProviderConfig: ProviderConfigWithSchedule = {
|
||||
name: 'qm',
|
||||
operations: {
|
||||
'create-sessions': async () => {
|
||||
logger.info('Creating QM sessions...');
|
||||
const { createSessions } = await import('./qm.tasks');
|
||||
await createSessions();
|
||||
logger.info('QM sessions created successfully');
|
||||
return { success: true, message: 'QM sessions created successfully' };
|
||||
},
|
||||
'search-symbols': async () => {
|
||||
logger.info('Starting QM symbol search...');
|
||||
const { fetchSymbols } = await import('./qm.tasks');
|
||||
const symbols = await fetchSymbols();
|
||||
|
||||
// Simulate network delay
|
||||
await new Promise(resolve => setTimeout(resolve, 100 + Math.random() * 200));
|
||||
|
||||
return mockData;
|
||||
if (symbols && symbols.length > 0) {
|
||||
logger.info('QM symbol search completed successfully', { count: symbols.length });
|
||||
return {
|
||||
success: true,
|
||||
message: 'QM symbol search completed successfully',
|
||||
count: symbols.length,
|
||||
symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
|
||||
};
|
||||
} else {
|
||||
logger.warn('QM symbol search returned no results');
|
||||
return {
|
||||
success: false,
|
||||
message: 'No symbols found',
|
||||
count: 0,
|
||||
};
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
'historical-data': async (payload: {
|
||||
symbol: string;
|
||||
from: Date;
|
||||
to: Date;
|
||||
interval?: string;
|
||||
fields?: string[];
|
||||
}) => {
|
||||
logger.info('Fetching historical data from qm', {
|
||||
symbol: payload.symbol,
|
||||
from: payload.from,
|
||||
to: payload.to,
|
||||
interval: payload.interval || '1d',
|
||||
});
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'create-sessions',
|
||||
operation: 'create-sessions',
|
||||
payload: {},
|
||||
cronPattern: '*/15 * * * * *', // Every minute
|
||||
priority: 7,
|
||||
immediately: true,
|
||||
description: 'Create and maintain QM sessions',
|
||||
},
|
||||
{
|
||||
type: 'search-symbols',
|
||||
operation: 'search-symbols',
|
||||
payload: {},
|
||||
cronPattern: '*/1 * * * *', // Every minute
|
||||
priority: 10,
|
||||
immediately: false,
|
||||
description: 'Comprehensive symbol search using QM API',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Generate mock historical data
|
||||
const days = Math.ceil(
|
||||
(payload.to.getTime() - payload.from.getTime()) / (1000 * 60 * 60 * 24)
|
||||
);
|
||||
const data = [];
|
||||
|
||||
for (let i = 0; i < Math.min(days, 100); i++) {
|
||||
const date = new Date(payload.from.getTime() + i * 24 * 60 * 60 * 1000);
|
||||
data.push({
|
||||
date: date.toISOString().split('T')[0],
|
||||
open: Math.random() * 1000 + 100,
|
||||
high: Math.random() * 1000 + 100,
|
||||
low: Math.random() * 1000 + 100,
|
||||
close: Math.random() * 1000 + 100,
|
||||
volume: Math.floor(Math.random() * 1000000),
|
||||
source: 'qm',
|
||||
});
|
||||
}
|
||||
|
||||
// Simulate network delay
|
||||
await new Promise(resolve => setTimeout(resolve, 200 + Math.random() * 300));
|
||||
|
||||
return {
|
||||
symbol: payload.symbol,
|
||||
interval: payload.interval || '1d',
|
||||
data,
|
||||
source: 'qm',
|
||||
totalRecords: data.length,
|
||||
};
|
||||
},
|
||||
'batch-quotes': async (payload: { symbols: string[]; fields?: string[] }) => {
|
||||
logger.info('Fetching batch quotes from qm', {
|
||||
symbols: payload.symbols,
|
||||
count: payload.symbols.length,
|
||||
});
|
||||
|
||||
const quotes = payload.symbols.map(symbol => ({
|
||||
symbol,
|
||||
price: Math.random() * 1000 + 100,
|
||||
volume: Math.floor(Math.random() * 1000000),
|
||||
change: (Math.random() - 0.5) * 20,
|
||||
timestamp: new Date().toISOString(),
|
||||
source: 'qm',
|
||||
}));
|
||||
|
||||
// Simulate network delay
|
||||
await new Promise(resolve => setTimeout(resolve, 300 + Math.random() * 200));
|
||||
|
||||
return {
|
||||
quotes,
|
||||
source: 'qm',
|
||||
timestamp: new Date().toISOString(),
|
||||
totalSymbols: payload.symbols.length,
|
||||
};
|
||||
},
|
||||
'company-profile': async (payload: { symbol: string }) => {
|
||||
logger.info('Fetching company profile from qm', { symbol: payload.symbol });
|
||||
|
||||
// Simulate company profile data
|
||||
const profile = {
|
||||
symbol: payload.symbol,
|
||||
companyName: `${payload.symbol} Corporation`,
|
||||
sector: 'Technology',
|
||||
industry: 'Software',
|
||||
description: `${payload.symbol} is a leading technology company.`,
|
||||
marketCap: Math.floor(Math.random() * 1000000000000),
|
||||
employees: Math.floor(Math.random() * 100000),
|
||||
website: `https://www.${payload.symbol.toLowerCase()}.com`,
|
||||
source: 'qm',
|
||||
};
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 150 + Math.random() * 100));
|
||||
|
||||
return profile;
|
||||
},
|
||||
'options-chain': async (payload: { symbol: string; expiration?: string }) => {
|
||||
logger.info('Fetching options chain from qm', {
|
||||
symbol: payload.symbol,
|
||||
expiration: payload.expiration,
|
||||
});
|
||||
|
||||
// Generate mock options data
|
||||
const strikes = Array.from({ length: 20 }, (_, i) => 100 + i * 5);
|
||||
const calls = strikes.map(strike => ({
|
||||
strike,
|
||||
bid: Math.random() * 10,
|
||||
ask: Math.random() * 10 + 0.5,
|
||||
volume: Math.floor(Math.random() * 1000),
|
||||
openInterest: Math.floor(Math.random() * 5000),
|
||||
}));
|
||||
|
||||
const puts = strikes.map(strike => ({
|
||||
strike,
|
||||
bid: Math.random() * 10,
|
||||
ask: Math.random() * 10 + 0.5,
|
||||
volume: Math.floor(Math.random() * 1000),
|
||||
openInterest: Math.floor(Math.random() * 5000),
|
||||
}));
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 400 + Math.random() * 300));
|
||||
return {
|
||||
symbol: payload.symbol,
|
||||
expiration:
|
||||
payload.expiration ||
|
||||
new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString().split('T')[0],
|
||||
calls,
|
||||
puts,
|
||||
source: 'qm',
|
||||
};
|
||||
},
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
// {
|
||||
// type: 'qm-premium-refresh',
|
||||
// operation: 'batch-quotes',
|
||||
// payload: { symbols: ['AAPL', 'GOOGL', 'MSFT'] },
|
||||
// cronPattern: '*/2 * * * *', // Every 2 minutes
|
||||
// priority: 7,
|
||||
// description: 'Refresh premium quotes with detailed market data'
|
||||
// },
|
||||
// {
|
||||
// type: 'qm-options-update',
|
||||
// operation: 'options-chain',
|
||||
// payload: { symbol: 'SPY' },
|
||||
// cronPattern: '*/10 * * * *', // Every 10 minutes
|
||||
// priority: 5,
|
||||
// description: 'Update options chain data for SPY ETF'
|
||||
// },
|
||||
// {
|
||||
// type: 'qm-profiles',
|
||||
// operation: 'company-profile',
|
||||
// payload: { symbol: 'AAPL' },
|
||||
// cronPattern: '0 9 * * 1-5', // Weekdays at 9 AM
|
||||
// priority: 3,
|
||||
// description: 'Update company profile data'
|
||||
// }
|
||||
],
|
||||
};
|
||||
providerRegistry.registerWithSchedule(qmProviderConfig);
|
||||
logger.info('IB provider registered successfully with scheduled jobs');
|
||||
}
|
||||
|
|
|
|||
220
apps/data-service/src/providers/qm.tasks.ts
Normal file
220
apps/data-service/src/providers/qm.tasks.ts
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
import { getRandomUserAgent } from '@stock-bot/http';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { SymbolSearchUtil } from '../utils/symbol-search.util';
|
||||
import { getProxy } from './webshare.provider';
|
||||
|
||||
// Shared instances (module-scoped, not global)
|
||||
let isInitialized = false; // Track if resources are initialized
|
||||
let logger: ReturnType<typeof getLogger>;
|
||||
// let cache: CacheProvider;
|
||||
|
||||
export interface QMSession {
|
||||
proxy: string;
|
||||
headers: Record<string, string>;
|
||||
successfulCalls: number;
|
||||
failedCalls: number;
|
||||
lastUsed: Date;
|
||||
}
|
||||
function getQmHeaders(): Record<string, string> {
|
||||
return {
|
||||
'User-Agent': getRandomUserAgent(),
|
||||
Accept: '*/*',
|
||||
'Accept-Language': 'en',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
Origin: 'https://www.quotemedia.com',
|
||||
Referer: 'https://www.quotemedia.com/',
|
||||
};
|
||||
}
|
||||
|
||||
const sessionCache: Record<string, QMSession[]> = {
|
||||
// '5ad521e05faf5778d567f6d0012ec34d6cdbaeb2462f41568f66558bc7b4ced9': [], //4488d072b
|
||||
// cc1cbdaf040f76db8f4c94f7d156b9b9b716e1a7509ec9c74a48a47f6b6b9f87: [], //97ff00cf3 // getQuotes
|
||||
// '74963ff42f1db2320d051762b5d3950ff9eab23f9d5c5b592551b4ca0441d086': [], //32ca24e394b // getSplitsBySymbol getBrokerRatingsBySymbol getDividendsBySymbol getEarningsSurprisesBySymbol getEarningsEventsBySymbol
|
||||
// '1e1d7cb1de1fd2fe52684abdea41a446919a5fe12776dfab88615ac1ce1ec2f6': [], //fb5721812d2c // getEnhancedQuotes getProfiles
|
||||
// a900a06cc6b3e8036afb9eeb1bbf9783f0007698ed8f5cb1e373dc790e7be2e5: [], //cc882cd95f9 // getEnhancedQuotes
|
||||
// a863d519e38f80e45d10e280fb1afc729816e23f0218db2f3e8b23005a9ad8dd: [], //05a09a41225 // getCompanyFilings getEnhancedQuotes
|
||||
// b3cdb1873f3682c5aeeac097be6181529bfb755945e5a412a24f4b9316291427: [], //6a63f56a6 // getHeadlinesTickerStory
|
||||
dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6: [], //fceb3c4bdd // lookup
|
||||
// '97b24911d7b034620aafad9441afdb2bc906ee5c992d86933c5903254ca29709': [], //c56424868d // detailed-quotes
|
||||
// '8a394f09cb8540c8be8988780660a7ae5b583c331a1f6cb12834f051a0169a8f': [], //2a86d214e50e5 // getGlobalIndustrySectorPeers getKeyRatiosBySymbol getGlobalIndustrySectorCodeList
|
||||
// '2f059f75e2a839437095c9e7e4991d2365bafa7bbb086672a87ae0cf8d92eb01': [], // 48fa36d // getNethouseBySymbol
|
||||
// d7ae7e0091dd1d7011948c3dc4af09b5ec552285d92bb188be2618968bc78e3f: [], // 63548ee //getRecentTradesBySymbol getQuotes getLevel2Quote getRecentTradesBySymbol
|
||||
// d22d1db8f67fe6e420b4028e5129b289ca64862aa6cee8459193747b68c01de3: [], // 84e9e
|
||||
// '6e0b22a7cbc02ac3fa07d45e2880b7696aaebeb29574dce81789e570570c9002': [], //
|
||||
};
|
||||
|
||||
export async function initializeQMResources(): Promise<void> {
|
||||
// Skip if already initialized
|
||||
if (isInitialized) {
|
||||
return;
|
||||
}
|
||||
logger = getLogger('qm-tasks');
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
export async function createSessions(): Promise<void> {
|
||||
try {
|
||||
//for each session, check array length, if less than 5, create new session
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
logger.info('Creating QM sessions...');
|
||||
for (const [sessionId, sessionArray] of Object.entries(sessionCache)) {
|
||||
// remove any sessions with failedCalls > 10
|
||||
// const filteredArray = sessionArray.filter(session => session.failedCalls <= 10);
|
||||
// sessionCache[sessionId] = filteredArray;
|
||||
// if sessionArray is empty or has less than 5 sessions, create a new session
|
||||
while (sessionArray.length < 2) {
|
||||
logger.info(`Creating new session for ${sessionId}`);
|
||||
const proxy = getProxy();
|
||||
if (proxy === null) {
|
||||
logger.error('No proxy available for QM session creation');
|
||||
break; // Skip session creation if no proxy is available
|
||||
}
|
||||
const newSession: QMSession = {
|
||||
proxy: proxy, // Placeholder, should be set to a valid proxy
|
||||
headers: getQmHeaders(),
|
||||
successfulCalls: 0,
|
||||
failedCalls: 0,
|
||||
lastUsed: new Date(),
|
||||
};
|
||||
const sessionResponse = await fetch(
|
||||
`https://app.quotemedia.com/auth/g/authenticate/dataTool/v0/500/${sessionId}`,
|
||||
{
|
||||
method: 'GET',
|
||||
proxy: newSession.proxy,
|
||||
headers: newSession.headers,
|
||||
}
|
||||
);
|
||||
|
||||
logger.debug('Session response received', {
|
||||
status: sessionResponse.status,
|
||||
sessionId,
|
||||
});
|
||||
if (!sessionResponse.ok) {
|
||||
logger.error('Failed to create QM session', {
|
||||
sessionId,
|
||||
status: sessionResponse.status,
|
||||
statusText: sessionResponse.statusText,
|
||||
});
|
||||
continue; // Skip this session if creation failed
|
||||
}
|
||||
const sessionData = await sessionResponse.json();
|
||||
logger.info('QM session created successfully', {
|
||||
sessionId,
|
||||
sessionData,
|
||||
});
|
||||
newSession.headers['Datatool-Token'] = sessionData.token;
|
||||
console.log(newSession.headers);
|
||||
sessionArray.push(newSession);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch QM session', { error });
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// API call function to search symbols via QM
|
||||
async function searchQMSymbolsAPI(query: string): Promise<string[]> {
|
||||
const proxy = getProxy();
|
||||
|
||||
if (!proxy) {
|
||||
throw new Error('No proxy available for QM API call');
|
||||
}
|
||||
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6'; // Use the session ID for symbol lookup
|
||||
const session =
|
||||
sessionCache[sessionId][Math.floor(Math.random() * sessionCache[sessionId].length)]; // lookup session
|
||||
if (!session) {
|
||||
throw new Error(`No active session found for QM API with ID: ${sessionId}`);
|
||||
}
|
||||
try {
|
||||
// QM lookup endpoint for symbol search
|
||||
const apiUrl = `https://app.quotemedia.com/datatool/lookup.json?marketType=equity&pathName=%2Fdemo%2Fportal%2Fcompany-summary.php&q=${encodeURIComponent(query)}&qmodTool=SmartSymbolLookup&searchType=symbol&showFree=false&showHisa=false&webmasterId=500`;
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
headers: session.headers,
|
||||
proxy: session.proxy,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const symbols = await response.json();
|
||||
|
||||
logger.info(`QM API returned ${symbols.length} symbols for query: ${query}`);
|
||||
return symbols;
|
||||
} catch (error) {
|
||||
logger.error(`Error searching QM symbols for query "${query}":`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchSymbols(): Promise<unknown[] | null> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
const sessionId = 'dc8c9930437f65d30f6597768800957017bac203a0a50342932757c8dfa158d6'; // Use the session ID for symbol lookup
|
||||
|
||||
const currentSessions = sessionCache[sessionId] || [];
|
||||
if (currentSessions.length === 0) {
|
||||
logger.info('No sessions found, creating sessions first...');
|
||||
await createSessions();
|
||||
|
||||
// Wait a bit for sessions to be ready
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
|
||||
const newSessions = sessionCache[sessionId] || [];
|
||||
if (newSessions.length === 0) {
|
||||
throw new Error('Failed to create sessions before symbol search');
|
||||
}
|
||||
logger.info(`Created ${newSessions.length} sessions for symbol search`);
|
||||
}
|
||||
|
||||
logger.info('🔄 Starting QM symbols fetch...');
|
||||
|
||||
// Create search function that uses our QM API
|
||||
const searchFunction = async (query: string): Promise<string[]> => {
|
||||
return await searchQMSymbolsAPI(query);
|
||||
};
|
||||
|
||||
// Use the utility to perform comprehensive search
|
||||
const symbols = await SymbolSearchUtil.search(
|
||||
searchFunction,
|
||||
50, // threshold
|
||||
4, // max depth (A -> AA -> AAA -> AAAA)
|
||||
200 // delay between requests in ms
|
||||
);
|
||||
|
||||
logger.info(`QM symbols fetch completed. Found ${symbols.length} total symbols`);
|
||||
return symbols;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch QM symbols', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchExchanges(): Promise<unknown[] | null> {
|
||||
try {
|
||||
if (!isInitialized) {
|
||||
await initializeQMResources();
|
||||
}
|
||||
|
||||
logger.info('🔄 QM exchanges fetch - not implemented yet');
|
||||
// TODO: Implement QM exchanges fetching logic
|
||||
return null;
|
||||
} catch (error) {
|
||||
logger.error('❌ Failed to fetch QM exchanges', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export const qmTasks = {
|
||||
createSessions,
|
||||
fetchSymbols,
|
||||
fetchExchanges,
|
||||
};
|
||||
151
apps/data-service/src/providers/webshare.provider.ts
Normal file
151
apps/data-service/src/providers/webshare.provider.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* WebShare Provider for proxy management
|
||||
*/
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { ProviderConfigWithSchedule } from '@stock-bot/queue';
|
||||
import { providerRegistry } from '@stock-bot/queue';
|
||||
|
||||
const logger = getLogger('webshare-provider');
|
||||
|
||||
// In-memory proxy storage
|
||||
let proxies: string[] = [];
|
||||
let lastFetchTime: Date | null = null;
|
||||
let currentProxyIndex = 0;
|
||||
|
||||
export function getProxy(): string | null {
|
||||
if (proxies.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const proxy = proxies[currentProxyIndex];
|
||||
currentProxyIndex = (currentProxyIndex + 1) % proxies.length;
|
||||
return proxy;
|
||||
}
|
||||
|
||||
// Initialize and register the WebShare provider
|
||||
export function initializeWebShareProvider() {
|
||||
logger.info('Registering WebShare provider with scheduled jobs...');
|
||||
|
||||
const webShareProviderConfig: ProviderConfigWithSchedule = {
|
||||
name: 'webshare',
|
||||
|
||||
operations: {
|
||||
'fetch-proxies': async _payload => {
|
||||
logger.info('Fetching proxies from WebShare API');
|
||||
|
||||
try {
|
||||
const fetchedProxies = await fetchProxiesFromWebShare();
|
||||
|
||||
if (fetchedProxies && fetchedProxies.length > 0) {
|
||||
proxies = fetchedProxies;
|
||||
lastFetchTime = new Date();
|
||||
|
||||
logger.info('Successfully updated proxy list', {
|
||||
count: proxies.length,
|
||||
lastFetchTime: lastFetchTime.toISOString(),
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
count: proxies.length,
|
||||
lastFetchTime: lastFetchTime.toISOString(),
|
||||
};
|
||||
} else {
|
||||
logger.warn('No proxies fetched from WebShare API');
|
||||
return {
|
||||
success: false,
|
||||
count: 0,
|
||||
error: 'No proxies returned from API',
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch proxies from WebShare', { error });
|
||||
return {
|
||||
success: false,
|
||||
count: proxies.length,
|
||||
error: error instanceof Error ? error.message : 'Unknown error',
|
||||
};
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'fetch-proxies',
|
||||
operation: 'fetch-proxies',
|
||||
payload: {},
|
||||
description: 'Fetch proxies from WebShare API',
|
||||
cronPattern: '*/5 * * * *', // Every 5 minutes
|
||||
priority: 2,
|
||||
immediately: true, // Fetch immediately on startup
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
// Register the provider
|
||||
providerRegistry.registerWithSchedule(webShareProviderConfig);
|
||||
|
||||
logger.info('WebShare provider registered successfully');
|
||||
}
|
||||
|
||||
export const webShareProvider = {
|
||||
initialize: initializeWebShareProvider,
|
||||
getProxy,
|
||||
};
|
||||
|
||||
async function fetchProxiesFromWebShare(): Promise<string[] | null> {
|
||||
try {
|
||||
const apiKey = process.env.WEBSHARE_API_KEY;
|
||||
const apiUrl = process.env.WEBSHARE_API_URL;
|
||||
|
||||
if (!apiKey || !apiUrl) {
|
||||
logger.error('Missing WebShare configuration', {
|
||||
hasApiKey: !!apiKey,
|
||||
hasApiUrl: !!apiUrl,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.info('Fetching proxies from WebShare API');
|
||||
|
||||
const response = await fetch(`${apiUrl}proxy/list/?mode=direct&page=1&page_size=100`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Token ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error('WebShare API request failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.results || !Array.isArray(data.results)) {
|
||||
logger.error('Invalid response format from WebShare API', { data });
|
||||
return null;
|
||||
}
|
||||
|
||||
// Transform proxy data to the format: http://username:password@host:port
|
||||
const fetchedProxies = data.results.map(
|
||||
(proxy: { username: string; password: string; proxy_address: string; port: number }) => {
|
||||
return `http://${proxy.username}:${proxy.password}@${proxy.proxy_address}:${proxy.port}`;
|
||||
}
|
||||
);
|
||||
|
||||
logger.info('Successfully fetched proxies from WebShare', {
|
||||
count: fetchedProxies.length,
|
||||
total: data.count || fetchedProxies.length,
|
||||
});
|
||||
// console.log('Fetched Proxies:', fetchedProxies);
|
||||
return fetchedProxies;
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch proxies from WebShare', { error });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,254 +0,0 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
|
||||
const logger = getLogger('yahoo-provider');
|
||||
|
||||
export const yahooProvider: ProviderConfig = {
|
||||
name: 'yahoo-finance',
|
||||
operations: {
|
||||
'live-data': async (payload: { symbol: string; modules?: string[] }) => {
|
||||
logger.info('Fetching live data from Yahoo Finance', { symbol: payload.symbol });
|
||||
|
||||
// Simulate Yahoo Finance API call
|
||||
const mockData = {
|
||||
symbol: payload.symbol,
|
||||
regularMarketPrice: Math.random() * 1000 + 100,
|
||||
regularMarketVolume: Math.floor(Math.random() * 1000000),
|
||||
regularMarketChange: (Math.random() - 0.5) * 20,
|
||||
regularMarketChangePercent: (Math.random() - 0.5) * 5,
|
||||
preMarketPrice: Math.random() * 1000 + 100,
|
||||
postMarketPrice: Math.random() * 1000 + 100,
|
||||
marketCap: Math.floor(Math.random() * 1000000000000),
|
||||
peRatio: Math.random() * 50 + 5,
|
||||
dividendYield: Math.random() * 0.1,
|
||||
fiftyTwoWeekHigh: Math.random() * 1200 + 100,
|
||||
fiftyTwoWeekLow: Math.random() * 800 + 50,
|
||||
timestamp: Date.now() / 1000,
|
||||
source: 'yahoo-finance',
|
||||
modules: payload.modules || ['price', 'summaryDetail'],
|
||||
};
|
||||
|
||||
// Simulate network delay
|
||||
await new Promise(resolve => setTimeout(resolve, 150 + Math.random() * 250));
|
||||
|
||||
return mockData;
|
||||
},
|
||||
|
||||
'historical-data': async (payload: {
|
||||
symbol: string;
|
||||
period1: number;
|
||||
period2: number;
|
||||
interval?: string;
|
||||
events?: string;
|
||||
}) => {
|
||||
const { getLogger } = await import('@stock-bot/logger');
|
||||
const logger = getLogger('yahoo-provider');
|
||||
|
||||
logger.info('Fetching historical data from Yahoo Finance', {
|
||||
symbol: payload.symbol,
|
||||
period1: payload.period1,
|
||||
period2: payload.period2,
|
||||
interval: payload.interval || '1d',
|
||||
});
|
||||
|
||||
// Generate mock historical data
|
||||
const days = Math.ceil((payload.period2 - payload.period1) / (24 * 60 * 60));
|
||||
const data = [];
|
||||
|
||||
for (let i = 0; i < Math.min(days, 100); i++) {
|
||||
const timestamp = payload.period1 + i * 24 * 60 * 60;
|
||||
data.push({
|
||||
timestamp,
|
||||
date: new Date(timestamp * 1000).toISOString().split('T')[0],
|
||||
open: Math.random() * 1000 + 100,
|
||||
high: Math.random() * 1000 + 100,
|
||||
low: Math.random() * 1000 + 100,
|
||||
close: Math.random() * 1000 + 100,
|
||||
adjClose: Math.random() * 1000 + 100,
|
||||
volume: Math.floor(Math.random() * 1000000),
|
||||
source: 'yahoo-finance',
|
||||
});
|
||||
}
|
||||
|
||||
// Simulate network delay
|
||||
await new Promise(resolve => setTimeout(resolve, 250 + Math.random() * 350));
|
||||
|
||||
return {
|
||||
symbol: payload.symbol,
|
||||
interval: payload.interval || '1d',
|
||||
timestamps: data.map(d => d.timestamp),
|
||||
indicators: {
|
||||
quote: [
|
||||
{
|
||||
open: data.map(d => d.open),
|
||||
high: data.map(d => d.high),
|
||||
low: data.map(d => d.low),
|
||||
close: data.map(d => d.close),
|
||||
volume: data.map(d => d.volume),
|
||||
},
|
||||
],
|
||||
adjclose: [
|
||||
{
|
||||
adjclose: data.map(d => d.adjClose),
|
||||
},
|
||||
],
|
||||
},
|
||||
source: 'yahoo-finance',
|
||||
totalRecords: data.length,
|
||||
};
|
||||
},
|
||||
search: async (payload: { query: string; quotesCount?: number; newsCount?: number }) => {
|
||||
const { getLogger } = await import('@stock-bot/logger');
|
||||
const logger = getLogger('yahoo-provider');
|
||||
|
||||
logger.info('Searching Yahoo Finance', { query: payload.query });
|
||||
|
||||
// Generate mock search results
|
||||
const quotes = Array.from({ length: payload.quotesCount || 5 }, (_, i) => ({
|
||||
symbol: `${payload.query.toUpperCase()}${i}`,
|
||||
shortname: `${payload.query} Company ${i}`,
|
||||
longname: `${payload.query} Corporation ${i}`,
|
||||
exchDisp: 'NASDAQ',
|
||||
typeDisp: 'Equity',
|
||||
source: 'yahoo-finance',
|
||||
}));
|
||||
|
||||
const news = Array.from({ length: payload.newsCount || 3 }, (_, i) => ({
|
||||
uuid: `news-${i}-${Date.now()}`,
|
||||
title: `${payload.query} News Article ${i}`,
|
||||
publisher: 'Financial News',
|
||||
providerPublishTime: Date.now() - i * 3600000,
|
||||
type: 'STORY',
|
||||
source: 'yahoo-finance',
|
||||
}));
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 200 + Math.random() * 200));
|
||||
|
||||
return {
|
||||
quotes,
|
||||
news,
|
||||
totalQuotes: quotes.length,
|
||||
totalNews: news.length,
|
||||
source: 'yahoo-finance',
|
||||
};
|
||||
},
|
||||
financials: async (payload: { symbol: string; type?: 'income' | 'balance' | 'cash' }) => {
|
||||
const { getLogger } = await import('@stock-bot/logger');
|
||||
const logger = getLogger('yahoo-provider');
|
||||
|
||||
logger.info('Fetching financials from Yahoo Finance', {
|
||||
symbol: payload.symbol,
|
||||
type: payload.type || 'income',
|
||||
});
|
||||
|
||||
// Generate mock financial data
|
||||
const financials = {
|
||||
symbol: payload.symbol,
|
||||
type: payload.type || 'income',
|
||||
currency: 'USD',
|
||||
annual: Array.from({ length: 4 }, (_, i) => ({
|
||||
fiscalYear: 2024 - i,
|
||||
revenue: Math.floor(Math.random() * 100000000000),
|
||||
netIncome: Math.floor(Math.random() * 10000000000),
|
||||
totalAssets: Math.floor(Math.random() * 500000000000),
|
||||
totalDebt: Math.floor(Math.random() * 50000000000),
|
||||
})),
|
||||
quarterly: Array.from({ length: 4 }, (_, i) => ({
|
||||
fiscalQuarter: `Q${4 - i} 2024`,
|
||||
revenue: Math.floor(Math.random() * 25000000000),
|
||||
netIncome: Math.floor(Math.random() * 2500000000),
|
||||
})),
|
||||
source: 'yahoo-finance',
|
||||
};
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 300 + Math.random() * 200));
|
||||
|
||||
return financials;
|
||||
},
|
||||
earnings: async (payload: { symbol: string; period?: 'annual' | 'quarterly' }) => {
|
||||
const { getLogger } = await import('@stock-bot/logger');
|
||||
const logger = getLogger('yahoo-provider');
|
||||
|
||||
logger.info('Fetching earnings from Yahoo Finance', {
|
||||
symbol: payload.symbol,
|
||||
period: payload.period || 'quarterly',
|
||||
});
|
||||
|
||||
// Generate mock earnings data
|
||||
const earnings = {
|
||||
symbol: payload.symbol,
|
||||
period: payload.period || 'quarterly',
|
||||
earnings: Array.from({ length: 8 }, (_, i) => ({
|
||||
quarter: `Q${(i % 4) + 1} ${2024 - Math.floor(i / 4)}`,
|
||||
epsEstimate: Math.random() * 5,
|
||||
epsActual: Math.random() * 5,
|
||||
revenueEstimate: Math.floor(Math.random() * 50000000000),
|
||||
revenueActual: Math.floor(Math.random() * 50000000000),
|
||||
surprise: (Math.random() - 0.5) * 2,
|
||||
})),
|
||||
source: 'yahoo-finance',
|
||||
};
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 250 + Math.random() * 150));
|
||||
|
||||
return earnings;
|
||||
},
|
||||
recommendations: async (payload: { symbol: string }) => {
|
||||
const { getLogger } = await import('@stock-bot/logger');
|
||||
const logger = getLogger('yahoo-provider');
|
||||
|
||||
logger.info('Fetching recommendations from Yahoo Finance', { symbol: payload.symbol });
|
||||
|
||||
// Generate mock recommendations
|
||||
const recommendations = {
|
||||
symbol: payload.symbol,
|
||||
current: {
|
||||
strongBuy: Math.floor(Math.random() * 10),
|
||||
buy: Math.floor(Math.random() * 15),
|
||||
hold: Math.floor(Math.random() * 20),
|
||||
sell: Math.floor(Math.random() * 5),
|
||||
strongSell: Math.floor(Math.random() * 3),
|
||||
},
|
||||
trend: Array.from({ length: 4 }, (_, i) => ({
|
||||
period: `${i}m`,
|
||||
strongBuy: Math.floor(Math.random() * 10),
|
||||
buy: Math.floor(Math.random() * 15),
|
||||
hold: Math.floor(Math.random() * 20),
|
||||
sell: Math.floor(Math.random() * 5),
|
||||
strongSell: Math.floor(Math.random() * 3),
|
||||
})),
|
||||
source: 'yahoo-finance',
|
||||
};
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 180 + Math.random() * 120));
|
||||
return recommendations;
|
||||
},
|
||||
},
|
||||
|
||||
scheduledJobs: [
|
||||
// {
|
||||
// type: 'yahoo-market-refresh',
|
||||
// operation: 'live-data',
|
||||
// payload: { symbol: 'AAPL' },
|
||||
// cronPattern: '*/1 * * * *', // Every minute
|
||||
// priority: 8,
|
||||
// description: 'Refresh Apple stock price from Yahoo Finance'
|
||||
// },
|
||||
// {
|
||||
// type: 'yahoo-sp500-update',
|
||||
// operation: 'live-data',
|
||||
// payload: { symbol: 'SPY' },
|
||||
// cronPattern: '*/2 * * * *', // Every 2 minutes
|
||||
// priority: 9,
|
||||
// description: 'Update S&P 500 ETF price'
|
||||
// },
|
||||
// {
|
||||
// type: 'yahoo-earnings-check',
|
||||
// operation: 'earnings',
|
||||
// payload: { symbol: 'AAPL' },
|
||||
// cronPattern: '0 16 * * 1-5', // Weekdays at 4 PM (market close)
|
||||
// priority: 6,
|
||||
// description: 'Check earnings data for Apple'
|
||||
// }
|
||||
],
|
||||
};
|
||||
109
apps/data-service/src/utils/symbol-search.util.ts
Normal file
109
apps/data-service/src/utils/symbol-search.util.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { getLogger } from '@stock-bot/logger';
|
||||
import { sleep } from '@stock-bot/utils';
|
||||
|
||||
const logger = getLogger('symbol-search-util');
|
||||
|
||||
export interface SearchFunction {
|
||||
(query: string): Promise<string[]>;
|
||||
}
|
||||
|
||||
export class SymbolSearchUtil {
|
||||
private threshold: number;
|
||||
private searchFunction: SearchFunction;
|
||||
private maxDepth: number;
|
||||
private delay: number;
|
||||
|
||||
constructor(
|
||||
searchFunction: SearchFunction,
|
||||
threshold: number = 50,
|
||||
maxDepth: number = 4,
|
||||
delay: number = 100
|
||||
) {
|
||||
this.searchFunction = searchFunction;
|
||||
this.threshold = threshold;
|
||||
this.maxDepth = maxDepth;
|
||||
this.delay = delay;
|
||||
}
|
||||
|
||||
async searchAllSymbols(): Promise<string[]> {
|
||||
logger.info('Starting comprehensive symbol search...');
|
||||
const allSymbols: string[] = [];
|
||||
|
||||
// Start with single letters A-Z
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const singleLetter = String.fromCharCode(65 + i);
|
||||
|
||||
try {
|
||||
const symbols = await this.searchRecursive(singleLetter, 1);
|
||||
allSymbols.push(...symbols);
|
||||
|
||||
// Add delay between top-level searches
|
||||
if (this.delay > 0) {
|
||||
await sleep(this.delay);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed to search for "${singleLetter}":`, error);
|
||||
// Continue with next letter
|
||||
}
|
||||
}
|
||||
|
||||
// Remove duplicates
|
||||
const uniqueSymbols = [...new Set(allSymbols)];
|
||||
logger.info(`Symbol search completed. Found ${uniqueSymbols.length} unique symbols`);
|
||||
return uniqueSymbols;
|
||||
}
|
||||
|
||||
private async searchRecursive(prefix: string, depth: number): Promise<string[]> {
|
||||
try {
|
||||
const symbols = await this.searchFunction(prefix);
|
||||
|
||||
logger.debug(`Query "${prefix}" returned ${symbols.length} symbols`);
|
||||
|
||||
// If we're at max depth or results are under threshold, return the symbols
|
||||
if (depth >= this.maxDepth || symbols.length < this.threshold) {
|
||||
logger.info(`Added ${symbols.length} symbols from query: ${prefix}`);
|
||||
return symbols;
|
||||
}
|
||||
|
||||
// If we have too many results, go deeper
|
||||
logger.info(
|
||||
`Query "${prefix}" returned ${symbols.length} results (>= ${this.threshold}), going deeper...`
|
||||
);
|
||||
const allSymbols: string[] = [];
|
||||
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const nextQuery = prefix + String.fromCharCode(65 + i);
|
||||
|
||||
try {
|
||||
const deeperSymbols = await this.searchRecursive(nextQuery, depth + 1);
|
||||
allSymbols.push(...deeperSymbols);
|
||||
|
||||
// Add delay between recursive calls
|
||||
if (this.delay > 0 && depth < 3) {
|
||||
// Only delay for first few levels
|
||||
await sleep(this.delay);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed recursive search for "${nextQuery}":`, error);
|
||||
// Continue with next combination
|
||||
}
|
||||
}
|
||||
|
||||
return allSymbols;
|
||||
} catch (error) {
|
||||
logger.error(`Error in recursive search for "${prefix}":`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Static method for one-off searches
|
||||
static async search(
|
||||
searchFunction: SearchFunction,
|
||||
threshold: number = 50,
|
||||
maxDepth: number = 4,
|
||||
delay: number = 100
|
||||
): Promise<string[]> {
|
||||
const util = new SymbolSearchUtil(searchFunction, threshold, maxDepth, delay);
|
||||
return util.searchAllSymbols();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue