work on qm filings
This commit is contained in:
parent
710577eb3d
commit
960daf4cad
17 changed files with 2319 additions and 32 deletions
448
apps/stock/data-ingestion/src/handlers/qm/actions/news.action.ts
Normal file
448
apps/stock/data-ingestion/src/handlers/qm/actions/news.action.ts
Normal file
|
|
@ -0,0 +1,448 @@
|
|||
/**
|
||||
* QM News Actions - Fetch symbol-specific and general market news
|
||||
*/
|
||||
|
||||
import type { ExecutionContext } from '@stock-bot/handlers';
|
||||
import type { QMHandler } from '../qm.handler';
|
||||
import { QM_CONFIG, QM_SESSION_IDS } from '../shared/config';
|
||||
import { QMSessionManager } from '../shared/session-manager';
|
||||
|
||||
interface NewsArticle {
|
||||
id: string;
|
||||
publishedDate: Date;
|
||||
title: string;
|
||||
summary: string;
|
||||
source: string;
|
||||
url: string;
|
||||
symbols?: string[];
|
||||
categories?: string[];
|
||||
sentiment?: {
|
||||
score: number;
|
||||
label: string; // positive, negative, neutral
|
||||
};
|
||||
imageUrl?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update news for a single symbol
|
||||
*/
|
||||
export async function updateSymbolNews(
|
||||
this: QMHandler,
|
||||
input: {
|
||||
symbol: string;
|
||||
symbolId: number;
|
||||
qmSearchCode: string;
|
||||
lookbackDays?: number;
|
||||
},
|
||||
_context?: ExecutionContext
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
symbol: string;
|
||||
message: string;
|
||||
data?: any;
|
||||
}> {
|
||||
const { symbol, symbolId, qmSearchCode, lookbackDays = 30 } = input;
|
||||
|
||||
this.logger.info('Fetching symbol news', { symbol, symbolId, lookbackDays });
|
||||
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
await sessionManager.initialize(this.cache, this.logger);
|
||||
|
||||
const sessionId = QM_SESSION_IDS.LOOKUP;
|
||||
const session = await sessionManager.getSession(sessionId);
|
||||
|
||||
if (!session || !session.uuid) {
|
||||
throw new Error(`No active session found for QM news`);
|
||||
}
|
||||
|
||||
try {
|
||||
// Calculate date range
|
||||
const endDate = new Date();
|
||||
const startDate = new Date();
|
||||
startDate.setDate(startDate.getDate() - lookbackDays);
|
||||
|
||||
// Build API request for symbol news
|
||||
const searchParams = new URLSearchParams({
|
||||
symbol: symbol,
|
||||
symbolId: symbolId.toString(),
|
||||
qmodTool: 'News',
|
||||
webmasterId: '500',
|
||||
startDate: startDate.toISOString().split('T')[0],
|
||||
endDate: endDate.toISOString().split('T')[0],
|
||||
includeContent: 'true',
|
||||
pageSize: '50'
|
||||
} as Record<string, string>);
|
||||
|
||||
const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/news.json?${searchParams.toString()}`;
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
headers: session.headers,
|
||||
proxy: session.proxy,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const newsData = await response.json();
|
||||
|
||||
// Update session success stats
|
||||
await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
|
||||
|
||||
// Process and store news data
|
||||
if (newsData && newsData.articles && newsData.articles.length > 0) {
|
||||
const processedArticles = newsData.articles.map((article: any) => ({
|
||||
articleId: article.id || `${symbol}_${article.publishedDate}_${article.title.substring(0, 20)}`,
|
||||
symbol,
|
||||
symbolId,
|
||||
publishedDate: new Date(article.publishedDate),
|
||||
title: article.title,
|
||||
summary: article.summary || article.content?.substring(0, 500),
|
||||
source: article.source || 'Unknown',
|
||||
url: article.url,
|
||||
symbols: article.symbols || [symbol],
|
||||
categories: article.categories || [],
|
||||
sentiment: article.sentiment ? {
|
||||
score: parseFloat(article.sentiment.score) || 0,
|
||||
label: article.sentiment.label || 'neutral'
|
||||
} : null,
|
||||
imageUrl: article.imageUrl,
|
||||
isSymbolSpecific: true,
|
||||
updated_at: new Date()
|
||||
}));
|
||||
|
||||
// Store in MongoDB
|
||||
await this.mongodb.batchUpsert(
|
||||
'qmNews',
|
||||
processedArticles,
|
||||
['articleId'] // Unique key
|
||||
);
|
||||
|
||||
// Calculate sentiment summary
|
||||
const sentimentCounts = processedArticles.reduce((acc: any, article: any) => {
|
||||
if (article.sentiment) {
|
||||
acc[article.sentiment.label] = (acc[article.sentiment.label] || 0) + 1;
|
||||
}
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Update operation tracking
|
||||
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'news_update', {
|
||||
status: 'success',
|
||||
lastRecordDate: endDate,
|
||||
recordCount: processedArticles.length,
|
||||
metadata: {
|
||||
sentimentCounts,
|
||||
uniqueSources: new Set(processedArticles.map((a: any) => a.source)).size,
|
||||
avgSentimentScore: processedArticles
|
||||
.filter((a: any) => a.sentiment?.score)
|
||||
.reduce((sum: number, a: any, i: number, arr: any[]) =>
|
||||
i === arr.length - 1 ? (sum + a.sentiment.score) / arr.length : sum + a.sentiment.score, 0
|
||||
)
|
||||
}
|
||||
});
|
||||
|
||||
this.logger.info('Symbol news updated successfully', {
|
||||
symbol,
|
||||
articleCount: processedArticles.length,
|
||||
sentimentCounts
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
symbol,
|
||||
message: `Updated ${processedArticles.length} news articles`,
|
||||
data: {
|
||||
count: processedArticles.length,
|
||||
sentimentCounts,
|
||||
sources: new Set(processedArticles.map((a: any) => a.source)).size
|
||||
}
|
||||
};
|
||||
} else {
|
||||
// No news found
|
||||
this.logger.info('No news articles found', { symbol });
|
||||
|
||||
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'news_update', {
|
||||
status: 'success',
|
||||
lastRecordDate: endDate,
|
||||
recordCount: 0
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
symbol,
|
||||
message: 'No news articles found',
|
||||
data: { count: 0 }
|
||||
};
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// Update session failure stats
|
||||
if (session.uuid) {
|
||||
await sessionManager.incrementFailedCalls(sessionId, session.uuid);
|
||||
}
|
||||
|
||||
this.logger.error('Error fetching symbol news', {
|
||||
symbol,
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
|
||||
// Update operation tracking for failure
|
||||
await this.operationRegistry.updateOperation('qm', qmSearchCode, 'news_update', {
|
||||
status: 'failure',
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
symbol,
|
||||
message: `Failed to fetch news: ${error instanceof Error ? error.message : 'Unknown error'}`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update general market news
|
||||
*/
|
||||
export async function updateGeneralNews(
|
||||
this: QMHandler,
|
||||
input: {
|
||||
categories?: string[];
|
||||
lookbackMinutes?: number;
|
||||
} = {},
|
||||
_context?: ExecutionContext
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
message: string;
|
||||
data?: any;
|
||||
}> {
|
||||
const { categories = ['market', 'economy', 'politics'], lookbackMinutes = 60 } = input;
|
||||
|
||||
this.logger.info('Fetching general news', { categories, lookbackMinutes });
|
||||
|
||||
const sessionManager = QMSessionManager.getInstance();
|
||||
await sessionManager.initialize(this.cache, this.logger);
|
||||
|
||||
const sessionId = QM_SESSION_IDS.LOOKUP;
|
||||
const session = await sessionManager.getSession(sessionId);
|
||||
|
||||
if (!session || !session.uuid) {
|
||||
throw new Error(`No active session found for QM general news`);
|
||||
}
|
||||
|
||||
try {
|
||||
// Calculate time range
|
||||
const endDate = new Date();
|
||||
const startDate = new Date();
|
||||
startDate.setMinutes(startDate.getMinutes() - lookbackMinutes);
|
||||
|
||||
// Build API request for general news
|
||||
const searchParams = new URLSearchParams({
|
||||
qmodTool: 'MarketNews',
|
||||
webmasterId: '500',
|
||||
categories: categories.join(','),
|
||||
startDateTime: startDate.toISOString(),
|
||||
endDateTime: endDate.toISOString(),
|
||||
includeContent: 'true',
|
||||
pageSize: '100'
|
||||
} as Record<string, string>);
|
||||
|
||||
const apiUrl = `${QM_CONFIG.BASE_URL}/datatool/marketnews.json?${searchParams.toString()}`;
|
||||
|
||||
const response = await fetch(apiUrl, {
|
||||
method: 'GET',
|
||||
headers: session.headers,
|
||||
proxy: session.proxy,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const newsData = await response.json();
|
||||
|
||||
// Update session success stats
|
||||
await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
|
||||
|
||||
// Process and store general news
|
||||
if (newsData && newsData.articles && newsData.articles.length > 0) {
|
||||
const processedArticles = newsData.articles.map((article: any) => ({
|
||||
articleId: article.id || `general_${article.publishedDate}_${article.title.substring(0, 20)}`,
|
||||
publishedDate: new Date(article.publishedDate),
|
||||
title: article.title,
|
||||
summary: article.summary || article.content?.substring(0, 500),
|
||||
source: article.source || 'Unknown',
|
||||
url: article.url,
|
||||
symbols: article.symbols || [],
|
||||
categories: article.categories || categories,
|
||||
sentiment: article.sentiment ? {
|
||||
score: parseFloat(article.sentiment.score) || 0,
|
||||
label: article.sentiment.label || 'neutral'
|
||||
} : null,
|
||||
imageUrl: article.imageUrl,
|
||||
isSymbolSpecific: false,
|
||||
isMarketMoving: article.isMarketMoving || false,
|
||||
importance: article.importance || 'medium',
|
||||
updated_at: new Date()
|
||||
}));
|
||||
|
||||
// Store in MongoDB
|
||||
await this.mongodb.batchUpsert(
|
||||
'qmNews',
|
||||
processedArticles,
|
||||
['articleId'] // Unique key
|
||||
);
|
||||
|
||||
// Find high-importance articles
|
||||
const highImportanceCount = processedArticles.filter((a: any) =>
|
||||
a.importance === 'high' || a.isMarketMoving
|
||||
).length;
|
||||
|
||||
// Update a general tracking document
|
||||
await this.mongodb.updateOne(
|
||||
'qmOperationStats',
|
||||
{ operation: 'general_news_update' },
|
||||
{
|
||||
$set: {
|
||||
lastRunAt: new Date(),
|
||||
lastRecordCount: processedArticles.length,
|
||||
highImportanceCount,
|
||||
categories,
|
||||
updated_at: new Date()
|
||||
}
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
|
||||
this.logger.info('General news updated successfully', {
|
||||
articleCount: processedArticles.length,
|
||||
highImportanceCount,
|
||||
categories
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Updated ${processedArticles.length} general news articles`,
|
||||
data: {
|
||||
count: processedArticles.length,
|
||||
highImportanceCount,
|
||||
categories,
|
||||
sources: new Set(processedArticles.map((a: any) => a.source)).size
|
||||
}
|
||||
};
|
||||
} else {
|
||||
// No news found
|
||||
this.logger.info('No general news articles found');
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'No general news articles found',
|
||||
data: { count: 0 }
|
||||
};
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// Update session failure stats
|
||||
if (session.uuid) {
|
||||
await sessionManager.incrementFailedCalls(sessionId, session.uuid);
|
||||
}
|
||||
|
||||
this.logger.error('Error fetching general news', {
|
||||
error: error instanceof Error ? error.message : 'Unknown error'
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
message: `Failed to fetch general news: ${error instanceof Error ? error.message : 'Unknown error'}`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule symbol news updates
|
||||
*/
|
||||
export async function scheduleSymbolNewsUpdates(
|
||||
this: QMHandler,
|
||||
input: {
|
||||
limit?: number;
|
||||
minHoursSinceRun?: number;
|
||||
forceUpdate?: boolean;
|
||||
} = {},
|
||||
_context?: ExecutionContext
|
||||
): Promise<{
|
||||
message: string;
|
||||
symbolsQueued: number;
|
||||
errors: number;
|
||||
}> {
|
||||
const { limit = 200, minHoursSinceRun = 24 * 7, forceUpdate = false } = input;
|
||||
|
||||
this.logger.info('Scheduling symbol news updates', { limit, minHoursSinceRun, forceUpdate });
|
||||
|
||||
try {
|
||||
// Get symbols that need news updates
|
||||
const staleSymbols = await this.operationRegistry.getStaleSymbols('qm', 'news_update', {
|
||||
minHoursSinceRun: forceUpdate ? 0 : minHoursSinceRun,
|
||||
limit
|
||||
});
|
||||
|
||||
if (staleSymbols.length === 0) {
|
||||
this.logger.info('No symbols need news updates');
|
||||
return {
|
||||
message: 'No symbols need news updates',
|
||||
symbolsQueued: 0,
|
||||
errors: 0
|
||||
};
|
||||
}
|
||||
|
||||
// Get full symbol data
|
||||
const symbolsToProcess = await this.mongodb.find('qmSymbols', {
|
||||
qmSearchCode: { $in: staleSymbols }
|
||||
}, {
|
||||
projection: { symbol: 1, symbolId: 1, qmSearchCode: 1 }
|
||||
});
|
||||
|
||||
this.logger.info(`Found ${symbolsToProcess.length} symbols for news updates`);
|
||||
|
||||
let symbolsQueued = 0;
|
||||
let errors = 0;
|
||||
|
||||
// Schedule update jobs
|
||||
for (const doc of symbolsToProcess) {
|
||||
try {
|
||||
if (!doc.symbolId) {
|
||||
this.logger.warn(`Symbol ${doc.symbol} missing symbolId, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.scheduleOperation('update-symbol-news', {
|
||||
symbol: doc.symbol,
|
||||
symbolId: doc.symbolId,
|
||||
qmSearchCode: doc.qmSearchCode
|
||||
}, {
|
||||
priority: 4, // Lower priority than price data
|
||||
delay: symbolsQueued * 500 // 0.5 seconds between jobs
|
||||
});
|
||||
|
||||
symbolsQueued++;
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to schedule news update for ${doc.symbol}`, { error });
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Symbol news update scheduling completed', {
|
||||
symbolsQueued,
|
||||
errors
|
||||
});
|
||||
|
||||
return {
|
||||
message: `Scheduled news updates for ${symbolsQueued} symbols`,
|
||||
symbolsQueued,
|
||||
errors
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error('Symbol news scheduling failed', { error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue