work on qm filings
This commit is contained in:
parent
710577eb3d
commit
960daf4cad
17 changed files with 2319 additions and 32 deletions
77
apps/stock/data-ingestion/test/intraday-crawl.test.ts
Normal file
77
apps/stock/data-ingestion/test/intraday-crawl.test.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Test script for intraday crawl functionality
|
||||
*/
|
||||
|
||||
import { createTestContext } from '../src/test-utils';
|
||||
import { QMHandler } from '../src/handlers/qm/qm.handler';
|
||||
|
||||
async function testIntradayCrawl() {
|
||||
console.log('Testing intraday crawl functionality...\n');
|
||||
|
||||
const context = await createTestContext();
|
||||
const handler = new QMHandler(context.services);
|
||||
|
||||
// Wait for operation registry to initialize
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
try {
|
||||
// Test 1: Schedule crawls for never-run symbols
|
||||
console.log('Test 1: Scheduling crawls for never-run symbols...');
|
||||
const result1 = await handler.scheduleIntradayCrawls({
|
||||
limit: 5,
|
||||
priorityMode: 'never_run',
|
||||
targetOldestDate: '2023-01-01' // Just 1 year for testing
|
||||
});
|
||||
console.log('Result:', result1);
|
||||
console.log('');
|
||||
|
||||
// Test 2: Check crawl state for a specific symbol
|
||||
console.log('Test 2: Checking crawl state for symbol X...');
|
||||
const tracker = handler.operationRegistry.getTracker('qm');
|
||||
const isComplete = await tracker.isIntradayCrawlComplete('X', 'intraday_bars', new Date('2023-01-01'));
|
||||
console.log('Is crawl complete for X?', isComplete);
|
||||
|
||||
// Get detailed state
|
||||
const symbols = await tracker.getSymbolsForIntradayCrawl('intraday_bars', {
|
||||
limit: 1,
|
||||
targetOldestDate: new Date('2023-01-01')
|
||||
});
|
||||
|
||||
const symbolX = symbols.find(s => s.symbol === 'X');
|
||||
if (symbolX) {
|
||||
console.log('Symbol X state:', JSON.stringify(symbolX, null, 2));
|
||||
}
|
||||
console.log('');
|
||||
|
||||
// Test 3: Manually crawl a single symbol
|
||||
console.log('Test 3: Manually crawling intraday data for X...');
|
||||
|
||||
// First get symbol data
|
||||
const symbolData = await context.services.mongodb.findOne('qmSymbols', {
|
||||
symbol: 'X'
|
||||
});
|
||||
|
||||
if (symbolData && symbolData.symbolId) {
|
||||
const crawlResult = await handler.crawlIntradayData({
|
||||
symbol: 'X',
|
||||
symbolId: symbolData.symbolId,
|
||||
qmSearchCode: symbolData.qmSearchCode,
|
||||
targetOldestDate: '2024-01-01', // Just current year for quick test
|
||||
batchSize: 7
|
||||
});
|
||||
console.log('Crawl result:', crawlResult);
|
||||
} else {
|
||||
console.log('Symbol X not found or missing symbolId');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Test failed:', error);
|
||||
} finally {
|
||||
await context.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
// Run the test
|
||||
testIntradayCrawl().catch(console.error);
|
||||
139
apps/stock/data-ingestion/test/test-modified-functions.ts
Normal file
139
apps/stock/data-ingestion/test/test-modified-functions.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
/**
|
||||
* Examples of modifying existing functions to only work with symbol 'X'
|
||||
*/
|
||||
|
||||
import { OperationTracker } from '../src/shared/operation-manager';
|
||||
|
||||
// Example 1: Modified getStaleSymbols to only return symbol X
|
||||
async function getStaleSymbolsOnlyX(
|
||||
operationTracker: OperationTracker,
|
||||
providerName: string,
|
||||
operationName: string,
|
||||
options: any = {}
|
||||
) {
|
||||
// Method 1: Add symbolFilter to options
|
||||
const modifiedOptions = {
|
||||
...options,
|
||||
symbolFilter: { symbol: 'X' }
|
||||
};
|
||||
|
||||
return operationTracker.getStaleSymbols(providerName, operationName, modifiedOptions);
|
||||
}
|
||||
|
||||
// Example 2: Modified sophisticated backtest function for symbol X only
|
||||
async function runSophisticatedBacktestOnlyX(orchestrator: any) {
|
||||
const symbols = ['X']; // Only test symbol X
|
||||
|
||||
const backtestConfig = {
|
||||
symbols,
|
||||
startDate: new Date('2023-01-01'),
|
||||
endDate: new Date('2024-01-01'),
|
||||
strategies: ['momentum', 'mean_reversion'],
|
||||
// ... rest of config
|
||||
};
|
||||
|
||||
return orchestrator.runBacktest(backtestConfig);
|
||||
}
|
||||
|
||||
// Example 3: Modified schedule function to only process symbol X
|
||||
async function scheduleOperationsOnlyX(handler: any) {
|
||||
// Get all symbols that need updates, then filter for X
|
||||
const staleSymbols = await handler.operationRegistry.getStaleSymbols('qm', 'price_update', {
|
||||
minHoursSinceRun: 24,
|
||||
limit: 1000
|
||||
});
|
||||
|
||||
// Filter to only include symbol X
|
||||
const symbolXOnly = staleSymbols.filter((s: string) => s.includes('X:'));
|
||||
|
||||
if (symbolXOnly.length > 0) {
|
||||
const symbolData = await handler.mongodb.find('qmSymbols', {
|
||||
qmSearchCode: symbolXOnly[0]
|
||||
});
|
||||
|
||||
if (symbolData.length > 0) {
|
||||
await handler.scheduleOperation('update-prices', {
|
||||
symbol: symbolData[0].symbol,
|
||||
symbolId: symbolData[0].symbolId,
|
||||
qmSearchCode: symbolData[0].qmSearchCode
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Example 4: Modified intraday crawl for symbol X only
|
||||
async function crawlIntradayOnlyX(handler: any) {
|
||||
// Direct approach - just process symbol X
|
||||
const symbolX = await handler.mongodb.findOne('qmSymbols', { symbol: 'X' });
|
||||
|
||||
if (symbolX) {
|
||||
return handler.crawlIntradayData({
|
||||
symbol: symbolX.symbol,
|
||||
symbolId: symbolX.symbolId,
|
||||
qmSearchCode: symbolX.qmSearchCode,
|
||||
mode: 'full',
|
||||
targetOldestDate: new Date('2020-01-01')
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Example 5: Test wrapper that ensures only symbol X is processed
|
||||
function createSymbolXTestWrapper(originalFunction: Function) {
|
||||
return async function(...args: any[]) {
|
||||
// Check if first argument has symbol property
|
||||
if (args[0] && typeof args[0] === 'object') {
|
||||
// If it's a symbol-specific call, only proceed if symbol is X
|
||||
if (args[0].symbol && args[0].symbol !== 'X') {
|
||||
console.log(`Skipping symbol ${args[0].symbol} - only testing X`);
|
||||
return { success: false, message: 'Test mode - only symbol X allowed' };
|
||||
}
|
||||
|
||||
// If it's a batch operation, filter to only X
|
||||
if (args[0].symbols && Array.isArray(args[0].symbols)) {
|
||||
args[0].symbols = args[0].symbols.filter((s: string) => s === 'X');
|
||||
}
|
||||
}
|
||||
|
||||
// Call original function with potentially modified args
|
||||
return originalFunction.apply(this, args);
|
||||
};
|
||||
}
|
||||
|
||||
// Example usage
|
||||
async function demonstrateUsage() {
|
||||
console.log('=== Demonstration of Symbol X Only Modifications ===\n');
|
||||
|
||||
// Mock tracker for demonstration
|
||||
const mockTracker = {
|
||||
getStaleSymbols: async (provider: string, operation: string, options: any) => {
|
||||
console.log(`Getting stale symbols with options:`, options);
|
||||
if (options.symbolFilter?.symbol === 'X') {
|
||||
return ['X:NYSE'];
|
||||
}
|
||||
return ['AAPL:NASDAQ', 'GOOGL:NASDAQ', 'X:NYSE', 'MSFT:NASDAQ'];
|
||||
}
|
||||
} as any;
|
||||
|
||||
// Test the modified function
|
||||
console.log('1. Testing getStaleSymbols with X filter:');
|
||||
const xOnlySymbols = await getStaleSymbolsOnlyX(mockTracker, 'qm', 'price_update', {
|
||||
minHoursSinceRun: 24
|
||||
});
|
||||
console.log('Results:', xOnlySymbols);
|
||||
|
||||
console.log('\n2. Example of wrapper usage:');
|
||||
const originalUpdate = async (input: any) => {
|
||||
console.log(`Processing symbol: ${input.symbol}`);
|
||||
return { success: true, symbol: input.symbol };
|
||||
};
|
||||
|
||||
const wrappedUpdate = createSymbolXTestWrapper(originalUpdate);
|
||||
|
||||
await wrappedUpdate({ symbol: 'X', symbolId: 123 });
|
||||
await wrappedUpdate({ symbol: 'AAPL', symbolId: 456 }); // Will be skipped
|
||||
|
||||
console.log('\n=== Demonstration Complete ===');
|
||||
}
|
||||
|
||||
// Run demonstration
|
||||
demonstrateUsage().catch(console.error);
|
||||
163
apps/stock/data-ingestion/test/test-qm-operations.ts
Normal file
163
apps/stock/data-ingestion/test/test-qm-operations.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
/**
|
||||
* Test script for QM operations
|
||||
*/
|
||||
|
||||
import { QMHandler } from '../src/handlers/qm/qm.handler';
|
||||
import type { DataIngestionServices } from '../src/types';
|
||||
|
||||
// Mock services for testing
|
||||
const mockServices: Partial<DataIngestionServices> = {
|
||||
mongodb: {
|
||||
batchUpsert: async (collection: string, data: any[], uniqueKeys: string[]) => {
|
||||
console.log(`Mock: Batch upsert to ${collection}`, {
|
||||
recordCount: data.length,
|
||||
uniqueKeys
|
||||
});
|
||||
return { insertedCount: data.length, modifiedCount: 0 };
|
||||
},
|
||||
find: async (collection: string, query: any, options?: any) => {
|
||||
console.log(`Mock: Find from ${collection}`, { query, options });
|
||||
// Return test symbol for testing
|
||||
if (collection === 'qmSymbols' && query.symbol === 'X') {
|
||||
return [{
|
||||
symbol: 'X',
|
||||
symbolId: 123456,
|
||||
qmSearchCode: 'X:NYSE',
|
||||
exchange: 'NYSE',
|
||||
name: 'United States Steel Corporation'
|
||||
}];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
updateOne: async (collection: string, filter: any, update: any, options?: any) => {
|
||||
console.log(`Mock: Update ${collection}`, { filter, update, options });
|
||||
return { modifiedCount: 1 };
|
||||
}
|
||||
},
|
||||
cache: {
|
||||
get: async (key: string) => {
|
||||
console.log(`Mock: Cache get ${key}`);
|
||||
return null;
|
||||
},
|
||||
set: async (key: string, value: any, ttl?: number) => {
|
||||
console.log(`Mock: Cache set ${key}`, { ttl });
|
||||
return true;
|
||||
}
|
||||
},
|
||||
logger: {
|
||||
info: (message: string, data?: any) => {
|
||||
console.log(`[INFO] ${message}`, data || '');
|
||||
},
|
||||
error: (message: string, data?: any) => {
|
||||
console.error(`[ERROR] ${message}`, data || '');
|
||||
},
|
||||
warn: (message: string, data?: any) => {
|
||||
console.warn(`[WARN] ${message}`, data || '');
|
||||
},
|
||||
debug: (message: string, data?: any) => {
|
||||
console.debug(`[DEBUG] ${message}`, data || '');
|
||||
}
|
||||
},
|
||||
// Mock operation registry
|
||||
operationRegistry: {
|
||||
updateOperation: async (provider: string, symbol: string, operation: string, data: any) => {
|
||||
console.log(`Mock: Update operation ${provider}/${operation} for ${symbol}`, data);
|
||||
return true;
|
||||
},
|
||||
getStaleSymbols: async (provider: string, operation: string, options: any) => {
|
||||
console.log(`Mock: Get stale symbols for ${provider}/${operation}`, options);
|
||||
// Return test symbol
|
||||
if (options.symbolFilter?.symbol === 'X') {
|
||||
return ['X:NYSE'];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}
|
||||
} as DataIngestionServices;
|
||||
|
||||
async function testQMOperations() {
|
||||
console.log('=== Testing QM Operations ===\n');
|
||||
|
||||
// Create handler instance
|
||||
const handler = new QMHandler(mockServices);
|
||||
|
||||
// Wait a bit for initialization
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
|
||||
// Test 1: Update Insiders for symbol X
|
||||
console.log('Test 1: Update Insiders for symbol X');
|
||||
console.log('-------------------------------------');
|
||||
try {
|
||||
const insidersResult = await handler.updateInsiders({
|
||||
symbol: 'X',
|
||||
symbolId: 123456,
|
||||
qmSearchCode: 'X:NYSE',
|
||||
lookbackDays: 30
|
||||
});
|
||||
console.log('Result:', JSON.stringify(insidersResult, null, 2));
|
||||
} catch (error) {
|
||||
console.error('Failed:', error);
|
||||
}
|
||||
|
||||
console.log('\n');
|
||||
|
||||
// Test 2: Update Symbol News for symbol X
|
||||
console.log('Test 2: Update Symbol News for symbol X');
|
||||
console.log('----------------------------------------');
|
||||
try {
|
||||
const newsResult = await handler.updateSymbolNews({
|
||||
symbol: 'X',
|
||||
symbolId: 123456,
|
||||
qmSearchCode: 'X:NYSE',
|
||||
lookbackDays: 7
|
||||
});
|
||||
console.log('Result:', JSON.stringify(newsResult, null, 2));
|
||||
} catch (error) {
|
||||
console.error('Failed:', error);
|
||||
}
|
||||
|
||||
console.log('\n');
|
||||
|
||||
// Test 3: Update General News
|
||||
console.log('Test 3: Update General News');
|
||||
console.log('---------------------------');
|
||||
try {
|
||||
const generalNewsResult = await handler.updateGeneralNews({
|
||||
categories: ['market', 'economy'],
|
||||
lookbackMinutes: 60
|
||||
});
|
||||
console.log('Result:', JSON.stringify(generalNewsResult, null, 2));
|
||||
} catch (error) {
|
||||
console.error('Failed:', error);
|
||||
}
|
||||
|
||||
console.log('\n');
|
||||
|
||||
// Test 4: Check available operations
|
||||
console.log('Test 4: List Available Operations');
|
||||
console.log('---------------------------------');
|
||||
const operations = [
|
||||
'create-session',
|
||||
'search-symbols',
|
||||
'update-symbol-info',
|
||||
'update-financials',
|
||||
'update-events',
|
||||
'update-filings',
|
||||
'update-prices',
|
||||
'update-intraday-bars',
|
||||
'crawl-intraday-data',
|
||||
'update-insiders',
|
||||
'update-symbol-news',
|
||||
'update-general-news'
|
||||
];
|
||||
|
||||
for (const op of operations) {
|
||||
const hasOperation = typeof (handler as any)[op.replace(/-/g, '')] === 'function';
|
||||
console.log(`${op}: ${hasOperation ? '✓' : '✗'}`);
|
||||
}
|
||||
|
||||
console.log('\n=== Tests Complete ===');
|
||||
}
|
||||
|
||||
// Run tests
|
||||
testQMOperations().catch(console.error);
|
||||
118
apps/stock/data-ingestion/test/test-symbol-x.ts
Normal file
118
apps/stock/data-ingestion/test/test-symbol-x.ts
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
/**
|
||||
* Test script specifically for symbol X operations
|
||||
*/
|
||||
|
||||
import { QMHandler } from '../src/handlers/qm/qm.handler';
|
||||
import { OperationTracker } from '../src/shared/operation-manager';
|
||||
import type { DataIngestionServices } from '../src/types';
|
||||
|
||||
// Simple test to check operations for symbol X
|
||||
async function testSymbolXOperations() {
|
||||
console.log('=== Testing Operations for Symbol X ===\n');
|
||||
|
||||
// Mock minimal services needed
|
||||
const mockServices: Partial<DataIngestionServices> = {
|
||||
mongodb: {
|
||||
collection: (name: string) => ({
|
||||
find: () => ({
|
||||
toArray: async () => {
|
||||
console.log(`Querying collection: ${name}`);
|
||||
if (name === 'qmSymbols') {
|
||||
return [{
|
||||
symbol: 'X',
|
||||
symbolId: 123456,
|
||||
qmSearchCode: 'X:NYSE',
|
||||
exchange: 'NYSE',
|
||||
name: 'United States Steel Corporation'
|
||||
}];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}),
|
||||
findOne: async (query: any) => {
|
||||
console.log(`Finding one in ${name}:`, query);
|
||||
return null;
|
||||
},
|
||||
updateOne: async (filter: any, update: any, options: any) => {
|
||||
console.log(`Updating ${name}:`, { filter, update });
|
||||
return { modifiedCount: 1 };
|
||||
}
|
||||
}),
|
||||
find: async (collection: string, query: any) => {
|
||||
console.log(`Direct find on ${collection}:`, query);
|
||||
if (collection === 'qmSymbols' && query.symbol === 'X') {
|
||||
return [{
|
||||
symbol: 'X',
|
||||
symbolId: 123456,
|
||||
qmSearchCode: 'X:NYSE'
|
||||
}];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
} as any,
|
||||
logger: {
|
||||
info: (msg: string, data?: any) => console.log(`[INFO] ${msg}`, data || ''),
|
||||
error: (msg: string, data?: any) => console.error(`[ERROR] ${msg}`, data || ''),
|
||||
warn: (msg: string, data?: any) => console.warn(`[WARN] ${msg}`, data || ''),
|
||||
debug: (msg: string, data?: any) => console.debug(`[DEBUG] ${msg}`, data || '')
|
||||
}
|
||||
} as DataIngestionServices;
|
||||
|
||||
// Test 1: Check stale operations for symbol X
|
||||
console.log('Test 1: Get stale operations for symbol X');
|
||||
console.log('------------------------------------------');
|
||||
|
||||
const tracker = new OperationTracker(mockServices as any);
|
||||
|
||||
try {
|
||||
// Check each operation type
|
||||
const operations = [
|
||||
'symbol_info',
|
||||
'price_update',
|
||||
'intraday_bars',
|
||||
'financials_update_quarterly',
|
||||
'financials_update_annual',
|
||||
'events_update',
|
||||
'filings_update',
|
||||
'insiders_update',
|
||||
'news_update'
|
||||
];
|
||||
|
||||
for (const operation of operations) {
|
||||
console.log(`\nChecking ${operation}:`);
|
||||
|
||||
const staleSymbols = await tracker.getStaleSymbols('qm', operation, {
|
||||
minHoursSinceRun: 0, // Get all symbols regardless of last run
|
||||
limit: 10,
|
||||
symbolFilter: { symbol: 'X' } // Only get symbol X
|
||||
});
|
||||
|
||||
console.log(`Found ${staleSymbols.length} stale symbols:`, staleSymbols);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error checking stale symbols:', error);
|
||||
}
|
||||
|
||||
// Test 2: Check intraday crawl status for symbol X
|
||||
console.log('\n\nTest 2: Check intraday crawl status for symbol X');
|
||||
console.log('------------------------------------------------');
|
||||
|
||||
try {
|
||||
const symbolsForCrawl = await tracker.getSymbolsForIntradayCrawl('intraday_bars', {
|
||||
limit: 10,
|
||||
symbolFilter: { symbol: 'X' }
|
||||
});
|
||||
|
||||
console.log(`Found ${symbolsForCrawl.length} symbols for intraday crawl`);
|
||||
if (symbolsForCrawl.length > 0) {
|
||||
console.log('Symbol details:', JSON.stringify(symbolsForCrawl[0], null, 2));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error checking intraday crawl:', error);
|
||||
}
|
||||
|
||||
console.log('\n=== Tests Complete ===');
|
||||
}
|
||||
|
||||
// Run the test
|
||||
testSymbolXOperations().catch(console.error);
|
||||
Loading…
Add table
Add a link
Reference in a new issue