work on qm spider
This commit is contained in:
parent
b767689470
commit
1d62343051
7 changed files with 333 additions and 146 deletions
|
|
@ -86,8 +86,8 @@
|
|||
"type": "exponential",
|
||||
"delay": 1000
|
||||
},
|
||||
"removeOnComplete": 100,
|
||||
"removeOnFail": 50,
|
||||
"removeOnComplete": 50000,
|
||||
"removeOnFail": 50000,
|
||||
"timeout": 300000
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -3,3 +3,4 @@
|
|||
*/
|
||||
|
||||
export { checkSessions, createSession } from './session.action';
|
||||
export { spiderSymbolSearch, searchSymbols } from './symbol.action';
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
/**
|
||||
* QM Spider Operations - Simple symbol discovery
|
||||
*/
|
||||
|
||||
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||
import type { SymbolSpiderJob } from '../shared/types';
|
||||
|
||||
export async function spiderSymbolSearch(
|
||||
services: IServiceContainer,
|
||||
config: SymbolSpiderJob
|
||||
): Promise<{ foundSymbols: number; depth: number }> {
|
||||
// Simple spider implementation
|
||||
// TODO: Implement actual API calls to discover symbols
|
||||
|
||||
// For now, just return mock results
|
||||
const foundSymbols = Math.floor(Math.random() * 10) + 1;
|
||||
|
||||
return {
|
||||
foundSymbols,
|
||||
depth: config.depth,
|
||||
};
|
||||
}
|
||||
|
||||
export async function queueSymbolDiscovery(
|
||||
services: IServiceContainer,
|
||||
searchTerms: string[]
|
||||
): Promise<void> {
|
||||
// Queue symbol discovery jobs
|
||||
for (const term of searchTerms) {
|
||||
// TODO: Queue actual discovery jobs
|
||||
await services.cache.set(`discovery:${term}`, { queued: true }, 3600);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,238 @@
|
|||
/**
|
||||
* QM Symbol Actions - Symbol search and spider operations
|
||||
*/
|
||||
|
||||
import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
|
||||
import type { Exchange, SymbolSpiderJob } from '../shared/types';
|
||||
|
||||
/**
|
||||
* Spider search for symbols - recursively searches QM API
|
||||
* Root job (no prefix) creates A-Z jobs
|
||||
* Each job searches its prefix and creates child jobs if needed
|
||||
*/
|
||||
export async function spiderSymbolSearch(
|
||||
this: BaseHandler,
|
||||
input: SymbolSpiderJob,
|
||||
_context: ExecutionContext
|
||||
): Promise<{
|
||||
message: string;
|
||||
symbolsFound?: number;
|
||||
jobsCreated?: number;
|
||||
}> {
|
||||
const { prefix, depth = 0, maxDepth = 4 } = input || {};
|
||||
|
||||
this.logger.info('Spider symbol search', { prefix, depth, maxDepth });
|
||||
console.log('Spider symbol search', { prefix, depth, maxDepth });
|
||||
|
||||
if (!prefix) {
|
||||
// Root job - create A-Z jobs
|
||||
let jobsCreated = 0;
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const letter = String.fromCharCode(65 + i); // A-Z
|
||||
await this.scheduleOperation('spider-symbols', {
|
||||
prefix: letter,
|
||||
depth: 1,
|
||||
source: 'qm',
|
||||
maxDepth
|
||||
}, {
|
||||
priority: 5
|
||||
});
|
||||
jobsCreated++;
|
||||
}
|
||||
|
||||
this.logger.info('Created root spider jobs', { jobsCreated });
|
||||
return {
|
||||
message: `Queued ${jobsCreated} root jobs (A-Z)`,
|
||||
jobsCreated
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// Search current prefix
|
||||
const symbols = await searchSymbols.call(this, { query: prefix });
|
||||
|
||||
if (!symbols || symbols.length === 0) {
|
||||
this.logger.debug('No symbols found for prefix', { prefix });
|
||||
return {
|
||||
message: `No symbols found for prefix: ${prefix}`,
|
||||
symbolsFound: 0
|
||||
};
|
||||
}
|
||||
|
||||
// Store symbols in MongoDB
|
||||
const processedSymbols = symbols.map(symbol => ({
|
||||
...symbol,
|
||||
spiderPrefix: prefix,
|
||||
spiderDepth: depth,
|
||||
discoveredAt: new Date()
|
||||
}));
|
||||
|
||||
await this.mongodb.batchUpsert('qm_symbols', processedSymbols, ['qmSearchCode']);
|
||||
|
||||
this.logger.info('Stored symbols from spider search', {
|
||||
prefix,
|
||||
count: symbols.length
|
||||
});
|
||||
|
||||
// Extract and store unique exchanges
|
||||
const exchanges: Exchange[] = [];
|
||||
for (const symbol of symbols) {
|
||||
if (symbol.exchange && !exchanges.some(ex => ex.exchange === symbol.exchange)) {
|
||||
exchanges.push({
|
||||
exchange: symbol.exchange,
|
||||
exchangeCode: symbol.exchangeCode || '',
|
||||
exchangeShortName: symbol.exchangeShortName || '',
|
||||
countryCode: symbol.countryCode || '',
|
||||
source: 'qm',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (exchanges.length > 0) {
|
||||
await this.mongodb.batchUpsert('qm_exchanges', exchanges, ['exchange']);
|
||||
this.logger.debug('Stored exchanges from spider search', {
|
||||
count: exchanges.length
|
||||
});
|
||||
}
|
||||
|
||||
// If not at max depth and we found symbols, create child jobs
|
||||
if (depth < maxDepth && symbols.length > 0) {
|
||||
let jobsCreated = 0;
|
||||
|
||||
// Only create child jobs if we found a significant number of symbols
|
||||
// This prevents excessive branching on sparse results
|
||||
if (symbols.length >= 10) {
|
||||
for (let i = 0; i < 26; i++) {
|
||||
const nextPrefix = prefix + String.fromCharCode(65 + i);
|
||||
await this.scheduleOperation('spider-symbols', {
|
||||
prefix: nextPrefix,
|
||||
depth: depth + 1,
|
||||
source: 'qm',
|
||||
maxDepth
|
||||
}, {
|
||||
delay: (i * 2000) + 10000, // Start after 10s, stagger by 2s
|
||||
priority: Math.max(1, 5 - depth) // Lower priority for deeper searches
|
||||
});
|
||||
jobsCreated++;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
message: `Found ${symbols.length} symbols for ${prefix}, queued ${jobsCreated} child jobs`,
|
||||
symbolsFound: symbols.length,
|
||||
jobsCreated
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
message: `Found ${symbols.length} symbols for ${prefix} (at max depth or too few results)`,
|
||||
symbolsFound: symbols.length
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error('Spider search failed', { prefix, error });
|
||||
return {
|
||||
message: `Spider search failed for prefix: ${prefix}`,
|
||||
symbolsFound: 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search QM symbols API directly
|
||||
*/
|
||||
export async function searchSymbols(
|
||||
this: BaseHandler,
|
||||
input: { query: string },
|
||||
_context?: ExecutionContext
|
||||
): Promise<any[]> {
|
||||
const { query } = input;
|
||||
|
||||
this.logger.debug('Searching QM symbols', { query });
|
||||
return []
|
||||
|
||||
// const sessionManager = QMSessionManager.getInstance();
|
||||
// sessionManager.initialize(this.cache, this.logger);
|
||||
|
||||
// // Get a session
|
||||
// const sessionId = QM_SESSION_IDS.LOOKUP;
|
||||
// const session = await sessionManager.getSession(sessionId);
|
||||
|
||||
// if (!session || !session.uuid) {
|
||||
// throw new Error(`No active session found for QM LOOKUP`);
|
||||
// }
|
||||
|
||||
// try {
|
||||
// // Check cache first
|
||||
// const cacheKey = `qm:symbol-search:${query}`;
|
||||
// const cachedResult = await this.cache?.get(cacheKey);
|
||||
// if (cachedResult) {
|
||||
// this.logger.trace('Using cached symbol search result', { query });
|
||||
// return cachedResult as any[];
|
||||
// }
|
||||
|
||||
// // Build API request
|
||||
// const searchParams = new URLSearchParams({
|
||||
// marketType: 'equity',
|
||||
// pathName: '/demo/portal/company-summary.php',
|
||||
// q: query,
|
||||
// qmodTool: 'SmartSymbolLookup',
|
||||
// searchType: 'symbol',
|
||||
// showFree: 'false',
|
||||
// showHisa: 'false',
|
||||
// webmasterId: '500'
|
||||
// });
|
||||
|
||||
// const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`;
|
||||
|
||||
// const response = await fetch(apiUrl, {
|
||||
// method: 'GET',
|
||||
// headers: session.headers,
|
||||
// proxy: session.proxy,
|
||||
// signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT),
|
||||
// });
|
||||
|
||||
// if (!response.ok) {
|
||||
// throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
|
||||
// }
|
||||
|
||||
// const symbols = await response.json();
|
||||
|
||||
// // Update session success stats
|
||||
// await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
|
||||
|
||||
// // Process symbol data
|
||||
// const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({
|
||||
// ...symbol,
|
||||
// qmSearchCode: symbol.symbol || '',
|
||||
// symbol: (symbol.symbol as string)?.split(':')[0] || '',
|
||||
// searchQuery: query,
|
||||
// fetchedAt: new Date()
|
||||
// })) : [];
|
||||
|
||||
// // Cache the result
|
||||
// if (processedSymbols.length > 0) {
|
||||
// await this.cache?.set(cacheKey, processedSymbols, 1800); // 30 minutes
|
||||
// }
|
||||
|
||||
// this.logger.info('QM API returned symbols', {
|
||||
// query,
|
||||
// count: processedSymbols.length
|
||||
// });
|
||||
|
||||
// return processedSymbols;
|
||||
|
||||
// } catch (error) {
|
||||
// // Update session failure stats
|
||||
// if (session.uuid) {
|
||||
// await sessionManager.incrementFailedCalls(sessionId, session.uuid);
|
||||
// }
|
||||
|
||||
// this.logger.error('Error searching QM symbols', {
|
||||
// query,
|
||||
// error: error instanceof Error ? error.message : 'Unknown error'
|
||||
// });
|
||||
|
||||
// throw error;
|
||||
// }
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
/**
|
||||
* QM Symbols Operations - Simple symbol fetching
|
||||
*/
|
||||
|
||||
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||
|
||||
interface QMSymbol {
|
||||
_id?: string;
|
||||
symbol: string;
|
||||
name: string;
|
||||
exchange: string;
|
||||
type?: string;
|
||||
sector?: string;
|
||||
industry?: string;
|
||||
}
|
||||
|
||||
export async function searchSymbols(services: IServiceContainer): Promise<QMSymbol[]> {
|
||||
// Get symbols from MongoDB
|
||||
const symbols = await services.mongodb
|
||||
.collection<QMSymbol>('qm_symbols')
|
||||
.find({})
|
||||
.limit(50)
|
||||
.toArray();
|
||||
|
||||
return symbols;
|
||||
}
|
||||
|
||||
export async function fetchSymbolData(
|
||||
services: IServiceContainer,
|
||||
symbol: string
|
||||
): Promise<QMSymbol | null> {
|
||||
// Fetch data for a specific symbol
|
||||
const symbolData = await services.mongodb.collection<QMSymbol>('qm_symbols').findOne({ symbol });
|
||||
|
||||
return symbolData;
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ import {
|
|||
Operation,
|
||||
ScheduledOperation,
|
||||
} from '@stock-bot/handlers';
|
||||
import { checkSessions, createSession } from './actions';
|
||||
import { checkSessions, createSession, searchSymbols, spiderSymbolSearch } from './actions';
|
||||
|
||||
@Handler('qm')
|
||||
export class QMHandler extends BaseHandler {
|
||||
|
|
@ -14,7 +14,7 @@ export class QMHandler extends BaseHandler {
|
|||
|
||||
@ScheduledOperation('check-sessions', '*/2 * * * *', {
|
||||
priority: 8,
|
||||
immediately: true,
|
||||
immediately: false,
|
||||
description: 'Check and maintain QM sessions every 2 minutes',
|
||||
})
|
||||
checkSessions = checkSessions;
|
||||
|
|
@ -22,79 +22,16 @@ export class QMHandler extends BaseHandler {
|
|||
@Operation('create-session')
|
||||
createSession = createSession;
|
||||
|
||||
// @Operation('search-symbols')
|
||||
// async searchSymbols(_input: unknown, _context: ExecutionContext): Promise<unknown> {
|
||||
// this.logger.info('Searching QM symbols with new DI pattern...');
|
||||
// try {
|
||||
// // Check existing symbols in MongoDB
|
||||
// const symbolsCollection = this.mongodb.collection('qm_symbols');
|
||||
// const symbols = await symbolsCollection.find({}).limit(100).toArray();
|
||||
@ScheduledOperation('spider-symbol-search', '* * * * *', {
|
||||
priority: 8,
|
||||
immediately: false,
|
||||
description: 'Weekly comprehensive symbol search using QM API spider - runs every Saturday at midnight'
|
||||
})
|
||||
spiderSymbolSchedule = spiderSymbolSearch;
|
||||
|
||||
// this.logger.info('QM symbol search completed', { count: symbols.length });
|
||||
@Operation('spider-symbols')
|
||||
spiderSymbolsJob = spiderSymbolSearch;
|
||||
|
||||
// if (symbols && symbols.length > 0) {
|
||||
// // Cache result for performance
|
||||
// await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800);
|
||||
|
||||
// return {
|
||||
// success: true,
|
||||
// message: 'QM symbol search completed successfully',
|
||||
// count: symbols.length,
|
||||
// symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
|
||||
// };
|
||||
// } else {
|
||||
// // No symbols found - this is expected initially
|
||||
// this.logger.info('No QM symbols found in database yet');
|
||||
// return {
|
||||
// success: true,
|
||||
// message: 'No symbols found yet - database is empty',
|
||||
// count: 0,
|
||||
// };
|
||||
// }
|
||||
|
||||
// } catch (error) {
|
||||
// this.logger.error('Failed to search QM symbols', { error });
|
||||
// throw error;
|
||||
// }
|
||||
// }
|
||||
|
||||
// @Operation('spider-symbol-search')
|
||||
// @QueueSchedule('0 0 * * 0', {
|
||||
// priority: 10,
|
||||
// immediately: false,
|
||||
// description: 'Comprehensive symbol search using QM API'
|
||||
// })
|
||||
// async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise<unknown> {
|
||||
// // Set default payload for scheduled runs
|
||||
// const jobPayload: SymbolSpiderJob = payload || {
|
||||
// prefix: null,
|
||||
// depth: 1,
|
||||
// source: 'qm',
|
||||
// maxDepth: 4
|
||||
// };
|
||||
|
||||
// this.logger.info('Starting QM spider symbol search', { payload: jobPayload });
|
||||
|
||||
// // Store spider job info in cache (temporary data)
|
||||
// const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`;
|
||||
// const spiderResult = {
|
||||
// payload: jobPayload,
|
||||
// startTime: new Date().toISOString(),
|
||||
// status: 'started',
|
||||
// jobId: spiderJobId
|
||||
// };
|
||||
|
||||
// // Store in cache with 1 hour TTL (temporary data)
|
||||
// await this.cache.set(spiderJobId, spiderResult, 3600);
|
||||
// this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 });
|
||||
|
||||
// // Schedule follow-up processing if needed
|
||||
// await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, { delay: 5000 });
|
||||
|
||||
// return {
|
||||
// success: true,
|
||||
// message: 'QM spider search initiated',
|
||||
// spiderJobId
|
||||
// };
|
||||
// }
|
||||
@Operation('search-symbols')
|
||||
searchSymbols = searchSymbols;
|
||||
}
|
||||
|
|
|
|||
80
apps/stock/data-ingestion/test-spider.md
Normal file
80
apps/stock/data-ingestion/test-spider.md
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
# QM Spider Symbol Search Test
|
||||
|
||||
## How the Spider Search Works
|
||||
|
||||
The spider search is a recursive symbol discovery system that explores the QM API systematically:
|
||||
|
||||
### 1. Root Job (Weekly Schedule)
|
||||
```typescript
|
||||
// Triggered every Saturday at midnight
|
||||
@ScheduledOperation('spider-symbol-search', '0 0 * * 6', {...})
|
||||
|
||||
// Creates 26 jobs: A, B, C, ... Z
|
||||
input: { prefix: null, depth: 0, maxDepth: 4 }
|
||||
```
|
||||
|
||||
### 2. Level 1 - Single Letters (A-Z)
|
||||
```typescript
|
||||
// Each job searches for symbols starting with that letter
|
||||
input: { prefix: 'A', depth: 1, maxDepth: 4 }
|
||||
|
||||
// If 10+ symbols found, creates: AA, AB, AC, ... AZ
|
||||
```
|
||||
|
||||
### 3. Level 2 - Two Letters (AA-ZZ)
|
||||
```typescript
|
||||
// Searches for symbols starting with two letters
|
||||
input: { prefix: 'AA', depth: 2, maxDepth: 4 }
|
||||
|
||||
// If 10+ symbols found, creates: AAA, AAB, AAC, ... AAZ
|
||||
```
|
||||
|
||||
### 4. Level 3 & 4 - Deeper Search
|
||||
Continues until `maxDepth` is reached or fewer than 10 symbols are found.
|
||||
|
||||
## Testing the Spider
|
||||
|
||||
### Manual Test - Root Job
|
||||
```bash
|
||||
# Trigger spider search root job
|
||||
curl -X POST http://localhost:3000/api/handlers/qm/operations/spider-symbol-search \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{}'
|
||||
```
|
||||
|
||||
### Manual Test - Specific Prefix
|
||||
```bash
|
||||
# Search for symbols starting with "APP"
|
||||
curl -X POST http://localhost:3000/api/handlers/qm/operations/spider-symbol-search \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"prefix": "APP", "depth": 3, "maxDepth": 4}'
|
||||
```
|
||||
|
||||
### Direct Symbol Search
|
||||
```bash
|
||||
# Search for specific symbols
|
||||
curl -X POST http://localhost:3000/api/handlers/qm/operations/search-symbols \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query": "AAPL"}'
|
||||
```
|
||||
|
||||
## Optimization Features
|
||||
|
||||
1. **Intelligent Branching**: Only creates child jobs if 10+ symbols found
|
||||
2. **Priority Management**: Deeper searches get lower priority
|
||||
3. **Staggered Execution**: Jobs are delayed to avoid API rate limits
|
||||
4. **Session Management**: Uses QM sessions with failure tracking
|
||||
5. **Caching**: Results cached for 30 minutes to avoid duplicate API calls
|
||||
|
||||
## MongoDB Collections
|
||||
|
||||
- `qm_symbols`: Stores discovered symbols with metadata
|
||||
- `qm_exchanges`: Stores unique exchanges found during searches
|
||||
|
||||
## Monitoring
|
||||
|
||||
Check logs for:
|
||||
- "Spider symbol search" - Job execution
|
||||
- "Created root spider jobs" - Initial A-Z creation
|
||||
- "Stored symbols from spider search" - Successful symbol storage
|
||||
- "Found X symbols for Y, queued Z child jobs" - Branching decisions
|
||||
Loading…
Add table
Add a link
Reference in a new issue