work on qm spider

apps/stock/config/config/default.json

@@ -86,8 +86,8 @@
         "type": "exponential",
         "delay": 1000
       },
-      "removeOnComplete": 100,
+      "removeOnComplete": 50000,
-      "removeOnFail": 50,
+      "removeOnFail": 50000,
       "timeout": 300000
     }
   },

apps/stock/data-ingestion/src/handlers/qm/actions/index.ts

@@ -3,3 +3,4 @@
  */
 
 export { checkSessions, createSession } from './session.action';
+export { spiderSymbolSearch, searchSymbols } from './symbol.action';

apps/stock/data-ingestion/src/handlers/qm/actions/spider.action.ts

@@ -1,33 +0,0 @@
-/**
- * QM Spider Operations - Simple symbol discovery
- */
-
-import type { IServiceContainer } from '@stock-bot/handlers';
-import type { SymbolSpiderJob } from '../shared/types';
-
-export async function spiderSymbolSearch(
-  services: IServiceContainer,
-  config: SymbolSpiderJob
-): Promise<{ foundSymbols: number; depth: number }> {
-  // Simple spider implementation
-  // TODO: Implement actual API calls to discover symbols
-
-  // For now, just return mock results
-  const foundSymbols = Math.floor(Math.random() * 10) + 1;
-
-  return {
-    foundSymbols,
-    depth: config.depth,
-  };
-}
-
-export async function queueSymbolDiscovery(
-  services: IServiceContainer,
-  searchTerms: string[]
-): Promise<void> {
-  // Queue symbol discovery jobs
-  for (const term of searchTerms) {
-    // TODO: Queue actual discovery jobs
-    await services.cache.set(`discovery:${term}`, { queued: true }, 3600);
-  }
-}

apps/stock/data-ingestion/src/handlers/qm/actions/symbol.action.ts

@@ -0,0 +1,238 @@
+/**
+ * QM Symbol Actions - Symbol search and spider operations
+ */
+
+import type { BaseHandler, ExecutionContext } from '@stock-bot/handlers';
+import type { Exchange, SymbolSpiderJob } from '../shared/types';
+
+/**
+ * Spider search for symbols - recursively searches QM API
+ * Root job (no prefix) creates A-Z jobs
+ * Each job searches its prefix and creates child jobs if needed
+ */
+export async function spiderSymbolSearch(
+  this: BaseHandler,
+  input: SymbolSpiderJob,
+  _context: ExecutionContext
+): Promise<{
+  message: string;
+  symbolsFound?: number;
+  jobsCreated?: number;
+}> {
+  const { prefix, depth = 0, maxDepth = 4 } = input || {};
+
+  this.logger.info('Spider symbol search', { prefix, depth, maxDepth });
+  console.log('Spider symbol search', { prefix, depth, maxDepth });
+
+  if (!prefix) {
+    // Root job - create A-Z jobs
+    let jobsCreated = 0;
+    for (let i = 0; i < 26; i++) {
+      const letter = String.fromCharCode(65 + i); // A-Z
+      await this.scheduleOperation('spider-symbols', {
+        prefix: letter,
+        depth: 1,
+        source: 'qm',
+        maxDepth
+      }, {
+        priority: 5
+      });
+      jobsCreated++;
+    }
+
+    this.logger.info('Created root spider jobs', { jobsCreated });
+    return { 
+      message: `Queued ${jobsCreated} root jobs (A-Z)`, 
+      jobsCreated 
+    };
+  }
+
+  try {
+    // Search current prefix
+    const symbols = await searchSymbols.call(this, { query: prefix });
+    
+    if (!symbols || symbols.length === 0) {
+      this.logger.debug('No symbols found for prefix', { prefix });
+      return { 
+        message: `No symbols found for prefix: ${prefix}`,
+        symbolsFound: 0 
+      };
+    }
+
+    // Store symbols in MongoDB
+    const processedSymbols = symbols.map(symbol => ({
+      ...symbol,
+      spiderPrefix: prefix,
+      spiderDepth: depth,
+      discoveredAt: new Date()
+    }));
+
+    await this.mongodb.batchUpsert('qm_symbols', processedSymbols, ['qmSearchCode']);
+    
+    this.logger.info('Stored symbols from spider search', { 
+      prefix, 
+      count: symbols.length 
+    });
+
+    // Extract and store unique exchanges
+    const exchanges: Exchange[] = [];
+    for (const symbol of symbols) {
+      if (symbol.exchange && !exchanges.some(ex => ex.exchange === symbol.exchange)) {
+        exchanges.push({
+          exchange: symbol.exchange,
+          exchangeCode: symbol.exchangeCode || '',
+          exchangeShortName: symbol.exchangeShortName || '',
+          countryCode: symbol.countryCode || '',
+          source: 'qm',
+        });
+      }
+    }
+
+    if (exchanges.length > 0) {
+      await this.mongodb.batchUpsert('qm_exchanges', exchanges, ['exchange']);
+      this.logger.debug('Stored exchanges from spider search', { 
+        count: exchanges.length 
+      });
+    }
+
+    // If not at max depth and we found symbols, create child jobs
+    if (depth < maxDepth && symbols.length > 0) {
+      let jobsCreated = 0;
+      
+      // Only create child jobs if we found a significant number of symbols
+      // This prevents excessive branching on sparse results
+      if (symbols.length >= 10) {
+        for (let i = 0; i < 26; i++) {
+          const nextPrefix = prefix + String.fromCharCode(65 + i);
+          await this.scheduleOperation('spider-symbols', {
+            prefix: nextPrefix,
+            depth: depth + 1,
+            source: 'qm',
+            maxDepth
+          }, {
+            delay: (i * 2000) + 10000, // Start after 10s, stagger by 2s
+            priority: Math.max(1, 5 - depth) // Lower priority for deeper searches
+          });
+          jobsCreated++;
+        }
+      }
+
+      return { 
+        message: `Found ${symbols.length} symbols for ${prefix}, queued ${jobsCreated} child jobs`,
+        symbolsFound: symbols.length,
+        jobsCreated 
+      };
+    }
+
+    return { 
+      message: `Found ${symbols.length} symbols for ${prefix} (at max depth or too few results)`,
+      symbolsFound: symbols.length 
+    };
+
+  } catch (error) {
+    this.logger.error('Spider search failed', { prefix, error });
+    return {
+      message: `Spider search failed for prefix: ${prefix}`,
+      symbolsFound: 0
+    };
+  }
+}
+
+/**
+ * Search QM symbols API directly
+ */
+export async function searchSymbols(
+  this: BaseHandler,
+  input: { query: string },
+  _context?: ExecutionContext
+): Promise<any[]> {
+  const { query } = input;
+  
+  this.logger.debug('Searching QM symbols', { query });
+  return []
+
+  // const sessionManager = QMSessionManager.getInstance();
+  // sessionManager.initialize(this.cache, this.logger);
+
+  // // Get a session
+  // const sessionId = QM_SESSION_IDS.LOOKUP;
+  // const session = await sessionManager.getSession(sessionId);
+  
+  // if (!session || !session.uuid) {
+  //   throw new Error(`No active session found for QM LOOKUP`);
+  // }
+
+  // try {
+  //   // Check cache first
+  //   const cacheKey = `qm:symbol-search:${query}`;
+  //   const cachedResult = await this.cache?.get(cacheKey);
+  //   if (cachedResult) {
+  //     this.logger.trace('Using cached symbol search result', { query });
+  //     return cachedResult as any[];
+  //   }
+
+  //   // Build API request
+  //   const searchParams = new URLSearchParams({
+  //     marketType: 'equity',
+  //     pathName: '/demo/portal/company-summary.php',
+  //     q: query,
+  //     qmodTool: 'SmartSymbolLookup',
+  //     searchType: 'symbol',
+  //     showFree: 'false',
+  //     showHisa: 'false',
+  //     webmasterId: '500'
+  //   });
+    
+  //   const apiUrl = `${QM_CONFIG.LOOKUP_URL}?${searchParams.toString()}`;
+
+  //   const response = await fetch(apiUrl, {
+  //     method: 'GET',
+  //     headers: session.headers,
+  //     proxy: session.proxy,
+  //     signal: AbortSignal.timeout(SESSION_CONFIG.API_TIMEOUT),
+  //   });
+
+  //   if (!response.ok) {
+  //     throw new Error(`QM API request failed: ${response.status} ${response.statusText}`);
+  //   }
+
+  //   const symbols = await response.json();
+    
+  //   // Update session success stats
+  //   await sessionManager.incrementSuccessfulCalls(sessionId, session.uuid);
+
+  //   // Process symbol data
+  //   const processedSymbols = Array.isArray(symbols) ? symbols.map((symbol: any) => ({
+  //     ...symbol,
+  //     qmSearchCode: symbol.symbol || '',
+  //     symbol: (symbol.symbol as string)?.split(':')[0] || '',
+  //     searchQuery: query,
+  //     fetchedAt: new Date()
+  //   })) : [];
+
+  //   // Cache the result
+  //   if (processedSymbols.length > 0) {
+  //     await this.cache?.set(cacheKey, processedSymbols, 1800); // 30 minutes
+  //   }
+
+  //   this.logger.info('QM API returned symbols', { 
+  //     query,
+  //     count: processedSymbols.length 
+  //   });
+
+  //   return processedSymbols;
+
+  // } catch (error) {
+  //   // Update session failure stats
+  //   if (session.uuid) {
+  //     await sessionManager.incrementFailedCalls(sessionId, session.uuid);
+  //   }
+
+  //   this.logger.error('Error searching QM symbols', { 
+  //     query,
+  //     error: error instanceof Error ? error.message : 'Unknown error'
+  //   });
+    
+  //   throw error;
+  // }
+}

apps/stock/data-ingestion/src/handlers/qm/actions/symbols.action.ts

@@ -1,36 +0,0 @@
-/**
- * QM Symbols Operations - Simple symbol fetching
- */
-
-import type { IServiceContainer } from '@stock-bot/handlers';
-
-interface QMSymbol {
-  _id?: string;
-  symbol: string;
-  name: string;
-  exchange: string;
-  type?: string;
-  sector?: string;
-  industry?: string;
-}
-
-export async function searchSymbols(services: IServiceContainer): Promise<QMSymbol[]> {
-  // Get symbols from MongoDB
-  const symbols = await services.mongodb
-    .collection<QMSymbol>('qm_symbols')
-    .find({})
-    .limit(50)
-    .toArray();
-
-  return symbols;
-}
-
-export async function fetchSymbolData(
-  services: IServiceContainer,
-  symbol: string
-): Promise<QMSymbol | null> {
-  // Fetch data for a specific symbol
-  const symbolData = await services.mongodb.collection<QMSymbol>('qm_symbols').findOne({ symbol });
-
-  return symbolData;
-}

apps/stock/data-ingestion/src/handlers/qm/qm.handler.ts

@@ -4,7 +4,7 @@ import {
   Operation,
   ScheduledOperation,
 } from '@stock-bot/handlers';
-import { checkSessions, createSession } from './actions';
+import { checkSessions, createSession, searchSymbols, spiderSymbolSearch } from './actions';
 
 @Handler('qm')
 export class QMHandler extends BaseHandler {
@@ -14,7 +14,7 @@ export class QMHandler extends BaseHandler {
 
   @ScheduledOperation('check-sessions', '*/2 * * * *', {
     priority: 8,
-    immediately: true,
+    immediately: false,
     description: 'Check and maintain QM sessions every 2 minutes',
   })
   checkSessions = checkSessions;
@@ -22,79 +22,16 @@ export class QMHandler extends BaseHandler {
   @Operation('create-session')
   createSession = createSession;
 
-  // @Operation('search-symbols')
+  @ScheduledOperation('spider-symbol-search', '* * * * *', {
-  // async searchSymbols(_input: unknown, _context: ExecutionContext): Promise<unknown> {
+    priority: 8,
-  //   this.logger.info('Searching QM symbols with new DI pattern...');
+    immediately: false,
-  //   try {
+    description: 'Weekly comprehensive symbol search using QM API spider - runs every Saturday at midnight'
-  //     // Check existing symbols in MongoDB
+  })
-  //     const symbolsCollection = this.mongodb.collection('qm_symbols');
+  spiderSymbolSchedule = spiderSymbolSearch;
-  //     const symbols = await symbolsCollection.find({}).limit(100).toArray();
 
-  //     this.logger.info('QM symbol search completed', { count: symbols.length });
+  @Operation('spider-symbols')
+  spiderSymbolsJob = spiderSymbolSearch;
 
-  //     if (symbols && symbols.length > 0) {
+  @Operation('search-symbols')
-  //       // Cache result for performance
+  searchSymbols = searchSymbols;
-  //       await this.cache.set('qm-symbols-sample', symbols.slice(0, 10), 1800);
-
-  //       return {
-  //         success: true,
-  //         message: 'QM symbol search completed successfully',
-  //         count: symbols.length,
-  //         symbols: symbols.slice(0, 10), // Return first 10 symbols as sample
-  //       };
-  //     } else {
-  //       // No symbols found - this is expected initially
-  //       this.logger.info('No QM symbols found in database yet');
-  //       return {
-  //         success: true,
-  //         message: 'No symbols found yet - database is empty',
-  //         count: 0,
-  //       };
-  //     }
-
-  //   } catch (error) {
-  //     this.logger.error('Failed to search QM symbols', { error });
-  //     throw error;
-  //   }
-  // }
-
-  // @Operation('spider-symbol-search')
-  // @QueueSchedule('0 0 * * 0', {
-  //   priority: 10,
-  //   immediately: false,
-  //   description: 'Comprehensive symbol search using QM API'
-  // })
-  // async spiderSymbolSearch(payload: SymbolSpiderJob | undefined, context: ExecutionContext): Promise<unknown> {
-  //   // Set default payload for scheduled runs
-  //   const jobPayload: SymbolSpiderJob = payload || {
-  //     prefix: null,
-  //     depth: 1,
-  //     source: 'qm',
-  //     maxDepth: 4
-  //   };
-
-  //   this.logger.info('Starting QM spider symbol search', { payload: jobPayload });
-
-  //   // Store spider job info in cache (temporary data)
-  //   const spiderJobId = `spider:qm:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`;
-  //   const spiderResult = {
-  //     payload: jobPayload,
-  //     startTime: new Date().toISOString(),
-  //     status: 'started',
-  //     jobId: spiderJobId
-  //   };
-
-  //   // Store in cache with 1 hour TTL (temporary data)
-  //   await this.cache.set(spiderJobId, spiderResult, 3600);
-  //   this.logger.debug('Spider job stored in cache', { spiderJobId, ttl: 3600 });
-
-  //   // Schedule follow-up processing if needed
-  //   await this.scheduleOperation('search-symbols', { source: 'spider', spiderJobId }, { delay: 5000 });
-
-  //   return {
-  //     success: true,
-  //     message: 'QM spider search initiated',
-  //     spiderJobId
-  //   };
-  // }
 }

apps/stock/data-ingestion/test-spider.md

@@ -0,0 +1,80 @@
+# QM Spider Symbol Search Test
+
+## How the Spider Search Works
+
+The spider search is a recursive symbol discovery system that explores the QM API systematically:
+
+### 1. Root Job (Weekly Schedule)
+```typescript
+// Triggered every Saturday at midnight
+@ScheduledOperation('spider-symbol-search', '0 0 * * 6', {...})
+
+// Creates 26 jobs: A, B, C, ... Z
+input: { prefix: null, depth: 0, maxDepth: 4 }
+```
+
+### 2. Level 1 - Single Letters (A-Z)
+```typescript
+// Each job searches for symbols starting with that letter
+input: { prefix: 'A', depth: 1, maxDepth: 4 }
+
+// If 10+ symbols found, creates: AA, AB, AC, ... AZ
+```
+
+### 3. Level 2 - Two Letters (AA-ZZ)
+```typescript
+// Searches for symbols starting with two letters
+input: { prefix: 'AA', depth: 2, maxDepth: 4 }
+
+// If 10+ symbols found, creates: AAA, AAB, AAC, ... AAZ
+```
+
+### 4. Level 3 & 4 - Deeper Search
+Continues until `maxDepth` is reached or fewer than 10 symbols are found.
+
+## Testing the Spider
+
+### Manual Test - Root Job
+```bash
+# Trigger spider search root job
+curl -X POST http://localhost:3000/api/handlers/qm/operations/spider-symbol-search \
+  -H "Content-Type: application/json" \
+  -d '{}'
+```
+
+### Manual Test - Specific Prefix
+```bash
+# Search for symbols starting with "APP"
+curl -X POST http://localhost:3000/api/handlers/qm/operations/spider-symbol-search \
+  -H "Content-Type: application/json" \
+  -d '{"prefix": "APP", "depth": 3, "maxDepth": 4}'
+```
+
+### Direct Symbol Search
+```bash
+# Search for specific symbols
+curl -X POST http://localhost:3000/api/handlers/qm/operations/search-symbols \
+  -H "Content-Type: application/json" \
+  -d '{"query": "AAPL"}'
+```
+
+## Optimization Features
+
+1. **Intelligent Branching**: Only creates child jobs if 10+ symbols found
+2. **Priority Management**: Deeper searches get lower priority
+3. **Staggered Execution**: Jobs are delayed to avoid API rate limits
+4. **Session Management**: Uses QM sessions with failure tracking
+5. **Caching**: Results cached for 30 minutes to avoid duplicate API calls
+
+## MongoDB Collections
+
+- `qm_symbols`: Stores discovered symbols with metadata
+- `qm_exchanges`: Stores unique exchanges found during searches
+
+## Monitoring
+
+Check logs for:
+- "Spider symbol search" - Job execution
+- "Created root spider jobs" - Initial A-Z creation
+- "Stored symbols from spider search" - Successful symbol storage
+- "Found X symbols for Y, queued Z child jobs" - Branching decisions