fixed up ratelimiting

apps/stock/config/config/default.json

@@ -182,7 +182,7 @@
   },
   "services": {
     "dataIngestion": {
-      "port": 2009,
+      "port": 2001,
       "workers": 5,
       "queues": {
         "ceo": { "concurrency": 2 },

apps/stock/data-ingestion/src/handlers/eod/eod.handler.ts

@@ -13,55 +13,63 @@ import {
 } from './actions';
 
 /**
- * EOD (End of Day) Handler for testing rate limits
- * This handler demonstrates different rate limit configurations
+ * EOD (End of Day) Handler demonstrating advanced rate limiting
  * 
- * Handler-level rate limit: 100 requests per minute for all operations
- * Individual operations can override this with their own limits
+ * Handler-level limits apply to all operations unless overridden
+ * Operations can specify just a cost to use handler limits, or override with custom limits
  */
 @Handler('eod')
-// @Disabled()
-@RateLimit({ points: 1, duration: 10, blockDuration: 10 })
+@RateLimit({
+  limits: [
+    { points: 10, duration: 1 },        // 100 points per second
+    { points: 10000, duration: 3600 },   // 10k points per hour
+    { points: 100000, duration: 86400 }, // 100k points per day
+  ],
+  cost: 1, // Default cost for operations using this handler
+})
 export class EodHandler extends BaseHandler<DataIngestionServices> {
   constructor(services: any) {
     super(services);
   }
 
   /**
-   * Fetch daily price data - High volume operation
-   * Rate limit: 50 requests per minute (overrides handler-level limit)
+   * Fetch daily price data - Low cost operation
+   * Uses handler rate limits but costs only 1 point
    */
   @Operation('fetch-daily-prices')
-  @RateLimit({ points: 3, duration: 10, blockDuration: 5 })
+  @RateLimit(1)  // Costs 1 point per call
   fetchDailyPrices = fetchDailyPrices;
 
   /**
-   * Fetch fundamental data - Medium volume operation
-   * Rate limit: 20 requests per minute
+   * Fetch fundamental data - Medium cost operation
+   * Uses handler rate limits but costs 10 points
    */
   @Operation('fetch-fundamentals')
-  @RateLimit({ points: 2, duration: 10, blockDuration: 10 })
+  @RateLimit(1)  // Costs 10 points per call
   fetchFundamentals = fetchFundamentals;
 
   /**
-   * Fetch news data - Low volume operation
-   * Rate limit: 10 requests per minute (most restrictive)
+   * Fetch news data - High cost operation
+   * Has custom limits AND high cost
    */
   @Operation('fetch-news')
+  @RateLimit(1)
   fetchNews = fetchNews;
 
   /**
    * Test burst operations - For testing rate limit behavior
-   * This doesn't have its own rate limit, so it uses the handler-level limit (100/min)
+   * Uses handler default cost (1 point)
    */
   @Operation('test-burst')
+  @RateLimit(0)
   async testBurstOperations(input: { operationsToTest: string[], burstSize: number }): Promise<unknown> {
     this.logger.info('Testing burst operations', input);
     
     const results = {
       attempted: 0,
       scheduled: 0,
-      failed: 0
+      failed: 0,
+      operations: {} as Record<string, number>
     };
 
     try {
@@ -69,11 +77,13 @@ export class EodHandler extends BaseHandler<DataIngestionServices> {
       for (let i = 0; i < input.burstSize; i++) {
         const operation = input.operationsToTest[i % input.operationsToTest.length] || 'fetch-news';
         results.attempted++;
+        results.operations[operation] = (results.operations[operation] || 0) + 1;
         
-        const promise = this.scheduleOperation(operation, {}).then(() => {
+        const promise = this.scheduleOperation(operation, { index: i }).then(() => {
           results.scheduled++;
-        }).catch(() => {
+        }).catch((error) => {
           results.failed++;
+          this.logger.debug('Failed to schedule operation', { operation, error: error.message });
         });
         
         promises.push(promise);
@@ -84,7 +94,8 @@ export class EodHandler extends BaseHandler<DataIngestionServices> {
       return {
         success: true,
         results,
-        message: `Scheduled ${results.scheduled}/${results.attempted} operations`
+        message: `Scheduled ${results.scheduled}/${results.attempted} operations`,
+        breakdown: results.operations
       };
     } catch (error) {
       this.logger.error('Burst test failed', { error });
@@ -92,34 +103,6 @@ export class EodHandler extends BaseHandler<DataIngestionServices> {
     }
   }
 
-  /**
-   * Scheduled job to fetch daily prices
-   * Runs every day at 6 PM (after market close)
-   */
-  @ScheduledOperation('eod-daily-prices', '0 18 * * *', {
-    priority: 5,
-    description: 'Fetch daily price data after market close',
-    immediately: false,
-  })
-  async scheduledFetchDailyPrices(): Promise<unknown> {
-    this.logger.info('Starting scheduled daily price fetch');
-    return this.fetchDailyPrices();
-  }
-
-  /**
-   * Scheduled job to fetch fundamentals
-   * Runs weekly on Sunday
-   */
-  @ScheduledOperation('eod-fundamentals', '0 0 * * 0', {
-    priority: 5,
-    description: 'Weekly fundamental data update',
-    immediately: false,
-  })
-  async scheduledFetchFundamentals(): Promise<unknown> {
-    this.logger.info('Starting scheduled fundamentals fetch');
-    return this.fetchFundamentals();
-  }
-
   /**
    * Scheduled job to test rate limits
    * Runs every 5 minutes for testing
@@ -129,6 +112,7 @@ export class EodHandler extends BaseHandler<DataIngestionServices> {
     description: 'Test rate limit behavior',
     immediately: true,
   })
+  @RateLimit(0) // No cost for this test operation
   async scheduledRateLimitTest(): Promise<unknown> {
     this.logger.info('Starting rate limit test');
     return this.testBurstOperations({ 

apps/stock/data-ingestion/test/test-advanced-rate-limit.ts

@@ -0,0 +1,100 @@
+#!/usr/bin/env bun
+
+import { getLogger } from '@stock-bot/logger';
+
+const logger = getLogger('test-advanced-rate-limit');
+
+async function testAdvancedRateLimits() {
+  logger.info('Testing advanced rate limit features...');
+  
+  const baseUrl = 'http://localhost:3001/api';
+  
+  logger.info('\n📋 Rate Limit Configuration:');
+  logger.info('Handler limits: 100pts/sec, 10k pts/hour, 100k pts/day');
+  logger.info('fetch-daily-prices: 1 point per call');
+  logger.info('fetch-fundamentals: 10 points per call');
+  logger.info('fetch-news: 50 points per call (custom limits: 10pts/min, 100pts/hour)');
+  
+  // First test: test-burst operation
+  logger.info('\n🚀 Testing burst operation with mixed costs...');
+  
+  const burstResponse = await fetch(`${baseUrl}/handlers/eod/operations/test-burst`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      operationsToTest: ['fetch-daily-prices', 'fetch-fundamentals', 'fetch-news'],
+      burstSize: 30
+    })
+  });
+  
+  const burstResult = await burstResponse.json();
+  logger.info('Burst test result:', burstResult);
+  
+  // Wait for jobs to process
+  logger.info('\n⏳ Waiting 10 seconds for jobs to process...');
+  await new Promise(resolve => setTimeout(resolve, 10000));
+  
+  // Test individual operations with different costs
+  logger.info('\n📊 Testing individual operations:');
+  
+  // Test cheap operation (1 point)
+  logger.info('\n1️⃣ Testing fetch-daily-prices (1 point each)...');
+  for (let i = 0; i < 5; i++) {
+    const response = await fetch(`${baseUrl}/handlers/eod/operations/fetch-daily-prices`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ test: true, index: i })
+    });
+    logger.info(`Request ${i + 1}: ${response.status} ${response.statusText}`);
+  }
+  
+  // Test medium cost operation (10 points)
+  logger.info('\n🔟 Testing fetch-fundamentals (10 points each)...');
+  for (let i = 0; i < 3; i++) {
+    const response = await fetch(`${baseUrl}/handlers/eod/operations/fetch-fundamentals`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ test: true, index: i })
+    });
+    logger.info(`Request ${i + 1}: ${response.status} ${response.statusText}`);
+  }
+  
+  // Test expensive operation (50 points)
+  logger.info('\n💰 Testing fetch-news (50 points each, custom limits)...');
+  for (let i = 0; i < 2; i++) {
+    const response = await fetch(`${baseUrl}/handlers/eod/operations/fetch-news`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ test: true, index: i })
+    });
+    logger.info(`Request ${i + 1}: ${response.status} ${response.statusText}`);
+  }
+  
+  logger.info('\n✅ Test completed!');
+  logger.info('Check the data-ingestion service logs to see rate limiting in action.');
+}
+
+// Health check before running tests
+async function checkService() {
+  try {
+    const response = await fetch('http://localhost:3001/health');
+    if (!response.ok) {
+      throw new Error('Service not healthy');
+    }
+    return true;
+  } catch (error) {
+    logger.error('Data ingestion service not running. Start it with: bun run dev');
+    return false;
+  }
+}
+
+async function main() {
+  if (await checkService()) {
+    await testAdvancedRateLimits();
+  }
+}
+
+main().catch(error => {
+  logger.error('Test failed:', error);
+  process.exit(1);
+});