queue work
This commit is contained in:
parent
c05a7413dc
commit
d3ef73ae00
9 changed files with 938 additions and 1086 deletions
|
|
@ -11,9 +11,10 @@ export interface RateLimitConfig {
|
|||
}
|
||||
|
||||
export interface RateLimitRule {
|
||||
level: 'global' | 'handler' | 'operation';
|
||||
handler?: string;
|
||||
operation?: string;
|
||||
level: 'global' | 'queue' | 'handler' | 'operation';
|
||||
queueName?: string; // For queue-level limits
|
||||
handler?: string; // For handler-level limits
|
||||
operation?: string; // For operation-level limits (most specific)
|
||||
config: RateLimitConfig;
|
||||
}
|
||||
|
||||
|
|
@ -29,7 +30,7 @@ export class QueueRateLimiter {
|
|||
addRule(rule: RateLimitRule): void {
|
||||
this.rules.push(rule);
|
||||
|
||||
const key = this.getRuleKey(rule.level, rule.handler, rule.operation);
|
||||
const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation);
|
||||
const limiter = new RateLimiterRedis({
|
||||
storeClient: this.redisClient,
|
||||
keyPrefix: rule.config.keyPrefix || `rl:${key}`,
|
||||
|
|
@ -42,6 +43,7 @@ export class QueueRateLimiter {
|
|||
|
||||
logger.info('Rate limit rule added', {
|
||||
level: rule.level,
|
||||
queueName: rule.queueName,
|
||||
handler: rule.handler,
|
||||
operation: rule.operation,
|
||||
points: rule.config.points,
|
||||
|
|
@ -51,44 +53,77 @@ export class QueueRateLimiter {
|
|||
|
||||
/**
|
||||
* Check if a job can be processed based on rate limits
|
||||
* Uses hierarchical precedence: operation > handler > queue > global
|
||||
* The most specific matching rule takes precedence
|
||||
*/
|
||||
async checkLimit(handler: string, operation: string): Promise<{
|
||||
async checkLimit(queueName: string, handler: string, operation: string): Promise<{
|
||||
allowed: boolean;
|
||||
retryAfter?: number;
|
||||
remainingPoints?: number;
|
||||
appliedRule?: RateLimitRule;
|
||||
}> {
|
||||
const limiters = this.getApplicableLimiters(handler, operation);
|
||||
const applicableRule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (limiters.length === 0) {
|
||||
if (!applicableRule) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const key = this.getRuleKey(applicableRule.level, applicableRule.queueName, applicableRule.handler, applicableRule.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
|
||||
if (!limiter) {
|
||||
logger.warn('Rate limiter not found for rule', { key, rule: applicableRule });
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
try {
|
||||
// Check all applicable rate limiters
|
||||
const results = await Promise.all(
|
||||
limiters.map(({ limiter, key }) => this.consumePoint(limiter, key))
|
||||
);
|
||||
|
||||
// All limiters must allow the request
|
||||
const blocked = results.find(r => !r.allowed);
|
||||
if (blocked) {
|
||||
return blocked;
|
||||
}
|
||||
|
||||
// Return the most restrictive remaining points
|
||||
const minRemainingPoints = Math.min(...results.map(r => r.remainingPoints || Infinity));
|
||||
const result = await this.consumePoint(limiter, this.getConsumerKey(queueName, handler, operation));
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
remainingPoints: minRemainingPoints === Infinity ? undefined : minRemainingPoints,
|
||||
...result,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Rate limit check failed', { handler, operation, error });
|
||||
logger.error('Rate limit check failed', { queueName, handler, operation, error });
|
||||
// On error, allow the request to proceed
|
||||
return { allowed: true };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the most specific rule that applies to this job
|
||||
* Precedence: operation > handler > queue > global
|
||||
*/
|
||||
private getMostSpecificRule(queueName: string, handler: string, operation: string): RateLimitRule | undefined {
|
||||
// 1. Check for operation-specific rule (most specific)
|
||||
let rule = this.rules.find(r =>
|
||||
r.level === 'operation' &&
|
||||
r.queueName === queueName &&
|
||||
r.handler === handler &&
|
||||
r.operation === operation
|
||||
);
|
||||
if (rule) return rule;
|
||||
|
||||
// 2. Check for handler-specific rule
|
||||
rule = this.rules.find(r =>
|
||||
r.level === 'handler' &&
|
||||
r.queueName === queueName &&
|
||||
r.handler === handler
|
||||
);
|
||||
if (rule) return rule;
|
||||
|
||||
// 3. Check for queue-specific rule
|
||||
rule = this.rules.find(r =>
|
||||
r.level === 'queue' &&
|
||||
r.queueName === queueName
|
||||
);
|
||||
if (rule) return rule;
|
||||
|
||||
// 4. Check for global rule (least specific)
|
||||
rule = this.rules.find(r => r.level === 'global');
|
||||
return rule;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consume a point from the rate limiter
|
||||
*/
|
||||
|
|
@ -120,148 +155,120 @@ export class QueueRateLimiter {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get applicable rate limiters for a handler/operation
|
||||
* Get rule key for storing rate limiter
|
||||
*/
|
||||
private getApplicableLimiters(handler: string, operation: string): Array<{ limiter: RateLimiterRedis; key: string }> {
|
||||
const applicable: Array<{ limiter: RateLimiterRedis; key: string }> = [];
|
||||
|
||||
for (const rule of this.rules) {
|
||||
let applies = false;
|
||||
let consumerKey = '';
|
||||
|
||||
switch (rule.level) {
|
||||
case 'global':
|
||||
// Global limit applies to all
|
||||
applies = true;
|
||||
consumerKey = 'global';
|
||||
break;
|
||||
|
||||
case 'handler':
|
||||
// Handler limit applies if handler matches
|
||||
if (rule.handler === handler) {
|
||||
applies = true;
|
||||
consumerKey = handler;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'operation':
|
||||
// Operation limit applies if both handler and operation match
|
||||
if (rule.handler === handler && rule.operation === operation) {
|
||||
applies = true;
|
||||
consumerKey = `${handler}:${operation}`;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (applies) {
|
||||
const ruleKey = this.getRuleKey(rule.level, rule.handler, rule.operation);
|
||||
const limiter = this.limiters.get(ruleKey);
|
||||
if (limiter) {
|
||||
applicable.push({ limiter, key: consumerKey });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return applicable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get rule key
|
||||
*/
|
||||
private getRuleKey(level: string, handler?: string, operation?: string): string {
|
||||
private getRuleKey(level: string, queueName?: string, handler?: string, operation?: string): string {
|
||||
switch (level) {
|
||||
case 'global':
|
||||
return 'global';
|
||||
case 'queue':
|
||||
return `queue:${queueName}`;
|
||||
case 'handler':
|
||||
return `handler:${handler}`;
|
||||
return `handler:${queueName}:${handler}`;
|
||||
case 'operation':
|
||||
return `operation:${handler}:${operation}`;
|
||||
return `operation:${queueName}:${handler}:${operation}`;
|
||||
default:
|
||||
return level;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current rate limit status for a handler/operation
|
||||
* Get consumer key for rate limiting (what gets counted)
|
||||
*/
|
||||
async getStatus(handler: string, operation: string): Promise<{
|
||||
private getConsumerKey(queueName: string, handler: string, operation: string): string {
|
||||
return `${queueName}:${handler}:${operation}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current rate limit status for a queue/handler/operation
|
||||
*/
|
||||
async getStatus(queueName: string, handler: string, operation: string): Promise<{
|
||||
queueName: string;
|
||||
handler: string;
|
||||
operation: string;
|
||||
limits: Array<{
|
||||
appliedRule?: RateLimitRule;
|
||||
limit?: {
|
||||
level: string;
|
||||
points: number;
|
||||
duration: number;
|
||||
remaining: number;
|
||||
resetIn: number;
|
||||
}>;
|
||||
}> {
|
||||
const applicable = this.getApplicableLimiters(handler, operation);
|
||||
|
||||
const limits = await Promise.all(
|
||||
applicable.map(async ({ limiter, key }) => {
|
||||
const rule = this.rules.find(r => {
|
||||
const ruleKey = this.getRuleKey(r.level, r.handler, r.operation);
|
||||
return this.limiters.get(ruleKey) === limiter;
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await limiter.get(key);
|
||||
if (!result) {
|
||||
return {
|
||||
level: rule?.level || 'unknown',
|
||||
points: limiter.points,
|
||||
duration: limiter.duration,
|
||||
remaining: limiter.points,
|
||||
resetIn: 0,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
level: rule?.level || 'unknown',
|
||||
points: limiter.points,
|
||||
duration: limiter.duration,
|
||||
remaining: result.remainingPoints,
|
||||
resetIn: result.msBeforeNext,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
level: rule?.level || 'unknown',
|
||||
points: limiter.points,
|
||||
duration: limiter.duration,
|
||||
remaining: 0,
|
||||
resetIn: 0,
|
||||
};
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
return {
|
||||
handler,
|
||||
operation,
|
||||
limits,
|
||||
};
|
||||
}> {
|
||||
const applicableRule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (!applicableRule) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
};
|
||||
}
|
||||
|
||||
const key = this.getRuleKey(applicableRule.level, applicableRule.queueName, applicableRule.handler, applicableRule.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
|
||||
if (!limiter) {
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule: applicableRule,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const consumerKey = this.getConsumerKey(queueName, handler, operation);
|
||||
const result = await limiter.get(consumerKey);
|
||||
|
||||
const limit = {
|
||||
level: applicableRule.level,
|
||||
points: limiter.points,
|
||||
duration: limiter.duration,
|
||||
remaining: result?.remainingPoints ?? limiter.points,
|
||||
resetIn: result?.msBeforeNext ?? 0,
|
||||
};
|
||||
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule,
|
||||
limit,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get rate limit status', { queueName, handler, operation, error });
|
||||
return {
|
||||
queueName,
|
||||
handler,
|
||||
operation,
|
||||
appliedRule,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset rate limits for a handler/operation
|
||||
* Reset rate limits for a specific consumer
|
||||
*/
|
||||
async reset(handler: string, operation?: string): Promise<void> {
|
||||
const applicable = operation
|
||||
? this.getApplicableLimiters(handler, operation)
|
||||
: this.rules
|
||||
.filter(r => !handler || r.handler === handler)
|
||||
.map(r => {
|
||||
const key = this.getRuleKey(r.level, r.handler, r.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
return limiter ? { limiter, key: handler || 'global' } : null;
|
||||
})
|
||||
.filter(Boolean) as Array<{ limiter: RateLimiterRedis; key: string }>;
|
||||
async reset(queueName: string, handler?: string, operation?: string): Promise<void> {
|
||||
if (handler && operation) {
|
||||
// Reset specific operation
|
||||
const consumerKey = this.getConsumerKey(queueName, handler, operation);
|
||||
const rule = this.getMostSpecificRule(queueName, handler, operation);
|
||||
|
||||
if (rule) {
|
||||
const key = this.getRuleKey(rule.level, rule.queueName, rule.handler, rule.operation);
|
||||
const limiter = this.limiters.get(key);
|
||||
if (limiter) {
|
||||
await limiter.delete(consumerKey);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Reset broader scope - this is more complex with the new hierarchy
|
||||
logger.warn('Broad reset not implemented yet', { queueName, handler, operation });
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
applicable.map(({ limiter, key }) => limiter.delete(key))
|
||||
);
|
||||
|
||||
logger.info('Rate limits reset', { handler, operation });
|
||||
logger.info('Rate limits reset', { queueName, handler, operation });
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -274,10 +281,11 @@ export class QueueRateLimiter {
|
|||
/**
|
||||
* Remove a rate limit rule
|
||||
*/
|
||||
removeRule(level: string, handler?: string, operation?: string): boolean {
|
||||
const key = this.getRuleKey(level, handler, operation);
|
||||
removeRule(level: string, queueName?: string, handler?: string, operation?: string): boolean {
|
||||
const key = this.getRuleKey(level, queueName, handler, operation);
|
||||
const ruleIndex = this.rules.findIndex(r =>
|
||||
r.level === level &&
|
||||
(!queueName || r.queueName === queueName) &&
|
||||
(!handler || r.handler === handler) &&
|
||||
(!operation || r.operation === operation)
|
||||
);
|
||||
|
|
@ -286,7 +294,7 @@ export class QueueRateLimiter {
|
|||
this.rules.splice(ruleIndex, 1);
|
||||
this.limiters.delete(key);
|
||||
|
||||
logger.info('Rate limit rule removed', { level, handler, operation });
|
||||
logger.info('Rate limit rule removed', { level, queueName, handler, operation });
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue