switched to new config and removed old
This commit is contained in:
parent
6b69bcbcaa
commit
269364fbc8
70 changed files with 889 additions and 2978 deletions
|
|
@ -1,15 +1,16 @@
|
|||
import { CacheProvider, createCache } from '@stock-bot/cache';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import type { QueueManager } from './queue-manager';
|
||||
import type { Queue } from './queue-instance';
|
||||
import type { BatchJobData, BatchResult, JobData, ProcessOptions } from './types';
|
||||
|
||||
const logger = getLogger('batch-processor');
|
||||
|
||||
const cacheProviders = new Map<string, CacheProvider>();
|
||||
|
||||
function getCache(queueName: string): CacheProvider {
|
||||
function getCache(queueName: string, redisConfig: any): CacheProvider {
|
||||
if (!cacheProviders.has(queueName)) {
|
||||
const cacheProvider = createCache({
|
||||
redisConfig,
|
||||
keyPrefix: `batch:${queueName}:`,
|
||||
ttl: 86400, // 24 hours default
|
||||
enableMetrics: true,
|
||||
|
|
@ -23,11 +24,12 @@ function getCache(queueName: string): CacheProvider {
|
|||
* Initialize the batch cache before any batch operations
|
||||
* This should be called during application startup
|
||||
*/
|
||||
export async function initializeBatchCache(queueManager: QueueManager): Promise<void> {
|
||||
const queueName = queueManager.getQueueName();
|
||||
export async function initializeBatchCache(queue: Queue): Promise<void> {
|
||||
const queueName = queue.getName();
|
||||
const redisConfig = queue.getRedisConfig();
|
||||
logger.info('Initializing batch cache...', { queueName });
|
||||
|
||||
const cache = getCache(queueName);
|
||||
const cache = getCache(queueName, redisConfig);
|
||||
await cache.waitForReady(10000);
|
||||
logger.info('Batch cache initialized successfully', { queueName });
|
||||
}
|
||||
|
|
@ -38,7 +40,7 @@ export async function initializeBatchCache(queueManager: QueueManager): Promise<
|
|||
*/
|
||||
export async function processItems<T>(
|
||||
items: T[],
|
||||
queue: QueueManager,
|
||||
queue: Queue,
|
||||
options: ProcessOptions
|
||||
): Promise<BatchResult> {
|
||||
const startTime = Date.now();
|
||||
|
|
@ -83,7 +85,7 @@ export async function processItems<T>(
|
|||
*/
|
||||
async function processDirect<T>(
|
||||
items: T[],
|
||||
queue: QueueManager,
|
||||
queue: Queue,
|
||||
options: ProcessOptions
|
||||
): Promise<Omit<BatchResult, 'duration'>> {
|
||||
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; // Convert hours to milliseconds
|
||||
|
|
@ -126,7 +128,7 @@ async function processDirect<T>(
|
|||
*/
|
||||
async function processBatched<T>(
|
||||
items: T[],
|
||||
queue: QueueManager,
|
||||
queue: Queue,
|
||||
options: ProcessOptions
|
||||
): Promise<Omit<BatchResult, 'duration'>> {
|
||||
const batchSize = options.batchSize || 100;
|
||||
|
|
@ -186,7 +188,7 @@ async function processBatched<T>(
|
|||
*/
|
||||
export async function processBatchJob(
|
||||
jobData: BatchJobData,
|
||||
queue: QueueManager
|
||||
queue: Queue
|
||||
): Promise<unknown> {
|
||||
const { payloadKey, batchIndex, totalBatches, itemCount } = jobData;
|
||||
|
||||
|
|
@ -250,14 +252,14 @@ function createBatches<T>(items: T[], batchSize: number): T[][] {
|
|||
|
||||
async function storeItems<T>(
|
||||
items: T[],
|
||||
queue: QueueManager,
|
||||
queue: Queue,
|
||||
options: ProcessOptions
|
||||
): Promise<string> {
|
||||
if (!queue) {
|
||||
throw new Error('Batch cache not initialized. Call initializeBatchCache() first.');
|
||||
}
|
||||
|
||||
const cache = getCache(queue.getQueueName());
|
||||
const cache = getCache(queue.getName(), queue.getRedisConfig());
|
||||
const payloadKey = `payload:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`;
|
||||
|
||||
const payload = {
|
||||
|
|
@ -280,7 +282,7 @@ async function storeItems<T>(
|
|||
|
||||
async function loadPayload<T>(
|
||||
key: string,
|
||||
queue: QueueManager
|
||||
queue: Queue
|
||||
): Promise<{
|
||||
items: T[];
|
||||
options: {
|
||||
|
|
@ -295,7 +297,7 @@ async function loadPayload<T>(
|
|||
throw new Error('Batch cache not initialized. Call initializeBatchCache() first.');
|
||||
}
|
||||
|
||||
const cache = getCache(queue.getQueueName());
|
||||
const cache = getCache(queue.getName(), queue.getRedisConfig());
|
||||
return (await cache.get(key)) as {
|
||||
items: T[];
|
||||
options: {
|
||||
|
|
@ -308,17 +310,17 @@ async function loadPayload<T>(
|
|||
} | null;
|
||||
}
|
||||
|
||||
async function cleanupPayload(key: string, queue: QueueManager): Promise<void> {
|
||||
async function cleanupPayload(key: string, queue: Queue): Promise<void> {
|
||||
if (!queue) {
|
||||
throw new Error('Batch cache not initialized. Call initializeBatchCache() first.');
|
||||
}
|
||||
|
||||
const cache = getCache(queue.getQueueName());
|
||||
const cache = getCache(queue.getName(), queue.getRedisConfig());
|
||||
await cache.del(key);
|
||||
}
|
||||
|
||||
async function addJobsInChunks(
|
||||
queue: QueueManager,
|
||||
queue: Queue,
|
||||
jobs: Array<{ name: string; data: JobData; opts?: Record<string, unknown> }>,
|
||||
chunkSize = 100
|
||||
): Promise<unknown[]> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue