switched to new config and removed old

This commit is contained in:
Boki 2025-06-18 21:03:45 -04:00
parent 6b69bcbcaa
commit 269364fbc8
70 changed files with 889 additions and 2978 deletions

View file

@ -1,15 +1,16 @@
import { CacheProvider, createCache } from '@stock-bot/cache';
import { getLogger } from '@stock-bot/logger';
import type { QueueManager } from './queue-manager';
import type { Queue } from './queue-instance';
import type { BatchJobData, BatchResult, JobData, ProcessOptions } from './types';
const logger = getLogger('batch-processor');
const cacheProviders = new Map<string, CacheProvider>();
function getCache(queueName: string): CacheProvider {
function getCache(queueName: string, redisConfig: any): CacheProvider {
if (!cacheProviders.has(queueName)) {
const cacheProvider = createCache({
redisConfig,
keyPrefix: `batch:${queueName}:`,
ttl: 86400, // 24 hours default
enableMetrics: true,
@ -23,11 +24,12 @@ function getCache(queueName: string): CacheProvider {
* Initialize the batch cache before any batch operations
* This should be called during application startup
*/
export async function initializeBatchCache(queueManager: QueueManager): Promise<void> {
const queueName = queueManager.getQueueName();
export async function initializeBatchCache(queue: Queue): Promise<void> {
const queueName = queue.getName();
const redisConfig = queue.getRedisConfig();
logger.info('Initializing batch cache...', { queueName });
const cache = getCache(queueName);
const cache = getCache(queueName, redisConfig);
await cache.waitForReady(10000);
logger.info('Batch cache initialized successfully', { queueName });
}
@ -38,7 +40,7 @@ export async function initializeBatchCache(queueManager: QueueManager): Promise<
*/
export async function processItems<T>(
items: T[],
queue: QueueManager,
queue: Queue,
options: ProcessOptions
): Promise<BatchResult> {
const startTime = Date.now();
@ -83,7 +85,7 @@ export async function processItems<T>(
*/
async function processDirect<T>(
items: T[],
queue: QueueManager,
queue: Queue,
options: ProcessOptions
): Promise<Omit<BatchResult, 'duration'>> {
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000; // Convert hours to milliseconds
@ -126,7 +128,7 @@ async function processDirect<T>(
*/
async function processBatched<T>(
items: T[],
queue: QueueManager,
queue: Queue,
options: ProcessOptions
): Promise<Omit<BatchResult, 'duration'>> {
const batchSize = options.batchSize || 100;
@ -186,7 +188,7 @@ async function processBatched<T>(
*/
export async function processBatchJob(
jobData: BatchJobData,
queue: QueueManager
queue: Queue
): Promise<unknown> {
const { payloadKey, batchIndex, totalBatches, itemCount } = jobData;
@ -250,14 +252,14 @@ function createBatches<T>(items: T[], batchSize: number): T[][] {
async function storeItems<T>(
items: T[],
queue: QueueManager,
queue: Queue,
options: ProcessOptions
): Promise<string> {
if (!queue) {
throw new Error('Batch cache not initialized. Call initializeBatchCache() first.');
}
const cache = getCache(queue.getQueueName());
const cache = getCache(queue.getName(), queue.getRedisConfig());
const payloadKey = `payload:${Date.now()}:${Math.random().toString(36).substr(2, 9)}`;
const payload = {
@ -280,7 +282,7 @@ async function storeItems<T>(
async function loadPayload<T>(
key: string,
queue: QueueManager
queue: Queue
): Promise<{
items: T[];
options: {
@ -295,7 +297,7 @@ async function loadPayload<T>(
throw new Error('Batch cache not initialized. Call initializeBatchCache() first.');
}
const cache = getCache(queue.getQueueName());
const cache = getCache(queue.getName(), queue.getRedisConfig());
return (await cache.get(key)) as {
items: T[];
options: {
@ -308,17 +310,17 @@ async function loadPayload<T>(
} | null;
}
async function cleanupPayload(key: string, queue: QueueManager): Promise<void> {
async function cleanupPayload(key: string, queue: Queue): Promise<void> {
if (!queue) {
throw new Error('Batch cache not initialized. Call initializeBatchCache() first.');
}
const cache = getCache(queue.getQueueName());
const cache = getCache(queue.getName(), queue.getRedisConfig());
await cache.del(key);
}
async function addJobsInChunks(
queue: QueueManager,
queue: Queue,
jobs: Array<{ name: string; data: JobData; opts?: Record<string, unknown> }>,
chunkSize = 100
): Promise<unknown[]> {