switching to generic queue lib
This commit is contained in:
parent
6c548416d1
commit
e5170b1c78
15 changed files with 500 additions and 1086 deletions
|
|
@ -1,42 +1,82 @@
|
|||
/**
|
||||
* Interactive Brokers Provider for new queue system
|
||||
*/
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
import type { ProviderConfigWithSchedule } from '@stock-bot/queue';
|
||||
import { providerRegistry } from '@stock-bot/queue';
|
||||
|
||||
const logger = getLogger('ib-provider');
|
||||
|
||||
export const ibProvider: ProviderConfig = {
|
||||
name: 'ib',
|
||||
operations: {
|
||||
'ib-exchanges-and-symbols': async () => {
|
||||
const { ibTasks } = await import('./ib.tasks');
|
||||
logger.info('Fetching symbol summary from IB');
|
||||
const sessionHeaders = await ibTasks.fetchSession();
|
||||
logger.info('Fetched symbol summary from IB');
|
||||
// Initialize and register the IB provider
|
||||
export function initializeIBProvider() {
|
||||
logger.info('Registering IB provider with scheduled jobs...');
|
||||
|
||||
if (sessionHeaders) {
|
||||
logger.info('Fetching exchanges from IB');
|
||||
const exchanges = await ibTasks.fetchExchanges(sessionHeaders);
|
||||
logger.info('Fetched exchanges from IB', { count: exchanges.lenght });
|
||||
const ibProviderConfig: ProviderConfigWithSchedule = {
|
||||
name: 'ib',
|
||||
operations: {
|
||||
'fetch-session': async _payload => {
|
||||
// payload contains session configuration (not used in current implementation)
|
||||
logger.debug('Processing session fetch request');
|
||||
const { fetchSession } = await import('./ib.tasks');
|
||||
return fetchSession();
|
||||
},
|
||||
|
||||
// do the same as above but for symbols
|
||||
logger.info('Fetching symbols from IB');
|
||||
const symbols = await ibTasks.fetchSymbols(sessionHeaders);
|
||||
logger.info('Fetched symbols from IB', { symbols });
|
||||
'fetch-exchanges': async _payload => {
|
||||
// payload should contain session headers
|
||||
logger.debug('Processing exchanges fetch request');
|
||||
const { fetchSession, fetchExchanges } = await import('./ib.tasks');
|
||||
const sessionHeaders = await fetchSession();
|
||||
if (sessionHeaders) {
|
||||
return fetchExchanges(sessionHeaders);
|
||||
}
|
||||
throw new Error('Failed to get session headers');
|
||||
},
|
||||
|
||||
return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length };
|
||||
}
|
||||
'fetch-symbols': async _payload => {
|
||||
// payload should contain session headers
|
||||
logger.debug('Processing symbols fetch request');
|
||||
const { fetchSession, fetchSymbols } = await import('./ib.tasks');
|
||||
const sessionHeaders = await fetchSession();
|
||||
if (sessionHeaders) {
|
||||
return fetchSymbols(sessionHeaders);
|
||||
}
|
||||
throw new Error('Failed to get session headers');
|
||||
},
|
||||
|
||||
'ib-exchanges-and-symbols': async _payload => {
|
||||
// Legacy operation for scheduled jobs
|
||||
logger.info('Fetching symbol summary from IB');
|
||||
const { fetchSession, fetchExchanges, fetchSymbols } = await import('./ib.tasks');
|
||||
|
||||
const sessionHeaders = await fetchSession();
|
||||
logger.info('Fetched symbol summary from IB');
|
||||
|
||||
if (sessionHeaders) {
|
||||
logger.info('Fetching exchanges from IB');
|
||||
const exchanges = await fetchExchanges(sessionHeaders);
|
||||
logger.info('Fetched exchanges from IB', { count: exchanges?.length });
|
||||
|
||||
logger.info('Fetching symbols from IB');
|
||||
const symbols = await fetchSymbols(sessionHeaders);
|
||||
logger.info('Fetched symbols from IB', { symbols });
|
||||
|
||||
return { exchangesCount: exchanges?.length, symbolsCount: symbols?.length };
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'ib-exchanges-and-symbols',
|
||||
operation: 'ib-exchanges-and-symbols',
|
||||
payload: {},
|
||||
cronPattern: '0 0 * * 0', // Every Sunday at midnight
|
||||
priority: 5,
|
||||
description: 'Fetch and update IB exchanges and symbols data',
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'ib-exchanges-and-symbols',
|
||||
operation: 'ib-exchanges-and-symbols',
|
||||
payload: {},
|
||||
// should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
cronPattern: '0 0 * * 0',
|
||||
priority: 5,
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
description: 'Fetch and validate proxy list from sources',
|
||||
},
|
||||
],
|
||||
};
|
||||
providerRegistry.registerWithSchedule(ibProviderConfig);
|
||||
logger.info('IB provider registered successfully with scheduled jobs');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,98 +1,48 @@
|
|||
import { ProxyInfo } from 'libs/http/src/types';
|
||||
/**
|
||||
* Proxy Provider for new queue system
|
||||
*/
|
||||
import { ProxyInfo } from '@stock-bot/http';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
import type { ProviderConfigWithSchedule } from '@stock-bot/queue';
|
||||
import { providerRegistry } from '@stock-bot/queue';
|
||||
|
||||
// Create logger for this provider
|
||||
const logger = getLogger('proxy-provider');
|
||||
|
||||
// This will run at the same time each day as when the app started
|
||||
const getEvery24HourCron = (): string => {
|
||||
const now = new Date();
|
||||
const hours = now.getHours();
|
||||
const minutes = now.getMinutes();
|
||||
return `${minutes} ${hours} * * *`; // Every day at startup time
|
||||
};
|
||||
// Initialize and register the Proxy provider
|
||||
export function initializeProxyProvider() {
|
||||
logger.info('Registering proxy provider with scheduled jobs...');
|
||||
|
||||
export const proxyProvider: ProviderConfig = {
|
||||
name: 'proxy-provider',
|
||||
operations: {
|
||||
'fetch-and-check': async (_payload: { sources?: string[] }) => {
|
||||
const { proxyService } = await import('./proxy.tasks');
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
const { processItems } = await import('../utils/batch-helpers');
|
||||
|
||||
const proxies = await proxyService.fetchProxiesFromSources();
|
||||
|
||||
if (proxies.length === 0) {
|
||||
return { proxiesFetched: 0, jobsCreated: 0 };
|
||||
}
|
||||
|
||||
// Use generic function with routing parameters
|
||||
const result = await processItems(
|
||||
proxies,
|
||||
(proxy, index) => ({
|
||||
proxy,
|
||||
index,
|
||||
source: 'batch-processing',
|
||||
}),
|
||||
queueManager,
|
||||
{
|
||||
totalDelayHours: 12, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
|
||||
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
|
||||
useBatching: process.env.PROXY_DIRECT_MODE !== 'true',
|
||||
provider: 'proxy-provider',
|
||||
operation: 'check-proxy',
|
||||
}
|
||||
);
|
||||
return result;
|
||||
},
|
||||
'process-batch-items': async (payload: any) => {
|
||||
// Process a batch using the simplified batch helpers
|
||||
const { processBatchJob } = await import('../utils/batch-helpers');
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
|
||||
return await processBatchJob(payload, queueManager);
|
||||
},
|
||||
|
||||
'check-proxy': async (payload: {
|
||||
proxy: ProxyInfo;
|
||||
source?: string;
|
||||
batchIndex?: number;
|
||||
itemIndex?: number;
|
||||
total?: number;
|
||||
}) => {
|
||||
const { checkProxy } = await import('./proxy.tasks');
|
||||
|
||||
try {
|
||||
const result = await checkProxy(payload.proxy);
|
||||
|
||||
logger.debug('Proxy validated', {
|
||||
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
||||
isWorking: result.isWorking,
|
||||
responseTime: result.responseTime,
|
||||
const proxyProviderConfig: ProviderConfigWithSchedule = {
|
||||
name: 'proxy',
|
||||
operations: {
|
||||
'check-proxy': async (payload: ProxyInfo) => {
|
||||
// payload is now the raw proxy info object
|
||||
logger.debug('Processing proxy check request', {
|
||||
proxy: `${payload.host}:${payload.port}`,
|
||||
});
|
||||
|
||||
return { result, proxy: payload.proxy };
|
||||
} catch (error) {
|
||||
logger.warn('Proxy validation failed', {
|
||||
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
|
||||
return { result: { isWorking: false, error: String(error) }, proxy: payload.proxy };
|
||||
}
|
||||
const { checkProxy } = await import('./proxy.tasks');
|
||||
return checkProxy(payload);
|
||||
},
|
||||
'fetch-from-sources': async _payload => {
|
||||
// Fetch proxies from all configured sources
|
||||
logger.info('Processing fetch proxies from sources request');
|
||||
const { fetchProxiesFromSources } = await import('./proxy.tasks');
|
||||
return fetchProxiesFromSources();
|
||||
},
|
||||
},
|
||||
},
|
||||
scheduledJobs: [
|
||||
// {
|
||||
// type: 'proxy-maintenance',
|
||||
// operation: 'fetch-and-check',
|
||||
// payload: {},
|
||||
// // should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
// cronPattern: getEvery24HourCron(),
|
||||
// priority: 5,
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
// description: 'Fetch and validate proxy list from sources',
|
||||
// },
|
||||
],
|
||||
};
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'proxy-fetch-and-check',
|
||||
operation: 'fetch-from-sources',
|
||||
payload: {},
|
||||
cronPattern: '0 */2 * * *', // Every 2 hours
|
||||
priority: 5,
|
||||
description: 'Fetch and validate proxy list from sources',
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
providerRegistry.registerWithSchedule(proxyProviderConfig);
|
||||
logger.info('Proxy provider registered successfully with scheduled jobs');
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue