boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fixed data-service apperently

Browse source
This commit is contained in:
Boki 2025-06-19 21:55:30 -04:00
parent 4b552e454c
commit 80c29283da
11 changed files with 106 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

8
apps/data-service/src/providers/exchange-sync.provider.ts
View file

@ -3,15 +3,15 @@
*/
import { getLogger } from '@stock-bot/logger';
import type { MasterExchange } from '@stock-bot/mongodb-client';
import type { ProviderConfigWithSchedule } from '@stock-bot/queue';
import { providerRegistry } from '@stock-bot/queue';
import type { HandlerConfigWithSchedule } from '@stock-bot/queue';
import { handlerRegistry } from '@stock-bot/queue';
const logger = getLogger('exchange-sync');
export function initializeExchangeSyncProvider() {
logger.info('Registering exchange sync provider...');
const exchangeSyncConfig: ProviderConfigWithSchedule = {
const exchangeSyncConfig: HandlerConfigWithSchedule = {
name: 'exchange-sync',
operations: {
@ -40,7 +40,7 @@ export function initializeExchangeSyncProvider() {
],
};
providerRegistry.registerWithSchedule(exchangeSyncConfig);
handlerRegistry.registerWithSchedule(exchangeSyncConfig);
logger.info('Exchange sync provider registered successfully');
}

8
apps/data-service/src/providers/ib.provider.ts
View file

@ -2,8 +2,8 @@
* Interactive Brokers Provider for new queue system
*/
import { getLogger } from '@stock-bot/logger';
import type { ProviderConfigWithSchedule } from '@stock-bot/queue';
import { providerRegistry } from '@stock-bot/queue';
import type { HandlerConfigWithSchedule } from '@stock-bot/queue';
import { handlerRegistry } from '@stock-bot/queue';
const logger = getLogger('ib-provider');
@ -11,7 +11,7 @@ const logger = getLogger('ib-provider');
export function initializeIBProvider() {
logger.info('Registering IB provider with scheduled jobs...');
const ibProviderConfig: ProviderConfigWithSchedule = {
const ibProviderConfig: HandlerConfigWithSchedule = {
name: 'ib',
operations: {
'fetch-session': async _payload => {
@ -77,6 +77,6 @@ export function initializeIBProvider() {
],
};
providerRegistry.registerWithSchedule(ibProviderConfig);
handlerRegistry.registerWithSchedule(ibProviderConfig);
logger.info('IB provider registered successfully with scheduled jobs');
}

14
apps/data-service/src/providers/proxy.provider.ts
View file

@ -3,8 +3,8 @@
*/
import { ProxyInfo } from '@stock-bot/http';
import { getLogger } from '@stock-bot/logger';
import type { ProviderConfigWithSchedule } from '@stock-bot/queue';
import { providerRegistry } from '@stock-bot/queue';
import type { HandlerConfigWithSchedule } from '@stock-bot/queue';
import { handlerRegistry } from '@stock-bot/queue';
const logger = getLogger('proxy-provider');
@ -12,7 +12,7 @@ const logger = getLogger('proxy-provider');
export function initializeProxyProvider() {
logger.info('Registering proxy provider with scheduled jobs...');
const proxyProviderConfig: ProviderConfigWithSchedule = {
const proxyProviderConfig: HandlerConfigWithSchedule = {
name: 'proxy',
operations: {
@ -20,7 +20,7 @@ export function initializeProxyProvider() {
// Fetch proxies from all configured sources
logger.info('Processing fetch proxies from sources request');
const { fetchProxiesFromSources } = await import('./proxy.tasks');
const { processItems, queueManager } = await import('../index');
const { processItems } = await import('@stock-bot/queue');
// Fetch all proxies from sources
const proxies = await fetchProxiesFromSources();
@ -32,8 +32,8 @@ export function initializeProxyProvider() {
}
// Batch process the proxies through check-proxy operation
const batchResult = await processItems(proxies, queueManager, {
provider: 'proxy',
const batchResult = await processItems(proxies, 'proxy', {
handler: 'proxy',
operation: 'check-proxy',
totalDelayHours: 0.083, // 5 minutes (5/60 hours)
batchSize: 50, // Process 50 proxies per batch
@ -83,6 +83,6 @@ export function initializeProxyProvider() {
],
};
providerRegistry.registerWithSchedule(proxyProviderConfig);
handlerRegistry.registerWithSchedule(proxyProviderConfig);
logger.info('Proxy provider registered successfully with scheduled jobs');
}

18
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -310,9 +310,12 @@ async function updateProxyInCache(proxy: ProxyInfo, isWorking: boolean): Promise
// Individual task functions
export async function queueProxyFetch(): Promise<string> {
const { queueManager } = await import('../index');
const job = await queueManager.add('proxy-fetch', {
type: 'proxy-fetch',
provider: 'proxy-service',
if (!queueManager) {
throw new Error('Queue manager not initialized');
}
const queue = queueManager.getQueue('proxy');
const job = await queue.add('proxy-fetch', {
handler: 'proxy',
operation: 'fetch-and-check',
payload: {},
priority: 5,
@ -325,9 +328,12 @@ export async function queueProxyFetch(): Promise<string> {
export async function queueProxyCheck(proxies: ProxyInfo[]): Promise<string> {
const { queueManager } = await import('../index');
const job = await queueManager.add('proxy-check', {
type: 'proxy-check',
provider: 'proxy-service',
if (!queueManager) {
throw new Error('Queue manager not initialized');
}
const queue = queueManager.getQueue('proxy');
const job = await queue.add('proxy-check', {
handler: 'proxy',
operation: 'check-specific',
payload: { proxies },
priority: 3,

6
apps/data-service/src/providers/qm.provider.ts
View file

@ -1,5 +1,5 @@
import { getLogger } from '@stock-bot/logger';
import { providerRegistry, type ProviderConfigWithSchedule } from '@stock-bot/queue';
import { handlerRegistry, type HandlerConfigWithSchedule } from '@stock-bot/queue';
import type { SymbolSpiderJob } from './qm.tasks';
const logger = getLogger('qm-provider');
@ -8,7 +8,7 @@ const logger = getLogger('qm-provider');
export function initializeQMProvider() {
logger.info('Registering IB provider with scheduled jobs...');
const qmProviderConfig: ProviderConfigWithSchedule = {
const qmProviderConfig: HandlerConfigWithSchedule = {
name: 'qm',
operations: {
'create-sessions': async () => {
@ -77,6 +77,6 @@ export function initializeQMProvider() {
],
};
providerRegistry.registerWithSchedule(qmProviderConfig);
handlerRegistry.registerWithSchedule(qmProviderConfig);
logger.info('IB provider registered successfully with scheduled jobs');
}

20
apps/data-service/src/providers/qm.tasks.ts
View file

@ -172,6 +172,10 @@ async function createAlphabetJobs(
): Promise<{ success: boolean; symbolsFound: number; jobsCreated: number }> {
try {
const { queueManager } = await import('../index');
if (!queueManager) {
throw new Error('Queue manager not initialized');
}
const queue = queueManager.getQueue('qm');
let jobsCreated = 0;
// Create jobs for A-Z
@ -185,10 +189,10 @@ async function createAlphabetJobs(
maxDepth,
};
await queueManager.add(
'qm',
await queue.add(
'spider-symbol-search',
{
provider: 'qm',
handler: 'qm',
operation: 'spider-symbol-search',
payload: job,
},
@ -239,6 +243,10 @@ async function searchAndSpawnJobs(
// If we have 50+ symbols and haven't reached max depth, spawn sub-jobs
if (symbolCount >= 50 && depth < maxDepth) {
const { queueManager } = await import('../index');
if (!queueManager) {
throw new Error('Queue manager not initialized');
}
const queue = queueManager.getQueue('qm');
logger.info(`Spawning sub-jobs for prefix "${prefix}" (${symbolCount} >= 50 symbols)`);
@ -254,10 +262,10 @@ async function searchAndSpawnJobs(
maxDepth,
};
await queueManager.add(
'qm',
await queue.add(
'spider-symbol-search',
{
provider: 'qm',
handler: 'qm',
operation: 'spider-symbol-search',
payload: job,
},

8
apps/data-service/src/providers/webshare.provider.ts
View file

@ -2,8 +2,8 @@
* WebShare Provider for proxy management
*/
import { getLogger } from '@stock-bot/logger';
import type { ProviderConfigWithSchedule } from '@stock-bot/queue';
import { providerRegistry } from '@stock-bot/queue';
import type { HandlerConfigWithSchedule } from '@stock-bot/queue';
import { handlerRegistry } from '@stock-bot/queue';
const logger = getLogger('webshare-provider');
@ -26,7 +26,7 @@ export function getProxy(): string | null {
export function initializeWebShareProvider() {
logger.info('Registering WebShare provider with scheduled jobs...');
const webShareProviderConfig: ProviderConfigWithSchedule = {
const webShareProviderConfig: HandlerConfigWithSchedule = {
name: 'webshare',
operations: {
@ -83,7 +83,7 @@ export function initializeWebShareProvider() {
};
// Register the provider
providerRegistry.registerWithSchedule(webShareProviderConfig);
handlerRegistry.registerWithSchedule(webShareProviderConfig);
logger.info('WebShare provider registered successfully');
}