diff --git a/SETUP-COMPLETE.md b/SETUP-COMPLETE.md index d8ce6af..9f9b345 100644 --- a/SETUP-COMPLETE.md +++ b/SETUP-COMPLETE.md @@ -50,10 +50,12 @@ Once running, access these services: |---------|-----|-------| | **QuestDB Console** | http://localhost:9000 | No login required | | **Redis Insight** | http://localhost:8001 | No login required | +| **Bull Board** | http://localhost:3001 | No login required | | **PgAdmin** | http://localhost:8080 | `admin@tradingbot.local` / `admin123` | | **Mongo Express** | http://localhost:8081 | `admin` / `admin123` | | **Prometheus** | http://localhost:9090 | No login required | | **Grafana** | http://localhost:3000 | `admin` / `admin123` | +| **Bull Board** | http://localhost:3001 | No login required | ## 📊 Database Connections diff --git a/apps/data-service/package.json b/apps/data-service/package.json index d56e9db..cf40b9d 100644 --- a/apps/data-service/package.json +++ b/apps/data-service/package.json @@ -21,6 +21,7 @@ "@stock-bot/questdb-client": "*", "@stock-bot/shutdown": "*", "@stock-bot/types": "*", + "bullmq": "^5.53.2", "hono": "^4.0.0", "p-limit": "^6.2.0", "ws": "^8.0.0" diff --git a/apps/data-service/src/api/queue.routes.ts b/apps/data-service/src/api/queue.routes.ts new file mode 100644 index 0000000..03990a0 --- /dev/null +++ b/apps/data-service/src/api/queue.routes.ts @@ -0,0 +1,115 @@ +/** + * Queue API Endpoints + * REST API for monitoring and controlling job queues + */ +import { Router } from 'express'; +import { getLogger } from '@stock-bot/logger'; +import { proxyQueueIntegration } from '../services/proxy-queue-integration'; + +const logger = getLogger('queue-api'); +const router = Router(); + +/** + * GET /api/queue/stats + * Get queue statistics + */ +router.get('/stats', async (req, res) => { + try { + const stats = await proxyQueueIntegration.getStats(); + res.json({ success: true, data: stats }); + } catch (error) { + logger.error('Failed to get queue stats', { error }); + res.status(500).json({ + success: false, + error: 'Failed to get queue statistics' + }); + } +}); + +/** + * POST /api/queue/proxy/fetch + * Manually trigger proxy fetching + */ +router.post('/proxy/fetch', async (req, res) => { + try { + const job = await proxyQueueIntegration.triggerProxyFetch(); + res.json({ + success: true, + data: { + jobId: job.id, + message: 'Proxy fetch job queued' + } + }); + } catch (error) { + logger.error('Failed to trigger proxy fetch', { error }); + res.status(500).json({ + success: false, + error: 'Failed to queue proxy fetch job' + }); + } +}); + +/** + * POST /api/queue/proxy/check + * Check specific proxies + */ +router.post('/proxy/check', async (req, res) => { + try { + const { proxies } = req.body; + + if (!Array.isArray(proxies) || proxies.length === 0) { + return res.status(400).json({ + success: false, + error: 'Proxies array is required' + }); + } + + const job = await proxyQueueIntegration.checkSpecificProxies(proxies); + res.json({ + success: true, + data: { + jobId: job.id, + proxiesCount: proxies.length, + message: 'Proxy check job queued' + } + }); + } catch (error) { + logger.error('Failed to queue proxy check', { error }); + res.status(500).json({ + success: false, + error: 'Failed to queue proxy check job' + }); + } +}); + +/** + * GET /api/queue/health + * Health check for queue service + */ +router.get('/health', async (req, res) => { + try { + const stats = await proxyQueueIntegration.getStats(); + const isHealthy = stats.active >= 0; // Basic health check + + res.status(isHealthy ? 200 : 503).json({ + success: isHealthy, + data: { + status: isHealthy ? 'healthy' : 'unhealthy', + stats, + timestamp: new Date().toISOString() + } + }); + } catch (error) { + logger.error('Queue health check failed', { error }); + res.status(503).json({ + success: false, + data: { + status: 'unhealthy', + error: 'Queue service unavailable', + timestamp: new Date().toISOString() + } + }); + } +}); + +export default router; diff --git a/apps/data-service/src/services/proxy-queue-integration.ts b/apps/data-service/src/services/proxy-queue-integration.ts new file mode 100644 index 0000000..b6f2a76 --- /dev/null +++ b/apps/data-service/src/services/proxy-queue-integration.ts @@ -0,0 +1,90 @@ +/** + * Example: Proxy Service with BullMQ Integration + * This shows how to integrate the queue service with your existing proxy service + */ +import { getLogger } from '@stock-bot/logger'; +import { queueService } from './queue.service'; +import type { ProxyInfo } from '@stock-bot/http'; + +const logger = getLogger('proxy-queue-integration'); + +export class ProxyQueueIntegration { + + constructor() { + // Initialize recurring tasks when service starts + this.initializeScheduledTasks(); + } + + private async initializeScheduledTasks() { + try { + await queueService.scheduleRecurringTasks(); + logger.info('Proxy scheduling tasks initialized'); + } catch (error) { + logger.error('Failed to initialize scheduled tasks', { error }); + } + } + + /** + * Manually trigger proxy fetching and checking + */ + async triggerProxyFetch() { + try { + const job = await queueService.addManualProxyFetch(); + logger.info('Manual proxy fetch job added', { jobId: job.id }); + return job; + } catch (error) { + logger.error('Failed to trigger proxy fetch', { error }); + throw error; + } + } + + /** + * Check specific proxies immediately + */ + async checkSpecificProxies(proxies: ProxyInfo[]) { + try { + const job = await queueService.addImmediateProxyCheck(proxies); + logger.info('Specific proxy check job added', { + jobId: job.id, + proxiesCount: proxies.length + }); + return job; + } catch (error) { + logger.error('Failed to check specific proxies', { error }); + throw error; + } + } + + /** + * Get queue statistics + */ + async getStats() { + try { + return await queueService.getQueueStats(); + } catch (error) { + logger.error('Failed to get queue stats', { error }); + throw error; + } + } + + /** + * Get the queue instance for Bull Board monitoring + */ + async getQueue() { + return await queueService.getQueue(); + } + + /** + * Shutdown queue service gracefully + */ + async shutdown() { + try { + await queueService.shutdown(); + logger.info('Proxy queue integration shut down'); + } catch (error) { + logger.error('Error during shutdown', { error }); + } + } +} + +export const proxyQueueIntegration = new ProxyQueueIntegration(); diff --git a/apps/data-service/src/services/proxy.service.ts b/apps/data-service/src/services/proxy.service.ts index e2bf0bc..d61b594 100644 --- a/apps/data-service/src/services/proxy.service.ts +++ b/apps/data-service/src/services/proxy.service.ts @@ -2,15 +2,16 @@ import { Logger } from '@stock-bot/logger'; import createCache, { type CacheProvider } from '@stock-bot/cache'; import { HttpClient, ProxyInfo } from '@stock-bot/http'; import pLimit from 'p-limit'; +import { queueService } from './queue.service'; export class ProxyService { private logger = new Logger('proxy-service'); private cache: CacheProvider = createCache('hybrid'); private httpClient: HttpClient; - private readonly concurrencyLimit = pLimit(250); + private readonly concurrencyLimit = pLimit(100); private readonly CACHE_KEY = 'proxy'; private readonly CACHE_TTL = 86400; // 24 hours - private readonly CHECK_TIMEOUT = 5000; + private readonly CHECK_TIMEOUT = 7000; private readonly CHECK_IP = '99.246.102.205' private readonly CHECK_URL = 'https://proxy-detection.stare.gg/?api_key=bd406bf53ddc6abe1d9de5907830a955'; private readonly PROXY_SOURCES = [ @@ -20,13 +21,10 @@ export class ProxyService { {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',protocol: 'http', }, - // {url: 'https://github.com/zloi-user/hideip.me/raw/refs/heads/master/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http' }, {url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', }, - // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',protocol: 'http', }, @@ -35,17 +33,15 @@ export class ProxyService { {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',protocol: 'http', }, {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',protocol: 'http', }, - {url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http',protocol: 'http', }, + // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/http.txt',protocol: 'http', }, - - {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', }, {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https' }, {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', }, {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', }, - // {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', }, + {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', }, + {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', }, // {url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/socks4.txt',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt',protocol: 'socks4', }, // {url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/socks4.txt',protocol: 'socks4', }, @@ -68,12 +64,24 @@ export class ProxyService { constructor() { this.httpClient = new HttpClient({ - timeout: this.CHECK_TIMEOUT, + timeout: 10000, }, this.logger); + + // Start scheduled tasks + this.initializeScheduling(); this.logger.info('ProxyService initialized'); } + private async initializeScheduling() { + try { + await queueService.scheduleRecurringTasks(); + this.logger.info('Proxy scheduling initialized'); + } catch (error) { + this.logger.error('Failed to initialize scheduling', { error }); + } + } + async fetchProxiesFromSources() : Promise { const sources = this.PROXY_SOURCES.map(source => diff --git a/apps/data-service/src/services/queue.service.ts b/apps/data-service/src/services/queue.service.ts new file mode 100644 index 0000000..20fe361 --- /dev/null +++ b/apps/data-service/src/services/queue.service.ts @@ -0,0 +1,198 @@ +/** + * BullMQ Queue Service + * Handles job scheduling and processing for the data service + */ +import { Queue, Worker, QueueEvents } from 'bullmq'; +import { getLogger } from '@stock-bot/logger'; +import type { ProxyInfo } from '@stock-bot/http'; + +const logger = getLogger('queue-service'); + +export interface ProxyJobData { + type: 'fetch-and-check' | 'check-specific' | 'clear-cache'; + proxies?: ProxyInfo[]; +} + +export class QueueService { + private queue: Queue; + private worker: Worker; + private queueEvents: QueueEvents; + + constructor() { + const connection = { + host: process.env.DRAGONFLY_HOST || 'localhost', + port: parseInt(process.env.DRAGONFLY_PORT || '6379'), + }; + + // Create queue + this.queue = new Queue('proxy-tasks', { connection }); + + // Create worker + this.worker = new Worker('proxy-tasks', this.processJob.bind(this), { + connection, + concurrency: 3, + }); + + // Create queue events for monitoring + this.queueEvents = new QueueEvents('proxy-tasks', { connection }); + + this.setupEventListeners(); + logger.info('Queue service initialized', { connection }); + } + + private async processJob(job: any) { + const { type, proxies }: ProxyJobData = job.data; + logger.info('Processing job', { + id: job.id, + type, + proxiesCount: proxies?.length + }); + + try { + switch (type) { + case 'fetch-and-check': + // Import proxy service dynamically to avoid circular dependencies + const { proxyService } = await import('./proxy.service'); + return await proxyService.fetchProxiesFromSources(); + + case 'check-specific': + if (!proxies) throw new Error('Proxies required for check-specific job'); + const { proxyService: ps } = await import('./proxy.service'); + return await ps.checkProxies(proxies); + + case 'clear-cache': + // Clear proxy cache + const { proxyService: pcs } = await import('./proxy.service'); + // Assuming you have a clearCache method + // return await pcs.clearCache(); + logger.info('Cache clear job processed'); + return { cleared: true }; + + default: + throw new Error(`Unknown job type: ${type}`); + } + } catch (error) { + logger.error('Job processing failed', { + id: job.id, + type, + error: error instanceof Error ? error.message : String(error) + }); + throw error; + } + } + + private setupEventListeners() { + this.worker.on('completed', (job) => { + logger.info('Job completed', { + id: job.id, + type: job.data.type, + result: job.returnvalue + }); + }); + + this.worker.on('failed', (job, err) => { + logger.error('Job failed', { + id: job?.id, + type: job?.data.type, + error: err.message + }); + }); + + this.worker.on('progress', (job, progress) => { + logger.debug('Job progress', { + id: job.id, + progress: `${progress}%` + }); + }); + + this.queueEvents.on('waiting', ({ jobId }) => { + logger.debug('Job waiting', { jobId }); + }); + + this.queueEvents.on('active', ({ jobId }) => { + logger.debug('Job active', { jobId }); + }); + } + + async scheduleRecurringTasks() { + // Fetch and check proxies every 15 minutes + await this.queue.add('fetch-and-check', + { type: 'fetch-and-check' }, + { + repeat: { pattern: '*/15 * * * *' }, + removeOnComplete: 10, + removeOnFail: 5, + jobId: 'recurring-proxy-fetch', // Use consistent ID to prevent duplicates + } + ); + + // Clear cache daily at midnight + await this.queue.add('clear-cache', + { type: 'clear-cache' }, + { + repeat: { pattern: '0 0 * * *' }, + removeOnComplete: 1, + removeOnFail: 1, + jobId: 'daily-cache-clear', + } + ); + + logger.info('Recurring tasks scheduled'); + } + + async addImmediateProxyCheck(proxies: ProxyInfo[]) { + return await this.queue.add('check-specific', + { type: 'check-specific', proxies }, + { + priority: 10, + removeOnComplete: 5, + removeOnFail: 3, + } + ); + } + + async addManualProxyFetch() { + return await this.queue.add('fetch-and-check', + { type: 'fetch-and-check' }, + { + priority: 5, + removeOnComplete: 5, + removeOnFail: 3, + } + ); + } + + async getQueueStats() { + const [waiting, active, completed, failed, delayed] = await Promise.all([ + this.queue.getWaiting(), + this.queue.getActive(), + this.queue.getCompleted(), + this.queue.getFailed(), + this.queue.getDelayed(), + ]); + + return { + waiting: waiting.length, + active: active.length, + completed: completed.length, + failed: failed.length, + delayed: delayed.length, + }; + } + + async getQueue() { + return this.queue; + } + + async shutdown() { + logger.info('Shutting down queue service...'); + + await this.worker.close(); + await this.queue.close(); + await this.queueEvents.close(); + + logger.info('Queue service shut down'); + } +} + +export const queueService = new QueueService(); diff --git a/bun.lock b/bun.lock index 4a6cc83..932e503 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,7 @@ "": { "name": "stock-bot", "dependencies": { + "bullmq": "^5.53.2", "envalid": "^8.0.0", "valibot": "^1.1.0", }, @@ -70,6 +71,7 @@ "@stock-bot/questdb-client": "*", "@stock-bot/shutdown": "*", "@stock-bot/types": "*", + "bullmq": "^5.53.2", "hono": "^4.0.0", "p-limit": "^6.2.0", "ws": "^8.0.0", @@ -1002,6 +1004,8 @@ "buildcheck": ["buildcheck@0.0.6", "", {}, "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A=="], + "bullmq": ["bullmq@5.53.2", "", { "dependencies": { "cron-parser": "^4.9.0", "ioredis": "^5.4.1", "msgpackr": "^1.11.2", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", "tslib": "^2.0.0", "uuid": "^9.0.0" } }, "sha512-xHgxrP/yNJHD7VCw1h+eRBh+2TCPBCM39uC9gCyksYc6ufcJP+HTZ/A2lzB2x7qMFWrvsX7tM40AT2BmdkYL/Q=="], + "bun-types": ["bun-types@1.2.15", "", { "dependencies": { "@types/node": "*" } }, "sha512-NarRIaS+iOaQU1JPfyKhZm4AsUOrwUOqRNHY0XxI8GI8jYxiLXLcdjYMG9UKS+fwWasc1uw1htV9AX24dD+p4w=="], "byline": ["byline@5.0.0", "", {}, "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q=="], @@ -1084,6 +1088,8 @@ "crc32-stream": ["crc32-stream@6.0.0", "", { "dependencies": { "crc-32": "^1.2.0", "readable-stream": "^4.0.0" } }, "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g=="], + "cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "css-select": ["css-select@5.1.0", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg=="], @@ -1524,6 +1530,8 @@ "lru-cache": ["lru-cache@6.0.0", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA=="], + "luxon": ["luxon@3.6.1", "", {}, "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ=="], + "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="], "make-dir": ["make-dir@4.0.0", "", { "dependencies": { "semver": "^7.5.3" } }, "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw=="], @@ -1610,6 +1618,8 @@ "new-find-package-json": ["new-find-package-json@2.0.0", "", { "dependencies": { "debug": "^4.3.4" } }, "sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew=="], + "node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="], + "node-addon-api": ["node-addon-api@6.1.0", "", {}, "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="], "node-gyp": ["node-gyp@11.2.0", "", { "dependencies": { "env-paths": "^2.2.0", "exponential-backoff": "^3.1.1", "graceful-fs": "^4.2.6", "make-fetch-happen": "^14.0.3", "nopt": "^8.0.0", "proc-log": "^5.0.0", "semver": "^7.3.5", "tar": "^7.4.3", "tinyglobby": "^0.2.12", "which": "^5.0.0" }, "bin": { "node-gyp": "bin/node-gyp.js" } }, "sha512-T0S1zqskVUSxcsSTkAsLc7xCycrRYmtDHadDinzocrThjyQCn5kMlEBSj6H4qDbgsIOSLmmlRIeb0lZXj+UArA=="], @@ -2042,7 +2052,7 @@ "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], - "uuid": ["uuid@10.0.0", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="], + "uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], "valibot": ["valibot@1.1.0", "", { "peerDependencies": { "typescript": ">=5" }, "optionalPeers": ["typescript"] }, "sha512-Nk8lX30Qhu+9txPYTwM0cFlWLdPFsFr6LblzqIySfbZph9+BFsAHsNvHOymEviUepeIW6KFHzpX8TKhbptBXXw=="], @@ -2226,6 +2236,8 @@ "dockerode/tar-fs": ["tar-fs@2.1.3", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg=="], + "dockerode/uuid": ["uuid@10.0.0", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="], + "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], "encoding/iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], diff --git a/docker-compose.yml b/docker-compose.yml index 7f754c4..7734bd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,11 +60,6 @@ services: environment: - QDB_TELEMETRY_ENABLED=false restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/status"] - interval: 30s - timeout: 10s - retries: 3 networks: - trading-bot-network @@ -179,7 +174,6 @@ services: restart: unless-stopped networks: - trading-bot-network - # Grafana - Visualization for logs and metrics grafana: image: grafana/grafana:10.2.0 @@ -201,6 +195,24 @@ services: networks: - trading-bot-network + # Bull Board - Queue monitoring + bull-board: + image: deadly0/bull-board + container_name: trading-bot-bull-board + ports: + - "3001:3000" + environment: + - REDIS_HOST=dragonfly + - REDIS_PORT=6379 + - REDIS_PASSWORD= + - REDIS_DB=0 + - REDIS_URL=redis://dragonfly:6379 + depends_on: + - dragonfly + restart: unless-stopped + networks: + - trading-bot-network + volumes: postgres_data: questdb_data: diff --git a/package.json b/package.json index 80afefa..4f6d204 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "clean": "turbo run clean", "clean:cache": "powershell ./scripts/clean.ps1 -cache", "clean:dist": "powershell ./scripts/clean.ps1 -dist", - "clean:modules": "powershell ./scripts/clean.ps1 -modules", + "clean:modules": "powershell ./scripts/clean.ps1 -modules", "clean:all": "powershell ./scripts/clean.ps1 -all", "clean:all:force": "powershell ./scripts/clean.ps1 -all -force", "clean:fresh": "powershell ./scripts/clean.ps1 -fresh", @@ -41,7 +41,6 @@ "dev:full": "npm run infra:up && npm run docker:admin && turbo run dev", "dev:clean": "npm run infra:reset && npm run dev:full", "proxy": "bun run ./apps/data-service/src/proxy-demo.ts" - }, "workspaces": [ "libs/*", @@ -67,6 +66,7 @@ "bun": ">=1.1.0" }, "dependencies": { + "bullmq": "^5.53.2", "envalid": "^8.0.0", "valibot": "^1.1.0" },