added redis connections script

This commit is contained in:
Boki 2025-06-10 12:56:22 -04:00
parent f52e3332df
commit ee57b66391
5 changed files with 3115 additions and 2610 deletions

View file

@ -16,8 +16,8 @@ export interface ProxySource {
// Shared configuration and utilities
const PROXY_CONFIG = {
CACHE_KEY: 'proxy',
CACHE_STATS_KEY: 'proxy:stats',
CACHE_KEY: 'active',
CACHE_STATS_KEY: 'stats',
CACHE_TTL: 86400, // 24 hours
CHECK_TIMEOUT: 7000,
CHECK_IP: '99.246.102.205',

5070
bun.lock

File diff suppressed because it is too large Load diff

View file

@ -1,75 +1,76 @@
{
"name": "stock-bot",
"private": true,
"version": "1.0.0",
"description": "Advanced trading bot with microservice architecture",
"type": "module",
"scripts": {
"dev": "turbo run dev",
"build": "powershell ./scripts/build-all.ps1",
"build:all:clean": "powershell ./scripts/build-all.ps1 -Clean",
"build:all:verbose": "powershell ./scripts/build-all.ps1 -Verbose",
"build:libs": "powershell ./scripts/build-libs.ps1",
"test": "turbo run test",
"test:watch": "bun test --watch",
"test:coverage": "bun test --coverage",
"test:unit": "bun test test/unit",
"test:integration": "bun test test/integration",
"test:e2e": "bun test test/e2e",
"test:libs": "turbo run test --filter='./libs/*'",
"test:apps": "turbo run test --filter=./apps/*/*",
"lint": "turbo run lint",
"start": "turbo run start",
"clean": "turbo run clean",
"clean:cache": "powershell ./scripts/clean.ps1 -cache",
"clean:dist": "powershell ./scripts/clean.ps1 -dist",
"clean:modules": "powershell ./scripts/clean.ps1 -modules",
"clean:all": "powershell ./scripts/clean.ps1 -all",
"clean:all:force": "powershell ./scripts/clean.ps1 -all -force",
"clean:fresh": "powershell ./scripts/clean.ps1 -fresh",
"clean:fresh:force": "powershell ./scripts/clean.ps1 -fresh -force",
"backtest": "turbo run backtest",
"docker:start": "powershell ./scripts/docker.ps1 start",
"docker:stop": "powershell ./scripts/docker.ps1 stop",
"docker:restart": "powershell ./scripts/docker.ps1 restart",
"docker:status": "powershell ./scripts/docker.ps1 status",
"docker:logs": "powershell ./scripts/docker.ps1 logs",
"docker:reset": "powershell ./scripts/docker.ps1 reset",
"docker:admin": "powershell ./scripts/docker.ps1 admin",
"docker:monitoring": "powershell ./scripts/docker.ps1 monitoring",
"infra:up": "docker-compose up -d dragonfly postgres questdb mongodb",
"infra:down": "docker-compose down",
"infra:reset": "docker-compose down -v && docker-compose up -d dragonfly postgres questdb mongodb",
"dev:full": "npm run infra:up && npm run docker:admin && turbo run dev",
"dev:clean": "npm run infra:reset && npm run dev:full",
"proxy": "bun run ./apps/data-service/src/proxy-demo.ts"
},
"workspaces": [
"libs/*",
"apps/*"
],
"devDependencies": {
"@testcontainers/mongodb": "^10.7.2",
"@testcontainers/postgresql": "^10.7.2",
"@types/bun": "latest",
"@types/node": "^22.15.30",
"@types/supertest": "^6.0.2",
"@types/yup": "^0.32.0",
"bun-types": "^1.2.15",
"mongodb-memory-server": "^9.1.6",
"pg-mem": "^2.8.1",
"supertest": "^6.3.4",
"turbo": "^2.5.4",
"typescript": "^5.8.3",
"yup": "^1.6.1"
},
"packageManager": "bun@1.1.12",
"engines": {
"node": ">=18.0.0",
"bun": ">=1.1.0"
},
"dependencies": {
"bullmq": "^5.53.2",
},
"trustedDependencies": ["mongodb"]
}
{
"name": "stock-bot",
"private": true,
"version": "1.0.0",
"description": "Advanced trading bot with microservice architecture",
"type": "module",
"scripts": {
"dev": "turbo run dev",
"build": "powershell ./scripts/build-all.ps1",
"build:all:clean": "powershell ./scripts/build-all.ps1 -Clean",
"build:all:verbose": "powershell ./scripts/build-all.ps1 -Verbose",
"build:libs": "powershell ./scripts/build-libs.ps1",
"test": "turbo run test",
"test:watch": "bun test --watch",
"test:coverage": "bun test --coverage",
"test:unit": "bun test test/unit",
"test:integration": "bun test test/integration",
"test:e2e": "bun test test/e2e",
"test:libs": "turbo run test --filter='./libs/*'",
"test:apps": "turbo run test --filter=./apps/*/*",
"lint": "turbo run lint",
"start": "turbo run start",
"clean": "turbo run clean",
"clean:cache": "powershell ./scripts/clean.ps1 -cache",
"clean:dist": "powershell ./scripts/clean.ps1 -dist",
"clean:modules": "powershell ./scripts/clean.ps1 -modules",
"clean:all": "powershell ./scripts/clean.ps1 -all",
"clean:all:force": "powershell ./scripts/clean.ps1 -all -force",
"clean:fresh": "powershell ./scripts/clean.ps1 -fresh",
"clean:fresh:force": "powershell ./scripts/clean.ps1 -fresh -force",
"backtest": "turbo run backtest",
"docker:start": "powershell ./scripts/docker.ps1 start",
"docker:stop": "powershell ./scripts/docker.ps1 stop",
"docker:restart": "powershell ./scripts/docker.ps1 restart",
"docker:status": "powershell ./scripts/docker.ps1 status",
"docker:logs": "powershell ./scripts/docker.ps1 logs",
"docker:reset": "powershell ./scripts/docker.ps1 reset",
"docker:admin": "powershell ./scripts/docker.ps1 admin",
"docker:monitoring": "powershell ./scripts/docker.ps1 monitoring",
"infra:up": "docker-compose up -d dragonfly postgres questdb mongodb",
"infra:down": "docker-compose down",
"infra:reset": "docker-compose down -v && docker-compose up -d dragonfly postgres questdb mongodb",
"dev:full": "npm run infra:up && npm run docker:admin && turbo run dev",
"dev:clean": "npm run infra:reset && npm run dev:full",
"proxy": "bun run ./apps/data-service/src/proxy-demo.ts"
},
"workspaces": [
"libs/*",
"apps/*"
],
"devDependencies": {
"@testcontainers/mongodb": "^10.7.2",
"@testcontainers/postgresql": "^10.7.2",
"@types/bun": "latest",
"@types/node": "^22.15.30",
"@types/supertest": "^6.0.2",
"@types/yup": "^0.32.0",
"bun-types": "^1.2.15",
"mongodb-memory-server": "^9.1.6",
"pg-mem": "^2.8.1",
"supertest": "^6.3.4",
"turbo": "^2.5.4",
"typescript": "^5.8.3",
"yup": "^1.6.1"
},
"packageManager": "bun@1.1.12",
"engines": {
"node": ">=18.0.0",
"bun": ">=1.1.0"
},
"dependencies": {
"bullmq": "^5.53.2",
"ioredis": "^5.6.1"
},
"trustedDependencies": ["mongodb"]
}

View file

@ -0,0 +1,112 @@
#!/bin/bash
# Simple script to get Redis/Dragonfly connections using redis-cli
# This provides a quick alternative to the TypeScript version
set -e
# Configuration
REDIS_HOST="${DRAGONFLY_HOST:-localhost}"
REDIS_PORT="${DRAGONFLY_PORT:-6379}"
REDIS_PASSWORD="${DRAGONFLY_PASSWORD:-}"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
echo -e "${BLUE}🐉 Dragonfly/Redis Connection Monitor${NC}"
echo "========================================"
# Check if redis-cli is available
if ! command -v redis-cli &> /dev/null; then
echo -e "${RED}❌ redis-cli not found. Please install redis-tools:${NC}"
echo " Ubuntu/Debian: sudo apt-get install redis-tools"
echo " macOS: brew install redis"
echo " Or use the TypeScript version: bun run scripts/get-redis-connections.ts"
exit 1
fi
# Build redis-cli command
REDIS_CMD="redis-cli -h $REDIS_HOST -p $REDIS_PORT"
if [ -n "$REDIS_PASSWORD" ]; then
REDIS_CMD="$REDIS_CMD -a $REDIS_PASSWORD"
fi
echo -e "${GREEN}📡 Connecting to Dragonfly at $REDIS_HOST:$REDIS_PORT${NC}"
# Test connection
if ! $REDIS_CMD ping > /dev/null 2>&1; then
echo -e "${RED}❌ Cannot connect to Dragonfly/Redis${NC}"
echo " Make sure the Docker container is running:"
echo " docker ps | grep dragonfly"
exit 1
fi
echo -e "${GREEN}✅ Connected successfully${NC}"
echo ""
# Get client list
echo -e "${YELLOW}📋 Active Client Connections:${NC}"
echo "----------------------------------------"
CLIENT_LIST=$($REDIS_CMD CLIENT LIST)
if [ -z "$CLIENT_LIST" ]; then
echo "No active connections found"
exit 0
fi
# Count total connections
TOTAL_CONNECTIONS=$(echo "$CLIENT_LIST" | wc -l)
echo -e "${BLUE}Total Connections: $TOTAL_CONNECTIONS${NC}"
echo ""
# Group by client name (simple grouping)
echo -e "${YELLOW}🔍 Connection Details:${NC}"
echo "$CLIENT_LIST" | while IFS= read -r line; do
if [ -n "$line" ]; then
# Extract key information
ID=$(echo "$line" | grep -o 'id=[0-9]*' | cut -d'=' -f2)
ADDR=$(echo "$line" | grep -o 'addr=[^[:space:]]*' | cut -d'=' -f2)
NAME=$(echo "$line" | grep -o 'name=[^[:space:]]*' | cut -d'=' -f2 || echo "unnamed")
AGE=$(echo "$line" | grep -o 'age=[0-9]*' | cut -d'=' -f2)
IDLE=$(echo "$line" | grep -o 'idle=[0-9]*' | cut -d'=' -f2)
CMD=$(echo "$line" | grep -o 'cmd=[^[:space:]]*' | cut -d'=' -f2 || echo "none")
echo "├─ Client ID: $ID | Name: $NAME | Address: $ADDR"
echo " Age: ${AGE}s | Idle: ${IDLE}s | Last Command: $CMD"
echo ""
fi
done
# Server info
echo -e "${YELLOW}🖥️ Server Information:${NC}"
echo "----------------------------------------"
$REDIS_CMD INFO server | grep -E "(dragonfly_version|redis_version|connected_clients|uptime_in_seconds)" | head -10
echo ""
echo -e "${YELLOW}💾 Memory Usage:${NC}"
echo "----------------------------------------"
$REDIS_CMD INFO memory | grep -E "(used_memory_human|maxmemory_human|used_memory_percentage)" | head -5
echo ""
echo -e "${GREEN}✅ Connection analysis complete${NC}"
# Optional: Monitor mode
if [ "$1" = "--monitor" ] || [ "$1" = "-m" ]; then
INTERVAL=${2:-5}
echo ""
echo -e "${BLUE}🔄 Starting monitoring mode (refresh every ${INTERVAL}s)${NC}"
echo "Press Ctrl+C to stop..."
while true; do
sleep $INTERVAL
clear
echo -e "${BLUE}🕐 Last updated: $(date)${NC}"
echo ""
exec "$0" # Re-run this script
done
fi

View file

@ -0,0 +1,388 @@
#!/usr/bin/env bun
/**
* Script to get all active connections to Dragonfly/Redis and group them by client name
*
* This script connects to the actual Dragonfly Docker container and retrieves:
* - All active client connections
* - Groups them by client name/type (removes timestamp suffixes)
* - Shows connection details like IP, idle time, commands executed
*
* Usage:
* bun run scripts/get-redis-connections.ts
*
* Requirements:
* - Dragonfly Docker container must be running
* - Script assumes default connection settings (localhost:6379)
*/
import Redis from 'ioredis';
interface ClientInfo {
id: string;
name?: string;
addr: string;
fd: string;
age: string;
idle: string;
flags: string;
db: string;
sub: string;
psub: string;
multi: string;
qbuf: string;
qbufFree: string;
obl: string;
oll: string;
omem: string;
events: string;
cmd: string;
argv?: string;
tot?: string;
[key: string]: string | undefined;
}
interface GroupedConnections {
[clientName: string]: {
count: number;
connections: ClientInfo[];
totalCommands: number;
avgIdleTime: number;
};
}
class DragonFlyConnectionMonitor {
private redis: Redis;
constructor() {
// Connect to Dragonfly using the same settings as the Docker setup
this.redis = new Redis({
host: process.env.DRAGONFLY_HOST || 'localhost',
port: parseInt(process.env.DRAGONFLY_PORT || '6379'),
password: process.env.DRAGONFLY_PASSWORD || undefined,
db: parseInt(process.env.DRAGONFLY_DATABASE || '0'),
maxRetriesPerRequest: 3,
lazyConnect: true,
});
}
/**
* Parse CLIENT LIST output into structured data
*/
private parseClientList(clientListOutput: string): ClientInfo[] {
const lines = clientListOutput.trim().split('\n');
const clients: ClientInfo[] = [];
for (const line of lines) {
if (!line.trim()) continue;
const client: ClientInfo = {
id: '',
addr: '',
fd: '',
age: '',
idle: '',
flags: '',
db: '',
sub: '',
psub: '',
multi: '',
qbuf: '',
qbufFree: '',
obl: '',
oll: '',
omem: '',
events: '',
cmd: '',
};
// Parse key=value pairs
const pairs = line.split(' ');
for (const pair of pairs) {
const [key, value] = pair.split('=');
if (key && value !== undefined) {
client[key] = value;
}
}
clients.push(client);
}
return clients;
}
/**
* Clean up client names by removing timestamp/number suffixes
*/
private cleanClientName(rawName: string): string {
if (!rawName || rawName === 'unnamed') {
return rawName;
}
// Remove timestamp/number suffixes (like BATCH-PROCESSOR-1749573709724)
// Pattern: Remove dash followed by numbers at the end
let cleanName = rawName.replace(/-\d+$/, '');
// Also handle patterns like NAME---1-1- (RedisInsight style)
cleanName = cleanName.replace(/---\d+-\d+-?$/, '');
// Convert to a more readable format
cleanName = cleanName
.replace(/-/g, ' ')
.toLowerCase()
.replace(/\b\w/g, l => l.toUpperCase());
return cleanName;
}
/**
* Group connections by client name/type
*/
private groupConnectionsByName(clients: ClientInfo[]): GroupedConnections {
const grouped: GroupedConnections = {};
for (const client of clients) {
// Determine client name - use 'name' field if set, otherwise infer from other properties
let clientName = client.name || 'unnamed';
// Clean up the client name by removing timestamps/numbers
if (clientName && clientName !== 'unnamed') {
clientName = this.cleanClientName(clientName);
}
// If no name is set, try to infer from the connection characteristics
if (clientName === 'unnamed' || !clientName) {
if (client.flags.includes('M')) {
clientName = 'Master Connection';
} else if (client.flags.includes('S')) {
clientName = 'Slave Connection';
} else if (client.flags.includes('P')) {
clientName = 'PubSub Connection';
} else if (client.cmd === 'monitor') {
clientName = 'Monitor Connection';
} else if (client.addr.includes('127.0.0.1') || client.addr.includes('localhost')) {
clientName = 'Local Client';
} else if (client.addr.includes('172.') || client.addr.includes('192.168.')) {
clientName = 'Docker Network Client';
} else {
clientName = `Client ${client.addr.split(':')[0]}`;
}
}
if (!grouped[clientName]) {
grouped[clientName] = {
count: 0,
connections: [],
totalCommands: 0,
avgIdleTime: 0,
};
}
grouped[clientName].count++;
grouped[clientName].connections.push(client);
// Parse command count if available
const cmdCount = client.tot ? parseInt(client.tot) : 0;
grouped[clientName].totalCommands += cmdCount;
}
// Calculate average idle times
for (const groupName in grouped) {
const group = grouped[groupName];
const totalIdle = group.connections.reduce((sum, client) => {
return sum + parseInt(client.idle || '0');
}, 0);
group.avgIdleTime = group.count > 0 ? Math.round(totalIdle / group.count) : 0;
}
return grouped;
}
/**
* Format connection details for display
*/
private formatConnectionDetails(client: ClientInfo): string {
const details = [
`ID: ${client.id}`,
`Address: ${client.addr}`,
`Age: ${client.age}s`,
`Idle: ${client.idle}s`,
`DB: ${client.db}`,
`Flags: ${client.flags}`,
`Last Command: ${client.cmd || 'none'}`,
];
if (client.tot) {
details.push(`Total Commands: ${client.tot}`);
}
if (client.name) {
details.unshift(`Original Name: ${client.name}`);
}
return details.join(', ');
}
/**
* Get and display all connections grouped by name
*/
async getConnectionsGroupedByName(): Promise<void> {
try {
console.log('🔍 Connecting to Dragonfly...');
await this.redis.connect();
console.log('📡 Fetching client connections...');
const clientListOutput = await this.redis.client('LIST') as string;
if (!clientListOutput) {
console.log('❌ No client information available');
return;
}
const clients = this.parseClientList(clientListOutput);
const grouped = this.groupConnectionsByName(clients);
console.log('\n🎯 DRAGONFLY CONNECTION SUMMARY');
console.log('═'.repeat(60));
console.log(`Total Active Connections: ${clients.length}`);
console.log(`Connection Groups: ${Object.keys(grouped).length}`);
console.log('');
// Sort groups by count (most connections first)
const sortedGroups = Object.entries(grouped).sort((a, b) => b[1].count - a[1].count);
// Display grouped connections
for (const [groupName, group] of sortedGroups) {
console.log(`📂 ${groupName.toUpperCase()}`);
console.log(` Count: ${group.count}`);
console.log(` Total Commands Executed: ${group.totalCommands}`);
console.log(` Average Idle Time: ${group.avgIdleTime}s`);
console.log(' Connections:');
for (const client of group.connections) {
console.log(` ├─ ${this.formatConnectionDetails(client)}`);
}
console.log('');
}
// Get additional server info
console.log('🖥️ DRAGONFLY SERVER INFO');
console.log('═'.repeat(60));
const info = await this.redis.info('server');
const infoLines = info.split('\n');
for (const line of infoLines) {
if (line.includes('dragonfly_version') ||
line.includes('connected_clients') ||
line.includes('used_memory_human') ||
line.includes('uptime_in_seconds')) {
console.log(` ${line}`);
}
}
// Get memory info
console.log('\n💾 MEMORY USAGE');
console.log('═'.repeat(60));
const memoryInfo = await this.redis.info('memory');
const memoryLines = memoryInfo.split('\n');
for (const line of memoryLines) {
if (line.includes('used_memory_human') ||
line.includes('maxmemory_human') ||
line.includes('used_memory_percentage')) {
console.log(` ${line}`);
}
}
} catch (error) {
console.error('❌ Error connecting to Dragonfly:', error);
console.log('\n💡 Troubleshooting:');
console.log(' - Make sure Dragonfly Docker container is running');
console.log(' - Check if port 6379 is accessible');
console.log(' - Verify environment variables if using custom connection settings');
console.log(' - Run: docker ps | grep dragonfly');
} finally {
await this.redis.disconnect();
}
}
/**
* Monitor connections in real-time
*/
async monitorConnections(intervalSeconds: number = 5): Promise<void> {
console.log(`🔄 Starting real-time monitoring (refresh every ${intervalSeconds}s)`);
console.log('Press Ctrl+C to stop...\n');
const monitor = async () => {
console.clear();
console.log(`🕐 Last updated: ${new Date().toLocaleTimeString()}\n`);
await this.getConnectionsGroupedByName();
};
// Initial run
await monitor();
// Set up interval
const interval = setInterval(monitor, intervalSeconds * 1000);
// Handle graceful shutdown
process.on('SIGINT', () => {
clearInterval(interval);
console.log('\n👋 Monitoring stopped');
process.exit(0);
});
}
}
// Main execution
async function main() {
const args = process.argv.slice(2);
const monitor = new DragonFlyConnectionMonitor();
if (args.includes('--monitor') || args.includes('-m')) {
const intervalArg = args.find(arg => arg.startsWith('--interval='));
const interval = intervalArg ? parseInt(intervalArg.split('=')[1]) : 5;
await monitor.monitorConnections(interval);
} else {
await monitor.getConnectionsGroupedByName();
}
}
// Help text
if (process.argv.includes('--help') || process.argv.includes('-h')) {
console.log(`
🐉 Dragonfly Connection Monitor
Usage:
bun run scripts/get-redis-connections.ts [options]
Options:
--help, -h Show this help message
--monitor, -m Monitor connections in real-time
--interval=N Set monitoring refresh interval in seconds (default: 5)
Examples:
bun run scripts/get-redis-connections.ts # One-time connection list
bun run scripts/get-redis-connections.ts --monitor # Real-time monitoring
bun run scripts/get-redis-connections.ts -m --interval=10 # Monitor every 10 seconds
Environment Variables:
DRAGONFLY_HOST Dragonfly host (default: localhost)
DRAGONFLY_PORT Dragonfly port (default: 6379)
DRAGONFLY_PASSWORD Dragonfly password (if auth enabled)
DRAGONFLY_DATABASE Database number (default: 0)
Features:
Groups connections by clean names (removes timestamp suffixes)
📊 Shows connection statistics and command counts
🔄 Real-time monitoring mode available
🎨 Color-coded output with connection details
`);
process.exit(0);
}
// Run the script
console.log('🚀 Starting Redis connection monitor...');
main().catch((error) => {
console.error('❌ Script error:', error);
process.exit(1);
});