refactoring continuing
This commit is contained in:
parent
742e590382
commit
a0a3b26177
20 changed files with 394 additions and 798 deletions
85
apps/data-ingestion/AWILIX-MIGRATION.md
Normal file
85
apps/data-ingestion/AWILIX-MIGRATION.md
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
# Awilix DI Container Migration Guide
|
||||||
|
|
||||||
|
This guide explains how to use the new Awilix dependency injection container in the data-ingestion service.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The Awilix container provides proper dependency injection for decoupled libraries, allowing them to be reused in other projects without stock-bot specific dependencies.
|
||||||
|
|
||||||
|
## Current Implementation
|
||||||
|
|
||||||
|
The data-ingestion service now uses a hybrid approach:
|
||||||
|
1. Awilix container for ProxyManager and other decoupled services
|
||||||
|
2. Legacy service factory for backward compatibility
|
||||||
|
|
||||||
|
## Usage Example
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// Create Awilix container
|
||||||
|
const awilixConfig = {
|
||||||
|
redis: {
|
||||||
|
host: config.database.dragonfly.host,
|
||||||
|
port: config.database.dragonfly.port,
|
||||||
|
db: config.database.dragonfly.db,
|
||||||
|
},
|
||||||
|
mongodb: {
|
||||||
|
uri: config.database.mongodb.uri,
|
||||||
|
database: config.database.mongodb.database,
|
||||||
|
},
|
||||||
|
postgres: {
|
||||||
|
host: config.database.postgres.host,
|
||||||
|
port: config.database.postgres.port,
|
||||||
|
database: config.database.postgres.database,
|
||||||
|
user: config.database.postgres.user,
|
||||||
|
password: config.database.postgres.password,
|
||||||
|
},
|
||||||
|
proxy: {
|
||||||
|
cachePrefix: 'proxy:',
|
||||||
|
ttl: 3600,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const container = createServiceContainer(awilixConfig);
|
||||||
|
await initializeServices(container);
|
||||||
|
|
||||||
|
// Access services from container
|
||||||
|
const proxyManager = container.resolve('proxyManager');
|
||||||
|
const cache = container.resolve('cache');
|
||||||
|
```
|
||||||
|
|
||||||
|
## Handler Integration
|
||||||
|
|
||||||
|
Handlers receive services through the enhanced service container:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// Create service adapter with proxy from Awilix
|
||||||
|
const serviceContainerWithProxy = createServiceAdapter(services);
|
||||||
|
Object.defineProperty(serviceContainerWithProxy, 'proxy', {
|
||||||
|
get: () => container.resolve('proxyManager'),
|
||||||
|
enumerable: true,
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handlers can now access proxy service
|
||||||
|
class MyHandler extends BaseHandler {
|
||||||
|
async myOperation() {
|
||||||
|
const proxy = this.proxy.getRandomProxy();
|
||||||
|
// Use proxy...
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Benefits
|
||||||
|
|
||||||
|
1. **Decoupled Libraries**: Libraries no longer depend on @stock-bot/config
|
||||||
|
2. **Reusability**: Libraries can be used in other projects
|
||||||
|
3. **Testability**: Easy to mock dependencies for testing
|
||||||
|
4. **Type Safety**: Full TypeScript support with Awilix
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
To fully migrate to Awilix:
|
||||||
|
1. Update HTTP library to accept dependencies via constructor
|
||||||
|
2. Update Queue library to accept Redis config via constructor
|
||||||
|
3. Create actual MongoDB, PostgreSQL, and QuestDB clients in the container
|
||||||
|
4. Remove legacy service factory once all services are migrated
|
||||||
|
|
@ -3,8 +3,7 @@
|
||||||
* Automatically discovers and registers all handlers
|
* Automatically discovers and registers all handlers
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { IDataIngestionServices } from '@stock-bot/di';
|
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||||
import { createServiceAdapter } from '@stock-bot/di';
|
|
||||||
import { autoRegisterHandlers } from '@stock-bot/handlers';
|
import { autoRegisterHandlers } from '@stock-bot/handlers';
|
||||||
import { getLogger } from '@stock-bot/logger';
|
import { getLogger } from '@stock-bot/logger';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
|
|
@ -19,10 +18,7 @@ const logger = getLogger('handler-init');
|
||||||
/**
|
/**
|
||||||
* Initialize and register all handlers automatically
|
* Initialize and register all handlers automatically
|
||||||
*/
|
*/
|
||||||
export async function initializeAllHandlers(services: IDataIngestionServices): Promise<void> {
|
export async function initializeAllHandlers(serviceContainer: IServiceContainer): Promise<void> {
|
||||||
// Create generic service container adapter
|
|
||||||
const serviceContainer = createServiceAdapter(services);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Auto-register all handlers in this directory
|
// Auto-register all handlers in this directory
|
||||||
const result = await autoRegisterHandlers(
|
const result = await autoRegisterHandlers(
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,6 @@ import {
|
||||||
type ExecutionContext,
|
type ExecutionContext,
|
||||||
type IServiceContainer
|
type IServiceContainer
|
||||||
} from '@stock-bot/handlers';
|
} from '@stock-bot/handlers';
|
||||||
import { updateProxies } from '@stock-bot/utils';
|
|
||||||
|
|
||||||
@Handler('webshare')
|
@Handler('webshare')
|
||||||
export class WebShareHandler extends BaseHandler {
|
export class WebShareHandler extends BaseHandler {
|
||||||
|
|
@ -28,8 +27,8 @@ export class WebShareHandler extends BaseHandler {
|
||||||
const proxies = await fetchWebShareProxies();
|
const proxies = await fetchWebShareProxies();
|
||||||
|
|
||||||
if (proxies.length > 0) {
|
if (proxies.length > 0) {
|
||||||
// Update the centralized proxy manager
|
// Update the centralized proxy manager using the injected service
|
||||||
await updateProxies(proxies);
|
await this.proxy.updateProxies(proxies);
|
||||||
|
|
||||||
this.logger.info('Updated proxy manager with WebShare proxies', {
|
this.logger.info('Updated proxy manager with WebShare proxies', {
|
||||||
count: proxies.length,
|
count: proxies.length,
|
||||||
|
|
|
||||||
|
|
@ -10,14 +10,17 @@ import { cors } from 'hono/cors';
|
||||||
|
|
||||||
// Library imports
|
// Library imports
|
||||||
import {
|
import {
|
||||||
|
createServiceContainer,
|
||||||
|
initializeServices as initializeAwilixServices,
|
||||||
|
createServiceAdapter,
|
||||||
createDataIngestionServices,
|
createDataIngestionServices,
|
||||||
disposeDataIngestionServices,
|
disposeDataIngestionServices,
|
||||||
type IDataIngestionServices
|
type IDataIngestionServices,
|
||||||
|
type ServiceContainer
|
||||||
} from '@stock-bot/di';
|
} from '@stock-bot/di';
|
||||||
import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
|
import { getLogger, setLoggerConfig, shutdownLoggers } from '@stock-bot/logger';
|
||||||
import { Shutdown } from '@stock-bot/shutdown';
|
import { Shutdown } from '@stock-bot/shutdown';
|
||||||
import { handlerRegistry } from '@stock-bot/types';
|
import { handlerRegistry } from '@stock-bot/types';
|
||||||
import { ProxyManager } from '@stock-bot/utils';
|
|
||||||
|
|
||||||
// Local imports
|
// Local imports
|
||||||
import { createRoutes } from './routes/create-routes';
|
import { createRoutes } from './routes/create-routes';
|
||||||
|
|
@ -43,6 +46,7 @@ const logger = getLogger('data-ingestion');
|
||||||
const PORT = serviceConfig.port;
|
const PORT = serviceConfig.port;
|
||||||
let server: ReturnType<typeof Bun.serve> | null = null;
|
let server: ReturnType<typeof Bun.serve> | null = null;
|
||||||
let services: IDataIngestionServices | null = null;
|
let services: IDataIngestionServices | null = null;
|
||||||
|
let container: ServiceContainer | null = null;
|
||||||
let app: Hono | null = null;
|
let app: Hono | null = null;
|
||||||
|
|
||||||
// Initialize shutdown manager
|
// Initialize shutdown manager
|
||||||
|
|
@ -53,7 +57,36 @@ async function initializeServices() {
|
||||||
logger.info('Initializing data-ingestion service with improved DI...');
|
logger.info('Initializing data-ingestion service with improved DI...');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Create all services using the service factory
|
// Create Awilix container with proper config structure
|
||||||
|
logger.debug('Creating Awilix DI container...');
|
||||||
|
const awilixConfig = {
|
||||||
|
redis: {
|
||||||
|
host: config.database.dragonfly.host,
|
||||||
|
port: config.database.dragonfly.port,
|
||||||
|
db: config.database.dragonfly.db,
|
||||||
|
},
|
||||||
|
mongodb: {
|
||||||
|
uri: config.database.mongodb.uri,
|
||||||
|
database: config.database.mongodb.database,
|
||||||
|
},
|
||||||
|
postgres: {
|
||||||
|
host: config.database.postgres.host,
|
||||||
|
port: config.database.postgres.port,
|
||||||
|
database: config.database.postgres.database,
|
||||||
|
user: config.database.postgres.user,
|
||||||
|
password: config.database.postgres.password,
|
||||||
|
},
|
||||||
|
proxy: {
|
||||||
|
cachePrefix: 'proxy:',
|
||||||
|
ttl: 3600,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
container = createServiceContainer(awilixConfig);
|
||||||
|
await initializeAwilixServices(container);
|
||||||
|
logger.info('Awilix container created and initialized');
|
||||||
|
|
||||||
|
// Create all services using the service factory (for backward compatibility)
|
||||||
logger.debug('Creating services using service factory...');
|
logger.debug('Creating services using service factory...');
|
||||||
services = await createDataIngestionServices(config);
|
services = await createDataIngestionServices(config);
|
||||||
logger.info('All services created successfully');
|
logger.info('All services created successfully');
|
||||||
|
|
@ -76,16 +109,20 @@ async function initializeServices() {
|
||||||
const routes = createRoutes(services);
|
const routes = createRoutes(services);
|
||||||
app.route('/', routes);
|
app.route('/', routes);
|
||||||
|
|
||||||
// Initialize proxy manager
|
// Initialize handlers with Awilix service container
|
||||||
logger.debug('Initializing proxy manager...');
|
logger.debug('Initializing data handlers with Awilix DI pattern...');
|
||||||
await ProxyManager.initialize();
|
|
||||||
logger.info('Proxy manager initialized');
|
|
||||||
|
|
||||||
// Initialize handlers with new DI pattern
|
// Create service adapter that includes proxy from Awilix container
|
||||||
logger.debug('Initializing data handlers with new DI pattern...');
|
const serviceContainerWithProxy = createServiceAdapter(services);
|
||||||
|
// Override the proxy service with the one from Awilix
|
||||||
|
Object.defineProperty(serviceContainerWithProxy, 'proxy', {
|
||||||
|
get: () => container!.resolve('proxyManager'),
|
||||||
|
enumerable: true,
|
||||||
|
configurable: true
|
||||||
|
});
|
||||||
|
|
||||||
// Auto-register all handlers
|
// Auto-register all handlers with the enhanced service container
|
||||||
await initializeAllHandlers(services);
|
await initializeAllHandlers(serviceContainerWithProxy);
|
||||||
|
|
||||||
logger.info('Data handlers initialized with new DI pattern');
|
logger.info('Data handlers initialized with new DI pattern');
|
||||||
|
|
||||||
|
|
|
||||||
11
bun.lock
11
bun.lock
|
|
@ -7,6 +7,7 @@
|
||||||
"@primeng/themes": "^19.1.3",
|
"@primeng/themes": "^19.1.3",
|
||||||
"@tanstack/table-core": "^8.21.3",
|
"@tanstack/table-core": "^8.21.3",
|
||||||
"@types/pg": "^8.15.4",
|
"@types/pg": "^8.15.4",
|
||||||
|
"awilix": "^12.0.5",
|
||||||
"bullmq": "^5.53.2",
|
"bullmq": "^5.53.2",
|
||||||
"ioredis": "^5.6.1",
|
"ioredis": "^5.6.1",
|
||||||
"pg": "^8.16.0",
|
"pg": "^8.16.0",
|
||||||
|
|
@ -986,6 +987,8 @@
|
||||||
|
|
||||||
"available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="],
|
"available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="],
|
||||||
|
|
||||||
|
"awilix": ["awilix@12.0.5", "", { "dependencies": { "camel-case": "^4.1.2", "fast-glob": "^3.3.3" } }, "sha512-Qf/V/hRo6DK0FoBKJ9QiObasRxHAhcNi0mV6kW2JMawxS3zq6Un+VsZmVAZDUfvB+MjTEiJ2tUJUl4cr0JiUAw=="],
|
||||||
|
|
||||||
"aws4": ["aws4@1.13.2", "", {}, "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw=="],
|
"aws4": ["aws4@1.13.2", "", {}, "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw=="],
|
||||||
|
|
||||||
"axios": ["axios@1.10.0", "", { "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } }, "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw=="],
|
"axios": ["axios@1.10.0", "", { "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } }, "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw=="],
|
||||||
|
|
@ -1060,6 +1063,8 @@
|
||||||
|
|
||||||
"callsites": ["callsites@3.1.0", "", {}, "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="],
|
"callsites": ["callsites@3.1.0", "", {}, "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ=="],
|
||||||
|
|
||||||
|
"camel-case": ["camel-case@4.1.2", "", { "dependencies": { "pascal-case": "^3.1.2", "tslib": "^2.0.3" } }, "sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw=="],
|
||||||
|
|
||||||
"camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="],
|
"camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="],
|
||||||
|
|
||||||
"camelcase-css": ["camelcase-css@2.0.1", "", {}, "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA=="],
|
"camelcase-css": ["camelcase-css@2.0.1", "", {}, "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA=="],
|
||||||
|
|
@ -1596,6 +1601,8 @@
|
||||||
|
|
||||||
"loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="],
|
"loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="],
|
||||||
|
|
||||||
|
"lower-case": ["lower-case@2.0.2", "", { "dependencies": { "tslib": "^2.0.3" } }, "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg=="],
|
||||||
|
|
||||||
"lowercase-keys": ["lowercase-keys@3.0.0", "", {}, "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="],
|
"lowercase-keys": ["lowercase-keys@3.0.0", "", {}, "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ=="],
|
||||||
|
|
||||||
"lru-cache": ["lru-cache@11.1.0", "", {}, "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A=="],
|
"lru-cache": ["lru-cache@11.1.0", "", {}, "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A=="],
|
||||||
|
|
@ -1694,6 +1701,8 @@
|
||||||
|
|
||||||
"new-find-package-json": ["new-find-package-json@2.0.0", "", { "dependencies": { "debug": "^4.3.4" } }, "sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew=="],
|
"new-find-package-json": ["new-find-package-json@2.0.0", "", { "dependencies": { "debug": "^4.3.4" } }, "sha512-lDcBsjBSMlj3LXH2v/FW3txlh2pYTjmbOXPYJD93HI5EwuLzI11tdHSIpUMmfq/IOsldj4Ps8M8flhm+pCK4Ew=="],
|
||||||
|
|
||||||
|
"no-case": ["no-case@3.0.4", "", { "dependencies": { "lower-case": "^2.0.2", "tslib": "^2.0.3" } }, "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg=="],
|
||||||
|
|
||||||
"node-abi": ["node-abi@3.75.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg=="],
|
"node-abi": ["node-abi@3.75.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg=="],
|
||||||
|
|
||||||
"node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="],
|
"node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="],
|
||||||
|
|
@ -1782,6 +1791,8 @@
|
||||||
|
|
||||||
"parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
|
"parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
|
||||||
|
|
||||||
|
"pascal-case": ["pascal-case@3.1.2", "", { "dependencies": { "no-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g=="],
|
||||||
|
|
||||||
"path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="],
|
"path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="],
|
||||||
|
|
||||||
"path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="],
|
"path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="],
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@
|
||||||
|
|
||||||
import type { IServiceContainer } from '@stock-bot/handlers';
|
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||||
import type { IDataIngestionServices } from '../service-interfaces';
|
import type { IDataIngestionServices } from '../service-interfaces';
|
||||||
import { ProxyManager } from '@stock-bot/proxy';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Adapter that converts IDataIngestionServices to IServiceContainer
|
* Adapter that converts IDataIngestionServices to IServiceContainer
|
||||||
|
|
@ -23,9 +22,10 @@ export class DataIngestionServiceAdapter implements IServiceContainer {
|
||||||
// HTTP client not in current data services - will be added when needed
|
// HTTP client not in current data services - will be added when needed
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
get proxy() {
|
get proxy(): any {
|
||||||
// Return singleton proxy manager instance
|
// Proxy manager should be injected via Awilix container
|
||||||
return ProxyManager.getInstance();
|
// This adapter is for legacy compatibility
|
||||||
|
throw new Error('ProxyManager must be provided through Awilix container');
|
||||||
}
|
}
|
||||||
|
|
||||||
// Database clients
|
// Database clients
|
||||||
|
|
|
||||||
174
libs/core/di/src/awilix-container.ts
Normal file
174
libs/core/di/src/awilix-container.ts
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
/**
|
||||||
|
* Awilix DI Container Setup
|
||||||
|
* Creates a decoupled, reusable dependency injection container
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createContainer, asFunction, asValue, InjectionMode, type AwilixContainer } from 'awilix';
|
||||||
|
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
||||||
|
import { ProxyManager } from '@stock-bot/proxy';
|
||||||
|
import { getLogger } from '@stock-bot/logger';
|
||||||
|
import type { IServiceContainer } from '@stock-bot/handlers';
|
||||||
|
|
||||||
|
// Configuration types
|
||||||
|
export interface AppConfig {
|
||||||
|
redis: {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
password?: string;
|
||||||
|
username?: string;
|
||||||
|
db?: number;
|
||||||
|
};
|
||||||
|
mongodb: {
|
||||||
|
uri: string;
|
||||||
|
database: string;
|
||||||
|
};
|
||||||
|
postgres: {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
database: string;
|
||||||
|
user: string;
|
||||||
|
password: string;
|
||||||
|
};
|
||||||
|
questdb?: {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
};
|
||||||
|
proxy?: {
|
||||||
|
cachePrefix?: string;
|
||||||
|
ttl?: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create and configure the DI container
|
||||||
|
*/
|
||||||
|
export function createServiceContainer(config: AppConfig): AwilixContainer {
|
||||||
|
const container = createContainer({
|
||||||
|
injectionMode: InjectionMode.PROXY,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Register configuration values
|
||||||
|
container.register({
|
||||||
|
// Configuration
|
||||||
|
config: asValue(config),
|
||||||
|
redisConfig: asValue(config.redis),
|
||||||
|
mongoConfig: asValue(config.mongodb),
|
||||||
|
postgresConfig: asValue(config.postgres),
|
||||||
|
questdbConfig: asValue(config.questdb || { host: 'localhost', port: 9009 }),
|
||||||
|
|
||||||
|
// Core services with dependency injection
|
||||||
|
logger: asFunction(() => getLogger('app')).singleton(),
|
||||||
|
|
||||||
|
// Cache with injected config and logger
|
||||||
|
cache: asFunction(({ redisConfig, logger }) =>
|
||||||
|
createCache({
|
||||||
|
redisConfig,
|
||||||
|
logger,
|
||||||
|
keyPrefix: 'cache:',
|
||||||
|
ttl: 3600,
|
||||||
|
enableMetrics: true,
|
||||||
|
})
|
||||||
|
).singleton(),
|
||||||
|
|
||||||
|
// Proxy manager with injected cache and logger
|
||||||
|
proxyManager: asFunction(({ cache, config, logger }) => {
|
||||||
|
const manager = new ProxyManager(
|
||||||
|
cache,
|
||||||
|
config.proxy || {},
|
||||||
|
logger
|
||||||
|
);
|
||||||
|
// Note: initialization happens in initializeServices function
|
||||||
|
return manager;
|
||||||
|
}).singleton(),
|
||||||
|
|
||||||
|
// HTTP client can be added here when decoupled
|
||||||
|
httpClient: asFunction(() => {
|
||||||
|
// TODO: Import and create HTTP client when decoupled
|
||||||
|
return null;
|
||||||
|
}).singleton(),
|
||||||
|
|
||||||
|
// Database clients - placeholders for now
|
||||||
|
mongoClient: asFunction(() => {
|
||||||
|
// TODO: Create MongoDB client
|
||||||
|
return null;
|
||||||
|
}).singleton(),
|
||||||
|
|
||||||
|
postgresClient: asFunction(() => {
|
||||||
|
// TODO: Create PostgreSQL client
|
||||||
|
return null;
|
||||||
|
}).singleton(),
|
||||||
|
|
||||||
|
questdbClient: asFunction(() => {
|
||||||
|
// TODO: Create QuestDB client
|
||||||
|
return null;
|
||||||
|
}).singleton(),
|
||||||
|
|
||||||
|
// Queue manager - placeholder
|
||||||
|
queueManager: asFunction(() => {
|
||||||
|
// TODO: Create queue manager when decoupled
|
||||||
|
return null;
|
||||||
|
}).singleton(),
|
||||||
|
|
||||||
|
// Build the IServiceContainer for handlers
|
||||||
|
serviceContainer: asFunction((cradle) => ({
|
||||||
|
logger: cradle.logger,
|
||||||
|
cache: cradle.cache,
|
||||||
|
proxy: cradle.proxyManager,
|
||||||
|
http: cradle.httpClient,
|
||||||
|
mongodb: cradle.mongoClient,
|
||||||
|
postgres: cradle.postgresClient,
|
||||||
|
questdb: cradle.questdbClient,
|
||||||
|
queue: cradle.queueManager,
|
||||||
|
} as IServiceContainer)).singleton(),
|
||||||
|
});
|
||||||
|
|
||||||
|
return container;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize async services after container creation
|
||||||
|
*/
|
||||||
|
export async function initializeServices(container: AwilixContainer): Promise<void> {
|
||||||
|
const logger = container.resolve('logger');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Wait for cache to be ready first
|
||||||
|
const cache = container.resolve('cache');
|
||||||
|
if (cache && typeof cache.waitForReady === 'function') {
|
||||||
|
await cache.waitForReady(10000);
|
||||||
|
logger.info('Cache is ready');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize proxy manager
|
||||||
|
const proxyManager = container.resolve('proxyManager');
|
||||||
|
if (proxyManager && typeof proxyManager.initialize === 'function') {
|
||||||
|
await proxyManager.initialize();
|
||||||
|
logger.info('Proxy manager initialized');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize other async services as needed
|
||||||
|
// ...
|
||||||
|
|
||||||
|
logger.info('All services initialized successfully');
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to initialize services', { error });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Type definitions for container resolution
|
||||||
|
export interface ServiceCradle {
|
||||||
|
config: AppConfig;
|
||||||
|
logger: any;
|
||||||
|
cache: CacheProvider;
|
||||||
|
proxyManager: ProxyManager;
|
||||||
|
httpClient: any;
|
||||||
|
mongoClient: any;
|
||||||
|
postgresClient: any;
|
||||||
|
questdbClient: any;
|
||||||
|
queueManager: any;
|
||||||
|
serviceContainer: IServiceContainer;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export typed container
|
||||||
|
export type ServiceContainer = AwilixContainer<ServiceCradle>;
|
||||||
|
|
@ -7,3 +7,12 @@ export * from './types';
|
||||||
export * from './service-interfaces';
|
export * from './service-interfaces';
|
||||||
export * from './service-factory';
|
export * from './service-factory';
|
||||||
export * from './adapters/service-adapter';
|
export * from './adapters/service-adapter';
|
||||||
|
|
||||||
|
// Awilix container exports
|
||||||
|
export {
|
||||||
|
createServiceContainer,
|
||||||
|
initializeServices,
|
||||||
|
type AppConfig,
|
||||||
|
type ServiceCradle,
|
||||||
|
type ServiceContainer
|
||||||
|
} from './awilix-container';
|
||||||
|
|
@ -5,7 +5,6 @@
|
||||||
import { getLogger } from '@stock-bot/logger';
|
import { getLogger } from '@stock-bot/logger';
|
||||||
import { ConnectionFactory } from './connection-factory';
|
import { ConnectionFactory } from './connection-factory';
|
||||||
import { PoolSizeCalculator } from './pool-size-calculator';
|
import { PoolSizeCalculator } from './pool-size-calculator';
|
||||||
import { ProxyManager } from '@stock-bot/proxy';
|
|
||||||
import type {
|
import type {
|
||||||
IDataIngestionServices,
|
IDataIngestionServices,
|
||||||
IServiceFactory,
|
IServiceFactory,
|
||||||
|
|
@ -45,9 +44,7 @@ export class DataIngestionServiceFactory implements IServiceFactory {
|
||||||
this.createQueueConnection(connectionFactory, config)
|
this.createQueueConnection(connectionFactory, config)
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Initialize proxy manager
|
// Note: Proxy manager initialization moved to Awilix container
|
||||||
logger.info('Initializing proxy manager...');
|
|
||||||
await ProxyManager.initialize();
|
|
||||||
|
|
||||||
const services: IDataIngestionServices = {
|
const services: IDataIngestionServices = {
|
||||||
mongodb: mongoPool.client,
|
mongodb: mongoPool.client,
|
||||||
|
|
|
||||||
9
libs/data/cache/src/connection-manager.ts
vendored
9
libs/data/cache/src/connection-manager.ts
vendored
|
|
@ -1,5 +1,4 @@
|
||||||
import Redis from 'ioredis';
|
import Redis from 'ioredis';
|
||||||
import { getLogger } from '@stock-bot/logger';
|
|
||||||
import type { RedisConfig } from './types';
|
import type { RedisConfig } from './types';
|
||||||
|
|
||||||
interface ConnectionConfig {
|
interface ConnectionConfig {
|
||||||
|
|
@ -7,6 +6,7 @@ interface ConnectionConfig {
|
||||||
singleton?: boolean;
|
singleton?: boolean;
|
||||||
db?: number;
|
db?: number;
|
||||||
redisConfig: RedisConfig;
|
redisConfig: RedisConfig;
|
||||||
|
logger?: any;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -16,7 +16,7 @@ export class RedisConnectionManager {
|
||||||
private connections = new Map<string, Redis>();
|
private connections = new Map<string, Redis>();
|
||||||
private static sharedConnections = new Map<string, Redis>();
|
private static sharedConnections = new Map<string, Redis>();
|
||||||
private static instance: RedisConnectionManager;
|
private static instance: RedisConnectionManager;
|
||||||
private logger = getLogger('redis-connection-manager');
|
private logger: any = console;
|
||||||
private static readyConnections = new Set<string>();
|
private static readyConnections = new Set<string>();
|
||||||
|
|
||||||
// Singleton pattern for the manager itself
|
// Singleton pattern for the manager itself
|
||||||
|
|
@ -33,7 +33,10 @@ export class RedisConnectionManager {
|
||||||
* @returns Redis connection instance
|
* @returns Redis connection instance
|
||||||
*/
|
*/
|
||||||
getConnection(config: ConnectionConfig): Redis {
|
getConnection(config: ConnectionConfig): Redis {
|
||||||
const { name, singleton = false, db, redisConfig } = config;
|
const { name, singleton = false, db, redisConfig, logger } = config;
|
||||||
|
if (logger) {
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
if (singleton) {
|
if (singleton) {
|
||||||
// Use shared connection across all instances
|
// Use shared connection across all instances
|
||||||
|
|
|
||||||
5
libs/data/cache/src/redis-cache.ts
vendored
5
libs/data/cache/src/redis-cache.ts
vendored
|
|
@ -1,5 +1,4 @@
|
||||||
import Redis from 'ioredis';
|
import Redis from 'ioredis';
|
||||||
import { getLogger } from '@stock-bot/logger';
|
|
||||||
import { RedisConnectionManager } from './connection-manager';
|
import { RedisConnectionManager } from './connection-manager';
|
||||||
import { CacheOptions, CacheProvider, CacheStats } from './types';
|
import { CacheOptions, CacheProvider, CacheStats } from './types';
|
||||||
|
|
||||||
|
|
@ -8,7 +7,7 @@ import { CacheOptions, CacheProvider, CacheStats } from './types';
|
||||||
*/
|
*/
|
||||||
export class RedisCache implements CacheProvider {
|
export class RedisCache implements CacheProvider {
|
||||||
private redis: Redis;
|
private redis: Redis;
|
||||||
private logger = getLogger('redis-cache');
|
private logger: any;
|
||||||
private defaultTTL: number;
|
private defaultTTL: number;
|
||||||
private keyPrefix: string;
|
private keyPrefix: string;
|
||||||
private enableMetrics: boolean;
|
private enableMetrics: boolean;
|
||||||
|
|
@ -29,6 +28,7 @@ export class RedisCache implements CacheProvider {
|
||||||
this.defaultTTL = options.ttl ?? 3600; // 1 hour default
|
this.defaultTTL = options.ttl ?? 3600; // 1 hour default
|
||||||
this.keyPrefix = options.keyPrefix ?? 'cache:';
|
this.keyPrefix = options.keyPrefix ?? 'cache:';
|
||||||
this.enableMetrics = options.enableMetrics ?? true;
|
this.enableMetrics = options.enableMetrics ?? true;
|
||||||
|
this.logger = options.logger || console; // Use provided logger or console as fallback
|
||||||
|
|
||||||
// Get connection manager instance
|
// Get connection manager instance
|
||||||
this.connectionManager = RedisConnectionManager.getInstance();
|
this.connectionManager = RedisConnectionManager.getInstance();
|
||||||
|
|
@ -47,6 +47,7 @@ export class RedisCache implements CacheProvider {
|
||||||
name: `${baseName}-SERVICE`,
|
name: `${baseName}-SERVICE`,
|
||||||
singleton: options.shared ?? true, // Default to shared connection for cache
|
singleton: options.shared ?? true, // Default to shared connection for cache
|
||||||
redisConfig: options.redisConfig,
|
redisConfig: options.redisConfig,
|
||||||
|
logger: this.logger,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Only setup event handlers for non-shared connections to avoid memory leaks
|
// Only setup event handlers for non-shared connections to avoid memory leaks
|
||||||
|
|
|
||||||
1
libs/data/cache/src/types.ts
vendored
1
libs/data/cache/src/types.ts
vendored
|
|
@ -85,6 +85,7 @@ export interface CacheOptions {
|
||||||
name?: string; // Name for connection identification
|
name?: string; // Name for connection identification
|
||||||
shared?: boolean; // Whether to use shared connection
|
shared?: boolean; // Whether to use shared connection
|
||||||
redisConfig: RedisConfig;
|
redisConfig: RedisConfig;
|
||||||
|
logger?: any; // Optional logger instance
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CacheStats {
|
export interface CacheStats {
|
||||||
|
|
|
||||||
|
|
@ -5,32 +5,15 @@
|
||||||
|
|
||||||
// Main classes
|
// Main classes
|
||||||
export { ProxyManager } from './proxy-manager';
|
export { ProxyManager } from './proxy-manager';
|
||||||
export { ProxySyncService } from './proxy-sync';
|
|
||||||
|
|
||||||
// Types
|
// Types
|
||||||
export type {
|
export type {
|
||||||
ProxyInfo,
|
ProxyInfo,
|
||||||
ProxyManagerConfig,
|
ProxyManagerConfig, ProxyStats, ProxySyncConfig
|
||||||
ProxySyncConfig,
|
|
||||||
ProxyStats
|
|
||||||
} from './types';
|
} from './types';
|
||||||
|
|
||||||
// Convenience functions
|
// Note: Convenience functions removed as ProxyManager is no longer a singleton
|
||||||
export {
|
// Create an instance and use its methods directly
|
||||||
getProxy,
|
|
||||||
getRandomProxy,
|
|
||||||
getAllProxies,
|
|
||||||
getWorkingProxies,
|
|
||||||
updateProxies,
|
|
||||||
getProxyStats
|
|
||||||
} from './proxy-manager';
|
|
||||||
|
|
||||||
export {
|
|
||||||
getProxySyncService,
|
|
||||||
startProxySync,
|
|
||||||
stopProxySync,
|
|
||||||
syncProxiesOnce
|
|
||||||
} from './proxy-sync';
|
|
||||||
|
|
||||||
// Default export
|
// Default export
|
||||||
export { ProxyManager as default } from './proxy-manager';
|
export { ProxyManager as default } from './proxy-manager';
|
||||||
|
|
@ -1,37 +1,21 @@
|
||||||
/**
|
/**
|
||||||
* Centralized Proxy Manager - Handles proxy storage, retrieval, and caching
|
* Centralized Proxy Manager - Handles proxy storage, retrieval, and caching
|
||||||
*/
|
*/
|
||||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
import type { CacheProvider } from '@stock-bot/cache';
|
||||||
import { getDatabaseConfig } from '@stock-bot/config';
|
|
||||||
import { getLogger } from '@stock-bot/logger';
|
|
||||||
import type { ProxyInfo, ProxyManagerConfig, ProxyStats } from './types';
|
import type { ProxyInfo, ProxyManagerConfig, ProxyStats } from './types';
|
||||||
|
|
||||||
const logger = getLogger('proxy-manager');
|
|
||||||
|
|
||||||
export class ProxyManager {
|
export class ProxyManager {
|
||||||
private static instance: ProxyManager | null = null;
|
|
||||||
private cache: CacheProvider;
|
private cache: CacheProvider;
|
||||||
private proxies: ProxyInfo[] = [];
|
private proxies: ProxyInfo[] = [];
|
||||||
private proxyIndex: number = 0;
|
private proxyIndex: number = 0;
|
||||||
private lastUpdate: Date | null = null;
|
private lastUpdate: Date | null = null;
|
||||||
private isInitialized = false;
|
private isInitialized = false;
|
||||||
private config: ProxyManagerConfig;
|
private logger: any;
|
||||||
|
|
||||||
private constructor(config: ProxyManagerConfig = {}) {
|
constructor(cache: CacheProvider, _config: ProxyManagerConfig = {}, logger?: any) {
|
||||||
this.config = {
|
this.cache = cache;
|
||||||
cachePrefix: 'proxies:',
|
this.logger = logger || console;
|
||||||
ttl: 86400, // 24 hours
|
// Config can be used in the future for customization
|
||||||
enableMetrics: true,
|
|
||||||
...config
|
|
||||||
};
|
|
||||||
|
|
||||||
const databaseConfig = getDatabaseConfig();
|
|
||||||
this.cache = createCache({
|
|
||||||
redisConfig: databaseConfig.dragonfly,
|
|
||||||
keyPrefix: this.config.cachePrefix,
|
|
||||||
ttl: this.config.ttl,
|
|
||||||
enableMetrics: this.config.enableMetrics,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -43,27 +27,27 @@ export class ProxyManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
logger.info('Initializing proxy manager...');
|
this.logger.info('Initializing proxy manager...');
|
||||||
|
|
||||||
// Wait for cache to be ready
|
// Wait for cache to be ready
|
||||||
await this.cache.waitForReady(10000); // Wait up to 10 seconds
|
await this.cache.waitForReady(10000); // Wait up to 10 seconds
|
||||||
logger.debug('Cache is ready');
|
this.logger.debug('Cache is ready');
|
||||||
|
|
||||||
await this.loadFromCache();
|
await this.loadFromCache();
|
||||||
this.isInitialized = true;
|
this.isInitialized = true;
|
||||||
logger.info('Proxy manager initialized', {
|
this.logger.info('Proxy manager initialized', {
|
||||||
proxiesLoaded: this.proxies.length,
|
proxiesLoaded: this.proxies.length,
|
||||||
lastUpdate: this.lastUpdate,
|
lastUpdate: this.lastUpdate,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Failed to initialize proxy manager', { error });
|
this.logger.error('Failed to initialize proxy manager', { error });
|
||||||
this.isInitialized = true; // Set to true anyway to avoid infinite retries
|
this.isInitialized = true; // Set to true anyway to avoid infinite retries
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
getProxy(): string | null {
|
getProxy(): string | null {
|
||||||
if (this.proxies.length === 0) {
|
if (this.proxies.length === 0) {
|
||||||
logger.warn('No proxies available in memory');
|
this.logger.warn('No proxies available in memory');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -97,7 +81,7 @@ export class ProxyManager {
|
||||||
|
|
||||||
// Return null if no proxies available
|
// Return null if no proxies available
|
||||||
if (this.proxies.length === 0) {
|
if (this.proxies.length === 0) {
|
||||||
logger.warn('No proxies available in memory');
|
this.logger.warn('No proxies available in memory');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -105,7 +89,7 @@ export class ProxyManager {
|
||||||
const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false);
|
const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false);
|
||||||
|
|
||||||
if (workingProxies.length === 0) {
|
if (workingProxies.length === 0) {
|
||||||
logger.warn('No working proxies available');
|
this.logger.warn('No working proxies available');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -122,11 +106,11 @@ export class ProxyManager {
|
||||||
const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)];
|
const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)];
|
||||||
|
|
||||||
if (!selectedProxy) {
|
if (!selectedProxy) {
|
||||||
logger.warn('No proxy selected from available pool');
|
this.logger.warn('No proxy selected from available pool');
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug('Selected proxy', {
|
this.logger.debug('Selected proxy', {
|
||||||
host: selectedProxy.host,
|
host: selectedProxy.host,
|
||||||
port: selectedProxy.port,
|
port: selectedProxy.port,
|
||||||
successRate: selectedProxy.successRate,
|
successRate: selectedProxy.successRate,
|
||||||
|
|
@ -178,8 +162,13 @@ export class ProxyManager {
|
||||||
* Update the proxy pool with new proxies
|
* Update the proxy pool with new proxies
|
||||||
*/
|
*/
|
||||||
async updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
async updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
||||||
|
// Ensure manager is initialized before updating
|
||||||
|
if (!this.isInitialized) {
|
||||||
|
await this.initializeInternal();
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
|
this.logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
|
||||||
|
|
||||||
this.proxies = proxies;
|
this.proxies = proxies;
|
||||||
this.lastUpdate = new Date();
|
this.lastUpdate = new Date();
|
||||||
|
|
@ -189,13 +178,13 @@ export class ProxyManager {
|
||||||
await this.cache.set('last-update', this.lastUpdate.toISOString());
|
await this.cache.set('last-update', this.lastUpdate.toISOString());
|
||||||
|
|
||||||
const workingCount = proxies.filter(p => p.isWorking !== false).length;
|
const workingCount = proxies.filter(p => p.isWorking !== false).length;
|
||||||
logger.info('Proxy pool updated successfully', {
|
this.logger.info('Proxy pool updated successfully', {
|
||||||
totalProxies: proxies.length,
|
totalProxies: proxies.length,
|
||||||
workingProxies: workingCount,
|
workingProxies: workingCount,
|
||||||
lastUpdate: this.lastUpdate,
|
lastUpdate: this.lastUpdate,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Failed to update proxy pool', { error });
|
this.logger.error('Failed to update proxy pool', { error });
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -210,10 +199,10 @@ export class ProxyManager {
|
||||||
|
|
||||||
if (existingIndex >= 0) {
|
if (existingIndex >= 0) {
|
||||||
this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy };
|
this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy };
|
||||||
logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
|
this.logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
|
||||||
} else {
|
} else {
|
||||||
this.proxies.push(proxy);
|
this.proxies.push(proxy);
|
||||||
logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
|
this.logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update cache
|
// Update cache
|
||||||
|
|
@ -231,7 +220,7 @@ export class ProxyManager {
|
||||||
|
|
||||||
if (this.proxies.length < initialLength) {
|
if (this.proxies.length < initialLength) {
|
||||||
await this.updateProxies(this.proxies);
|
await this.updateProxies(this.proxies);
|
||||||
logger.debug('Removed proxy', { host, port, protocol });
|
this.logger.debug('Removed proxy', { host, port, protocol });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -245,7 +234,7 @@ export class ProxyManager {
|
||||||
await this.cache.del('active-proxies');
|
await this.cache.del('active-proxies');
|
||||||
await this.cache.del('last-update');
|
await this.cache.del('last-update');
|
||||||
|
|
||||||
logger.info('Cleared all proxies');
|
this.logger.info('Cleared all proxies');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -267,79 +256,29 @@ export class ProxyManager {
|
||||||
this.proxies = cachedProxies;
|
this.proxies = cachedProxies;
|
||||||
this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null;
|
this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null;
|
||||||
|
|
||||||
logger.debug('Loaded proxies from cache', {
|
this.logger.debug('Loaded proxies from cache', {
|
||||||
count: this.proxies.length,
|
count: this.proxies.length,
|
||||||
lastUpdate: this.lastUpdate,
|
lastUpdate: this.lastUpdate,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
logger.debug('No cached proxies found');
|
this.logger.debug('No cached proxies found');
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Failed to load proxies from cache', { error });
|
this.logger.error('Failed to load proxies from cache', { error });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the singleton instance
|
* Initialize the proxy manager
|
||||||
*/
|
*/
|
||||||
static async initialize(config?: ProxyManagerConfig): Promise<void> {
|
async initialize(): Promise<void> {
|
||||||
if (!ProxyManager.instance) {
|
await this.initializeInternal();
|
||||||
ProxyManager.instance = new ProxyManager(config);
|
|
||||||
await ProxyManager.instance.initializeInternal();
|
|
||||||
|
|
||||||
// Perform initial sync with proxy:active:* storage
|
// Note: Initial proxy sync should be handled by the container or application
|
||||||
try {
|
// that creates ProxyManager instance
|
||||||
const { syncProxiesOnce } = await import('./proxy-sync');
|
this.logger.info('ProxyManager initialized - proxy sync should be handled externally');
|
||||||
await syncProxiesOnce();
|
|
||||||
logger.info('Initial proxy sync completed');
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Failed to perform initial proxy sync', { error });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the singleton instance (must be initialized first)
|
|
||||||
*/
|
|
||||||
static getInstance(): ProxyManager {
|
|
||||||
if (!ProxyManager.instance) {
|
|
||||||
throw new Error('ProxyManager not initialized. Call ProxyManager.initialize() first.');
|
|
||||||
}
|
|
||||||
return ProxyManager.instance;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reset the singleton instance (for testing)
|
|
||||||
*/
|
|
||||||
static reset(): void {
|
|
||||||
ProxyManager.instance = null;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export the class as default
|
// Export the class as default
|
||||||
export default ProxyManager;
|
export default ProxyManager;
|
||||||
|
|
||||||
// Convenience functions for easier imports
|
|
||||||
export function getProxy(): string | null {
|
|
||||||
return ProxyManager.getInstance().getProxy();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getRandomProxy(): ProxyInfo | null {
|
|
||||||
return ProxyManager.getInstance().getRandomProxy();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getAllProxies(): ProxyInfo[] {
|
|
||||||
return ProxyManager.getInstance().getAllProxies();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getWorkingProxies(): ProxyInfo[] {
|
|
||||||
return ProxyManager.getInstance().getWorkingProxies();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
|
||||||
return ProxyManager.getInstance().updateProxies(proxies);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getProxyStats(): ProxyStats {
|
|
||||||
return ProxyManager.getInstance().getStats();
|
|
||||||
}
|
|
||||||
|
|
@ -1,170 +0,0 @@
|
||||||
/**
|
|
||||||
* Proxy Storage Synchronization Service
|
|
||||||
*
|
|
||||||
* This service bridges the gap between two proxy storage systems:
|
|
||||||
* 1. proxy:active:* keys (used by proxy tasks for individual proxy storage)
|
|
||||||
* 2. proxies:active-proxies (used by ProxyManager for centralized storage)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
|
||||||
import { getDatabaseConfig } from '@stock-bot/config';
|
|
||||||
import { getLogger } from '@stock-bot/logger';
|
|
||||||
import type { ProxyInfo, ProxySyncConfig } from './types';
|
|
||||||
import { ProxyManager } from './proxy-manager';
|
|
||||||
|
|
||||||
const logger = getLogger('proxy-sync');
|
|
||||||
|
|
||||||
export class ProxySyncService {
|
|
||||||
private cache: CacheProvider;
|
|
||||||
private syncInterval: Timer | null = null;
|
|
||||||
private isRunning = false;
|
|
||||||
private config: ProxySyncConfig;
|
|
||||||
|
|
||||||
constructor(config: ProxySyncConfig = {}) {
|
|
||||||
this.config = {
|
|
||||||
intervalMs: 300000, // 5 minutes
|
|
||||||
enableAutoSync: true,
|
|
||||||
...config
|
|
||||||
};
|
|
||||||
|
|
||||||
const databaseConfig = getDatabaseConfig();
|
|
||||||
this.cache = createCache({
|
|
||||||
redisConfig: databaseConfig.dragonfly,
|
|
||||||
keyPrefix: '', // No prefix to access all keys
|
|
||||||
ttl: 86400,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start the synchronization service
|
|
||||||
* @param intervalMs - Sync interval in milliseconds (default: 5 minutes)
|
|
||||||
*/
|
|
||||||
async start(intervalMs?: number): Promise<void> {
|
|
||||||
const interval = intervalMs || this.config.intervalMs!;
|
|
||||||
|
|
||||||
if (this.isRunning) {
|
|
||||||
logger.warn('Proxy sync service is already running');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.isRunning = true;
|
|
||||||
logger.info('Starting proxy sync service', { intervalMs: interval });
|
|
||||||
|
|
||||||
// Wait for cache to be ready before initial sync
|
|
||||||
await this.cache.waitForReady(10000);
|
|
||||||
|
|
||||||
// Initial sync
|
|
||||||
await this.syncProxies();
|
|
||||||
|
|
||||||
// Set up periodic sync if enabled
|
|
||||||
if (this.config.enableAutoSync) {
|
|
||||||
this.syncInterval = setInterval(async () => {
|
|
||||||
try {
|
|
||||||
await this.syncProxies();
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Error during periodic sync', { error });
|
|
||||||
}
|
|
||||||
}, interval);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stop the synchronization service
|
|
||||||
*/
|
|
||||||
stop(): void {
|
|
||||||
if (this.syncInterval) {
|
|
||||||
clearInterval(this.syncInterval);
|
|
||||||
this.syncInterval = null;
|
|
||||||
}
|
|
||||||
this.isRunning = false;
|
|
||||||
logger.info('Stopped proxy sync service');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Perform a one-time synchronization
|
|
||||||
*/
|
|
||||||
async syncProxies(): Promise<void> {
|
|
||||||
try {
|
|
||||||
logger.debug('Starting proxy synchronization');
|
|
||||||
|
|
||||||
// Wait for cache to be ready
|
|
||||||
await this.cache.waitForReady(5000);
|
|
||||||
|
|
||||||
// Collect all proxies from proxy:active:* storage
|
|
||||||
const proxyKeys = await this.cache.keys('proxy:active:*');
|
|
||||||
|
|
||||||
if (proxyKeys.length === 0) {
|
|
||||||
logger.debug('No proxies found in proxy:active:* storage');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const allProxies: ProxyInfo[] = [];
|
|
||||||
|
|
||||||
// Fetch all proxies in parallel for better performance
|
|
||||||
const proxyPromises = proxyKeys.map(key => this.cache.get<ProxyInfo>(key));
|
|
||||||
const proxyResults = await Promise.all(proxyPromises);
|
|
||||||
|
|
||||||
for (const proxy of proxyResults) {
|
|
||||||
if (proxy) {
|
|
||||||
allProxies.push(proxy);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const workingCount = allProxies.filter(p => p.isWorking).length;
|
|
||||||
|
|
||||||
logger.info('Collected proxies from storage', {
|
|
||||||
total: allProxies.length,
|
|
||||||
working: workingCount,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Update ProxyManager with all proxies
|
|
||||||
const manager = ProxyManager.getInstance();
|
|
||||||
await manager.updateProxies(allProxies);
|
|
||||||
|
|
||||||
logger.info('Proxy synchronization completed', {
|
|
||||||
synchronized: allProxies.length,
|
|
||||||
working: workingCount,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Failed to sync proxies', { error });
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get synchronization status
|
|
||||||
*/
|
|
||||||
getStatus(): { isRunning: boolean; config: ProxySyncConfig } {
|
|
||||||
return {
|
|
||||||
isRunning: this.isRunning,
|
|
||||||
config: this.config
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export singleton instance
|
|
||||||
let syncServiceInstance: ProxySyncService | null = null;
|
|
||||||
|
|
||||||
export function getProxySyncService(config?: ProxySyncConfig): ProxySyncService {
|
|
||||||
if (!syncServiceInstance) {
|
|
||||||
syncServiceInstance = new ProxySyncService(config);
|
|
||||||
}
|
|
||||||
return syncServiceInstance;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convenience functions
|
|
||||||
export async function startProxySync(intervalMs?: number, config?: ProxySyncConfig): Promise<void> {
|
|
||||||
const service = getProxySyncService(config);
|
|
||||||
await service.start(intervalMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function stopProxySync(): void {
|
|
||||||
if (syncServiceInstance) {
|
|
||||||
syncServiceInstance.stop();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function syncProxiesOnce(): Promise<void> {
|
|
||||||
const service = getProxySyncService();
|
|
||||||
await service.syncProxies();
|
|
||||||
}
|
|
||||||
|
|
@ -2,4 +2,3 @@ export * from './calculations/index';
|
||||||
export * from './common';
|
export * from './common';
|
||||||
export * from './dateUtils';
|
export * from './dateUtils';
|
||||||
export * from './generic-functions';
|
export * from './generic-functions';
|
||||||
export * from './proxy';
|
|
||||||
|
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
/**
|
|
||||||
* Proxy management utilities
|
|
||||||
*/
|
|
||||||
export {
|
|
||||||
default as ProxyManager,
|
|
||||||
getProxy,
|
|
||||||
getRandomProxy,
|
|
||||||
getAllProxies,
|
|
||||||
getWorkingProxies,
|
|
||||||
updateProxies
|
|
||||||
} from './proxy-manager';
|
|
||||||
|
|
||||||
export {
|
|
||||||
ProxySyncService,
|
|
||||||
getProxySyncService,
|
|
||||||
startProxySync,
|
|
||||||
stopProxySync,
|
|
||||||
syncProxiesOnce
|
|
||||||
} from './proxy-sync';
|
|
||||||
|
|
||||||
export type { ProxyInfo } from '@stock-bot/http'; // Re-export for convenience
|
|
||||||
|
|
@ -1,291 +0,0 @@
|
||||||
/**
|
|
||||||
* Centralized Proxy Manager - Handles proxy storage, retrieval, and caching
|
|
||||||
*/
|
|
||||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
|
||||||
import { getDatabaseConfig } from '@stock-bot/config';
|
|
||||||
import { getLogger } from '@stock-bot/logger';
|
|
||||||
import type { ProxyInfo } from '@stock-bot/http';
|
|
||||||
|
|
||||||
const logger = getLogger('proxy-manager');
|
|
||||||
|
|
||||||
export class ProxyManager {
|
|
||||||
private static instance: ProxyManager | null = null;
|
|
||||||
private cache: CacheProvider;
|
|
||||||
private proxies: ProxyInfo[] = [];
|
|
||||||
private lastUpdate: Date | null = null;
|
|
||||||
private isInitialized = false;
|
|
||||||
|
|
||||||
private constructor() {
|
|
||||||
const databaseConfig = getDatabaseConfig();
|
|
||||||
this.cache = createCache({
|
|
||||||
redisConfig: databaseConfig.dragonfly,
|
|
||||||
keyPrefix: 'proxies:',
|
|
||||||
ttl: 86400, // 24 hours
|
|
||||||
enableMetrics: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Internal initialization - loads existing proxies from cache
|
|
||||||
*/
|
|
||||||
private async initializeInternal(): Promise<void> {
|
|
||||||
if (this.isInitialized) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
logger.info('Initializing proxy manager...');
|
|
||||||
|
|
||||||
// Wait for cache to be ready
|
|
||||||
await this.cache.waitForReady(10000); // Wait up to 10 seconds
|
|
||||||
logger.debug('Cache is ready');
|
|
||||||
|
|
||||||
await this.loadFromCache();
|
|
||||||
this.isInitialized = true;
|
|
||||||
logger.info('Proxy manager initialized', {
|
|
||||||
proxiesLoaded: this.proxies.length,
|
|
||||||
lastUpdate: this.lastUpdate,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Failed to initialize proxy manager', { error });
|
|
||||||
this.isInitialized = true; // Set to true anyway to avoid infinite retries
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get a random working proxy from the available pool (synchronous)
|
|
||||||
*/
|
|
||||||
getRandomProxy(): ProxyInfo | null {
|
|
||||||
// Ensure initialized
|
|
||||||
if (!this.isInitialized) {
|
|
||||||
throw new Error('ProxyManager not initialized');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return null if no proxies available
|
|
||||||
if (this.proxies.length === 0) {
|
|
||||||
logger.warn('No proxies available in memory');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Filter for working proxies (not explicitly marked as non-working)
|
|
||||||
const workingProxies = this.proxies.filter(proxy => proxy.isWorking !== false);
|
|
||||||
|
|
||||||
if (workingProxies.length === 0) {
|
|
||||||
logger.warn('No working proxies available');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return random proxy with preference for recently successful ones
|
|
||||||
const sortedProxies = workingProxies.sort((a, b) => {
|
|
||||||
// Prefer proxies with better success rates
|
|
||||||
const aRate = a.successRate || 0;
|
|
||||||
const bRate = b.successRate || 0;
|
|
||||||
return bRate - aRate;
|
|
||||||
});
|
|
||||||
|
|
||||||
// Take from top 50% of best performing proxies
|
|
||||||
const topProxies = sortedProxies.slice(0, Math.max(1, Math.floor(sortedProxies.length * 0.5)));
|
|
||||||
const selectedProxy = topProxies[Math.floor(Math.random() * topProxies.length)];
|
|
||||||
|
|
||||||
if (!selectedProxy) {
|
|
||||||
logger.warn('No proxy selected from available pool');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug('Selected proxy', {
|
|
||||||
host: selectedProxy.host,
|
|
||||||
port: selectedProxy.port,
|
|
||||||
successRate: selectedProxy.successRate,
|
|
||||||
totalAvailable: workingProxies.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
return selectedProxy;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all working proxies (synchronous)
|
|
||||||
*/
|
|
||||||
getWorkingProxies(): ProxyInfo[] {
|
|
||||||
if (!this.isInitialized) {
|
|
||||||
throw new Error('ProxyManager not initialized');
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.proxies.filter(proxy => proxy.isWorking !== false);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all proxies (working and non-working)
|
|
||||||
*/
|
|
||||||
getAllProxies(): ProxyInfo[] {
|
|
||||||
if (!this.isInitialized) {
|
|
||||||
throw new Error('ProxyManager not initialized');
|
|
||||||
}
|
|
||||||
|
|
||||||
return [...this.proxies];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update the proxy pool with new proxies
|
|
||||||
*/
|
|
||||||
async updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
|
||||||
try {
|
|
||||||
logger.info('Updating proxy pool', { newCount: proxies.length, existingCount: this.proxies.length });
|
|
||||||
|
|
||||||
this.proxies = proxies;
|
|
||||||
this.lastUpdate = new Date();
|
|
||||||
|
|
||||||
// Store to cache
|
|
||||||
await this.cache.set('active-proxies', proxies);
|
|
||||||
await this.cache.set('last-update', this.lastUpdate.toISOString());
|
|
||||||
|
|
||||||
const workingCount = proxies.filter(p => p.isWorking !== false).length;
|
|
||||||
logger.info('Proxy pool updated successfully', {
|
|
||||||
totalProxies: proxies.length,
|
|
||||||
workingProxies: workingCount,
|
|
||||||
lastUpdate: this.lastUpdate,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Failed to update proxy pool', { error });
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add or update a single proxy in the pool
|
|
||||||
*/
|
|
||||||
async updateProxy(proxy: ProxyInfo): Promise<void> {
|
|
||||||
const existingIndex = this.proxies.findIndex(
|
|
||||||
p => p.host === proxy.host && p.port === proxy.port && p.protocol === proxy.protocol
|
|
||||||
);
|
|
||||||
|
|
||||||
if (existingIndex >= 0) {
|
|
||||||
this.proxies[existingIndex] = { ...this.proxies[existingIndex], ...proxy };
|
|
||||||
logger.debug('Updated existing proxy', { host: proxy.host, port: proxy.port });
|
|
||||||
} else {
|
|
||||||
this.proxies.push(proxy);
|
|
||||||
logger.debug('Added new proxy', { host: proxy.host, port: proxy.port });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update cache
|
|
||||||
await this.updateProxies(this.proxies);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Remove a proxy from the pool
|
|
||||||
*/
|
|
||||||
async removeProxy(host: string, port: number, protocol: string): Promise<void> {
|
|
||||||
const initialLength = this.proxies.length;
|
|
||||||
this.proxies = this.proxies.filter(
|
|
||||||
p => !(p.host === host && p.port === port && p.protocol === protocol)
|
|
||||||
);
|
|
||||||
|
|
||||||
if (this.proxies.length < initialLength) {
|
|
||||||
await this.updateProxies(this.proxies);
|
|
||||||
logger.debug('Removed proxy', { host, port, protocol });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clear all proxies from memory and cache
|
|
||||||
*/
|
|
||||||
async clearProxies(): Promise<void> {
|
|
||||||
this.proxies = [];
|
|
||||||
this.lastUpdate = null;
|
|
||||||
|
|
||||||
await this.cache.del('active-proxies');
|
|
||||||
await this.cache.del('last-update');
|
|
||||||
|
|
||||||
logger.info('Cleared all proxies');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if proxy manager is ready
|
|
||||||
*/
|
|
||||||
isReady(): boolean {
|
|
||||||
return this.isInitialized;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Load proxies from cache storage
|
|
||||||
*/
|
|
||||||
private async loadFromCache(): Promise<void> {
|
|
||||||
try {
|
|
||||||
const cachedProxies = await this.cache.get<ProxyInfo[]>('active-proxies');
|
|
||||||
const lastUpdateStr = await this.cache.get<string>('last-update');
|
|
||||||
|
|
||||||
if (cachedProxies && Array.isArray(cachedProxies)) {
|
|
||||||
this.proxies = cachedProxies;
|
|
||||||
this.lastUpdate = lastUpdateStr ? new Date(lastUpdateStr) : null;
|
|
||||||
|
|
||||||
logger.debug('Loaded proxies from cache', {
|
|
||||||
count: this.proxies.length,
|
|
||||||
lastUpdate: this.lastUpdate,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
logger.debug('No cached proxies found');
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Failed to load proxies from cache', { error });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize the singleton instance
|
|
||||||
*/
|
|
||||||
static async initialize(): Promise<void> {
|
|
||||||
if (!ProxyManager.instance) {
|
|
||||||
ProxyManager.instance = new ProxyManager();
|
|
||||||
await ProxyManager.instance.initializeInternal();
|
|
||||||
|
|
||||||
// Perform initial sync with proxy:active:* storage
|
|
||||||
try {
|
|
||||||
const { syncProxiesOnce } = await import('./proxy-sync');
|
|
||||||
await syncProxiesOnce();
|
|
||||||
logger.info('Initial proxy sync completed');
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Failed to perform initial proxy sync', { error });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the singleton instance (must be initialized first)
|
|
||||||
*/
|
|
||||||
static getInstance(): ProxyManager {
|
|
||||||
if (!ProxyManager.instance) {
|
|
||||||
throw new Error('ProxyManager not initialized. Call ProxyManager.initialize() first.');
|
|
||||||
}
|
|
||||||
return ProxyManager.instance;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reset the singleton instance (for testing)
|
|
||||||
*/
|
|
||||||
static reset(): void {
|
|
||||||
ProxyManager.instance = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export the class as default
|
|
||||||
export default ProxyManager;
|
|
||||||
|
|
||||||
// Convenience functions for easier imports
|
|
||||||
export function getProxy(): ProxyInfo | null {
|
|
||||||
return ProxyManager.getInstance().getRandomProxy();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getRandomProxy(): ProxyInfo | null {
|
|
||||||
return ProxyManager.getInstance().getRandomProxy();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getAllProxies(): ProxyInfo[] {
|
|
||||||
return ProxyManager.getInstance().getAllProxies();
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getWorkingProxies(): ProxyInfo[] {
|
|
||||||
return ProxyManager.getInstance().getWorkingProxies();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function updateProxies(proxies: ProxyInfo[]): Promise<void> {
|
|
||||||
return ProxyManager.getInstance().updateProxies(proxies);
|
|
||||||
}
|
|
||||||
|
|
@ -1,157 +0,0 @@
|
||||||
/**
|
|
||||||
* Proxy Storage Synchronization Service
|
|
||||||
*
|
|
||||||
* This service bridges the gap between two proxy storage systems:
|
|
||||||
* 1. proxy:active:* keys (used by proxy tasks for individual proxy storage)
|
|
||||||
* 2. proxies:active-proxies (used by ProxyManager for centralized storage)
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { createCache, type CacheProvider } from '@stock-bot/cache';
|
|
||||||
import { getDatabaseConfig } from '@stock-bot/config';
|
|
||||||
import { getLogger } from '@stock-bot/logger';
|
|
||||||
import type { ProxyInfo } from '@stock-bot/http';
|
|
||||||
import { ProxyManager } from './proxy-manager';
|
|
||||||
|
|
||||||
const logger = getLogger('proxy-sync');
|
|
||||||
|
|
||||||
export class ProxySyncService {
|
|
||||||
private cache: CacheProvider;
|
|
||||||
private syncInterval: Timer | null = null;
|
|
||||||
private isRunning = false;
|
|
||||||
|
|
||||||
constructor() {
|
|
||||||
const databaseConfig = getDatabaseConfig();
|
|
||||||
this.cache = createCache({
|
|
||||||
redisConfig: databaseConfig.dragonfly,
|
|
||||||
keyPrefix: '', // No prefix to access all keys
|
|
||||||
ttl: 86400,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start the synchronization service
|
|
||||||
* @param intervalMs - Sync interval in milliseconds (default: 5 minutes)
|
|
||||||
*/
|
|
||||||
async start(intervalMs: number = 300000): Promise<void> {
|
|
||||||
if (this.isRunning) {
|
|
||||||
logger.warn('Proxy sync service is already running');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.isRunning = true;
|
|
||||||
logger.info('Starting proxy sync service', { intervalMs });
|
|
||||||
|
|
||||||
// Wait for cache to be ready before initial sync
|
|
||||||
await this.cache.waitForReady(10000);
|
|
||||||
|
|
||||||
// Initial sync
|
|
||||||
await this.syncProxies();
|
|
||||||
|
|
||||||
// Set up periodic sync
|
|
||||||
this.syncInterval = setInterval(async () => {
|
|
||||||
try {
|
|
||||||
await this.syncProxies();
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Error during periodic sync', { error });
|
|
||||||
}
|
|
||||||
}, intervalMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stop the synchronization service
|
|
||||||
*/
|
|
||||||
stop(): void {
|
|
||||||
if (this.syncInterval) {
|
|
||||||
clearInterval(this.syncInterval);
|
|
||||||
this.syncInterval = null;
|
|
||||||
}
|
|
||||||
this.isRunning = false;
|
|
||||||
logger.info('Stopped proxy sync service');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Perform a one-time synchronization
|
|
||||||
*/
|
|
||||||
async syncProxies(): Promise<void> {
|
|
||||||
try {
|
|
||||||
logger.debug('Starting proxy synchronization');
|
|
||||||
|
|
||||||
// Wait for cache to be ready
|
|
||||||
await this.cache.waitForReady(5000);
|
|
||||||
|
|
||||||
// Collect all proxies from proxy:active:* storage
|
|
||||||
const proxyKeys = await this.cache.keys('proxy:active:*');
|
|
||||||
|
|
||||||
if (proxyKeys.length === 0) {
|
|
||||||
logger.debug('No proxies found in proxy:active:* storage');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const allProxies: ProxyInfo[] = [];
|
|
||||||
|
|
||||||
// Fetch all proxies in parallel for better performance
|
|
||||||
const proxyPromises = proxyKeys.map(key => this.cache.get<ProxyInfo>(key));
|
|
||||||
const proxyResults = await Promise.all(proxyPromises);
|
|
||||||
|
|
||||||
for (const proxy of proxyResults) {
|
|
||||||
if (proxy) {
|
|
||||||
allProxies.push(proxy);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const workingCount = allProxies.filter(p => p.isWorking).length;
|
|
||||||
|
|
||||||
logger.info('Collected proxies from storage', {
|
|
||||||
total: allProxies.length,
|
|
||||||
working: workingCount,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Update ProxyManager with all proxies
|
|
||||||
const manager = ProxyManager.getInstance();
|
|
||||||
await manager.updateProxies(allProxies);
|
|
||||||
|
|
||||||
logger.info('Proxy synchronization completed', {
|
|
||||||
synchronized: allProxies.length,
|
|
||||||
working: workingCount,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
logger.error('Failed to sync proxies', { error });
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get synchronization status
|
|
||||||
*/
|
|
||||||
getStatus(): { isRunning: boolean; lastSync?: Date } {
|
|
||||||
return {
|
|
||||||
isRunning: this.isRunning,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export singleton instance
|
|
||||||
let syncServiceInstance: ProxySyncService | null = null;
|
|
||||||
|
|
||||||
export function getProxySyncService(): ProxySyncService {
|
|
||||||
if (!syncServiceInstance) {
|
|
||||||
syncServiceInstance = new ProxySyncService();
|
|
||||||
}
|
|
||||||
return syncServiceInstance;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convenience functions
|
|
||||||
export async function startProxySync(intervalMs?: number): Promise<void> {
|
|
||||||
const service = getProxySyncService();
|
|
||||||
await service.start(intervalMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function stopProxySync(): void {
|
|
||||||
const service = getProxySyncService();
|
|
||||||
service.stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function syncProxiesOnce(): Promise<void> {
|
|
||||||
const service = getProxySyncService();
|
|
||||||
await service.syncProxies();
|
|
||||||
}
|
|
||||||
|
|
@ -96,6 +96,7 @@
|
||||||
"@primeng/themes": "^19.1.3",
|
"@primeng/themes": "^19.1.3",
|
||||||
"@tanstack/table-core": "^8.21.3",
|
"@tanstack/table-core": "^8.21.3",
|
||||||
"@types/pg": "^8.15.4",
|
"@types/pg": "^8.15.4",
|
||||||
|
"awilix": "^12.0.5",
|
||||||
"bullmq": "^5.53.2",
|
"bullmq": "^5.53.2",
|
||||||
"ioredis": "^5.6.1",
|
"ioredis": "^5.6.1",
|
||||||
"pg": "^8.16.0",
|
"pg": "^8.16.0",
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue