boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fixed up delay time

Browse source
This commit is contained in:
Boki 2025-06-11 08:33:36 -04:00
parent 16599c86da
commit be807378a3
4 changed files with 61 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

22
apps/data-service/src/providers/proxy.provider.ts
View file

@ -35,7 +35,7 @@ export const proxyProvider: ProviderConfig = {
source: 'batch-processing' source: 'batch-processing'
}), }),
queueManager, { queueManager, {
totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '7') * 60 * 60 * 1000, totalDelayHours: 0.1,//parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'), batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
useBatching: process.env.PROXY_DIRECT_MODE !== 'true', useBatching: process.env.PROXY_DIRECT_MODE !== 'true',
priority: 2, priority: 2,
@ -114,16 +114,16 @@ export const proxyProvider: ProviderConfig = {
} }
}, },
scheduledJobs: [ scheduledJobs: [
// { {
// type: 'proxy-maintenance', type: 'proxy-maintenance',
// operation: 'fetch-and-check', operation: 'fetch-and-check',
// payload: {}, payload: {},
// // should remove and just run at the same time so app restarts dont keeping adding same jobs // should remove and just run at the same time so app restarts dont keeping adding same jobs
// cronPattern: getEvery24HourCron(), cronPattern: getEvery24HourCron(),
// priority: 5, priority: 5,
// immediately: true, // Don't run immediately during startup to avoid conflicts immediately: true, // Don't run immediately during startup to avoid conflicts
// description: 'Fetch and validate proxy list from sources' description: 'Fetch and validate proxy list from sources'
// } }
] ]
}; };

38
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -26,30 +26,32 @@ const PROXY_CONFIG = {
PROXY_SOURCES: [ PROXY_SOURCES: [
{id: 'prxchk', url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt', protocol: 'http'}, {id: 'prxchk', url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt', protocol: 'http'},
{id: 'casals', url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http', protocol: 'http'}, {id: 'casals', url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http', protocol: 'http'},
{id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'},
{id: 'vakhov-fresh', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', protocol: 'http'},
{id: 'sunny9577', url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', protocol: 'http'}, {id: 'sunny9577', url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', protocol: 'http'},
{id: 'kangproxy', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', protocol: 'http'},
{id: 'gfpcom', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http'},
{id: 'dpangestuw', url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', protocol: 'http'},
{id: 'gitrecon', url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', protocol: 'http'},
{id: 'themiralay', url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt', protocol: 'http'}, {id: 'themiralay', url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt', protocol: 'http'},
{id: 'vakhov-master', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
{id: 'casa-ls', url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http', protocol: 'http'}, {id: 'casa-ls', url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http', protocol: 'http'},
{id: 'databay', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt', protocol: 'http'}, {id: 'databay', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
{id: 'breaking-tech', url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http'},
{id: 'speedx', url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', protocol: 'http'}, {id: 'speedx', url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', protocol: 'http'},
{id: 'ercindedeoglu', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', protocol: 'http'},
{id: 'monosans', url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', protocol: 'http'}, {id: 'monosans', url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', protocol: 'http'},
{id: 'tuanminpay', url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', protocol: 'http'},
// {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', }, // {id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'},
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https' }, // {id: 'vakhov-fresh', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', protocol: 'http'},
// {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', }, // {id: 'kangproxy', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', protocol: 'http'},
// {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',protocol: 'https', }, // {id: 'gfpcom', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http'},
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', }, // {id: 'dpangestuw', url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', protocol: 'http'},
// {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', }, // {id: 'gitrecon', url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', protocol: 'http'},
// {id: 'vakhov-master', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
// {id: 'breaking-tech', url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http'},
// {id: 'ercindedeoglu', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', protocol: 'http'},
// {id: 'tuanminpay', url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', protocol: 'http'},
// {id: 'r00tee-https', url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt', protocol: 'https'},
// {id: 'ercindedeoglu-https', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt', protocol: 'https'},
{id: 'vakhov-fresh-https', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https'},
// {id: 'databay-https', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt', protocol: 'https'},
// {id: 'kangproxy-https', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt', protocol: 'https'},
// {id: 'zloi-user-https', url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt', protocol: 'https'},
// {id: 'gfpcom-https', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt', protocol: 'https'},
] ]
}; };
@ -289,7 +291,7 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) { if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
success = true; success = true;
await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL); await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, { ttl: PROXY_CONFIG.CACHE_TTL });
} else { } else {
await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`); await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
} }

34
apps/data-service/src/routes/test.routes.ts
View file

@ -12,22 +12,30 @@ export const testRoutes = new Hono();
// Test endpoint for new functional batch processing // Test endpoint for new functional batch processing
testRoutes.post('/api/test/batch-symbols', async (c) => { testRoutes.post('/api/test/batch-symbols', async (c) => {
try { try {
const { symbols, useBatching = false, totalDelayMs = 60000 } = await c.req.json(); const { symbols, useBatching = false, totalDelayHours = 1 } = await c.req.json();
const { processSymbols } = await import('../utils/batch-helpers'); const { processItems } = await import('../utils/batch-helpers');
if (!symbols || !Array.isArray(symbols)) { if (!symbols || !Array.isArray(symbols)) {
return c.json({ status: 'error', message: 'symbols array is required' }, 400); return c.json({ status: 'error', message: 'symbols array is required' }, 400);
} }
const result = await processSymbols(symbols, queueManager, { const result = await processItems(
operation: 'live-data', symbols,
service: 'test', (symbol, index) => ({
provider: 'test-provider', symbol,
totalDelayMs, index,
timestamp: new Date().toISOString()
}),
queueManager,
{
totalDelayHours,
useBatching, useBatching,
batchSize: 10, batchSize: 10,
priority: 1 priority: 1,
}); provider: 'test-provider',
operation: 'live-data'
}
);
return c.json({ return c.json({
status: 'success', status: 'success',
@ -42,7 +50,7 @@ testRoutes.post('/api/test/batch-symbols', async (c) => {
testRoutes.post('/api/test/batch-custom', async (c) => { testRoutes.post('/api/test/batch-custom', async (c) => {
try { try {
const { items, useBatching = false, totalDelayMs = 30000 } = await c.req.json(); const { items, useBatching = false, totalDelayHours = 0.5 } = await c.req.json();
const { processItems } = await import('../utils/batch-helpers'); const { processItems } = await import('../utils/batch-helpers');
if (!items || !Array.isArray(items)) { if (!items || !Array.isArray(items)) {
@ -58,10 +66,12 @@ testRoutes.post('/api/test/batch-custom', async (c) => {
}), }),
queueManager, queueManager,
{ {
totalDelayMs, totalDelayHours,
useBatching, useBatching,
batchSize: 5, batchSize: 5,
priority: 1 priority: 1,
provider: 'test-provider',
operation: 'custom-test'
} }
); );

10
apps/data-service/src/utils/batch-helpers.ts
View file

@ -6,7 +6,7 @@ const logger = getLogger('batch-helpers');
// Simple interfaces // Simple interfaces
export interface ProcessOptions { export interface ProcessOptions {
totalDelayMs: number; totalDelayHours: number;
batchSize?: number; batchSize?: number;
priority?: number; priority?: number;
useBatching?: boolean; useBatching?: boolean;
@ -76,7 +76,7 @@ export async function processItems<T>(
totalItems: items.length, totalItems: items.length,
mode: options.useBatching ? 'batch' : 'direct', mode: options.useBatching ? 'batch' : 'direct',
batchSize: options.batchSize, batchSize: options.batchSize,
totalDelayHours: (options.totalDelayMs / 1000 / 60 / 60).toFixed(1) totalDelayHours: options.totalDelayHours
}); });
try { try {
@ -109,7 +109,8 @@ async function processDirect<T>(
options: ProcessOptions options: ProcessOptions
): Promise<Omit<BatchResult, 'duration'>> { ): Promise<Omit<BatchResult, 'duration'>> {
const delayPerItem = Math.floor(options.totalDelayMs / items.length); const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000;
const delayPerItem = Math.floor(totalDelayMs / items.length);
logger.info('Creating direct jobs', { logger.info('Creating direct jobs', {
totalItems: items.length, totalItems: items.length,
@ -155,7 +156,8 @@ async function processBatched<T>(
const batchSize = options.batchSize || 100; const batchSize = options.batchSize || 100;
const batches = createBatches(items, batchSize); const batches = createBatches(items, batchSize);
const delayPerBatch = Math.floor(options.totalDelayMs / batches.length); const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000;
const delayPerBatch = Math.floor(totalDelayMs / batches.length);
logger.info('Creating batch jobs', { logger.info('Creating batch jobs', {
totalItems: items.length, totalItems: items.length,