fixed up delay time
This commit is contained in:
parent
16599c86da
commit
be807378a3
4 changed files with 61 additions and 47 deletions
|
|
@ -35,7 +35,7 @@ export const proxyProvider: ProviderConfig = {
|
|||
source: 'batch-processing'
|
||||
}),
|
||||
queueManager, {
|
||||
totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '7') * 60 * 60 * 1000,
|
||||
totalDelayHours: 0.1,//parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
|
||||
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
|
||||
useBatching: process.env.PROXY_DIRECT_MODE !== 'true',
|
||||
priority: 2,
|
||||
|
|
@ -114,16 +114,16 @@ export const proxyProvider: ProviderConfig = {
|
|||
}
|
||||
},
|
||||
scheduledJobs: [
|
||||
// {
|
||||
// type: 'proxy-maintenance',
|
||||
// operation: 'fetch-and-check',
|
||||
// payload: {},
|
||||
// // should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
// cronPattern: getEvery24HourCron(),
|
||||
// priority: 5,
|
||||
// immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
// description: 'Fetch and validate proxy list from sources'
|
||||
// }
|
||||
{
|
||||
type: 'proxy-maintenance',
|
||||
operation: 'fetch-and-check',
|
||||
payload: {},
|
||||
// should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
cronPattern: getEvery24HourCron(),
|
||||
priority: 5,
|
||||
immediately: true, // Don't run immediately during startup to avoid conflicts
|
||||
description: 'Fetch and validate proxy list from sources'
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -26,30 +26,32 @@ const PROXY_CONFIG = {
|
|||
PROXY_SOURCES: [
|
||||
{id: 'prxchk', url: 'https://raw.githubusercontent.com/prxchk/proxy-list/main/http.txt', protocol: 'http'},
|
||||
{id: 'casals', url: 'https://raw.githubusercontent.com/casals-ar/proxy-list/main/http', protocol: 'http'},
|
||||
{id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'},
|
||||
{id: 'vakhov-fresh', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', protocol: 'http'},
|
||||
{id: 'sunny9577', url: 'https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt', protocol: 'http'},
|
||||
{id: 'kangproxy', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', protocol: 'http'},
|
||||
{id: 'gfpcom', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http'},
|
||||
{id: 'dpangestuw', url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', protocol: 'http'},
|
||||
{id: 'gitrecon', url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', protocol: 'http'},
|
||||
{id: 'themiralay', url: 'https://raw.githubusercontent.com/themiralay/Proxy-List-World/refs/heads/master/data.txt', protocol: 'http'},
|
||||
{id: 'vakhov-master', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
|
||||
{id: 'casa-ls', url: 'https://raw.githubusercontent.com/casa-ls/proxy-list/refs/heads/main/http', protocol: 'http'},
|
||||
{id: 'databay', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
|
||||
{id: 'breaking-tech', url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http'},
|
||||
{id: 'speedx', url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt', protocol: 'http'},
|
||||
{id: 'ercindedeoglu', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', protocol: 'http'},
|
||||
{id: 'monosans', url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt', protocol: 'http'},
|
||||
{id: 'tuanminpay', url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', protocol: 'http'},
|
||||
|
||||
// {url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https' },
|
||||
// {url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',protocol: 'https', },
|
||||
// {url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',protocol: 'https', },
|
||||
|
||||
// {id: 'murong', url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt', protocol: 'http'},
|
||||
// {id: 'vakhov-fresh', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt', protocol: 'http'},
|
||||
// {id: 'kangproxy', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt', protocol: 'http'},
|
||||
// {id: 'gfpcom', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt', protocol: 'http'},
|
||||
// {id: 'dpangestuw', url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt', protocol: 'http'},
|
||||
// {id: 'gitrecon', url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt', protocol: 'http'},
|
||||
// {id: 'vakhov-master', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt', protocol: 'http'},
|
||||
// {id: 'breaking-tech', url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt', protocol: 'http'},
|
||||
// {id: 'ercindedeoglu', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt', protocol: 'http'},
|
||||
// {id: 'tuanminpay', url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt', protocol: 'http'},
|
||||
|
||||
// {id: 'r00tee-https', url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt', protocol: 'https'},
|
||||
// {id: 'ercindedeoglu-https', url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt', protocol: 'https'},
|
||||
{id: 'vakhov-fresh-https', url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt', protocol: 'https'},
|
||||
// {id: 'databay-https', url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt', protocol: 'https'},
|
||||
// {id: 'kangproxy-https', url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt', protocol: 'https'},
|
||||
// {id: 'zloi-user-https', url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt', protocol: 'https'},
|
||||
// {id: 'gfpcom-https', url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt', protocol: 'https'},
|
||||
]
|
||||
};
|
||||
|
||||
|
|
@ -289,7 +291,7 @@ export async function checkProxy(proxy: ProxyInfo): Promise<ProxyInfo> {
|
|||
|
||||
if (isWorking && !JSON.stringify(response.data).includes(PROXY_CONFIG.CHECK_IP)) {
|
||||
success = true;
|
||||
await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, PROXY_CONFIG.CACHE_TTL);
|
||||
await cache.set(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`, result, { ttl: PROXY_CONFIG.CACHE_TTL });
|
||||
} else {
|
||||
await cache.del(`${PROXY_CONFIG.CACHE_KEY}:${proxy.protocol}://${proxy.host}:${proxy.port}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,22 +12,30 @@ export const testRoutes = new Hono();
|
|||
// Test endpoint for new functional batch processing
|
||||
testRoutes.post('/api/test/batch-symbols', async (c) => {
|
||||
try {
|
||||
const { symbols, useBatching = false, totalDelayMs = 60000 } = await c.req.json();
|
||||
const { processSymbols } = await import('../utils/batch-helpers');
|
||||
const { symbols, useBatching = false, totalDelayHours = 1 } = await c.req.json();
|
||||
const { processItems } = await import('../utils/batch-helpers');
|
||||
|
||||
if (!symbols || !Array.isArray(symbols)) {
|
||||
return c.json({ status: 'error', message: 'symbols array is required' }, 400);
|
||||
}
|
||||
|
||||
const result = await processSymbols(symbols, queueManager, {
|
||||
operation: 'live-data',
|
||||
service: 'test',
|
||||
provider: 'test-provider',
|
||||
totalDelayMs,
|
||||
useBatching,
|
||||
batchSize: 10,
|
||||
priority: 1
|
||||
});
|
||||
const result = await processItems(
|
||||
symbols,
|
||||
(symbol, index) => ({
|
||||
symbol,
|
||||
index,
|
||||
timestamp: new Date().toISOString()
|
||||
}),
|
||||
queueManager,
|
||||
{
|
||||
totalDelayHours,
|
||||
useBatching,
|
||||
batchSize: 10,
|
||||
priority: 1,
|
||||
provider: 'test-provider',
|
||||
operation: 'live-data'
|
||||
}
|
||||
);
|
||||
|
||||
return c.json({
|
||||
status: 'success',
|
||||
|
|
@ -42,7 +50,7 @@ testRoutes.post('/api/test/batch-symbols', async (c) => {
|
|||
|
||||
testRoutes.post('/api/test/batch-custom', async (c) => {
|
||||
try {
|
||||
const { items, useBatching = false, totalDelayMs = 30000 } = await c.req.json();
|
||||
const { items, useBatching = false, totalDelayHours = 0.5 } = await c.req.json();
|
||||
const { processItems } = await import('../utils/batch-helpers');
|
||||
|
||||
if (!items || !Array.isArray(items)) {
|
||||
|
|
@ -58,10 +66,12 @@ testRoutes.post('/api/test/batch-custom', async (c) => {
|
|||
}),
|
||||
queueManager,
|
||||
{
|
||||
totalDelayMs,
|
||||
totalDelayHours,
|
||||
useBatching,
|
||||
batchSize: 5,
|
||||
priority: 1
|
||||
priority: 1,
|
||||
provider: 'test-provider',
|
||||
operation: 'custom-test'
|
||||
}
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ const logger = getLogger('batch-helpers');
|
|||
|
||||
// Simple interfaces
|
||||
export interface ProcessOptions {
|
||||
totalDelayMs: number;
|
||||
totalDelayHours: number;
|
||||
batchSize?: number;
|
||||
priority?: number;
|
||||
useBatching?: boolean;
|
||||
|
|
@ -76,7 +76,7 @@ export async function processItems<T>(
|
|||
totalItems: items.length,
|
||||
mode: options.useBatching ? 'batch' : 'direct',
|
||||
batchSize: options.batchSize,
|
||||
totalDelayHours: (options.totalDelayMs / 1000 / 60 / 60).toFixed(1)
|
||||
totalDelayHours: options.totalDelayHours
|
||||
});
|
||||
|
||||
try {
|
||||
|
|
@ -109,7 +109,8 @@ async function processDirect<T>(
|
|||
options: ProcessOptions
|
||||
): Promise<Omit<BatchResult, 'duration'>> {
|
||||
|
||||
const delayPerItem = Math.floor(options.totalDelayMs / items.length);
|
||||
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000;
|
||||
const delayPerItem = Math.floor(totalDelayMs / items.length);
|
||||
|
||||
logger.info('Creating direct jobs', {
|
||||
totalItems: items.length,
|
||||
|
|
@ -155,7 +156,8 @@ async function processBatched<T>(
|
|||
|
||||
const batchSize = options.batchSize || 100;
|
||||
const batches = createBatches(items, batchSize);
|
||||
const delayPerBatch = Math.floor(options.totalDelayMs / batches.length);
|
||||
const totalDelayMs = options.totalDelayHours * 60 * 60 * 1000;
|
||||
const delayPerBatch = Math.floor(totalDelayMs / batches.length);
|
||||
|
||||
logger.info('Creating batch jobs', {
|
||||
totalItems: items.length,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue