boki/stock-bot
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

fixed batching and waiting priority plus cleanup

Browse source
This commit is contained in:
Boki 2025-06-11 12:56:07 -04:00
parent e4d5dba73a
commit 6fb98c69f2
6 changed files with 121 additions and 155 deletions
Download patch file Download diff file Expand all files Collapse all files

61
apps/data-service/src/providers/proxy.provider.ts
View file

@ -37,21 +37,14 @@ export const proxyProvider: ProviderConfig = {
}),
queueManager,
{
totalDelayHours: 0.1, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
totalDelayHours: 12, //parseFloat(process.env.PROXY_VALIDATION_HOURS || '1'),
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
useBatching: process.env.PROXY_DIRECT_MODE !== 'true',
priority: 2,
provider: 'proxy-provider',
operation: 'check-proxy',
}
);
return {
proxiesFetched: result.totalItems,
jobsCreated: result.jobsCreated,
mode: result.mode,
batchesCreated: result.batchesCreated,
processingTimeMs: result.duration,
};
return result;
},
'process-batch-items': async (payload: any) => {
// Process a batch using the simplified batch helpers
@ -77,55 +70,29 @@ export const proxyProvider: ProviderConfig = {
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
isWorking: result.isWorking,
responseTime: result.responseTime,
batchIndex: payload.batchIndex,
});
return {
result,
proxy: payload.proxy,
// Only include batch info if it exists (for batch mode)
...(payload.batchIndex !== undefined && {
batchInfo: {
batchIndex: payload.batchIndex,
itemIndex: payload.itemIndex,
total: payload.total,
source: payload.source,
},
}),
};
return { result, proxy: payload.proxy };
} catch (error) {
logger.warn('Proxy validation failed', {
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
error: error instanceof Error ? error.message : String(error),
batchIndex: payload.batchIndex,
});
return {
result: { isWorking: false, error: String(error) },
proxy: payload.proxy,
// Only include batch info if it exists (for batch mode)
...(payload.batchIndex !== undefined && {
batchInfo: {
batchIndex: payload.batchIndex,
itemIndex: payload.itemIndex,
total: payload.total,
source: payload.source,
},
}),
};
return { result: { isWorking: false, error: String(error) }, proxy: payload.proxy };
}
},
},
scheduledJobs: [
{
type: 'proxy-maintenance',
operation: 'fetch-and-check',
payload: {},
// should remove and just run at the same time so app restarts dont keeping adding same jobs
cronPattern: getEvery24HourCron(),
priority: 5,
immediately: true, // Don't run immediately during startup to avoid conflicts
description: 'Fetch and validate proxy list from sources',
},
// {
// type: 'proxy-maintenance',
// operation: 'fetch-and-check',
// payload: {},
// // should remove and just run at the same time so app restarts dont keeping adding same jobs
// cronPattern: getEvery24HourCron(),
// priority: 5,
// immediately: true, // Don't run immediately during startup to avoid conflicts
// description: 'Fetch and validate proxy list from sources',
// },
],
};

191
apps/data-service/src/providers/proxy.tasks.ts
View file

@ -54,103 +54,102 @@ const PROXY_CONFIG = {
url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/http.txt',
protocol: 'http',
},
// {
// id: 'speedx',
// url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',
// protocol: 'http',
// },
// {
// id: 'monosans',
// url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',
// protocol: 'http',
// },
{
id: 'speedx',
url: 'https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt',
protocol: 'http',
},
{
id: 'monosans',
url: 'https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt',
protocol: 'http',
},
{
id: 'murong',
url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',
protocol: 'http',
},
{
id: 'vakhov-fresh',
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',
protocol: 'http',
},
{
id: 'kangproxy',
url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',
protocol: 'http',
},
{
id: 'gfpcom',
url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt',
protocol: 'http',
},
{
id: 'dpangestuw',
url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',
protocol: 'http',
},
{
id: 'gitrecon',
url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',
protocol: 'http',
},
{
id: 'vakhov-master',
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',
protocol: 'http',
},
{
id: 'breaking-tech',
url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt',
protocol: 'http',
},
{
id: 'ercindedeoglu',
url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',
protocol: 'http',
},
{
id: 'tuanminpay',
url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',
protocol: 'http',
},
// {
// id: 'murong',
// url: 'https://raw.githubusercontent.com/MuRongPIG/Proxy-Master/main/http.txt',
// protocol: 'http',
// },
// {
// id: 'vakhov-fresh',
// url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt',
// protocol: 'http',
// },
// {
// id: 'kangproxy',
// url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/http/http.txt',
// protocol: 'http',
// },
// {
// id: 'gfpcom',
// url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/http.txt',
// protocol: 'http',
// },
// {
// id: 'dpangestuw',
// url: 'https://raw.githubusercontent.com/dpangestuw/Free-Proxy/refs/heads/main/http_proxies.txt',
// protocol: 'http',
// },
// {
// id: 'gitrecon',
// url: 'https://raw.githubusercontent.com/gitrecon1455/fresh-proxy-list/refs/heads/main/proxylist.txt',
// protocol: 'http',
// },
// {
// id: 'vakhov-master',
// url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/http.txt',
// protocol: 'http',
// },
// {
// id: 'breaking-tech',
// url: 'https://raw.githubusercontent.com/BreakingTechFr/Proxy_Free/refs/heads/main/proxies/http.txt',
// protocol: 'http',
// },
// {
// id: 'ercindedeoglu',
// url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt',
// protocol: 'http',
// },
// {
// id: 'tuanminpay',
// url: 'https://raw.githubusercontent.com/TuanMinPay/live-proxy/master/http.txt',
// protocol: 'http',
// },
// {
// id: 'r00tee-https',
// url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',
// protocol: 'https',
// },
// {
// id: 'ercindedeoglu-https',
// url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',
// protocol: 'https',
// },
// {
// id: 'vakhov-fresh-https',
// url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt',
// protocol: 'https',
// },
// {
// id: 'databay-https',
// url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',
// protocol: 'https',
// },
// {
// id: 'kangproxy-https',
// url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',
// protocol: 'https',
// },
// {
// id: 'zloi-user-https',
// url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',
// protocol: 'https',
// },
// {
// id: 'gfpcom-https',
// url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',
// protocol: 'https',
// },
{
id: 'r00tee-https',
url: 'https://raw.githubusercontent.com/r00tee/Proxy-List/refs/heads/main/Https.txt',
protocol: 'https',
},
{
id: 'ercindedeoglu-https',
url: 'https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/https.txt',
protocol: 'https',
},
{
id: 'vakhov-fresh-https',
url: 'https://raw.githubusercontent.com/vakhov/fresh-proxy-list/refs/heads/master/https.txt',
protocol: 'https',
},
{
id: 'databay-https',
url: 'https://raw.githubusercontent.com/databay-labs/free-proxy-list/refs/heads/master/https.txt',
protocol: 'https',
},
{
id: 'kangproxy-https',
url: 'https://raw.githubusercontent.com/officialputuid/KangProxy/refs/heads/KangProxy/https/https.txt',
protocol: 'https',
},
{
id: 'zloi-user-https',
url: 'https://raw.githubusercontent.com/zloi-user/hideip.me/refs/heads/master/https.txt',
protocol: 'https',
},
{
id: 'gfpcom-https',
url: 'https://raw.githubusercontent.com/gfpcom/free-proxy-list/refs/heads/main/list/https.txt',
protocol: 'https',
},
],
};