linxus fs fixes
This commit is contained in:
parent
ac23b70146
commit
0b7846fe67
292 changed files with 41947 additions and 41947 deletions
|
|
@ -1,140 +1,140 @@
|
|||
import { ProxyInfo } from 'libs/http/src/types';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { BatchProcessor } from '../utils/batch-processor';
|
||||
|
||||
// Create logger for this provider
|
||||
const logger = getLogger('proxy-provider');
|
||||
|
||||
// This will run at the same time each day as when the app started
|
||||
const getEvery24HourCron = (): string => {
|
||||
const now = new Date();
|
||||
const hours = now.getHours();
|
||||
const minutes = now.getMinutes();
|
||||
return `${minutes} ${hours} * * *`; // Every day at startup time
|
||||
};
|
||||
|
||||
export const proxyProvider: ProviderConfig = {
|
||||
name: 'proxy-service',
|
||||
service: 'proxy',
|
||||
operations: {
|
||||
'fetch-and-check': async (payload: { sources?: string[] }) => {
|
||||
const { proxyService } = await import('./proxy.tasks');
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
|
||||
const proxies = await proxyService.fetchProxiesFromSources();
|
||||
|
||||
if (proxies.length === 0) {
|
||||
return { proxiesFetched: 0, jobsCreated: 0 };
|
||||
}
|
||||
|
||||
const batchProcessor = new BatchProcessor(queueManager);
|
||||
|
||||
// Simplified configuration
|
||||
const result = await batchProcessor.processItems({
|
||||
items: proxies,
|
||||
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
|
||||
totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '4') * 60 * 60 * 1000 ,
|
||||
jobNamePrefix: 'proxy',
|
||||
operation: 'check-proxy',
|
||||
service: 'proxy',
|
||||
provider: 'proxy-service',
|
||||
priority: 2,
|
||||
useBatching: process.env.PROXY_DIRECT_MODE !== 'true', // Simple boolean flag
|
||||
createJobData: (proxy: ProxyInfo) => ({
|
||||
proxy,
|
||||
source: 'fetch-and-check'
|
||||
}),
|
||||
removeOnComplete: 5,
|
||||
removeOnFail: 3
|
||||
});
|
||||
|
||||
return {
|
||||
proxiesFetched: result.totalItems,
|
||||
...result
|
||||
};
|
||||
},
|
||||
|
||||
'process-proxy-batch': async (payload: any) => {
|
||||
// Process a batch of proxies - uses the fetch-and-check JobNamePrefix process-(proxy)-batch
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
const batchProcessor = new BatchProcessor(queueManager);
|
||||
return await batchProcessor.processBatch(
|
||||
payload,
|
||||
(proxy: ProxyInfo) => ({
|
||||
proxy,
|
||||
source: payload.config?.source || 'batch-processing'
|
||||
})
|
||||
);
|
||||
},
|
||||
|
||||
'check-proxy': async (payload: {
|
||||
proxy: ProxyInfo,
|
||||
source?: string,
|
||||
batchIndex?: number,
|
||||
itemIndex?: number,
|
||||
total?: number
|
||||
}) => {
|
||||
const { checkProxy } = await import('./proxy.tasks');
|
||||
|
||||
try {
|
||||
const result = await checkProxy(payload.proxy);
|
||||
|
||||
logger.debug('Proxy validated', {
|
||||
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
||||
isWorking: result.isWorking,
|
||||
responseTime: result.responseTime,
|
||||
batchIndex: payload.batchIndex
|
||||
});
|
||||
|
||||
return {
|
||||
result,
|
||||
proxy: payload.proxy,
|
||||
// Only include batch info if it exists (for batch mode)
|
||||
...(payload.batchIndex !== undefined && {
|
||||
batchInfo: {
|
||||
batchIndex: payload.batchIndex,
|
||||
itemIndex: payload.itemIndex,
|
||||
total: payload.total,
|
||||
source: payload.source
|
||||
}
|
||||
})
|
||||
};
|
||||
} catch (error) {
|
||||
logger.warn('Proxy validation failed', {
|
||||
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
batchIndex: payload.batchIndex
|
||||
});
|
||||
|
||||
return {
|
||||
result: { isWorking: false, error: String(error) },
|
||||
proxy: payload.proxy,
|
||||
// Only include batch info if it exists (for batch mode)
|
||||
...(payload.batchIndex !== undefined && {
|
||||
batchInfo: {
|
||||
batchIndex: payload.batchIndex,
|
||||
itemIndex: payload.itemIndex,
|
||||
total: payload.total,
|
||||
source: payload.source
|
||||
}
|
||||
})
|
||||
};
|
||||
}
|
||||
}
|
||||
},
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'proxy-maintenance',
|
||||
operation: 'fetch-and-check',
|
||||
payload: {},
|
||||
// should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
cronPattern: getEvery24HourCron(),
|
||||
priority: 5,
|
||||
immediately: true,
|
||||
description: 'Fetch and validate proxy list from sources'
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
|
||||
import { ProxyInfo } from 'libs/http/src/types';
|
||||
import { ProviderConfig } from '../services/provider-registry.service';
|
||||
import { getLogger } from '@stock-bot/logger';
|
||||
import { BatchProcessor } from '../utils/batch-processor';
|
||||
|
||||
// Create logger for this provider
|
||||
const logger = getLogger('proxy-provider');
|
||||
|
||||
// This will run at the same time each day as when the app started
|
||||
const getEvery24HourCron = (): string => {
|
||||
const now = new Date();
|
||||
const hours = now.getHours();
|
||||
const minutes = now.getMinutes();
|
||||
return `${minutes} ${hours} * * *`; // Every day at startup time
|
||||
};
|
||||
|
||||
export const proxyProvider: ProviderConfig = {
|
||||
name: 'proxy-service',
|
||||
service: 'proxy',
|
||||
operations: {
|
||||
'fetch-and-check': async (payload: { sources?: string[] }) => {
|
||||
const { proxyService } = await import('./proxy.tasks');
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
|
||||
const proxies = await proxyService.fetchProxiesFromSources();
|
||||
|
||||
if (proxies.length === 0) {
|
||||
return { proxiesFetched: 0, jobsCreated: 0 };
|
||||
}
|
||||
|
||||
const batchProcessor = new BatchProcessor(queueManager);
|
||||
|
||||
// Simplified configuration
|
||||
const result = await batchProcessor.processItems({
|
||||
items: proxies,
|
||||
batchSize: parseInt(process.env.PROXY_BATCH_SIZE || '200'),
|
||||
totalDelayMs: parseInt(process.env.PROXY_VALIDATION_HOURS || '4') * 60 * 60 * 1000 ,
|
||||
jobNamePrefix: 'proxy',
|
||||
operation: 'check-proxy',
|
||||
service: 'proxy',
|
||||
provider: 'proxy-service',
|
||||
priority: 2,
|
||||
useBatching: process.env.PROXY_DIRECT_MODE !== 'true', // Simple boolean flag
|
||||
createJobData: (proxy: ProxyInfo) => ({
|
||||
proxy,
|
||||
source: 'fetch-and-check'
|
||||
}),
|
||||
removeOnComplete: 5,
|
||||
removeOnFail: 3
|
||||
});
|
||||
|
||||
return {
|
||||
proxiesFetched: result.totalItems,
|
||||
...result
|
||||
};
|
||||
},
|
||||
|
||||
'process-proxy-batch': async (payload: any) => {
|
||||
// Process a batch of proxies - uses the fetch-and-check JobNamePrefix process-(proxy)-batch
|
||||
const { queueManager } = await import('../services/queue.service');
|
||||
const batchProcessor = new BatchProcessor(queueManager);
|
||||
return await batchProcessor.processBatch(
|
||||
payload,
|
||||
(proxy: ProxyInfo) => ({
|
||||
proxy,
|
||||
source: payload.config?.source || 'batch-processing'
|
||||
})
|
||||
);
|
||||
},
|
||||
|
||||
'check-proxy': async (payload: {
|
||||
proxy: ProxyInfo,
|
||||
source?: string,
|
||||
batchIndex?: number,
|
||||
itemIndex?: number,
|
||||
total?: number
|
||||
}) => {
|
||||
const { checkProxy } = await import('./proxy.tasks');
|
||||
|
||||
try {
|
||||
const result = await checkProxy(payload.proxy);
|
||||
|
||||
logger.debug('Proxy validated', {
|
||||
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
||||
isWorking: result.isWorking,
|
||||
responseTime: result.responseTime,
|
||||
batchIndex: payload.batchIndex
|
||||
});
|
||||
|
||||
return {
|
||||
result,
|
||||
proxy: payload.proxy,
|
||||
// Only include batch info if it exists (for batch mode)
|
||||
...(payload.batchIndex !== undefined && {
|
||||
batchInfo: {
|
||||
batchIndex: payload.batchIndex,
|
||||
itemIndex: payload.itemIndex,
|
||||
total: payload.total,
|
||||
source: payload.source
|
||||
}
|
||||
})
|
||||
};
|
||||
} catch (error) {
|
||||
logger.warn('Proxy validation failed', {
|
||||
proxy: `${payload.proxy.host}:${payload.proxy.port}`,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
batchIndex: payload.batchIndex
|
||||
});
|
||||
|
||||
return {
|
||||
result: { isWorking: false, error: String(error) },
|
||||
proxy: payload.proxy,
|
||||
// Only include batch info if it exists (for batch mode)
|
||||
...(payload.batchIndex !== undefined && {
|
||||
batchInfo: {
|
||||
batchIndex: payload.batchIndex,
|
||||
itemIndex: payload.itemIndex,
|
||||
total: payload.total,
|
||||
source: payload.source
|
||||
}
|
||||
})
|
||||
};
|
||||
}
|
||||
}
|
||||
},
|
||||
scheduledJobs: [
|
||||
{
|
||||
type: 'proxy-maintenance',
|
||||
operation: 'fetch-and-check',
|
||||
payload: {},
|
||||
// should remove and just run at the same time so app restarts dont keeping adding same jobs
|
||||
cronPattern: getEvery24HourCron(),
|
||||
priority: 5,
|
||||
immediately: true,
|
||||
description: 'Fetch and validate proxy list from sources'
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue