thing im pretty much done with extracting the queue and making it reususable, maybe just a few more change to be able to making the batch names a bit more specific

apps/data-service/src/providers/proxy.provider.ts

@@ -14,7 +14,57 @@ export function initializeProxyProvider() {
 
   const proxyProviderConfig: ProviderConfigWithSchedule = {
     name: 'proxy',
+
     operations: {
+      'fetch-from-sources': async _payload => {
+        // Fetch proxies from all configured sources
+        logger.info('Processing fetch proxies from sources request');
+        const { fetchProxiesFromSources } = await import('./proxy.tasks');
+        const { processItems, queueManager } = await import('../index');
+        
+        // Fetch all proxies from sources
+        const proxies = await fetchProxiesFromSources();
+        logger.info('Fetched proxies from sources', { count: proxies.length });
+
+        if (proxies.length === 0) {
+          logger.warn('No proxies fetched from sources');
+          return { processed: 0, successful: 0 };
+        }
+
+        // Batch process the proxies through check-proxy operation
+        const batchResult = await processItems(
+          proxies,
+          queueManager,
+          { 
+            provider: 'proxy', 
+            operation: 'check-proxy',
+            totalDelayHours: 0.083, // 5 minutes (5/60 hours)
+            batchSize: 50, // Process 50 proxies per batch
+            priority: 3,
+            useBatching: true,
+            retries: 1,
+            ttl: 30000, // 30 second timeout per proxy check
+            removeOnComplete: 5,
+            removeOnFail: 3,
+          }
+        );
+
+        logger.info('Batch proxy validation completed', {
+          totalProxies: proxies.length,
+          jobsCreated: batchResult.jobsCreated,
+          mode: batchResult.mode,
+          batchesCreated: batchResult.batchesCreated,
+          duration: `${batchResult.duration}ms`
+        });
+
+        return {
+          processed: proxies.length,
+          jobsCreated: batchResult.jobsCreated,
+          batchesCreated: batchResult.batchesCreated,
+          mode: batchResult.mode
+        };
+      },
+
       'check-proxy': async (payload: ProxyInfo) => {
         // payload is now the raw proxy info object
         logger.debug('Processing proxy check request', {
@@ -23,12 +73,6 @@ export function initializeProxyProvider() {
         const { checkProxy } = await import('./proxy.tasks');
         return checkProxy(payload);
       },
-      'fetch-from-sources': async _payload => {
-        // Fetch proxies from all configured sources
-        logger.info('Processing fetch proxies from sources request');
-        const { fetchProxiesFromSources } = await import('./proxy.tasks');
-        return fetchProxiesFromSources();
-      },
     },
     scheduledJobs: [
       {
@@ -38,7 +82,7 @@ export function initializeProxyProvider() {
         cronPattern: '0 */2 * * *', // Every 2 hours
         priority: 5,
         description: 'Fetch and validate proxy list from sources',
-        // immediately: true, // Don't run immediately during startup to avoid conflicts
+        immediately: true, // Don't run immediately during startup to avoid conflicts
       },
     ],
   };