import type { TeHandler } from '../te.handler';

const STALE_DAYS = 30;

export async function crawlScheduler(this: TeHandler): Promise<{ scheduled: number }> {
  const { logger, mongodb } = this;

  const cutoff = new Date(Date.now() - STALE_DAYS * 24 * 60 * 60 * 1000);

  const staleUrls = await mongodb?.find('teUrls', {
    $or: [
      { lastCrawled: { $exists: false } },
      { lastCrawled: { $lt: cutoff } }
    ],
    skipReason: { $exists: false }
  }, {
    sort: { lastCrawled: 1 },
    projection: { url: 1 }
  });

  if (!staleUrls?.length) {
    logger.debug('No stale URLs to schedule');
    return { scheduled: 0 };
  }

  logger.info(`Scheduling ${staleUrls.length} stale/uncrawled URLs for spidering`);

  for (let i = 0; i < staleUrls.length; i++) {
    const { url } = staleUrls[i];
    await this.scheduleOperation('te-spider', { url }, {
      jobId: `spider-${url.replace(/\//g, '-')}`,
      priority: 8,
      delay: i * 200
    });
  }

  return { scheduled: staleUrls.length };
}
