import { getRandomUserAgent } from '@stock-bot/utils';
import * as cheerio from 'cheerio';
import { TE_CONFIG } from '../shared/config';
import type { TeHandler } from '../te.handler';

const SKIP_GROUPS = new Set(['World Bank', 'News', 'Calendar']);

export async function fetchIndicators(this: TeHandler): Promise<{ count: number }> {
  const { logger, mongodb, proxy } = this;

  const response = await this.http.get(`${TE_CONFIG.MAIN_URL}/indicators`, {
    proxy: proxy?.getProxy(),
    headers: {
      'User-Agent': getRandomUserAgent(),
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
      'Accept-Language': 'en-US,en;q=0.5',
    },
  });

  if (!response.ok) {
    throw new Error(`Failed to fetch indicators page: ${response.status}`);
  }

  const html = await response.text();
  const $ = cheerio.load(html);

  const indicators: Array<{ group: string; slug: string; name: string; url: string }> = [];

  $('li.list-group-item.active').each((_, groupEl) => {
    const group = $(groupEl).text().trim();
    if (SKIP_GROUPS.has(group)) return;

    $(groupEl).nextAll('li.list-group-item').each((_, li) => {
      if ($(li).hasClass('active')) return false; // next group
      const $a = $(li).find('a[href*="/country-list/"]');
      if ($a.length) {
        const href = $a.attr('href') || '';
        const name = $a.text().trim();
        const slug = href.replace('/country-list/', '');
        if (slug && name) {
          indicators.push({ group, slug, name, url: href });
        }
      }
    });
  });

  if (!indicators.length) {
    throw new Error('No indicators found on page');
  }

  logger.info(`Found ${indicators.length} indicators across groups`);

  await mongodb?.batchUpsert('teIndicators', indicators, ['slug']);

  logger.info(`Upserted ${indicators.length} indicators into teIndicators`);
  return { count: indicators.length };
}
