import { getRandomUserAgent } from '@stock-bot/utils';
import * as cheerio from 'cheerio';
import { TE_CONFIG } from '../shared/config';
import type { TeHandler } from '../te.handler';

export async function fetchCountries(this: TeHandler): Promise<{ count: number }> {
  const { logger, mongodb, proxy } = this;

  const response = await this.http.get(TE_CONFIG.COUNTRIES_URL, {
    proxy: proxy?.getProxy(),
    headers: {
      'User-Agent': getRandomUserAgent(),
      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
      'Accept-Language': 'en-US,en;q=0.5',
    },
  });

  if (!response.ok) {
    throw new Error(`Failed to fetch countries page: ${response.status}`);
  }

  const html = await response.text();
  const $ = cheerio.load(html);

  // Country links end with /indicators e.g. /united-states/indicators
  const countries = new Map<string, string>();
  $('a[href$="/indicators"]').each((_, el) => {
    const href = $(el).attr('href') || '';
    const name = $(el).text().trim();
    const slug = href.replace('/indicators', '').replace('/', '');
    if (slug && name) {
      countries.set(slug, name);
    }
  });

  if (!countries.size) {
    throw new Error('No countries found on page');
  }

  logger.info(`Found ${countries.size} countries`);

  const docs = Array.from(countries.entries()).map(([slug, name]) => ({
    slug,
    name,
    url: `/${slug}`,
  }));

  await mongodb?.batchUpsert('teCountries', docs, ['slug']);

  logger.info(`Upserted ${docs.length} countries into teCountries`);
  return { count: docs.length };
}
