import { getRandomUserAgent } from '@stock-bot/utils';
import * as cheerio from 'cheerio';
import { TE_CONFIG } from '../shared/config';
import type { TeHandler } from '../te.handler';

export async function spiderUrl(this: TeHandler, payload: { url: string }): Promise<string[] | null> {
  const { logger, mongodb } = this;
  const reqUrl = payload && payload.url ? TE_CONFIG.MAIN_URL + payload.url : TE_CONFIG.MAIN_URL;
  this.logger.info(`Spiderring URL: ${reqUrl}`, {reqUrl});

  // if( mongoRecord){
  //   const url = mongoRecord.url;
  //   if (shouldSkipUrl(url)) {
  //     logger.info(`Skipping URL ${url} as its too deep`);
  //     return null;
  //   }
  // }
  const mongoRecord = await mongodb?.findOne('teUrls', { url: payload?.url || '/' });
  if(payload && payload.url && mongoRecord && mongoRecord.lastCrawled && mongoRecord.lastCrawled.getTime() > Date.now() - 30 * 24 * 60 * 60 * 1000) {
    this.logger.info(`Skipping URL ${reqUrl} as it was already crawled in the last 24 hours`);
    return null; // Skip if already crawled in the last 24 hours
  }


  if (!payload) {
    const oneDayAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
    const records = await mongodb?.find('teUrls', {
      $or: [
        { lastCrawled: { $lt: oneDayAgo } },  // Crawled more than 24 hours ago
        { lastCrawled: { $exists: false } }   // Never crawled
      ]
    });
    this.logger.info(`Found ${records?.length || 0} records to process`);
    for (const record of records || []) {
      const url = record.url;
      if (shouldSkipUrl(url)) {
        logger.info(`Skipping URL ${url} as its too deep`);
        continue;
      }

      await this.scheduleOperation('te-spider', {
        url: record.url,
      }, {
        jobId: `te-spider-${record.url}`,
        priority: 5, // Lower priority than financial data
      });
    }
  }
  
  try {
    // 1. Fetch the HTML page
    const reqInfo = {
      proxy: 'http://5.79.66.2:13010',//this.proxy.getProxy(),
      headers: {
        'User-Agent': getRandomUserAgent(),
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Accept-Encoding': 'gzip, deflate, br',
      },
    }
    const response = await fetch(reqUrl, reqInfo);
    
    logger.debug('Response status:', { 
      status: response.status, 
      statusText: response.statusText,
      url: response.url 
    });
    
    if (!response.ok) {
      throw new Error(`Failed to fetch: ${response.status} ${response.statusText}`);
    }

    const html = await response.text();
    

    let match = html.match(/TESymbol = '([^']+)'/);
    const teSymbol = match ? match[1] : undefined;
    match = html.match(/;TELastUpdate = '([^']+)'/);
    const teLastUpdate = match ? match[1] : undefined;
    match = html.match(/; var TEChartsDatasource = '([^']+)'/);
    const teChartUrl =   match ? match[1] : undefined;
    match = html.match(/; var TEChartsToken = '([^']+)'/);
    const teChartToken = match ? match[1] : undefined;

    console.log(teSymbol, teLastUpdate, teChartUrl, teChartToken);

    const $ = cheerio.load(html);
    const urls: string[] = [];
    
    $('.list-group-item, a[href^="/"]').each((_, element) => {
      const $el = $(element);
      let url: string | undefined;
      if ($el.is('a')) {
        const href = $el.attr('href');
        if (href && href.startsWith('/') && !href.includes('.aspx')) {
          url = href;
        }
      }

      if (url && urls.indexOf(url) === -1) {
        urls.push(url);
      }
    });
    
    if (urls.length === 0) {
      throw new Error('No urls found in HTML');
    }
    
    // 3. Save to MongoDB
    try {
      if (urls.length > 0) {
        const urlMap: {url: string, lastCrawled?: Date, teSymbol? : string, teLastUpdate? : string, teChartUrl? : string, teChartToken? : string}[] = urls.map(url => ({url}));
        if( payload && payload.url) {
          urlMap.push({
            url: payload.url,
            lastCrawled: new Date(),
            teSymbol,
            teLastUpdate,
            teChartUrl,
            teChartToken,})
        }else {
          urlMap.push({url: '/', lastCrawled: new Date()})
        }

        const result = await mongodb?.batchUpsert('teUrls', urlMap, ['url']);
        logger.info('TE URLs saved to MongoDB', {
          matched: result.matchedCount,
          modified: result.modifiedCount,
          upserted: result.upsertedCount,
        });
      }
    } catch (dbError) {
      logger.error('Failed to save urls to MongoDB', { error: dbError });
      throw dbError;
    }

    for (const url of urls) {
      if (shouldSkipUrl(url)) {
        logger.info(`Skipping URL ${url} as its too deep`);
        continue; // Skip if it's a subpage or already crawled
      }
      this.scheduleOperation('te-spider', {
        url: url,
      }, {
        jobId: `te-spider-${url}`,
        priority: 5, // Lower priority than financial data
      })
    }
    
    return urls;
  } catch (error) {
    logger.error(`Failed to fetch Trading Economics URLs ${reqUrl}`, { 
      error: error instanceof Error ? error.message : String(error),
      stack: error instanceof Error ? error.stack : undefined,
    });
    return null;
  }
}

function shouldSkipUrl(url?: string): boolean {
  // Skip if it's a subpage or already crawled in the last 24 hours
  if (!url) {
    return false;
  }
  const matches = url.match(/\//g);
  return matches !== null && matches.length >= 4;
}