import { getRandomUserAgent } from '@stock-bot/utils';
import * as cheerio from 'cheerio';
import { TE_CONFIG } from '../shared/config';
import type { TeHandler } from '../te.handler';

export async function spiderUrl(this: TeHandler, payload: { url: string }): Promise<string[] | null> {
  const { logger, mongodb, proxy, http } = this;
  const urlPath = payload?.url || '/';
  const fullUrl = TE_CONFIG.MAIN_URL + urlPath;
  
  // 1. Check if already crawled recently (30 days)
  const record = await mongodb?.findOne('teUrls', { url: urlPath });
  if (record?.lastCrawled) {
    const daysSinceCrawl = (Date.now() - record.lastCrawled.getTime()) / (1000 * 60 * 60 * 24 * 30);
    if (daysSinceCrawl < 30) {
      logger.info(`Skipping ${urlPath} - crawled ${(daysSinceCrawl * 30).toFixed(1)} days ago`);
      return null;
    }
  }
  
  // 2. Fetch the page
  logger.info(`Crawling ${fullUrl}`);
  const html = await fetchPage.call(this, fullUrl);
  if (!html) {
    logger.error(`Failed to fetch ${fullUrl}`);
    // Mark as crawled anyway to avoid retrying immediately
    await mongodb?.updateOne(
      'teUrls',
      { url: urlPath },
      { 
        $set: { 
          lastCrawled: new Date(),
          foundLinks: 0,
          error: true
        }
      },
      { upsert: true }
    );
    return null;
  }
  
  // 3. Extract TE data from the HTML
  let match = html.match(/var TESymbol = '([^']+)'/);
  const teSymbol = match ? match[1] : undefined;
  
  match = html.match(/TELastUpdate = '([^']+)'/);
  const teLastUpdate = match ? match[1] : undefined;
  
  match = html.match(/TEChart = '([^']+)'/);
  const teChart = match ? match[1] : undefined;
  
  match = html.match(/var TEAlertsName\s*=\s*'([^']+)'/);
  const teAlertsName = match ? match[1] : undefined;
  
  // Extract from script tags - these appear after TEChartUrl
  match = html.match(/TEChartUrl = '([^']+)'/);
  const teChartUrl = match ? match[1] : undefined;
  
  match = html.match(/TECountry = '([^']+)'/);
  const teCountry = match ? match[1] : undefined;
  
  match = html.match(/TECategory = '([^']+)'/);
  const teCategory = match ? match[1] : undefined;
  
  match = html.match(/TEType = '([^']+)'/);
  const teType = match ? match[1] : undefined;
  
  match = html.match(/TEFrequency = '([^']+)'/);
  const teFrequency = match ? match[1] : undefined;
  
  // Extract array data
  match = html.match(/TEForecast\s*=\s*\[([^\]]+)\]/);
  const teForecast = match ? JSON.parse('[' + match[1] + ']') : undefined;
  
  // Extract JSON metadata - use a more flexible regex
  match = html.match(/TEChartsMeta = (\[[\s\S]*?\]);/);
  const teChartsMeta = match ? (() => {
    try {
      return JSON.parse(match[1]);
    } catch (e) {
      logger.warn('Failed to parse TEChartsMeta:', e);
      return undefined;
    }
  })() : undefined;
  
  // Extract symbol data
  match = html.match(/symbol = '([^']+)'/);
  const symbol = match ? match[1] : undefined;
  
  match = html.match(/symbolType = '([^']+)'/);
  const symbolType = match ? match[1] : undefined;
  
  // TEChartsDatasource is the CDN URL for chart data
  match = html.match(/; var TEChartsDatasource = '([^']+)'/);
  const teChartsDatasource = match ? match[1] : undefined;
  
  match = html.match(/; var TEChartsToken = '([^']+)'/);
  const teChartToken = match ? match[1] : undefined;
  
  // Log if we found any TE data
  const foundData = teSymbol || teLastUpdate || teChartUrl || teChartsDatasource || 
                    teChartToken || teChart || teAlertsName || teCountry || 
                    teCategory || teType || teFrequency || teForecast || 
                    teChartsMeta || symbol || symbolType;
                    
  if (foundData) {
    logger.debug('Found TE data:', { 
      teSymbol, 
      teLastUpdate, 
      teChart,
      teAlertsName,
      teChartUrl,
      teCountry,
      teCategory,
      teType,
      teFrequency,
      teForecast: teForecast ? `[${teForecast.length} values]` : undefined,
      teChartsMeta: teChartsMeta ? `[${teChartsMeta.length} items]` : undefined,
      symbol,
      symbolType,
      teChartsDatasource,
      teChartToken 
    });
  }
  
  // 4. Extract all links
  const $ = cheerio.load(html);
  const foundUrls = new Set<string>();
  
  // Add some seed URLs if this is the root page
  if (urlPath === '/') {
    const seedUrls = [
      '/united-states', '/china', '/eurozone', '/japan', '/united-kingdom',
      '/indicators', '/calendar', '/forecasts', '/countries',
      '/gdp', '/inflation', '/unemployment', '/interest-rate',
      '/stocks', '/bonds', '/commodity', '/currency'
    ];
    seedUrls.forEach(url => foundUrls.add(url));
  }
  
  // Extract links from the page
  $('a[href]').each((_, element) => {
    const href = $(element).attr('href');
    if (!href) {
      return;
    }
    
    // Convert to absolute path
    let path: string;
    if (href.startsWith('/')) {
      path = href;
    } else if (href.includes('tradingeconomics.com')) {
      try {
        path = new URL(href).pathname;
      } catch {
        return; // Invalid URL
      }
    } else {
      return; // Skip external links
    }
    
    // Clean the path
    path = path.split('?')[0].split('#')[0];
    
    // Basic filtering
    if (shouldCrawl(path)) {
      foundUrls.add(path);
    }
  });
  
  const urls = Array.from(foundUrls);
  logger.info(`Found ${urls.length} URLs on ${urlPath}`);
  
  // 5. Update database
  // Mark current URL as crawled with TE data if found
  const updateData: any = { 
    lastCrawled: new Date(),
    foundLinks: urls.length,
    error: false
  };
  
  // Add TE data if found
  if (teSymbol) {
    updateData.teSymbol = teSymbol;
  }
  if (teLastUpdate) {
    updateData.teLastUpdate = teLastUpdate;
  }
  if (teChart) {
    updateData.teChart = teChart;
  }
  if (teAlertsName) {
    updateData.teAlertsName = teAlertsName;
  }
  if (teChartUrl) {
    updateData.teChartUrl = teChartUrl;  // Full chart URL with PNG
  }
  if (teCountry) {
    updateData.teCountry = teCountry;
  }
  if (teCategory) {
    updateData.teCategory = teCategory;
  }
  if (teType) {
    updateData.teType = teType;
  }
  if (teFrequency) {
    updateData.teFrequency = teFrequency;
  }
  if (teForecast) {
    updateData.teForecast = teForecast;
  }
  if (teChartsMeta) {
    updateData.teChartsMeta = teChartsMeta;
  }
  if (symbol) {
    updateData.symbol = symbol;
  }
  if (symbolType) {
    updateData.symbolType = symbolType;
  }
  if (teChartsDatasource) {
    updateData.teChartsDatasource = teChartsDatasource;  // CDN URL for chart data
  }
  if (teChartToken) {
    updateData.teChartToken = teChartToken;
  }
  
  await mongodb?.updateOne(
    'teUrls',
    { url: urlPath },
    { $set: updateData },
    { upsert: true }
  );
  
  // Insert new URLs (without lastCrawled so they'll be picked up for crawling)
  const newUrls: string[] = [];
  for (const url of urls) {
    const result = await mongodb?.updateOne(
      'teUrls',
      { url },
      { 
        $setOnInsert: { 
          url, 
          createdAt: new Date(),
          source: urlPath 
        } 
      },
      { upsert: true }
    );
    // Only schedule if it was actually inserted (not already existing)
    if (result?.upsertedCount > 0) {
      newUrls.push(url);
    }
  }
  
  // 6. Schedule individual jobs for each new URL
  logger.info(`Scheduling ${newUrls.length} new URLs for crawling`);
  for (const url of newUrls) {
    await this.scheduleOperation('te-spider', { url }, {
      jobId: `spider-${url.replace(/\//g, '-')}`,
      priority: 10,
      delay: Math.floor(Math.random() * 10000) // Spread requests over 10 seconds
    });
  }
  
  return urls;
}

// Simple fetch with retry using http service with proxy
async function fetchPage(this: TeHandler, url: string): Promise<string | null> {
  const { http, proxy, logger } = this;
  
  for (let attempt = 1; attempt <= 3; attempt++) {
    try {
      // Get a proxy URL for this request
      const proxyUrl = proxy?.getProxy();
      
      const response = await http.get(url, {
        proxy: proxyUrl,
        headers: {
          'User-Agent': getRandomUserAgent(),
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.5',
          'Accept-Encoding': 'gzip, deflate, br',
          'Connection': 'keep-alive',
          'Upgrade-Insecure-Requests': '1',
          'Cache-Control': 'max-age=0'
        }
      });
      
      if (response.ok) {
        return await response.text();
      }
      
      // If not OK, log the status
      logger.error(`HTTP ${response.status} for ${url}`);
      
    } catch (error) {
      logger.error(`Attempt ${attempt}/3 failed for ${url}:`, error);
      if (attempt === 3) {
        return null;
      }
      
      // Wait before retrying (exponential backoff)
      await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
    }
  }
  return null;
}

// Simple URL filtering
function shouldCrawl(url: string): boolean {
  // Skip empty or root
  if (!url || url === '/') {
    return false;
  }
  
  // Skip static files
  if (/\.(pdf|jpg|jpeg|png|gif|svg|css|js|ico|xml|rss|json|txt|csv|xlsx|xls|doc|docx|zip)$/i.test(url)) {
    return false;
  }
  
  // Skip deep URLs (more than 4 levels)
  const depth = (url.match(/\//g) || []).length;
  if (depth > 4) {
    return false;
  }
  
  // Skip common non-content pages
  const skipPatterns = [
    /\/api\//,
    /\/login/,
    /\/register/,
    /\/logout/,
    /\/admin/,
    /\/search/,
    /\/print\//,
    /\/download\//,
    /\/embed\//,
    /\/widget\//,
    /\/stream\//,
    /\/rss\//,
    /#/,
  ];
  
  for (const pattern of skipPatterns) {
    if (pattern.test(url)) {
      return false;
    }
  }
  
  // Accept everything else
  return true;
}