<?php
if ( ! defined( 'ABSPATH' ) ) exit;
/**
 * Class AISK_URLContentFetcher
 * Handles fetching content from URLs
 */
class AISK_URLContentFetcher {
    /**
     * User agent sent with every crawl request
     */
    const USER_AGENT = 'Mozilla/5.0 (compatible; AiskBot/1.0; +https://aisk.chat)';

    /**
     * Product token matched against robots.txt User-agent lines
     */
    const ROBOTS_TOKEN = 'aiskbot';

    /**
     * Parsed robots.txt rules per host for the current request
     *
     * @var array
     */
    private static $robots_rules = [];

    /**
     * Fetch content from a URL
     *
     * @param string $url URL to fetch
     * @param array $include_selectors Optional CSS selectors to include
     * @param array $exclude_selectors Optional CSS selectors to exclude
     * @return string|WP_Error Content or WP_Error on failure
     * @throws Exception When bot protection is detected or other errors occur
     */
    public function fetch($url, $include_selectors = [], $exclude_selectors = []) {
        if (!$this->is_allowed_by_robots($url)) {
            throw new Exception('This URL is disallowed by the website\'s robots.txt.');
        }

        $response = wp_remote_get($url, $this->get_request_args($url));

        if (is_wp_error($response)) {
            throw new Exception('Failed to fetch URL: ' . esc_html($response->get_error_message()));
        }

        $status_code = wp_remote_retrieve_response_code($response);
        $body = wp_remote_retrieve_body($response);
        $headers = wp_remote_retrieve_headers($response);

        // Check for bot protection/captcha indicators
        if ($this->detect_bot_protection($body, $headers, $status_code)) {
            throw new Exception('Bot protection detected. Unable to crawl this URL.');
        }

        if ($status_code !== 200) {
            throw new Exception('HTTP error: ' . esc_html($status_code));
        }

        $content_type = wp_remote_retrieve_header($response, 'content-type');
        if (!preg_match('/(text\/html|application\/xhtml\+xml)/i', $content_type)) {
            return false;
        }

        if (empty($body)) {
            return false;
        }

        return $body;
    }

    /**
     * Request arguments used for crawl requests
     *
     * @param string $url URL being requested
     * @return array
     */
    public function get_request_args($url) {
        return [
            'timeout' => 30,
            'user-agent' => self::USER_AGENT,
            'sslverify' => false,
            'headers' => [
                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language' => 'en-US,en;q=0.5',
                'Referer' => wp_parse_url($url, PHP_URL_SCHEME) . '://' . wp_parse_url($url, PHP_URL_HOST)
            ]
        ];
    }

    /**
     * Detect if the response is a bot protection / challenge page.
     *
     * Only real challenge signals are checked. A CDN header such as
     * "Server: cloudflare" or ordinary words in the page text (e.g. "blocked",
     * "please wait") are not treated as protection.
     *
     * @param string $body Response body
     * @param array|ArrayAccess $headers Response headers
     * @param int $status_code HTTP status code
     * @return bool True if bot protection detected
     */
    public function detect_bot_protection($body, $headers, $status_code) {
        // Explicit challenge headers (Cloudflare managed challenge, Sucuri, Incapsula)
        if (isset($headers['cf-mitigated']) && stripos((string) $headers['cf-mitigated'], 'challenge') !== false) {
            return true;
        }
        if (isset($headers['x-sucuri-block']) || isset($headers['x-iinfo'])) {
            return true;
        }

        // Status codes typically returned by firewalls and rate limiters
        if (in_array((int) $status_code, [403, 429, 503], true)) {
            return true;
        }

        $body = (string) $body;

        // Titles used by known challenge / block pages
        if (preg_match('/<title[^>]*>(.*?)<\/title>/is', $body, $matches)) {
            $title = trim(html_entity_decode(wp_strip_all_tags($matches[1])));
            $challenge_titles = [
                'just a moment',
                'attention required! | cloudflare',
                'ddos-guard',
                'pardon our interruption',
                'access denied | sucuri',
                'sucuri website firewall',
                'human verification',
                'are you a robot',
                'verify you are human',
            ];
            foreach ($challenge_titles as $challenge_title) {
                if (stripos($title, $challenge_title) === 0) {
                    return true;
                }
            }
        }

        // Markup that only appears on challenge pages
        $challenge_markers = [
            'cf_chl_opt',
            'cf-browser-verification',
            'cf-challenge-running',
            'checking your browser before accessing',
            '_Incapsula_Resource',
            'Incapsula incident ID',
            'captcha-delivery.com',
            'id="px-captcha"',
            'ddos-guard.net/check',
        ];
        foreach ($challenge_markers as $marker) {
            if (stripos($body, $marker) !== false) {
                return true;
            }
        }

        return false;
    }

    /**
     * Check whether robots.txt allows AiskBot to crawl a URL.
     *
     * Rules from a group naming AiskBot take priority over the "*" group.
     * The most specific (longest) matching rule wins, with Allow winning ties.
     * A missing or unreachable robots.txt allows crawling.
     *
     * @param string $url URL to check
     * @return bool
     */
    public function is_allowed_by_robots($url) {
        $parts = wp_parse_url($url);
        if (empty($parts['host'])) {
            return true;
        }

        $rules = $this->get_robots_rules($parts);
        if (empty($rules)) {
            return true;
        }

        $path = (isset($parts['path']) ? $parts['path'] : '/') . (isset($parts['query']) ? '?' . $parts['query'] : '');

        $matched_length = -1;
        $allowed = true;
        foreach ($rules as $rule) {
            if (!$this->robots_path_matches($rule['path'], $path)) {
                continue;
            }
            $length = strlen($rule['path']);
            if ($length > $matched_length || ($length === $matched_length && $rule['allow'])) {
                $matched_length = $length;
                $allowed = $rule['allow'];
            }
        }

        return $allowed;
    }

    /**
     * Fetch and parse robots.txt rules for a host, cached per request and for 12 hours
     *
     * @param array $parts Parsed URL parts
     * @return array List of ['allow' => bool, 'path' => string]
     */
    private function get_robots_rules($parts) {
        $scheme = isset($parts['scheme']) ? $parts['scheme'] : 'https';
        $origin = $scheme . '://' . $parts['host'] . (isset($parts['port']) ? ':' . $parts['port'] : '');

        if (isset(self::$robots_rules[$origin])) {
            return self::$robots_rules[$origin];
        }

        $transient_key = 'aisk_robots_' . md5($origin);
        $rules = get_transient($transient_key);

        if (false === $rules) {
            $rules = [];
            $response = wp_remote_get($origin . '/robots.txt', [
                'timeout' => 10,
                'user-agent' => self::USER_AGENT,
                'sslverify' => false,
            ]);

            if (!is_wp_error($response) && wp_remote_retrieve_response_code($response) === 200) {
                $rules = $this->parse_robots(wp_remote_retrieve_body($response));
            }

            set_transient($transient_key, $rules, 12 * HOUR_IN_SECONDS);
        }

        self::$robots_rules[$origin] = $rules;
        return $rules;
    }

    /**
     * Parse robots.txt and return the rules that apply to AiskBot
     *
     * @param string $content robots.txt body
     * @return array
     */
    private function parse_robots($content) {
        $groups = [];
        $current = null;
        $last_was_agent = false;

        foreach (preg_split('/\r\n|\r|\n/', (string) $content) as $line) {
            $line = trim(preg_replace('/#.*$/', '', $line));
            if ($line === '' || strpos($line, ':') === false) {
                continue;
            }

            list($field, $value) = array_map('trim', explode(':', $line, 2));
            $field = strtolower($field);

            if ($field === 'user-agent') {
                // Consecutive User-agent lines share one group
                if (!$last_was_agent) {
                    $groups[] = ['agents' => [], 'rules' => []];
                    $current = count($groups) - 1;
                }
                $groups[$current]['agents'][] = strtolower($value);
                $last_was_agent = true;
                continue;
            }

            $last_was_agent = false;
            if ($current === null || !in_array($field, ['allow', 'disallow'], true)) {
                continue;
            }

            // An empty Disallow means everything is allowed
            if ($value === '') {
                continue;
            }

            $groups[$current]['rules'][] = [
                'allow' => $field === 'allow',
                'path' => $value,
            ];
        }

        $specific = [];
        $wildcard = [];
        $has_specific = false;
        foreach ($groups as $group) {
            foreach ($group['agents'] as $agent) {
                if ($agent === '*') {
                    $wildcard = array_merge($wildcard, $group['rules']);
                } elseif ($agent === self::ROBOTS_TOKEN) {
                    $specific = array_merge($specific, $group['rules']);
                    $has_specific = true;
                }
            }
        }

        return $has_specific ? $specific : $wildcard;
    }

    /**
     * Match a robots.txt path pattern (supports * and $) against a URL path
     *
     * @param string $pattern Rule path
     * @param string $path URL path with query
     * @return bool
     */
    private function robots_path_matches($pattern, $path) {
        $anchored = substr($pattern, -1) === '$';
        if ($anchored) {
            $pattern = substr($pattern, 0, -1);
        }

        $regex = str_replace('\*', '.*', preg_quote($pattern, '#'));
        return (bool) preg_match('#^' . $regex . ($anchored ? '$' : '') . '#', $path);
    }
}