<?php
declare(strict_types=1);
namespace AetosSeo\Collectors;

if (!defined('ABSPATH')) {
    exit;
}
use AetosSeo\Security\Redactor;
use AetosSeo\Utils\Url;
class CrawlCollector
{
    /** @var HttpCollector */
    private $http;
    /** @var HtmlCollector */
    private $html;
    public function __construct()
    {
        $this->http = new HttpCollector();
        $this->html = new HtmlCollector();
    }

    /**
     * Backward-compatible alias used by Audit\Runner.
     *
     * Some builds call discover_seeds(), while this collector exposes get_seed_urls().
     * Keep both to avoid fatal errors during Run Audit / Crawl.
     */
    public function discover_seeds(int $limit = 10): array
    {
        return $this->get_seed_urls($limit);
    }

    private function is_probably_non_html_asset_url(string $url): bool
    {
        return Url::is_probably_non_html_url($url);
    }

    /**
     * Try to detect a sitemap (internal) and return a list of URLs.
     *
     * @return array{source:string,sitemap_url:string,urls:array<int,string>,checked:array<int,array<string,mixed>>}
     */
    private function sitemap_url_cap(): int
    {
        $cap = (int) apply_filters('aetos_seo_sitemap_url_cap', 10000);
        // Safety clamp.
        $cap = max(1, min(200000, $cap));
        return $cap;
    }

    public function get_seed_urls(int $limit = 10): array
    {
        $cap = $this->sitemap_url_cap();
        $limit = $limit <= 0 ? $cap : min($cap, $limit);
        $home = home_url('/');
        $candidates = [
            home_url('/wp-sitemap.xml'),
            home_url('/sitemap.xml'),
            home_url('/sitemap_index.xml'),
        ];
        $checked = [];
        foreach ($candidates as $cand) {
            $res = $this->http->request($cand, ['timeout' => 10], 2);
            $status = (int) ($res['status_code'] ?? 0);
            $checked[] = [
                'url' => $cand,
                'status_code' => $status,
            ];
            if ($status >= 200 && $status < 300) {
                $xml = (string) ($res['body_sample'] ?? '');
                $urls = $this->parse_sitemap_xml($xml, $limit);
                if ($urls) {
                    return [
                        'source' => 'sitemap',
                        'sitemap_url' => $cand,
                        'urls' => $urls,
                        'checked' => $checked,
                    ];
                }
            }
        }
        // Fallback: if sitemap discovery fails, attempt to discover internal URLs from the home page.
        $urls = [$home];
        $source = 'home';

        if ($limit > 1) {
            try {
                $info = $this->crawl_url($home, 'seed');
                $more = is_array($info['internal_links'] ?? null) ? (array) $info['internal_links'] : [];
                // Normalize + keep internal links only.
                $clean = [];
                foreach ($more as $u) {
                    if (is_string($u) && Url::is_internal_url($u) && !$this->is_probably_non_html_asset_url($u)) {
                        $norm = Url::normalize($u);
                        if ($norm !== '') {
                            $clean[] = $norm;
                        }
                    }
                }
                $more = array_values(array_unique($clean));
                if (!empty($more)) {
                    $urls = array_values(array_unique(array_merge($urls, $more)));
                    $urls = array_slice($urls, 0, $limit);
                    if (count($urls) > 1) {
                        $source = 'home_links';
                    }
                }
            } catch (\Throwable $e) {
                // Swallow errors; keep home-only seed.
            }
        }

        return [
            'source' => $source,
            'sitemap_url' => '',
            'urls' => $urls,
            'checked' => $checked,
        ];
    }
    /**
     * @return array<int,string>
     */
    public function parse_sitemap_xml(string $xml, int $limit = 50): array
    {
        $cap = $this->sitemap_url_cap();
        $limit = $limit <= 0 ? $cap : min($cap, $limit);

        $xml = $this->maybe_decode_gzip($xml);
        $xml = trim($xml);
        if ($xml === '') {
            return [];
        }

        // Fast checks to avoid attempting XML parsing on HTML.
        $looksLikeXml = (stripos($xml, '<?xml') !== false) || (stripos($xml, '<urlset') !== false) || (stripos($xml, '<sitemapindex') !== false);
        if (!$looksLikeXml) {
            return [];
        }

        $isIndex = (stripos($xml, '<sitemapindex') !== false);
        $depthLimit = (int) apply_filters('aetos_seo_sitemap_depth_limit', 4);

        $seen = [];
        return $this->parse_sitemap_xml_recursive($xml, $limit, 0, $isIndex, $seen, '');
    }

    /**
     * Best-effort gzip decode for sitemap bodies.
     */
    private function maybe_decode_gzip(string $body): string
    {
        if ($body === '') {
            return '';
        }
        // GZIP magic header: 1F 8B
        if (strlen($body) >= 2 && ord($body[0]) === 0x1f && ord($body[1]) === 0x8b) {
            if (function_exists('gzdecode')) {
                $decoded = @gzdecode($body);
                if (is_string($decoded) && $decoded !== '') {
                    return $decoded;
                }
            }
        }
        return $body;
    }

    /**
     * Extract <loc> values from XML in a streaming-safe way.
     *
     * @return array<int,string>
     */
    private function extract_sitemap_locs(string $xml, int $maxLocs): array
    {
        $maxLocs = max(1, min(20000, $maxLocs));
        $locs = [];

        // Prefer XMLReader (streaming) when available.
        if (class_exists('XMLReader')) {
            $reader = new \XMLReader();
            // Suppress warnings for malformed XML; we fallback to regex.
            $ok = @$reader->XML($xml, null, LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_NONET);
            if ($ok) {
                while ($reader->read()) {
                    if ($reader->nodeType === \XMLReader::ELEMENT && strtolower((string) $reader->localName) === 'loc') {
                        $reader->read();
                        $val = '';
                        if ($reader->nodeType === \XMLReader::TEXT || $reader->nodeType === \XMLReader::CDATA) {
                            $val = (string) $reader->value;
                        }
                        $val = trim(html_entity_decode($val, ENT_QUOTES | ENT_XML1, 'UTF-8'));
                        if ($val !== '') {
                            $locs[] = $val;
                            if (count($locs) >= $maxLocs) {
                                break;
                            }
                        }
                    }
                }
                $reader->close();
                if (!empty($locs)) {
                    return $locs;
                }
            }
        }

        // Fallback regex (best effort).
        if (preg_match_all('~<loc>\s*([^<]+)\s*</loc>~i', $xml, $m)) {
            foreach ($m[1] as $raw) {
                $val = trim(html_entity_decode((string) $raw, ENT_QUOTES | ENT_XML1, 'UTF-8'));
                if ($val !== '') {
                    $locs[] = $val;
                    if (count($locs) >= $maxLocs) {
                        break;
                    }
                }
            }
        }

        return $locs;
    }

    /**
     * Recursive sitemap parsing that supports sitemapindex (including nested) and .gz.
     *
     * @param array<string,bool> $seen
     * @return array<int,string>
     */
    private function parse_sitemap_xml_recursive(string $xml, int $limit, int $depth, bool $isIndex, array &$seen, string $currentUrl): array
    {
        $cap = $this->sitemap_url_cap();
        $limit = $limit <= 0 ? $cap : min($cap, $limit);
        $depthLimit = (int) apply_filters('aetos_seo_sitemap_depth_limit', 4);
        if ($depth > $depthLimit) {
            return [];
        }

        $xml = $this->maybe_decode_gzip($xml);
        $xml = trim($xml);
        if ($xml === '') {
            return [];
        }

        $isIndex = (stripos($xml, '<sitemapindex') !== false);

        // For sitemapindex, we may need many locs to reach the final URL limit.
        $maxLocs = $isIndex ? min(5000, $limit * 50) : $limit;
        $locs = $this->extract_sitemap_locs($xml, $maxLocs);
        if (empty($locs)) {
            return [];
        }

        $urls = [];

        if (!$isIndex) {
            foreach ($locs as $u) {
                if (Url::is_internal_url($u) && !$this->is_probably_non_html_asset_url($u)) {
                    $norm = Url::normalize($u);
                    if ($norm === '') {
                        continue;
                    }
                    $urls[] = $norm;
                    if (count($urls) >= $limit) {
                        break;
                    }
                }
            }
            return array_values(array_unique($urls));
        }

        // Index: each <loc> points to another sitemap.
        foreach ($locs as $smUrl) {
            if (!Url::is_internal_url($smUrl)) {
                continue;
            }
            $smKey = strtolower(trim($smUrl));
            if (isset($seen[$smKey])) {
                continue;
            }
            $seen[$smKey] = true;

            $res = $this->http->request($smUrl, ['timeout' => 15], 2);
            $xml2 = (string) ($res['body_sample'] ?? '');
            if ($xml2 === '') {
                continue;
            }

            $more = $this->parse_sitemap_xml_recursive($xml2, $limit - count($urls), $depth + 1, false, $seen, $smUrl);
            foreach ($more as $m) {
                $urls[] = $m;
                if (count($urls) >= $limit) {
                    break 2;
                }
            }
        }

        return array_values(array_unique($urls));
    }
    /**
     * Crawl a single URL and return extracted info + discovered links.
     *
     * @return array<string,mixed>
     */
    public function crawl_url(string $url, string $discovered_from = ''): array
    {
        // v2.1.1 — cache-bypass query param + no-cache headers so the crawler
        // sees a fresh server-side response (Cloudflare/page-cache plugins).
        // Also add a unique query token to defeat opaque CDN caches.
        $bypass_url = $this->add_cache_bypass($url);
        // v3.3.29 — tighter HTTP budget. 8s timeout × 3 redirects = 24s
        // worst-case per URL (was 12 × 5 = 60s, single slow URL could blow
        // through the Cloudflare/SiteGround proxy cutoff).
        $req_args = [
            'timeout' => 8,
            // HttpCollector follows redirects manually (up to the 3rd arg below)
            // and enforces a same-site guard on every hop, so WP's own auto-follow
            // stays off here.
            'redirection' => 0,
            'headers' => [
                'Cache-Control' => 'no-cache',
                'Pragma' => 'no-cache',
            ],
        ];
        $res = $this->http->request($bypass_url, $req_args, 3);
        $final_url = (string) ($res['final_url'] ?? $url);
        // Strip the bypass token from the final URL so downstream comparisons
        // match the canonical version of the URL.
        $final_url = $this->strip_cache_bypass($final_url);
        $headers = isset($res['headers']) && is_array($res['headers']) ? $res['headers'] : [];
        $content_type = $this->extract_content_type($headers);
        $body = (string) ($res['body_sample'] ?? '');
        // 4.0.17: the page may echo the request URL (core's
        // cancel-comment-reply-link does), so remove the bypass token from the
        // body before extracting links, canonical and social URLs.
        $body = self::scrub_cache_bypass_token($body);
        $is_html = $this->should_treat_as_html($final_url, $content_type, $body);
        $analysis = $is_html ? $this->html->analyze($body, $final_url) : [];

        return [
            'url' => $url,
            'normalized_url' => Url::normalize($final_url),
            'status_code' => (int) ($res['status_code'] ?? 0),
            'redirect_chain' => $res['redirect_chain'] ?? [],
            'content_type' => $content_type,
            'html_analyzed' => $is_html,
            // v3.3.46 — true only when the HTML body actually parsed into a
            // DOM. Distinct from html_analyzed (which just means "looked like
            // HTML"): a page can be html_analyzed=true yet fail DOM parsing,
            // in which case analyze() returns default signals. Checks exclude
            // _html_parsed=false pages from ratio calculations so a degraded
            // crawl never turns into a false-positive finding.
            '_html_parsed' => !empty($analysis['_html_parsed']),
            // v3.3.48 — surface BOM-before-DOCTYPE signal so RenderingCheck
            // can emit a finding when any sampled page started with the
            // UTF-8 byte order mark.
            'has_bom_before_doctype' => !empty($analysis['has_bom_before_doctype']),
            'canonical' => (string) ($analysis['canonical'] ?? ''),
            'robots_noindex' => !empty($analysis['robots_noindex']),
            'password_protected' => !empty($analysis['password_protected']),
            'title' => (string) ($analysis['title'] ?? ''),
            'meta_desc' => (string) ($analysis['meta_desc'] ?? ''),
            'robots' => (string) ($analysis['robots'] ?? ''),
            'viewport' => (string) ($analysis['viewport'] ?? ''),
            'hreflang_count' => (int) ($analysis['hreflang_count'] ?? 0),
            'list_count' => (int) ($analysis['list_count'] ?? 0),
            'table_count' => (int) ($analysis['table_count'] ?? 0),
            'dl_count' => (int) ($analysis['dl_count'] ?? 0),
            'h1_count' => (int) ($analysis['h1_count'] ?? 0),
            'word_count' => (int) ($analysis['word_count'] ?? 0),
            'images_missing_alt' => (int) ($analysis['images_missing_alt'] ?? 0),
            // Bug fix v1.0.55: images_total used to live only inside asset_counts.images_total,
            // so Repository::insert_crawl_url() (which reads top-level $data['images_total'])
            // always stored 0. Surface it at top-level too while keeping the legacy nested copy
            // below for any check still reading from asset_counts.
            'images_total' => (int) ($analysis['images_total'] ?? 0),
            'jsonld_types' => $analysis['jsonld_types'] ?? [],
            'internal_links' => isset($analysis['internal_links']) && is_array($analysis['internal_links']) ? $analysis['internal_links'] : [],
            'nav_links' => isset($analysis['nav_links']) && is_array($analysis['nav_links']) ? array_slice($analysis['nav_links'], 0, 100) : [],
            'asset_counts' => [
                'css_count' => (int) ($analysis['css_count'] ?? 0),
                // v1.0.64: split CSS counters
                'external_css_count' => (int) ($analysis['external_css_count'] ?? 0),
                'inline_css_count'   => (int) ($analysis['inline_css_count'] ?? 0),
                'inline_css_bytes'   => (int) ($analysis['inline_css_bytes'] ?? 0),
                'largest_inline_css_bytes' => (int) ($analysis['largest_inline_css_bytes'] ?? 0),
                'js_count' => (int) ($analysis['js_count'] ?? 0),
                'js_blocking_head' => (int) ($analysis['js_blocking_head'] ?? 0),
                'lazyload_detected' => !empty($analysis['lazyload_detected']),
                'webp_images' => (int) ($analysis['webp_images'] ?? 0),
                'images_total' => (int) ($analysis['images_total'] ?? 0),
            ],
            'asset_details' => [
                'css_origins' => isset($analysis['css_origins']) && is_array($analysis['css_origins']) ? $analysis['css_origins'] : [],
                'js_origins' => isset($analysis['js_origins']) && is_array($analysis['js_origins']) ? $analysis['js_origins'] : [],
                // v1.0.65: same base URL loaded 2+ times (real render-blocker)
                'duplicate_assets' => isset($analysis['duplicate_assets']) && is_array($analysis['duplicate_assets']) ? $analysis['duplicate_assets'] : [],
            ],
            // v3.3.51 — persist inline analytics signals into the stored crawl
            // row. HtmlCollector::analyze() computes analytics_signals (GTM/GA4/
            // Clarity/dataLayer detected by raw-HTML regex), but this curated row
            // previously dropped the field, so AnalyticsStackCheck always read an
            // empty value and false-flagged instrumented sites as "measurement
            // stack missing" — confirmed on spa-cz (GTM-PKLP5WSW live) even after
            // the v3.3.50 detection code landed. Persisting it here closes the gap.
            'analytics_signals' => isset($analysis['analytics_signals']) && is_array($analysis['analytics_signals']) ? $analysis['analytics_signals'] : [],
            'discovered_from' => $discovered_from,
            // Social / E-E-A-T / structural signals (from HtmlCollector)
            'h2_count' => (int) ($analysis['h2_count'] ?? 0),
            'h3_count' => (int) ($analysis['h3_count'] ?? 0),
            'og_title' => (string) ($analysis['og_title'] ?? ''),
            'og_description' => (string) ($analysis['og_description'] ?? ''),
            'og_image' => (string) ($analysis['og_image'] ?? ''),
            'og_type' => (string) ($analysis['og_type'] ?? ''),
            'twitter_card' => (string) ($analysis['twitter_card'] ?? ''),
            'twitter_title' => (string) ($analysis['twitter_title'] ?? ''),
            'lang' => (string) ($analysis['lang'] ?? ''),
            'has_favicon' => !empty($analysis['has_favicon']),
            'author_name' => (string) ($analysis['author_name'] ?? ''),
            'has_author_link' => !empty($analysis['has_author_link']),
            'article_published_time' => (string) ($analysis['article_published_time'] ?? ''),
            'article_modified_time' => (string) ($analysis['article_modified_time'] ?? ''),
            'images_without_dimensions' => (int) ($analysis['images_without_dimensions'] ?? 0),
            'outbound_links_count' => (int) ($analysis['outbound_links_count'] ?? 0),
            'jsonld_raw' => $analysis['jsonld_raw'] ?? [],
            'jsonld_raw_truncated' => !empty($analysis['jsonld_raw_truncated']), // v3.3.52 P2.3
            // Gap-closing fields
            'text_to_html_ratio' => (float) ($analysis['text_to_html_ratio'] ?? 0.0),
            'heading_hierarchy_valid' => (bool) ($analysis['heading_hierarchy_valid'] ?? true),
            'heading_sequence' => $analysis['heading_sequence'] ?? [],
            // v2.1.4 — propagate the skip position + offending heading text
            // so OnPageCheck evidence can show "H2 'Section A' → H4 'Sub'
            // at position 17" instead of a truncated sequence and `?` skip.
            'heading_skip_at' => (int) ($analysis['heading_skip_at'] ?? -1),
            'heading_skip_from_text' => (string) ($analysis['heading_skip_from_text'] ?? ''),
            'heading_skip_to_text'   => (string) ($analysis['heading_skip_to_text'] ?? ''),
            'has_srcset' => !empty($analysis['has_srcset']),
            'srcset_count' => (int) ($analysis['srcset_count'] ?? 0),
            'avif_images' => (int) ($analysis['avif_images'] ?? 0),
            'font_display_swap_detected' => !empty($analysis['font_display_swap_detected']),
            'has_breadcrumb_nav' => !empty($analysis['has_breadcrumb_nav']),
            'has_breadcrumb_schema' => !empty($analysis['has_breadcrumb_schema']),
            'nofollow_internal_count' => (int) ($analysis['nofollow_internal_count'] ?? 0),
            'has_publisher_schema' => !empty($analysis['has_publisher_schema']),
            'schema_types_detailed' => $analysis['schema_types_detailed'] ?? [],
            // HTTP header signals
            'x_robots_tag' => isset($headers['x-robots-tag'])
                ? (is_array($headers['x-robots-tag']) ? implode(', ', $headers['x-robots-tag']) : (string) $headers['x-robots-tag'])
                : '',
            'last_modified' => isset($headers['last-modified'])
                ? (is_array($headers['last-modified']) ? (string) end($headers['last-modified']) : (string) $headers['last-modified'])
                : '',
            // v2.1.1 — CDN/cache observability so audits can avoid re-opening
            // findings that are caused by stale public cache rather than the
            // origin HTML.
            'cf_cache_status' => $this->header_value($headers, 'cf-cache-status'),
            'x_cache' => $this->header_value($headers, 'x-cache'),
            'age_seconds' => (int) $this->header_value($headers, 'age'),
            'cdn_signature' => $this->detect_cdn_signature($headers),
            // v2.1.1 — pass-through of the new HtmlCollector heading-text + first
            // paragraph fields so QuestionCoverageCheck / DirectAnswerPattern /
            // CannibalizationCheck have data to operate on.
            'h1_texts' => isset($analysis['h1_texts']) && is_array($analysis['h1_texts']) ? $analysis['h1_texts'] : [],
            'h2_texts' => isset($analysis['h2_texts']) && is_array($analysis['h2_texts']) ? $analysis['h2_texts'] : [],
            'h3_texts' => isset($analysis['h3_texts']) && is_array($analysis['h3_texts']) ? $analysis['h3_texts'] : [],
            // v2.1.5 — H4 for FAQ accordion question detection.
            'h4_texts' => isset($analysis['h4_texts']) && is_array($analysis['h4_texts']) ? $analysis['h4_texts'] : [],
            'first_paragraph' => (string) ($analysis['first_paragraph'] ?? ''),
            // v2.1.1 — actual video element detection (replaces slug/title heuristic).
            'has_video_embed' => !empty($analysis['has_video_embed']),
            'video_provider'  => (string) ($analysis['video_provider'] ?? ''),
            // v2.1.1 — collector schema version so audits can detect stale rows
            // and prompt the user to re-crawl before drawing conclusions.
            'collector_version' => defined('AETOS_SEO_VERSION') ? (string) AETOS_SEO_VERSION : '2.1.1',
        ];
    }

    /**
     * v2.1.1 — best-effort cache-bypass: append a no-op query param that
     * doesn't conflict with the page's own query string, so we cause a cache
     * miss on Cloudflare/CDN/page-cache plugins. We strip it from the final
     * URL before comparing for canonical matches.
     */
    private function add_cache_bypass(string $url): string
    {
        $token = 'aetos_audit_t=' . substr(md5((string) microtime(true) . $url), 0, 10);
        if (strpos($url, '?') === false) {
            return $url . '?' . $token;
        }
        return $url . '&' . $token;
    }

    private function strip_cache_bypass(string $url): string
    {
        $clean = preg_replace('~([?&])aetos_audit_t=[^&]*(&|$)~', '$1', $url);
        if (!is_string($clean)) {
            return $url;
        }
        $clean = preg_replace('~[?&]$~', '', $clean) ?? $clean;
        return $clean;
    }

    /**
     * Remove every echoed copy of the cache-bypass token from a response body.
     * Handles raw, HTML-escaped (&amp; / &#038;) and percent-encoded separators.
     */
    public static function scrub_cache_bypass_token(string $body): string
    {
        if ($body === '' || stripos($body, 'aetos_audit_t') === false) {
            return $body;
        }
        $sep = '(?:\?|&amp;|&#0?38;|&|%3F|%26)';
        $tok = 'aetos_audit_t(?:=|%3D)[A-Za-z0-9]*';
        // "?token&rest" -> "?rest" keeps the remaining query valid.
        $out = preg_replace('~(\?|%3F)' . $tok . '(?:&amp;|&#0?38;|&|%26)~i', '$1', $body);
        if (!is_string($out)) {
            return $body;
        }
        $out = preg_replace('~' . $sep . $tok . '~i', '', $out);
        return is_string($out) ? $out : $body;
    }

    /**
     * Read a header value from a wp_remote_get header bag.
     *
     * @param array<string,mixed> $headers
     */
    private function header_value(array $headers, string $name): string
    {
        $key = strtolower($name);
        if (!isset($headers[$key])) {
            return '';
        }
        $v = $headers[$key];
        if (is_array($v)) {
            return (string) end($v);
        }
        return (string) $v;
    }

    /**
     * Determine which CDN (if any) returned the response. Helps SecurityCheck
     * decide whether to grade headers against the public origin (CF) or the
     * loopback (PHP) response.
     *
     * @param array<string,mixed> $headers
     */
    private function detect_cdn_signature(array $headers): string
    {
        $server = strtolower($this->header_value($headers, 'server'));
        if ($this->header_value($headers, 'cf-ray') !== '') {
            return 'cloudflare';
        }
        if ($this->header_value($headers, 'x-amz-cf-id') !== '' || strpos($server, 'cloudfront') !== false) {
            return 'cloudfront';
        }
        if ($this->header_value($headers, 'x-fastly-request-id') !== '' || strpos($server, 'fastly') !== false) {
            return 'fastly';
        }
        if ($this->header_value($headers, 'x-bunnycdn-cache-status') !== '') {
            return 'bunnycdn';
        }
        return '';
    }

    /**
     * @param array<string,mixed> $headers
     */
    private function extract_content_type(array $headers): string
    {
        if (!isset($headers['content-type'])) {
            return '';
        }

        $content_type = $headers['content-type'];
        if (is_array($content_type)) {
            $content_type = end($content_type);
        }

        $value = strtolower(trim((string) $content_type));
        if ($value === '') {
            return '';
        }

        if (strpos($value, ';') !== false) {
            $value = trim((string) strtok($value, ';'));
        }

        return $value;
    }

    private function should_treat_as_html(string $url, string $content_type, string $body): bool
    {
        if ($content_type !== '') {
            if (strpos($content_type, 'text/html') !== false || strpos($content_type, 'application/xhtml+xml') !== false) {
                return true;
            }

            if (
                strpos($content_type, 'image/') === 0
                || strpos($content_type, 'audio/') === 0
                || strpos($content_type, 'video/') === 0
                || strpos($content_type, 'font/') === 0
                || strpos($content_type, 'application/pdf') !== false
                || strpos($content_type, 'application/json') !== false
                || strpos($content_type, 'application/xml') !== false
                || strpos($content_type, 'text/xml') !== false
                || strpos($content_type, 'text/plain') !== false
                || strpos($content_type, 'text/css') !== false
                || strpos($content_type, 'javascript') !== false
            ) {
                return false;
            }
        }

        if (Url::is_probably_non_html_url($url)) {
            return false;
        }

        $body_l = strtolower(substr(ltrim($body), 0, 512));
        return strpos($body_l, '<html') !== false || strpos($body_l, '<!doctype html') !== false || strpos($body_l, '<head') !== false || strpos($body_l, '<body') !== false;
    }
}