2097152) { return new \WP_Error('a1local_mdi_size', 'This page exceeds the 2 MB conversion limit.'); } $dom = new DOMDocument('1.0', 'UTF-8'); $previous = libxml_use_internal_errors(true); // Never load external entities. The encoding declaration keeps non-ASCII content intact. $loaded = $dom->loadHTML('' . $html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING); libxml_clear_errors(); libxml_use_internal_errors($previous); if (!$loaded) { return new \WP_Error('a1local_mdi_html', 'The page could not be read as HTML.'); } $xpath = new DOMXPath($dom); foreach ($xpath->query('//meta[@name]') as $meta) { if (in_array(strtolower($meta->getAttribute('name')), ['robots', 'googlebot'], true) && preg_match('/\b(?:noindex|none)\b/i', $meta->getAttribute('content'))) { return new \WP_Error('a1local_mdi_noindex', 'This page asks crawlers not to index it. Markdown is excluded.'); } } // An explicit exclusion covers its whole subtree, including schema. foreach (iterator_to_array($xpath->query('//*[@data-a1local-mdi-exclude or @data-mfa-exclude]')) as $node) { if ($node->parentNode) { $node->parentNode->removeChild($node); } } $structured = []; foreach ($xpath->query('//script[@type="application/ld+json"]') as $script) { $data = json_decode($script->textContent, true); if (json_last_error() === JSON_ERROR_NONE && is_array($data)) { $structured[] = $data; } } $remove = '//script|//style|//noscript|//template|//svg|//nav|//aside|//form|//button|//input|//select|//textarea|//iframe|//comment()|//*[@hidden]|//*[@aria-hidden="true"]|//*[@data-a1local-mdi-exclude or @data-mfa-exclude]'; $nodes = iterator_to_array($xpath->query($remove)); foreach ($nodes as $node) { if ($node->parentNode) { $node->parentNode->removeChild($node); } } foreach (iterator_to_array($xpath->query('//*[@style]')) as $node) { if (preg_match('/(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden)\s*(?:!important)?\s*(?:;|$)/i', $node->getAttribute('style'))) { $node->parentNode->removeChild($node); } } $root = null; $selection = ''; $selectors = [ 'main' => '//main', 'main landmark' => '//*[@role="main"]', 'article' => '//article', 'post content' => '//*[contains(concat(" ",normalize-space(@class)," ")," entry-content ") or contains(concat(" ",normalize-space(@class)," ")," wp-block-post-content ")]', 'page body' => '//body', ]; foreach ($selectors as $label => $expression) { $matches = $xpath->query($expression); if ($matches->length) { $root = $matches->item(0); $selection = $label; break; } } if (!$root) { return new \WP_Error('a1local_mdi_content', 'No readable content container was found.'); } // Site chrome outside the selected main content is not part of the document. if ($selection === 'page body') { foreach (iterator_to_array($xpath->query('.//header|.//footer', $root)) as $node) { $node->parentNode->removeChild($node); } } $plain = trim(preg_replace('/\s+/u', ' ', $root->textContent)); if (strlen($plain) < 20) { return new \WP_Error('a1local_mdi_empty', 'The selected page content is empty or too short. Check the page template.'); } $canonical = get_permalink($post); foreach ($xpath->query('.//*[@href or @src]', $root) as $element) { foreach (['href', 'src'] as $attr) { if (!$element->hasAttribute($attr)) { continue; } $value = trim($element->getAttribute($attr)); if ($value === '') { continue; } $absolute = \WP_Http::make_absolute_url($value, $canonical); $element->setAttribute($attr, esc_url_raw($absolute, ['http', 'https', 'mailto', 'tel'])); } } $converter = new HtmlConverter(['strip_tags' => true, 'hard_break' => true, 'header_style' => 'atx', 'remove_nodes' => 'script style iframe']); $converter->getEnvironment()->addConverter(new TableConverter()); $body = trim($converter->convert($dom->saveHTML($root))); if ($body === '') { return new \WP_Error('a1local_mdi_empty', 'Conversion produced no readable content.'); } $metadata = [ 'title' => html_entity_decode(get_the_title($post), ENT_QUOTES | ENT_HTML5, 'UTF-8'), 'url' => $canonical, 'updated' => get_post_modified_time('c', true, $post), 'language' => get_bloginfo('language'), ]; $markdown = "---\n"; foreach ($metadata as $key => $value) { $markdown .= $key . ': ' . wp_json_encode($value, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE) . "\n"; } $markdown .= "---\n\n" . $body . "\n"; if ($structured) { // Encode backticks to stop JSON-LD strings from terminating a Markdown code fence. $json = str_replace('`', '\\u0060', wp_json_encode(count($structured) === 1 ? $structured[0] : $structured, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); $markdown .= "\n## Structured data\n\n```json\n" . $json . "\n```\n"; } return ['markdown' => $markdown, 'source_bytes' => strlen($html), 'markdown_bytes' => strlen($markdown), 'container' => $selection, 'structured_count' => count($structured)]; } }