get_bots() as $bot ) { if ( empty( $bot['enabled'] ) || empty( $bot['ua_pattern'] ) ) { continue; } $ua = self::pattern_to_ua( $bot['ua_pattern'] ); $p = self::probe( $url, $ua, $token, true ); if ( isset( $p['error'] ) || 403 === $p['code'] ) { continue; } return array( 'ua' => $ua, 'label' => isset( $bot['label'] ) ? $bot['label'] : '', 'probe' => $p, ); } return null; } /** * Interpret the engine probe (the bot is already guaranteed to reach the server). * * @param array $p probe() result (cache-buster). * @return array */ private static function interpret_engine( $p ) { $label = __( 'Markdown for bots (engine)', 'adg-markdown-for-ai-crawlers' ); if ( self::is_markdown( $p ) ) { return self::r( $label, 'ok', __( 'The bot receives clean Markdown — the engine works.', 'adg-markdown-for-ai-crawlers' ), '' ); } if ( false !== stripos( $p['ctype'], 'text/html' ) ) { return self::r( $label, 'fail', __( 'The bot reached the server but received HTML instead of Markdown.', 'adg-markdown-for-ai-crawlers' ), __( 'Check the post type and that another plugin is not intercepting template_redirect earlier.', 'adg-markdown-for-ai-crawlers' ) ); } return self::r( $label, 'warn', sprintf( /* translators: 1: HTTP code, 2: content-type */ __( 'Unexpected response: HTTP %1$d, %2$s.', 'adg-markdown-for-ai-crawlers' ), $p['code'], $p['ctype'] ? $p['ctype'] : '—' ), '' ); } /** * Cache: the same bot without a cache-buster must not get HTML from cache. */ private static function check_cache( $url, $bot_ua, $token ) { $label = __( 'Page cache does not intercept bots', 'adg-markdown-for-ai-crawlers' ); $p = self::probe( $url, $bot_ua, $token, false ); if ( isset( $p['error'] ) ) { return self::r( $label, 'warn', __( 'Could not check (loopback error).', 'adg-markdown-for-ai-crawlers' ), '' ); } if ( self::is_markdown( $p ) ) { return self::r( $label, 'ok', __( 'Bots reach the plugin even on a "hot" URL — the cache does not interfere.', 'adg-markdown-for-ai-crawlers' ), '' ); } // PHP with a valid token would have served Markdown; HTML means a cached response. $fix = __( 'Exclude AI-bot UAs from the page cache (at the nginx/hosting level or in the cache plugin) so their requests reach PHP.', 'adg-markdown-for-ai-crawlers' ); if ( '' !== $p['cache'] ) { return self::r( $label, 'fail', sprintf( /* translators: %s: cache header */ __( 'The page is served from cache (%s), bypassing the plugin — bots will get HTML.', 'adg-markdown-for-ai-crawlers' ), $p['cache'] ), $fix ); } return self::r( $label, 'warn', __( 'On a "hot" URL the bot received HTML with no explicit cache header — likely a cache without a status header.', 'adg-markdown-for-ai-crawlers' ), $fix ); } /** * A regular visitor must get HTML, not Markdown. */ private static function check_human( $url, $token ) { $label = __( 'Regular visitors get HTML', 'adg-markdown-for-ai-crawlers' ); $p = self::probe( $url, self::BROWSER_UA, $token, true ); if ( isset( $p['error'] ) ) { return self::r( $label, 'warn', __( 'Could not check (loopback error).', 'adg-markdown-for-ai-crawlers' ), '' ); } if ( self::is_markdown( $p ) ) { return self::r( $label, 'fail', __( 'A regular visitor received Markdown — that should not happen.', 'adg-markdown-for-ai-crawlers' ), __( 'Check the bot UA patterns: a too-broad regex may be matching browsers.', 'adg-markdown-for-ai-crawlers' ) ); } return self::r( $label, 'ok', __( 'Humans get regular HTML (Markdown is not served to them).', 'adg-markdown-for-ai-crawlers' ), '' ); } /** * Server-level (WAF/CDN) bot blocks — iterate over enabled bots. */ private static function check_server_blocks( $token ) { $label = __( 'Bots are not blocked by the server', 'adg-markdown-for-ai-crawlers' ); $home = home_url( '/' ); $blocked = array(); foreach ( ( new ADG_AIM_Bot_Detector() )->get_bots() as $bot ) { if ( empty( $bot['enabled'] ) || empty( $bot['ua_pattern'] ) ) { continue; } $ua = self::pattern_to_ua( $bot['ua_pattern'] ); $p = self::probe( $home, $ua, $token, true ); if ( ! isset( $p['error'] ) && 403 === $p['code'] ) { $blocked[] = isset( $bot['label'] ) ? $bot['label'] : $bot['ua_pattern']; } } if ( empty( $blocked ) ) { return self::r( $label, 'ok', __( 'No enabled bot is blocked by the server (no 403 detected).', 'adg-markdown-for-ai-crawlers' ), '' ); } return self::r( $label, 'warn', sprintf( /* translators: %s: list of bots */ __( 'Blocked at the server level (HTTP 403): %s.', 'adg-markdown-for-ai-crawlers' ), implode( ', ', $blocked ) ), __( 'If access for these bots is needed, unblock their UAs on the server/CDN (nginx, Cloudflare "Block AI bots", Wordfence).', 'adg-markdown-for-ai-crawlers' ) ); } /** * Availability of llms.txt / llms-full.txt. * * @return array List of results. */ private static function check_llms( $token ) { if ( ! ADG_AIM_Plugin::get_setting( 'llms_enabled', true ) ) { return array( self::r( __( 'llms.txt', 'adg-markdown-for-ai-crawlers' ), 'warn', __( 'llms.txt generation is turned off in the settings.', 'adg-markdown-for-ai-crawlers' ), __( 'Enable "Serve /llms.txt and /llms-full.txt" if you need the files.', 'adg-markdown-for-ai-crawlers' ) ), ); } // Probe robots.txt to detect server-level blocking of .txt requests. $robots_probe = self::probe( home_url( '/robots.txt' ), self::BROWSER_UA, $token, true ); $robots_ok = ! isset( $robots_probe['error'] ) && 200 === $robots_probe['code'] && '' !== trim( (string) $robots_probe['ctype'] ); $out = array(); foreach ( array( '/llms.txt', '/llms-full.txt' ) as $path ) { $p = self::probe( home_url( $path ), self::BROWSER_UA, $token, true ); if ( isset( $p['error'] ) ) { $out[] = self::r( $path, 'warn', __( 'Could not check (loopback error).', 'adg-markdown-for-ai-crawlers' ), '' ); continue; } if ( 200 === $p['code'] && '' !== trim( $p['body'] ) ) { $out[] = self::r( $path, 'ok', __( 'Served correctly (HTTP 200).', 'adg-markdown-for-ai-crawlers' ), '' ); } else { $out[] = self::build_blocked_result( $path, $p['code'], $robots_ok ); } } return $out; } /** * Build the "llms.txt unavailable" report row, tailored to the detected * hosting environment when recognizable. * * @param string $path '/llms.txt' or '/llms-full.txt'. * @param int $code HTTP code received. * @param bool $robots_ok Whether /robots.txt worked (rules out a generic rewrite failure). * @return array */ private static function build_blocked_result( $path, $code, $robots_ok ) { if ( ! $robots_ok ) { // robots.txt is also broken — a generic rewrite/permalink issue, not a // host-specific block. Host detection below assumes robots.txt works. return self::r( $path, 'fail', sprintf( /* translators: %d: HTTP code */ __( 'Unavailable (HTTP %d) — the rewrite rule did not fire, and /robots.txt is unavailable too.', 'adg-markdown-for-ai-crawlers' ), $code ), __( 'Go to Settings → Permalinks and click Save to flush rewrite rules. If that does not help, this site\'s webserver config needs a look (see readme for Nginx).', 'adg-markdown-for-ai-crawlers' ) ); } $host = ADG_AIM_Host_Detector::detect(); $detail = sprintf( /* translators: %d: HTTP code */ __( 'Unavailable (HTTP %d), while /robots.txt works — something on this server intercepts /llms.txt before WordPress runs.', 'adg-markdown-for-ai-crawlers' ), $code ); switch ( $host ) { case 'newfold': $detail .= ' ' . __( 'Detected: Newfold-family hosting (Bluehost/HostGator/iPage) — a known "NFD EPC" edge-cache block.', 'adg-markdown-for-ai-crawlers' ); $fix = __( 'Use "Try automatic fix" below, or edit .htaccess by hand: in the "# BEGIN NFD EPC" block, add llms\.txt and llms-full\.txt to the RewriteCond %{REQUEST_URI} !(robots\.txt|...) exception list.', 'adg-markdown-for-ai-crawlers' ); $fix_code = 'RewriteCond %{REQUEST_URI} !(robots\.txt|llms\.txt|llms-full\.txt|.*sitemap.*\.xml)'; return self::r_with_code( $path, 'fail', $detail, $fix, $fix_code ); case 'siteground': $detail .= ' ' . __( 'Detected: SiteGround hosting — likely their Dynamic Caching layer.', 'adg-markdown-for-ai-crawlers' ); return self::r( $path, 'fail', $detail, __( 'In Site Tools → Speed → Caching, add /llms.txt and /llms-full.txt to the Dynamic Cache exclusions, then purge the cache. "Try automatic fix" below may also help if the block is in .htaccess.', 'adg-markdown-for-ai-crawlers' ) ); case 'wpengine': $detail .= ' ' . __( 'Detected: WP Engine hosting (Nginx-based — .htaccess has no effect here).', 'adg-markdown-for-ai-crawlers' ); return self::r( $path, 'fail', $detail, __( 'Open a WP Engine support chat and ask them to allow direct access to /llms.txt and /llms-full.txt (their edge currently routes unrecognized .txt paths away from your site).', 'adg-markdown-for-ai-crawlers' ) ); case 'kinsta': $detail .= ' ' . __( 'Detected: Kinsta hosting (Nginx-based — .htaccess has no effect here).', 'adg-markdown-for-ai-crawlers' ); return self::r( $path, 'fail', $detail, __( 'Open a Kinsta support chat and ask them to allow direct access to /llms.txt and /llms-full.txt at their edge.', 'adg-markdown-for-ai-crawlers' ) ); case 'pantheon': $detail .= ' ' . __( 'Detected: Pantheon hosting (platform-level edge — .htaccess has no effect here).', 'adg-markdown-for-ai-crawlers' ); return self::r( $path, 'fail', $detail, __( 'Ask Pantheon support to allow direct access to /llms.txt and /llms-full.txt, or add the needed rule to pantheon.yml.', 'adg-markdown-for-ai-crawlers' ) ); case 'nginx-generic': $detail .= ' ' . __( 'Detected: Nginx.', 'adg-markdown-for-ai-crawlers' ); $fix = __( 'Add this to your Nginx server block (ask your host to apply it if you cannot edit Nginx config yourself), then reload Nginx:', 'adg-markdown-for-ai-crawlers' ); $fix_code = "location = /llms.txt { try_files \$uri =404; }\nlocation = /llms-full.txt { try_files \$uri =404; }"; return self::r_with_code( $path, 'fail', $detail, $fix, $fix_code ); default: $fix = __( 'Use "Try automatic fix" below (Apache/.htaccess), or contact your host\'s support with this: requests to /llms.txt and /llms-full.txt must be allowed through like /robots.txt is.', 'adg-markdown-for-ai-crawlers' ); $fix_code = 'RewriteCond %{REQUEST_URI} !(robots\.txt|llms\.txt|llms-full\.txt|.*sitemap.*\.xml)'; return self::r_with_code( $path, 'fail', $detail, $fix, $fix_code ); } } /* --------------------------------------------------------------------- * * Utilities * --------------------------------------------------------------------- */ /** * Loopback request carrying the self-test token. * * @param string $url URL. * @param string $ua User-Agent. * @param string $token Token. * @param bool $bust Whether to add a cache-buster. * @return array */ private static function probe( $url, $ua, $token, $bust ) { if ( $bust ) { $url = add_query_arg( 'adg_selftest', (string) wp_rand( 100000, 999999 ), $url ); } $resp = wp_remote_get( $url, array( 'timeout' => 12, 'redirection' => 3, 'sslverify' => false, 'user-agent' => $ua, 'headers' => array( self::HEADER => $token ), ) ); if ( is_wp_error( $resp ) ) { return array( 'error' => $resp->get_error_message() ); } return array( 'code' => (int) wp_remote_retrieve_response_code( $resp ), 'ctype' => (string) wp_remote_retrieve_header( $resp, 'content-type' ), 'vary' => (string) wp_remote_retrieve_header( $resp, 'vary' ), 'cache' => self::cache_header( $resp ), 'body' => (string) wp_remote_retrieve_body( $resp ), ); } /** * Whether the response is Markdown. * * @param array $p probe() result. * @return bool */ private static function is_markdown( $p ) { return false !== stripos( $p['ctype'], 'text/markdown' ); } /** * Find a cache-hit indicator among the common headers. * * @param array|WP_Error $resp Response. * @return string 'header: value' or ''. */ private static function cache_header( $resp ) { $candidates = array( 'x-cache-status', 'x-cache', 'x-proxy-cache', 'cf-cache-status', 'x-litespeed-cache', 'x-nginx-cache' ); foreach ( $candidates as $h ) { $v = wp_remote_retrieve_header( $resp, $h ); if ( $v && false !== stripos( (string) $v, 'HIT' ) ) { return $h . ': ' . $v; } } $age = wp_remote_retrieve_header( $resp, 'age' ); if ( '' !== (string) $age && (int) $age > 0 ) { return 'age: ' . $age; } return ''; } /** * Pick a published post of an allowed type to test. * * @return WP_Post|null */ private static function pick_test_post() { $types = (array) ADG_AIM_Plugin::get_setting( 'post_types', array( 'post', 'page' ) ); $posts = get_posts( array( 'post_type' => $types, 'post_status' => 'publish', 'numberposts' => 20, 'orderby' => 'date', 'order' => 'DESC', ) ); foreach ( $posts as $p ) { if ( get_post_meta( $p->ID, '_adg_hide_from_ai_bots', true ) ) { continue; } if ( post_password_required( $p ) ) { continue; } return $p; } return null; } /** * Build a plausible UA from a bot UA pattern (extract the literal core). * * @param string $pattern Pattern (regex or substring). * @return string */ private static function pattern_to_ua( $pattern ) { $pattern = trim( (string) $pattern ); // Strip /.../flags delimiters. if ( strlen( $pattern ) > 1 ) { $first = $pattern[0]; if ( in_array( $first, array( '/', '#', '~', '@' ), true ) ) { $last = strrpos( $pattern, $first ); if ( $last > 0 ) { $pattern = substr( $pattern, 1, $last - 1 ); } } } // Keep only a "safe" core (letters/digits/._- and space). $core = preg_replace( '/[^\w.\- ]+/', '', $pattern ); $core = trim( $core ); if ( '' === $core ) { $core = 'AIBot'; } return 'Mozilla/5.0 (compatible; ' . $core . '/1.0; +adg-selftest)'; } /** * Build a report entry. * * @param string $label Check name. * @param string $status ok|warn|fail. * @param string $detail Explanation. * @param string $fix Recommendation (may be ''). * @return array */ private static function r( $label, $status, $detail, $fix ) { return array( 'label' => $label, 'status' => $status, 'detail' => $detail, 'fix' => $fix, ); } /** * Build a report entry with a copyable code block. * * @param string $label Check name. * @param string $status ok|warn|fail. * @param string $detail Explanation. * @param string $fix Recommendation. * @param string $fix_code Copyable code snippet. * @return array */ private static function r_with_code( $label, $status, $detail, $fix, $fix_code ) { return array( 'label' => $label, 'status' => $status, 'detail' => $detail, 'fix' => $fix, 'fix_code' => $fix_code, ); } }