label */ private static function public_post_type_options() { $objects = get_post_types( array( 'public' => true ), 'objects' ); unset( $objects['attachment'] ); $options = array(); foreach ( $objects as $slug => $object ) { $options[ $slug ] = ! empty( $object->labels->name ) ? $object->labels->name : $slug; } return $options; } /** * Return the cached audit, or an empty report if the audit has not run yet. * * @return array */ public static function audit() { $audit = get_option( self::AUDIT_OPTION, array() ); if ( empty( $audit ) || ! is_array( $audit ) ) { return self::empty_audit(); } $audit = wp_parse_args( $audit, self::empty_audit() ); if ( empty( $audit['files'] ) || ! is_array( $audit['files'] ) ) { $audit['files'] = array(); } return $audit; } /** * Empty audit payload. * * @return array */ private static function empty_audit() { $urls = self::guidance_urls(); $settings = self::settings(); return array( 'checked_at' => '', 'rank_math_active' => self::is_rank_math_active(), 'rank_math_header_fix_enabled' => ! empty( $settings['llms_rank_math_header_fix'] ), 'overall_status' => 'not_checked', 'files' => array( 'llms_txt' => array( 'label' => 'llms.txt', 'url' => $urls['llms_txt'], 'required' => true, 'status' => 'not_checked', ), ), ); } /** * Run a fresh audit against the live files and cache the result. * * @return array */ public static function run_audit() { $settings = self::settings(); $urls = self::guidance_urls(); $files = array( 'llms_txt' => array( 'label' => 'llms.txt', 'url' => $urls['llms_txt'], 'required' => true, 'full' => false, ), ); if ( ! empty( $settings['llms_audit_check_full'] ) || ! empty( $settings['llms_generate_full'] ) ) { $files['llms_full_txt'] = array( 'label' => 'llms-full.txt', 'url' => $urls['llms_full_txt'], 'required' => ! empty( $settings['llms_generate_full'] ), 'full' => true, ); } $warnings = 0; $failures = 0; foreach ( $files as $key => $file ) { $files[ $key ] = self::audit_file( $file, $settings ); if ( 'fail' === $files[ $key ]['status'] ) { $failures++; } elseif ( 'warning' === $files[ $key ]['status'] ) { $warnings++; } } $overall = 'pass'; if ( $failures > 0 ) { $overall = 'fail'; } elseif ( $warnings > 0 ) { $overall = 'warning'; } $audit = array( 'checked_at' => current_time( 'mysql', true ), 'rank_math_active' => self::is_rank_math_active(), 'rank_math_header_fix_enabled' => ! empty( $settings['llms_rank_math_header_fix'] ), 'overall_status' => $overall, 'files' => $files, 'version' => AI_O_Optimiser_VERSION, ); update_option( self::AUDIT_OPTION, $audit, false ); return $audit; } /** * Audit one guidance file URL. * * @param array $file File definition. * @param array $settings Plugin settings. * @return array */ /** * Who published the file that is actually live. * * Until 1.32.0 the generated files ended with a line naming this plugin. * That line sat in public output on the site owner's own domain, which they * never opted into, and it answered the question badly anyway: a footer in a * machine-readable file is not where somebody looks when they are working out * why llms.txt says something unexpected. * * This answers it where the question is actually asked, on the screen that * already fetches the live file -- and answers it better, because comparing * the published bytes to what this plugin would generate can also say when * something else is serving the file. On a site running another plugin that * writes llms.txt, that is the fact worth knowing. * * @param string $body The fetched file. * @param array $file File definition, including whether it is the full variant. * @param array $settings Plugin settings. * @return string ours|differs|not_generating */ private static function identify_generator( $body, $file, $settings ) { if ( empty( $settings['llms_generate_enabled'] ) ) { return 'not_generating'; } $ours = AI_O_Optimiser_LLMS::generate_guidance_content( ! empty( $file['full'] ), $settings ); return trim( self::normalise_line_endings( $body ) ) === trim( self::normalise_line_endings( $ours ) ) ? 'ours' : 'differs'; } private static function audit_file( $file, $settings ) { $result = array( 'label' => $file['label'], 'url' => $file['url'], 'required' => ! empty( $file['required'] ), 'status' => 'not_checked', 'http_code' => 0, 'content_type' => '', 'x_robots_tag' => '', 'bytes' => 0, 'words' => 0, 'links' => 0, 'internal_links' => 0, 'duplicate_links' => 0, 'checked_links' => 0, 'broken_links' => array(), 'noindex_links' => array(), 'privateish_links' => array(), 'warnings' => array(), 'notes' => array(), 'snippet' => '', 'generator' => 'unknown', ); $fetched = AI_O_Optimiser_HTTP::get_with_fallback( $file['url'], array( 'timeout' => 8, 'redirection' => 3, 'headers' => array( 'Accept' => 'text/plain,text/markdown,text/*,*/*;q=0.8', ), ), 'identified', 'llms.txt Audit' ); $response = $fetched['response']; if ( is_wp_error( $response ) ) { $result['status'] = ! empty( $file['required'] ) ? 'fail' : 'warning'; $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Could not fetch %1$s: %2$s', 'ai-o-optimiser' ), $file['label'], $response->get_error_message() ); return $result; } if ( ! empty( $fetched['recovered'] ) ) { $result['warnings'][] = AI_O_Optimiser_HTTP::edge_block_note( $fetched['primary_code'], 'identified' ); } $code = (int) wp_remote_retrieve_response_code( $response ); $content_type = wp_remote_retrieve_header( $response, 'content-type' ); $x_robots = wp_remote_retrieve_header( $response, 'x-robots-tag' ); $body = (string) wp_remote_retrieve_body( $response ); $result['http_code'] = $code; $result['content_type'] = is_array( $content_type ) ? implode( ', ', $content_type ) : (string) $content_type; $result['x_robots_tag'] = is_array( $x_robots ) ? implode( ', ', $x_robots ) : (string) $x_robots; $result['bytes'] = strlen( $body ); $result['snippet'] = self::safe_substr( self::normalise_line_endings( $body ), 0, 1500 ); if ( 200 !== $code ) { if ( ! empty( $file['required'] ) ) { $result['status'] = 'fail'; $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%1$s returned HTTP %2$d. The root file should return 200 OK.', 'ai-o-optimiser' ), $file['label'], $code ); } else { $result['status'] = 'pass'; $result['notes'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%1$s is optional and returned HTTP %2$d.', 'ai-o-optimiser' ), $file['label'], $code ); } return $result; } self::analyse_headers( $result ); $result['generator'] = self::identify_generator( $body, $file, $settings ); self::analyse_content( $result, $body, $settings ); if ( ! empty( $result['warnings'] ) ) { $result['status'] = 'warning'; } else { $result['status'] = 'pass'; } return $result; } /** * Analyse HTTP response headers for guidance-file crawler usefulness. * * @param array $result Audit result, passed by reference. */ private static function analyse_headers( &$result ) { $x_robots = isset( $result['x_robots_tag'] ) ? strtolower( (string) $result['x_robots_tag'] ) : ''; if ( '' === $x_robots ) { $result['notes'][] = __( 'No X-Robots-Tag header was found. For llms.txt, consider noindex without nofollow so the file does not appear in SERPs while its links remain usable.', 'ai-o-optimiser' ); return; } if ( false !== strpos( $x_robots, 'nofollow' ) || preg_match( '/(^|[,\s])none([,\s]|$)/i', $x_robots ) ) { $result['warnings'][] = __( 'X-Robots-Tag contains nofollow/none. llms.txt should not block link following; use noindex without nofollow if you do not want the file indexed.', 'ai-o-optimiser' ); return; } if ( false !== strpos( $x_robots, 'noindex' ) ) { $result['notes'][] = __( 'X-Robots-Tag noindex detected without nofollow. This keeps the guidance file out of SERPs while preserving link discovery.', 'ai-o-optimiser' ); } } /** * Analyse Markdown/text body quality. * * @param array $result Audit result, passed by reference. * @param string $body File content. * @param array $settings Plugin settings. */ private static function analyse_content( &$result, $body, $settings ) { $body = self::normalise_line_endings( $body ); $plain = wp_strip_all_tags( $body ); $max_bytes = max( 10000, (int) $settings['llms_audit_max_bytes'] ); $min_links = max( 0, (int) $settings['llms_audit_min_links'] ); $max_links = max( 1, (int) $settings['llms_audit_max_links'] ); $result['words'] = str_word_count( $plain ); preg_match_all( '/\[[^\]]+\]\(([^\)\s]+)(?:\s+"[^"]*")?\)/i', $body, $markdown_matches ); $links = array(); if ( ! empty( $markdown_matches[1] ) ) { foreach ( $markdown_matches[1] as $url ) { $links[] = html_entity_decode( trim( $url ), ENT_QUOTES, get_bloginfo( 'charset' ) ); } } preg_match_all( '/(?)"\']+/i', $body, $raw_matches ); if ( ! empty( $raw_matches[0] ) ) { foreach ( $raw_matches[0] as $url ) { $links[] = html_entity_decode( rtrim( trim( $url ), '.,;' ), ENT_QUOTES, get_bloginfo( 'charset' ) ); } } $links = array_values( array_filter( array_map( 'esc_url_raw', $links ) ) ); $unique_links = array_values( array_unique( $links ) ); $result['links'] = count( $links ); $result['duplicate_links'] = max( 0, count( $links ) - count( $unique_links ) ); $home_host = wp_parse_url( home_url(), PHP_URL_HOST ); $internal = array(); foreach ( $unique_links as $url ) { $host = wp_parse_url( $url, PHP_URL_HOST ); if ( $home_host && $host && strtolower( $home_host ) === strtolower( $host ) ) { $internal[] = $url; } } $result['internal_links'] = count( $internal ); if ( '' === trim( $body ) || $result['bytes'] < 50 ) { $result['warnings'][] = __( 'The file is empty or extremely thin.', 'ai-o-optimiser' ); } if ( $result['bytes'] > $max_bytes && 'llms.txt' === $result['label'] ) { $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'The file is larger than %s. Keep llms.txt concise and use llms-full.txt for expanded content.', 'ai-o-optimiser' ), size_format( $max_bytes ) ); } if ( ! preg_match( '/^#\s+.+/m', $body ) ) { $result['warnings'][] = __( 'No top-level Markdown heading was found. Add a clear # Site Name heading.', 'ai-o-optimiser' ); } if ( false !== stripos( $body, ' 0 && count( $unique_links ) < $min_links ) { $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Only %1$d unique link(s) found. Recommended minimum is %2$d for useful AI discovery.', 'ai-o-optimiser' ), count( $unique_links ), $min_links ); } if ( $result['duplicate_links'] > 0 ) { $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%d duplicate link reference(s) found.', 'ai-o-optimiser' ), $result['duplicate_links'] ); } if ( $result['content_type'] && ! preg_match( '/text\/(plain|markdown|x-markdown|md|html)|application\/octet-stream/i', $result['content_type'] ) ) { $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Unexpected Content-Type: %s.', 'ai-o-optimiser' ), $result['content_type'] ); } foreach ( $internal as $url ) { if ( self::looks_private_or_admin_url( $url ) ) { $result['privateish_links'][] = $url; } $post_id = url_to_postid( $url ); if ( $post_id && self::post_has_noindex( $post_id ) ) { $result['noindex_links'][] = $url; } } if ( ! empty( $result['privateish_links'] ) ) { $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%d admin, preview, login or non-public looking internal URL(s) found.', 'ai-o-optimiser' ), count( $result['privateish_links'] ) ); } if ( ! empty( $result['noindex_links'] ) ) { $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%d internal URL(s) appear to be noindex in WordPress SEO metadata.', 'ai-o-optimiser' ), count( $result['noindex_links'] ) ); } if ( ! empty( $settings['llms_audit_check_links'] ) && ! empty( $internal ) ) { $to_check = array_slice( $internal, 0, $max_links ); foreach ( $to_check as $url ) { $result['checked_links']++; $check = self::check_link_status( $url ); if ( $check['code'] >= 400 || 0 === $check['code'] ) { $result['broken_links'][] = array( 'url' => $url, 'code' => $check['code'], 'message' => $check['message'], ); } } if ( ! empty( $result['broken_links'] ) ) { $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%d checked internal link(s) returned an error status.', 'ai-o-optimiser' ), count( $result['broken_links'] ) ); } } } /** * Check one link via HEAD, falling back to GET where needed. * * @param string $url URL to check. * @return array code/message. */ private static function check_link_status( $url ) { $args = array( 'timeout' => 5, 'redirection' => 3, 'headers' => array(), ); $response = wp_safe_remote_head( $url, AI_O_Optimiser_HTTP::profile_args( $args, 'identified', 'llms.txt Link Check' ) ); if ( is_wp_error( $response ) ) { return array( 'code' => 0, 'message' => $response->get_error_message() ); } $code = (int) wp_remote_retrieve_response_code( $response ); if ( 405 === $code ) { // Server rejects HEAD; try a normal GET with the same identity. $response = wp_safe_remote_get( $url, AI_O_Optimiser_HTTP::profile_args( $args, 'identified', 'llms.txt Link Check' ) ); if ( is_wp_error( $response ) ) { return array( 'code' => 0, 'message' => $response->get_error_message() ); } $code = (int) wp_remote_retrieve_response_code( $response ); } if ( AI_O_Optimiser_HTTP::is_waf_block_code( $code ) && apply_filters( 'ai_o_optimiser_browser_fallback', true, $url, $code ) ) { // WAF-style refusal of the identified UA. Retry with a browser // profile before reporting the link broken — the target may be // perfectly reachable for visitors and AI assistants with // verified crawlers, just not for unknown bots. $response = wp_safe_remote_get( $url, AI_O_Optimiser_HTTP::profile_args( $args, 'browser', 'llms.txt Link Check' ) ); if ( ! is_wp_error( $response ) ) { $browser_code = (int) wp_remote_retrieve_response_code( $response ); if ( $browser_code >= 200 && $browser_code < 400 ) { return array( 'code' => $browser_code, 'message' => sprintf( /* translators: %d: HTTP status code */ __( 'Reachable for browsers, but HTTP %d to unrecognised bots.', 'ai-o-optimiser' ), $code ), ); } } } return array( 'code' => $code, 'message' => '' ); } /** * Determine whether Rank Math appears active. * * @return bool */ private static function is_rank_math_active() { if ( defined( 'RANK_MATH_VERSION' ) || class_exists( 'RankMath' ) || class_exists( '\\RankMath\\Helper' ) ) { return true; } $active = (array) get_option( 'active_plugins', array() ); if ( is_multisite() ) { $network_active = (array) get_site_option( 'active_sitewide_plugins', array() ); $active = array_merge( $active, array_keys( $network_active ) ); } foreach ( $active as $plugin ) { if ( false !== strpos( (string) $plugin, 'seo-by-rank-math/' ) || false !== strpos( (string) $plugin, 'rank-math' ) ) { return true; } } return false; } /** * Multibyte-safe substring with a PHP core fallback. * * @param string $text Text. * @param int $start Start offset. * @param int $length Length. * @return string */ private static function safe_substr( $text, $start, $length ) { $text = (string) $text; return function_exists( 'mb_substr' ) ? mb_substr( $text, $start, $length ) : substr( $text, $start, $length ); } /** * Format a UTC MySQL datetime in the WordPress site timezone. * * @param string $utc_mysql UTC datetime. * @return string */ private static function format_site_time( $utc_mysql ) { try { return ( new DateTimeImmutable( $utc_mysql, new DateTimeZone( 'UTC' ) ) )->setTimezone( wp_timezone() )->format( 'Y-m-d H:i:s T' ); } catch ( Exception $e ) { return $utc_mysql; } } /** * Small status label for the admin page. * * @param string $status Status key. * @return string */ private static function status_label( $status ) { switch ( $status ) { case 'pass': return __( 'Pass', 'ai-o-optimiser' ); case 'warning': return __( 'Warnings', 'ai-o-optimiser' ); case 'fail': return __( 'Fail', 'ai-o-optimiser' ); default: return __( 'Not checked', 'ai-o-optimiser' ); } } /** * CSS class suffix for status badges. * * @param string $status Status key. * @return string */ private static function status_class( $status ) { $status = str_replace( '_', '-', sanitize_key( $status ) ); if ( ! in_array( $status, array( 'pass', 'warning', 'fail', 'not-checked' ), true ) ) { $status = 'not-checked'; } return 'ai-o-optimiser-badge-' . $status; } /** * Provider mode labels. * * @return array */ private static function provider_mode_labels() { return array( 'auto' => __( 'Auto: Rank Math owns llms.txt when active', 'ai-o-optimiser' ), 'rank_math' => __( 'Rank Math owns llms.txt; ACT fixes/audits headers', 'ai-o-optimiser' ), 'act' => __( 'AI-O Optimiser owns llms.txt', 'ai-o-optimiser' ), 'disabled' => __( 'Disabled: do not serve llms.txt from ACT', 'ai-o-optimiser' ), ); } /** * Render the AI guidance files admin screen. */ public static function render_admin_page() { // phpcs:disable WordPress.Security.NonceVerification.Recommended -- Read-only screen state (filters, tab selection, pagination, redirect result notices). Nothing is written here, so there is no form submission to carry a nonce. if ( ! current_user_can( AI_O_Optimiser_Admin::CAPABILITY ) ) { return; } $settings = self::settings(); $provider_mode = isset( $settings['llms_provider_mode'] ) ? $settings['llms_provider_mode'] : 'auto'; $provider_labels = self::provider_mode_labels(); $audit = self::audit(); $urls = self::guidance_urls(); $post_type_options = self::public_post_type_options(); $preview_llms = self::generate_guidance_content( false, $settings ); $preview_full = self::generate_guidance_content( true, $settings ); ?>