'ai-o-optimiser-system-check' ), admin_url( 'admin.php' ) ) . '#ai-o-optimiser-nightly-health-monitor' ); exit; } /** * Add a stable seven-day schedule for the weekly health-check option. * WordPress core does not ship a "weekly" recurrence by default. * * @param array $schedules Existing schedules. * @return array */ public static function add_weekly_schedule( $schedules ) { if ( ! isset( $schedules['ai_o_optimiser_health_weekly'] ) ) { $schedules['ai_o_optimiser_health_weekly'] = array( 'interval' => 7 * DAY_IN_SECONDS, 'display' => __( 'Once weekly', 'ai-o-optimiser' ), ); } return $schedules; } /** * Return the cached report or an empty not-checked payload. * * @return array */ public static function report() { $report = get_option( self::OPTION, array() ); if ( empty( $report ) || ! is_array( $report ) ) { return self::empty_report(); } return wp_parse_args( $report, self::empty_report() ); } /** * Default settings for the nightly health monitor. * * @return array */ public static function health_default_settings() { return AI_O_Optimiser_Settings::health_defaults(); } /** * Return latest automated health monitor report. * * @return array */ public static function health_report() { $report = get_option( self::HEALTH_OPTION, array() ); if ( empty( $report ) || ! is_array( $report ) ) { return array( 'checked_at' => '', 'overall' => 'not_checked', 'items' => array(), 'version' => AI_O_Optimiser_VERSION, ); } return wp_parse_args( $report, array( 'checked_at' => '', 'overall' => 'not_checked', 'items' => array(), 'version' => AI_O_Optimiser_VERSION, ) ); } /** * Schedule or clear the nightly health check based on current settings. */ public static function reschedule_nightly_health_check() { wp_clear_scheduled_hook( self::NIGHTLY_HOOK ); $settings = wp_parse_args( get_option( 'ai_o_optimiser_settings', array() ), self::health_default_settings() ); if ( empty( $settings['health_check_enabled'] ) ) { return; } // Registered here rather than only in the constructor: this method also // runs during activation/upgrade, which are not guaranteed to be requests // where AI_O_Optimiser_System_Check has been instantiated yet. wp_schedule_event() // resolves the schedule name through this same filter, so it must be // registered before that call in whichever request actually makes it. add_filter( 'cron_schedules', array( __CLASS__, 'add_weekly_schedule' ) ); // phpcs:ignore WordPress.WP.CronInterval.ChangeDetected -- registers a schedule choice, not a fixed interval change. $frequency = isset( $settings['health_check_frequency'] ) && 'weekly' === $settings['health_check_frequency'] ? 'weekly' : 'daily'; $schedule = 'weekly' === $frequency ? 'ai_o_optimiser_health_weekly' : 'daily'; $hour = isset( $settings['health_check_hour'] ) ? min( 23, max( 17, (int) $settings['health_check_hour'] ) ) : 20; $next = new DateTimeImmutable( 'now', wp_timezone() ); $next = $next->setTime( $hour, 0, 0 ); if ( $next->getTimestamp() <= time() ) { $next = $next->modify( '+1 day' ); } wp_schedule_event( $next->getTimestamp(), $schedule, self::NIGHTLY_HOOK ); } /** * Scheduled AI crawlability and SEO sanity check. * * @return array */ public static function run_scheduled_health_check() { $settings = wp_parse_args( get_option( 'ai_o_optimiser_settings', array() ), AI_O_Optimiser_Settings::defaults() ); if ( empty( $settings['health_check_enabled'] ) ) { return array(); } $items = array(); $system = self::run_check(); $items[] = self::health_item( 'System check', $system['overall'], self::collect_section_messages( $system, array( 'dns', 'http', 'robots' ) ) ); if ( class_exists( 'AI_O_Optimiser_LLMS_Admin' ) ) { $llms = AI_O_Optimiser_LLMS_Admin::run_audit(); $items[] = self::health_item( 'llms.txt quality check', isset( $llms['overall_status'] ) ? $llms['overall_status'] : 'warning', self::summarise_llms_audit( $llms ) ); } $items[] = self::check_ai_model_health(); $hub_item = self::check_hub_compatibility(); if ( $hub_item ) { $items[] = $hub_item; } $items[] = self::check_file_access_for_user_agents( '/robots.txt', true ); $items[] = self::check_file_access_for_user_agents( '/llms.txt', true ); $items[] = self::check_sitemap_health(); foreach ( self::check_key_page_health() as $item ) { $items[] = $item; } $overall = self::derive_health_status( $items ); $report = array( 'checked_at' => current_time( 'mysql', true ), 'overall' => $overall, 'items' => $items, 'version' => AI_O_Optimiser_VERSION, ); update_option( self::HEALTH_OPTION, $report, false ); if ( 'fail' === $overall ) { self::send_health_email( $report, $settings ); } return $report; } /** * Check the Connector Hub still accepts this plugin's Google sign-in returns. * * A Hub that has fallen behind this plugin stops stamping a purpose on the * OAuth return, and every connection attempt is rejected with a message that * does not name the cause — which is how Hub 1.6.2 took Search Console, GA4 * and Business Profile down together without anything reporting why. This * asks the deployed Hub what it recognises, which no repository test can see. * * Returns null for sites with no Hub connection: there is nothing to judge, * and an empty row would just add noise to their report. * * @return array|null Health item, or null when there is no Hub connection. */ private static function check_hub_compatibility() { $label = 'Connector Hub compatibility'; if ( function_exists( 'ai_o_optimiser_guidance_load_connector_client' ) ) { ai_o_optimiser_guidance_load_connector_client(); } require_once AI_O_Optimiser_PLUGIN_DIR . 'includes/class-ai-o-optimiser-hub-compatibility.php'; $verdict = AI_O_Optimiser_Hub_Compatibility::refresh(); if ( ! is_array( $verdict ) ) { return null; } // health_item() knows pass/warning/fail/not_checked and coerces anything // else to warning, so the verdict's own 'ok' must be translated here or // a fully compatible Hub would put a warning on every clean report. $status = 'pass'; if ( 'incompatible' === $verdict['state'] ) { $status = 'fail'; } elseif ( 'unknown' === $verdict['state'] ) { $status = 'warning'; } return self::health_item( $label, $status, array( $verdict['message'] ) ); } /** * Check that the configured direct-provider model still exists. * * A model that is withdrawn does not announce itself: the next scan simply * starts failing, and the error that surfaces is a generic HTTP failure * rather than "this model is gone". Listing models costs no tokens, so it * is cheap to confirm nightly that the configured one is still offered. * * A list that cannot be read means "unknown", never "the model is gone" -- * a provider outage must not be reported as a broken configuration. * * @return array Health item. */ private static function check_ai_model_health() { $label = 'AI model availability'; // A manual run happens on the System Check screen, where the metadata // runtime is deliberately not booted. Load it here rather than reporting // "not checked" for a question that can be answered. if ( ! class_exists( 'AI_O_Optimiser_Metadata_Provider' ) && function_exists( 'ai_o_optimiser_metadata_load_classes' ) ) { ai_o_optimiser_metadata_load_classes(); } if ( ! class_exists( 'AI_O_Optimiser_Metadata_Provider' ) || ! class_exists( 'AI_O_Optimiser_AI_Models' ) ) { return self::health_item( $label, 'not_checked', array( 'The metadata runtime is not available in this context.' ) ); } $settings = get_option( 'ai_o_optimiser_metadata_settings', array() ); $settings = is_array( $settings ) ? $settings : array(); $provider = 'gemini' === ( $settings['provider'] ?? 'openai' ) ? 'gemini' : 'openai'; $name = 'gemini' === $provider ? 'Gemini' : 'OpenAI'; if ( ! AI_O_Optimiser_Metadata_Provider::direct_key_configured( $provider, $settings ) ) { // Nothing to verify: either WordPress's connector is handling AI, or // AI features are simply not set up. Neither is a fault. return self::health_item( $label, 'not_checked', array( sprintf( 'No direct %s key is configured, so there is no fallback model to verify.', $name ) ) ); } $model = AI_O_Optimiser_AI_Models::configured( $provider, $settings ); $available = AI_O_Optimiser_Metadata_Provider::refresh_models( $provider, $settings ); if ( ! $available ) { return self::health_item( $label, 'not_checked', array( sprintf( 'Could not read the %s model list, so %s could not be confirmed. This usually means a temporary provider or network problem.', $name, $model ) ) ); } if ( in_array( $model, $available, true ) ) { return self::health_item( $label, 'pass', array( sprintf( '%s still offers %s.', $name, $model ) ) ); } return self::health_item( $label, 'warning', array( sprintf( '%s no longer lists %s among the models available to this account. AI metadata and schema assessment will fail until a current model is selected in Metadata settings.', $name, $model ), sprintf( 'Currently offered: %s.', implode( ', ', array_slice( $available, 0, 8 ) ) ), ) ); } /** * Build a health item payload. */ private static function health_item( $label, $status, $messages ) { return array( 'label' => (string) $label, 'status' => in_array( $status, array( 'pass', 'warning', 'fail', 'not_checked' ), true ) ? $status : 'warning', 'messages' => array_values( array_filter( array_map( 'wp_strip_all_tags', (array) $messages ) ) ), ); } /** * Derive overall monitor status from health items. */ private static function derive_health_status( $items ) { $overall = 'pass'; foreach ( (array) $items as $item ) { if ( isset( $item['status'] ) && 'fail' === $item['status'] ) { return 'fail'; } if ( isset( $item['status'] ) && 'warning' === $item['status'] ) { $overall = 'warning'; } } return $overall; } /** * Collect warnings and notes from system check sections. */ private static function collect_section_messages( $report, $sections ) { $messages = array(); foreach ( $sections as $section ) { if ( ! empty( $report[ $section ]['warnings'] ) ) { $messages = array_merge( $messages, $report[ $section ]['warnings'] ); } } return $messages ? $messages : array( 'No critical system check issues detected.' ); } /** * Summarise llms.txt audit warnings/failures for email and monitor log. */ private static function summarise_llms_audit( $audit ) { $messages = array(); if ( ! empty( $audit['files'] ) && is_array( $audit['files'] ) ) { foreach ( $audit['files'] as $file ) { if ( empty( $file['warnings'] ) || ! is_array( $file['warnings'] ) ) { continue; } foreach ( $file['warnings'] as $warning ) { $messages[] = $file['label'] . ': ' . $warning; } } } return $messages ? $messages : array( 'llms.txt audit did not report failures.' ); } /** * Test robots.txt or llms.txt with crawler-specific user-agents. */ private static function check_file_access_for_user_agents( $path, $require_plain_text ) { $url = home_url( $path ); $agents = array( 'ClaudeBot' => 'ClaudeBot/1.0', 'anthropic-ai' => 'anthropic-ai', 'GPTBot' => 'GPTBot/1.0', 'OAI-SearchBot' => 'OAI-SearchBot', 'PerplexityBot' => 'PerplexityBot/1.0', 'Googlebot' => 'Googlebot/2.1', 'Google-CloudVertexBot' => 'Google-CloudVertexBot', 'Google-GeminiNotebook' => 'Google-GeminiNotebook', 'Google-NotebookLM' => 'Google-NotebookLM', 'Google-Agent / Gemini' => 'Google-Agent', ); $status = 'pass'; $messages = array(); foreach ( $agents as $label => $ua ) { $response = wp_safe_remote_head( $url, array( 'timeout' => 8, 'redirection' => 3, 'headers' => array( 'User-Agent' => $ua, 'Accept' => 'text/plain,*/*;q=0.8', ), ) ); if ( is_wp_error( $response ) ) { $status = 'fail'; $messages[] = $label . ' could not fetch ' . $path . ': ' . $response->get_error_message(); continue; } $code = (int) wp_remote_retrieve_response_code( $response ); $headers = self::normalise_headers( wp_remote_retrieve_headers( $response ) ); $content_type = isset( $headers['content-type'] ) ? strtolower( $headers['content-type'] ) : ''; $x_robots = isset( $headers['x-robots-tag'] ) ? strtolower( $headers['x-robots-tag'] ) : ''; $length = isset( $headers['content-length'] ) ? (int) $headers['content-length'] : 0; if ( $code >= 400 || 0 === $code ) { $status = 'fail'; $messages[] = $label . ' received HTTP ' . $code . ' for ' . $path . '.'; } if ( $require_plain_text && $code >= 200 && $code < 400 && $content_type && false === strpos( $content_type, 'text/plain' ) && false === strpos( $content_type, 'text/markdown' ) ) { if ( 'fail' !== $status ) { $status = 'warning'; } $messages[] = $label . ' received unexpected Content-Type for ' . $path . ': ' . $content_type . '.'; } if ( false !== strpos( $x_robots, 'nofollow' ) || preg_match( '/(^|[,\s])none([,\s]|$)/i', $x_robots ) ) { if ( 'fail' !== $status ) { $status = '/llms.txt' === $path ? 'warning' : 'fail'; } $messages[] = $label . ' received X-Robots-Tag with nofollow/none for ' . $path . '. For llms.txt, use noindex without nofollow so internal guidance links remain followable.'; } elseif ( false !== strpos( $x_robots, 'noindex' ) && '/llms.txt' !== $path ) { $status = 'fail'; $messages[] = $label . ' received X-Robots-Tag noindex for ' . $path . '.'; } if ( '/llms.txt' === $path && $length > 0 && $length < 200 ) { if ( 'fail' !== $status ) { $status = 'warning'; } $messages[] = $label . ' received a very small llms.txt response: ' . $length . ' bytes.'; } } $default_message = '/llms.txt' === $path ? 'llms.txt returned acceptable headers for tested crawler user-agents. X-Robots-Tag noindex is acceptable here as long as nofollow is not present.' : ltrim( $path, '/' ) . ' returned acceptable headers for tested crawler user-agents.'; return self::health_item( ltrim( $path, '/' ) . ' crawler access', $status, $messages ? $messages : array( $default_message ) ); } /** * Check XML sitemap availability. */ private static function check_sitemap_health() { $candidates = array( home_url( '/sitemap_index.xml' ), home_url( '/sitemap.xml' ) ); $messages = array(); foreach ( $candidates as $url ) { $response = wp_safe_remote_head( $url, array( 'timeout' => 8, 'redirection' => 3, 'headers' => array( 'User-Agent' => 'AI-O Optimiser Health Check/' . AI_O_Optimiser_VERSION ) ) ); if ( is_wp_error( $response ) ) { $messages[] = $url . ': ' . $response->get_error_message(); continue; } $code = (int) wp_remote_retrieve_response_code( $response ); if ( $code >= 200 && $code < 400 ) { return self::health_item( 'Sitemap', 'pass', array( 'Sitemap responded at ' . $url . ' with HTTP ' . $code . '.' ) ); } $messages[] = $url . ' returned HTTP ' . $code . '.'; } return self::health_item( 'Sitemap', 'warning', $messages ); } /** * Check homepage, likely contact/blog pages and a small sample of public pages. */ private static function check_key_page_health() { $urls = array( home_url( '/' ) ); foreach ( array( 'contact', 'contact-us' ) as $slug ) { $page = get_page_by_path( $slug ); if ( $page && 'publish' === get_post_status( $page ) ) { $urls[] = get_permalink( $page ); break; } } $posts_page = (int) get_option( 'page_for_posts' ); if ( $posts_page && 'publish' === get_post_status( $posts_page ) ) { $urls[] = get_permalink( $posts_page ); } $sample_pages = get_pages( array( 'sort_column' => 'menu_order,post_title', 'number' => 3, 'post_status' => 'publish' ) ); foreach ( $sample_pages as $page ) { $urls[] = get_permalink( $page ); } $urls = array_values( array_unique( apply_filters( 'ai_o_optimiser_health_check_urls', $urls ) ) ); $items = array(); foreach ( $urls as $url ) { $items[] = self::check_single_public_page( $url ); } return $items; } /** * Check one public HTML URL for status and noindex headers. */ private static function check_single_public_page( $url ) { $response = wp_safe_remote_head( $url, array( 'timeout' => 8, 'redirection' => 3, 'headers' => array( 'User-Agent' => 'AI-O Optimiser Health Check/' . AI_O_Optimiser_VERSION ) ) ); if ( is_wp_error( $response ) ) { return self::health_item( 'Public page: ' . $url, 'fail', array( $response->get_error_message() ) ); } $code = (int) wp_remote_retrieve_response_code( $response ); $headers = self::normalise_headers( wp_remote_retrieve_headers( $response ) ); $messages = array(); $status = 'pass'; if ( $code >= 400 || 0 === $code ) { $status = 'fail'; $messages[] = 'HTTP ' . $code . ' returned.'; } elseif ( $code >= 300 ) { $status = 'warning'; $messages[] = 'Redirect status HTTP ' . $code . ' returned.'; } if ( ! empty( $headers['x-robots-tag'] ) && false !== stripos( $headers['x-robots-tag'], 'noindex' ) ) { $status = 'fail'; $messages[] = 'X-Robots-Tag contains noindex.'; } return self::health_item( 'Public page: ' . $url, $status, $messages ? $messages : array( 'HTTP ' . $code . ' and no blocking X-Robots-Tag detected.' ) ); } /** * Send failure email to the configured tech address. */ private static function send_health_email( $report, $settings ) { $email = ! empty( $settings['health_check_email'] ) && is_email( $settings['health_check_email'] ) ? $settings['health_check_email'] : get_option( 'admin_email' ); if ( ! $email || ! is_email( $email ) ) { return false; } $subject = sprintf( '[%s] AI crawlability health check failed', wp_parse_url( home_url( '/' ), PHP_URL_HOST ) ); $lines = array(); $lines[] = 'The nightly AI crawlability and SEO health check found at least one failure.'; $lines[] = ''; $lines[] = 'Site: ' . home_url( '/' ); $lines[] = 'Checked: ' . self::format_site_time( $report['checked_at'] ); $lines[] = 'Overall status: ' . strtoupper( $report['overall'] ); $lines[] = ''; foreach ( (array) $report['items'] as $item ) { if ( empty( $item['status'] ) || 'pass' === $item['status'] ) { continue; } $lines[] = strtoupper( $item['status'] ) . ': ' . $item['label']; foreach ( (array) $item['messages'] as $message ) { $lines[] = ' - ' . $message; } $lines[] = ''; } $lines[] = 'Review in WordPress: ' . admin_url( 'admin.php?page=ai-o-optimiser-system-check' ); return wp_mail( $email, $subject, implode( "\n", $lines ) ); } /** * Empty report payload. * * @return array */ private static function empty_report() { return array( 'checked_at' => '', 'overall' => 'not_checked', 'environment' => array(), 'dns' => array( 'status' => 'not_checked', 'notes' => array(), 'warnings' => array() ), 'http' => array( 'status' => 'not_checked', 'notes' => array(), 'warnings' => array(), 'headers' => array() ), 'robots' => array( 'status' => 'not_checked', 'notes' => array(), 'warnings' => array(), 'bots' => array(), 'snippet' => '' ), 'version' => AI_O_Optimiser_VERSION, ); } /** * Run every diagnostic check and cache the result. * * @return array */ public static function run_check() { $environment = self::check_environment(); $dns = self::check_dns(); $http = self::check_http_layer(); $robots = self::check_robots(); $overall = 'pass'; foreach ( array( $dns, $http, $robots ) as $section ) { if ( isset( $section['status'] ) && 'fail' === $section['status'] ) { $overall = 'fail'; break; } if ( isset( $section['status'] ) && 'warning' === $section['status'] && 'fail' !== $overall ) { $overall = 'warning'; } } $report = array( 'checked_at' => current_time( 'mysql', true ), 'overall' => $overall, 'environment' => $environment, 'dns' => $dns, 'http' => $http, 'robots' => $robots, 'version' => AI_O_Optimiser_VERSION, ); update_option( self::OPTION, $report, false ); return $report; } /** * Runtime / plugin environment diagnostics. * * @return array */ private static function check_environment() { $settings = wp_parse_args( get_option( 'ai_o_optimiser_settings', array() ), AI_O_Optimiser_Settings::defaults() ); return array( 'home_url' => home_url( '/' ), 'site_timezone' => wp_timezone_string() ? wp_timezone_string() : 'UTC offset ' . get_option( 'gmt_offset' ), 'php_version' => PHP_VERSION, 'fastcgi_finish_request' => function_exists( 'fastcgi_finish_request' ), 'tracker_enabled' => ! empty( $settings['enabled'] ), 'referral_tracking' => ! empty( $settings['enable_referrals'] ), 'llms_request_tracking' => ! empty( $settings['llms_track_hits'] ), 'rank_math_detected' => self::is_rank_math_active(), 'cache_plugins_detected' => self::detect_cache_plugins(), 'wordpress_version' => get_bloginfo( 'version' ), 'plugin_version' => AI_O_Optimiser_VERSION, ); } /** * DNS and public record hints. This cannot inspect provider dashboards. * * @return array */ private static function check_dns() { $host = wp_parse_url( home_url( '/' ), PHP_URL_HOST ); $host = $host ? strtolower( preg_replace( '/^www\./i', '', $host ) ) : ''; $result = array( 'status' => 'pass', 'host' => $host, 'nameservers' => array(), 'mx' => array(), 'a' => array(), 'aaaa' => array(), 'provider' => __( 'Unknown / not detected', 'ai-o-optimiser' ), 'notes' => array(), 'warnings' => array(), ); if ( empty( $host ) ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'Could not determine the site host from the WordPress Home URL.', 'ai-o-optimiser' ); return $result; } if ( ! function_exists( 'dns_get_record' ) ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'PHP dns_get_record() is unavailable, so public DNS records could not be inspected from WordPress.', 'ai-o-optimiser' ); return $result; } $ns_records = @dns_get_record( $host, DNS_NS ); $mx_records = @dns_get_record( $host, DNS_MX ); $a_records = @dns_get_record( $host, DNS_A ); $aaaa_records = @dns_get_record( $host, DNS_AAAA ); if ( is_array( $ns_records ) ) { foreach ( $ns_records as $record ) { if ( ! empty( $record['target'] ) ) { $result['nameservers'][] = rtrim( strtolower( $record['target'] ), '.' ); } } } if ( is_array( $mx_records ) ) { foreach ( $mx_records as $record ) { if ( ! empty( $record['target'] ) ) { $result['mx'][] = rtrim( strtolower( $record['target'] ), '.' ); } } } if ( is_array( $a_records ) ) { foreach ( $a_records as $record ) { if ( ! empty( $record['ip'] ) ) { $result['a'][] = $record['ip']; } } } if ( is_array( $aaaa_records ) ) { foreach ( $aaaa_records as $record ) { if ( ! empty( $record['ipv6'] ) ) { $result['aaaa'][] = $record['ipv6']; } } } $ns_blob = implode( ' ', $result['nameservers'] ); $mx_blob = implode( ' ', $result['mx'] ); // phpcs:ignore PluginCheck.CodeAnalysis.Offloading.OffloadedContent -- This only recognises a public DNS provider name; AI-O Optimiser does not offload plugin assets or content to Cloudflare. if ( false !== strpos( $ns_blob, 'cloudflare' ) ) { $result['provider'] = 'Cloudflare DNS'; $result['notes'][] = __( 'Cloudflare nameservers detected. Cloudflare WAF, bot rules and edge cache can affect whether AI crawlers reach WordPress.', 'ai-o-optimiser' ); } elseif ( false !== strpos( $ns_blob, 'azure-dns' ) || false !== strpos( $ns_blob, 'microsoft' ) ) { $result['provider'] = 'Microsoft / Azure DNS'; $result['notes'][] = __( 'Microsoft/Azure DNS detected. WordPress can see public records, but it cannot inspect tenant, DNS or security policies inside Microsoft/Azure.', 'ai-o-optimiser' ); } elseif ( false !== strpos( $ns_blob, 'awsdns' ) ) { $result['provider'] = 'Amazon Route 53'; } elseif ( false !== strpos( $ns_blob, 'domaincontrol' ) ) { $result['provider'] = 'GoDaddy DNS'; } elseif ( false !== strpos( $ns_blob, 'dns-parking' ) ) { $result['provider'] = 'Hostinger DNS'; } if ( false !== strpos( $mx_blob, 'mail.protection.outlook' ) ) { $result['notes'][] = __( 'Microsoft 365 mail records detected. This usually affects email routing only, not AI crawler access to the website.', 'ai-o-optimiser' ); } if ( empty( $result['nameservers'] ) ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'No nameserver records were returned. This may be a DNS lookup limitation on the server, not necessarily a live DNS issue.', 'ai-o-optimiser' ); } if ( empty( $result['a'] ) && empty( $result['aaaa'] ) ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'No A or AAAA records were returned for the root domain. If the site uses only a www host or a proxy, verify DNS manually.', 'ai-o-optimiser' ); } $result['notes'][] = __( 'DNS checks are public-record diagnostics only. They cannot read Cloudflare, Microsoft tenant, WAF or hosting control-panel settings without a separate API integration.', 'ai-o-optimiser' ); return $result; } /** * Fetch the home page externally and inspect CDN/cache/security headers. * * @return array */ private static function check_http_layer() { $result = array( 'status' => 'pass', 'url' => home_url( '/' ), 'http_code' => 0, 'headers' => array(), 'signals' => array(), 'notes' => array(), 'warnings' => array(), ); $fetched = AI_O_Optimiser_HTTP::get_with_fallback( $result['url'], array( 'timeout' => 8, 'redirection' => 3, 'headers' => array( 'Accept' => 'text/html,application/xhtml+xml,*/*;q=0.8', ), ), 'identified', 'System Check' ); $response = $fetched['response']; if ( is_wp_error( $response ) ) { $result['status'] = 'fail'; $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Could not fetch the home page over HTTP: %s', 'ai-o-optimiser' ), $response->get_error_message() ); return $result; } if ( ! empty( $fetched['recovered'] ) ) { // Identified UA was refused; the page itself is fine for browsers. // Surface the edge policy instead of failing the health check. $result['status'] = 'warning'; $result['warnings'][] = AI_O_Optimiser_HTTP::edge_block_note( $fetched['primary_code'], 'identified' ); } elseif ( AI_O_Optimiser_HTTP::is_waf_block_code( (int) wp_remote_retrieve_response_code( $response ) ) ) { $result['status'] = 'fail'; $result['warnings'][] = sprintf( /* translators: %d: HTTP status code */ __( 'The home page returned HTTP %d to every request profile from this server — the edge firewall is refusing this server\'s requests (bot management, IP reputation or rate limiting). Allowlist this server before trusting other checks.', 'ai-o-optimiser' ), (int) wp_remote_retrieve_response_code( $response ) ); } $result['http_code'] = (int) wp_remote_retrieve_response_code( $response ); $result['headers'] = self::normalise_headers( wp_remote_retrieve_headers( $response ) ); if ( $result['http_code'] >= 400 || 0 === $result['http_code'] ) { $result['status'] = 'fail'; $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'The home page returned HTTP %d to the system check request.', 'ai-o-optimiser' ), $result['http_code'] ); } $headers = $result['headers']; $server = isset( $headers['server'] ) ? strtolower( $headers['server'] ) : ''; $cf_ray = isset( $headers['cf-ray'] ) ? $headers['cf-ray'] : ''; $cf_cache = isset( $headers['cf-cache-status'] ) ? strtoupper( $headers['cf-cache-status'] ) : ''; $x_cache = isset( $headers['x-cache'] ) ? strtoupper( $headers['x-cache'] ) : ''; $via = isset( $headers['via'] ) ? strtolower( $headers['via'] ) : ''; if ( $cf_ray || false !== strpos( $server, 'cloudflare' ) || $cf_cache ) { $result['signals'][] = 'Cloudflare'; $result['notes'][] = __( 'Cloudflare response headers detected.', 'ai-o-optimiser' ); if ( 'HIT' === $cf_cache ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'Cloudflare returned CF-Cache-Status: HIT for the home page. If HTML is served from edge cache, WordPress may not log every AI crawler request.', 'ai-o-optimiser' ); } } if ( isset( $headers['x-varnish'] ) || false !== strpos( $via, 'varnish' ) ) { $result['signals'][] = 'Varnish'; $result['notes'][] = __( 'Varnish/cache proxy headers detected. Cloudways or server-level caching may serve some pages before WordPress runs.', 'ai-o-optimiser' ); } if ( false !== strpos( $server, 'litespeed' ) || isset( $headers['x-litespeed-cache'] ) ) { $result['signals'][] = 'LiteSpeed'; $result['notes'][] = __( 'LiteSpeed cache/server headers detected. Confirm whether AI crawler user-agents bypass page cache if complete tracking is required.', 'ai-o-optimiser' ); } if ( false !== strpos( $x_cache, 'HIT' ) ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'An X-Cache HIT header was detected. Cached HTML can reduce the number of requests that reach WordPress tracking.', 'ai-o-optimiser' ); } if ( empty( $result['signals'] ) ) { $result['notes'][] = __( 'No obvious CDN/page-cache response headers were detected on the home page request.', 'ai-o-optimiser' ); } return $result; } /** * Fetch and parse robots.txt for known AI crawler tokens. * * @return array */ private static function check_robots() { $url = home_url( '/robots.txt' ); $result = array( 'status' => 'pass', 'url' => $url, 'http_code' => 0, 'notes' => array(), 'warnings' => array(), 'bots' => array(), 'recommendations' => array(), 'snippet' => '', ); $fetched = AI_O_Optimiser_HTTP::get_with_fallback( $url, array( 'timeout' => 8, 'redirection' => 3, 'headers' => array( 'Accept' => 'text/plain,*/*;q=0.8', ), ), 'identified', 'robots.txt Audit' ); $response = $fetched['response']; if ( is_wp_error( $response ) ) { $result['status'] = 'fail'; $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Could not fetch robots.txt: %s', 'ai-o-optimiser' ), $response->get_error_message() ); return $result; } if ( ! empty( $fetched['recovered'] ) ) { $result['warnings'][] = AI_O_Optimiser_HTTP::edge_block_note( $fetched['primary_code'], 'identified' ); } $body = (string) wp_remote_retrieve_body( $response ); $result['http_code'] = (int) wp_remote_retrieve_response_code( $response ); $result['snippet'] = self::safe_substr( self::normalise_line_endings( $body ), 0, 2000 ); if ( 404 === $result['http_code'] ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'robots.txt returned 404. Most crawlers will treat this as no robots restrictions, but a deliberate robots.txt file is recommended.', 'ai-o-optimiser' ); } elseif ( $result['http_code'] >= 400 || 0 === $result['http_code'] ) { $result['status'] = 'fail'; $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'robots.txt returned HTTP %d. Crawlers may be unable to read your access rules.', 'ai-o-optimiser' ), $result['http_code'] ); } $groups = self::parse_robots( $body ); if ( 200 === $result['http_code'] && empty( $groups ) ) { $result['status'] = 'warning'; $result['warnings'][] = __( 'robots.txt was fetched, but no User-agent groups were parsed.', 'ai-o-optimiser' ); } $settings = wp_parse_args( get_option( 'ai_o_optimiser_settings', array() ), AI_O_Optimiser_Settings::defaults() ); $tracked_bots = isset( $settings['tracked_bots'] ) && is_array( $settings['tracked_bots'] ) ? $settings['tracked_bots'] : array(); $blocked = 0; $allowed = 0; $google_extended_root_allowed = null; $robots_subjects = array_merge( AI_O_Optimiser_Bots::all(), AI_O_Optimiser_Bots::robots_policy_tokens() ); foreach ( $robots_subjects as $slug => $bot ) { $needle = isset( $bot['needle'] ) ? $bot['needle'] : $slug; $root = self::robots_allowed_for_path( $groups, $needle, '/' ); $llms = self::robots_allowed_for_path( $groups, $needle, '/llms.txt' ); $is_policy = isset( $bot['group'] ) && 'policy' === $bot['group']; $is_on = $is_policy ? false : ( empty( $tracked_bots ) || ! empty( $tracked_bots[ $slug ] ) ); if ( ! empty( $root['allowed'] ) ) { $allowed++; } else { $blocked++; } $notes = array(); if ( 'googlebot-search' === $slug ) { $notes[] = __( 'Googlebot is a real Google Search crawler, but its requests are not counted as confirmed AI activity because the same crawler serves standard Search and Google AI Search features.', 'ai-o-optimiser' ); } elseif ( 'google-extended' === $slug ) { $google_extended_root_allowed = ! empty( $root['allowed'] ); $notes[] = __( 'Google-Extended is a robots.txt control token, not a standalone HTTP User-Agent in server logs. It governs Gemini / Vertex training and grounding uses, not Google Search inclusion.', 'ai-o-optimiser' ); } elseif ( $is_policy ) { $notes[] = __( 'Policy/status subject only; not included in measured AI activity totals.', 'ai-o-optimiser' ); } if ( ! $is_policy && ! $is_on ) { $notes[] = __( 'Disabled in plugin tracking settings.', 'ai-o-optimiser' ); } $result['bots'][] = array( 'slug' => $slug, 'label' => isset( $bot['label'] ) ? $bot['label'] : $slug, 'needle' => $needle, 'group' => isset( $bot['group'] ) ? $bot['group'] : 'other', 'tracked' => $is_on, 'root' => $root, 'llms' => $llms, 'notes' => $notes, 'diagnostic' => self::crawler_access_diagnostic( $slug, $bot, $root, $llms, $is_on ), ); } $result['summary'] = array( 'allowed' => $allowed, 'blocked' => $blocked, 'total' => count( $result['bots'] ), ); if ( $blocked > 0 && 'fail' !== $result['status'] ) { $result['status'] = 'warning'; $result['warnings'][] = sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%d crawler or AI policy token(s) appear blocked from the site root by robots.txt. This may be intentional, but it should be reviewed.', 'ai-o-optimiser' ), $blocked ); } if ( preg_match( '/^\s*#\s*ai-train\s*:\s*no\b/im', $body ) ) { $result['notes'][] = __( 'The “ai-train: no” line is an informational comment, not a standard robots.txt directive that Google enforces.', 'ai-o-optimiser' ); if ( true === $google_extended_root_allowed ) { if ( 'fail' !== $result['status'] ) { $result['status'] = 'warning'; } $result['warnings'][] = __( 'robots.txt commentary says “ai-train: no”, but Google-Extended is allowed. Google-Extended currently governs both future Gemini model training and Gemini / Vertex grounding, so the explanatory comment does not match Google’s active robots control.', 'ai-o-optimiser' ); } } $result['recommendations'] = self::robots_recommendations( $result, $body ); $result['notes'][] = __( 'robots.txt controls access rules. It does not prove that a request passed through Cloudflare, Microsoft, hosting WAF or server-level cache to WordPress.', 'ai-o-optimiser' ); return $result; } /** * Turn the live robots.txt result into conservative, visibility-first advice. * The plugin never writes robots.txt; suggested rules are for manual review. * * @param array $result Robots check result. * @param string $body Live robots.txt body. * @return array */ private static function robots_recommendations( $result, $body ) { $recommendations = array(); if ( empty( $result['http_code'] ) || 200 !== (int) $result['http_code'] ) { $recommendations[] = array( 'priority' => ( (int) $result['http_code'] >= 500 || 0 === (int) $result['http_code'] ) ? 'high' : 'medium', 'title' => __( 'Serve a stable live robots.txt response', 'ai-o-optimiser' ), 'detail' => sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'The live robots.txt response is HTTP %d. Resolve the public response before tuning individual crawler rules.', 'ai-o-optimiser' ), (int) $result['http_code'] ), 'snippet' => '', ); } foreach ( isset( $result['bots'] ) && is_array( $result['bots'] ) ? $result['bots'] : array() as $bot ) { if ( empty( $bot['root'] ) || ! empty( $bot['root']['allowed'] ) ) { continue; } $needle = isset( $bot['needle'] ) ? trim( (string) $bot['needle'] ) : ''; if ( '' === $needle ) { continue; } if ( 'googlebot-search' === $bot['slug'] ) { $recommendations[] = array( 'priority' => 'high', 'title' => __( 'Restore Google Search crawl access', 'ai-o-optimiser' ), 'detail' => __( 'Googlebot is blocked. If organic Search and Google AI Search visibility are intended, review the matching rule and allow Googlebot.', 'ai-o-optimiser' ), 'snippet' => "User-agent: Googlebot\nAllow: /", ); } elseif ( 'google-extended' === $bot['slug'] ) { $recommendations[] = array( 'priority' => 'policy', 'title' => __( 'Review Gemini / Vertex policy choice', 'ai-o-optimiser' ), 'detail' => __( 'Google-Extended is blocked. Keep it blocked if intentional; allow it only if the Gemini / Vertex uses governed by this token are part of the site visibility strategy.', 'ai-o-optimiser' ), 'snippet' => "User-agent: Google-Extended\nAllow: /", ); } elseif ( isset( $bot['group'] ) && 'user' === $bot['group'] ) { $recommendations[] = array( 'priority' => 'medium', 'title' => sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Review live-fetch access for %s', 'ai-o-optimiser' ), $bot['label'] ), 'detail' => __( 'This user-triggered AI fetcher is blocked. Allow it when live assistant retrieval/citation is a visibility goal; otherwise the block can remain deliberate.', 'ai-o-optimiser' ), 'snippet' => 'User-agent: ' . $needle . "\nAllow: /", ); } } return array_slice( $recommendations, 0, 12 ); } /** * Translate one robots access result into a stable user-facing diagnostic. * * The diagnostic contract is deliberately shared across crawler types: * state -> meaning -> impact -> recommended action -> caveat/evidence. * Blocking is not automatically treated as a mistake; training/policy controls * can be intentional, while search or live-fetch blocks may have different impact. * * @param string $slug Bot/token slug. * @param array $bot Bot registry entry. * @param array $root Root robots result. * @param array $llms llms.txt robots result. * @param bool $tracked Whether runtime tracking is enabled for this crawler. * @return array */ private static function crawler_access_diagnostic( $slug, $bot, $root, $llms, $tracked ) { $allowed = ! empty( $root['allowed'] ); $group = isset( $bot['group'] ) ? (string) $bot['group'] : 'other'; $label = isset( $bot['label'] ) ? (string) $bot['label'] : (string) $slug; $rule = ! empty( $root['matched_rule'] ) ? (string) $root['matched_rule'] : __( 'No matching path rule', 'ai-o-optimiser' ); $agent = ! empty( $root['matched_agent'] ) ? (string) $root['matched_agent'] : __( 'No matching robots group', 'ai-o-optimiser' ); $diagnostic = array( 'status' => $allowed ? 'pass' : 'warning', 'title' => $allowed ? __( 'Access allowed', 'ai-o-optimiser' ) : __( 'Access blocked', 'ai-o-optimiser' ), 'meaning' => $allowed ? sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%s can access the site root under the current robots.txt rules.', 'ai-o-optimiser' ), $label ) : sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%s is restricted from the site root by robots.txt.', 'ai-o-optimiser' ), $label ), 'impact' => '', 'action' => '', 'caveat' => '', 'evidence'=> sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( '%1$s · matched agent: %2$s', 'ai-o-optimiser' ), $rule, $agent ), ); if ( 'googlebot-search' === $slug ) { $diagnostic['status'] = $allowed ? 'pass' : 'fail'; $diagnostic['title'] = $allowed ? __( 'Google Search crawl eligibility available', 'ai-o-optimiser' ) : __( 'Google Search crawl eligibility blocked', 'ai-o-optimiser' ); $diagnostic['meaning'] = $allowed ? __( 'Googlebot is permitted to crawl the site under robots.txt.', 'ai-o-optimiser' ) : __( 'Googlebot is blocked at the site root by robots.txt.', 'ai-o-optimiser' ); $diagnostic['impact'] = $allowed ? __( 'This supports crawl eligibility for standard Google Search and Google AI Search features.', 'ai-o-optimiser' ) : __( 'This can prevent Google Search crawling and therefore restrict eligibility for Google Search AI features as well.', 'ai-o-optimiser' ); $diagnostic['action'] = $allowed ? __( 'No robots.txt change is indicated unless you intentionally want to restrict Google Search.', 'ai-o-optimiser' ) : __( 'Review the matching Googlebot or wildcard rule and allow Googlebot if organic Search visibility is intended.', 'ai-o-optimiser' ); $diagnostic['caveat'] = __( 'Allowed crawling does not prove indexing, ranking, AI Overview appearance or citation.', 'ai-o-optimiser' ); return $diagnostic; } if ( 'google-extended' === $slug ) { $diagnostic['status'] = $allowed ? 'pass' : 'warning'; $diagnostic['title'] = $allowed ? __( 'Gemini / Vertex policy allowed', 'ai-o-optimiser' ) : __( 'Gemini / Vertex policy restricted', 'ai-o-optimiser' ); $diagnostic['meaning'] = $allowed ? __( 'The uses governed by Google-Extended are permitted by robots.txt.', 'ai-o-optimiser' ) : __( 'The uses governed by Google-Extended are restricted by robots.txt.', 'ai-o-optimiser' ); $diagnostic['impact'] = $allowed ? __( 'Gemini / Vertex uses controlled by Google-Extended are not being blocked by this robots policy.', 'ai-o-optimiser' ) : __( 'Gemini / Vertex uses governed by Google-Extended are restricted. This does not block inclusion in Google Search.', 'ai-o-optimiser' ); $diagnostic['action'] = $allowed ? __( 'Keep this setting only if the permitted Google-Extended uses match your content policy.', 'ai-o-optimiser' ) : __( 'Leave blocked if this restriction is intentional; allow Google-Extended if those Gemini / Vertex uses are a visibility priority.', 'ai-o-optimiser' ); $diagnostic['caveat'] = __( 'Google-Extended is a robots.txt control token rather than a separately measurable server-log crawler.', 'ai-o-optimiser' ); return $diagnostic; } if ( 'user' === $group ) { $diagnostic['title'] = $allowed ? __( 'Live AI fetch access allowed', 'ai-o-optimiser' ) : __( 'Live AI fetch access blocked', 'ai-o-optimiser' ); $diagnostic['impact'] = $allowed ? __( 'User-triggered assistant requests can reach public pages when the provider uses this fetcher.', 'ai-o-optimiser' ) : __( 'A user asking the relevant assistant to inspect, browse or cite a page may receive reduced or failed live access through this fetcher.', 'ai-o-optimiser' ); $diagnostic['action'] = $allowed ? __( 'No change is needed if live AI assistant access is desired.', 'ai-o-optimiser' ) : __( 'Allow this user-triggered fetcher if live assistant retrieval and citation are desired; otherwise the block may be intentional.', 'ai-o-optimiser' ); $diagnostic['caveat'] = __( 'This result describes robots.txt access only; WAF, CDN or rate-limit rules can still block the request before it reaches WordPress.', 'ai-o-optimiser' ); } elseif ( 'training' === $group ) { $is_search_like = false !== stripos( $label, 'search' ) || false !== stripos( $label, 'index' ); $diagnostic['title'] = $is_search_like ? ( $allowed ? __( 'Discovery / index crawler allowed', 'ai-o-optimiser' ) : __( 'Discovery / index crawler blocked', 'ai-o-optimiser' ) ) : ( $allowed ? __( 'Training / discovery crawler allowed', 'ai-o-optimiser' ) : __( 'Training / discovery crawler blocked', 'ai-o-optimiser' ) ); $diagnostic['impact'] = $allowed ? ( $is_search_like ? __( 'This provider can use the crawler for discovery or index refresh where supported.', 'ai-o-optimiser' ) : __( 'This provider is not restricted by robots.txt from the crawler use represented by this user-agent.', 'ai-o-optimiser' ) ) : ( $is_search_like ? __( 'Discovery or index refresh through this crawler may be reduced, which can affect how current your content is within that provider.', 'ai-o-optimiser' ) : __( 'The crawler use represented by this user-agent is restricted. For training-oriented crawlers this may be an intentional content-policy choice.', 'ai-o-optimiser' ) ); $diagnostic['action'] = $allowed ? __( 'Keep allowed if this provider is part of your AI discovery strategy; otherwise review your content-use policy.', 'ai-o-optimiser' ) : __( 'Decide deliberately: allow it when provider discovery is a priority, or keep it blocked when the restriction matches your policy.', 'ai-o-optimiser' ); $diagnostic['caveat'] = __( 'A blocked discovery/training crawler does not automatically mean the provider cannot know about the site through other indexes, referrals or user-triggered fetchers.', 'ai-o-optimiser' ); } else { $diagnostic['title'] = $allowed ? __( 'Crawler / validation access allowed', 'ai-o-optimiser' ) : __( 'Crawler / validation access blocked', 'ai-o-optimiser' ); $diagnostic['impact'] = $allowed ? __( 'The platform-specific crawler or validator is not restricted by robots.txt.', 'ai-o-optimiser' ) : __( 'The platform-specific crawler or validator cannot access the site root under the current robots policy.', 'ai-o-optimiser' ); $diagnostic['action'] = $allowed ? __( 'No change is needed unless this access conflicts with your policy.', 'ai-o-optimiser' ) : __( 'Review the crawler purpose and allow it only if that platform-specific access is useful to the site.', 'ai-o-optimiser' ); $diagnostic['caveat'] = __( 'Custom and validation bots can have provider-specific behaviour; confirm the crawler purpose before changing policy.', 'ai-o-optimiser' ); } if ( ! $tracked && 'policy' !== $group ) { $diagnostic['caveat'] .= ' ' . __( 'This crawler is currently disabled in AI-O Optimiser tracking, so allowed access will not create plugin activity records until tracking is enabled for it.', 'ai-o-optimiser' ); } if ( empty( $llms['allowed'] ) ) { $diagnostic['caveat'] .= ' ' . __( 'It is also blocked from /llms.txt.', 'ai-o-optimiser' ); } return $diagnostic; } /** * Parse robots.txt into groups. * * @param string $body Robots.txt content. * @return array */ private static function parse_robots( $body ) { $groups = array(); $agents = array(); $rules = array(); $saw_rule = false; $lines = preg_split( '/\r\n|\r|\n/', (string) $body ); foreach ( $lines as $line ) { $line = preg_replace( '/\s*#.*$/', '', trim( $line ) ); if ( '' === $line || false === strpos( $line, ':' ) ) { continue; } list( $field, $value ) = array_map( 'trim', explode( ':', $line, 2 ) ); $field = strtolower( $field ); if ( 'user-agent' === $field ) { if ( $saw_rule && ! empty( $agents ) ) { $groups[] = array( 'agents' => $agents, 'rules' => $rules ); $agents = array(); $rules = array(); $saw_rule = false; } if ( '' !== $value ) { $agents[] = strtolower( $value ); } } elseif ( in_array( $field, array( 'allow', 'disallow' ), true ) && ! empty( $agents ) ) { $saw_rule = true; $rules[] = array( 'directive' => $field, 'path' => $value, ); } } if ( ! empty( $agents ) ) { $groups[] = array( 'agents' => $agents, 'rules' => $rules ); } return $groups; } /** * Determine whether a robots token is allowed for a path. * * @param array $groups Parsed robots groups. * @param string $ua_token User-agent token. * @param string $path URL path. * @return array */ private static function robots_allowed_for_path( $groups, $ua_token, $path ) { if ( empty( $groups ) ) { return array( 'allowed' => true, 'matched_agent' => __( 'No parsed robots group', 'ai-o-optimiser' ), 'matched_rule' => '', 'reason' => __( 'No matching robots rule was found.', 'ai-o-optimiser' ), ); } $ua_token = strtolower( (string) $ua_token ); $matches = array(); $max_len = -1; foreach ( $groups as $group ) { foreach ( $group['agents'] as $agent ) { $agent = strtolower( trim( $agent ) ); if ( '*' === $agent || false !== strpos( $ua_token, $agent ) || false !== strpos( $agent, $ua_token ) ) { $len = '*' === $agent ? 0 : strlen( $agent ); if ( $len > $max_len ) { $matches = array(); $max_len = $len; } if ( $len === $max_len ) { $matches[] = array( 'agent' => $agent, 'rules' => $group['rules'] ); } } } } if ( empty( $matches ) ) { return array( 'allowed' => true, 'matched_agent' => __( 'None', 'ai-o-optimiser' ), 'matched_rule' => '', 'reason' => __( 'No matching user-agent group. Crawler is treated as allowed.', 'ai-o-optimiser' ), ); } $best_rule = null; foreach ( $matches as $match ) { foreach ( $match['rules'] as $rule ) { if ( 'disallow' === $rule['directive'] && '' === trim( $rule['path'] ) ) { continue; } if ( self::robots_path_matches( $rule['path'], $path ) ) { $length = strlen( rtrim( $rule['path'], '$' ) ); if ( null === $best_rule || $length > $best_rule['length'] || ( $length === $best_rule['length'] && 'allow' === $rule['directive'] ) ) { $best_rule = array( 'directive' => $rule['directive'], 'path' => $rule['path'], 'agent' => $match['agent'], 'length' => $length, ); } } } } if ( null === $best_rule ) { return array( 'allowed' => true, 'matched_agent' => $matches[0]['agent'], 'matched_rule' => '', 'reason' => __( 'Matching user-agent group has no rule for this path.', 'ai-o-optimiser' ), ); } $allowed = 'allow' === $best_rule['directive']; return array( 'allowed' => $allowed, 'matched_agent' => $best_rule['agent'], 'matched_rule' => ucfirst( $best_rule['directive'] ) . ': ' . $best_rule['path'], 'reason' => $allowed ? __( 'Allowed by the most specific matching robots rule.', 'ai-o-optimiser' ) : __( 'Blocked by the most specific matching robots rule.', 'ai-o-optimiser' ), ); } /** * Match a robots path pattern against a URL path. * * @param string $pattern Robots path. * @param string $path URL path. * @return bool */ private static function robots_path_matches( $pattern, $path ) { $pattern = trim( (string) $pattern ); if ( '' === $pattern ) { return false; } $path = '/' . ltrim( (string) $path, '/' ); $end_anchor = false; if ( '$' === substr( $pattern, -1 ) ) { $end_anchor = true; $pattern = substr( $pattern, 0, -1 ); } $regex = preg_quote( $pattern, '#' ); $regex = str_replace( '\\*', '.*', $regex ); $regex = '#^' . $regex . ( $end_anchor ? '$' : '' ) . '#'; return 1 === preg_match( $regex, $path ); } /** * Convert WP HTTP headers to a lower-case string array. * * @param mixed $headers Header collection. * @return array */ private static function normalise_headers( $headers ) { if ( is_object( $headers ) && method_exists( $headers, 'getAll' ) ) { $headers = $headers->getAll(); } $normalised = array(); foreach ( (array) $headers as $key => $value ) { $key = strtolower( (string) $key ); if ( is_array( $value ) ) { $value = implode( ', ', $value ); } $normalised[ $key ] = (string) $value; } return $normalised; } /** * Detect common WordPress cache/CDN plugins. * * @return array */ private static function detect_cache_plugins() { require_once AI_O_Optimiser_PLUGIN_DIR . 'includes/class-ai-o-optimiser-cache-layers.php'; // One list, shared with the Visibility dashboard's measurement caveat. return AI_O_Optimiser_Cache_Layers::active_plugins(); } /** * Determine whether Rank Math appears active. * * @return bool */ private static function is_rank_math_active() { if ( defined( 'RANK_MATH_VERSION' ) || class_exists( 'RankMath' ) || class_exists( '\\RankMath\\Helper' ) ) { return true; } $active = (array) get_option( 'active_plugins', array() ); if ( is_multisite() ) { $network_active = (array) get_site_option( 'active_sitewide_plugins', array() ); $active = array_merge( $active, array_keys( $network_active ) ); } foreach ( $active as $plugin ) { if ( false !== strpos( (string) $plugin, 'seo-by-rank-math/' ) || false !== strpos( (string) $plugin, 'rank-math' ) ) { return true; } } return false; } /** * Multibyte-safe substring with a PHP core fallback. * * @param string $text Text. * @param int $start Start offset. * @param int $length Length. * @return string */ private static function safe_substr( $text, $start, $length ) { $text = (string) $text; return function_exists( 'mb_substr' ) ? mb_substr( $text, $start, $length ) : substr( $text, $start, $length ); } /** * Normalise text line endings. * * @param string $text Raw text. * @return string */ private static function normalise_line_endings( $text ) { return str_replace( array( "\r\n", "\r" ), "\n", (string) $text ); } /** * Status label. * * @param string $status Status key. * @return string */ private static function status_label( $status ) { $labels = array( 'pass' => __( 'Pass', 'ai-o-optimiser' ), 'warning' => __( 'Warnings', 'ai-o-optimiser' ), 'fail' => __( 'Fail', 'ai-o-optimiser' ), 'not_checked' => __( 'Not checked', 'ai-o-optimiser' ), ); return isset( $labels[ $status ] ) ? $labels[ $status ] : $status; } /** * Status badge CSS class. * * @param string $status Status key. * @return string */ private static function status_class( $status ) { if ( 'pass' === $status ) { return 'ai-o-optimiser-badge-pass'; } if ( 'warning' === $status ) { return 'ai-o-optimiser-badge-warning'; } if ( 'fail' === $status ) { return 'ai-o-optimiser-badge-fail'; } return 'ai-o-optimiser-badge-not-checked'; } /** * Render an allowed/blocked badge. * * @param bool $allowed Allowed flag. */ private static function render_allowed_badge( $allowed ) { $class = $allowed ? 'ai-o-optimiser-badge-pass' : 'ai-o-optimiser-badge-fail'; $text = $allowed ? __( 'Allowed', 'ai-o-optimiser' ) : __( 'Blocked', 'ai-o-optimiser' ); echo '' . esc_html( $text ) . ''; } /** * Render the System Check admin page. */ public static function render_admin_page() { if ( ! current_user_can( AI_O_Optimiser_Admin::CAPABILITY ) ) { return; } $report = self::report(); $health = self::health_report(); $settings = wp_parse_args( get_option( 'ai_o_optimiser_settings', array() ), AI_O_Optimiser_Settings::defaults() ); ?>



__( 'Start', 'ai-o-optimiser' ), 'status' => 'not_checked', 'area' => __( 'System check', 'ai-o-optimiser' ), 'check' => __( 'Run System Check to collect the first live robots.txt, DNS and HTTP/cache snapshot.', 'ai-o-optimiser' ), 'why' => __( 'Without a fresh snapshot, warning counts and crawler-access assumptions may be stale.', 'ai-o-optimiser' ), 'url' => admin_url( 'admin.php?page=ai-o-optimiser-system-check#ai-o-optimiser-system-check-run' ), 'label' => __( 'Run check', 'ai-o-optimiser' ), ); } if ( empty( $settings['enabled'] ) ) { $actions[] = array( 'priority' => __( 'High', 'ai-o-optimiser' ), 'status' => 'fail', 'area' => __( 'Tracking', 'ai-o-optimiser' ), 'check' => __( 'Enable AI crawler tracking in Settings if this site should collect live AI activity.', 'ai-o-optimiser' ), 'why' => __( 'The dashboard cannot build useful crawler history while tracking is disabled.', 'ai-o-optimiser' ), 'url' => admin_url( 'admin.php?page=ai-o-optimiser-settings#ai-o-optimiser-crawler-tracking-setting' ), 'label' => __( 'Open setting', 'ai-o-optimiser' ), ); } if ( empty( $settings['health_check_enabled'] ) ) { $actions[] = array( 'priority' => __( 'Medium', 'ai-o-optimiser' ), 'status' => 'warning', 'area' => __( 'Nightly monitor', 'ai-o-optimiser' ), 'check' => __( 'Enable the nightly health monitor once the baseline check is clean.', 'ai-o-optimiser' ), 'why' => __( 'It catches robots.txt, llms.txt and page-access changes before they quietly affect AI visibility.', 'ai-o-optimiser' ), 'url' => admin_url( 'admin.php?page=ai-o-optimiser-settings#ai-o-optimiser-nightly-health-monitor-setting' ), 'label' => __( 'Enable monitor', 'ai-o-optimiser' ), ); } if ( ! empty( $report['robots']['summary']['blocked'] ) ) { $actions[] = array( 'priority' => __( 'High', 'ai-o-optimiser' ), 'status' => 'warning', 'area' => __( 'robots.txt', 'ai-o-optimiser' ), 'check' => sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Review %d blocked AI crawler or policy tokens in the robots.txt table below.', 'ai-o-optimiser' ), (int) $report['robots']['summary']['blocked'] ), 'why' => __( 'Some blocks may be deliberate, but unintended AI crawler blocks reduce discovery and live fetch access.', 'ai-o-optimiser' ), 'url' => self::robots_resolution_url( $report ), 'label' => self::robots_resolution_label( $report ), ); } $robots_warnings = ! empty( $report['robots']['warnings'] ) && is_array( $report['robots']['warnings'] ) ? implode( ' ', $report['robots']['warnings'] ) : ''; if ( false !== stripos( $robots_warnings, '404' ) ) { $actions[] = array( 'priority' => __( 'Medium', 'ai-o-optimiser' ), 'status' => 'warning', 'area' => __( 'robots.txt', 'ai-o-optimiser' ), 'check' => __( 'Add a deliberate robots.txt file instead of relying on a 404/default behaviour.', 'ai-o-optimiser' ), 'why' => __( 'A clear robots.txt policy is easier to audit, explain to clients and monitor over time.', 'ai-o-optimiser' ), 'url' => self::robots_resolution_url( $report ), 'label' => self::robots_resolution_label( $report ), ); } $http_signals = ! empty( $report['http']['signals'] ) && is_array( $report['http']['signals'] ) ? implode( ' ', $report['http']['signals'] ) : ''; $http_warnings = ! empty( $report['http']['warnings'] ) && is_array( $report['http']['warnings'] ) ? implode( ' ', $report['http']['warnings'] ) : ''; if ( false !== stripos( $http_signals . ' ' . $http_warnings, 'cache' ) || false !== stripos( $http_signals . ' ' . $http_warnings, 'Cloudflare' ) ) { $actions[] = array( 'priority' => __( 'Medium', 'ai-o-optimiser' ), 'status' => 'warning', 'area' => __( 'CDN/cache', 'ai-o-optimiser' ), 'check' => __( 'Check whether HTML is being served from edge/page cache before WordPress runs.', 'ai-o-optimiser' ), 'why' => __( 'Requests served entirely from cache may not be logged by WordPress even when crawlers can access the page.', 'ai-o-optimiser' ), 'url' => admin_url( 'admin.php?page=ai-o-optimiser-system-check#ai-o-optimiser-http-cache' ), 'label' => __( 'Review cache', 'ai-o-optimiser' ), ); } if ( ! empty( $report['http']['http_code'] ) && (int) $report['http']['http_code'] >= 400 ) { $actions[] = array( 'priority' => __( 'High', 'ai-o-optimiser' ), 'status' => 'fail', 'area' => __( 'Homepage access', 'ai-o-optimiser' ), 'check' => sprintf( /* translators: Placeholder values are replaced with contextual values shown in this message. */ __( 'Investigate the HTTP %d response returned to the plugin check.', 'ai-o-optimiser' ), (int) $report['http']['http_code'] ), 'why' => __( 'AI crawler and citation readiness cannot be assessed reliably if public pages are inaccessible.', 'ai-o-optimiser' ), 'url' => admin_url( 'admin.php?page=ai-o-optimiser-system-check#ai-o-optimiser-http-cache' ), 'label' => __( 'Review response', 'ai-o-optimiser' ), ); } if ( ! empty( $health['items'] ) && is_array( $health['items'] ) ) { foreach ( $health['items'] as $item ) { if ( empty( $item['status'] ) || 'pass' === $item['status'] ) { continue; } $health_target = self::health_item_target( $item ); $actions[] = array( 'priority' => 'fail' === $item['status'] ? __( 'High', 'ai-o-optimiser' ) : __( 'Medium', 'ai-o-optimiser' ), 'status' => $item['status'], 'area' => isset( $item['label'] ) ? $item['label'] : __( 'Health monitor', 'ai-o-optimiser' ), 'check' => ! empty( $item['messages'] ) ? implode( ' ', array_slice( (array) $item['messages'], 0, 2 ) ) : __( 'Review the latest scheduled health warning.', 'ai-o-optimiser' ), 'why' => __( 'Scheduled health warnings mean the problem may have appeared after the last manual setup pass.', 'ai-o-optimiser' ), 'url' => $health_target['url'], 'label' => $health_target['label'], ); } } // De-duplicate identical area/check combinations and keep the panel short. $seen = array(); $out = array(); foreach ( $actions as $action ) { $key = md5( $action['area'] . '|' . $action['check'] ); if ( isset( $seen[ $key ] ) ) { continue; } $seen[ $key ] = true; $out[] = $action; } return array_slice( $out, 0, 8 ); } /** * Best available destination for resolving robots.txt findings. * * Rank Math is used when active because it owns the virtual robots.txt editor * on many AI-O Optimiser sites. A physical robots.txt file cannot be edited safely * from this plugin, so those sites stay on the detailed diagnostic instead. * * @param array $report System check report. * @return string */ private static function robots_resolution_url( $report ) { $has_static = file_exists( ABSPATH . 'robots.txt' ); $rank_math = ! empty( $report['environment']['rank_math_detected'] ); if ( $rank_math && ! $has_static ) { return admin_url( 'admin.php?page=rank-math-options-general&view=edit-robots' ); } if ( $has_static ) { return admin_url( 'admin.php?page=ai-o-optimiser-environment-check' ); } return admin_url( 'admin.php?page=ai-o-optimiser-system-check#ai-o-optimiser-robots-access' ); } /** * Label for the robots.txt resolution destination. * * @param array $report System check report. * @return string */ private static function robots_resolution_label( $report ) { $has_static = file_exists( ABSPATH . 'robots.txt' ); $rank_math = ! empty( $report['environment']['rank_math_detected'] ); if ( $rank_math && ! $has_static ) { return __( 'Edit robots.txt', 'ai-o-optimiser' ); } return $has_static ? __( 'Review static file', 'ai-o-optimiser' ) : __( 'Review robots', 'ai-o-optimiser' ); } /** * Map a nightly-health item to the nearest useful resolution/review screen. * * @param array $item Health item. * @return array{url:string,label:string} */ private static function health_item_target( $item ) { $label = strtolower( isset( $item['label'] ) ? (string) $item['label'] : '' ); if ( false !== strpos( $label, 'llms.txt' ) ) { return array( 'url' => admin_url( 'admin.php?page=ai-o-optimiser-guidance' ), 'label' => __( 'Open guidance', 'ai-o-optimiser' ), ); } if ( false !== strpos( $label, 'robots.txt' ) ) { return array( 'url' => admin_url( 'admin.php?page=ai-o-optimiser-system-check#ai-o-optimiser-robots-access' ), 'label' => __( 'Review robots', 'ai-o-optimiser' ), ); } if ( false !== strpos( $label, 'sitemap' ) ) { return array( 'url' => home_url( '/sitemap_index.xml' ), 'label' => __( 'Open sitemap', 'ai-o-optimiser' ), ); } if ( 0 === strpos( $label, 'public page:' ) ) { $url = trim( substr( isset( $item['label'] ) ? (string) $item['label'] : '', strlen( 'Public page:' ) ) ); return array( 'url' => wp_http_validate_url( $url ) ? $url : admin_url( 'admin.php?page=ai-o-optimiser-system-check#ai-o-optimiser-http-cache' ), 'label' => __( 'Open page', 'ai-o-optimiser' ), ); } return array( 'url' => admin_url( 'admin.php?page=ai-o-optimiser-system-check#ai-o-optimiser-system-check-run' ), 'label' => __( 'Open check', 'ai-o-optimiser' ), ); } /** * Render collected warnings and notes. * * @param array $report Report. */ private static function render_messages( $report ) { $warnings = array(); $notes = array(); foreach ( array( 'dns', 'http', 'robots' ) as $section ) { if ( ! empty( $report[ $section ]['warnings'] ) ) { $warnings = array_merge( $warnings, $report[ $section ]['warnings'] ); } if ( ! empty( $report[ $section ]['notes'] ) ) { $notes = array_merge( $notes, $report[ $section ]['notes'] ); } } if ( ! empty( $warnings ) ) { echo '

' . esc_html__( 'Review these items:', 'ai-o-optimiser' ) . '

'; } if ( ! empty( $notes ) ) { echo '

' . esc_html__( 'Diagnostic notes:', 'ai-o-optimiser' ) . '

'; } } /** * Render a table row. * * @param string $key Label. * @param mixed $value Value. */ private static function render_key_value_row( $key, $value ) { echo '' . esc_html( $key ) . '' . esc_html( (string) $value ) . ''; } /** * Format UTC time in site timezone. * * @param string $utc UTC time. * @return string */ private static function format_site_time( $utc ) { if ( empty( $utc ) ) { return ''; } try { return ( new DateTimeImmutable( $utc, new DateTimeZone( 'UTC' ) ) )->setTimezone( wp_timezone() )->format( 'Y-m-d H:i:s T' ); } catch ( Exception $e ) { return $utc; } } /** * Handle admin-post request for running the system check. */ public function handle_run_check() { if ( ! current_user_can( AI_O_Optimiser_Admin::CAPABILITY ) ) { wp_die( esc_html__( 'You do not have permission to do this.', 'ai-o-optimiser' ) ); } check_admin_referer( 'ai_o_optimiser_run_system_check' ); self::run_check(); wp_safe_redirect( add_query_arg( 'checked', '1', admin_url( 'admin.php?page=ai-o-optimiser-system-check' ) ) ); exit; } }