'llms_txt',
'name' => __('llms.txt File', 'ai-ready-core'),
'status' => 'fail',
'message' => '',
'recommendation' => '',
'priority' => 'high'
];
$llms_url = home_url('/llms.txt');
// Test the virtual endpoint
$response = wp_remote_get($llms_url, [
'timeout' => 10,
'user-agent' => 'AI-Ready-Diagnostics/1.0'
]);
if (is_wp_error($response)) {
$check['status'] = 'fail';
/* translators: %s: error message */
$check['message'] = sprintf(__('llms.txt endpoint is not accessible: %s', 'ai-ready-core'), $response->get_error_message());
$check['recommendation'] = __('Ensure the AI-Ready plugin is active and rewrite rules are flushed.', 'ai-ready-core');
return $check;
}
$status_code = wp_remote_retrieve_response_code($response);
$content = wp_remote_retrieve_body($response);
$content_type = wp_remote_retrieve_header($response, 'content-type');
if ($status_code !== 200) {
$check['status'] = 'fail';
/* translators: %d: HTTP status code */
$check['message'] = sprintf(__('llms.txt returns HTTP %d instead of 200', 'ai-ready-core'), $status_code);
$check['recommendation'] = __('Check your WordPress rewrite rules and plugin configuration.', 'ai-ready-core');
return $check;
}
if (strpos($content_type, 'text/plain') === false) {
$check['status'] = 'warning';
/* translators: %s: content type header value */
$check['message'] = sprintf(__('llms.txt should return Content-Type: text/plain (currently: %s)', 'ai-ready-core'), $content_type);
$check['recommendation'] = __('Ensure proper content-type headers are set in the virtual endpoint.', 'ai-ready-core');
}
if (empty($content) || strlen($content) < 50) {
$check['status'] = 'warning';
$check['message'] = __('llms.txt content appears to be empty or too short', 'ai-ready-core');
$check['recommendation'] = __('Add meaningful content to your llms.txt file through the AI Ready settings page.', 'ai-ready-core');
return $check;
}
// Validate content format
if (function_exists('ai_ready_validate_llms_txt')) {
$validation = ai_ready_validate_llms_txt($content);
if (!$validation['valid']) {
$check['status'] = 'warning';
/* translators: %s: comma-separated list of validation errors */
$check['message'] = sprintf(__('llms.txt format issues: %s', 'ai-ready-core'), implode(', ', $validation['errors']));
$check['recommendation'] = __('Fix the format issues or use the "Reset to Defaults" option in AI Ready settings.', 'ai-ready-core');
return $check;
}
}
$check['status'] = 'pass';
/* translators: %d: length of llms.txt content in characters */
$check['message'] = sprintf(__('llms.txt is accessible and properly formatted (%d characters)', 'ai-ready-core'), strlen($content));
$check['recommendation'] = __('Great! Your llms.txt file is working correctly.', 'ai-ready-core');
return $check;
}
/**
* Check robots.txt for AI bot permissions and Agent Gate configuration
*
* @return array Diagnostic result
*/
function ai_ready_check_robots_txt() {
$check = [
'id' => 'robots_txt',
'name' => __('AI Bot Access (robots.txt)', 'ai-ready-core'),
'status' => 'unknown',
'message' => '',
'recommendation' => '',
'priority' => 'high'
];
// Use WordPress's robots.txt filter to get the actual content that would be served
// This ensures we're checking the filtered content that includes Agent Gate rules
$is_public = get_option('blog_public');
$robots_content = apply_filters('robots_txt', '', $is_public);
// If no content from filter, fall back to HTTP request
if (empty($robots_content)) {
$robots_url = home_url('/robots.txt');
$response = wp_remote_get($robots_url, ['timeout' => 10]);
if (is_wp_error($response) || wp_remote_retrieve_response_code($response) !== 200) {
$check['status'] = 'warning';
$check['message'] = __('robots.txt not found or not accessible', 'ai-ready-core');
$check['recommendation'] = __('Consider creating a robots.txt file to provide clear guidance to AI crawlers. You can configure AI bot access using the Agent Gate tab.', 'ai-ready-core');
return $check;
}
$robots_content = wp_remote_retrieve_body($response);
}
// Check for global block first (highest priority)
if (preg_match('/User-agent:\s*\*\s*\n\s*Disallow:\s*\//i', $robots_content)) {
$check['status'] = 'info';
$check['message'] = __('🔒 GLOBAL BLOCK: robots.txt blocks ALL bots (User-agent: * Disallow: /)', 'ai-ready-core');
$check['recommendation'] = __('Your site blocks all crawlers including AI bots. This is a valid choice for privacy, but will prevent AI discovery. Use the Agent Gate tab to adjust AI bot access.', 'ai-ready-core');
return $check;
}
// Use Agent Gate module for diagnostics if available
global $ai_ready_agent_gate;
if (isset($ai_ready_agent_gate) && method_exists($ai_ready_agent_gate, 'get_diagnostics')) {
$agent_gate_check = $ai_ready_agent_gate->get_diagnostics();
// Merge Agent Gate diagnostics with our basic check structure
$check['status'] = $agent_gate_check['status'];
$check['message'] = $agent_gate_check['message'];
$check['recommendation'] = $agent_gate_check['recommendation'];
// Add link to Agent Gate tab in recommendation if not already present
if (strpos($check['recommendation'], 'agent-gate') === false) {
/* translators: %s: URL to Agent Gate settings tab */
$check['recommendation'] .= sprintf(__(' Configure AI bot access in the Agent Gate tab →', 'ai-ready-core'), '?page=ai-ready-settings&tab=agent-gate');
}
} else {
// Fallback: basic Agent Gate check using direct option access
$gate_mode = get_option('ai_ready_agent_gate_mode', 'allow_all');
$custom_bots = get_option('ai_ready_agent_gate_custom', []);
switch ($gate_mode) {
case 'allow_all':
$check['status'] = 'pass';
$check['message'] = __('🟢 Agent Gate: All AI bots are allowed', 'ai-ready-core');
$check['recommendation'] = __('Excellent! All AI agents can access your content for maximum discoverability.', 'ai-ready-core');
break;
case 'block_all':
$check['status'] = 'info';
$check['message'] = __('🔒 Agent Gate: All AI bots are blocked', 'ai-ready-core');
$check['recommendation'] = __('You are blocking all AI bots as configured. This protects privacy but prevents AI discovery.', 'ai-ready-core');
break;
case 'custom':
$allowed_count = count($custom_bots);
$check['status'] = 'pass';
/* translators: %d: number of allowed bots */
$check['message'] = sprintf(__('⚙️ Agent Gate: Custom configuration - %d bots allowed', 'ai-ready-core'), $allowed_count);
$check['recommendation'] = __('You have selectively configured AI bot access.', 'ai-ready-core');
break;
default:
$check['status'] = 'warning';
$check['message'] = __('⚠️ Agent Gate: Unknown configuration', 'ai-ready-core');
$check['recommendation'] = __('Agent Gate configuration appears to be invalid. Please check your settings.', 'ai-ready-core');
break;
}
// Add link to Agent Gate tab
/* translators: %s: URL to Agent Gate settings tab */
$check['recommendation'] .= sprintf(__(' Configure AI bot access in the Agent Gate tab →', 'ai-ready-core'), '?page=ai-ready-settings&tab=agent-gate');
}
return $check;
}
/**
* Check for noai meta tags that block AI crawling
*
* @return array Diagnostic result
*/
function ai_ready_check_noai_meta() {
$check = [
'id' => 'noai_meta',
'name' => __('No-AI Meta Tags', 'ai-ready-core'),
'status' => 'unknown',
'message' => '',
'recommendation' => '',
'priority' => 'medium'
];
// Check homepage for noai meta tags
$homepage_response = wp_remote_get(home_url(), ['timeout' => 10]);
if (is_wp_error($homepage_response)) {
$check['status'] = 'unknown';
$check['message'] = __('Could not fetch homepage to check for noai meta tags', 'ai-ready-core');
$check['recommendation'] = __('Manually check your site for meta name="robots" content="noai" tags.', 'ai-ready-core');
return $check;
}
$homepage_content = wp_remote_retrieve_body($homepage_response);
// Check for various noai patterns
$noai_patterns = [
'/meta\s+name=["\']robots["\']\s+content=["\'][^"\']*noai/i',
'/meta\s+name=["\']googlebot["\']\s+content=["\'][^"\']*noai/i',
'/meta\s+name=["\']ai["\']\s+content=["\'][^"\']*noindex/i'
];
$found_noai = false;
foreach ($noai_patterns as $pattern) {
if (preg_match($pattern, $homepage_content)) {
$found_noai = true;
break;
}
}
if ($found_noai) {
$check['status'] = 'warning';
$check['message'] = __('Homepage contains noai meta tags that may block AI crawlers', 'ai-ready-core');
$check['recommendation'] = __('Remove noai meta tags if you want your content to be discoverable by AI systems.', 'ai-ready-core');
return $check;
}
$check['status'] = 'pass';
$check['message'] = __('No blocking noai meta tags found on homepage', 'ai-ready-core');
$check['recommendation'] = __('Good! Your pages are not explicitly blocking AI crawlers.', 'ai-ready-core');
return $check;
}
/**
* Check for canonical URLs
*
* @return array Diagnostic result
*/
function ai_ready_check_canonical_urls() {
$check = [
'id' => 'canonical_urls',
'name' => __('Canonical URLs', 'ai-ready-core'),
'status' => 'unknown',
'message' => '',
'recommendation' => '',
'priority' => 'low'
];
// Check homepage for canonical link
$homepage_response = wp_remote_get(home_url(), ['timeout' => 10]);
if (is_wp_error($homepage_response)) {
$check['status'] = 'unknown';
$check['message'] = __('Could not fetch homepage to check for canonical URLs', 'ai-ready-core');
$check['recommendation'] = __('Manually check your site for tags.', 'ai-ready-core');
return $check;
}
$homepage_content = wp_remote_retrieve_body($homepage_response);
// Check for canonical link tag
if (preg_match('/ 'structured_data',
'name' => __('Structured Data (JSON-LD)', 'ai-ready-core'),
'status' => 'unknown',
'message' => '',
'recommendation' => '',
'priority' => 'medium'
];
// Check homepage for JSON-LD structured data
$homepage_response = wp_remote_get(home_url(), ['timeout' => 10]);
if (is_wp_error($homepage_response)) {
$check['status'] = 'unknown';
$check['message'] = __('Could not fetch homepage to check for structured data', 'ai-ready-core');
$check['recommendation'] = __('Manually check your site for JSON-LD structured data.', 'ai-ready-core');
return $check;
}
$homepage_content = wp_remote_retrieve_body($homepage_response);
// Count JSON-LD blocks
$jsonld_count = preg_match_all('/