*/ private const BOT_FAMILIES = array( // OpenAI. 'GPTBot' => 'openai', 'ChatGPT-User' => 'openai', 'OAI-SearchBot' => 'openai', // Anthropic. 'ClaudeBot' => 'anthropic', 'Claude-Web' => 'anthropic', 'anthropic-ai' => 'anthropic', 'Claude-User' => 'anthropic', 'Claude-SearchBot' => 'anthropic', // Google. 'Google-Extended' => 'google', 'GoogleOther' => 'google', 'Google-CloudVertexBot' => 'google', // Microsoft / Bing. 'Bingbot' => 'microsoft', // Perplexity. 'PerplexityBot' => 'perplexity', 'Perplexity-User' => 'perplexity', // Apple. 'Applebot-Extended' => 'apple', // Amazon. 'Amazonbot' => 'amazon', // Meta. 'Meta-ExternalAgent' => 'meta', 'Meta-ExternalFetcher' => 'meta', 'FacebookBot' => 'meta', // ByteDance / TikTok. 'Bytespider' => 'bytedance', // Common Crawl (feeds many LLM training sets). 'CCBot' => 'commoncrawl', // DuckDuckGo AI. 'DuckAssistBot' => 'duckduckgo', // Cohere. 'cohere-ai' => 'cohere', 'cohere-training-data-crawler' => 'cohere', // Others. 'YouBot' => 'you', 'Diffbot' => 'diffbot', 'PetalBot' => 'petal', 'Timpibot' => 'timpi', 'ImagesiftBot' => 'imagesift', 'AI2Bot' => 'ai2', ); /** * Paths that should never be exposed to crawlers, even when allowed. * * @var string[] */ private const COMMERCE_DISALLOW = array( '/wp-admin/', '/cart/', '/checkout/', '/my-account/', '/*add-to-cart=*', ); /** * Filter the robots.txt output to inject AI bot rules. * * Hooked to the 'robots_txt' WordPress filter. * * @param string $output The existing robots.txt content. * @param bool $is_public Whether the site is public. * * @return string Modified robots.txt content. */ public function filter_robots_txt( string $output, bool $is_public ): string { if ( ! $is_public ) { return $output; } $cached = get_transient( self::CACHE_KEY ); if ( false !== $cached ) { return $output . "\n" . $cached; } $rules_text = $this->build_rules(); set_transient( self::CACHE_KEY, $rules_text, DAY_IN_SECONDS ); return $output . "\n" . $rules_text; } /** * Build the AI bot robots.txt rules string. * * @return string The formatted rules block. */ public function build_rules(): string { $rules = $this->get_rules(); $blocked = get_option( 'aicommerce_blocked_bots', array() ); $sections = array(); $sections[] = '# Tyralix AI Discoverability — AI Bot Rules'; $sections[] = '# Generated: ' . gmdate( 'c' ); foreach ( $rules as $bot_name => $directives ) { $sections[] = ''; $sections[] = 'User-agent: ' . $bot_name; // Check if this bot's family is blocked. $family = $this->get_bot_family( $bot_name ); $is_blocked = is_array( $blocked ) && in_array( $family, $blocked, true ); if ( $is_blocked ) { $sections[] = 'Disallow: /'; } else { foreach ( $directives['disallow'] as $path ) { $sections[] = 'Disallow: ' . $path; } foreach ( $directives['allow'] as $path ) { $sections[] = 'Allow: ' . $path; } } } // Add sitemap reference. $sitemap_url = $this->get_sitemap_url(); if ( ! empty( $sitemap_url ) ) { $sections[] = ''; $sections[] = 'Sitemap: ' . $sitemap_url; } return implode( "\n", $sections ) . "\n"; } /** * Build the default rule set for every known AI crawler. * * @return array */ public function get_default_rules(): array { $defaults = array(); foreach ( array_keys( self::BOT_FAMILIES ) as $bot_name ) { $defaults[ $bot_name ] = array( 'allow' => array( '/' ), 'disallow' => self::COMMERCE_DISALLOW, ); } /** * Filters the default AI-crawler robots.txt rule set. * * @param array $defaults Default rules keyed by user-agent. */ return apply_filters( 'aicommerce_default_robots_rules', $defaults ); } /** * Get the current rules, merging defaults with user customizations. * * @return array */ public function get_rules(): array { $defaults = $this->get_default_rules(); $custom_rules = get_option( 'aicommerce_robots_rules', array() ); if ( ! empty( $custom_rules ) && is_array( $custom_rules ) ) { return array_merge( $defaults, $custom_rules ); } return $defaults; } /** * Get the list of known AI-crawler families. * * @return array Unique family slugs. */ public function get_bot_families(): array { return array_values( array_unique( array_values( self::BOT_FAMILIES ) ) ); } /** * Get the full user-agent → family map. * * @return array */ public function get_bot_map(): array { return self::BOT_FAMILIES; } /** * Update rules for a specific bot. * * @param string $bot_name The bot user-agent name. * @param array $allow Array of allowed paths. * @param array $disallow Array of disallowed paths. * * @return void */ public function update_rule( string $bot_name, array $allow, array $disallow ): void { $custom_rules = get_option( 'aicommerce_robots_rules', array() ); $custom_rules[ $bot_name ] = array( 'allow' => array_map( 'sanitize_text_field', $allow ), 'disallow' => array_map( 'sanitize_text_field', $disallow ), ); update_option( 'aicommerce_robots_rules', $custom_rules, false ); $this->clear_cache(); } /** * Remove a custom rule, reverting to default. * * @param string $bot_name The bot user-agent name. * * @return void */ public function remove_rule( string $bot_name ): void { $custom_rules = get_option( 'aicommerce_robots_rules', array() ); unset( $custom_rules[ $bot_name ] ); update_option( 'aicommerce_robots_rules', $custom_rules, false ); $this->clear_cache(); } /** * Clear the cached robots rules. * * @return void */ public function clear_cache(): void { delete_transient( self::CACHE_KEY ); } /** * Get the sitemap URL. * * @return string The sitemap URL. */ private function get_sitemap_url(): string { return home_url( '/wp-sitemap.xml' ); } /** * Map a bot user-agent name to its family. * * @param string $bot_name The bot name. * * @return string The bot family. */ private function get_bot_family( string $bot_name ): string { return self::BOT_FAMILIES[ $bot_name ] ?? 'unknown'; } }