detector = $detector; } /** * Register hooks. */ public function hooks() { add_filter( 'robots_txt', array( $this, 'filter_robots_txt' ), 20, 2 ); } /** * Append the AI-crawler rules to robots.txt output. * * @param string $output The robots.txt content so far. * @param bool $public Whether the site is set to be indexed (blog_public). * @return string */ public function filter_robots_txt( $output, $public ) { // Pro-gated, and off by default. if ( ! ADG_AIM_Plugin::is_pro() || ! (bool) ADG_AIM_Plugin::get_setting( 'robots_manage', false ) ) { return $output; } // Non-public sites: core already emits a global Disallow; do not fight it. if ( ! $public ) { return $output; } $block = $this->build_block(); if ( '' === $block ) { return $output; } $output = rtrim( (string) $output, "\n" ); return $output . "\n\n" . $block . "\n"; } /** * Build the AI-crawler rules block (without leading/trailing blank lines). * * @return string */ public function build_block() { $lines = array( '# AI crawler policy — managed by ADG Markdown for AI Crawlers' ); $seen = array(); $rules = 0; foreach ( $this->detector->get_bots() as $bot ) { $token = ADG_AIM_Bot_Detector::robots_token( $bot ); if ( '' === $token ) { continue; } $key = strtolower( $token ); if ( isset( $seen[ $key ] ) ) { continue; } $seen[ $key ] = true; $category = ADG_AIM_Bot_Detector::bot_category( $bot ); $allowed = ! empty( $bot['enabled'] ) && ADG_AIM_Plugin::category_allowed( $category ); $lines[] = ''; $lines[] = 'User-agent: ' . $token; $lines[] = $allowed ? 'Allow: /' : 'Disallow: /'; ++$rules; } if ( 0 === $rules ) { return ''; } if ( (bool) ADG_AIM_Plugin::get_setting( 'llms_enabled', true ) ) { $lines[] = ''; $lines[] = '# Structured index for AI: ' . home_url( '/llms.txt' ); } return implode( "\n", $lines ); } /** * Whether a physical robots.txt exists at the web root (bypasses these rules). * * @return bool */ public static function physical_robots_exists() { $path = untrailingslashit( ABSPATH ) . '/robots.txt'; return @file_exists( $path ); // phpcs:ignore WordPress.PHP.NoSilencedErrors, WordPress.WP.AlternativeFunctions } }