$output, 'sitemaps' => [ SiteData::file_urls()['sitemap'] ], ]; return rtrim( $output ) . "\n\n" . ( new RobotsGenerator() )->generate( $policy ); } /** * Audit checks contributed by this module. * * @return array Checks. */ public function checks(): array { $slug = $this->slug(); $url = home_url( '/robots.txt' ); return [ Checks::reachable( 'robots_reachable', $slug, __( 'robots.txt is reachable', 'acid-aeo' ), $url, [ 'weight' => Check::WEIGHT_CRITICAL, 'fix' => 'flush_rewrite', ] ), Checks::observed( 'robots_has_ai_policy', $slug, __( 'AI crawler rules in robots.txt', 'acid-aeo' ), $url, static fn( string $body ): bool => false !== strpos( $body, 'User-agent: GPTBot' ), [ 'ok' => __( 'robots.txt names the AI crawlers your policy decides about, one group each.', 'acid-aeo' ), 'bad' => __( 'robots.txt is served but carries no AI crawler rules, so every assistant falls back to its own defaults.', 'acid-aeo' ), 'weight' => Check::WEIGHT_CRITICAL, 'fix' => 'flush_rewrite', ] ), Checks::observed( 'robots_content_signal', $slug, __( 'Content-Signal in robots.txt', 'acid-aeo' ), $url, static fn( string $body ): bool => false !== strpos( $body, 'Content-Signal:' ), [ 'ok' => __( 'robots.txt states what your content may be used for.', 'acid-aeo' ), 'bad' => __( 'robots.txt carries no Content-Signal line, so a crawler learns which bots may read the site but not what it may do with what it reads.', 'acid-aeo' ), 'status' => Check::WARN, 'weight' => Check::WEIGHT_IMPORTANT, 'fix' => 'flush_rewrite', ] ), $this->physical_file(), ]; } /** * Whether a robots.txt on disk is shadowing the generated one. * * The web server answers a request for a file that exists before WordPress * ever sees it, so a robots.txt in the site root silently replaces the * whole policy. * * @return Check The finding. */ private function physical_file(): Check { $files = Environment::detect()['physical_files'] ?? []; $label = __( 'No robots.txt on disk', 'acid-aeo' ); if ( empty( $files['robots.txt'] ) ) { return Check::pass( 'robots_physical_file', $this->slug(), $label, __( 'Nothing on disk shadows the robots.txt WordPress serves.', 'acid-aeo' ), Check::WEIGHT_CRITICAL ); } return Check::fail( 'robots_physical_file', $this->slug(), $label, __( 'A robots.txt file sits in the site root. The web server answers with that file, so nothing this plugin generates is ever read. Delete or rename it.', 'acid-aeo' ), Check::WEIGHT_CRITICAL ); } }