loopback = $loopback; $this->robots = $robots; } /** * Returns the check identifier. * * @return string Identifier. */ public function get_id(): string { return 'served_robots'; } /** * Whether a loopback request is required. * * @return bool True. */ public function needs_loopback(): bool { return true; } /** * Runs the check. * * @return Result The finding. */ public function run(): Result { $label = __( 'What crawlers actually receive at /robots.txt', 'ai-blackout' ); $response = $this->loopback->get( '/robots.txt' ); if ( is_wp_error( $response ) ) { return new Result( $this->get_id(), $label, Result::UNKNOWN, __( 'Your robots.txt could not be fetched, so we cannot show what crawlers receive.', 'ai-blackout' ), __( 'Some hosts stop a site from requesting its own pages. This does not mean your robots.txt is wrong, only that we could not read it back.', 'ai-blackout' ), __( 'Open /robots.txt in your browser to see it yourself.', 'ai-blackout' ), array( 'error' => $response->get_error_message() ) ); } $body = $response['body']; $has_block = false !== strpos( $body, RobotsTxt::BEGIN_MARKER ); $expected = $this->robots->build_block(); $want_block = '' !== $expected; $detail = array( 'url' => $response['url'], 'status_code' => $response['code'], 'bytes' => strlen( $body ), 'served' => $body, 'has_block' => $has_block, ); if ( 200 !== $response['code'] ) { return new Result( $this->get_id(), $label, Result::BROKEN, sprintf( /* translators: %d: HTTP status code. */ __( 'Requesting /robots.txt returned %d instead of 200.', 'ai-blackout' ), $response['code'] ), __( 'Crawlers cannot read your robots.txt, so none of your rules are reaching them, whatever the settings screen shows.', 'ai-blackout' ), __( 'Check whether a redirect, security rule or server setting is blocking access to /robots.txt.', 'ai-blackout' ), $detail ); } if ( $want_block && ! $has_block ) { return new Result( $this->get_id(), $label, Result::BROKEN, __( 'Your robots.txt is being served, but your AI crawler rules are not in it.', 'ai-blackout' ), __( 'This is what crawlers actually receive, and your rules are missing from it. Something between WordPress and the visitor is replacing the file. Usually that is a robots.txt on disk, a CDN, or a server rule.', 'ai-blackout' ), __( 'Check whether a robots.txt file exists on disk, and whether a CDN or caching service is serving an older copy.', 'ai-blackout' ), $detail ); } if ( ! $want_block ) { return new Result( $this->get_id(), $label, Result::INFO, __( 'This is exactly what crawlers receive at /robots.txt.', 'ai-blackout' ), __( 'No AI crawler rules are set, so none appear in the file.', 'ai-blackout' ), __( 'Choose one of the blocking answers at the top of this screen.', 'ai-blackout' ), $detail ); } return new Result( $this->get_id(), $label, Result::DECLARED, __( 'This is exactly what crawlers receive, and your AI rules are in it.', 'ai-blackout' ), __( 'Confirmed by fetching the file the same way a crawler would. Remember this is a request that crawlers can ignore, not a barrier that stops them.', 'ai-blackout' ), '', $detail ); } }