settings = Plugin::get_settings( 'security' ); add_filter( 'adminease_settings_fields', [ $this, 'adminease_settings_fields' ], 21 ); add_action( 'adminease_settings_saved', [ $this, 'adminease_settings_saved' ] ); if( !empty( $this->settings['crawl_delay_enabled'] ) ) { add_filter( 'robots_txt', [ $this, 'robots_txt' ], 20, 2 ); if( !empty( $this->settings['crawl_delay_enforce'] ) ) { add_action( 'init', [ $this, 'throttle_bots' ], 1 ); } } } /** * Adds the crawl delay configuration options to the settings fields array. * * @param array $fields The current fields array to be modified with additional settings. * * @return array The modified fields array including the crawl delay options. */ public function adminease_settings_fields( array $fields ): array { $fields['security']['fields'][] = [ 'type' => 'switch', 'id' => 'crawl-delay-enabled', 'name' => 'adminease[security][crawl_delay_enabled]', 'value' => $this->settings['crawl_delay_enabled'] ?? '', 'label_class' => 'pwp-switch', 'input_class' => 'pwp-field-input toggle-field', 'label' => __( 'Enable crawl delay', 'adminease' ), 'description' => __( 'Slow aggressive crawlers down instead of banning them. Blocking a search engine costs you the indexing; a crawl delay keeps you listed while capping how fast bots can hit your server. The delay is advertised in robots.txt for well-behaved crawlers, and can additionally be enforced in PHP for the ones that ignore it.', 'adminease' ), 'field_description' => $this->get_enable_field_description(), 'child_fields' => [ [ 'type' => 'select', 'id' => 'crawl-delay-bots', 'name' => 'adminease[security][crawl_delay_bots][]', 'value' => $this->settings['crawl_delay_bots'] ?? [], 'options' => Utils::get_bots_list(), 'has_optgroups' => true, 'label_class' => 'pwp-field-label', 'input_class' => 'pwp-field-input adminease-choices', 'wrapper_class' => 'form-group-child', 'label' => __( 'Bots to slow down', 'adminease' ), 'field_description' => __( 'Select the bots that should be given a crawl delay. Use the Network Viewer Log to find out which bots are actually hitting your server.', 'adminease' ), 'attributes' => [ 'data-parent' => 'crawl-delay-enabled', 'multiple' => 'multiple', 'data-allow_clear' => true, 'data-allow_select_all' => true, ], ], [ 'type' => 'number', 'id' => 'crawl-delay-seconds', 'name' => 'adminease[security][crawl_delay_seconds]', 'value' => $this->settings['crawl_delay_seconds'] ?? 10, 'label_class' => 'pwp-field-label', 'input_class' => 'pwp-field-input', 'wrapper_class' => 'form-group-child', 'label' => __( 'Delay in seconds', 'adminease' ), 'description' => __( 'Minimum number of seconds a selected bot must wait between requests.', 'adminease' ), /* translators: 1: minimum delay in seconds, 2: maximum delay in seconds. */ 'field_description' => sprintf( __( 'Applied to every bot selected above unless it has its own value below. Allowed range: %1$d to %2$d seconds.', 'adminease' ), self::MIN_DELAY, self::MAX_DELAY ), 'attributes' => [ 'data-parent' => 'crawl-delay-enabled', 'min' => self::MIN_DELAY, 'max' => self::MAX_DELAY, ], ], [ 'type' => 'textarea', 'id' => 'crawl-delay-overrides', 'name' => 'adminease[security][crawl_delay_overrides]', 'value' => $this->settings['crawl_delay_overrides'] ?? '', 'label_class' => 'pwp-field-label', 'input_class' => 'pwp-field-input', 'wrapper_class' => 'form-group-child', 'label' => __( 'Per-bot delays', 'adminease' ), 'field_description' => __( 'Give individual bots their own delay, one per line, as BotName:seconds. These override the shared delay above, and a bot listed here is throttled even if it is not selected in the list above.
Example:
Bingbot:10
GPTBot:30
AhrefsBot:60', 'adminease' ), 'attributes' => [ 'data-parent' => 'crawl-delay-enabled', ], ], [ 'type' => 'number', 'id' => 'crawl-delay-default', 'name' => 'adminease[security][crawl_delay_default]', 'value' => $this->settings['crawl_delay_default'] ?? 0, 'label_class' => 'pwp-field-label', 'input_class' => 'pwp-field-input', 'wrapper_class' => 'form-group-child', 'label' => __( 'Default delay for all other bots', 'adminease' ), 'description' => __( 'Adds a catch-all "User-agent: *" crawl delay to robots.txt.', 'adminease' ), 'field_description' => __( 'Published in robots.txt only and never enforced in PHP, because enforcing a delay against every user agent would throttle real visitors too. Set to 0 to leave the line out.', 'adminease' ), 'attributes' => [ 'data-parent' => 'crawl-delay-enabled', 'min' => 0, 'max' => self::MAX_DELAY, ], ], [ 'type' => 'switch', 'id' => 'crawl-delay-enforce', 'name' => 'adminease[security][crawl_delay_enforce]', 'value' => $this->settings['crawl_delay_enforce'] ?? '', 'label_class' => 'pwp-switch', 'input_class' => 'pwp-field-input toggle-field', 'wrapper_class' => 'form-group-child', 'label' => __( 'Enforce the delay', 'adminease' ), 'field_description' => __( 'Answer bots that come back too soon with 429 Too Many Requests and a Retry-After header, instead of only asking them politely in robots.txt. Leave this off if you want advertising only.', 'adminease' ), 'attributes' => [ 'data-parent' => 'crawl-delay-enabled', ], ], ], ]; return $fields; } /** * Build the description shown under the parent toggle. * Reports how the delays will actually reach crawlers on this particular site, * since that depends on whether a physical robots.txt shadows the virtual one. * * @return string */ private function get_enable_field_description(): string { $description = __( 'Bingbot and YandexBot honour the Crawl-delay directive. Googlebot ignores it by design, and so do most scrapers, which is what the enforcement option below is for.', 'adminease' ); $robots_txt = FileHandler::get_instance()->get_robots_txt_status(); if( empty( $robots_txt['exists'] ) ) { return $description; } if( !empty( $robots_txt['writable'] ) ) { $description .= '
' . __( 'Your site has a physical robots.txt file, which replaces the one WordPress generates. AdminEase writes the delays straight into it and keeps them up to date, leaving the rest of the file untouched.', 'adminease' ); return $description; } $description .= '
' . sprintf( /* translators: %s: absolute path to the robots.txt file. */ __( 'Your robots.txt file is not writable, so the delays cannot be added to it: %s. Make the file writable, or copy the lines in by hand.', 'adminease' ), esc_html( $robots_txt['path'] ) ) . ''; return $description; } /** * Write the Crawl-delay directives into a physical robots.txt, if the site has one. * * Only relevant when a real robots.txt exists: that file shadows the virtual one * entirely, so the `robots_txt` filter never runs and the delays would otherwise * never be served. FileHandler no-ops when there is no physical file, and removes * the block when the rules come back empty. * * @param array $settings The sanitized settings being saved. * * @return void */ public function adminease_settings_saved( array $settings ): void { if( !current_user_can( 'manage_options' ) ) { return; } $security = $settings['security'] ?? []; $rules = ''; if( !empty( $security['crawl_delay_enabled'] ) ) { $rules = $this->build_crawl_delay_rules( $security ); } FileHandler::get_instance()->set_robots_txt_block( 'CRAWL_DELAY', $rules ); } /** * Append the configured Crawl-delay groups to the virtual robots.txt. * * Core's do_robots() already prints a "User-agent: *" group. Appending a second * one is safe: RFC 9309 requires parsers to merge records that share a user agent, * and both Google's and Bing's parsers do. Rewriting core's group in place would * be fragile against SEO plugins that filter `robots_txt` as well. * * @param string $output The robots.txt output built so far. * @param bool $is_public Whether the site is asking to be indexed. Unused: a * crawl delay is harmless on a non-indexed site, where * core already prints "Disallow: /". * * @return string */ public function robots_txt( string $output, bool $is_public ): string { return $output . $this->build_crawl_delay_rules(); } /** * Build the Crawl-delay directives. * * Single source of truth for both delivery paths, so what the virtual robots.txt * advertises and what gets written into a physical robots.txt can never drift. * Returns the bare directives with no marker comments around them. * * @param array|null $settings Security settings to read, or null for the stored ones. * * @return string Directives, or an empty string when there is nothing to publish. */ private function build_crawl_delay_rules( ?array $settings = null ): string { $settings = $settings ?? $this->settings; $delay_map = $this->get_delay_map( $settings ); $default = (int) ( $settings['crawl_delay_default'] ?? 0 ); if( $default > self::MAX_DELAY ) { $default = self::MAX_DELAY; } if( empty( $delay_map ) && $default < 1 ) { return ''; } $rules = ''; foreach( $delay_map as $bot => $delay ) { $rules .= "\nUser-agent: {$bot}\nCrawl-delay: {$delay}\n"; } if( $default > 0 ) { $rules .= "\nUser-agent: *\nCrawl-delay: {$default}\n"; } return $rules; } /** * Enforce the configured delay for bots that ignore robots.txt. * * Runs on 'init' at priority 1, matching BlockSpecificBots::php_block_bots(). An * earlier hook is not usable: pluggable.php has not loaded during 'plugins_loaded', * so is_user_logged_in() would not exist yet. * * @return void */ public function throttle_bots(): void { if( is_admin() || wp_doing_ajax() || wp_doing_cron() ) { return; } if( defined( 'WP_CLI' ) && WP_CLI ) { return; } if( is_user_logged_in() ) { return; } // phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Sanitized below $user_agent = isset( $_SERVER['HTTP_USER_AGENT'] ) ? sanitize_text_field( wp_unslash( $_SERVER['HTTP_USER_AGENT'] ) ) : ''; if( empty( $user_agent ) ) { return; } $delay_map = $this->get_delay_map(); if( empty( $delay_map ) ) { return; } foreach( $delay_map as $bot => $delay ) { if( false === stripos( $user_agent, $bot ) ) { continue; } $transient_key = self::TRANSIENT_PREFIX . md5( $bot ); if( false !== get_transient( $transient_key ) ) { $this->send_too_many_requests( $delay ); } set_transient( $transient_key, 1, $delay ); return; } } /** * Send a 429 response with a Retry-After header and stop the request. * Retry-After is what makes this a crawl delay rather than a random rejection: * Bing and Google both back off politely when it is present. * * @param int $delay Number of seconds the bot should wait. * * @return void */ private function send_too_many_requests( int $delay ): void { nocache_headers(); header( 'Retry-After: ' . $delay ); header( 'Content-Type: text/plain; charset=utf-8' ); status_header( 429 ); echo 'Too Many Requests'; exit; } /** * Build the map of throttled bots. * Keys are lower-cased bot tokens, values are the delay in seconds. * * @param array|null $settings Security settings to read, or null for the stored ones. * * @return array */ private function get_delay_map( ?array $settings = null ): array { $settings = $settings ?? $this->settings; $delay_map = []; $bots = $settings['crawl_delay_bots'] ?? []; $overrides = $settings['crawl_delay_overrides'] ?? ''; $seconds = $this->clamp_delay( $settings['crawl_delay_seconds'] ?? 10 ); if( is_array( $bots ) ) { foreach( $bots as $bot ) { $bot = $this->sanitize_bot_name( (string) $bot ); if( '' === $bot ) { continue; } $delay_map[ strtolower( $bot ) ] = $seconds; } } if( !empty( $overrides ) ) { $lines = preg_split( '/\r\n|\r|\n/', (string) $overrides ); if( false === $lines ) { $lines = []; } foreach( $lines as $line ) { if( !preg_match( '/^\s*(.+?)\s*:\s*(\d+)\s*$/', $line, $matches ) ) { continue; } $bot = $this->sanitize_bot_name( $matches[1] ); if( '' === $bot ) { continue; } $delay_map[ strtolower( $bot ) ] = $this->clamp_delay( $matches[2] ); } } if( count( $delay_map ) > self::MAX_BOTS ) { $delay_map = array_slice( $delay_map, 0, self::MAX_BOTS, true ); } return $delay_map; } /** * Constrain a delay to the supported range. * * @param mixed $delay Raw delay value. * * @return int Delay in seconds. */ private function clamp_delay( $delay ): int { $delay = (int) $delay; if( $delay < self::MIN_DELAY ) { return self::MIN_DELAY; } if( $delay > self::MAX_DELAY ) { return self::MAX_DELAY; } return $delay; } /** * Sanitize a single bot name. * Strips disallowed characters and enforces a maximum length. * * @param string $bot Raw bot name. * * @return string Sanitized bot name. */ private function sanitize_bot_name( string $bot ): string { $bot = trim( $bot ); $bot = preg_replace( '/[^a-zA-Z0-9\s\-_.]/', '', $bot ); if( strlen( $bot ) > 100 ) { $bot = substr( $bot, 0, 100 ); } return trim( $bot ); } }