settings = Plugin::get_settings( 'security' );
add_filter( 'adminease_settings_fields', [ $this, 'adminease_settings_fields' ], 21 );
add_action( 'adminease_settings_saved', [ $this, 'adminease_settings_saved' ] );
if( !empty( $this->settings['crawl_delay_enabled'] ) ) {
add_filter( 'robots_txt', [ $this, 'robots_txt' ], 20, 2 );
if( !empty( $this->settings['crawl_delay_enforce'] ) ) {
add_action( 'init', [ $this, 'throttle_bots' ], 1 );
}
}
}
/**
* Adds the crawl delay configuration options to the settings fields array.
*
* @param array $fields The current fields array to be modified with additional settings.
*
* @return array The modified fields array including the crawl delay options.
*/
public function adminease_settings_fields( array $fields ): array {
$fields['security']['fields'][] = [
'type' => 'switch',
'id' => 'crawl-delay-enabled',
'name' => 'adminease[security][crawl_delay_enabled]',
'value' => $this->settings['crawl_delay_enabled'] ?? '',
'label_class' => 'pwp-switch',
'input_class' => 'pwp-field-input toggle-field',
'label' => __( 'Enable crawl delay', 'adminease' ),
'description' => __( 'Slow aggressive crawlers down instead of banning them. Blocking a search engine costs you the indexing; a crawl delay keeps you listed while capping how fast bots can hit your server. The delay is advertised in robots.txt for well-behaved crawlers, and can additionally be enforced in PHP for the ones that ignore it.', 'adminease' ),
'field_description' => $this->get_enable_field_description(),
'child_fields' => [
[
'type' => 'select',
'id' => 'crawl-delay-bots',
'name' => 'adminease[security][crawl_delay_bots][]',
'value' => $this->settings['crawl_delay_bots'] ?? [],
'options' => Utils::get_bots_list(),
'has_optgroups' => true,
'label_class' => 'pwp-field-label',
'input_class' => 'pwp-field-input adminease-choices',
'wrapper_class' => 'form-group-child',
'label' => __( 'Bots to slow down', 'adminease' ),
'field_description' => __( 'Select the bots that should be given a crawl delay. Use the Network Viewer Log to find out which bots are actually hitting your server.', 'adminease' ),
'attributes' => [
'data-parent' => 'crawl-delay-enabled',
'multiple' => 'multiple',
'data-allow_clear' => true,
'data-allow_select_all' => true,
],
],
[
'type' => 'number',
'id' => 'crawl-delay-seconds',
'name' => 'adminease[security][crawl_delay_seconds]',
'value' => $this->settings['crawl_delay_seconds'] ?? 10,
'label_class' => 'pwp-field-label',
'input_class' => 'pwp-field-input',
'wrapper_class' => 'form-group-child',
'label' => __( 'Delay in seconds', 'adminease' ),
'description' => __( 'Minimum number of seconds a selected bot must wait between requests.', 'adminease' ),
/* translators: 1: minimum delay in seconds, 2: maximum delay in seconds. */
'field_description' => sprintf( __( 'Applied to every bot selected above unless it has its own value below. Allowed range: %1$d to %2$d seconds.', 'adminease' ), self::MIN_DELAY, self::MAX_DELAY ),
'attributes' => [
'data-parent' => 'crawl-delay-enabled',
'min' => self::MIN_DELAY,
'max' => self::MAX_DELAY,
],
],
[
'type' => 'textarea',
'id' => 'crawl-delay-overrides',
'name' => 'adminease[security][crawl_delay_overrides]',
'value' => $this->settings['crawl_delay_overrides'] ?? '',
'label_class' => 'pwp-field-label',
'input_class' => 'pwp-field-input',
'wrapper_class' => 'form-group-child',
'label' => __( 'Per-bot delays', 'adminease' ),
'field_description' => __( 'Give individual bots their own delay, one per line, as BotName:seconds. These override the shared delay above, and a bot listed here is throttled even if it is not selected in the list above.
Example:
Bingbot:10
GPTBot:30
AhrefsBot:60', 'adminease' ),
'attributes' => [
'data-parent' => 'crawl-delay-enabled',
],
],
[
'type' => 'number',
'id' => 'crawl-delay-default',
'name' => 'adminease[security][crawl_delay_default]',
'value' => $this->settings['crawl_delay_default'] ?? 0,
'label_class' => 'pwp-field-label',
'input_class' => 'pwp-field-input',
'wrapper_class' => 'form-group-child',
'label' => __( 'Default delay for all other bots', 'adminease' ),
'description' => __( 'Adds a catch-all "User-agent: *" crawl delay to robots.txt.', 'adminease' ),
'field_description' => __( 'Published in robots.txt only and never enforced in PHP, because enforcing a delay against every user agent would throttle real visitors too. Set to 0 to leave the line out.', 'adminease' ),
'attributes' => [
'data-parent' => 'crawl-delay-enabled',
'min' => 0,
'max' => self::MAX_DELAY,
],
],
[
'type' => 'switch',
'id' => 'crawl-delay-enforce',
'name' => 'adminease[security][crawl_delay_enforce]',
'value' => $this->settings['crawl_delay_enforce'] ?? '',
'label_class' => 'pwp-switch',
'input_class' => 'pwp-field-input toggle-field',
'wrapper_class' => 'form-group-child',
'label' => __( 'Enforce the delay', 'adminease' ),
'field_description' => __( 'Answer bots that come back too soon with 429 Too Many Requests and a Retry-After header, instead of only asking them politely in robots.txt. Leave this off if you want advertising only.', 'adminease' ),
'attributes' => [
'data-parent' => 'crawl-delay-enabled',
],
],
],
];
return $fields;
}
/**
* Build the description shown under the parent toggle.
* Reports how the delays will actually reach crawlers on this particular site,
* since that depends on whether a physical robots.txt shadows the virtual one.
*
* @return string
*/
private function get_enable_field_description(): string {
$description = __( 'Bingbot and YandexBot honour the Crawl-delay directive. Googlebot ignores it by design, and so do most scrapers, which is what the enforcement option below is for.', 'adminease' );
$robots_txt = FileHandler::get_instance()->get_robots_txt_status();
if( empty( $robots_txt['exists'] ) ) {
return $description;
}
if( !empty( $robots_txt['writable'] ) ) {
$description .= '
' . __( 'Your site has a physical robots.txt file, which replaces the one WordPress generates. AdminEase writes the delays straight into it and keeps them up to date, leaving the rest of the file untouched.', 'adminease' );
return $description;
}
$description .= '
' . sprintf(
/* translators: %s: absolute path to the robots.txt file. */
__( 'Your robots.txt file is not writable, so the delays cannot be added to it: %s. Make the file writable, or copy the lines in by hand.', 'adminease' ),
esc_html( $robots_txt['path'] )
) . '';
return $description;
}
/**
* Write the Crawl-delay directives into a physical robots.txt, if the site has one.
*
* Only relevant when a real robots.txt exists: that file shadows the virtual one
* entirely, so the `robots_txt` filter never runs and the delays would otherwise
* never be served. FileHandler no-ops when there is no physical file, and removes
* the block when the rules come back empty.
*
* @param array $settings The sanitized settings being saved.
*
* @return void
*/
public function adminease_settings_saved( array $settings ): void {
if( !current_user_can( 'manage_options' ) ) {
return;
}
$security = $settings['security'] ?? [];
$rules = '';
if( !empty( $security['crawl_delay_enabled'] ) ) {
$rules = $this->build_crawl_delay_rules( $security );
}
FileHandler::get_instance()->set_robots_txt_block( 'CRAWL_DELAY', $rules );
}
/**
* Append the configured Crawl-delay groups to the virtual robots.txt.
*
* Core's do_robots() already prints a "User-agent: *" group. Appending a second
* one is safe: RFC 9309 requires parsers to merge records that share a user agent,
* and both Google's and Bing's parsers do. Rewriting core's group in place would
* be fragile against SEO plugins that filter `robots_txt` as well.
*
* @param string $output The robots.txt output built so far.
* @param bool $is_public Whether the site is asking to be indexed. Unused: a
* crawl delay is harmless on a non-indexed site, where
* core already prints "Disallow: /".
*
* @return string
*/
public function robots_txt( string $output, bool $is_public ): string {
return $output . $this->build_crawl_delay_rules();
}
/**
* Build the Crawl-delay directives.
*
* Single source of truth for both delivery paths, so what the virtual robots.txt
* advertises and what gets written into a physical robots.txt can never drift.
* Returns the bare directives with no marker comments around them.
*
* @param array|null $settings Security settings to read, or null for the stored ones.
*
* @return string Directives, or an empty string when there is nothing to publish.
*/
private function build_crawl_delay_rules( ?array $settings = null ): string {
$settings = $settings ?? $this->settings;
$delay_map = $this->get_delay_map( $settings );
$default = (int) ( $settings['crawl_delay_default'] ?? 0 );
if( $default > self::MAX_DELAY ) {
$default = self::MAX_DELAY;
}
if( empty( $delay_map ) && $default < 1 ) {
return '';
}
$rules = '';
foreach( $delay_map as $bot => $delay ) {
$rules .= "\nUser-agent: {$bot}\nCrawl-delay: {$delay}\n";
}
if( $default > 0 ) {
$rules .= "\nUser-agent: *\nCrawl-delay: {$default}\n";
}
return $rules;
}
/**
* Enforce the configured delay for bots that ignore robots.txt.
*
* Runs on 'init' at priority 1, matching BlockSpecificBots::php_block_bots(). An
* earlier hook is not usable: pluggable.php has not loaded during 'plugins_loaded',
* so is_user_logged_in() would not exist yet.
*
* @return void
*/
public function throttle_bots(): void {
if( is_admin() || wp_doing_ajax() || wp_doing_cron() ) {
return;
}
if( defined( 'WP_CLI' ) && WP_CLI ) {
return;
}
if( is_user_logged_in() ) {
return;
}
// phpcs:ignore WordPress.Security.ValidatedSanitizedInput.InputNotSanitized -- Sanitized below
$user_agent = isset( $_SERVER['HTTP_USER_AGENT'] ) ? sanitize_text_field( wp_unslash( $_SERVER['HTTP_USER_AGENT'] ) ) : '';
if( empty( $user_agent ) ) {
return;
}
$delay_map = $this->get_delay_map();
if( empty( $delay_map ) ) {
return;
}
foreach( $delay_map as $bot => $delay ) {
if( false === stripos( $user_agent, $bot ) ) {
continue;
}
$transient_key = self::TRANSIENT_PREFIX . md5( $bot );
if( false !== get_transient( $transient_key ) ) {
$this->send_too_many_requests( $delay );
}
set_transient( $transient_key, 1, $delay );
return;
}
}
/**
* Send a 429 response with a Retry-After header and stop the request.
* Retry-After is what makes this a crawl delay rather than a random rejection:
* Bing and Google both back off politely when it is present.
*
* @param int $delay Number of seconds the bot should wait.
*
* @return void
*/
private function send_too_many_requests( int $delay ): void {
nocache_headers();
header( 'Retry-After: ' . $delay );
header( 'Content-Type: text/plain; charset=utf-8' );
status_header( 429 );
echo 'Too Many Requests';
exit;
}
/**
* Build the map of throttled bots.
* Keys are lower-cased bot tokens, values are the delay in seconds.
*
* @param array|null $settings Security settings to read, or null for the stored ones.
*
* @return array
*/
private function get_delay_map( ?array $settings = null ): array {
$settings = $settings ?? $this->settings;
$delay_map = [];
$bots = $settings['crawl_delay_bots'] ?? [];
$overrides = $settings['crawl_delay_overrides'] ?? '';
$seconds = $this->clamp_delay( $settings['crawl_delay_seconds'] ?? 10 );
if( is_array( $bots ) ) {
foreach( $bots as $bot ) {
$bot = $this->sanitize_bot_name( (string) $bot );
if( '' === $bot ) {
continue;
}
$delay_map[ strtolower( $bot ) ] = $seconds;
}
}
if( !empty( $overrides ) ) {
$lines = preg_split( '/\r\n|\r|\n/', (string) $overrides );
if( false === $lines ) {
$lines = [];
}
foreach( $lines as $line ) {
if( !preg_match( '/^\s*(.+?)\s*:\s*(\d+)\s*$/', $line, $matches ) ) {
continue;
}
$bot = $this->sanitize_bot_name( $matches[1] );
if( '' === $bot ) {
continue;
}
$delay_map[ strtolower( $bot ) ] = $this->clamp_delay( $matches[2] );
}
}
if( count( $delay_map ) > self::MAX_BOTS ) {
$delay_map = array_slice( $delay_map, 0, self::MAX_BOTS, true );
}
return $delay_map;
}
/**
* Constrain a delay to the supported range.
*
* @param mixed $delay Raw delay value.
*
* @return int Delay in seconds.
*/
private function clamp_delay( $delay ): int {
$delay = (int) $delay;
if( $delay < self::MIN_DELAY ) {
return self::MIN_DELAY;
}
if( $delay > self::MAX_DELAY ) {
return self::MAX_DELAY;
}
return $delay;
}
/**
* Sanitize a single bot name.
* Strips disallowed characters and enforces a maximum length.
*
* @param string $bot Raw bot name.
*
* @return string Sanitized bot name.
*/
private function sanitize_bot_name( string $bot ): string {
$bot = trim( $bot );
$bot = preg_replace( '/[^a-zA-Z0-9\s\-_.]/', '', $bot );
if( strlen( $bot ) > 100 ) {
$bot = substr( $bot, 0, 100 );
}
return trim( $bot );
}
}