8, 'redirection' => 0, 'sslverify' => false ] ); if ( is_wp_error( $response ) || wp_remote_retrieve_response_code( $response ) !== 200 ) { return ''; } return wp_remote_retrieve_body( $response ); } /** * Preset directives for common AI training bots. * Returns an array keyed by preset slug. */ public static function get_presets() { return [ 'block_ai_training' => [ 'label' => 'Block AI training bots', 'description' => 'Blocks GPTBot, CCBot, Google-Extended, anthropic-ai, Claude-Web, PerplexityBot, Amazonbot, and Meta-ExternalAgent from crawling your site for training data.', 'rules' => implode( "\n", [ '# Block AI training crawlers', 'User-agent: GPTBot', 'Disallow: /', '', 'User-agent: CCBot', 'Disallow: /', '', 'User-agent: Google-Extended', 'Disallow: /', '', 'User-agent: anthropic-ai', 'Disallow: /', '', 'User-agent: Claude-Web', 'Disallow: /', '', 'User-agent: PerplexityBot', 'Disallow: /', '', 'User-agent: Amazonbot', 'Disallow: /', '', 'User-agent: Meta-ExternalAgent', 'Disallow: /', ] ), ], 'block_wp_admin' => [ 'label' => 'Block /wp-admin/ (recommended)', 'description' => 'Standard rule to disallow bots from crawling the admin area.', 'rules' => implode( "\n", [ 'User-agent: *', 'Disallow: /wp-admin/', 'Allow: /wp-admin/admin-ajax.php', ] ), ], 'add_sitemap' => [ 'label' => 'Add sitemap reference', 'description' => 'Tells crawlers where your XML sitemap lives.', 'rules' => 'Sitemap: ' . home_url( '/sitemap.xml' ), ], 'block_search' => [ 'label' => 'Block internal search pages', 'description' => 'Prevents crawlers from indexing paginated search result pages.', 'rules' => implode( "\n", [ 'User-agent: *', 'Disallow: /?s=', 'Disallow: /search/', ] ), ], ]; } // ── AJAX ───────────────────────────────────────────────────────────────── public static function ajax_save() { check_ajax_referer( 'awbseo_admin', 'nonce' ); if ( ! current_user_can( 'manage_options' ) ) wp_send_json_error( 'Forbidden' ); // wp_unslash + allow newlines; strip any HTML. $content = wp_strip_all_tags( wp_unslash( $_POST['content'] ?? '' ) ); update_option( self::OPTION_KEY, $content, false ); wp_send_json_success( [ 'message' => 'robots.txt saved.' ] ); } public static function ajax_reset() { check_ajax_referer( 'awbseo_admin', 'nonce' ); if ( ! current_user_can( 'manage_options' ) ) wp_send_json_error( 'Forbidden' ); delete_option( self::OPTION_KEY ); wp_send_json_success( [ 'message' => 'Reset to WordPress default.', 'content' => self::get_wp_default(), ] ); } }