user-agent => one-line origin label */ public static function bot_list() { return [ 'GPTBot' => 'OpenAI — model training crawler', 'ChatGPT-User' => 'OpenAI — ChatGPT on-demand browsing', 'OAI-SearchBot' => 'OpenAI — search index', 'ClaudeBot' => 'Anthropic — Claude training crawler', 'Claude-Web' => 'Anthropic — Claude on-demand browsing', 'anthropic-ai' => 'Anthropic — legacy training UA', 'cohere-ai' => 'Cohere — training crawler', 'PerplexityBot' => 'Perplexity AI — answer engine', 'YouBot' => 'You.com — AI search', 'Google-Extended' => 'Google — Bard / Gemini training opt-out', 'CCBot' => 'Common Crawl (used by most LLMs)', 'Bytespider' => 'ByteDance (TikTok) — AI training', 'FacebookBot' => 'Meta AI training crawler', 'Meta-ExternalAgent' => 'Meta — LLaMA & related training', 'Applebot-Extended' => 'Apple — Apple Intelligence training', 'Diffbot' => 'Diffbot — knowledge graph', 'ImagesiftBot' => 'ImageSift — reverse-image AI', 'Omgilibot' => 'Webz.io — content aggregation', 'Omgili' => 'Webz.io legacy', 'Timpibot' => 'Timpi search', 'Amazonbot' => 'Amazon — Alexa AI', 'PetalBot' => 'Huawei — Petal search & AI', ]; } public static function init() { if ( ! get_option( 'aiep_scraper_defense_enabled', 0 ) ) { return; } add_filter( 'robots_txt', [ __CLASS__, 'append_robots' ], 20, 2 ); add_action( 'init', [ __CLASS__, 'maybe_track_request' ] ); } /** * Append our disallow rules to whatever WordPress already returned. */ public static function append_robots( $output, $public ) { if ( '1' !== (string) $public ) { return $output; } $lines = [ '', '# AI Explainer Pro — AI scraper defense', '# https://aiexplainerpro.noveralis.com' ]; foreach ( array_keys( self::bot_list() ) as $ua ) { $lines[] = 'User-agent: ' . $ua; $lines[] = 'Disallow: /'; $lines[] = ''; } return $output . "\n" . implode( "\n", $lines ); } /** * Lightweight request tracking. Counts hits per UA over a rolling * 30-day window. No IP storage — privacy first. Pro uses this for * the stats card on the defense admin tab. */ public static function maybe_track_request() { if ( ! isset( $_SERVER['HTTP_USER_AGENT'] ) ) { return; } $ua_raw = sanitize_text_field( wp_unslash( $_SERVER['HTTP_USER_AGENT'] ) ); if ( '' === $ua_raw ) { return; } foreach ( self::bot_list() as $needle => $_label ) { if ( false !== stripos( $ua_raw, $needle ) ) { self::increment_counter( $needle ); return; } } } private static function increment_counter( $needle ) { $stats = get_option( 'aiep_scraper_stats', [] ); $period = gmdate( 'Y-m' ); $current = isset( $stats[ $period ][ $needle ] ) ? (int) $stats[ $period ][ $needle ] : 0; // Throttle: never write more than once every 5 minutes to avoid // hammering the options cache on busy sites with many crawl hits. $tkey = 'aiep_scraper_tick_' . md5( $needle . '|' . $period ); if ( get_transient( $tkey ) ) { $stats[ $period ][ $needle ] = $current + 1; update_option( 'aiep_scraper_stats', self::trim_stats( $stats ), false ); return; } set_transient( $tkey, 1, 5 * MINUTE_IN_SECONDS ); $stats[ $period ][ $needle ] = $current + 1; update_option( 'aiep_scraper_stats', self::trim_stats( $stats ), false ); } /** * Keep only the last 6 months of stats so the option doesn't bloat. */ private static function trim_stats( $stats ) { if ( count( $stats ) <= 6 ) { return $stats; } krsort( $stats ); return array_slice( $stats, 0, 6, true ); } /** * Returns the totals for the current and previous month. */ public static function stats_summary() { $stats = get_option( 'aiep_scraper_stats', [] ); $current = gmdate( 'Y-m' ); $previous = gmdate( 'Y-m', strtotime( 'first day of last month' ) ); $by_bot = isset( $stats[ $current ] ) ? $stats[ $current ] : []; $total = array_sum( $by_bot ); $prev = isset( $stats[ $previous ] ) ? array_sum( $stats[ $previous ] ) : 0; return [ 'total_month' => (int) $total, 'total_previous' => (int) $prev, 'by_bot' => $by_bot, ]; } /** * Produces a ready-to-copy .htaccess block for sites that want * server-level enforcement (Pro UI surfaces this; Lite shows it * as informational copy in the settings tab). */ public static function htaccess_snippet() { $bots = array_keys( self::bot_list() ); $regex = '(' . implode( '|', array_map( 'preg_quote', $bots ) ) . ')'; return "# BEGIN AI Explainer Pro — AI scraper defense\n" . "\n" . "RewriteEngine On\n" . "RewriteCond %{HTTP_USER_AGENT} " . $regex . " [NC]\n" . "RewriteRule .* - [F,L]\n" . "\n" . "# END AI Explainer Pro — AI scraper defense\n"; } }