'gptbot', 'label' => 'OpenAI GPTBot', 'ua_pattern' => '/GPTBot/i', 'category' => 'training', 'robots_token' => 'GPTBot', 'ip_ranges_url' => 'https://openai.com/gptbot.json', 'enabled' => true, ), array( 'id' => 'oai-searchbot', 'label' => 'OpenAI OAI-SearchBot', 'ua_pattern' => '/OAI-SearchBot/i', 'category' => 'citation', 'robots_token' => 'OAI-SearchBot', 'ip_ranges_url' => 'https://openai.com/searchbot.json', 'enabled' => true, ), array( 'id' => 'chatgpt-user', 'label' => 'OpenAI ChatGPT-User', 'ua_pattern' => '/ChatGPT-User/i', 'category' => 'citation', 'robots_token' => 'ChatGPT-User', 'ip_ranges_url' => 'https://openai.com/chatgpt-user.json', 'enabled' => true, ), array( 'id' => 'claudebot', 'label' => 'Anthropic ClaudeBot', 'ua_pattern' => '/ClaudeBot/i', 'category' => 'training', 'robots_token' => 'ClaudeBot', 'ip_ranges_url' => '', 'rdns_suffixes' => array( 'anthropic.com' ), 'enabled' => true, ), array( 'id' => 'claude-user', 'label' => 'Anthropic Claude-User', 'ua_pattern' => '/Claude-User/i', 'category' => 'citation', 'robots_token' => 'Claude-User', 'ip_ranges_url' => '', 'rdns_suffixes' => array( 'anthropic.com' ), 'enabled' => true, ), array( 'id' => 'claude-searchbot', 'label' => 'Anthropic Claude-SearchBot', 'ua_pattern' => '/Claude-SearchBot/i', 'category' => 'citation', 'robots_token' => 'Claude-SearchBot', 'ip_ranges_url' => '', 'rdns_suffixes' => array( 'anthropic.com' ), 'enabled' => true, ), array( 'id' => 'perplexitybot', 'label' => 'PerplexityBot', 'ua_pattern' => '/PerplexityBot/i', 'category' => 'citation', 'robots_token' => 'PerplexityBot', 'enabled' => true, ), array( 'id' => 'perplexity-user', 'label' => 'Perplexity-User', 'ua_pattern' => '/Perplexity-User/i', 'category' => 'citation', 'robots_token' => 'Perplexity-User', 'enabled' => true, ), array( 'id' => 'google-extended', 'label' => 'Google-Extended', 'ua_pattern' => '/Google-Extended/i', 'category' => 'training', 'robots_token' => 'Google-Extended', 'enabled' => true, ), ); } /** * Valid access categories. * * training — crawlers gathering content for model-training corpora. * citation — assistants/search indexes that read content to answer and cite. * * @return string[] */ public static function categories() { return array( 'training', 'citation' ); } /** * Normalize a bot's category, defaulting unknown/legacy rows to 'citation'. * * @param array $bot Bot entry. * @return string 'training'|'citation'. */ public static function bot_category( $bot ) { $cat = isset( $bot['category'] ) ? strtolower( (string) $bot['category'] ) : ''; return in_array( $cat, self::categories(), true ) ? $cat : 'citation'; } /** * The robots.txt product token for a bot: explicit robots_token, else derived * from the UA pattern (strip regex delimiters/anchors). * * @param array $bot Bot entry. * @return string Token, or '' if none can be derived. */ public static function robots_token( $bot ) { $token = isset( $bot['robots_token'] ) ? trim( (string) $bot['robots_token'] ) : ''; if ( '' !== $token ) { return $token; } $ua = isset( $bot['ua_pattern'] ) ? trim( (string) $bot['ua_pattern'] ) : ''; if ( '' === $ua ) { return ''; } $first = $ua[0]; $delims = array( '/', '#', '~', '@' ); if ( in_array( $first, $delims, true ) ) { $last = strrpos( $ua, $first ); if ( $last > 0 ) { $ua = substr( $ua, 1, $last - 1 ); } } $ua = str_replace( array( '\\', '^', '$', '.*', '.' ), '', $ua ); $ua = trim( $ua ); return preg_match( '/^[A-Za-z0-9._-]+$/', $ua ) ? $ua : ''; } /** * Install the initial list on activation, merging with saved user edits (by id). * User values take precedence. */ public static function install_default_bots() { $saved = get_option( ADG_AIM_OPT_BOTS, array() ); $defaults = self::default_bots(); if ( ! is_array( $saved ) || empty( $saved ) ) { update_option( ADG_AIM_OPT_BOTS, $defaults ); return; } $by_id = array(); foreach ( $saved as $bot ) { if ( isset( $bot['id'] ) ) { $by_id[ $bot['id'] ] = $bot; } } // Add new bots; for existing ones, backfill only missing keys (e.g. // category / robots_token on upgrade) without overwriting user edits. foreach ( $defaults as $bot ) { if ( ! isset( $by_id[ $bot['id'] ] ) ) { $by_id[ $bot['id'] ] = $bot; } else { foreach ( $bot as $key => $val ) { if ( ! array_key_exists( $key, $by_id[ $bot['id'] ] ) ) { $by_id[ $bot['id'] ][ $key ] = $val; } } } } update_option( ADG_AIM_OPT_BOTS, array_values( $by_id ) ); } /** * Get the bot list from the option. * * @return array */ public function get_bots() { $bots = get_option( ADG_AIM_OPT_BOTS, array() ); if ( ! is_array( $bots ) || empty( $bots ) ) { $bots = self::default_bots(); } return $bots; } /** * The current request's User-Agent (sanitized). * * @return string */ public function get_user_agent() { if ( ! isset( $_SERVER['HTTP_USER_AGENT'] ) ) { return ''; } return sanitize_text_field( wp_unslash( $_SERVER['HTTP_USER_AGENT'] ) ); } /** * Determine whether the current request is from a known AI bot. * * Free tier: User-Agent match only. Pro with IP verification enabled: the * matched bot's IP is checked against its published CIDR feed or by * forward-confirmed reverse DNS; a spoofed UA from a non-owner IP is rejected * (and recorded as suspicious). The result carries a '_verified' flag. * * @return array|false Bot entry or false. */ public function detect() { if ( null !== $this->detected ) { return $this->detected; } $bot = $this->match_ua(); if ( false === $bot ) { $this->detected = false; return false; } $verified = true; if ( ADG_AIM_Plugin::is_pro() && (bool) ADG_AIM_Plugin::get_setting( 'ip_verification', false ) ) { $ip = $this->get_client_ip(); if ( ! empty( $bot['ip_ranges_url'] ) ) { $check = $this->ip_in_published_ranges( $bot['ip_ranges_url'], $ip ); } elseif ( ! empty( $bot['rdns_suffixes'] ) ) { $check = $this->verify_fcrdns( $ip, (array) $bot['rdns_suffixes'] ); } else { $check = null; // Nothing to verify against — trust the UA. } if ( false === $check ) { // IP not confirmed — treat as UA spoofing. $this->record_suspicious( $bot ); $this->detected = false; return false; } $verified = ( true === $check ); // null (DNS/feed unavailable) → unverified, not blocked. } $bot['_verified'] = $verified; $this->detected = $bot; return $bot; } /** * Record a spoofed (suspicious) bot hit into analytics, when Pro. * * @param array $bot Bot entry. */ private function record_suspicious( $bot ) { if ( ! class_exists( 'ADG_AIM_Analytics' ) ) { return; } $label = isset( $bot['label'] ) ? $bot['label'] : ( isset( $bot['id'] ) ? $bot['id'] : 'unknown' ); $uri = isset( $_SERVER['REQUEST_URI'] ) ? esc_url_raw( wp_unslash( $_SERVER['REQUEST_URI'] ) ) : ''; ADG_AIM_Analytics::record( $label, 'suspicious', $uri ); } /** * Match the current User-Agent against the bot list. * * @return array|false Bot entry or false. */ public function match_ua() { $ua = $this->get_user_agent(); if ( '' === $ua ) { return false; } foreach ( $this->get_bots() as $bot ) { if ( empty( $bot['enabled'] ) || empty( $bot['ua_pattern'] ) ) { continue; } $pattern = $this->safe_pattern( $bot['ua_pattern'] ); if ( '' === $pattern ) { continue; } if ( 1 === preg_match( $pattern, $ua ) ) { return $bot; } } return false; } /** * Safely normalize a regex from settings: guarantee a valid delimiter. * * @param string $pattern Pattern. * @return string Valid pattern or '' if invalid. */ private function safe_pattern( $pattern ) { $pattern = (string) $pattern; if ( '' === $pattern ) { return ''; } // If the admin entered a ready /.../i — honor it only if it is a well-formed // delimited regex. Otherwise wrap the input as a literal substring. $first = $pattern[0]; $delims = array( '/', '#', '~', '@' ); if ( in_array( $first, $delims, true ) && strrpos( $pattern, $first ) > 0 && $this->is_valid_regex( $pattern ) ) { return $pattern; } return '/' . preg_quote( $pattern, '/' ) . '/i'; } /** * Whether a delimited pattern is a compilable regex. * * Uses a plain preg_match probe (no error suppression, so nothing to flag). * preg_match returns false on an invalid pattern; a valid pattern returns 0/1. * * @param string $pattern Delimited pattern (e.g. /GPTBot/i). * @return bool */ private function is_valid_regex( $pattern ) { return false !== preg_match( $pattern, '' ); } /** * Client IP (sanitized, simple logic without trusting proxy headers). * * @return string */ public function get_client_ip() { if ( ! isset( $_SERVER['REMOTE_ADDR'] ) ) { return ''; } $ip = sanitize_text_field( wp_unslash( $_SERVER['REMOTE_ADDR'] ) ); return filter_var( $ip, FILTER_VALIDATE_IP ) ? $ip : ''; } /** * Forward-confirmed reverse DNS. * * Steps: 1) PTR of the requesting IP → hostname; 2) the hostname must end * with one of the allowed domains (e.g. anthropic.com); 3) forward-resolve * the hostname back to IP addresses — the original IP must be among them. * The result is cached per IP (positive/negative — for a long time, * "unknown" due to DNS unavailability — briefly), to avoid hammering the resolver. * * @param string $ip Client IP. * @param array $suffixes Allowed domain suffixes. * @return bool|null true confirmed, false spoofing, null DNS unavailable. */ private function verify_fcrdns( $ip, array $suffixes ) { if ( '' === $ip || empty( $suffixes ) ) { return null; } if ( ! function_exists( 'gethostbyaddr' ) ) { return null; // DNS functions disabled on the host — verification impossible. } $cache_key = 'adg_aim_rdns_' . md5( $ip . '|' . implode( ',', $suffixes ) ); $cached = get_transient( $cache_key ); if ( false !== $cached ) { if ( 'y' === $cached ) { return true; } if ( 'n' === $cached ) { return false; } return null; } $result = $this->do_fcrdns( $ip, $suffixes ); $flag = ( true === $result ) ? 'y' : ( ( false === $result ) ? 'n' : '?' ); $ttl = ( '?' === $flag ) ? HOUR_IN_SECONDS : 12 * HOUR_IN_SECONDS; set_transient( $cache_key, $flag, $ttl ); return $result; } /** * Actually run the FCrDNS check (without cache). * * @param string $ip Client IP. * @param array $suffixes Allowed suffixes. * @return bool|null */ private function do_fcrdns( $ip, array $suffixes ) { // Step 1: PTR (reverse lookup). // @codingStandardsIgnoreLine — network call, errors silenced intentionally. $host = @gethostbyaddr( $ip ); if ( ! $host || $host === $ip ) { return null; // PTR unavailable — cannot verify. } $host = rtrim( strtolower( $host ), '.' ); // Step 2: the suffix must match one of the allowed domains. $matched = false; foreach ( $suffixes as $suffix ) { $suffix = strtolower( trim( ltrim( trim( (string) $suffix ), '.' ) ) ); if ( '' === $suffix ) { continue; } if ( $host === $suffix || substr( $host, - ( strlen( $suffix ) + 1 ) ) === '.' . $suffix ) { $matched = true; break; } } if ( ! $matched ) { return false; // PTR host not in an allowed domain — spoofing. } // Step 3: forward-resolve the hostname and compare with the original IP. $forward = $this->forward_ips( $host ); if ( empty( $forward ) ) { return null; // Forward DNS unavailable. } $target = @inet_pton( $ip ); // phpcs:ignore WordPress.PHP.NoSilencedErrors foreach ( $forward as $fip ) { // Compare in binary form — correct for IPv6 canonicalization. if ( @inet_pton( $fip ) === $target ) { // phpcs:ignore WordPress.PHP.NoSilencedErrors return true; } } return false; } /** * Forward-resolve a hostname into a list of IPs (A + AAAA). * * @param string $host Host. * @return string[] List of IP strings. */ private function forward_ips( $host ) { $ips = array(); if ( function_exists( 'gethostbynamel' ) ) { // @codingStandardsIgnoreLine $a = @gethostbynamel( $host ); if ( is_array( $a ) ) { $ips = array_merge( $ips, $a ); } } if ( function_exists( 'dns_get_record' ) && defined( 'DNS_AAAA' ) ) { // @codingStandardsIgnoreLine $aaaa = @dns_get_record( $host, DNS_AAAA ); if ( is_array( $aaaa ) ) { foreach ( $aaaa as $rec ) { if ( ! empty( $rec['ipv6'] ) ) { $ips[] = $rec['ipv6']; } } } } return array_values( array_unique( $ips ) ); } /** * Check an IP against a published JSON feed of ranges (cached 24h). * * @param string $url Feed URL. * @param string $ip Client IP. * @return bool|null true in range, false out of range, null if feed unavailable/empty. */ private function ip_in_published_ranges( $url, $ip ) { if ( '' === $ip || ! filter_var( $url, FILTER_VALIDATE_URL ) ) { return null; } $cache_key = 'adg_aim_iprng_' . md5( $url ); $ranges = get_transient( $cache_key ); if ( false === $ranges ) { $ranges = array(); $resp = wp_remote_get( $url, array( 'timeout' => 5 ) ); if ( ! is_wp_error( $resp ) && 200 === wp_remote_retrieve_response_code( $resp ) ) { $data = json_decode( wp_remote_retrieve_body( $resp ), true ); $ranges = $this->extract_cidrs( $data ); } // Cache even an empty result (for 24h) to avoid hitting the feed on every request. set_transient( $cache_key, $ranges, DAY_IN_SECONDS ); } if ( empty( $ranges ) ) { return null; // Feed unavailable/empty — verification impossible. } foreach ( $ranges as $cidr ) { if ( $this->ip_in_cidr( $ip, $cidr ) ) { return true; } } return false; } /** * Extract CIDR prefixes from common feed formats (OpenAI-style prefixes[].ipv4Prefix, etc.). * * @param mixed $data Decoded JSON. * @return array List of CIDR strings. */ private function extract_cidrs( $data ) { $out = array(); if ( ! is_array( $data ) ) { return $out; } // OpenAI / Google format: { "prefixes": [ { "ipv4Prefix": "1.2.3.0/24" }, ... ] }. if ( isset( $data['prefixes'] ) && is_array( $data['prefixes'] ) ) { foreach ( $data['prefixes'] as $p ) { if ( isset( $p['ipv4Prefix'] ) ) { $out[] = $p['ipv4Prefix']; } if ( isset( $p['ipv6Prefix'] ) ) { $out[] = $p['ipv6Prefix']; } } } // Flat array of strings. foreach ( $data as $v ) { if ( is_string( $v ) && false !== strpos( $v, '/' ) ) { $out[] = $v; } } return array_values( array_unique( $out ) ); } /** * Check whether an IP is in a CIDR (IPv4 and IPv6). * * @param string $ip IP. * @param string $cidr CIDR. * @return bool */ private function ip_in_cidr( $ip, $cidr ) { if ( false === strpos( $cidr, '/' ) ) { return $ip === $cidr; } list( $subnet, $bits ) = explode( '/', $cidr, 2 ); $bits = (int) $bits; $ip_bin = @inet_pton( $ip ); $subnet_bin = @inet_pton( $subnet ); if ( false === $ip_bin || false === $subnet_bin || strlen( $ip_bin ) !== strlen( $subnet_bin ) ) { return false; } $bytes = intdiv( $bits, 8 ); $rem = $bits % 8; if ( $bytes > 0 && 0 !== substr_compare( $ip_bin, $subnet_bin, 0, $bytes ) ) { return false; } if ( $rem > 0 ) { $mask = chr( 0xff << ( 8 - $rem ) & 0xff ); if ( ( ord( $ip_bin[ $bytes ] ) & ord( $mask ) ) !== ( ord( $subnet_bin[ $bytes ] ) & ord( $mask ) ) ) { return false; } } return true; } }