array( 'label' => 'OpenAI', 'method' => 'ranges', 'slugs' => array( 'gptbot', 'chatgpt-user', 'oai-searchbot', 'oai-adsbot' ), 'urls' => array( 'https://openai.com/gptbot.json', 'https://openai.com/chatgpt-user.json', 'https://openai.com/searchbot.json', ), ), 'anthropic' => array( 'label' => 'Anthropic', 'method' => 'ranges', 'slugs' => array( 'claudebot', 'claude-user', 'claude-searchbot', 'anthropic-ai' ), 'urls' => array( 'https://www.anthropic.com/ips.json' ), ), 'perplexity' => array( 'label' => 'Perplexity', 'method' => 'ranges', 'slugs' => array( 'perplexitybot', 'perplexity-user' ), 'urls' => array( 'https://www.perplexity.com/perplexitybot.json', 'https://www.perplexity.com/perplexity-user.json', ), ), 'google' => array( 'label' => 'Google', 'method' => 'rdns', 'slugs' => array( 'google-agent', 'google-gemininotebook', 'google-notebooklm', 'google-cloudvertexbot', 'googleother', 'googleother-image', 'googleother-video', ), /* * Reverse-DNS suffixes, not asset hosts. Verification resolves the * request IP and requires the resulting hostname to end in one of * these, which is how Google documents proving a crawler is theirs. * Nothing is ever fetched from them. The Offloading sniff reads * googleusercontent.com as a CDN a plugin might load assets from and * reports an error; dropping the suffix would silently disable * verification for Google's crawlers, which is the opposite of safe. */ // phpcs:ignore PluginCheck.CodeAnalysis.Offloading.OffloadedContent -- Reverse-DNS suffix allowlist for verifying Google crawlers; no content is loaded from these hosts. 'suffixes' => array( '.google.com', '.googlebot.com', '.googleusercontent.com' ), ), 'apple' => array( 'label' => 'Apple', 'method' => 'rdns', 'slugs' => array( 'applebot' ), 'suffixes' => array( '.applebot.apple.com' ), ), ); $filtered = apply_filters( 'ai_o_optimiser_bot_verification_vendors', $vendors ); return is_array( $filtered ) ? $filtered : $vendors; } /** @return bool Whether the site owner has turned verification on. */ public static function enabled() { $settings = get_option( 'ai_o_optimiser_settings', array() ); return ! empty( $settings['verify_bot_identity'] ); } /** * The vendor definition covering one bot slug, or null. * * @param string $slug Bot slug. * @return array|null */ public static function vendor_for_slug( $slug ) { $slug = (string) $slug; foreach ( self::vendors() as $key => $vendor ) { if ( in_array( $slug, (array) $vendor['slugs'], true ) ) { $vendor['key'] = $key; return $vendor; } } return null; } /** * Verdict for one request. * * @param string $slug Matched bot slug. * @param string $ip Resolved client address. * @return string One of the verdict constants, or '' when not checked. */ public static function verdict( $slug, $ip ) { if ( ! self::enabled() ) { return ''; } if ( '' === (string) $slug || ! filter_var( $ip, FILTER_VALIDATE_IP ) ) { return ''; } $vendor = self::vendor_for_slug( $slug ); if ( null === $vendor ) { return self::UNVERIFIABLE; } if ( 'ranges' === $vendor['method'] ) { $ranges = self::cached_ranges( $vendor['key'] ); // No list yet -- first run, or every fetch failed. Saying "unverified" // here would report a spoof that was never actually looked for. if ( null === $ranges ) { return self::UNVERIFIABLE; } return AI_O_Optimiser_Client_IP::in_any_range( $ip, $ranges ) ? self::VERIFIED : self::UNVERIFIED; } if ( 'rdns' === $vendor['method'] ) { return self::rdns_verdict( $ip, (array) $vendor['suffixes'] ); } return self::UNVERIFIABLE; } /** * Forward-confirmed reverse DNS. * * Runs after the response has been flushed to the visitor, and the verdict * for an address is cached, so a crawler making a hundred requests costs one * pair of lookups rather than a hundred. * * @param string $ip Client address. * @param string[] $suffixes Hostname suffixes the vendor owns. * @return string */ private static function rdns_verdict( $ip, $suffixes ) { $cache_key = self::RDNS_CACHE_PREFIX . md5( $ip . '|' . implode( ',', $suffixes ) ); $cached = get_transient( $cache_key ); if ( is_string( $cached ) && '' !== $cached ) { return $cached; } $verdict = self::resolve_rdns_verdict( $ip, $suffixes ); set_transient( $cache_key, $verdict, self::RDNS_CACHE_TTL ); return $verdict; } /** * @param string $ip Client address. * @param string[] $suffixes Hostname suffixes the vendor owns. * @return string */ private static function resolve_rdns_verdict( $ip, $suffixes ) { $host = self::reverse_lookup( $ip ); if ( '' === $host ) { return self::UNVERIFIED; } $host = strtolower( rtrim( $host, '.' ) ); $owns = false; foreach ( $suffixes as $suffix ) { $suffix = strtolower( (string) $suffix ); if ( $host === ltrim( $suffix, '.' ) || substr( $host, -strlen( $suffix ) ) === $suffix ) { $owns = true; break; } } if ( ! $owns ) { return self::UNVERIFIED; } // The forward half. Reverse DNS is set by whoever controls the address, // so without this anyone could name themselves crawl-1.googlebot.com. return in_array( $ip, self::forward_lookup( $host ), true ) ? self::VERIFIED : self::UNVERIFIED; } /** * @param string $ip Address. * @return string Hostname, or '' when there is none. */ private static function reverse_lookup( $ip ) { if ( ! function_exists( 'gethostbyaddr' ) ) { return ''; } $previous = ini_get( 'default_socket_timeout' ); @ini_set( 'default_socket_timeout', self::RDNS_TIMEOUT ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged,WordPress.PHP.IniSet.Risky,Squiz.PHP.DiscouragedFunctions.Discouraged -- bounds a DNS lookup on the shutdown path; restored immediately. $host = @gethostbyaddr( $ip ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- a failed lookup is an unverified verdict, not a warning. @ini_set( 'default_socket_timeout', $previous ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged,WordPress.PHP.IniSet.Risky,Squiz.PHP.DiscouragedFunctions.Discouraged -- restoring the caller's value. // gethostbyaddr() returns the address unchanged when there is no record. return ( is_string( $host ) && '' !== $host && $host !== $ip ) ? $host : ''; } /** * @param string $host Hostname. * @return string[] Every address the name resolves to. */ private static function forward_lookup( $host ) { $addresses = array(); if ( ! function_exists( 'dns_get_record' ) ) { return $addresses; } $records = @dns_get_record( $host, DNS_A | DNS_AAAA ); // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- a failed lookup is an unverified verdict, not a warning. foreach ( (array) $records as $record ) { if ( isset( $record['ip'] ) ) { $addresses[] = $record['ip']; } if ( isset( $record['ipv6'] ) ) { $addresses[] = $record['ipv6']; } } return $addresses; } /** * Cached published ranges for one vendor. * * @param string $vendor_key Vendor key. * @return array|null Ranges, or null when none have been fetched. */ public static function cached_ranges( $vendor_key ) { $store = get_option( self::RANGES_OPTION, array() ); if ( ! is_array( $store ) || empty( $store[ $vendor_key ]['ranges'] ) ) { return null; } $entry = $store[ $vendor_key ]; // A list this stale may no longer describe the vendor's fleet, and // judging a real crawler a spoof is worse than declining to judge. if ( ! isset( $entry['fetched'] ) || ( time() - (int) $entry['fetched'] ) > self::RANGE_TTL ) { return null; } return (array) $entry['ranges']; } /** * Refresh every range-published vendor. Runs on the daily cron. * * @return void */ public static function refresh_ranges() { if ( ! self::enabled() ) { return; } $store = get_option( self::RANGES_OPTION, array() ); $store = is_array( $store ) ? $store : array(); foreach ( self::vendors() as $key => $vendor ) { if ( 'ranges' !== $vendor['method'] ) { continue; } $ranges = array(); foreach ( (array) $vendor['urls'] as $url ) { $ranges = array_merge( $ranges, self::fetch_ranges( $url ) ); } $ranges = array_values( array_unique( $ranges ) ); // A failed fetch keeps whatever is already stored. Replacing a good // list with an empty one would turn every request from that vendor // into an unverifiable, silently, until the next success. if ( empty( $ranges ) ) { continue; } $store[ $key ] = array( 'ranges' => $ranges, 'fetched' => time(), ); } update_option( self::RANGES_OPTION, $store, false ); } /** * Fetch and parse one published range list. * * @param string $url Vendor URL. * @return string[] */ private static function fetch_ranges( $url ) { $response = wp_safe_remote_get( $url, array( 'timeout' => 15, 'redirection' => 2, 'limit_response_size' => 524288, 'headers' => array( 'Accept' => 'application/json' ), ) ); if ( is_wp_error( $response ) || 200 !== (int) wp_remote_retrieve_response_code( $response ) ) { return array(); } $data = json_decode( wp_remote_retrieve_body( $response ), true ); return is_array( $data ) ? self::collect_prefixes( $data ) : array(); } /** * Pull every CIDR block out of a decoded range document. * * Vendors converged on Google's shape -- a "prefixes" list of objects with * ipv4Prefix / ipv6Prefix keys -- but not exactly, and none of them promise * to keep it. Walking the structure for anything that is a valid block * survives a vendor nesting it differently or renaming the wrapper. * * @param mixed $node Decoded JSON node. * @param int $depth Recursion guard. * @return string[] */ private static function collect_prefixes( $node, $depth = 0 ) { if ( $depth > 6 || ! is_array( $node ) ) { return array(); } $found = array(); foreach ( $node as $value ) { if ( is_array( $value ) ) { $found = array_merge( $found, self::collect_prefixes( $value, $depth + 1 ) ); continue; } if ( is_string( $value ) && false !== strpos( $value, '/' ) && AI_O_Optimiser_Client_IP::is_valid_range( $value ) ) { $found[] = $value; } } return $found; } /** * Human-readable verdict label. * * @param string $verdict Verdict. * @return string */ public static function verdict_label( $verdict ) { switch ( $verdict ) { case self::VERIFIED: return __( 'Verified', 'ai-o-optimiser' ); case self::UNVERIFIED: return __( 'Failed verification', 'ai-o-optimiser' ); case self::UNVERIFIABLE: return __( 'No published proof', 'ai-o-optimiser' ); default: return __( 'Not checked', 'ai-o-optimiser' ); } } }