$bot ) { $token = strtolower( (string) $bot['ua'] ); if ( '' === $token || ! str_contains( $haystack, $token ) ) { continue; } // Longest match wins: "Claude-SearchBot" contains "ClaudeBot" only // in the loosest sense, but "Perplexity-User" genuinely contains // "Perplexity", and picking the shorter one would file every // on-demand fetch under the training crawler. if ( strlen( $token ) > $best_len ) { $best = (string) $id; $best_len = strlen( $token ); } } return $best; } /** * Record one visit. * * UPDATE first, INSERT only when the (bot, day) row does not exist yet — * so the hot path is a single atomic increment, and the insert race (two * first-hits of the same day) is settled by the primary key: the loser's * INSERT fails and its UPDATE retry lands on the winner's row. */ public static function record( string $bot_id, string $path, int $status = 200 ): void { global $wpdb; $table = \AiRankSuite\Support\Database::crawler_hits(); $day = gmdate( 'Y-m-d' ); $blocked = 403 === $status ? 1 : 0; $now = time(); $path = mb_substr( $path, 0, 190 ); // phpcs:disable WordPress.DB.DirectDatabaseQuery -- purpose-built table. $updated = $wpdb->query( $wpdb->prepare( "UPDATE {$table} SET hits = hits + 1, blocked = blocked + %d, last = %d, last_path = %s WHERE bot = %s AND day = %s", $blocked, $now, $path, $bot_id, $day ) ); if ( 0 === $updated ) { $inserted = $wpdb->query( $wpdb->prepare( "INSERT INTO {$table} (bot, day, hits, blocked, last, last_path) VALUES (%s, %s, 1, %d, %d, %s)", $bot_id, $day, $blocked, $now, $path ) ); if ( false === $inserted ) { // Lost the race: the row exists now, and the increment applies. $wpdb->query( $wpdb->prepare( "UPDATE {$table} SET hits = hits + 1, blocked = blocked + %d, last = %d, last_path = %s WHERE bot = %s AND day = %s", $blocked, $now, $path, $bot_id, $day ) ); } else { // First hit of a new day: the cheapest moment to prune, because // it runs at most once per bot per day, never on the hot path. $wpdb->query( $wpdb->prepare( "DELETE FROM {$table} WHERE day < %s", gmdate( 'Y-m-d', time() - ( self::DAYS + 30 ) * DAY_IN_SECONDS ) ) ); } } // phpcs:enable } /** @return array> */ public static function all(): array { global $wpdb; $table = \AiRankSuite\Support\Database::crawler_hits(); // phpcs:ignore WordPress.DB.DirectDatabaseQuery $rows = $wpdb->get_results( $wpdb->prepare( "SELECT bot, day, hits, blocked, last, last_path FROM %i ORDER BY day ASC", $table ), ARRAY_A ); if ( ! is_array( $rows ) ) { return array(); } $log = array(); foreach ( $rows as $row ) { $bot = (string) $row['bot']; $entry = $log[ $bot ] ?? array( 'hits' => 0, 'blocked' => 0, 'first' => 0, 'last' => 0, 'last_path' => '', 'days' => array(), ); $entry['hits'] += (int) $row['hits']; $entry['blocked'] += (int) $row['blocked']; $day_start = (int) strtotime( $row['day'] . ' 00:00:00 UTC' ); if ( 0 === $entry['first'] || $day_start < $entry['first'] ) { $entry['first'] = $day_start; } if ( (int) $row['last'] > $entry['last'] ) { $entry['last'] = (int) $row['last']; $entry['last_path'] = (string) $row['last_path']; } $entry['days'][ (string) $row['day'] ] = (int) $row['hits']; $log[ $bot ] = $entry; } return $log; } /** * The report: every known bot, what it did, and whether the rule holds. * * @param array $blocked Bot ids currently disallowed. * @return array */ public static function report( array $blocked ): array { $log = self::all(); $blocked = array_flip( $blocked ); $rows = array(); $seen = 0; $total = 0; $defiant = 0; $since = time() - ( 7 * DAY_IN_SECONDS ); foreach ( CrawlerControlsModule::bots() as $id => $bot ) { $entry = $log[ $id ] ?? null; $hits = (int) ( $entry['hits'] ?? 0 ); $last = (int) ( $entry['last'] ?? 0 ); // Hits in the last seven days, from the daily buckets. $recent = 0; foreach ( (array) ( $entry['days'] ?? array() ) as $day => $count ) { if ( strtotime( $day . ' 00:00:00 UTC' ) >= $since ) { $recent += (int) $count; } } $is_blocked = isset( $blocked[ $id ] ); // The interesting state: disallowed, and still turning up. Given a // day's grace, because a crawler that read robots.txt an hour // before the rule changed is not defying anything. $ignoring = $is_blocked && $last > 0 && ( time() - $last ) < DAY_IN_SECONDS && $recent > 0; if ( $ignoring ) { ++$defiant; } if ( $hits > 0 ) { ++$seen; $total += $hits; } $rows[ (string) $id ] = array( 'hits' => $hits, 'recent' => $recent, 'blocked' => (int) ( $entry['blocked'] ?? 0 ), 'first' => (int) ( $entry['first'] ?? 0 ), 'last' => $last, 'last_path' => (string) ( $entry['last_path'] ?? '' ), 'ignoring' => $ignoring, 'series' => self::series( (array) ( $entry['days'] ?? array() ) ), ); } return array( 'bots' => $rows, 'seen' => $seen, 'total' => $total, 'defiant' => $defiant, ); } /** * Fourteen days of counts, oldest first, with the gaps filled in. * * A sparkline built from only the days that had traffic would compress the * quiet weeks out of existence and make every crawler look constant. * * @param array $days Day => count. * @return array */ private static function series( array $days ): array { $out = array(); for ( $i = 13; $i >= 0; $i-- ) { $key = gmdate( 'Y-m-d', time() - ( $i * DAY_IN_SECONDS ) ); $out[] = (int) ( $days[ $key ] ?? 0 ); } return $out; } public static function reset(): void { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery $wpdb->query( $wpdb->prepare( 'DELETE FROM %i', \AiRankSuite\Support\Database::crawler_hits() ) ); // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared -- table name from this plugin's own Database class/adapter, never from input; values go through $wpdb->prepare(). delete_option( self::OPTION ); } /** * Read the live robots.txt and check the rules are really being served. * * Writing a rule and serving it are different things: a static robots.txt * file on disk wins over the WordPress filter, a CDN can serve a cached * copy for a day, and a security plugin can rewrite the whole file. None * of that is visible from inside the admin, and all of it silently undoes * the switches on this screen. * * @param array $blocked Bot ids that should be disallowed. * @return array */ public static function verify( array $blocked ): array { $response = wp_remote_get( home_url( '/robots.txt' ), array( 'timeout' => 12, ) ); if ( is_wp_error( $response ) ) { return array( 'ok' => false, 'reason' => $response->get_error_message(), ); } $code = (int) wp_remote_retrieve_response_code( $response ); $body = (string) wp_remote_retrieve_body( $response ); if ( $code >= 400 || '' === trim( $body ) ) { return array( 'ok' => false, 'status' => $code, 'reason' => __( 'The site did not return a robots.txt. Nothing on this screen is being enforced.', 'ai-rank-suite' ), ); } $bots = CrawlerControlsModule::bots(); $missing = array(); $present = 0; foreach ( $blocked as $id ) { $token = (string) ( $bots[ (string) $id ]['ua'] ?? '' ); if ( '' === $token ) { continue; } // The agent must appear AND be followed by a Disallow before the // next User-agent line. A file that merely names the bot proves // nothing. if ( self::disallowed( $body, $token ) ) { ++$present; } else { $missing[] = array( 'id' => (string) $id, 'agent' => $token, ); } } return array( 'ok' => true, 'status' => $code, 'bytes' => strlen( $body ), 'present' => $present, 'missing' => $missing, 'body' => mb_substr( $body, 0, 8000 ), ); } /** * Is this agent disallowed anywhere in the file? */ private static function disallowed( string $robots, string $agent ): bool { $lines = preg_split( '/\r\n|\r|\n/', $robots ) ?: array(); $in_group = false; foreach ( $lines as $line ) { $line = trim( $line ); if ( '' === $line || str_starts_with( $line, '#' ) ) { continue; } if ( preg_match( '/^user-agent\s*:\s*(.+)$/i', $line, $m ) ) { $in_group = 0 === strcasecmp( trim( $m[1] ), $agent ); continue; } if ( $in_group && preg_match( '/^disallow\s*:\s*(.*)$/i', $line, $m ) ) { // `Disallow:` with nothing after it means "allow everything", // which is the opposite of what we are checking for. if ( '' !== trim( $m[1] ) ) { return true; } } } return false; } }