self::evaluate_root($body, 'OAI-SearchBot'), 'general' => self::evaluate_root($body, '*'), 'gptBot' => self::evaluate_root($body, 'GPTBot'), ]; } /** * @return list, rules: list}> */ public static function parse_groups(string $body): array { $lines = preg_split('/\r\n|\r|\n/', $body) ?: []; $groups = []; $current_agents = []; $current_rules = []; $flush = static function () use (&$groups, &$current_agents, &$current_rules): void { if ($current_agents === [] && $current_rules === []) { return; } $groups[] = [ 'agents' => $current_agents === [] ? ['*'] : $current_agents, 'rules' => $current_rules, ]; $current_agents = []; $current_rules = []; }; foreach ($lines as $line) { $trim = trim((string) $line); if ($trim === '' || str_starts_with($trim, '#')) { continue; } $parts = explode(':', $trim, 2); if (count($parts) !== 2) { continue; } $directive = strtolower(trim($parts[0])); $value = trim(self::strip_comment($parts[1])); if ($directive === 'user-agent') { if ($current_rules !== []) { $flush(); } $current_agents[] = $value; continue; } if ($directive === 'disallow' || $directive === 'allow') { $current_rules[] = ['type' => $directive, 'path' => $value]; } } $flush(); return $groups; } /** * @param list, rules: list}> $groups * @return list|null */ private static function merged_rules_for_agent(array $groups, string $agent): ?array { $specific_rules = []; $wildcard_rules = []; $saw_specific_group = false; $saw_wildcard_group = false; foreach ($groups as $group) { if (self::group_has_agent($group['agents'], $agent)) { $saw_specific_group = true; foreach ($group['rules'] as $rule) { $specific_rules[] = $rule; } continue; } if (self::group_has_agent($group['agents'], '*')) { $saw_wildcard_group = true; foreach ($group['rules'] as $rule) { $wildcard_rules[] = $rule; } } } if ($saw_specific_group) { return $specific_rules; } if ($saw_wildcard_group) { return $wildcard_rules; } return $groups === [] ? null : null; } /** @param list $agents */ private static function group_has_agent(array $agents, string $agent): bool { foreach ($agents as $listed) { if (strcasecmp((string) $listed, $agent) === 0) { return true; } } return false; } /** * @param list $rules */ private static function root_decision_for_rules(array $rules): string { $best = null; $best_len = -1; $url_path = '/'; foreach ($rules as $rule) { $path = (string) ($rule['path'] ?? ''); if ($rule['type'] === 'disallow' && $path === '') { continue; } if (!self::path_matches_pattern($path, $url_path)) { continue; } $len = strlen($path); if ($len > $best_len) { $best_len = $len; $best = $rule; } elseif ($len === $best_len && $best !== null && $rule['type'] === 'allow' && $best['type'] === 'disallow') { $best = $rule; } } if ($best === null) { return 'allow'; } if ($best['type'] === 'allow') { return 'allow'; } return 'disallow'; } private static function path_matches_pattern(string $pattern, string $url_path): bool { if ($pattern === '') { return false; } $ends_with_dollar = str_ends_with($pattern, '$'); if ($ends_with_dollar) { $pattern = substr($pattern, 0, -1); } $regex = ''; $length = strlen($pattern); for ($i = 0; $i < $length; ++$i) { $char = $pattern[$i]; if ($char === '*') { $regex .= '.*'; continue; } $regex .= preg_quote($char, '/'); } if ($ends_with_dollar) { return preg_match('/^' . $regex . '$/', $url_path) === 1; } return preg_match('/^' . $regex . '/', $url_path) === 1; } private static function strip_comment(string $value): string { $pos = strpos($value, '#'); if ($pos === false) { return trim($value); } return trim(substr($value, 0, $pos)); } }