$status, 'timestamp' => time(), 'message' => $message, 'http_status' => $http_status, ]); } private static function http_status_from_exception(\Throwable $e) { if ($e instanceof CloudRequestException) { return $e->httpStatusCode; } return null; } public static function fetch_from_cloud() { $resource_key = get_option(Options::RESOURCE_KEY, ''); if (empty($resource_key)) { return false; } $all_categories = FiftyOneDegreesCloudMetadata::fetch_crawler_usage_values(); $effective_denied = self::get_effective_denied_for_cloud($all_categories); $params = ['resource' => $resource_key]; if (!empty($all_categories)) { foreach (array_keys($all_categories) as $cat) { $val = in_array($cat, $effective_denied) ? 'disallow' : 'allow'; $params['robotstxt.' . strtolower($cat)] = $val; } } else { foreach ($effective_denied as $cat) { $params['robotstxt.' . strtolower($cat)] = 'disallow'; } } $tdl_values = self::get_effective_tdl_values(); if (!empty($tdl_values)) { $params['robotstxt.tdl'] = implode(',', $tdl_values); } try { $engine = new CloudRequestEngine(['resourceKey' => $resource_key]); $url = rtrim($engine->baseURL, '/'); if (!preg_match('#/json$#', $url)) { $url .= '/json'; } $url .= '?' . http_build_query($params); } catch (\Throwable $e) { error_log('51Degrees robots.txt Cloud API request failed: ' . $e->getMessage()); set_transient('fiftyonedegrees_robots_cloud_error', $e->getMessage(), 60); self::record_last_refresh('error', $e->getMessage(), self::http_status_from_exception($e)); return false; } try { $body = self::do_cloud_request($url); } catch (\Throwable $e) { error_log('51Degrees robots.txt Cloud API request failed: ' . $e->getMessage()); set_transient('fiftyonedegrees_robots_cloud_error', $e->getMessage(), 60); self::record_last_refresh('error', $e->getMessage(), self::http_status_from_exception($e)); return false; } $data = json_decode($body, true); if (!is_array($data)) { $msg = 'Invalid JSON response'; error_log('51Degrees robots.txt Cloud API: ' . $msg); set_transient('fiftyonedegrees_robots_cloud_error', $msg, 60); self::record_last_refresh('error', $msg, null); return false; } // Discriminator: presence of the `robotstxt` section means the // engine ran server-side. Empty plaintext from a present section // is a legitimate "no Disallow lines" robots.txt; absence of the // section means the engine isn't advertised by the resource key. if (!isset($data['robotstxt'])) { $msg = 'Cloud response did not include robots.txt content — check resource key permissions'; error_log('51Degrees robots.txt Cloud API: ' . $msg); set_transient('fiftyonedegrees_robots_cloud_error', $msg, 60); self::record_last_refresh('error', $msg, null); return false; } $plaintext = isset($data['robotstxt']['plaintext']) ? $data['robotstxt']['plaintext'] : ''; delete_transient('fiftyonedegrees_robots_cloud_error'); update_option(Options::ROBOTS_PLAINTEXT_CACHE, $plaintext); self::record_last_refresh('success', 'Updated', null); return true; } protected static function do_cloud_request(string $url): string { $ip = isset($_SERVER['REMOTE_ADDR']) ? $_SERVER['REMOTE_ADDR'] : null; if (!empty($ip)) { $url .= (strpos($url, '?') !== false ? '&' : '?') . 'client-ip=' . urlencode($ip); } $httpClient = new FiftyOneDegreesWpHttpClient(); return $httpClient->makeCloudRequest('GET', $url, null, FiftyOneDegreesWpHttpClient::defaultOrigin()); } public static function generate_robots_txt_content($public) { $result = ''; $custom_top = get_option(Options::ROBOTS_CUSTOM_TOP, ''); $custom_bottom = get_option(Options::ROBOTS_CUSTOM_BOTTOM, ''); $cache = get_option(Options::ROBOTS_PLAINTEXT_CACHE, ''); if (!empty($custom_top)) { $result .= rtrim($custom_top) . "\n\n"; } if (!empty($cache)) { $result .= $cache; } if (!empty($custom_bottom)) { $result .= rtrim($custom_bottom) . "\n"; } return $result; } public static function generate_robots_txt($output, $public) { if (get_option(Options::ROBOTS_ENABLE, 'off') !== 'on') { return $output; } $content = self::generate_robots_txt_content($public); if (empty(trim($content)) && !$public) { return $output; } return $content; } public static function enforce_crawler_redirect() { if (get_option(Options::ROBOTS_ENFORCE, 'off') !== 'on') { return; } if (get_option(Options::PIPELINE_ENABLE, 'on') !== 'on') { return; } if (is_admin() || (function_exists('wp_doing_ajax') && wp_doing_ajax()) || (function_exists('wp_doing_cron') && wp_doing_cron())) { return; } if (php_sapi_name() === 'cli') { return; } // /robots.txt must be reachable to crawlers regardless of crawler- // category gating; redirecting it defeats the policy it advertises. // Mirrors the matching skip in suspicious-activity.php. $path = wp_parse_url(wp_unslash($_SERVER['REQUEST_URI'] ?? '/'), PHP_URL_PATH); if (is_string($path) && strtolower(rtrim($path, '/')) === '/robots.txt') { return; } $is_crawler = Pipeline::get('device', 'iscrawler'); if ($is_crawler !== true) { return; } $saved_allowed = get_option(Options::ROBOTS_ALLOWED_CATEGORIES, null); if ($saved_allowed === null || $saved_allowed === false) { return; } if (!FiftyOneDegreesCloudMetadata::supports_crawler_usage()) { return; } $crawler_usage = Pipeline::get('device', 'crawlerusage'); if (!is_array($crawler_usage) || empty($crawler_usage)) { return; } $allowed = is_array($saved_allowed) ? $saved_allowed : []; if (empty(array_diff($crawler_usage, $allowed))) { return; } $redirect_url = get_option(Options::ROBOTS_REDIRECT_URL, ''); if (!empty($redirect_url)) { $request_uri = sanitize_text_field(wp_unslash($_SERVER['REQUEST_URI'] ?? '/')); $current_url = home_url($request_uri); if (trailingslashit($current_url) === trailingslashit($redirect_url)) { return; } if (function_exists('url_to_postid')) { $target_id = url_to_postid($redirect_url); if ($target_id > 0 && url_to_postid($current_url) === $target_id) { return; } } wp_redirect($redirect_url, 302); exit; } else { nocache_headers(); $title = esc_html(FiftyOneDegreesStrings::get('robots.bot_denied.title')); $message = esc_html(FiftyOneDegreesStrings::get('robots.bot_denied.message')); echo '
' . $message . '
'; exit; } } public static function delete_options() { delete_option(Options::ROBOTS_ENABLE); delete_option(Options::ROBOTS_ENFORCE); delete_option(Options::ROBOTS_REDIRECT_URL); delete_option(Options::ROBOTS_STANDARD_TDL_SELECTED); delete_option(Options::ROBOTS_CUSTOM_TDL); delete_option(Options::ROBOTS_CUSTOM_TOP); delete_option(Options::ROBOTS_CUSTOM_BOTTOM); delete_option(Options::ROBOTS_ALLOWED_CATEGORIES); delete_option(Options::ROBOTS_PLAINTEXT_CACHE); delete_option(Options::ROBOTS_LAST_REFRESH); } }