$status, 'timestamp' => time(), 'message' => $message, 'http_status' => $http_status, ]); } private static function http_status_from_exception(\Throwable $e) { if ($e instanceof CloudRequestException) { return $e->httpStatusCode; } return null; } public static function fetch_from_cloud() { $resource_key = get_option(Options::RESOURCE_KEY, ''); if (empty($resource_key)) { return false; } $all_categories = FiftyOneDegreesCloudMetadata::fetch_crawler_usage_values(); $effective_denied = self::get_effective_denied_for_cloud($all_categories); $params = ['resource' => $resource_key]; if (!empty($all_categories)) { foreach (array_keys($all_categories) as $cat) { $val = in_array($cat, $effective_denied) ? 'disallow' : 'allow'; $params['robotstxt.' . strtolower($cat)] = $val; } } else { foreach ($effective_denied as $cat) { $params['robotstxt.' . strtolower($cat)] = 'disallow'; } } $tdl_values = self::get_effective_tdl_values(); if (!empty($tdl_values)) { $params['robotstxt.tdl'] = implode(',', $tdl_values); } try { $engine = new CloudRequestEngine(['resourceKey' => $resource_key]); $url = rtrim($engine->baseURL, '/'); if (!preg_match('#/json$#', $url)) { $url .= '/json'; } $url .= '?' . http_build_query($params); } catch (\Throwable $e) { error_log('51Degrees robots.txt Cloud API request failed: ' . $e->getMessage()); set_transient('fiftyonedegrees_robots_cloud_error', $e->getMessage(), 60); self::record_last_refresh('error', $e->getMessage(), self::http_status_from_exception($e)); return false; } try { $body = self::do_cloud_request($url); } catch (\Throwable $e) { error_log('51Degrees robots.txt Cloud API request failed: ' . $e->getMessage()); set_transient('fiftyonedegrees_robots_cloud_error', $e->getMessage(), 60); self::record_last_refresh('error', $e->getMessage(), self::http_status_from_exception($e)); return false; } $data = json_decode($body, true); if (!is_array($data)) { $msg = 'Invalid JSON response'; error_log('51Degrees robots.txt Cloud API: ' . $msg); set_transient('fiftyonedegrees_robots_cloud_error', $msg, 60); self::record_last_refresh('error', $msg, null); return false; } // Discriminator: presence of the `robotstxt` section means the // engine ran server-side. Empty plaintext from a present section // is a legitimate "no Disallow lines" robots.txt; absence of the // section means the engine isn't advertised by the resource key. if (!isset($data['robotstxt'])) { $msg = 'Cloud response did not include robots.txt content — check resource key permissions'; error_log('51Degrees robots.txt Cloud API: ' . $msg); set_transient('fiftyonedegrees_robots_cloud_error', $msg, 60); self::record_last_refresh('error', $msg, null); return false; } $plaintext = isset($data['robotstxt']['plaintext']) ? $data['robotstxt']['plaintext'] : ''; delete_transient('fiftyonedegrees_robots_cloud_error'); update_option(Options::ROBOTS_PLAINTEXT_CACHE, $plaintext); self::record_last_refresh('success', 'Updated', null); return true; } protected static function do_cloud_request(string $url): string { $ip = isset($_SERVER['REMOTE_ADDR']) ? $_SERVER['REMOTE_ADDR'] : null; if (!empty($ip)) { $url .= (strpos($url, '?') !== false ? '&' : '?') . 'client-ip=' . urlencode($ip); } $httpClient = new FiftyOneDegreesWpHttpClient(); return $httpClient->makeCloudRequest('GET', $url, null, FiftyOneDegreesWpHttpClient::defaultOrigin()); } public static function generate_robots_txt_content($public) { $result = ''; $custom_top = get_option(Options::ROBOTS_CUSTOM_TOP, ''); $custom_bottom = get_option(Options::ROBOTS_CUSTOM_BOTTOM, ''); $cache = get_option(Options::ROBOTS_PLAINTEXT_CACHE, ''); if (!empty($custom_top)) { $result .= rtrim($custom_top) . "\n\n"; } if (!empty($cache)) { $result .= $cache; } if (!empty($custom_bottom)) { $result .= rtrim($custom_bottom) . "\n"; } return $result; } public static function generate_robots_txt($output, $public) { if (get_option(Options::ROBOTS_ENABLE, 'off') !== 'on') { return $output; } $content = self::generate_robots_txt_content($public); if (empty(trim($content)) && !$public) { return $output; } return $content; } /** * Extracts the Terms Document Locator (TDL) URLs from the robots.txt * content this plugin serves. Standard TDLs are stored locally as * macros and only resolved to URLs by 51Degrees Cloud, so the served * robots.txt content (custom top + cloud plaintext + custom bottom) * is the one place the final URLs exist. One URL per "tdl:" line, * kept in order of first appearance with duplicates removed. * * @return array */ public static function get_tdl_urls_from_robots_txt() { $content = self::generate_robots_txt_content(get_option('blog_public')); $urls = []; foreach (preg_split('/\r\n|\r|\n/', $content) as $line) { if (preg_match('/^\s*tdl\s*:\s*(\S+)/i', $line, $matches)) { if (!in_array($matches[1], $urls, true)) { $urls[] = $matches[1]; } } } return $urls; } /** * Echoes one element into the HTML * section for each TDL present in the generated robots.txt, * so pages advertise the same terms documents as the robots.txt. * See https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Attributes/rel#terms-of-service * * Runs on wp_head and emits nothing when the robots.txt feature is * disabled or no TDLs are configured. */ public static function render_terms_of_service_links() { if (get_option(Options::ROBOTS_ENABLE, 'off') !== 'on') { return; } foreach (self::get_tdl_urls_from_robots_txt() as $url) { $escaped = esc_url($url); if (empty($escaped)) { continue; } echo '' . "\n"; } } public static function enforce_crawler_redirect() { if (get_option(Options::ROBOTS_ENFORCE, 'off') !== 'on') { return; } if (get_option(Options::PIPELINE_ENABLE, 'on') !== 'on') { return; } if (is_admin() || (function_exists('wp_doing_ajax') && wp_doing_ajax()) || (function_exists('wp_doing_cron') && wp_doing_cron())) { return; } if (php_sapi_name() === 'cli') { return; } // Bot-facing policy files (robots.txt, ads.txt, /.well-known/*) must // be reachable to crawlers regardless of crawler-category gating; // redirecting them defeats the policies they advertise. Mirrors the // matching skip in suspicious-activity.php. if (fiftyonedegrees_is_bot_exempt_path(wp_unslash($_SERVER['REQUEST_URI'] ?? '/'))) { return; } $is_crawler = Pipeline::get('device', 'iscrawler'); if ($is_crawler !== true) { return; } $saved_allowed = get_option(Options::ROBOTS_ALLOWED_CATEGORIES, null); if ($saved_allowed === null || $saved_allowed === false) { return; } if (!FiftyOneDegreesCloudMetadata::supports_crawler_usage()) { return; } $crawler_usage = Pipeline::get('device', 'crawlerusage'); if (!is_array($crawler_usage) || empty($crawler_usage)) { return; } $allowed = is_array($saved_allowed) ? $saved_allowed : []; if (empty(array_diff($crawler_usage, $allowed))) { return; } $redirect_url = get_option(Options::ROBOTS_REDIRECT_URL, ''); if (!empty($redirect_url)) { $request_uri = sanitize_text_field(wp_unslash($_SERVER['REQUEST_URI'] ?? '/')); $current_url = home_url($request_uri); if (trailingslashit($current_url) === trailingslashit($redirect_url)) { return; } if (function_exists('url_to_postid')) { $target_id = url_to_postid($redirect_url); if ($target_id > 0 && url_to_postid($current_url) === $target_id) { return; } } wp_redirect($redirect_url, 302); exit; } else { nocache_headers(); $title = esc_html(FiftyOneDegreesStrings::get('robots.bot_denied.title')); $message = esc_html(FiftyOneDegreesStrings::get('robots.bot_denied.message')); echo '' . $title . '' . '

' . $message . '

'; exit; } } public static function delete_options() { delete_option(Options::ROBOTS_ENABLE); delete_option(Options::ROBOTS_ENFORCE); delete_option(Options::ROBOTS_REDIRECT_URL); delete_option(Options::ROBOTS_STANDARD_TDL_SELECTED); delete_option(Options::ROBOTS_CUSTOM_TDL); delete_option(Options::ROBOTS_CUSTOM_TOP); delete_option(Options::ROBOTS_CUSTOM_BOTTOM); delete_option(Options::ROBOTS_ALLOWED_CATEGORIES); delete_option(Options::ROBOTS_PLAINTEXT_CACHE); delete_option(Options::ROBOTS_LAST_REFRESH); } }