$headers */ private function max_body_sample_bytes(string $url, array $headers): int { $ct = ''; if (isset($headers['content-type'])) { $ct = is_array($headers['content-type']) ? (string) end($headers['content-type']) : (string) $headers['content-type']; } $ct_l = strtolower($ct); $u_l = strtolower($url); $looks_xml = ( strpos($ct_l, 'xml') !== false || strpos($ct_l, 'rss') !== false || strpos($u_l, 'sitemap') !== false || strpos($u_l, 'robots.txt') !== false || preg_match('~\.(xml|rss|atom)(\.gz)?$~', $u_l) ); // Defaults (can be overridden by filters). $xml_max = (int) apply_filters('aetos_seo_http_max_body_xml', 20 * 1024 * 1024, $url, $headers); $html_max = (int) apply_filters('aetos_seo_http_max_body_html', 1 * 1024 * 1024, $url, $headers); $max = $looks_xml ? $xml_max : $html_max; // Safety clamps. $max = max(20000, min(50 * 1024 * 1024, $max)); return $max; } /** * Best-effort gzip decode for sitemap/robots bodies. */ private function maybe_gzdecode(string $body): string { if ($body === '') { return ''; } // GZIP magic header: 1F 8B if (strlen($body) >= 2 && ord($body[0]) === 0x1f && ord($body[1]) === 0x8b) { if (function_exists('gzdecode')) { $decoded = @gzdecode($body); if (is_string($decoded) && $decoded !== '') { return $decoded; } } } return $body; } /** * Fetch URL and follow its redirect chain manually, checking every hop * against the same-site guarantee (no auto-follow, no off-site fetch). * * @param array $args * @return array */ public function request(string $url, array $args = [], int $max_redirects = 5): array { $max_redirects = max(0, min(10, $max_redirects)); $ua = \AetosSeo\Utils\Http::user_agent(); $defaults = [ 'timeout' => 10, 'redirection' => 0, 'user-agent' => $ua, // Tag our own requests as a first-party internal audit request. 'aetos_seo' => 1, 'headers' => [ 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', ], ]; $args = array_merge($defaults, $args); // v3.3.52+ — this collector follows redirects MANUALLY in the loop below // so it can enforce the same-site guarantee on every hop. WP's own // auto-follow must stay disabled regardless of what the caller passed, // otherwise an external redirect would be fetched before we inspect it. $args['redirection'] = 0; $chain = []; $current = $url; $final = $url; $status = 0; $headers = []; $body_sample = ''; for ($i = 0; $i <= $max_redirects; $i++) { if (!Url::is_internal_url($current)) { $chain[] = ['url' => $current, 'error' => 'external_url_not_followed']; break; } // Best-effort response size cap (prevents huge responses from filling memory). // We guess based on URL first; after we get headers we will cap reading to the final max bytes. $limit_guess = $this->max_body_sample_bytes($current, []); $req_args = $args; $req_args['limit_response_size'] = $limit_guess; // Stream to a temp file to avoid loading large bodies into memory. $tmp_file = ''; if (function_exists('wp_tempnam')) { $tmp = wp_tempnam($current); if (is_string($tmp) && $tmp !== '') { $tmp_file = $tmp; $req_args['stream'] = true; $req_args['filename'] = $tmp_file; } } $resp = wp_remote_get($current, $req_args); if (is_wp_error($resp)) { $status = 0; $headers = []; $body_sample = ''; if ($tmp_file !== '' && file_exists($tmp_file)) { wp_delete_file($tmp_file); } $chain[] = [ 'url' => $current, 'error' => $resp->get_error_message(), ]; $final = $current; break; } $status = (int) wp_remote_retrieve_response_code($resp); $headers_obj = wp_remote_retrieve_headers($resp); $headers = []; if (is_object($headers_obj) && method_exists($headers_obj, 'getAll')) { $headers = (array) $headers_obj->getAll(); } elseif (is_array($headers_obj)) { $headers = $headers_obj; } $location = ''; if (isset($headers['location'])) { $location = is_array($headers['location']) ? (string) end($headers['location']) : (string) $headers['location']; } $chain[] = [ 'url' => $current, 'status' => $status, 'location' => $location, ]; $is_redirect = in_array($status, [301,302,303,307,308], true) && $location !== ''; // Only capture body sample for the final response (or non-redirect). if (!$is_redirect) { $max_bytes = $this->max_body_sample_bytes($current, $headers); $max_read = min($limit_guess, $max_bytes); $max_read = max(0, (int) $max_read); if ($tmp_file !== '' && file_exists($tmp_file)) { $body_sample = $this->read_body_sample_from_file($tmp_file, $max_read); wp_delete_file($tmp_file); } else { $body = (string) wp_remote_retrieve_body($resp); if ($body !== '') { if (strlen($body) > $max_read) { $body = substr($body, 0, $max_read); } $body2 = $this->maybe_gzdecode($body); $body_sample = (strlen($body2) > $max_read) ? substr($body2, 0, $max_read) : $body2; } } } else { if ($tmp_file !== '' && file_exists($tmp_file)) { wp_delete_file($tmp_file); } } if ($is_redirect) { $next = Url::resolve($current, $location); // v3.3.52+ — same-site guarantee. The free build has no separate // outbound-request guard, so the collector itself must ensure an // audit never leaves this site: a redirect that points to an // external host is recorded and the chain stops here — never fetched. if (!Url::is_internal_url($next)) { $chain[] = [ 'url' => $next, 'error' => 'external_redirect_not_followed', ]; $final = $current; break; } if ($i === $max_redirects) { $chain[] = ['url' => $next, 'error' => 'redirect_limit_reached']; $final = $current; break; } $current = $next; $final = $current; continue; } $final = $current; break; } return [ 'requested_url' => $url, 'final_url' => $final, 'status_code' => $status, 'headers' => $headers, 'redirect_chain' => $chain, 'body_sample' => $body_sample, ]; } /** * Read up to $max_bytes from a streamed response file. * If the file is a gzip file (e.g. sitemap.xml.gz), this will attempt to read/decompress safely. */ private function read_body_sample_from_file(string $filepath, int $max_bytes): string { $max_bytes = max(0, $max_bytes); if ($max_bytes <= 0 || !file_exists($filepath)) { return ''; } // phpcs:disable WordPress.WP.AlternativeFunctions.file_system_operations_fopen,WordPress.WP.AlternativeFunctions.file_system_operations_fread,WordPress.WP.AlternativeFunctions.file_system_operations_fclose -- Reading the first 2 bytes of a plugin-owned temp file (from wp_remote_get stream:true) to detect a gzip magic number; chunked binary peek that WP_Filesystem::get_contents cannot do cheaply. $fh = @fopen($filepath, 'rb'); if (!$fh) { return ''; } $magic = fread($fh, 2); fclose($fh); // phpcs:enable WordPress.WP.AlternativeFunctions.file_system_operations_fopen,WordPress.WP.AlternativeFunctions.file_system_operations_fread,WordPress.WP.AlternativeFunctions.file_system_operations_fclose $is_gzip_file = ($magic === "\x1f\x8b"); // If this is a .gz file, read a *decompressed* sample without loading everything into memory. if ($is_gzip_file && function_exists('gzopen')) { $gz = @gzopen($filepath, 'rb'); if (!$gz) { return ''; } $out = ''; while (!gzeof($gz) && strlen($out) < $max_bytes) { $need = $max_bytes - strlen($out); $chunk = gzread($gz, min(8192, $need)); if ($chunk === false) { break; } $out .= $chunk; } gzclose($gz); return $out; } // Normal file: read a raw sample. // phpcs:disable WordPress.WP.AlternativeFunctions.file_system_operations_fopen,WordPress.WP.AlternativeFunctions.file_system_operations_fread,WordPress.WP.AlternativeFunctions.file_system_operations_fclose -- Streamed chunked read (cap $max_bytes) of a plugin-owned temp file fetched via wp_remote_get; avoids loading huge response bodies into memory. WP_Filesystem::get_contents has no bounded-read primitive. $fh = @fopen($filepath, 'rb'); if (!$fh) { return ''; } $out = ''; while (!feof($fh) && strlen($out) < $max_bytes) { $need = $max_bytes - strlen($out); $chunk = fread($fh, min(8192, $need)); if ($chunk === false) { break; } $out .= $chunk; } fclose($fh); // phpcs:enable WordPress.WP.AlternativeFunctions.file_system_operations_fopen,WordPress.WP.AlternativeFunctions.file_system_operations_fread,WordPress.WP.AlternativeFunctions.file_system_operations_fclose // If it looks gzip but we couldn't gzopen, try best-effort decode. if ($is_gzip_file) { $out2 = $this->maybe_gzdecode($out); return $out2; } return $out; } }