*/ private $maxBytesByMime; /** * Decoded body size cap used for an accepted MIME type with no entry in * {@see $maxBytesByMime}. * * @var int */ private $defaultMaxBytes; /** * Transport-level `limit_response_size` used for a hop before its * response's actual MIME type is known: one byte past the SMALLEST cap * this fetcher accepts, never the largest. Sizing this to the largest * configured per-MIME cap would let a response that merely claims to be * a small-cap type (e.g. `text/html`) buffer all the way up to a much * larger cap meant for a different, deliberately-larger-cap type (e.g. a * PDF) before the per-type check in {@see fetch()} ever runs — a * bandwidth/memory amplification a hostile or compromised source could * trigger on every fetch. {@see fetch()} re-requests a hop with the * type's own larger budget only once that type is confirmed from the * response, so a legitimately larger-cap type is never truncated either. * * @var int */ private $transportMaxBytes; /** * Injectable HTTP transport: fn( string $url, array $wpArgs ): array. * * Real implementation performs `wp_remote_get()`; see {@see defaultHttpRequest()}. * * @var callable */ private $http; /** * Injectable transport-availability check: fn(): bool. * * @var callable */ private $transportCheck; /** * Constructor. * * @param string[] $allowedMimeTypes Content types this fetcher accepts, Accept-header order. * @param array $maxBytesByMime Per-MIME decoded body size cap; see {@see $maxBytesByMime}. * @param int $defaultMaxBytes Fallback decoded body size cap for an accepted MIME with no entry above. * @param UrlSafetyValidator|null $validator Validator override for tests. * @param callable|null $http HTTP transport override for tests. * @param callable|null $transportCheck cURL-availability check override for tests. * * @throws \InvalidArgumentException When $allowedMimeTypes is empty: a fetcher that can accept * nothing is a caller misconfiguration, not a runtime input. */ public function __construct( array $allowedMimeTypes, array $maxBytesByMime, int $defaultMaxBytes, ?UrlSafetyValidator $validator = null, ?callable $http = null, ?callable $transportCheck = null ) { if ( empty( $allowedMimeTypes ) ) { throw new \InvalidArgumentException( 'SafeUrlFetcher requires at least one allowed MIME type.' ); } $this->allowedMimeTypes = array_values( $allowedMimeTypes ); $this->maxBytesByMime = $maxBytesByMime; $this->defaultMaxBytes = $defaultMaxBytes; $this->transportMaxBytes = min( array_merge( array_values( $maxBytesByMime ), array( $defaultMaxBytes ) ) ) + 1; $this->validator = $validator ?? new UrlSafetyValidator(); $this->http = $http ?? array( $this, 'defaultHttpRequest' ); $this->transportCheck = $transportCheck ?? array( $this, 'defaultTransportCheck' ); } /** * Fetch a URL through the safe, bounded, re-validated redirect loop. * * @param string $url Candidate URL. * @param array $opts { * @type int $timeout Per-hop timeout in seconds (default 10). * @type int $maxRedirects Maximum redirect hops (default 3); 0 means a 3xx * response is always a failure (`too_many_redirects`) * and is never followed with a second request. * @type string[] $allowedHosts Host allowlist re-checked on every hop (empty = any public host). * @type bool $httpsOnly When true, an `http` URL is refused (`https_required`) * before any network call, on the initial URL and on * every redirect hop alike. * } * @return array{ok:bool,status:string,httpStatus:int,finalUrl:string,contentType:string,body:string,bytes:int,redirects:int,errorCode:string,errorMessage:string} */ public function fetch( string $url, array $opts = array() ): array { if ( ! $this->transportIsSafe() ) { return $this->failure( 'blocked', 0, $url, '', 0, __( 'Secure page reading is not available on this server (cURL is required).', 'ai-product-tools' ), 'transport_unavailable' ); } $maxRedirects = isset( $opts['maxRedirects'] ) ? max( 0, (int) $opts['maxRedirects'] ) : self::MAX_REDIRECTS; $allowedHosts = ( isset( $opts['allowedHosts'] ) && is_array( $opts['allowedHosts'] ) ) ? $opts['allowedHosts'] : array(); $timeout = isset( $opts['timeout'] ) ? max( 1, (int) $opts['timeout'] ) : self::DEFAULT_TIMEOUT; $httpsOnly = ! empty( $opts['httpsOnly'] ); $currentUrl = $url; $redirects = 0; while ( true ) { $validation = $this->validator->validate( $currentUrl, $allowedHosts, $httpsOnly ); if ( ! $validation['ok'] ) { return $this->failure( $this->statusForValidationCode( $validation['code'] ), 0, $currentUrl, '', 0, (string) $validation['message'], (string) $validation['code'], $redirects ); } $response = $this->requestWithPin( $validation, $timeout ); if ( ! empty( $response['error'] ) ) { return $this->failure( 'unreachable', 0, $currentUrl, '', 0, (string) ( $response['errorMessage'] ?? '' ), (string) ( $response['errorCode'] ?? 'request_failed' ), $redirects ); } $httpStatus = (int) ( $response['status'] ?? 0 ); $location = (string) ( $response['location'] ?? '' ); if ( in_array( $httpStatus, self::REDIRECT_STATUSES, true ) && '' !== $location ) { ++$redirects; if ( $redirects > $maxRedirects ) { return $this->failure( 'error', $httpStatus, $currentUrl, '', 0, __( 'Too many redirects.', 'ai-product-tools' ), 'too_many_redirects', $redirects ); } $currentUrl = $this->resolveRedirectUrl( $validation['normalizedUrl'], $location ); continue; } if ( $httpStatus < 200 || $httpStatus >= 300 ) { return $this->failure( 'unreachable', $httpStatus, $currentUrl, '', 0, sprintf( /* translators: %d: HTTP status code. */ __( 'The page responded with HTTP %d.', 'ai-product-tools' ), $httpStatus ), 'http_error', $redirects ); } $contentType = $this->extractMime( (string) ( $response['contentType'] ?? '' ) ); $body = (string) ( $response['body'] ?? '' ); $bytes = strlen( $body ); if ( ! in_array( $contentType, $this->allowedMimeTypes, true ) ) { return $this->failure( 'unsupported', $httpStatus, $currentUrl, $contentType, $bytes, __( 'This content type is not supported.', 'ai-product-tools' ), 'unsupported_mime', $redirects ); } $cap = $this->maxBytesByMime[ $contentType ] ?? $this->defaultMaxBytes; if ( $cap >= $this->transportMaxBytes && $bytes >= $this->transportMaxBytes ) { $response = $this->requestWithPin( $validation, $timeout, $cap + 1 ); if ( ! empty( $response['error'] ) ) { return $this->failure( 'unreachable', 0, $currentUrl, '', 0, (string) ( $response['errorMessage'] ?? '' ), (string) ( $response['errorCode'] ?? 'request_failed' ), $redirects ); } $httpStatus = (int) ( $response['status'] ?? 0 ); if ( in_array( $httpStatus, self::REDIRECT_STATUSES, true ) && '' !== (string) ( $response['location'] ?? '' ) ) { return $this->failure( 'error', $httpStatus, $currentUrl, '', 0, __( 'The page changed while it was being read.', 'ai-product-tools' ), 'unstable_response', $redirects ); } if ( $httpStatus < 200 || $httpStatus >= 300 ) { return $this->failure( 'unreachable', $httpStatus, $currentUrl, '', 0, sprintf( /* translators: %d: HTTP status code. */ __( 'The page responded with HTTP %d.', 'ai-product-tools' ), $httpStatus ), 'http_error', $redirects ); } $contentType = $this->extractMime( (string) ( $response['contentType'] ?? '' ) ); $body = (string) ( $response['body'] ?? '' ); $bytes = strlen( $body ); if ( ! in_array( $contentType, $this->allowedMimeTypes, true ) ) { return $this->failure( 'unsupported', $httpStatus, $currentUrl, $contentType, $bytes, __( 'This content type is not supported.', 'ai-product-tools' ), 'unsupported_mime', $redirects ); } $cap = $this->maxBytesByMime[ $contentType ] ?? $this->defaultMaxBytes; } if ( $bytes > $cap ) { return $this->failure( 'too_large', $httpStatus, $currentUrl, $contentType, $bytes, __( 'The page is too large to read.', 'ai-product-tools' ), 'too_large', $redirects ); } return array( 'ok' => true, 'status' => 'ready', 'httpStatus' => $httpStatus, 'finalUrl' => $currentUrl, 'contentType' => $contentType, 'body' => $body, 'bytes' => $bytes, 'redirects' => $redirects, 'errorCode' => '', 'errorMessage' => '', ); } } /** * Perform one hop's request with a request-scoped `CURLOPT_RESOLVE` pin. * * The pin is added immediately before the request and removed in * `finally`, so a pin from an earlier hop can never apply to this one. * * @param array{host:string,port:int,ips:string[],normalizedUrl:string} $validation Validated hop target. * @param int $timeout Timeout in seconds. * @param int|null $maxBytesOverride Transport byte cap for this one request; defaults to {@see $transportMaxBytes} (the conservative, type-unaware budget) when omitted. * @return array */ private function requestWithPin( array $validation, int $timeout, ?int $maxBytesOverride = null ): array { $entry = sprintf( '%s:%d:%s', $validation['host'], $validation['port'], implode( ',', $validation['ips'] ) ); $callback = $this->buildPinCallback( $entry ); $pinned = false; if ( function_exists( 'add_action' ) ) { \add_action( 'http_api_curl', $callback, 10, 1 ); $pinned = true; } try { $wpArgs = array( 'timeout' => $timeout, 'redirection' => 0, 'limit_response_size' => $maxBytesOverride ?? $this->transportMaxBytes, 'sslverify' => true, 'reject_unsafe_urls' => true, 'cookies' => array(), 'headers' => array( 'Accept' => $this->acceptHeader() ), 'user-agent' => $this->userAgent(), ); return (array) call_user_func( $this->http, $validation['normalizedUrl'], $wpArgs ); } finally { if ( $pinned && function_exists( 'remove_action' ) ) { \remove_action( 'http_api_curl', $callback, 10 ); } } } /** * Build the `http_api_curl` action callback that pins curl's own DNS * resolution to the address we already validated, closing the gap * between {@see UrlSafetyValidator::validate()} and the actual connect. * * @param string $entry "host:port:ip[,ip...]" CURLOPT_RESOLVE entry. * @return callable */ private function buildPinCallback( string $entry ): callable { return static function ( $handle ) use ( $entry ) { if ( function_exists( 'curl_setopt' ) ) { curl_setopt( $handle, CURLOPT_RESOLVE, array( $entry ) ); } }; } /** * Default `$http` implementation: a real `wp_remote_get()` call. * * @param string $url Final, validated URL for this hop. * @param array $wpArgs WP HTTP API arguments. * @return array */ private function defaultHttpRequest( string $url, array $wpArgs ): array { if ( ! function_exists( 'wp_remote_get' ) ) { return array( 'error' => true, 'errorCode' => 'transport_unavailable', 'errorMessage' => __( 'The HTTP transport is unavailable.', 'ai-product-tools' ), ); } $response = \wp_remote_get( $url, $wpArgs ); if ( $response instanceof \WP_Error ) { return array( 'error' => true, 'errorCode' => 'request_failed', 'errorMessage' => $response->get_error_message(), ); } return array( 'error' => false, 'status' => (int) \wp_remote_retrieve_response_code( $response ), 'location' => (string) \wp_remote_retrieve_header( $response, 'location' ), 'contentType' => (string) \wp_remote_retrieve_header( $response, 'content-type' ), 'body' => (string) \wp_remote_retrieve_body( $response ), ); } /** * Whether WordPress will use the cURL transport for the next request. * * Fails closed: no cURL extension, and no known Requests cURL transport * class to confirm with, both mean "not safe to fetch". * * @return bool */ private function transportIsSafe(): bool { return (bool) call_user_func( $this->transportCheck ); } /** * Default transport-availability check. * * @return bool */ private function defaultTransportCheck(): bool { if ( ! function_exists( 'curl_init' ) ) { return false; } if ( class_exists( '\WpOrg\Requests\Transport\Curl' ) ) { return (bool) \WpOrg\Requests\Transport\Curl::test(); } if ( class_exists( '\Requests_Transport_cURL' ) ) { return (bool) \Requests_Transport_cURL::test(); } return false; } /** * Resolve a (possibly relative) redirect Location header against the hop it came from. * * @param string $baseUrl The URL that produced this redirect. * @param string $location Raw Location header value. * @return string Absolute URL. */ private function resolveRedirectUrl( string $baseUrl, string $location ): string { $location = trim( $location ); if ( '' === $location ) { return $baseUrl; } if ( preg_match( '#^[a-zA-Z][a-zA-Z0-9+.\-]*://#', $location ) ) { return $location; } $base = parse_url( $baseUrl ); if ( ! is_array( $base ) || empty( $base['scheme'] ) || empty( $base['host'] ) ) { return $location; } $authority = $base['scheme'] . '://' . $base['host'] . ( isset( $base['port'] ) ? ':' . $base['port'] : '' ); if ( isset( $location[0] ) && '/' === $location[0] ) { if ( isset( $location[1] ) && '/' === $location[1] ) { return $base['scheme'] . ':' . $location; } return $authority . $location; } $basePath = isset( $base['path'] ) ? (string) $base['path'] : '/'; $slashPos = strrpos( $basePath, '/' ); $dir = false !== $slashPos ? substr( $basePath, 0, $slashPos + 1 ) : '/'; return $authority . $dir . $location; } /** * Map a {@see UrlSafetyValidator} failure code to a source status. * * @param string $code Validator failure code. * @return string */ private function statusForValidationCode( string $code ): string { switch ( $code ) { case 'private_ip': case 'host_not_allowed': case 'invalid_scheme': case 'invalid_port': case 'userinfo_not_allowed': case 'https_required': return 'blocked'; case 'dns_resolution_failed': return 'unreachable'; default: return 'error'; } } /** * Strip parameters (e.g. "; charset=UTF-8") from a Content-Type header value. * * @param string $header Raw Content-Type header value. * @return string Lowercase MIME type only. */ private function extractMime( string $header ): string { $mime = strtolower( trim( $header ) ); $semi = strpos( $mime, ';' ); if ( false !== $semi ) { $mime = trim( substr( $mime, 0, $semi ) ); } return $mime; } /** * Build a consistent failure result. * * @param string $status Source-status-shaped outcome. * @param int $httpStatus HTTP status code, or 0 when none was received. * @param string $finalUrl URL that produced this failure. * @param string $contentType Content type, when known. * @param int $bytes Byte count, when known. * @param string $message User-safe message. * @param string $code Internal error code. * @param int $redirects Redirect hops followed so far. * @return array{ok:bool,status:string,httpStatus:int,finalUrl:string,contentType:string,body:string,bytes:int,redirects:int,errorCode:string,errorMessage:string} */ private function failure( string $status, int $httpStatus, string $finalUrl, string $contentType, int $bytes, string $message, string $code, int $redirects = 0 ): array { return array( 'ok' => false, 'status' => $status, 'httpStatus' => $httpStatus, 'finalUrl' => $finalUrl, 'contentType' => $contentType, 'body' => '', 'bytes' => $bytes, 'redirects' => $redirects, 'errorCode' => $code, 'errorMessage' => $message, ); } /** * Accept header sent on every request, built from the accepted MIME list * so it always matches what {@see fetch()} will actually allow through. * * @return string */ private function acceptHeader(): string { return implode( ',', $this->allowedMimeTypes ); } /** * User agent prefix this fetcher identifies itself with. A subclass that * needs its own identity (e.g. the content-structure source reader) * overrides this. * * @return string */ protected function userAgentPrefix(): string { return self::DEFAULT_USER_AGENT_PREFIX; } /** * Build the fetcher's user agent string. * * @return string */ private function userAgent(): string { $version = defined( 'AIPT_VERSION' ) ? AIPT_VERSION : '0'; return $this->userAgentPrefix() . $version; } }