(.*?)<\/title>/is', $html, $matches ) ) { $title = trim( html_entity_decode( $matches[1], ENT_QUOTES | ENT_HTML5, 'UTF-8' ) ); } if ( empty( $title ) ) { $title = $url; } // Clean HTML content to obtain readable text. $text = self::clean_html( $html ); return array( 'title' => $title, 'text' => $text, ); } /** * Strip out style tags, scripts, layout/headers, navigation menus and keep readable content. * * @param string $html Raw HTML content. * @return string Plain text representation. */ private static function clean_html( string $html ): string { // Strip comments. $html = preg_replace( '//ms', '', $html ); // Strip style, script and template blocks. $html = preg_replace( '/<(script|style|template|svg|noscript|iframe)[^>]*>.*?<\/\1>/is', '', $html ); // Strip headers, footers, and sidebars to target the main article/body text as much as possible. $html = preg_replace( '/<(header|footer|nav|aside)[^>]*>.*?<\/\1>/is', '', $html ); // Replace structural blocks with newlines to preserve spacing/paragraphs. $html = preg_replace( '/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/i', "\n", $html ); // Strip remaining HTML tags. $text = wp_strip_all_tags( $html ); // Decode entities. $text = html_entity_decode( $text, ENT_QUOTES | ENT_HTML5, 'UTF-8' ); // Clean up extra spaces/newlines. $text = preg_replace( "/[ \t]+/u", " ", $text ); $text = preg_replace( "/\n\s*\n+/u", "\n\n", $text ); return trim( $text ); } }

30, 'user-agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 WordPress-ai-support-assistant/' . ( defined( 'AI_SUPPORT_ASSISTANT_VERSION' ) ? AI_SUPPORT_ASSISTANT_VERSION : '1.0.1' ), ) ); if ( is_wp_error( $response ) ) { $err = $response->get_error_message(); Logger::error( "URL Crawl failed for $url: " . $err ); /* translators: %s: Crawl error details. */ throw new \Exception( esc_html( sprintf( __( 'Failed to fetch URL: %s', 'ai-support-assistant' ), $err ) ) ); } $code = wp_remote_retrieve_response_code( $response ); if ( 200 !== $code ) { Logger::error( "URL Crawl failed with HTTP $code for $url" ); /* translators: %d: HTTP status code. */ throw new \Exception( esc_html( sprintf( __( 'Failed to fetch URL, HTTP error code: %d', 'ai-support-assistant' ), $code ) ) ); } $html = wp_remote_retrieve_body( $response ); if ( empty( $html ) ) { throw new \Exception( esc_html__( 'Webpage content is empty.', 'ai-support-assistant' ) ); } // Parse Title. $title = ''; if ( preg_match( '/(.*?)<\/title>/is', $html, $matches ) ) { $title = trim( html_entity_decode( $matches[1], ENT_QUOTES | ENT_HTML5, 'UTF-8' ) ); } if ( empty( $title ) ) { $title = $url; } // Clean HTML content to obtain readable text. $text = self::clean_html( $html ); return array( 'title' => $title, 'text' => $text, ); } /** * Strip out style tags, scripts, layout/headers, navigation menus and keep readable content. * * @param string $html Raw HTML content. * @return string Plain text representation. */ private static function clean_html( string $html ): string { // Strip comments. $html = preg_replace( '//ms', '', $html ); // Strip style, script and template blocks. $html = preg_replace( '/<(script|style|template|svg|noscript|iframe)[^>]*>.*?<\/\1>/is', '', $html ); // Strip headers, footers, and sidebars to target the main article/body text as much as possible. $html = preg_replace( '/<(header|footer|nav|aside)[^>]*>.*?<\/\1>/is', '', $html ); // Replace structural blocks with newlines to preserve spacing/paragraphs. $html = preg_replace( '/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/i', "\n", $html ); // Strip remaining HTML tags. $text = wp_strip_all_tags( $html ); // Decode entities. $text = html_entity_decode( $text, ENT_QUOTES | ENT_HTML5, 'UTF-8' ); // Clean up extra spaces/newlines. $text = preg_replace( "/[ \t]+/u", " ", $text ); $text = preg_replace( "/\n\s*\n+/u", "\n\n", $text ); return trim( $text ); } }