*/ private const KEYWORD_TYPES = array( 'features' => array( 'feature' ), 'benefits' => array( 'benefit' ), 'specs' => array( 'specification', 'spec', 'technical detail', 'tech detail' ), 'warranty' => array( 'warranty', 'guarantee' ), 'care' => array( 'how to use', 'maintenance', 'care' ), 'compatibility' => array( 'compatib' ), 'shipping_returns' => array( 'shipping', 'delivery', 'return' ), 'use_cases' => array( 'use case', 'uses' ), 'summary' => array( 'summary', 'conclusion' ), ); /** * Needles short enough to collide with unrelated words as a bare * substring (e.g. "spec" inside "special", "uses" inside "excuses") and * so are matched as a whole word instead. Every other needle keeps the * deliberate partial-word match (e.g. "feature" matching "Features", * "compatib" matching "Compatibility"). * * @var string[] */ private const WORD_BOUNDARY_NEEDLES = array( 'spec', 'uses' ); /** * Detect a draft section list from one example description. * * @param string $raw_input The example text, HTML or plain text. * @param string $product_name The example product's own name, when * known (used only to keep a verbatim * closing paragraph from being proposed as * fixed text when it names that product). * @return array{sections: array>, notices: array} */ public function detect( string $raw_input, string $product_name = '' ): array { $notices = array(); if ( '' === trim( $raw_input ) ) { return array( 'sections' => array(), 'notices' => $notices ); } if ( strlen( $raw_input ) > self::MAX_INPUT_BYTES ) { $raw_input = substr( $raw_input, 0, self::MAX_INPUT_BYTES ); } $is_html = 1 === preg_match( '/<[a-z][^>]*>/i', $raw_input ); $blocks = $is_html ? $this->parseHtmlBlocks( $raw_input ) : $this->parsePlainTextBlocks( $raw_input ); $blocks = $this->mergeLabelValueRuns( $blocks ); if ( count( $blocks ) > self::MAX_BLOCKS ) { $blocks = array_slice( $blocks, 0, self::MAX_BLOCKS ); } $content_blocks = $this->associateHeadings( $blocks ); $sections = $this->buildSections( $content_blocks, $product_name ); if ( count( $sections ) > DefinitionPrimitives::SECTIONS_MAX ) { $sections = array_slice( $sections, 0, DefinitionPrimitives::SECTIONS_MAX ); $notices[] = DefinitionPrimitives::issue( 'structure.sections', 'example_truncated', __( 'This example produced more sections than a template can hold, so only the first ones were kept.', 'ai-product-tools' ) ); } return array( 'sections' => $sections, 'notices' => $notices ); } /** * @param string $html Raw HTML. * @return array> Ordered blocks. */ private function parseHtmlBlocks( string $html ): array { $html = (string) preg_replace( '/^\s*(?:<\?xml[^>]*\?>\s*)?(?:]|\[[^\]]*\])*>)?/i', '', $html ); $doc = new \DOMDocument(); $prior_setting = libxml_use_internal_errors( true ); $loaded = $doc->loadHTML( '' . $html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING ); libxml_clear_errors(); libxml_use_internal_errors( $prior_setting ); if ( ! $loaded ) { return array(); } $this->removeUnwantedElements( $doc ); $body = $doc->getElementsByTagName( 'body' )->item( 0 ); $root = $body ?? $doc->documentElement; $blocks = array(); if ( null !== $root ) { $this->collectOrderedBlocks( $root, $blocks ); } return $blocks; } /** * @param \DOMDocument $doc Loaded document. * @return void */ private function removeUnwantedElements( \DOMDocument $doc ): void { foreach ( self::REMOVED_TAGS as $tag ) { $nodes = $doc->getElementsByTagName( $tag ); for ( $i = $nodes->length - 1; $i >= 0; $i-- ) { $node = $nodes->item( $i ); if ( null !== $node && null !== $node->parentNode ) { $node->parentNode->removeChild( $node ); } } } } /** * Walk $node's children in document order, turning headings, paragraphs, * lists and tables/definition lists into one flat, ordered block list. * Every other element (div, section, span, …) is recursed into so its * own children stay in the same overall order. * * @param \DOMNode $node Current node. * @param array> &$blocks Accumulator, by reference. * @return void */ private function collectOrderedBlocks( \DOMNode $node, array &$blocks ): void { foreach ( $node->childNodes as $child ) { if ( count( $blocks ) >= self::MAX_BLOCKS ) { return; } if ( XML_ELEMENT_NODE !== $child->nodeType ) { continue; } $tag = strtolower( $child->nodeName ); if ( in_array( $tag, array( 'h2', 'h3', 'h4' ), true ) ) { $text = $this->collapseWhitespace( $child->textContent ); if ( '' !== $text ) { $blocks[] = array( 'kind' => 'heading', 'text' => $text ); } continue; } if ( in_array( $tag, array( 'h1', 'h5', 'h6' ), true ) ) { continue; } if ( 'p' === $tag ) { $this->appendParagraphOrSpecsBlock( $child, $blocks ); continue; } if ( in_array( $tag, array( 'ul', 'ol' ), true ) ) { $this->appendListBlock( $tag, $child, $blocks ); continue; } if ( 'table' === $tag ) { $rows = $this->tableRows( $child ); if ( ! empty( $rows ) ) { $blocks[] = array( 'kind' => 'specs', 'rows' => $rows ); } continue; } if ( 'dl' === $tag ) { $rows = $this->definitionListRows( $child ); if ( ! empty( $rows ) ) { $blocks[] = array( 'kind' => 'specs', 'rows' => $rows ); } continue; } $this->collectOrderedBlocks( $child, $blocks ); } } /** * A `

` normally becomes one paragraph block, but a `

` whose * `
`-separated lines are each "Label: value" shaped (a common way to * fake a spec table without a real ``) becomes a specs block * instead, exactly like a real table would. * * @param \DOMNode $node The `

` element. * @param array> &$blocks Accumulator, by reference. * @return void */ private function appendParagraphOrSpecsBlock( \DOMNode $node, array &$blocks ): void { $raw = $this->nodeTextWithBreaks( $node ); $lines = array_values( array_filter( array_map( array( $this, 'collapseWhitespace' ), preg_split( '/\n+/', $raw ) ), static function ( string $line ): bool { return '' !== $line; } ) ); if ( empty( $lines ) ) { return; } $rows = $this->labelValueRows( $lines ); if ( count( $rows ) >= 2 && count( $rows ) === count( $lines ) ) { $blocks[] = array( 'kind' => 'specs', 'rows' => $rows ); return; } $text = $this->collapseWhitespace( implode( ' ', $lines ) ); if ( '' !== $text ) { $blocks[] = array( 'kind' => 'paragraph', 'text' => $text ); } } /** * @param string $tag 'ul' | 'ol'. * @param \DOMNode $node The list element. * @param array> &$blocks Accumulator, by reference. * @return void */ private function appendListBlock( string $tag, \DOMNode $node, array &$blocks ): void { $items = array(); foreach ( $node->childNodes as $li ) { if ( XML_ELEMENT_NODE === $li->nodeType && 'li' === strtolower( $li->nodeName ) ) { $text = $this->collapseWhitespace( $li->textContent ); if ( '' !== $text ) { $items[] = $text; } } } if ( ! empty( $items ) ) { $blocks[] = array( 'kind' => 'list', 'ordered' => ( 'ol' === $tag ), 'items' => $items ); } } /** * @param \DOMNode $table Table element. * @return array */ private function tableRows( \DOMNode $table ): array { $rows = array(); foreach ( $table->getElementsByTagName( 'tr' ) as $tr ) { $cells = array(); foreach ( $tr->childNodes as $cell ) { if ( XML_ELEMENT_NODE === $cell->nodeType && in_array( strtolower( $cell->nodeName ), array( 'th', 'td' ), true ) ) { $cells[] = $cell; } } if ( count( $cells ) >= 2 ) { $label = $this->collapseWhitespace( $cells[0]->textContent ); $value = $this->collapseWhitespace( $cells[1]->textContent ); if ( '' !== $label && '' !== $value ) { $rows[] = array( 'label' => $label, 'value' => $value ); } } } return $rows; } /** * @param \DOMNode $dl Definition list element. * @return array */ private function definitionListRows( \DOMNode $dl ): array { $rows = array(); $pending_label = null; foreach ( $dl->childNodes as $child ) { if ( XML_ELEMENT_NODE !== $child->nodeType ) { continue; } $tag = strtolower( $child->nodeName ); if ( 'dt' === $tag ) { $pending_label = $this->collapseWhitespace( $child->textContent ); continue; } if ( 'dd' === $tag && null !== $pending_label ) { $value = $this->collapseWhitespace( $child->textContent ); if ( '' !== $pending_label && '' !== $value ) { $rows[] = array( 'label' => $pending_label, 'value' => $value ); } $pending_label = null; } } return $rows; } /** * Node text with `
` rendered as a newline (HtmlExtractor's own * `nodeTextWithBreaks()` pattern), so "Label: value" lines separated by * `
` inside one `

` are still individually detectable. * * @param \DOMNode $node Node to read. * @return string */ private function nodeTextWithBreaks( \DOMNode $node ): string { $out = ''; foreach ( $node->childNodes as $child ) { if ( XML_ELEMENT_NODE === $child->nodeType && 'br' === strtolower( $child->nodeName ) ) { $out .= "\n"; continue; } if ( XML_TEXT_NODE === $child->nodeType || XML_CDATA_SECTION_NODE === $child->nodeType ) { $out .= $child->nodeValue; continue; } if ( XML_ELEMENT_NODE === $child->nodeType ) { $out .= $this->nodeTextWithBreaks( $child ); } } return $out; } /** * Plain text has no heading concept: blank lines, bullet markers * (-, *, •, "1.") and "Label: value" lines are read the same way HTML's * lists/specs are, but no block ever becomes a `heading` kind. * * @param string $text Raw plain text. * @return array> Ordered blocks. */ private function parsePlainTextBlocks( string $text ): array { $text = str_replace( array( "\r\n", "\r" ), "\n", $text ); $chunks = preg_split( '/\n[ \t]*\n+/', $text ); $blocks = array(); foreach ( (array) $chunks as $chunk ) { if ( count( $blocks ) >= self::MAX_BLOCKS ) { break; } $lines = array_values( array_filter( array_map( 'trim', explode( "\n", $chunk ) ), static function ( string $line ): bool { return '' !== $line; } ) ); if ( empty( $lines ) ) { continue; } $block = $this->classifyPlainTextChunk( $lines ); if ( null !== $block ) { $blocks[] = $block; } } return $blocks; } /** * @param string[] $lines Non-empty, trimmed lines of one blank-line-separated chunk. * @return array|null */ private function classifyPlainTextChunk( array $lines ): ?array { $bullet_items = array(); $all_bulleted = true; $any_numbered = false; foreach ( $lines as $line ) { if ( preg_match( '/^(?:[-*\x{2022}]|\d+\.)\s+(.+)$/u', $line, $m ) ) { $bullet_items[] = trim( $m[1] ); if ( preg_match( '/^\d+\./', $line ) ) { $any_numbered = true; } } else { $all_bulleted = false; } } if ( $all_bulleted && ! empty( $bullet_items ) ) { return array( 'kind' => 'list', 'ordered' => $any_numbered, 'items' => $bullet_items ); } $rows = $this->labelValueRows( $lines ); if ( count( $rows ) >= 2 && count( $rows ) === count( $lines ) ) { return array( 'kind' => 'specs', 'rows' => $rows ); } $text = $this->collapseWhitespace( implode( ' ', $lines ) ); return '' !== $text ? array( 'kind' => 'paragraph', 'text' => $text ) : null; } /** * @param string[] $lines Candidate lines. * @return array Only the lines that matched; caller compares the count back against `count($lines)`. */ private function labelValueRows( array $lines ): array { $rows = array(); foreach ( $lines as $line ) { if ( preg_match( '/^([^:]{1,60}):\s*(.{1,200})$/u', $line, $m ) ) { $label = trim( $m[1] ); $value = trim( $m[2] ); if ( '' !== $label && '' !== $value ) { $rows[] = array( 'label' => $label, 'value' => $value ); } } } return $rows; } /** * Collapse consecutive "paragraph" blocks that are EACH individually * "Label: value" shaped into one `specs` block, when there are at least * two of them in a row ("several … lines") — covers the common * `

Weight: 2kg

Color: Black

` shape a real `
` never * takes. A lone match is left as an ordinary paragraph (not "several"). * * @param array> $blocks Ordered blocks (headings included). * @return array> */ private function mergeLabelValueRuns( array $blocks ): array { $result = array(); $count = count( $blocks ); $i = 0; while ( $i < $count ) { $block = $blocks[ $i ]; if ( 'paragraph' !== $block['kind'] ) { $result[] = $block; ++$i; continue; } $rows = $this->labelValueRows( array( $block['text'] ) ); if ( empty( $rows ) ) { $result[] = $block; ++$i; continue; } $run = $rows; $j = $i + 1; while ( $j < $count && 'paragraph' === $blocks[ $j ]['kind'] ) { $next_rows = $this->labelValueRows( array( $blocks[ $j ]['text'] ) ); if ( empty( $next_rows ) ) { break; } $run = array_merge( $run, $next_rows ); ++$j; } if ( count( $run ) >= 2 ) { $result[] = array( 'kind' => 'specs', 'rows' => $run ); $i = $j; } else { $result[] = $block; ++$i; } } return $result; } /** * Attach each heading's text to the next non-heading block as * `headingText` (null when none precedes it). A trailing heading with * nothing after it names nothing and is dropped. * * @param array> $blocks Ordered blocks. * @return array> Content blocks only, each carrying `headingText`. */ private function associateHeadings( array $blocks ): array { $pending = null; $result = array(); foreach ( $blocks as $block ) { if ( 'heading' === $block['kind'] ) { $pending = $block['text']; continue; } $block['headingText'] = $pending; $result[] = $block; $pending = null; } return $result; } /** * @param array> $content_blocks Blocks with `headingText` attached. * @param string $product_name Example product's own name, if known. * @return array> BlueprintSection-shaped arrays. */ private function buildSections( array $content_blocks, string $product_name ): array { $first_paragraph_index = null; $last_paragraph_index = null; foreach ( $content_blocks as $i => $block ) { if ( 'paragraph' === $block['kind'] ) { if ( null === $first_paragraph_index ) { $first_paragraph_index = $i; } $last_paragraph_index = $i; } } $sections = array(); foreach ( $content_blocks as $i => $block ) { $type = $this->resolveType( $block, $i, $first_paragraph_index, $last_paragraph_index ); if ( 'summary' === $type && 'paragraph' === $block['kind'] ) { $sections[] = $this->buildSummarySection( $block, $product_name ); continue; } $sections[] = $this->buildSection( $type, $block ); } return $sections; } /** * @param array $block One content block. * @param int $index Its index in the content-block list. * @param int|null $first_paragraph_index See buildSections(). * @param int|null $last_paragraph_index See buildSections(). * @return string SectionType. */ private function resolveType( array $block, int $index, ?int $first_paragraph_index, ?int $last_paragraph_index ): string { if ( 'specs' === $block['kind'] ) { return 'specs'; } $keyword_type = null !== $block['headingText'] ? $this->matchKeyword( $block['headingText'] ) : null; if ( null !== $keyword_type ) { return $keyword_type; } if ( 'paragraph' !== $block['kind'] ) { return 'custom'; } if ( $index === $first_paragraph_index ) { return 'opening'; } if ( $index === $last_paragraph_index && $this->wordCount( $block['text'] ) <= self::SHORT_SUMMARY_MAX_WORDS ) { return 'summary'; } if ( null === $block['headingText'] && null !== $first_paragraph_index && $index === $first_paragraph_index + 1 ) { return 'supporting'; } return 'custom'; } /** * @param string $heading_text Heading text. * @return string|null Matched SectionType, or null when no keyword group matched. */ private function matchKeyword( string $heading_text ): ?string { foreach ( self::KEYWORD_TYPES as $type => $needles ) { foreach ( $needles as $needle ) { if ( $this->needleMatches( $heading_text, $needle ) ) { return $type; } } } return null; } /** * @param string $heading_text Heading text to search. * @param string $needle One keyword from KEYWORD_TYPES. * @return bool */ private function needleMatches( string $heading_text, string $needle ): bool { if ( in_array( $needle, self::WORD_BOUNDARY_NEEDLES, true ) ) { return 1 === preg_match( '/\b' . preg_quote( $needle, '/' ) . '\b/iu', $heading_text ); } return false !== mb_stripos( $heading_text, $needle, 0, 'UTF-8' ); } /** * Build any non-summary section from its matched type and content block. * * @param string $type SectionType. * @param array $block One content block. * @return array */ private function buildSection( string $type, array $block ): array { $entry = SectionCatalog::get( $type ) ?? SectionCatalog::get( 'custom' ); $fixed_text = $this->fixedTextForBlock( $block ); $mode = $this->resolveMode( $type, $block, $entry, $fixed_text ); $allowed_formats = $entry['formats'][ $mode ] ?? array( 'paragraph' ); $overrides = array( 'mode' => $mode ); if ( 'fixed' === $mode ) { $overrides['fixedText'] = $fixed_text; } if ( 'specs' === $block['kind'] ) { $overrides['format'] = in_array( 'table', $allowed_formats, true ) ? 'table' : $allowed_formats[0]; $overrides['fields'] = array( 'mode' => 'all', 'keys' => array() ); } elseif ( 'list' === $block['kind'] ) { $this->applyListOverrides( $block, $mode, $allowed_formats, $overrides ); } else { $this->applyParagraphOverrides( $block, $allowed_formats, $overrides ); } if ( null !== $block['headingText'] ) { $overrides['heading'] = array( 'mode' => 'fixed', 'text' => $this->capText( $block['headingText'], DefinitionPrimitives::HEADING_TEXT_MAX_LEN ) ); $overrides['name'] = $this->capText( $block['headingText'], DefinitionPrimitives::NAME_MAX_LEN ); } return SectionCatalog::newSection( $type, $overrides ); } /** * @param string $type SectionType. * @param array $block One content block. * @param array $entry SectionCatalog::get($type) (or its 'custom' fallback). * @param string $fixed_text $block's own fixedTextForBlock() result, computed once by the caller. * @return string ContentMode. */ private function resolveMode( string $type, array $block, array $entry, string $fixed_text ): string { if ( 'specs' === $block['kind'] ) { return 'data'; } if ( 'compatibility' === $type ) { return 'ai'; } $mode = $entry['modes'][0] ?? 'ai'; $unusable = '' === $fixed_text || mb_strlen( $fixed_text, 'UTF-8' ) > DefinitionPrimitives::FIXED_TEXT_MAX_LEN; if ( 'fixed' === $mode && $unusable && in_array( 'ai', $entry['modes'], true ) ) { return 'ai'; } return $mode; } /** * The detected block's own text, reusable verbatim as a fixed-mode * section's `fixedText`: a paragraph's text as-is, or a list's items as * one paragraph of sentences (the section only allows the paragraph * format). '' when the block has no usable text (resolveMode() then * falls back to 'ai'). * * @param array $block One content block. * @return string */ private function fixedTextForBlock( array $block ): string { if ( 'list' === $block['kind'] ) { $sentences = array(); foreach ( (array) ( $block['items'] ?? array() ) as $item ) { $item = is_string( $item ) ? $this->collapseWhitespace( $item ) : ''; if ( '' === $item ) { continue; } $sentences[] = preg_match( '/[.!?]$/u', $item ) ? $item : $item . '.'; } return implode( ' ', $sentences ); } return isset( $block['text'] ) && is_string( $block['text'] ) ? trim( $block['text'] ) : ''; } /** * @param array $block A 'list' content block. * @param string $mode Resolved ContentMode. * @param string[] $allowed_formats Formats allowed for $mode. * @param array &$overrides Section overrides, mutated in place. * @return void */ private function applyListOverrides( array $block, string $mode, array $allowed_formats, array &$overrides ): void { $preferred = $block['ordered'] ? 'numbered' : 'bullets'; if ( in_array( $preferred, $allowed_formats, true ) ) { $format = $preferred; } elseif ( in_array( 'bullets', $allowed_formats, true ) ) { $format = 'bullets'; } elseif ( in_array( 'numbered', $allowed_formats, true ) ) { $format = 'numbered'; } else { $format = null; } if ( null !== $format ) { $overrides['format'] = $format; $overrides['items'] = array( 'max' => $this->clampItems( count( $block['items'] ), $mode ) ); return; } $overrides['format'] = $allowed_formats[0] ?? 'paragraph'; $overrides['length'] = $this->wordRangeFor( implode( '. ', $block['items'] ) ); } /** * @param array $block A 'paragraph' content block. * @param string[] $allowed_formats Formats allowed for the resolved mode. * @param array &$overrides Section overrides, mutated in place. * @return void */ private function applyParagraphOverrides( array $block, array $allowed_formats, array &$overrides ): void { if ( in_array( 'paragraph', $allowed_formats, true ) ) { $overrides['format'] = 'paragraph'; $overrides['length'] = $this->wordRangeFor( $block['text'] ); return; } $overrides['format'] = $allowed_formats[0] ?? 'bullets'; } /** * @param array $block A 'paragraph' content block. * @param string $product_name Example product's own name, if known. * @return array */ private function buildSummarySection( array $block, string $product_name ): array { $text = $block['text']; $has_digit = 1 === preg_match( '/\d/', $text ); $has_product_name = '' !== $product_name && false !== mb_stripos( $text, $product_name, 0, 'UTF-8' ); $fits = mb_strlen( $text, 'UTF-8' ) <= DefinitionPrimitives::FIXED_TEXT_MAX_LEN; if ( ! $has_digit && ! $has_product_name && $fits ) { $overrides = array( 'mode' => 'fixed', 'format' => 'paragraph', 'fixedText' => $text ); } else { $overrides = array( 'mode' => 'ai', 'format' => 'paragraph', 'length' => $this->wordRangeFor( $text ) ); } if ( null !== $block['headingText'] ) { $overrides['heading'] = array( 'mode' => 'fixed', 'text' => $this->capText( $block['headingText'], DefinitionPrimitives::HEADING_TEXT_MAX_LEN ) ); $overrides['name'] = $this->capText( $block['headingText'], DefinitionPrimitives::NAME_MAX_LEN ); } return SectionCatalog::newSection( 'summary', $overrides ); } /** * @param string $text Source text. * @return array{targetMin:int,targetMax:int,hardMax:null} A word target ~20% either side of the measured length, clamped to the normalizer's own bounds. */ private function wordRangeFor( string $text ): array { $measured = $this->wordCount( $text ); if ( $measured < 1 ) { return array( 'targetMin' => 0, 'targetMax' => 0, 'hardMax' => null ); } $min = (int) round( $measured * 0.8 ); $max = (int) round( $measured * 1.2 ); $min = max( 0, min( $min, DefinitionPrimitives::WORD_TARGET_MAX ) ); $max = max( $min, min( $max, DefinitionPrimitives::WORD_TARGET_MAX ) ); return array( 'targetMin' => $min, 'targetMax' => $max, 'hardMax' => null ); } /** * @param int $count Measured item/row count. * @param string $mode 'data' | 'ai'. * @return int Clamped into the normalizer's own bounds for that mode. */ private function clampItems( int $count, string $mode ): int { $min = 'data' === $mode ? DefinitionPrimitives::DATA_ITEMS_MIN : DefinitionPrimitives::AI_ITEMS_MIN; $max = 'data' === $mode ? DefinitionPrimitives::DATA_ITEMS_MAX : DefinitionPrimitives::AI_ITEMS_MAX; return max( $min, min( $count, $max ) ); } /** * @param string $text Text to measure. * @return int */ private function wordCount( string $text ): int { $text = trim( $text ); if ( '' === $text ) { return 0; } return count( preg_split( '/\s+/u', $text ) ); } /** * @param string $text Candidate text. * @param int $cap Maximum characters. * @return string */ private function capText( string $text, int $cap ): string { return mb_strlen( $text, 'UTF-8' ) > $cap ? rtrim( mb_substr( $text, 0, $cap, 'UTF-8' ) ) : $text; } /** * Collapse all whitespace (including newlines) to single spaces and trim * (HtmlExtractor's own `collapseWhitespace()` pattern). * * @param string $text Raw text. * @return string */ private function collapseWhitespace( string $text ): string { return trim( preg_replace( '/[\s\x{00A0}]+/u', ' ', $text ) ); } }