self::POLICY_ALLOW, AI_Bot_Registry::CATEGORY_AGENT => self::POLICY_ALLOW, AI_Bot_Registry::CATEGORY_TRAINING => self::POLICY_BLOCK ]; } public static function sanitize_policy( array $raw ): array { $clean = []; foreach ( self::default_policy() as $category => $fallback ) { $value = $raw[ $category ] ?? $fallback; $clean[ $category ] = in_array( $value, [ self::POLICY_ALLOW, self::POLICY_BLOCK ], true ) ? $value : $fallback; } return $clean; } /** Return the canonical Content-Signal directive for User-agent: *. */ public static function content_signal( array $policy ): string { $policy = self::sanitize_policy( $policy ); $fields = []; foreach ( AI_Bot_Registry::CONTENT_SIGNAL_FIELD_MAP as $category => $field ) { $fields[] = $field . '=' . ( self::POLICY_ALLOW === $policy[ $category ] ? 'yes' : 'no' ); } return 'Content-Signal: ' . implode( ', ', $fields ); } /** Build only explicit fallback crawler rules; an all-allow policy needs none. */ public static function build_block( array $policy ): string { $policy = self::sanitize_policy( $policy ); $lines = []; foreach ( array_keys( AI_Bot_Registry::CONTENT_SIGNAL_FIELD_MAP ) as $category ) { if ( self::POLICY_BLOCK !== $policy[ $category ] ) { continue; } $bots = wp_list_pluck( AI_Bot_Registry::get_by_category( $category ), 'user_agent' ); if ( empty( $bots ) ) { continue; } if ( empty( $lines ) ) { $lines[] = self::BLOCK_START; $lines[] = '# AI training crawlers that may not honor Content-Signal'; } else { $lines[] = ''; } foreach ( $bots as $agent ) { $lines[] = 'User-agent: ' . $agent; } $lines[] = 'Disallow: /'; } if ( empty( $lines ) ) { return ''; } $lines[] = self::BLOCK_END; return implode( "\n", $lines ); } /** @return array{status: string, policy?: array} */ public static function detect_policy( string $content ): array { $groups = self::wildcard_groups( $content ); if ( 1 !== count( $groups ) || 1 !== count( $groups[0]['signals'] ) || self::managed_block_count( $content ) > 1 ) { return [ 'status' => 'custom' ]; } $values = self::parse_signal( $groups[0]['signals'][0] ); if ( null === $values ) { return [ 'status' => 'custom' ]; } return [ 'status' => 'detected', 'policy' => [ AI_Bot_Registry::CATEGORY_SEARCH => 'yes' === $values['search'] ? self::POLICY_ALLOW : self::POLICY_BLOCK, AI_Bot_Registry::CATEGORY_AGENT => 'yes' === $values['ai-input'] ? self::POLICY_ALLOW : self::POLICY_BLOCK, AI_Bot_Registry::CATEGORY_TRAINING => 'yes' === $values['ai-train'] ? self::POLICY_ALLOW : self::POLICY_BLOCK, ] ]; } /** Merge only AO additions, preserving unrelated directives and ordering. */ public static function apply_to_content( string $content, array $policy ): string { $policy = self::sanitize_policy( $policy ); if ( count( self::wildcard_groups( $content ) ) > 1 ) { return $content; // Multiple wildcard groups are unsafe to normalize. } $content = self::remove_managed_blocks( $content ); $groups = self::wildcard_groups( $content ); $lines = preg_split( '/\r\n|\r|\n/', $content ); $signal = self::content_signal( $policy ); if ( empty( $groups ) ) { $lines = trim( $content ) === '' ? [ 'User-agent: *', $signal ] : array_merge( $lines, [ '', 'User-agent: *', $signal ] ); } else { $group = $groups[0]; foreach ( array_reverse( $group['signal_indexes'] ) as $index ) { array_splice( $lines, $index, 1 ); } array_splice( $lines, $group['start'] + 1, 0, [ $signal ] ); } $content = rtrim( implode( "\n", $lines ) ); $block = self::build_block( $policy ); return '' === $block ? $content : $content . "\n\n" . $block; } /** @return array, signal_indexes: array}> */ private static function wildcard_groups( string $content ): array { $lines = preg_split( '/\r\n|\r|\n/', $content ); $groups = []; $current = null; foreach ( $lines as $index => $line ) { if ( preg_match( '/^\s*User-agent\s*:\s*\*\s*(?:#.*)?$/i', $line ) ) { $groups[] = [ 'start' => $index, 'signals' => [], 'signal_indexes' => [] ]; $current = count( $groups ) - 1; continue; } if ( preg_match( '/^\s*User-agent\s*:/i', $line ) ) { $current = null; continue; } if ( null !== $current && preg_match( '/^\s*Content-Signal\s*:\s*(.*?)\s*$/i', $line, $matches ) ) { $groups[ $current ]['signals'][] = $matches[1]; $groups[ $current ]['signal_indexes'][] = $index; } } return $groups; } /** @return array|null */ private static function parse_signal( string $signal ): ?array { $values = []; foreach ( explode( ',', $signal ) as $part ) { if ( ! preg_match( '/^\s*(search|ai-train|ai-input)\s*=\s*(yes|no)\s*$/i', $part, $matches ) || isset( $values[ strtolower( $matches[1] ) ] ) ) { return null; } $values[ strtolower( $matches[1] ) ] = strtolower( $matches[2] ); } return 3 === count( $values ) && isset( $values['search'], $values['ai-train'], $values['ai-input'] ) ? $values : null; } private static function remove_managed_blocks( string $content ): string { $pattern = '/' . preg_quote( self::BLOCK_START, '/' ) . '.*?' . preg_quote( self::BLOCK_END, '/' ) . '\s*/s'; return preg_replace( $pattern, '', $content ); } private static function managed_block_count( string $content ): int { return preg_match_all( '/' . preg_quote( self::BLOCK_START, '/' ) . '.*?' . preg_quote( self::BLOCK_END, '/' ) . '/s', $content ); } }