*/ public static function chunk(string $title, string $body, int $modelMaxTokens = 8192): array { $body = trim($body); if ($body === '') { return []; } $headerBudget = Models::estimateTokens('Title: ' . $title . "\nSection: \n\n") + 24; $target = max(64, min(self::TARGET_TOKENS, $modelMaxTokens - $headerBudget - self::OVERLAP_TOKENS - 32)); $overlap = min(self::OVERLAP_TOKENS, (int) floor($target / 5)); $chunks = []; foreach (self::sections($body) as [$heading, $text]) { foreach (self::packSection($text, $target, $overlap) as $piece) { $full = self::header($title, $heading) . $piece; $chunks[] = ['text' => $full, 'section' => $heading, 'tokens' => Models::estimateTokens($full)]; } } return $chunks; } /** * @return list [heading, text] pairs; the text before the first heading has heading ''. */ private static function sections(string $body): array { $out = []; $heading = ''; $buffer = []; foreach (preg_split('/\n/', $body) ?: [] as $line) { if (preg_match('/^## +(.+)$/', $line, $m)) { if (trim(implode("\n", $buffer)) !== '') { $out[] = [$heading, trim(implode("\n", $buffer))]; } $heading = trim($m[1]); $buffer = []; continue; } $buffer[] = $line; } if (trim(implode("\n", $buffer)) !== '') { $out[] = [$heading, trim(implode("\n", $buffer))]; } return $out; } /** * @return list */ private static function packSection(string $text, int $target, int $overlap): array { $units = []; foreach (preg_split('/\n{2,}/', $text) ?: [] as $paragraph) { $paragraph = trim($paragraph); if ($paragraph !== '') { array_push($units, ...self::split($paragraph, $target)); } } $pieces = []; $current = ''; foreach ($units as $unit) { $candidate = $current === '' ? $unit : $current . "\n\n" . $unit; if ($current !== '' && Models::estimateTokens($candidate) > $target) { $pieces[] = $current; $tail = self::tail($current, $overlap); $current = $tail === '' ? $unit : $tail . "\n\n" . $unit; continue; } $current = $candidate; } if ($current !== '') { $last = count($pieces) - 1; // A small remainder reads better as part of the previous chunk than on its own. if ($last >= 0 && Models::estimateTokens($current) < self::MIN_TAIL_TOKENS) { $merged = $pieces[$last] . "\n\n" . self::withoutOverlap($current, $pieces[$last]); if (Models::estimateTokens($merged) <= $target + $overlap) { $pieces[$last] = $merged; return $pieces; } } $pieces[] = $current; } return $pieces; } /** * A paragraph as units no bigger than the target: itself, its sentences, or word runs. * * @return list */ private static function split(string $paragraph, int $target): array { if (Models::estimateTokens($paragraph) <= $target) { return [$paragraph]; } $lines = explode("\n", $paragraph); if (count($lines) > 1) { // Lines (table rows, list items) are packed together, keeping their breaks. $units = []; $run = ''; foreach ($lines as $line) { if (Models::estimateTokens($line) > $target) { if ($run !== '') { $units[] = $run; $run = ''; } array_push($units, ...self::split($line, $target)); continue; } $next = $run === '' ? $line : $run . "\n" . $line; if ($run !== '' && Models::estimateTokens($next) > $target) { $units[] = $run; $run = $line; } else { $run = $next; } } if ($run !== '') { $units[] = $run; } return $units; } $units = []; foreach (preg_split('/(?<=[.!?।॥。！？])\s+/u', $paragraph) ?: [] as $sentence) { if (Models::estimateTokens($sentence) <= $target) { $units[] = $sentence; continue; } // Run-on text: word runs; a single "word" longer than the target is cut by characters. $run = ''; foreach (preg_split('/\s+/u', $sentence) ?: [] as $word) { foreach (self::cutLong($word, $target) as $part) { $next = $run === '' ? $part : $run . ' ' . $part; if ($run !== '' && Models::estimateTokens($next) > $target) { $units[] = $run; $run = $part; } else { $run = $next; } } } if ($run !== '') { $units[] = $run; } } // Sentences are packed back together by packSection(). return $units; } /** * @return list */ private static function cutLong(string $word, int $target): array { if (Models::estimateTokens($word) <= $target) { return [$word]; } return mb_str_split($word, $target * 2) ?: [$word]; } /** * The last ~$overlap tokens of a chunk, starting at a sentence (else word) boundary. */ private static function tail(string $text, int $overlap): string { if ($overlap <= 0) { return ''; } $sentences = preg_split('/(?<=[.!?।॥。！？])\s+/u', $text) ?: []; $tail = ''; for ($i = count($sentences) - 1; $i >= 0; $i--) { $next = trim($sentences[$i] . ' ' . $tail); if (Models::estimateTokens($next) > $overlap) { break; } $tail = $next; } if ($tail !== '') { return $tail; } $words = preg_split('/\s+/u', $text) ?: []; $last = (string) end($words); $size = Models::estimateTokens($last); if ($size > $overlap) { // One huge "word" (a URL, an encoded blob): overlap with its end only. return mb_substr($last, -max(1, (int) floor(mb_strlen($last) * $overlap / $size))); } $take = []; for ($i = count($words) - 1; $i >= 0; $i--) { array_unshift($take, $words[$i]); if (Models::estimateTokens(implode(' ', $take)) >= $overlap) { break; } } return implode(' ', $take); } /** * Drops the overlap a piece starts with when it is merged back into its predecessor. */ private static function withoutOverlap(string $piece, string $previous): string { $parts = explode("\n\n", $piece, 2); if (count($parts) === 2 && $parts[0] !== '' && substr($previous, -strlen($parts[0])) === $parts[0]) { return $parts[1]; } return $piece; } private static function header(string $title, string $section): string { $lines = []; if ($title !== '') { $lines[] = 'Title: ' . $title; } if ($section !== '') { $lines[] = 'Section: ' . $section; } return $lines === [] ? '' : implode("\n", $lines) . "\n\n"; } }