$lines JSONL lines, oldest first, newline-free. * @param int $budgetBytes Hard ceiling for the returned lines including their newlines. * @param array $knownFailingIds Request ids condemned in a DIFFERENT * journal, keyed by id, from * ABJ_404_Solution_DiagnosticClientVerdict::requestIdsIn(). * @return array{lines: array, summary: array} */ public static function select(array $lines, int $budgetBytes, array $knownFailingIds = array()): array { $groups = self::group($lines); self::applyKnownFailures($groups, $knownFailingIds); $classification = ABJ_404_Solution_DiagnosticFailureClassification::fromGroups($groups); $ordered = self::orderByPriority($groups, $classification); $selection = ABJ_404_Solution_DiagnosticEvidenceBudget::allocate( $lines, $groups, $ordered['ids'], $ordered['prioritized'], $budgetBytes); return array( 'lines' => $selection['lines'], 'summary' => self::summarize($lines, $groups, $classification, $selection, $budgetBytes), ); } /** * One group per request id, in first-seen order. * * Decoded records are deliberately not retained: each one is folded into * its group's classification and then dropped, so a megabyte of JSONL does * not become many megabytes of live PHP arrays inside an admin request. * * @param array $lines * @return array */ private static function group(array $lines): array { $groups = array(); foreach ($lines as $index => $line) { $record = json_decode($line, true); $id = self::UNJOINABLE_KEY; if (is_array($record) && isset($record['request_id']) && is_scalar($record['request_id']) && (string)$record['request_id'] !== '') { $id = (string)$record['request_id']; } if (!isset($groups[$id])) { $groups[$id] = new ABJ_404_Solution_DiagnosticRequestGroup( $id, $id !== self::UNJOINABLE_KEY); } $groups[$id]->addLine($index, strlen($line) + 1); if (!is_array($record)) { continue; } $groups[$id]->applyRecord($record); self::applyClientVerdict($groups, $record); } return $groups; } /** * Fold a record's verdict about another request into that request's group. * * The verdict always names an id other than the one whose envelope carries * it, so a record can condemn a group these lines have not reached yet, or * one they never reach at all. * * @param array $groups * @param array $record */ private static function applyClientVerdict(array &$groups, array $record): void { $reportedId = ABJ_404_Solution_DiagnosticClientVerdict::condemnedRequestId($record); if ($reportedId === '') { return; } if (!isset($groups[$reportedId])) { // The condemned attempt wrote nothing here at all -- it may never // have reached PHP. Recorded as a known-failing id with no records // so the summary can say so out loud instead of it being absent. $groups[$reportedId] = new ABJ_404_Solution_DiagnosticRequestGroup($reportedId, true); } $groups[$reportedId]->markFailed(); } /** * Fold verdicts found in OTHER journals into the groups this one holds. * * Only groups that exist here are promoted; a condemned id this journal * never recorded mints nothing. That asymmetry with the in-journal pass is * deliberate: a request that died before its first stage legitimately * writes nothing to the stage trace, so minting a placeholder for it there * would report a failing request the excerpt could never carry and make * the accounting claim evidence was lost when none ever existed. The * journal that DID record the verdict still mints its own placeholder, so * the "condemned and completely absent" case stays stated exactly once. * * @param array $groups * @param array $knownFailingIds */ private static function applyKnownFailures(array $groups, array $knownFailingIds): void { foreach (array_keys($knownFailingIds) as $id) { $id = (string)$id; if (isset($groups[$id]) && $groups[$id]->hasRecords()) { $groups[$id]->markFailed(); } } } /** * Request ids in the order their budget is granted, plus how many of them * are prioritized (condemned, chain and suspected) and therefore hold a reserved share. * * @param array $groups * @return array{ids: array, prioritized: int} */ private static function orderByPriority(array $groups, ABJ_404_Solution_DiagnosticFailureClassification $classification): array { $chainIds = self::retryChainOf($groups, $classification->failingIds()); $condemned = array(); $chain = array(); $suspected = array(); $rest = array(); foreach ($groups as $id => $group) { if (!$group->hasRecords()) { continue; } if ($classification->isCondemned((string)$id)) { $condemned[] = $id; } elseif (isset($chainIds[$id])) { $chain[] = $id; } elseif ($classification->isSuspected((string)$id)) { $suspected[] = $id; } else { $rest[] = $id; } } // Tier 4 newest first: recent context is what a reader can still // correlate with the moment the admin clicked "send". Unjoinable lines // lead it, because a torn journal is itself a finding. // // Partitioned by hand rather than sorted: usort() is only stable from // PHP 8.0, and this plugin still runs on 7.4, where an equal-comparing // sort would quietly scramble the newest-first order that decides // which requests count as recent context. $unjoinable = array(); $joinable = array(); foreach (array_reverse($rest) as $id) { if ($id === self::UNJOINABLE_KEY) { $unjoinable[] = $id; } else { $joinable[] = $id; } } return array( 'ids' => array_merge($condemned, $chain, $suspected, $unjoinable, $joinable), 'prioritized' => count($condemned) + count($chain) + count($suspected), ); } /** * Transitive retry-chain closure around the failing requests: parents of a * failing request, and any request that names a chain member as its parent. * * @param array $groups * @param array $failingIds * @return array */ private static function retryChainOf(array $groups, array $failingIds): array { $children = array(); foreach ($groups as $id => $group) { foreach ($group->parentIds() as $parentId) { $children[$parentId][(string)$id] = true; } } $chain = array(); $pending = array_keys($failingIds); while ($pending !== array()) { $id = (string)array_pop($pending); $neighbours = isset($groups[$id]) ? $groups[$id]->parentIds() : array(); foreach (array_keys($children[$id] ?? array()) as $childId) { $neighbours[] = (string)$childId; } foreach ($neighbours as $neighbourId) { $neighbourId = (string)$neighbourId; if (isset($failingIds[$neighbourId]) || isset($chain[$neighbourId]) || !isset($groups[$neighbourId])) { continue; } $chain[$neighbourId] = true; $pending[] = $neighbourId; } } return $chain; } /** * The accounting that rides the excerpt. Without it a reader cannot tell * "this session only had four requests" from "sixty-eight were dropped in * transit", which is the exact ambiguity that made the previous excerpt * look like complete evidence when it was 3% of it. * * @param array $lines * @param array $groups * @param array{requests: int, records: int, bytes: int, includedIds: array, elided: int} $selection * @return array */ private static function summarize(array $lines, array $groups, ABJ_404_Solution_DiagnosticFailureClassification $classification, array $selection, int $budgetBytes): array { $failingIds = $classification->failingIds(); $condemnedIds = $classification->condemnedIds(); $bytesOnDisk = 0; $withRecords = 0; $unjoinable = 0; foreach ($groups as $id => $group) { $bytesOnDisk += $group->bytes(); if ($group->hasRecords()) { $withRecords++; } if ($id === self::UNJOINABLE_KEY) { $unjoinable = $group->recordCount(); } } $failingIncluded = 0; $failingWithoutRecords = 0; foreach (array_keys($failingIds) as $id) { if (isset($selection['includedIds'][$id])) { $failingIncluded++; } if (!isset($groups[$id]) || !$groups[$id]->hasRecords()) { $failingWithoutRecords++; } } $condemnedIncluded = 0; foreach (array_keys($condemnedIds) as $id) { if (isset($selection['includedIds'][$id])) { $condemnedIncluded++; } } return array( 'requests_on_disk' => $withRecords, 'requests_included' => $selection['requests'], 'records_on_disk' => count($lines), 'records_included' => $selection['records'], 'records_elided' => $selection['elided'], 'records_unjoinable' => $unjoinable, 'failing_requests' => count($failingIds), 'failing_requests_included' => $failingIncluded, 'failing_requests_without_records' => $failingWithoutRecords, 'condemned_requests' => count($condemnedIds), 'condemned_requests_included' => $condemnedIncluded, 'bytes_on_disk' => $bytesOnDisk, 'bytes_included' => $selection['bytes'], 'bytes_budget' => $budgetBytes, ); } }