pi-tuck 1.0.1 → 1.0.3

package/src/tuck/core/shrink.ts

@@ -0,0 +1,218 @@
+import type { NormalizedBlock } from "../types";
+import { blockRecords, decorateRecord, isDialogue, isOmittable, omissionRecord, type Clip, type DialogueBlock } from "./brief";
+import { DIALOGUE_SUFFIX } from "./format";
+import { orderByOriginalSource } from "./source-order";
+import { serializeRecord, serializeTranscript, TRANSCRIPT_HEADER, type TranscriptRecord } from "./transcript";
+
+type Role = "user" | "assistant" | "notice";
+const ROLES: readonly Role[] = ["user", "assistant", "notice"];
+
+/** The lossy variants of a unit, least clipped first: the words kept from the start of its text and from its end. */
+const TIERS: Record<Role, readonly Clip[]> = {
+  assistant: [[80, 120], [40, 40], [16, 0]],
+  user: [[256, 0], [64, 0], [24, 0]],
+  notice: [[256, 0]],
+};
+const TOOLS_OMITTED: TranscriptRecord = { kind: "omission", reason: "Historical tool actions and observations omitted; read original sources with tuck_recall" };
+
+export interface DialogueInput {
+  /** Normalized, noise-filtered blocks of the pre-cut history in chronological order: dialogue and tool blocks alike. */
+  blocks: NormalizedBlock[];
+  /** Sources the adapter quotes in its own base: they render as nothing here and are not counted as omitted. */
+  mandatorySourceIndices?: ReadonlySet<number>;
+  /** The source index of each original message and, when known, its entry id, for the original order of the records. */
+  sourceIndices: readonly (number | undefined)[];
+  sourceEntryIds?: readonly string[];
+}
+
+/** The configurations the ladder tries, in order; "none" is no configuration that fits. */
+export type Step = "whole" | `A${1 | 2 | 3}` | "B" | `C${1 | 2 | 3}` | "D" | "E1" | "E2" | "none";
+export interface Shrunk {
+  /** The summary text, or "" when nothing fits. */
+  text: string;
+  step: Step;
+  /** The length the ladder computed from per-unit lengths before it rendered `text`. */
+  size: number;
+}
+
+/** What one unit renders in a configuration, and its serialized size: each record's line plus a newline. */
+interface Rendered { records: TranscriptRecord[]; cost: number }
+const rendered = (records: TranscriptRecord[]): Rendered => ({ records, cost: records.reduce((n, r) => n + serializeRecord(r).length + 1, 0) });
+
+/** Units [0, drop) of a role are dropped, [drop, end) are clipped to `tier`, and the rest stay whole. */
+interface Setting { drop: number; tier: number; end: number }
+type Config = Record<Role, Setting>;
+const WHOLE: Setting = { drop: 0, tier: 0, end: 0 };
+
+/** The units of one role in chronological order, with each variant's cost summed over every prefix. */
+const createLane = (role: Role, units: DialogueBlock[], mandatory: ReadonlySet<number> | undefined, decorate: (record: TranscriptRecord) => TranscriptRecord) => {
+  const n = units.length;
+  const records = (block: DialogueBlock, clip?: Clip) => rendered(blockRecords(block, mandatory, clip).map(decorate));
+  const prefix = (level: Rendered[]) => { const sums = new Float64Array(n + 1); level.forEach((r, i) => sums[i + 1] = sums[i]! + r.cost); return sums; };
+  const levels: Rendered[][] = [units.map(block => records(block))];
+  const sums = [prefix(levels[0]!)];
+  // A clip keeps whole passages, so a unit that is one long sentence, code block or JSON region can keep no text at all. Such a clip renders
+  // nothing and its unit is counted in the role's one omission notice, like a dropped unit: a notice per empty clip would cost more than text.
+  const NONE: Rendered = { records: [], cost: 0 };
+  const emptied: Uint8Array[] = [new Uint8Array(n)];
+  const emptyCounts: Int32Array[] = [new Int32Array(n + 1)];
+  /** Level t holds each unit's tier-t variant. A clip that would not make the unit smaller is not taken, so sizes never grow with t. */
+  const level = (t: number): Rendered[] => {
+    while (levels.length <= t) {
+      const previous = levels[levels.length - 1]!, wasEmpty = emptied[emptied.length - 1]!, clip = TIERS[role][levels.length - 1]!;
+      const empty = new Uint8Array(n);
+      const next = units.map((block, i) => {
+        if (wasEmpty[i]) { empty[i] = 1; return previous[i]!; }
+        const clipped = records(block, clip);
+        if (role !== "notice" && isOmittable(block, mandatory) && !clipped.records.some(record => record.kind === "excerpt")) { empty[i] = 1; return NONE; }
+        return clipped.cost < previous[i]!.cost ? clipped : previous[i]!;
+      });
+      const counts = new Int32Array(n + 1);
+      empty.forEach((e, i) => counts[i + 1] = counts[i]! + e);
+      levels.push(next); sums.push(prefix(next)); emptied.push(empty); emptyCounts.push(counts);
+    }
+    return levels[t]!;
+  };
+  const sum = (t: number) => { level(t); return sums[t]!; };
+
+  // The notice for a dropped prefix of p units: how many unselected text blocks, the first locators, and whether some have none.
+  let tally: { counts: Int32Array; unlocated: Uint8Array; smallest: number[][] } | undefined;
+  /** The notice for the units a setting omits: the dropped prefix, and the clips in [drop, end) that kept no text. */
+  const omission = ({ drop, tier, end }: Setting): TranscriptRecord | undefined => {
+    if (role === "notice") return undefined;
+    if (!tally) {
+      tally = { counts: new Int32Array(n + 1), unlocated: new Uint8Array(n + 1), smallest: new Array(n + 1) };
+      let smallest: number[] = [], count = 0, unlocated = 0;
+      tally.smallest[0] = smallest;
+      units.forEach((block, i) => {
+        if (isOmittable(block, mandatory)) {
+          count++;
+          const source = block.sourceIndex;
+          if (source === undefined) unlocated = 1;
+          else if (!smallest.includes(source) && (smallest.length < 9 || source < smallest[8]!)) {
+            smallest = [...smallest, source].sort((a, b) => a - b).slice(0, 9);
+          }
+        }
+        tally!.counts[i + 1] = count; tally!.unlocated[i + 1] = unlocated; tally!.smallest[i + 1] = smallest;
+      });
+    }
+    let count = tally.counts[drop]!, unlocated = tally.unlocated[drop] === 1, sources = tally.smallest[drop]!;
+    if (end > drop && tier > 0) {
+      level(tier);
+      const empty = emptied[tier]!, counts = emptyCounts[tier]!;
+      count += counts[end]! - counts[drop]!;
+      if (counts[end]! > counts[drop]!) {
+        const extra: number[] = [];
+        for (let i = drop; i < end && extra.length < 9; i++) if (empty[i]) {
+          const source = units[i]!.sourceIndex;
+          if (source === undefined) unlocated = true; else extra.push(source);
+        }
+        for (let i = drop; i < end && !unlocated; i++) if (empty[i] && units[i]!.sourceIndex === undefined) unlocated = true;
+        sources = [...new Set([...sources, ...extra])].sort((a, b) => a - b).slice(0, 9);
+      }
+    }
+    return count ? decorate(omissionRecord(role, count, sources, unlocated)) : undefined;
+  };
+  const omissionCosts = new Map<string, number>();
+  const omissionCost = (setting: Setting) => {
+    const key = `${setting.drop}:${setting.tier}:${setting.end}`;
+    let cost = omissionCosts.get(key);
+    if (cost === undefined) { const record = omission(setting); omissionCosts.set(key, cost = record ? rendered([record]).cost : 0); }
+    return cost;
+  };
+
+  return {
+    n, level, sum, omission,
+    cost: ({ drop, tier, end }: Setting) => sums[0]![n]! - sums[0]![Math.max(drop, end)]! + (end > drop ? sum(tier)[end]! - sum(tier)[drop]! : 0) + omissionCost({ drop, tier, end }),
+  };
+};
+
+/** The units of a dialogue and what any configuration of them renders, by arithmetic over per-unit lengths or as the summary text. */
+const createDialogue = (input: DialogueInput) => {
+  const dialogue = input.blocks.filter(isDialogue);
+  const times = dialogue.find(block => block.sourceTimes)?.sourceTimes;
+  const decorate = (record: TranscriptRecord) => decorateRecord(record, times);
+  const roleOf = (block: DialogueBlock): Role => block.kind;
+  const lanes = Object.fromEntries(ROLES.map(role => [role, createLane(role, dialogue.filter(block => roleOf(block) === role), input.mandatorySourceIndices, decorate)])) as
+    Record<Role, ReturnType<typeof createLane>>;
+  const seen: Record<Role, number> = { user: 0, assistant: 0, notice: 0 };
+  const order = dialogue.map(block => ({ role: roleOf(block), i: seen[roleOf(block)]++ }));
+  const tools = input.blocks.length > dialogue.length ? rendered([TOOLS_OMITTED]) : undefined;
+  const overhead = TRANSCRIPT_HEADER.length + DIALOGUE_SUFFIX.length;
+
+  /** The length of the summary text of a configuration: its header, one line per record, and the closing section; 0 when it has no record. */
+  const size = (config: Config): number => {
+    const cost = ROLES.reduce((n, role) => n + lanes[role].cost(config[role]), tools?.cost ?? 0);
+    return cost === 0 ? 0 : overhead + cost;
+  };
+  const text = (config: Config): string => {
+    const records: TranscriptRecord[] = [];
+    for (const { role, i } of order) {
+      const { drop, tier, end } = config[role];
+      if (i >= drop) records.push(...lanes[role].level(i < end ? tier : 0)[i]!.records);
+    }
+    for (const role of ROLES) { const record = lanes[role].omission(config[role]); if (record) records.push(record); }
+    if (tools) records.push(...tools.records);
+    orderByOriginalSource(records, input.sourceIndices, input.sourceEntryIds);
+    return records.length ? serializeTranscript(records) + DIALOGUE_SUFFIX : "";
+  };
+  return { lanes, size, text };
+};
+
+/** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted. "" when it renders nothing. */
+export const wholeDialogue = (input: DialogueInput): string => {
+  const { text } = createDialogue(input);
+  return text({ user: WHOLE, assistant: WHOLE, notice: WHOLE });
+};
+
+/**
+ * The same dialogue shrunk until its summary text is at most `limit` characters, oldest first. The first configuration that fits wins:
+ *   A. the newest k assistant units whole, the older ones clipped to tier 1, 2, 3 (the largest k that fits, for each tier in turn);
+ *   B. the newest assistant units that fit a third of the limit whole, the older ones at tier 3 with the oldest d dropped;
+ *   C. the newest assistant units that fit a third of the limit at tier 3, the rest dropped; then the oldest u user units clipped to tier 1, 2, 3;
+ *   D. every user unit at tier 3, the oldest dropped;
+ *   E. the oldest notices dropped, then the oldest of the remaining assistant units.
+ * Every step is a one-parameter family whose size is read from prefix sums of the per-unit lengths, so a step costs O(units). A notice is clipped
+ * to its one tier in every configuration but the whole dialogue. The size is arithmetic over unit lengths; the rendered text has the last say:
+ * a configuration whose text is over the limit is passed over.
+ */
+export const shrinkDialogue = (input: DialogueInput, limit: number): Shrunk => {
+  const { lanes, size, text } = createDialogue(input);
+  const { user, assistant, notice } = lanes;
+  const clipped: Setting = { drop: 0, tier: 1, end: notice.n };
+  const found = (step: Step, config: Config): Shrunk | undefined => {
+    const length = size(config);
+    if (length > limit) return undefined;
+    const summary = text(config);
+    return summary.length <= limit ? { text: summary, step, size: length } : undefined;
+  };
+  const gone = (lane: { n: number }): Setting => ({ drop: lane.n, tier: 3, end: lane.n });
+  const tiers = [1, 2, 3] as const;
+
+  let hit = found("whole", { user: WHOLE, assistant: WHOLE, notice: WHOLE });
+  if (hit) return hit;
+  for (const t of tiers) for (let k = assistant.n; k >= 0; k--)
+    if (hit = found(`A${t}`, { user: WHOLE, assistant: { drop: 0, tier: t, end: assistant.n - k }, notice: clipped })) return hit;
+  // The newest assistant units that fit a third of the limit whole are kept until users have given way: B clips and drops only older ones.
+  // A clip can keep no text at all, so the third is sized on whole units; when the newest alone is over the third, it keeps its largest clip
+  // that fits and still has text.
+  let kept = 0;
+  while (kept < assistant.n && assistant.sum(0)[assistant.n]! - assistant.sum(0)[kept]! > limit / 3) kept++;
+  let recent: Setting = { drop: kept, tier: 3, end: kept };
+  if (kept === assistant.n && assistant.n > 0) for (const t of tiers) {
+    const cost = assistant.level(t)[assistant.n - 1]!.cost;
+    if (cost > 0 && cost <= limit / 3) { recent = { drop: assistant.n - 1, tier: t, end: assistant.n }; break; }
+  }
+  for (let d = 0; d <= recent.drop; d++)
+    if (hit = found("B", { user: WHOLE, assistant: { ...recent, drop: d, end: Math.max(recent.end, recent.drop) }, notice: clipped })) return hit;
+  // From here users give way, and the assistant keeps only that recent third.
+  for (const t of tiers) for (let u = 0; u <= user.n; u++)
+    if (hit = found(`C${t}`, { user: { drop: 0, tier: t, end: u }, assistant: recent, notice: clipped })) return hit;
+  for (let d = 0; d <= user.n; d++)
+    if (hit = found("D", { user: { drop: d, tier: 3, end: user.n }, assistant: recent, notice: clipped })) return hit;
+  for (let d = 0; d <= notice.n; d++)
+    if (hit = found("E1", { user: gone(user), assistant: recent, notice: { drop: d, tier: 1, end: notice.n } })) return hit;
+  for (let d = recent.drop; d <= assistant.n; d++)
+    if (hit = found("E2", { user: gone(user), assistant: { drop: d, tier: recent.tier, end: Math.max(recent.end, d) }, notice: gone(notice) })) return hit;
+  return { text: "", step: "none", size: 0 };
+};

package/src/tuck/core/summarize.ts

@@ -1,41 +1,36 @@
-import { orderByOriginalSource } from "./source-order";
 import type { CompilerMessage } from "../types";
 import { normalize } from "./normalize";
 import { filterNoise } from "./filter-noise";
-import { compileBrief, planBrief, buildBriefRecords } from "./brief";
-import { DIALOGUE_SUFFIX, RANKED_SUFFIX } from "./format";
-import { parseRecords, serializeTranscript } from "./transcript";
-import { selectRankedBriefBlocks, type BriefRankingOptions } from "./rank";
+import { shrinkDialogue, wholeDialogue, type DialogueInput } from "./shrink";
 
-export interface RankedCompileInput {
+export interface CompileInput {
   messages: CompilerMessage[];
-  /** Eligible original branch only; receipt identity/order, not extra ranked content. */
+  /** Eligible original branch only: receipt identity and order. */
   receiptContext?: { messages: CompilerMessage[]; sourceIndices: Array<number | undefined> };
   /** Native session-global source indices parallel to messages. */
   sourceIndices?: Array<number | undefined>;
   /** Actual native entry IDs parallel to originals, not inferred from text. */
   sourceEntryIds?: readonly string[];
   sourceTimes?: readonly import("./source-time").SourceTime[];
-  ranking?: BriefRankingOptions;
+  ranking?: {
+    /** The complete serialized length a summary of the dialogue may have, header and closing section included. */
+    maxBriefChars?: number;
+    /** Full sources rendered separately by the caller: they render as nothing here. */
+    mandatorySourceIndices?: ReadonlySet<number>;
+  };
 }
 
-/** Select eligible originals once. No inferred state or prior-view merging. */
-export const compileRanked = (input: RankedCompileInput): string => {
-  const blocks = filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes));
-  const selected = selectRankedBriefBlocks(blocks, input.ranking);
-  const transcript = serializeTranscript(parseRecords(compileBrief(selected)));
-  return transcript ? transcript + RANKED_SUFFIX : "";
-};
+const dialogueInput = (input: CompileInput): DialogueInput => ({
+  blocks: filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes)),
+  mandatorySourceIndices: input.ranking?.mandatorySourceIndices,
+  sourceIndices: input.sourceIndices ?? input.messages.map((_, index) => index),
+  sourceEntryIds: input.sourceEntryIds,
+});
 
 /** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted.
  * This disposable view grants neither present authority nor native tool execution. */
-export const compileDialogue = (input: RankedCompileInput): string => {
-  const originals = filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes));
-  const blocks = originals.filter(block => ["user", "assistant", "notice"].includes(block.kind));
-  planBrief(blocks, new Set(blocks), originals, input.ranking?.mandatorySourceIndices);
-  const records = buildBriefRecords(blocks);
-  if (blocks.length < originals.length) records.push({ kind: "omission",
-    reason: "Historical tool actions and observations omitted; read original sources with tuck_recall" });
-  orderByOriginalSource(records, input.sourceIndices ?? input.messages.map((_, index) => index), input.sourceEntryIds);
-  return records.length ? serializeTranscript(records) + DIALOGUE_SUFFIX : "";
-};
+export const compileDialogue = (input: CompileInput): string => wholeDialogue(dialogueInput(input));
+
+/** The dialogue shrunk to `ranking.maxBriefChars`, oldest assistant text first (see `shrinkDialogue`); "" when nothing fits. */
+export const compileShrunkDialogue = (input: CompileInput & { ranking: { maxBriefChars: number } }): string =>
+  shrinkDialogue(dialogueInput(input), input.ranking.maxBriefChars).text;

package/src/tuck/core/transcript.ts

@@ -1,4 +1,4 @@
-import { validSourceTime } from "./source-time";
+import { validSourceTime, type SourceTime } from "./source-time";
 import { isDeepStrictEqual } from "node:util";
 import type { HistoricalReceipt } from "../types";
 
@@ -27,6 +27,7 @@ interface RecordFields {
   count?: number;
   title?: string;
 }
+/** The tool_call, tool_receipt, tool_excerpt and bash kinds are v1 only: summaries saved by earlier builds hold them, and this build no longer writes them. */
 export type TranscriptRecord = RecordFields & (
   { kind: "excerpt"; role: "user" | "assistant" | "custom"; text: string } |
   { kind: "tool_call"; role: "assistant"; text: string; arguments?: Record<string, unknown>;
@@ -38,96 +39,247 @@ export type TranscriptRecord = RecordFields & (
   { kind: "section"; title: string; text: string }
 );
 
-export const TRANSCRIPT_HEADER = "TUCK-TRANSCRIPT/1";
-export const TRANSCRIPT_UNIT_CHARACTERS = 120;
-/** UTF-16 serialized characters, including JSON escapes, labels and framing. */
-export const lineUnits = (characters: number): number => Math.max(1, Math.ceil(characters / TRANSCRIPT_UNIT_CHARACTERS));
-export const transcriptUnits = (text: string): number => text === "" ? 0 :
-  text.split("\n").reduce((n, line) => n + lineUnits(line.length), 0);
-export const serializeRecords = (records: TranscriptRecord[]): string => records.map(r => JSON.stringify(r)).join("\n");
-export const serializeTranscript = (records: TranscriptRecord[]): string => records.length ?
-  `${TRANSCRIPT_HEADER}\n${serializeRecords(records)}` : "";
+/** The format this build writes. */
+export const TRANSCRIPT_HEADER = "TUCK-TRANSCRIPT/2";
+/** The format of 1.0.x summaries: one JSON object per line. Read, never written. */
+export const TRANSCRIPT_HEADER_V1 = "TUCK-TRANSCRIPT/1";
+/** The header of summaries saved before the rename to pi-tuck: the v1 format. */
+export const LEGACY_TRANSCRIPT_HEADER = "VCC-TRANSCRIPT/1";
+export type TranscriptVersion = 1 | 2;
+/** The header a summary of `version` is written with; a legacy one is rewritten as v1. */
+export const transcriptHeader = (version: TranscriptVersion): string => version === 2 ? TRANSCRIPT_HEADER : TRANSCRIPT_HEADER_V1;
+
 /** Starts a summary's closing section, after the record lines. */
 export const CLOSING_SEPARATOR = "\n\n---\n\n";
-/** A summary in this format, split by structure: each record is one JSON line, so the first blank line after the header starts the
- * closing section. Any closing wording, from this build or an earlier one, is accepted; the records are returned byte for byte. */
-export const splitSummary = (summary: string): { body: string; closing: string } | undefined => {
-  const header = [TRANSCRIPT_HEADER, LEGACY_TRANSCRIPT_HEADER].find(h => summary.startsWith(h + "\n"));
+/** A summary in either format, split by structure: each record is one line, and no record line is empty, so the first blank line after the
+ * header starts the closing section. Any closing wording, from this build or an earlier one, is accepted; the records are returned byte
+ * for byte, with the format their header names. */
+export const splitSummary = (summary: string): { body: string; closing: string; version: TranscriptVersion } | undefined => {
+  const header = [TRANSCRIPT_HEADER, TRANSCRIPT_HEADER_V1, LEGACY_TRANSCRIPT_HEADER].find(h => summary.startsWith(h + "\n"));
   if (!header) return undefined;
   const at = summary.indexOf(CLOSING_SEPARATOR, header.length);
   if (at < 0) return undefined;
   // A summary saved before the rename keeps its records byte for byte (their lengths are checked); only its closing is pointed at the
   // tools the model has now.
-  const closing = header === TRANSCRIPT_HEADER ? summary.slice(at) : summary.slice(at).replaceAll("vcc_recall", "tuck_recall").replaceAll("vcc_search", "tuck_search");
-  return { body: summary.slice(header.length + 1, at), closing };
+  const closing = header === LEGACY_TRANSCRIPT_HEADER ? summary.slice(at).replaceAll("vcc_recall", "tuck_recall").replaceAll("vcc_search", "tuck_search") : summary.slice(at);
+  return { body: summary.slice(header.length + 1, at), closing, version: header === TRANSCRIPT_HEADER ? 2 : 1 };
 };
-/** The header of summaries saved before the rename to pi-tuck. */
-export const LEGACY_TRANSCRIPT_HEADER = "VCC-TRANSCRIPT/1";
 const LEGACY_RECEIPT_RANGE_NOTICE = RECEIPT_RANGE_NOTICE.replace("tuck_recall", "vcc_recall");
 
-/** Accept our explicit record protocol only. Never recover roles from payload text. */
-export const parseRecords = (text: string): TranscriptRecord[] => {
+/*
+ * TUCK-TRANSCRIPT/2: the header line, one line per record, then the closing section.
+ *
+ *   line      = head ":" [ " " text ]          no " " and no text when the text is empty
+ *   record    = [ locator " " ] role [ " " id ] [ " " time ] [ " " span ] ":" ...                      excerpt
+ *             | [ locator " " ] [ role [ " " id ] " " ] "omitted" [ " " time ] [ " " span ] ":" ...      omission of part of one source,
+ *                                                                                                         or, with no role, of the tool history
+ *             | role " omitted " count [ " " locator-time { " " locator-time } ] ":" ...               omission of unselected text blocks
+ *             | [ locator " " ] title ":" ...                                                          section (file ledger, action trail)
+ *   locator   = "#" index                      the source's session-wide index, as tuck_recall {"index"} takes it
+ *   locator-time = locator [ "@" time-value ]  every locator of a record has a time, or none does
+ *   role      = "user" | "assistant" | "custom"
+ *   id        = a JSON string                  the native entry id of a custom notice (role "custom" only)
+ *   time      = "@" time-value                 the recorded time of the locator that precedes the role (a record that has one)
+ *   time-value = UTC | "journal:" UTC | "unknown"   UTC is the message time, "journal:" UTC the session file's time of the entry
+ *   span      = "[chars " start "-" end " of " total "]"     the characters of the source text block the text covers or the omission leaves out
+ *   count     = index                          how many text blocks the omission counts
+ *   title     = capital letter, then letters, digits, "_" and "-"
+ *   index     = "0" | nonzero digit, then digits   a safe integer, no sign, no leading zero
+ *
+ * The text of an excerpt or section, or the reason of an omission, is the rest of the line: "\" is written "\\", a line feed "\n" and a
+ * carriage return "\r"; every other character is written as it is.
+ *
+ * The head is made of the tokens above only, and the text is the rest of the line after the first ":" that follows them, so no text can be
+ * mistaken for structure: a text that looks like a record is the text of one line. A role is lowercase and a title is capitalized, so the
+ * first word of a head tells which record it is. Lines that follow no production are rejected, as are fields the record shape does not
+ * have (a time without a locator, an id on a role other than "custom") and every check parseRecords makes of a v1 record.
+ *
+ * Examples:
+ *   #12 user @2026-10-30T14:00:00Z: Please fix the parser.
+ *   #12 assistant [chars 0-523 of 2004]: I will start with the tokenizer.
+ *   #12 assistant omitted [chars 523-1900 of 2004]: tuck_recall {"index":12,"offset":523}
+ *   #7 custom "a1b2c3d4": A note.
+ *   assistant omitted 40 #3 #5 #9: Completely unselected historical text blocks; ...
+ *   user omitted 3 #1@2026-10-30T14:00:00Z #4@unknown #8@journal:2026-10-30T14:00:05Z: Completely unselected ...
+ *   omitted: Historical tool actions and observations omitted; read original sources with tuck_recall
+ *   #4 Files: - read src/a.ts (tuck_recall {"index":4})
+ */
+const escapeText = (text: string): string => text.replace(/[\\\n\r]/g, c => c === "\\" ? "\\\\" : c === "\n" ? "\\n" : "\\r");
+/** The length of `text` as a record line carries it: one character more for each "\", line feed and carriage return. */
+export const escapedLength = (text: string): number => text.length + (text.match(/[\\\n\r]/g)?.length ?? 0);
+const TITLE = /^[A-Z][A-Za-z0-9_-]*$/;
+const refuse = (record: TranscriptRecord): never => { throw new Error(`Cannot write a ${record.kind} record as ${TRANSCRIPT_HEADER}`); };
+
+/** One record as a line of TUCK-TRANSCRIPT/2 (without its newline). The records this build emits only: a record whose fields the grammar
+ * cannot carry exactly is refused, never written lossily. */
+export const serializeRecord = (record: TranscriptRecord): string => {
+  const fields = record as unknown as Record<string, unknown>;
+  const present = Object.keys(fields).filter(key => fields[key] !== undefined);
+  const only = (...allowed: string[]) => { if (!present.every(key => allowed.includes(key))) refuse(record); };
+  const index = (n: unknown): string => typeof n === "number" && Number.isSafeInteger(n) && n >= 0 ? String(n) : refuse(record);
+  const time = (t: SourceTime): string => !validSourceTime(t) ? refuse(record) : t.basis === "unknown" ? "unknown" : t.basis === "journal" ? `journal:${t.utc}` : t.utc;
+  /** The record's times, one for each locator in order; none when it has none. */
+  const times = (locators: readonly number[]): string[] | undefined => {
+    const { sourceTimes } = record;
+    if (!sourceTimes) return undefined;
+    if (sourceTimes.length !== locators.length) refuse(record);
+    return sourceTimes.map(({ sourceIndex, ...value }, i) => sourceIndex === locators[i] ? time(value as SourceTime) : refuse(record));
+  };
+  const line = (head: string, text: string | undefined): string => head + ":" + (text ? " " + escapeText(text) : "");
+  if (record.kind === "section") {
+    only("kind", "title", "text", "sourceIndex");
+    if (typeof record.title !== "string" || !TITLE.test(record.title) || typeof record.text !== "string") refuse(record);
+    return line((record.sourceIndex === undefined ? "" : `#${index(record.sourceIndex)} `) + record.title, record.text);
+  }
+  if (record.kind !== "excerpt" && record.kind !== "omission") return refuse(record);
+  const { role, sourceIndex, sourceEntryId, span } = record, omission = record.kind === "omission";
+  if (role !== undefined && !["user", "assistant", "custom"].includes(role) || role === undefined && (!omission || sourceEntryId !== undefined) ||
+    sourceEntryId !== undefined && (role !== "custom" || typeof sourceEntryId !== "string")) refuse(record);
+  const who = (sourceEntryId === undefined ? "" : ` ${JSON.stringify(sourceEntryId)}`);
+  if (omission && record.count !== undefined) {
+    only("kind", "role", "count", "sourceIndices", "reason", "sourceTimes");
+    const locators = record.sourceIndices ?? [], stamped = times(locators);
+    if (role === undefined || typeof record.reason !== "string") refuse(record);
+    return line(`${role} omitted ${index(record.count)}` + locators.map((n, i) => ` #${index(n)}${stamped ? "@" + stamped[i] : ""}`).join(""), record.reason);
+  }
+  only("kind", "role", "sourceIndex", omission ? "reason" : "text", "span", "sourceEntryId", "sourceTimes");
+  const text = omission ? record.reason : record.text;
+  if (typeof text !== "string") refuse(record);
+  const stamped = times(sourceIndex === undefined ? [] : [sourceIndex]);
+  const where = (sourceIndex === undefined ? "" : `#${index(sourceIndex)} `) + (role === undefined ? "" : role + who + (omission ? " " : ""));
+  return line(where + (omission ? "omitted" : "") + (stamped ? ` @${stamped[0]}` : "") +
+    (span ? ` [chars ${index(span.start)}-${index(span.end)} of ${index(span.totalCharacters)}]` : ""), text);
+};
+export const serializeRecords = (records: TranscriptRecord[]): string => records.map(serializeRecord).join("\n");
+export const serializeTranscript = (records: TranscriptRecord[]): string => records.length ?
+  `${TRANSCRIPT_HEADER}\n${serializeRecords(records)}` : "";
+
+const invalid = (): never => { throw new Error("Invalid pi-tuck transcript record"); };
+const unescapeText = (text: string): string => text.replace(/\\([\\nr])?/g, (_, c?: string) => c === undefined ? invalid() : c === "n" ? "\n" : c === "r" ? "\r" : "\\");
+// Sticky patterns: each is matched at the position the line has been read up to.
+const INDEX = "(0|[1-9]\\d*)";
+const TIME = "(unknown|(?:journal:)?\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,3})?Z)";
+const LOCATOR = new RegExp(`#${INDEX} `, "y"), ROLE = /user|assistant|custom/y, OMITTED = /omitted/y;
+const ID = / ("(?:[^"\\\u0000-\u001f]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*")/y, SPACE_OMITTED = / omitted/y, COUNT = new RegExp(` ${INDEX}`, "y");
+const LOCATOR_TIME = new RegExp(` #${INDEX}(?:@${TIME})?`, "y"), TIME_TOKEN = new RegExp(` @${TIME}`, "y");
+const SPAN = new RegExp(` \\[chars ${INDEX}-${INDEX} of ${INDEX}\\]`, "y"), TITLE_TOKEN = /[A-Z][A-Za-z0-9_-]*/y;
+
+/** One line of TUCK-TRANSCRIPT/2, as the record the build that wrote it held (its fields in the order that build emits them). */
+const parseLine = (line: string): unknown => {
+  let at = 0;
+  const take = (pattern: RegExp): RegExpExecArray | undefined => {
+    pattern.lastIndex = at;
+    const match = pattern.exec(line);
+    if (match) at = pattern.lastIndex;
+    return match ?? undefined;
+  };
+  const integer = (digits: string): number => Number.isSafeInteger(Number(digits)) ? Number(digits) : invalid();
+  const timeOf = (value: string): SourceTime => value === "unknown" ? { basis: "unknown" } :
+    value.startsWith("journal:") ? { basis: "journal", utc: value.slice(8) } : { basis: "message", utc: value };
+  const located = take(LOCATOR), sourceIndex = located ? integer(located[1]!) : undefined;
+  const title = take(TITLE_TOKEN)?.[0], role = title === undefined ? take(ROLE)?.[0] : undefined;
+  const id = role === "custom" ? take(ID)?.[1] : undefined;
+  const omitted = title === undefined && take(role === undefined ? OMITTED : SPACE_OMITTED) !== undefined;
+  if (title === undefined && role === undefined && !omitted) invalid();
+  const common = { ...(sourceIndex !== undefined ? { sourceIndex } : {}) };
+  let record: Record<string, unknown>, text: string;
+  const end = (): string => {
+    if (line[at++] !== ":") return invalid();
+    const rest = line.slice(at);
+    return rest === "" ? "" : rest[0] === " " && rest.length > 1 && !rest.includes("\r") ? unescapeText(rest.slice(1)) : invalid();
+  };
+  const count = omitted && role !== undefined && sourceIndex === undefined && id === undefined ? take(COUNT) : undefined;
+  if (title !== undefined) {
+    text = end();
+    record = { kind: "section", title, text, ...common };
+  } else if (count) {
+    const items: { sourceIndex: number; time?: SourceTime }[] = [];
+    for (let item; (item = take(LOCATOR_TIME));) items.push({ sourceIndex: integer(item[1]!), ...(item[2] !== undefined ? { time: timeOf(item[2]) } : {}) });
+    const stamped = items.filter(item => item.time).length;
+    if (stamped && stamped !== items.length) invalid();
+    record = { kind: "omission", role, count: integer(count[1]!), ...(items.length ? { sourceIndices: items.map(item => item.sourceIndex) } : {}), reason: end(),
+      ...(stamped ? { sourceTimes: items.map(({ sourceIndex: n, time }) => ({ sourceIndex: n, ...time! })) } : {}) };
+  } else {
+    const time = take(TIME_TOKEN), span = take(SPAN);
+    if (time && sourceIndex === undefined) invalid();
+    const fields = {
+      ...(span ? { span: { coordinateSpace: "sanitized-block" as const, start: integer(span[1]!), end: integer(span[2]!), totalCharacters: integer(span[3]!) } } : {}),
+      ...(id !== undefined ? { sourceEntryId: JSON.parse(id) as string } : {}),
+      ...(time ? { sourceTimes: [{ sourceIndex: sourceIndex!, ...timeOf(time[1]!) }] } : {}),
+    };
+    text = end();
+    record = omitted ? { kind: "omission", ...(role !== undefined ? { role } : {}), ...common, reason: text, ...fields } :
+      { kind: "excerpt", role, ...common, text, ...fields };
+  }
+  return record;
+};
+
+/** The structure of a record, whichever format it was read from: roles, source times, spans and, for the v1-only kinds, receipts. */
+const validate = (r: unknown): TranscriptRecord => {
+  if (!r || typeof r !== "object" || Array.isArray(r)) throw new Error("Invalid pi-tuck transcript record");
+  const v = r as TranscriptRecord;
+  const roles = ["user", "assistant", "custom", "bashExecution", "toolResult"];
+  const integer = (n: unknown) => typeof n === "number" && Number.isSafeInteger(n) && n >= 0;
+  if (!["excerpt", "tool_call", "tool_receipt", "tool_excerpt", "bash", "omission", "section"].includes(v.kind) ||
+    (v.role !== undefined && !roles.includes(v.role)) ||
+    (v.sourceTime !== undefined && (v.sourceEntryId === undefined || !validSourceTime(v.sourceTime))) ||
+    (v.sourceTimes !== undefined && (!Array.isArray(v.sourceTimes) || !v.sourceTimes.length || v.sourceTimes.some(t => !integer(t.sourceIndex) || !validSourceTime(Object.fromEntries(Object.entries(t).filter(([key]) => key !== "sourceIndex")))))) ||
+    (v.sourceIndex !== undefined && !integer(v.sourceIndex)) ||
+    (v.sourceEntryId !== undefined && (typeof v.sourceEntryId !== "string" || !v.sourceEntryId || v.sourceEntryId.length > 200 ||
+      v.role !== "custom" || !["excerpt", "omission"].includes(v.kind))) ||
+    (v.sourceIndices !== undefined && (!Array.isArray(v.sourceIndices) || !v.sourceIndices.every(integer))) ||
+    (v.kind !== "omission" && typeof v.text !== "string") ||
+    (v.kind === "omission" && typeof v.reason !== "string") ||
+    (v.kind === "excerpt" && !["user", "assistant", "custom"].includes(v.role)) ||
+    (v.kind === "tool_call" && v.role !== "assistant") ||
+    (v.kind === "bash" && v.role !== "bashExecution") ||
+    (v.kind === "section" && typeof v.title !== "string") ||
+    (v.count !== undefined && !integer(v.count)) ||
+    (v.span !== undefined && (v.span.coordinateSpace !== "sanitized-block" || !integer(v.span.start) || !integer(v.span.end) || !integer(v.span.totalCharacters) ||
+      v.span.start > v.span.end || v.span.end > v.span.totalCharacters ||
+      (v.kind === "excerpt" && v.text!.length !== v.span.end - v.span.start))))
+    throw new Error("Invalid pi-tuck transcript record");
+  if (v.sourceTimes) {
+    const expected = v.sourceIndices ?? (v.sourceIndex === undefined ? [] : [v.sourceIndex]);
+    if (v.sourceTimes.length !== expected.length || v.sourceTimes.some((t, i) => t.sourceIndex !== expected[i])) throw new Error("Mismatched source times");
+  }
+  if (["arguments", "argumentsJSONPrefix", "completeArgumentMembers"].some(key => Object.hasOwn(v, key))) {
+    if (v.kind !== "tool_call" ||
+      (Object.hasOwn(v, "arguments") && (!smallOriginalArguments(v.arguments, Number.MAX_SAFE_INTEGER) ||
+        Object.hasOwn(v, "argumentsJSONPrefix") || Object.hasOwn(v, "completeArgumentMembers"))) ||
+      (Object.hasOwn(v, "argumentsJSONPrefix") && (typeof v.argumentsJSONPrefix !== "string" || !v.argumentsJSONPrefix)) ||
+      (Object.hasOwn(v, "completeArgumentMembers") && (!Object.hasOwn(v, "argumentsJSONPrefix") ||
+        !smallOriginalArguments(v.completeArgumentMembers, Number.MAX_SAFE_INTEGER) || !Object.keys(v.completeArgumentMembers).length)))
+      throw new Error("Invalid historical argument fields");
+  }
+  if (v.kind === "tool_receipt" || v.kind === "tool_excerpt") {
+    const p = v.producer;
+    const keys = [...(v.sourceTimes ? ["sourceTimes"] : []), "kind", "role", "sourceIndex", "sourceIndices", "historical", "isError", "text", "producer", ...(v.kind === "tool_excerpt" ? ["complete", "notice", "range"] : [])];
+    if (Object.keys(v).length !== keys.length || !Object.keys(v).every(k => keys.includes(k)) ||
+      v.role !== "toolResult" || v.historical !== true || typeof v.isError !== "boolean" ||
+      !v.text || v.text.length > MAX_RECEIPT_CHARACTERS || !integer(v.sourceIndex) ||
+      !p || typeof p !== "object" || Array.isArray(p) || Object.keys(p).length !== 4 ||
+      !Object.keys(p).every(k => ["sourceIndex", "toolName", "toolCallId", "arguments", "argumentsOmitted"].includes(k)) ||
+      !integer(p.sourceIndex) || p.sourceIndex >= v.sourceIndex ||
+      typeof p.toolName !== "string" || !p.toolName.trim() || p.toolName.length > 160 ||
+      typeof p.toolCallId !== "string" || !p.toolCallId || p.toolCallId.length > 160 ||
+      !(Object.hasOwn(p, "arguments") && !Object.hasOwn(p, "argumentsOmitted") && smallOriginalArguments(p.arguments) ||
+        p.argumentsOmitted === true && !Object.hasOwn(p, "arguments")) ||
+      !Array.isArray(v.sourceIndices) || v.sourceIndices.length !== 2 ||
+      v.sourceIndices[0] !== p.sourceIndex || v.sourceIndices[1] !== v.sourceIndex)
+      throw new Error("Invalid historical receipt record");
+    if (v.kind === "tool_excerpt" && (v.complete !== false || ![RECEIPT_RANGE_NOTICE, LEGACY_RECEIPT_RANGE_NOTICE].includes(v.notice) || !v.range ||
+      Object.keys(v.range).length !== 4 || !Object.keys(v.range).every(k => ["coordinateSpace", "start", "end", "totalCharacters"].includes(k)) ||
+      v.range.coordinateSpace !== "readable-entry-utf16" || !integer(v.range.start) || !integer(v.range.end) || !integer(v.range.totalCharacters) ||
+      v.range.start >= v.range.end || v.range.end > v.range.totalCharacters || v.range.totalCharacters <= MAX_RECEIPT_CHARACTERS ||
+      v.text.length !== v.range.end - v.range.start)) throw new Error("Invalid historical receipt range");
+  }
+  return v;
+};
+
+/** Accept our explicit record protocol only. Never recover roles from payload text. `version` is the summary header's: v1 reads one JSON
+ * object per line, v2 the lines described above. Left out, it is taken from the first line, which a v1 record always begins with "{" and a v2 never. */
+export const parseRecords = (text: string, version: TranscriptVersion = text.startsWith("{") ? 1 : 2): TranscriptRecord[] => {
   if (!text) return [];
-  return text.split("\n").map(line => {
-    const r: unknown = JSON.parse(line);
-    if (!r || typeof r !== "object" || Array.isArray(r)) throw new Error("Invalid pi-tuck transcript record");
-    const v = r as TranscriptRecord;
-    const roles = ["user", "assistant", "custom", "bashExecution", "toolResult"];
-    const integer = (n: unknown) => typeof n === "number" && Number.isSafeInteger(n) && n >= 0;
-    if (!["excerpt", "tool_call", "tool_receipt", "tool_excerpt", "bash", "omission", "section"].includes(v.kind) ||
-      (v.role !== undefined && !roles.includes(v.role)) ||
-      (v.sourceTime !== undefined && (v.sourceEntryId === undefined || !validSourceTime(v.sourceTime))) ||
-      (v.sourceTimes !== undefined && (!Array.isArray(v.sourceTimes) || !v.sourceTimes.length || v.sourceTimes.some(t => !integer(t.sourceIndex) || !validSourceTime(Object.fromEntries(Object.entries(t).filter(([key]) => key !== "sourceIndex")))))) ||
-      (v.sourceIndex !== undefined && !integer(v.sourceIndex)) ||
-      (v.sourceEntryId !== undefined && (typeof v.sourceEntryId !== "string" || !v.sourceEntryId || v.sourceEntryId.length > 200 ||
-        v.role !== "custom" || !["excerpt", "omission"].includes(v.kind))) ||
-      (v.sourceIndices !== undefined && (!Array.isArray(v.sourceIndices) || !v.sourceIndices.every(integer))) ||
-      (v.kind !== "omission" && typeof v.text !== "string") ||
-      (v.kind === "omission" && typeof v.reason !== "string") ||
-      (v.kind === "excerpt" && !["user", "assistant", "custom"].includes(v.role)) ||
-      (v.kind === "tool_call" && v.role !== "assistant") ||
-      (v.kind === "bash" && v.role !== "bashExecution") ||
-      (v.kind === "section" && typeof v.title !== "string") ||
-      (v.count !== undefined && !integer(v.count)) ||
-      (v.span !== undefined && (v.span.coordinateSpace !== "sanitized-block" || !integer(v.span.start) || !integer(v.span.end) || !integer(v.span.totalCharacters) ||
-        v.span.start > v.span.end || v.span.end > v.span.totalCharacters ||
-        (v.kind === "excerpt" && v.text!.length !== v.span.end - v.span.start))))
-      throw new Error("Invalid pi-tuck transcript record");
-    if (v.sourceTimes) {
-      const expected = v.sourceIndices ?? (v.sourceIndex === undefined ? [] : [v.sourceIndex]);
-      if (v.sourceTimes.length !== expected.length || v.sourceTimes.some((t, i) => t.sourceIndex !== expected[i])) throw new Error("Mismatched source times");
-    }
-    if (["arguments", "argumentsJSONPrefix", "completeArgumentMembers"].some(key => Object.hasOwn(v, key))) {
-      if (v.kind !== "tool_call" ||
-        (Object.hasOwn(v, "arguments") && (!smallOriginalArguments(v.arguments, Number.MAX_SAFE_INTEGER) ||
-          Object.hasOwn(v, "argumentsJSONPrefix") || Object.hasOwn(v, "completeArgumentMembers"))) ||
-        (Object.hasOwn(v, "argumentsJSONPrefix") && (typeof v.argumentsJSONPrefix !== "string" || !v.argumentsJSONPrefix)) ||
-        (Object.hasOwn(v, "completeArgumentMembers") && (!Object.hasOwn(v, "argumentsJSONPrefix") ||
-          !smallOriginalArguments(v.completeArgumentMembers, Number.MAX_SAFE_INTEGER) || !Object.keys(v.completeArgumentMembers).length)))
-        throw new Error("Invalid historical argument fields");
-    }
-    if (v.kind === "tool_receipt" || v.kind === "tool_excerpt") {
-      const p = v.producer;
-      const keys = [...(v.sourceTimes ? ["sourceTimes"] : []), "kind", "role", "sourceIndex", "sourceIndices", "historical", "isError", "text", "producer", ...(v.kind === "tool_excerpt" ? ["complete", "notice", "range"] : [])];
-      if (Object.keys(v).length !== keys.length || !Object.keys(v).every(k => keys.includes(k)) ||
-        v.role !== "toolResult" || v.historical !== true || typeof v.isError !== "boolean" ||
-        !v.text || v.text.length > MAX_RECEIPT_CHARACTERS || !integer(v.sourceIndex) ||
-        !p || typeof p !== "object" || Array.isArray(p) || Object.keys(p).length !== 4 ||
-        !Object.keys(p).every(k => ["sourceIndex", "toolName", "toolCallId", "arguments", "argumentsOmitted"].includes(k)) ||
-        !integer(p.sourceIndex) || p.sourceIndex >= v.sourceIndex ||
-        typeof p.toolName !== "string" || !p.toolName.trim() || p.toolName.length > 160 ||
-        typeof p.toolCallId !== "string" || !p.toolCallId || p.toolCallId.length > 160 ||
-        !(Object.hasOwn(p, "arguments") && !Object.hasOwn(p, "argumentsOmitted") && smallOriginalArguments(p.arguments) ||
-          p.argumentsOmitted === true && !Object.hasOwn(p, "arguments")) ||
-        !Array.isArray(v.sourceIndices) || v.sourceIndices.length !== 2 ||
-        v.sourceIndices[0] !== p.sourceIndex || v.sourceIndices[1] !== v.sourceIndex)
-        throw new Error("Invalid historical receipt record");
-      if (v.kind === "tool_excerpt" && (v.complete !== false || ![RECEIPT_RANGE_NOTICE, LEGACY_RECEIPT_RANGE_NOTICE].includes(v.notice) || !v.range ||
-        Object.keys(v.range).length !== 4 || !Object.keys(v.range).every(k => ["coordinateSpace", "start", "end", "totalCharacters"].includes(k)) ||
-        v.range.coordinateSpace !== "readable-entry-utf16" || !integer(v.range.start) || !integer(v.range.end) || !integer(v.range.totalCharacters) ||
-        v.range.start >= v.range.end || v.range.end > v.range.totalCharacters || v.range.totalCharacters <= MAX_RECEIPT_CHARACTERS ||
-        v.text.length !== v.range.end - v.range.start)) throw new Error("Invalid historical receipt range");
-    }
-    return v;
-  });
+  return text.split("\n").map(line => validate(version === 1 ? JSON.parse(line) : parseLine(line)));
 };