pi-tuck 1.0.1 → 1.0.4

package/src/tuck/core/shrink.ts

@@ -0,0 +1,217 @@
+import type { NormalizedBlock } from "../types";
+import { blockRecords, decorateRecord, isDialogue, isOmittable, omissionRecord, type Clip, type DialogueBlock } from "./brief";
+import { DIALOGUE_SUFFIX } from "./format";
+import { orderByOriginalSource } from "./source-order";
+import { serializeRecord, serializeTranscript, TRANSCRIPT_HEADER, type TranscriptRecord } from "./transcript";
+
+type Role = "user" | "assistant" | "notice";
+const ROLES: readonly Role[] = ["user", "assistant", "notice"];
+
+/** The lossy variants of a unit, least clipped first: the words kept from the start of its text and from its end. */
+const TIERS: Record<Role, readonly Clip[]> = {
+  assistant: [[80, 120], [40, 40], [16, 0]],
+  user: [[256, 0], [64, 0], [24, 0]],
+  notice: [[256, 0]],
+};
+const TOOLS_OMITTED: TranscriptRecord = { kind: "omission", reason: "Historical tool actions and observations omitted; read original sources with tuck_recall" };
+
+export interface DialogueInput {
+  /** Normalized, noise-filtered blocks of the pre-cut history in chronological order: dialogue and tool blocks alike. */
+  blocks: NormalizedBlock[];
+  /** Sources the adapter quotes in its own base: they render as nothing here and are not counted as omitted. */
+  mandatorySourceIndices?: ReadonlySet<number>;
+  /** The source index of each original message and, when known, its entry id, for the original order of the records. */
+  sourceIndices: readonly (number | undefined)[];
+  sourceEntryIds?: readonly string[];
+}
+
+/** The configurations the ladder tries, in order; "none" is no configuration that fits. */
+export type Step = "whole" | `A${1 | 2 | 3}` | "B" | `C${1 | 2 | 3}` | "D" | "E1" | "E2" | "none";
+export interface Shrunk {
+  /** The summary text, or "" when nothing fits. */
+  text: string;
+  step: Step;
+  /** The length the ladder computed from per-unit lengths before it rendered `text`. */
+  size: number;
+}
+
+/** What one unit renders in a configuration, and its serialized size: each record's line plus a newline. */
+interface Rendered { records: TranscriptRecord[]; cost: number }
+const rendered = (records: TranscriptRecord[]): Rendered => ({ records, cost: records.reduce((n, r) => n + serializeRecord(r).length + 1, 0) });
+
+/** Units [0, drop) of a role are dropped, [drop, end) are clipped to `tier`, and the rest stay whole. */
+interface Setting { drop: number; tier: number; end: number }
+type Config = Record<Role, Setting>;
+const WHOLE: Setting = { drop: 0, tier: 0, end: 0 };
+
+/** The units of one role in chronological order, with each variant's cost summed over every prefix. */
+const createLane = (role: Role, units: DialogueBlock[], mandatory: ReadonlySet<number> | undefined, decorate: (record: TranscriptRecord) => TranscriptRecord) => {
+  const n = units.length;
+  const records = (block: DialogueBlock, clip?: Clip) => rendered(blockRecords(block, mandatory, clip).map(decorate));
+  const prefix = (level: Rendered[]) => { const sums = new Float64Array(n + 1); level.forEach((r, i) => sums[i + 1] = sums[i]! + r.cost); return sums; };
+  const levels: Rendered[][] = [units.map(block => records(block))];
+  const sums = [prefix(levels[0]!)];
+  // A clip keeps whole passages, so a unit that is one long sentence, code block or JSON region can keep no text at all. Such a clip renders
+  // nothing and its unit is counted in the role's one omission notice, like a dropped unit: a notice per empty clip would cost more than text.
+  const NONE: Rendered = { records: [], cost: 0 };
+  const emptied: Uint8Array[] = [new Uint8Array(n)];
+  const emptyCounts: Int32Array[] = [new Int32Array(n + 1)];
+  /** Level t holds each unit's tier-t variant. A clip that would not make the unit smaller is not taken, so sizes never grow with t. */
+  const level = (t: number): Rendered[] => {
+    while (levels.length <= t) {
+      const previous = levels[levels.length - 1]!, wasEmpty = emptied[emptied.length - 1]!, clip = TIERS[role][levels.length - 1]!;
+      const empty = new Uint8Array(n);
+      const next = units.map((block, i) => {
+        if (wasEmpty[i]) { empty[i] = 1; return previous[i]!; }
+        const clipped = records(block, clip);
+        if (role !== "notice" && isOmittable(block, mandatory) && !clipped.records.some(record => record.kind === "excerpt")) { empty[i] = 1; return NONE; }
+        return clipped.cost < previous[i]!.cost ? clipped : previous[i]!;
+      });
+      const counts = new Int32Array(n + 1);
+      empty.forEach((e, i) => counts[i + 1] = counts[i]! + e);
+      levels.push(next); sums.push(prefix(next)); emptied.push(empty); emptyCounts.push(counts);
+    }
+    return levels[t]!;
+  };
+  const sum = (t: number) => { level(t); return sums[t]!; };
+
+  // The notice for a dropped prefix of p units: how many unselected text blocks, the first locators, and whether some have none.
+  let tally: { counts: Int32Array; unlocated: Uint8Array; smallest: number[][] } | undefined;
+  /** The notice for the units a setting omits: the dropped prefix, and the clips in [drop, end) that kept no text. */
+  const omission = ({ drop, tier, end }: Setting): TranscriptRecord | undefined => {
+    if (role === "notice") return undefined;
+    if (!tally) {
+      tally = { counts: new Int32Array(n + 1), unlocated: new Uint8Array(n + 1), smallest: new Array(n + 1) };
+      let smallest: number[] = [], count = 0, unlocated = 0;
+      tally.smallest[0] = smallest;
+      units.forEach((block, i) => {
+        if (isOmittable(block, mandatory)) {
+          count++;
+          const source = block.sourceIndex;
+          if (source === undefined) unlocated = 1;
+          else if (!smallest.includes(source) && (smallest.length < 9 || source < smallest[8]!)) {
+            smallest = [...smallest, source].sort((a, b) => a - b).slice(0, 9);
+          }
+        }
+        tally!.counts[i + 1] = count; tally!.unlocated[i + 1] = unlocated; tally!.smallest[i + 1] = smallest;
+      });
+    }
+    let count = tally.counts[drop]!, unlocated = tally.unlocated[drop] === 1, sources = tally.smallest[drop]!;
+    if (end > drop && tier > 0) {
+      level(tier);
+      const empty = emptied[tier]!, counts = emptyCounts[tier]!;
+      count += counts[end]! - counts[drop]!;
+      if (counts[end]! > counts[drop]!) {
+        // Text blocks of one message share a locator: collect distinct ones, enough to tell whether the shown list is complete.
+        const extra = new Set<number>();
+        for (let i = drop; i < end && (extra.size < 9 || !unlocated); i++) if (empty[i]) {
+          const source = units[i]!.sourceIndex;
+          if (source === undefined) unlocated = true; else if (extra.size < 9) extra.add(source);
+        }
+        sources = [...new Set([...sources, ...extra])].sort((a, b) => a - b).slice(0, 9);
+      }
+    }
+    return count ? decorate(omissionRecord(role, count, sources, unlocated)) : undefined;
+  };
+  const omissionCosts = new Map<string, number>();
+  const omissionCost = (setting: Setting) => {
+    const key = `${setting.drop}:${setting.tier}:${setting.end}`;
+    let cost = omissionCosts.get(key);
+    if (cost === undefined) { const record = omission(setting); omissionCosts.set(key, cost = record ? rendered([record]).cost : 0); }
+    return cost;
+  };
+
+  return {
+    n, level, sum, omission,
+    cost: ({ drop, tier, end }: Setting) => sums[0]![n]! - sums[0]![Math.max(drop, end)]! + (end > drop ? sum(tier)[end]! - sum(tier)[drop]! : 0) + omissionCost({ drop, tier, end }),
+  };
+};
+
+/** The units of a dialogue and what any configuration of them renders, by arithmetic over per-unit lengths or as the summary text. */
+const createDialogue = (input: DialogueInput) => {
+  const dialogue = input.blocks.filter(isDialogue);
+  const times = dialogue.find(block => block.sourceTimes)?.sourceTimes;
+  const decorate = (record: TranscriptRecord) => decorateRecord(record, times);
+  const lanes = Object.fromEntries(ROLES.map(role => [role, createLane(role, dialogue.filter(block => block.kind === role), input.mandatorySourceIndices, decorate)])) as
+    Record<Role, ReturnType<typeof createLane>>;
+  const seen: Record<Role, number> = { user: 0, assistant: 0, notice: 0 };
+  const order = dialogue.map(block => ({ role: block.kind, i: seen[block.kind]++ }));
+  const tools = input.blocks.length > dialogue.length ? rendered([TOOLS_OMITTED]) : undefined;
+  const overhead = TRANSCRIPT_HEADER.length + DIALOGUE_SUFFIX.length;
+
+  /** The length of the summary text of a configuration: its header, one line per record, and the closing section; 0 when it has no record. */
+  const size = (config: Config): number => {
+    const cost = ROLES.reduce((n, role) => n + lanes[role].cost(config[role]), tools?.cost ?? 0);
+    return cost === 0 ? 0 : overhead + cost;
+  };
+  const text = (config: Config): string => {
+    const records: TranscriptRecord[] = [];
+    for (const { role, i } of order) {
+      const { drop, tier, end } = config[role];
+      if (i >= drop) records.push(...lanes[role].level(i < end ? tier : 0)[i]!.records);
+    }
+    for (const role of ROLES) { const record = lanes[role].omission(config[role]); if (record) records.push(record); }
+    if (tools) records.push(...tools.records);
+    orderByOriginalSource(records, input.sourceIndices, input.sourceEntryIds);
+    return records.length ? serializeTranscript(records) + DIALOGUE_SUFFIX : "";
+  };
+  return { lanes, size, text };
+};
+
+/** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted. "" when it renders nothing. */
+export const wholeDialogue = (input: DialogueInput): string => {
+  const { text } = createDialogue(input);
+  return text({ user: WHOLE, assistant: WHOLE, notice: WHOLE });
+};
+
+/**
+ * The same dialogue shrunk until its summary text is at most `limit` characters, oldest first. The first configuration that fits wins:
+ *   A. the newest k assistant units whole, the older ones clipped to tier 1, 2, 3 (the largest k that fits, for each tier in turn);
+ *   B. the newest assistant units that fit a third of the limit whole, the older ones at tier 3 with the oldest d dropped;
+ *   C. the newest assistant units that fit a third of the limit at tier 3, the rest dropped; then the oldest u user units clipped to tier 1, 2, 3;
+ *   D. every user unit at tier 3, the oldest dropped;
+ *   E. the oldest notices dropped, then the oldest of the remaining assistant units.
+ * Every step is a one-parameter family whose size is read from prefix sums of the per-unit lengths, so a step costs O(units). A notice is clipped
+ * to its one tier in every configuration but the whole dialogue. The size is arithmetic over unit lengths; the rendered text has the last say:
+ * a configuration whose text is over the limit is passed over.
+ */
+export const shrinkDialogue = (input: DialogueInput, limit: number): Shrunk => {
+  const { lanes, size, text } = createDialogue(input);
+  const { user, assistant, notice } = lanes;
+  const clipped: Setting = { drop: 0, tier: 1, end: notice.n };
+  const found = (step: Step, config: Config): Shrunk | undefined => {
+    const length = size(config);
+    if (length > limit) return undefined;
+    const summary = text(config);
+    return summary.length <= limit ? { text: summary, step, size: length } : undefined;
+  };
+  const gone = (lane: { n: number }): Setting => ({ drop: lane.n, tier: 3, end: lane.n });
+  const tiers = [1, 2, 3] as const;
+
+  let hit = found("whole", { user: WHOLE, assistant: WHOLE, notice: WHOLE });
+  if (hit) return hit;
+  for (const t of tiers) for (let k = assistant.n; k >= 0; k--)
+    if (hit = found(`A${t}`, { user: WHOLE, assistant: { drop: 0, tier: t, end: assistant.n - k }, notice: clipped })) return hit;
+  // The newest assistant units that fit a third of the limit whole are kept until users have given way: B clips and drops only older ones.
+  // A clip can keep no text at all, so the third is sized on whole units; when the newest alone is over the third, it keeps its largest clip
+  // that fits and still has text.
+  let kept = 0;
+  while (kept < assistant.n && assistant.sum(0)[assistant.n]! - assistant.sum(0)[kept]! > limit / 3) kept++;
+  let recent: Setting = { drop: kept, tier: 3, end: kept };
+  if (kept === assistant.n && assistant.n > 0) for (const t of tiers) {
+    const cost = assistant.level(t)[assistant.n - 1]!.cost;
+    if (cost > 0 && cost <= limit / 3) { recent = { drop: assistant.n - 1, tier: t, end: assistant.n }; break; }
+  }
+  for (let d = 0; d <= recent.drop; d++)
+    if (hit = found("B", { user: WHOLE, assistant: { ...recent, drop: d }, notice: clipped })) return hit;
+  // From here users give way, and the assistant keeps only that recent third.
+  for (const t of tiers) for (let u = 0; u <= user.n; u++)
+    if (hit = found(`C${t}`, { user: { drop: 0, tier: t, end: u }, assistant: recent, notice: clipped })) return hit;
+  for (let d = 0; d <= user.n; d++)
+    if (hit = found("D", { user: { drop: d, tier: 3, end: user.n }, assistant: recent, notice: clipped })) return hit;
+  for (let d = 0; d <= notice.n; d++)
+    if (hit = found("E1", { user: gone(user), assistant: recent, notice: { drop: d, tier: 1, end: notice.n } })) return hit;
+  for (let d = recent.drop; d <= assistant.n; d++)
+    if (hit = found("E2", { user: gone(user), assistant: { ...recent, drop: d }, notice: gone(notice) })) return hit;
+  return { text: "", step: "none", size: 0 };
+};

package/src/tuck/core/summarize.ts

@@ -1,41 +1,43 @@
-import { orderByOriginalSource } from "./source-order";
 import type { CompilerMessage } from "../types";
 import { normalize } from "./normalize";
 import { filterNoise } from "./filter-noise";
-import { compileBrief, planBrief, buildBriefRecords } from "./brief";
-import { DIALOGUE_SUFFIX, RANKED_SUFFIX } from "./format";
-import { parseRecords, serializeTranscript } from "./transcript";
-import { selectRankedBriefBlocks, type BriefRankingOptions } from "./rank";
+import { shrinkDialogue, wholeDialogue, type DialogueInput } from "./shrink";
 
-export interface RankedCompileInput {
+export interface CompileInput {
   messages: CompilerMessage[];
-  /** Eligible original branch only; receipt identity/order, not extra ranked content. */
+  /** Eligible original branch only: receipt identity and order. */
   receiptContext?: { messages: CompilerMessage[]; sourceIndices: Array<number | undefined> };
   /** Native session-global source indices parallel to messages. */
   sourceIndices?: Array<number | undefined>;
   /** Actual native entry IDs parallel to originals, not inferred from text. */
   sourceEntryIds?: readonly string[];
   sourceTimes?: readonly import("./source-time").SourceTime[];
-  ranking?: BriefRankingOptions;
+  ranking?: {
+    /** The complete serialized length a summary of the dialogue may have, header and closing section included. */
+    maxBriefChars?: number;
+    /** Full sources rendered separately by the caller: they render as nothing here. */
+    mandatorySourceIndices?: ReadonlySet<number>;
+  };
 }
 
-/** Select eligible originals once. No inferred state or prior-view merging. */
-export const compileRanked = (input: RankedCompileInput): string => {
-  const blocks = filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes));
-  const selected = selectRankedBriefBlocks(blocks, input.ranking);
-  const transcript = serializeTranscript(parseRecords(compileBrief(selected)));
-  return transcript ? transcript + RANKED_SUFFIX : "";
+// A cut offers the whole dialogue and then, when it does not fit, the shrunk one, from the same input: normalize it once. The blocks are
+// never mutated, and keeping the same objects also keeps their cached passage analyses.
+const inputs = new WeakMap<CompileInput, DialogueInput>();
+const dialogueInput = (input: CompileInput): DialogueInput => {
+  let prepared = inputs.get(input);
+  if (!prepared) inputs.set(input, prepared = {
+    blocks: filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes)),
+    mandatorySourceIndices: input.ranking?.mandatorySourceIndices,
+    sourceIndices: input.sourceIndices ?? input.messages.map((_, index) => index),
+    sourceEntryIds: input.sourceEntryIds,
+  });
+  return prepared;
 };
 
 /** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted.
  * This disposable view grants neither present authority nor native tool execution. */
-export const compileDialogue = (input: RankedCompileInput): string => {
-  const originals = filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes));
-  const blocks = originals.filter(block => ["user", "assistant", "notice"].includes(block.kind));
-  planBrief(blocks, new Set(blocks), originals, input.ranking?.mandatorySourceIndices);
-  const records = buildBriefRecords(blocks);
-  if (blocks.length < originals.length) records.push({ kind: "omission",
-    reason: "Historical tool actions and observations omitted; read original sources with tuck_recall" });
-  orderByOriginalSource(records, input.sourceIndices ?? input.messages.map((_, index) => index), input.sourceEntryIds);
-  return records.length ? serializeTranscript(records) + DIALOGUE_SUFFIX : "";
-};
+export const compileDialogue = (input: CompileInput): string => wholeDialogue(dialogueInput(input));
+
+/** The dialogue shrunk to `ranking.maxBriefChars`, oldest assistant text first (see `shrinkDialogue`); "" when nothing fits. */
+export const compileShrunkDialogue = (input: CompileInput & { ranking: { maxBriefChars: number } }): string =>
+  shrinkDialogue(dialogueInput(input), input.ranking.maxBriefChars).text;