pi-tuck 1.0.1 → 1.0.4

package/dist/src/tuck/core/shrink.js

@@ -0,0 +1,218 @@
+import { blockRecords, decorateRecord, isDialogue, isOmittable, omissionRecord } from "./brief.js";
+import { DIALOGUE_SUFFIX } from "./format.js";
+import { orderByOriginalSource } from "./source-order.js";
+import { serializeRecord, serializeTranscript, TRANSCRIPT_HEADER } from "./transcript.js";
+const ROLES = ["user", "assistant", "notice"];
+/** The lossy variants of a unit, least clipped first: the words kept from the start of its text and from its end. */
+const TIERS = {
+    assistant: [[80, 120], [40, 40], [16, 0]],
+    user: [[256, 0], [64, 0], [24, 0]],
+    notice: [[256, 0]],
+};
+const TOOLS_OMITTED = { kind: "omission", reason: "Historical tool actions and observations omitted; read original sources with tuck_recall" };
+const rendered = (records) => ({ records, cost: records.reduce((n, r) => n + serializeRecord(r).length + 1, 0) });
+const WHOLE = { drop: 0, tier: 0, end: 0 };
+/** The units of one role in chronological order, with each variant's cost summed over every prefix. */
+const createLane = (role, units, mandatory, decorate) => {
+    const n = units.length;
+    const records = (block, clip) => rendered(blockRecords(block, mandatory, clip).map(decorate));
+    const prefix = (level) => { const sums = new Float64Array(n + 1); level.forEach((r, i) => sums[i + 1] = sums[i] + r.cost); return sums; };
+    const levels = [units.map(block => records(block))];
+    const sums = [prefix(levels[0])];
+    // A clip keeps whole passages, so a unit that is one long sentence, code block or JSON region can keep no text at all. Such a clip renders
+    // nothing and its unit is counted in the role's one omission notice, like a dropped unit: a notice per empty clip would cost more than text.
+    const NONE = { records: [], cost: 0 };
+    const emptied = [new Uint8Array(n)];
+    const emptyCounts = [new Int32Array(n + 1)];
+    /** Level t holds each unit's tier-t variant. A clip that would not make the unit smaller is not taken, so sizes never grow with t. */
+    const level = (t) => {
+        while (levels.length <= t) {
+            const previous = levels[levels.length - 1], wasEmpty = emptied[emptied.length - 1], clip = TIERS[role][levels.length - 1];
+            const empty = new Uint8Array(n);
+            const next = units.map((block, i) => {
+                if (wasEmpty[i]) {
+                    empty[i] = 1;
+                    return previous[i];
+                }
+                const clipped = records(block, clip);
+                if (role !== "notice" && isOmittable(block, mandatory) && !clipped.records.some(record => record.kind === "excerpt")) {
+                    empty[i] = 1;
+                    return NONE;
+                }
+                return clipped.cost < previous[i].cost ? clipped : previous[i];
+            });
+            const counts = new Int32Array(n + 1);
+            empty.forEach((e, i) => counts[i + 1] = counts[i] + e);
+            levels.push(next);
+            sums.push(prefix(next));
+            emptied.push(empty);
+            emptyCounts.push(counts);
+        }
+        return levels[t];
+    };
+    const sum = (t) => { level(t); return sums[t]; };
+    // The notice for a dropped prefix of p units: how many unselected text blocks, the first locators, and whether some have none.
+    let tally;
+    /** The notice for the units a setting omits: the dropped prefix, and the clips in [drop, end) that kept no text. */
+    const omission = ({ drop, tier, end }) => {
+        if (role === "notice")
+            return undefined;
+        if (!tally) {
+            tally = { counts: new Int32Array(n + 1), unlocated: new Uint8Array(n + 1), smallest: new Array(n + 1) };
+            let smallest = [], count = 0, unlocated = 0;
+            tally.smallest[0] = smallest;
+            units.forEach((block, i) => {
+                if (isOmittable(block, mandatory)) {
+                    count++;
+                    const source = block.sourceIndex;
+                    if (source === undefined)
+                        unlocated = 1;
+                    else if (!smallest.includes(source) && (smallest.length < 9 || source < smallest[8])) {
+                        smallest = [...smallest, source].sort((a, b) => a - b).slice(0, 9);
+                    }
+                }
+                tally.counts[i + 1] = count;
+                tally.unlocated[i + 1] = unlocated;
+                tally.smallest[i + 1] = smallest;
+            });
+        }
+        let count = tally.counts[drop], unlocated = tally.unlocated[drop] === 1, sources = tally.smallest[drop];
+        if (end > drop && tier > 0) {
+            level(tier);
+            const empty = emptied[tier], counts = emptyCounts[tier];
+            count += counts[end] - counts[drop];
+            if (counts[end] > counts[drop]) {
+                // Text blocks of one message share a locator: collect distinct ones, enough to tell whether the shown list is complete.
+                const extra = new Set();
+                for (let i = drop; i < end && (extra.size < 9 || !unlocated); i++)
+                    if (empty[i]) {
+                        const source = units[i].sourceIndex;
+                        if (source === undefined)
+                            unlocated = true;
+                        else if (extra.size < 9)
+                            extra.add(source);
+                    }
+                sources = [...new Set([...sources, ...extra])].sort((a, b) => a - b).slice(0, 9);
+            }
+        }
+        return count ? decorate(omissionRecord(role, count, sources, unlocated)) : undefined;
+    };
+    const omissionCosts = new Map();
+    const omissionCost = (setting) => {
+        const key = `${setting.drop}:${setting.tier}:${setting.end}`;
+        let cost = omissionCosts.get(key);
+        if (cost === undefined) {
+            const record = omission(setting);
+            omissionCosts.set(key, cost = record ? rendered([record]).cost : 0);
+        }
+        return cost;
+    };
+    return {
+        n, level, sum, omission,
+        cost: ({ drop, tier, end }) => sums[0][n] - sums[0][Math.max(drop, end)] + (end > drop ? sum(tier)[end] - sum(tier)[drop] : 0) + omissionCost({ drop, tier, end }),
+    };
+};
+/** The units of a dialogue and what any configuration of them renders, by arithmetic over per-unit lengths or as the summary text. */
+const createDialogue = (input) => {
+    const dialogue = input.blocks.filter(isDialogue);
+    const times = dialogue.find(block => block.sourceTimes)?.sourceTimes;
+    const decorate = (record) => decorateRecord(record, times);
+    const lanes = Object.fromEntries(ROLES.map(role => [role, createLane(role, dialogue.filter(block => block.kind === role), input.mandatorySourceIndices, decorate)]));
+    const seen = { user: 0, assistant: 0, notice: 0 };
+    const order = dialogue.map(block => ({ role: block.kind, i: seen[block.kind]++ }));
+    const tools = input.blocks.length > dialogue.length ? rendered([TOOLS_OMITTED]) : undefined;
+    const overhead = TRANSCRIPT_HEADER.length + DIALOGUE_SUFFIX.length;
+    /** The length of the summary text of a configuration: its header, one line per record, and the closing section; 0 when it has no record. */
+    const size = (config) => {
+        const cost = ROLES.reduce((n, role) => n + lanes[role].cost(config[role]), tools?.cost ?? 0);
+        return cost === 0 ? 0 : overhead + cost;
+    };
+    const text = (config) => {
+        const records = [];
+        for (const { role, i } of order) {
+            const { drop, tier, end } = config[role];
+            if (i >= drop)
+                records.push(...lanes[role].level(i < end ? tier : 0)[i].records);
+        }
+        for (const role of ROLES) {
+            const record = lanes[role].omission(config[role]);
+            if (record)
+                records.push(record);
+        }
+        if (tools)
+            records.push(...tools.records);
+        orderByOriginalSource(records, input.sourceIndices, input.sourceEntryIds);
+        return records.length ? serializeTranscript(records) + DIALOGUE_SUFFIX : "";
+    };
+    return { lanes, size, text };
+};
+/** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted. "" when it renders nothing. */
+export const wholeDialogue = (input) => {
+    const { text } = createDialogue(input);
+    return text({ user: WHOLE, assistant: WHOLE, notice: WHOLE });
+};
+/**
+ * The same dialogue shrunk until its summary text is at most `limit` characters, oldest first. The first configuration that fits wins:
+ *   A. the newest k assistant units whole, the older ones clipped to tier 1, 2, 3 (the largest k that fits, for each tier in turn);
+ *   B. the newest assistant units that fit a third of the limit whole, the older ones at tier 3 with the oldest d dropped;
+ *   C. the newest assistant units that fit a third of the limit at tier 3, the rest dropped; then the oldest u user units clipped to tier 1, 2, 3;
+ *   D. every user unit at tier 3, the oldest dropped;
+ *   E. the oldest notices dropped, then the oldest of the remaining assistant units.
+ * Every step is a one-parameter family whose size is read from prefix sums of the per-unit lengths, so a step costs O(units). A notice is clipped
+ * to its one tier in every configuration but the whole dialogue. The size is arithmetic over unit lengths; the rendered text has the last say:
+ * a configuration whose text is over the limit is passed over.
+ */
+export const shrinkDialogue = (input, limit) => {
+    const { lanes, size, text } = createDialogue(input);
+    const { user, assistant, notice } = lanes;
+    const clipped = { drop: 0, tier: 1, end: notice.n };
+    const found = (step, config) => {
+        const length = size(config);
+        if (length > limit)
+            return undefined;
+        const summary = text(config);
+        return summary.length <= limit ? { text: summary, step, size: length } : undefined;
+    };
+    const gone = (lane) => ({ drop: lane.n, tier: 3, end: lane.n });
+    const tiers = [1, 2, 3];
+    let hit = found("whole", { user: WHOLE, assistant: WHOLE, notice: WHOLE });
+    if (hit)
+        return hit;
+    for (const t of tiers)
+        for (let k = assistant.n; k >= 0; k--)
+            if (hit = found(`A${t}`, { user: WHOLE, assistant: { drop: 0, tier: t, end: assistant.n - k }, notice: clipped }))
+                return hit;
+    // The newest assistant units that fit a third of the limit whole are kept until users have given way: B clips and drops only older ones.
+    // A clip can keep no text at all, so the third is sized on whole units; when the newest alone is over the third, it keeps its largest clip
+    // that fits and still has text.
+    let kept = 0;
+    while (kept < assistant.n && assistant.sum(0)[assistant.n] - assistant.sum(0)[kept] > limit / 3)
+        kept++;
+    let recent = { drop: kept, tier: 3, end: kept };
+    if (kept === assistant.n && assistant.n > 0)
+        for (const t of tiers) {
+            const cost = assistant.level(t)[assistant.n - 1].cost;
+            if (cost > 0 && cost <= limit / 3) {
+                recent = { drop: assistant.n - 1, tier: t, end: assistant.n };
+                break;
+            }
+        }
+    for (let d = 0; d <= recent.drop; d++)
+        if (hit = found("B", { user: WHOLE, assistant: { ...recent, drop: d }, notice: clipped }))
+            return hit;
+    // From here users give way, and the assistant keeps only that recent third.
+    for (const t of tiers)
+        for (let u = 0; u <= user.n; u++)
+            if (hit = found(`C${t}`, { user: { drop: 0, tier: t, end: u }, assistant: recent, notice: clipped }))
+                return hit;
+    for (let d = 0; d <= user.n; d++)
+        if (hit = found("D", { user: { drop: d, tier: 3, end: user.n }, assistant: recent, notice: clipped }))
+            return hit;
+    for (let d = 0; d <= notice.n; d++)
+        if (hit = found("E1", { user: gone(user), assistant: recent, notice: { drop: d, tier: 1, end: notice.n } }))
+            return hit;
+    for (let d = recent.drop; d <= assistant.n; d++)
+        if (hit = found("E2", { user: gone(user), assistant: { ...recent, drop: d }, notice: gone(notice) }))
+            return hit;
+    return { text: "", step: "none", size: 0 };
+};

package/dist/src/tuck/core/summarize.d.ts

@@ -1,8 +1,7 @@
 import type { CompilerMessage } from "../types.js";
-import { type BriefRankingOptions } from "./rank.js";
-export interface RankedCompileInput {
+export interface CompileInput {
     messages: CompilerMessage[];
-    /** Eligible original branch only; receipt identity/order, not extra ranked content. */
+    /** Eligible original branch only: receipt identity and order. */
     receiptContext?: {
         messages: CompilerMessage[];
         sourceIndices: Array<number | undefined>;
@@ -12,10 +11,19 @@ export interface RankedCompileInput {
     /** Actual native entry IDs parallel to originals, not inferred from text. */
     sourceEntryIds?: readonly string[];
     sourceTimes?: readonly import("./source-time.js").SourceTime[];
-    ranking?: BriefRankingOptions;
+    ranking?: {
+        /** The complete serialized length a summary of the dialogue may have, header and closing section included. */
+        maxBriefChars?: number;
+        /** Full sources rendered separately by the caller: they render as nothing here. */
+        mandatorySourceIndices?: ReadonlySet<number>;
+    };
 }
-/** Select eligible originals once. No inferred state or prior-view merging. */
-export declare const compileRanked: (input: RankedCompileInput) => string;
 /** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted.
  * This disposable view grants neither present authority nor native tool execution. */
-export declare const compileDialogue: (input: RankedCompileInput) => string;
+export declare const compileDialogue: (input: CompileInput) => string;
+/** The dialogue shrunk to `ranking.maxBriefChars`, oldest assistant text first (see `shrinkDialogue`); "" when nothing fits. */
+export declare const compileShrunkDialogue: (input: CompileInput & {
+    ranking: {
+        maxBriefChars: number;
+    };
+}) => string;

package/dist/src/tuck/core/summarize.js

@@ -1,27 +1,22 @@
-import { orderByOriginalSource } from "./source-order.js";
 import { normalize } from "./normalize.js";
 import { filterNoise } from "./filter-noise.js";
-import { compileBrief, planBrief, buildBriefRecords } from "./brief.js";
-import { DIALOGUE_SUFFIX, RANKED_SUFFIX } from "./format.js";
-import { parseRecords, serializeTranscript } from "./transcript.js";
-import { selectRankedBriefBlocks } from "./rank.js";
-/** Select eligible originals once. No inferred state or prior-view merging. */
-export const compileRanked = (input) => {
-    const blocks = filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes));
-    const selected = selectRankedBriefBlocks(blocks, input.ranking);
-    const transcript = serializeTranscript(parseRecords(compileBrief(selected)));
-    return transcript ? transcript + RANKED_SUFFIX : "";
+import { shrinkDialogue, wholeDialogue } from "./shrink.js";
+// A cut offers the whole dialogue and then, when it does not fit, the shrunk one, from the same input: normalize it once. The blocks are
+// never mutated, and keeping the same objects also keeps their cached passage analyses.
+const inputs = new WeakMap();
+const dialogueInput = (input) => {
+    let prepared = inputs.get(input);
+    if (!prepared)
+        inputs.set(input, prepared = {
+            blocks: filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes)),
+            mandatorySourceIndices: input.ranking?.mandatorySourceIndices,
+            sourceIndices: input.sourceIndices ?? input.messages.map((_, index) => index),
+            sourceEntryIds: input.sourceEntryIds,
+        });
+    return prepared;
 };
 /** The whole pre-cut dialogue: user, assistant and notice records; tool actions and observations are omitted.
  * This disposable view grants neither present authority nor native tool execution. */
-export const compileDialogue = (input) => {
-    const originals = filterNoise(normalize(input.messages, input.sourceIndices, input.receiptContext, input.sourceEntryIds, input.sourceTimes));
-    const blocks = originals.filter(block => ["user", "assistant", "notice"].includes(block.kind));
-    planBrief(blocks, new Set(blocks), originals, input.ranking?.mandatorySourceIndices);
-    const records = buildBriefRecords(blocks);
-    if (blocks.length < originals.length)
-        records.push({ kind: "omission",
-            reason: "Historical tool actions and observations omitted; read original sources with tuck_recall" });
-    orderByOriginalSource(records, input.sourceIndices ?? input.messages.map((_, index) => index), input.sourceEntryIds);
-    return records.length ? serializeTranscript(records) + DIALOGUE_SUFFIX : "";
-};
+export const compileDialogue = (input) => wholeDialogue(dialogueInput(input));
+/** The dialogue shrunk to `ranking.maxBriefChars`, oldest assistant text first (see `shrinkDialogue`); "" when nothing fits. */
+export const compileShrunkDialogue = (input) => shrinkDialogue(dialogueInput(input), input.ranking.maxBriefChars).text;

package/dist/src/tuck/core/transcript.d.ts

@@ -24,6 +24,7 @@ interface RecordFields {
     count?: number;
     title?: string;
 }
+/** The tool_call, tool_receipt, tool_excerpt and bash kinds are v1 only: summaries saved by earlier builds hold them, and this build no longer writes them. */
 export type TranscriptRecord = RecordFields & ({
     kind: "excerpt";
     role: "user" | "assistant" | "custom";
@@ -73,23 +74,33 @@ export type TranscriptRecord = RecordFields & ({
     title: string;
     text: string;
 });
-export declare const TRANSCRIPT_HEADER = "TUCK-TRANSCRIPT/1";
-export declare const TRANSCRIPT_UNIT_CHARACTERS = 120;
-/** UTF-16 serialized characters, including JSON escapes, labels and framing. */
-export declare const lineUnits: (characters: number) => number;
-export declare const transcriptUnits: (text: string) => number;
-export declare const serializeRecords: (records: TranscriptRecord[]) => string;
-export declare const serializeTranscript: (records: TranscriptRecord[]) => string;
+/** The format this build writes. */
+export declare const TRANSCRIPT_HEADER = "TUCK-TRANSCRIPT/2";
+/** The format of 1.0.x summaries: one JSON object per line. Read, never written. */
+export declare const TRANSCRIPT_HEADER_V1 = "TUCK-TRANSCRIPT/1";
+/** The header of summaries saved before the rename to pi-tuck: the v1 format. */
+export declare const LEGACY_TRANSCRIPT_HEADER = "VCC-TRANSCRIPT/1";
+export type TranscriptVersion = 1 | 2;
+/** The header a summary of `version` is written with; a legacy one is rewritten as v1. */
+export declare const transcriptHeader: (version: TranscriptVersion) => string;
 /** Starts a summary's closing section, after the record lines. */
 export declare const CLOSING_SEPARATOR = "\n\n---\n\n";
-/** A summary in this format, split by structure: each record is one JSON line, so the first blank line after the header starts the
- * closing section. Any closing wording, from this build or an earlier one, is accepted; the records are returned byte for byte. */
+/** A summary in either format, split by structure: each record is one line, and no record line is empty, so the first blank line after the
+ * header starts the closing section. Any closing wording, from this build or an earlier one, is accepted; the records are returned byte
+ * for byte, with the format their header names. */
 export declare const splitSummary: (summary: string) => {
     body: string;
     closing: string;
+    version: TranscriptVersion;
 } | undefined;
-/** The header of summaries saved before the rename to pi-tuck. */
-export declare const LEGACY_TRANSCRIPT_HEADER = "VCC-TRANSCRIPT/1";
-/** Accept our explicit record protocol only. Never recover roles from payload text. */
-export declare const parseRecords: (text: string) => TranscriptRecord[];
+/** The length of `text` as a record line carries it: one character more for each "\", line feed and carriage return. */
+export declare const escapedLength: (text: string) => number;
+/** One record as a line of TUCK-TRANSCRIPT/2 (without its newline). The records this build emits only: a record whose fields the grammar
+ * cannot carry exactly is refused, never written lossily. */
+export declare const serializeRecord: (record: TranscriptRecord) => string;
+export declare const serializeRecords: (records: TranscriptRecord[]) => string;
+export declare const serializeTranscript: (records: TranscriptRecord[]) => string;
+/** Accept our explicit record protocol only. Never recover roles from payload text. `version` is the summary header's: v1 reads one JSON
+ * object per line, v2 the lines described above. Left out, it is taken from the first line, which a v1 record always begins with "{" and a v2 never. */
+export declare const parseRecords: (text: string, version?: TranscriptVersion) => TranscriptRecord[];
 export {};