supercompact 1.0.0

package/dist/dialogue.js

@@ -0,0 +1,93 @@
+// Telling apart the two things a session file holds: what people said, and what
+// the machine fetched.
+import { isRecord } from './transcript.js';
+/** The words in a user turn, whether Claude Code stored them as a plain string
+ * or as content blocks.
+ *
+ * A prompt with a pasted image arrives as blocks, so reading only the string
+ * form silently loses every prompt that had a screenshot in it. Tool results
+ * wear the same shape and are not prompts. */
+export function promptText(content) {
+    if (typeof content === 'string')
+        return content;
+    if (!Array.isArray(content))
+        return undefined;
+    const parts = [];
+    for (const block of content) {
+        if (!isRecord(block))
+            continue;
+        if (block.type === 'tool_result')
+            return undefined;
+        if (block.type === 'text' && typeof block.text === 'string')
+            parts.push(block.text);
+    }
+    return parts.length > 0 ? parts.join('\n') : undefined;
+}
+/** What an assistant turn said, ignoring the calls it made. */
+export function assistantText(content) {
+    if (!Array.isArray(content))
+        return '';
+    const parts = [];
+    for (const block of content) {
+        if (isRecord(block) && block.type === 'text' && typeof block.text === 'string') {
+            parts.push(block.text);
+        }
+    }
+    return parts.join('\n');
+}
+// Claude Code files its own runtime chatter under the user's name, wrapped in
+// tags. Nobody typed any of it, so none of it is dialogue.
+const noise = /<(system-reminder|local-command-caveat|command-stdout|command-stderr)\b[^>]*>[\s\S]*?<\/\1>/g;
+export function stripNoise(text) {
+    return text.replace(noise, '').replace(/\n{3,}/g, '\n\n').trim();
+}
+const maxValueChars = 120;
+const maxParamsChars = 240;
+/** A call written out as a sentence, so a rewritten session still shows what
+ * was done without carrying what came back.
+ *
+ *     You used tool Bash(command: npm test, description: Run the tests) */
+export function summarize(block) {
+    const name = typeof block.name === 'string' ? block.name : '';
+    if (name === '')
+        return '';
+    const input = block.input;
+    if (!isRecord(input) || Object.keys(input).length === 0)
+        return `You used tool ${name}`;
+    const parts = [];
+    for (const key of Object.keys(input).sort()) {
+        const value = describe(input[key]);
+        if (value === '')
+            continue;
+        parts.push(`${key}: ${value}`);
+    }
+    if (parts.length === 0)
+        return `You used tool ${name}`;
+    let params = parts.join(', ');
+    if (params.length > maxParamsChars)
+        params = params.slice(0, maxParamsChars) + '…';
+    return `You used tool ${name}(${params})`;
+}
+function describe(value) {
+    if (typeof value === 'string') {
+        const flat = value.replace(/\n/g, ' ');
+        return flat.length > maxValueChars ? flat.slice(0, maxValueChars) + '…' : flat;
+    }
+    if (typeof value === 'number' || typeof value === 'boolean')
+        return String(value);
+    if (Array.isArray(value))
+        return `[${value.length} items]`;
+    if (isRecord(value))
+        return `{${Object.keys(value).length} keys}`;
+    return '';
+}
+/** Two calls are the same call when the tool and its arguments match. */
+export function signature(block) {
+    const name = typeof block.name === 'string' ? block.name : 'tool';
+    try {
+        return name + '(' + JSON.stringify(block.input).slice(0, 400) + ')';
+    }
+    catch {
+        return name;
+    }
+}

package/dist/estimate.js

@@ -0,0 +1,217 @@
+// Token arithmetic, and the one place a session records how big it is.
+//
+// Claude Code, the context reading in the status line and any warning hook all
+// size a session from the usage block on an assistant message. A rewrite that
+// strips those and stops leaves every one of them with nothing to read, so the
+// new size goes back on before the file is written.
+import { decode, encode, isRecord } from './transcript.js';
+/** A character rate, not a tokenizer. Anthropic ships no offline tokenizer, so
+ * anything derived from this is called an estimate wherever a person sees it. */
+export const charsPerToken = 3.7;
+export function tokensInText(text) {
+    return Math.round(text.length / charsPerToken);
+}
+/** Prices a message body, following tool results into whatever they carried. */
+export function tokensInContent(content) {
+    if (typeof content === 'string')
+        return tokensInText(content);
+    if (!Array.isArray(content))
+        return 0;
+    let total = 0;
+    let size;
+    for (const block of content) {
+        if (!isRecord(block))
+            continue;
+        if (block.type === 'text') {
+            const text = typeof block.text === 'string' ? block.text : '';
+            size ??= imageSize(text);
+            total += tokensInText(text);
+        }
+        else if (block.type === 'image') {
+            total += imageTokens(size);
+        }
+        else if (block.type === 'tool_result') {
+            total += tokensInContent(block.content);
+        }
+    }
+    return total;
+}
+export function tokensInJsonl(jsonl) {
+    let total = 0;
+    for (const line of jsonl.split('\n')) {
+        if (line === '')
+            continue;
+        const record = decode(line);
+        if (record === undefined)
+            continue;
+        if (record.type !== 'user' && record.type !== 'assistant')
+            continue;
+        const message = record.message;
+        if (!isRecord(message))
+            continue;
+        total += tokensInContent(message.content);
+    }
+    return total;
+}
+/** Anthropic's own sizing: width times height over 750. A tool result usually
+ * prints the dimensions on the line above the data, which is cheaper to read
+ * than the base64 and much closer to the truth than its length. */
+export function imageTokens(size) {
+    if (size === undefined)
+        return 1500;
+    return Math.min(Math.max(Math.floor((size[0] * size[1]) / 750), 200), 2400);
+}
+function imageSize(text) {
+    const match = /\((\d{2,5})x(\d{2,5})/.exec(text);
+    if (match === null)
+        return undefined;
+    return [Number(match[1]), Number(match[2])];
+}
+/** Below this figure a request never carried a real starting context, so it is
+ * something else and not worth reading. */
+const leastReportedRequest = 5_000;
+/** The subtraction below stays sound only while the transcript in front of the
+ * request is short. Past this the character rate has drifted too far. */
+const mostTranscriptInFront = 25_000;
+/** Above this the session is being charged for a history the file does not
+ * contain, which is what a resumed session looks like from the inside. */
+const mostStartingContext = 150_000;
+/** What a request carried, rather than what a person said.
+ *
+ * `tokensInContent` prices a message for a reader, so it passes over thinking
+ * and over the arguments a tool was called with. The subtraction in
+ * `startingContext` cannot pass over anything, because the figure it subtracts
+ * from is what the API charged for the whole request. */
+export function tokensAsCharged(content) {
+    if (typeof content === 'string')
+        return tokensInText(content);
+    if (!Array.isArray(content))
+        return 0;
+    let total = 0;
+    let size;
+    for (const block of content) {
+        if (!isRecord(block))
+            continue;
+        if (block.type === 'text') {
+            const text = typeof block.text === 'string' ? block.text : '';
+            size ??= imageSize(text);
+            total += tokensInText(text);
+        }
+        else if (block.type === 'thinking') {
+            total += tokensInText(typeof block.thinking === 'string' ? block.thinking : '');
+        }
+        else if (block.type === 'tool_use') {
+            total += tokensInText(JSON.stringify(block.input ?? {}));
+        }
+        else if (block.type === 'image') {
+            total += imageTokens(size);
+        }
+        else if (block.type === 'tool_result') {
+            total += tokensAsCharged(block.content);
+        }
+    }
+    return total;
+}
+/** The context a session carries before anyone has typed anything: the system
+ * prompt, the tool schemas, the CLAUDE.md files, the memory index and the skill
+ * listing. No rewrite removes any of it, and it is paid again every time the
+ * session opens, so a projection that leaves it out reads far below what the
+ * window will show.
+ *
+ * Nothing in the file states the figure. It is the difference between what the
+ * API charged for a request and the weight of the transcript in front of that
+ * request. That subtraction is only sound early: the character rate undercounts
+ * JSON and tool output, the error grows with the transcript, and a long file
+ * hands back its own estimator error along with the answer. Reading it while
+ * the transcript is still short keeps that error out of it.
+ *
+ * Returns 0 when the session has no request worth reading. That is the honest
+ * answer, and every caller treats it as one. */
+export function startingContext(entries) {
+    const reader = new StartingContext();
+    for (const entry of entries) {
+        if (!reader.looking)
+            break;
+        if (entry.type !== 'user' && entry.type !== 'assistant')
+            continue;
+        const message = entry.message;
+        if (message === undefined)
+            continue;
+        // A figure this tool stamped is its own arithmetic rather than something
+        // the API charged, so subtracting a transcript from it proves nothing.
+        if (entry.type === 'assistant' && entry.data.usageIsEstimate !== true) {
+            const usage = message.usage;
+            if (isRecord(usage)) {
+                reader.request(asNumber(usage.input_tokens) +
+                    asNumber(usage.cache_read_input_tokens) +
+                    asNumber(usage.cache_creation_input_tokens));
+                if (!reader.looking)
+                    break;
+            }
+        }
+        reader.carried(message.content);
+    }
+    return reader.value;
+}
+/** The rule above, worked out as a session is read rather than after.
+ *
+ * Two callers need it from different directions. The preview holds the whole
+ * session in memory. `measure` sees one line at a time and never holds two.
+ * Both feed this, so the rule itself exists once. */
+export class StartingContext {
+    inFront = 0;
+    found = 0;
+    done = false;
+    /** Whether there is still any point in reading. */
+    get looking() {
+        return !this.done && this.inFront < mostTranscriptInFront;
+    }
+    get value() {
+        return this.found;
+    }
+    /** A request the API charged for, with everything so far in front of it. */
+    request(reported) {
+        if (!this.looking || reported < leastReportedRequest)
+            return;
+        this.done = true;
+        const starting = reported - this.inFront;
+        if (starting > 0 && starting <= mostStartingContext)
+            this.found = starting;
+    }
+    /** A message the next request will carry. */
+    carried(content) {
+        if (this.looking)
+            this.inFront += tokensAsCharged(content);
+    }
+    /** The same, for a line too large to parse and priced from the outside. */
+    carriedTokens(tokens) {
+        if (this.looking)
+            this.inFront += tokens;
+    }
+}
+function asNumber(value) {
+    return typeof value === 'number' ? value : 0;
+}
+/** Writes the new size onto the last assistant message. */
+export function stampUsage(jsonl, tokens) {
+    const lines = jsonl.replace(/\n+$/, '').split('\n');
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const record = decode(lines[i]);
+        if (record === undefined || record.type !== 'assistant')
+            continue;
+        const message = record.message;
+        if (!isRecord(message))
+            continue;
+        message.usage = {
+            input_tokens: tokens,
+            cache_creation_input_tokens: 0,
+            cache_read_input_tokens: 0,
+            output_tokens: 0,
+        };
+        record.message = message;
+        record.usageIsEstimate = true;
+        lines[i] = encode(record);
+        return lines.join('\n') + '\n';
+    }
+    return jsonl;
+}

package/dist/keep.js

@@ -0,0 +1,76 @@
+// What survives untouched.
+//
+// Dialogue alone is cheap, and it also throws away the state the session was
+// standing in: the file it just read, the page it was just looking at. These
+// hold part of that back.
+import { promptText } from './dialogue.js';
+import { signature } from './dialogue.js';
+export const noKeep = { lastMessages: 0, toolCalls: 0, unique: false };
+export function keepIsEmpty(keep) {
+    return keep.lastMessages <= 0 && keep.toolCalls <= 0;
+}
+export function inWindow(selection, index) {
+    return selection.from !== undefined && index >= selection.from;
+}
+export function plan(transcript, keep) {
+    const selection = { calls: new Set(), picked: 0 };
+    if (keepIsEmpty(keep))
+        return selection;
+    const messages = [];
+    const calls = [];
+    const answered = new Set();
+    for (const entry of transcript.entries) {
+        if (entry.type === 'user') {
+            if (promptText(entry.content) !== undefined) {
+                messages.push(entry.index);
+                continue;
+            }
+            for (const block of entry.blocks) {
+                if (block.type !== 'tool_result')
+                    continue;
+                if (typeof block.tool_use_id === 'string')
+                    answered.add(block.tool_use_id);
+            }
+        }
+        else if (entry.type === 'assistant') {
+            messages.push(entry.index);
+            for (const block of entry.blocks) {
+                if (block.type !== 'tool_use')
+                    continue;
+                if (typeof block.id === 'string') {
+                    calls.push({ index: entry.index, id: block.id, sig: signature(block) });
+                }
+            }
+        }
+    }
+    if (keep.lastMessages > 0 && messages.length > 0) {
+        selection.from = messages[Math.max(0, messages.length - keep.lastMessages)];
+    }
+    // A kept call needs the answer that went with it. A tool_use with nothing
+    // replying to it makes the API refuse the whole conversation, so a call
+    // nobody answered is never chosen.
+    if (keep.toolCalls > 0) {
+        const seen = new Set();
+        for (let i = calls.length - 1; i >= 0; i--) {
+            const call = calls[i];
+            if (inWindow(selection, call.index))
+                continue;
+            if (!answered.has(call.id))
+                continue;
+            if (keep.unique) {
+                if (seen.has(call.sig))
+                    continue;
+                seen.add(call.sig);
+            }
+            selection.calls.add(call.id);
+            selection.picked++;
+            if (selection.picked >= keep.toolCalls)
+                break;
+        }
+    }
+    for (const call of calls) {
+        if (inWindow(selection, call.index))
+            selection.calls.add(call.id);
+    }
+    return selection;
+}

package/dist/measure.js

@@ -0,0 +1,204 @@
+// How much of a context window is the conversation, and how much of the rest
+// can actually be given back.
+//
+// Nothing here is estimated on the heavy side. Every assistant turn carries the
+// token count the API itself reported for that request, so the size of the
+// context is a measured fact rather than a guess. Walk the file, add up the
+// words both sides actually said, and compare the two at the turn where the
+// context was largest.
+//
+// What is removed here is what the tool removes with no options given: every
+// tool call, every tool result, and the images inside them. The keep options
+// hold some of that back, and a session rewritten with them is larger by
+// exactly the amount the preview prices.
+import { createReadStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { assistantText, promptText } from './dialogue.js';
+import { charsPerToken, StartingContext } from './estimate.js';
+import { isRecord } from './transcript.js';
+/** A line this long is always tool output and never anything anyone said. */
+const skipLinesOver = 120_000;
+/** Below this a session never filled anything up and says nothing useful about
+ * a context window. */
+export const minimumContext = 20_000;
+/** An example worth printing comes from a session someone worked in. */
+const exampleContext = 100_000;
+/** A session that never got big says little about a tool for sessions that did.
+ * Below this, most of the context is the starting context, and no rewrite gives
+ * that back. */
+export const heavyContext = 200_000;
+export function reclaimable(size) {
+    return Math.max(0, size.context - size.dialogue - size.starting);
+}
+export function reclaimableTotal(m) {
+    return Math.max(0, m.context - m.dialogue - m.starting);
+}
+export function pooledShare(m) {
+    return m.context === 0 ? 0 : reclaimableTotal(m) / m.context;
+}
+export function medianShare(m) {
+    if (m.shares.length === 0)
+        return 0;
+    return m.shares[Math.floor(m.shares.length / 2)] ?? 0;
+}
+/** The middle session among those that filled up, which is the only group the
+ * tool is for. */
+export function heavyMedianShare(m) {
+    if (m.heavyShares.length === 0)
+        return 0;
+    return m.heavyShares[Math.floor(m.heavyShares.length / 2)] ?? 0;
+}
+/** How many sessions could give back at least this share of their context. */
+export function sessionsPast(m, share) {
+    return m.shares.filter((value) => value >= share).length;
+}
+export async function measure(paths, onProgress) {
+    const m = {
+        sessions: 0,
+        context: 0,
+        dialogue: 0,
+        starting: 0,
+        sizes: [],
+        shares: [],
+        heavyShares: [],
+    };
+    // Reading is the slow part and the files are independent, so a handful are
+    // in flight at once.
+    const width = 8;
+    let next = 0;
+    let done = 0;
+    async function worker() {
+        for (;;) {
+            const index = next++;
+            if (index >= paths.length)
+                return;
+            const path = paths[index];
+            let size;
+            try {
+                size = await measureOne(path);
+            }
+            catch {
+                size = undefined;
+            }
+            done++;
+            if (onProgress && done % 200 === 0)
+                onProgress(done);
+            if (size === undefined)
+                continue;
+            m.sessions++;
+            m.context += size.context;
+            m.dialogue += size.dialogue;
+            m.starting += size.starting;
+            m.sizes.push(size);
+            const share = reclaimable(size) / size.context;
+            m.shares.push(share);
+            if (size.context >= heavyContext)
+                m.heavyShares.push(share);
+            if (size.context >= exampleContext &&
+                size.dialogue > 0 &&
+                (m.heaviest === undefined || size.context > m.heaviest.context)) {
+                m.heaviest = size;
+            }
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(width, paths.length) }, worker));
+    m.shares.sort((a, b) => a - b);
+    m.heavyShares.sort((a, b) => a - b);
+    return m;
+}
+/** The turn where a session's context was largest, how much of it had been said
+ * out loud by then, and how much of it the session was carrying before either
+ * of them had said anything. */
+export async function measureOne(path) {
+    let spoken = 0;
+    const starting = new StartingContext();
+    let best;
+    const lines = createInterface({
+        input: createReadStream(path),
+        crlfDelay: Infinity,
+    });
+    for await (const line of lines) {
+        // A line this long is a screenshot or a very large file read. Parsing it
+        // costs more than the rest of the file put together, so it is priced from
+        // the outside instead.
+        if (line.length > skipLinesOver) {
+            starting.carriedTokens(Math.round(line.length / charsPerToken));
+            continue;
+        }
+        const isUser = line.includes('"type":"user"');
+        const isAssistant = !isUser && line.includes('"type":"assistant"');
+        if (!isUser && !isAssistant)
+            continue;
+        if (isUser && line.includes('"tool_result"')) {
+            let record;
+            try {
+                record = JSON.parse(line);
+            }
+            catch {
+                continue;
+            }
+            if (!isRecord(record))
+                continue;
+            const message = record.message;
+            if (!isRecord(message))
+                continue;
+            starting.carried(message.content);
+            continue;
+        }
+        let record;
+        try {
+            record = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (!isRecord(record))
+            continue;
+        if (record.isSidechain === true)
+            continue;
+        const message = record.message;
+        if (!isRecord(message))
+            continue;
+        if (record.type === 'user') {
+            const text = promptText(message.content);
+            if (text !== undefined)
+                spoken += text.length / charsPerToken;
+            starting.carried(message.content);
+            continue;
+        }
+        if (record.type !== 'assistant')
+            continue;
+        spoken += assistantText(message.content).length / charsPerToken;
+        // A session this tool rewrote carries a stamped estimate. Reading that back
+        // would be measuring our own arithmetic.
+        if (record.usageIsEstimate === true) {
+            starting.carried(message.content);
+            continue;
+        }
+        const usage = message.usage;
+        if (!isRecord(usage)) {
+            starting.carried(message.content);
+            continue;
+        }
+        const context = count(usage.input_tokens) +
+            count(usage.cache_read_input_tokens) +
+            count(usage.cache_creation_input_tokens);
+        starting.request(context);
+        starting.carried(message.content);
+        if (best === undefined || context > best.context) {
+            best = { context, dialogue: Math.min(Math.trunc(spoken), context) };
+        }
+    }
+    if (best === undefined || best.context < minimumContext)
+        return undefined;
+    const room = Math.max(0, best.context - best.dialogue);
+    return {
+        path,
+        context: best.context,
+        dialogue: best.dialogue,
+        starting: Math.min(starting.value, room),
+    };
+}
+function count(value) {
+    return typeof value === 'number' ? value : 0;
+}