pi-mega-compact 0.4.5 → 0.4.6

package/dist/src/store.test.js

@@ -0,0 +1,142 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { gzipSync } from "node:zlib";
+import { compressSmart, decompressSmart, readGzJson, writeGzJson, normalizeSessionId, nextCheckpointId, } from "./store.js";
+const baseTmp = mkdtempSync(join(tmpdir(), "mc-store-"));
+let counter = 0;
+function tmpDir() {
+    return join(baseTmp, `d-${counter++}`);
+}
+// ---------------------------------------------------------------------------
+// compressSmart / decompressSmart
+// ---------------------------------------------------------------------------
+test("compressSmart: tiny payload (<512B) uses RAW tier", () => {
+    const data = Buffer.from(JSON.stringify({ hello: "world" }));
+    assert.ok(data.length < 512);
+    const compressed = compressSmart(data);
+    // Versioned header: 0xEC 0x01 [version=1] [tier=0x00 RAW]; payload after byte 4.
+    assert.equal(compressed[0], 0xec, "magic hi");
+    assert.equal(compressed[1], 0x01, "magic lo / version marker");
+    assert.equal(compressed[2], 0x01, "format version 1");
+    assert.equal(compressed[3], 0x00, "tag RAW");
+    // Payload after the 4-byte header should be identical to original
+    assert.deepEqual(compressed.subarray(4), data);
+});
+test("compressSmart: medium payload (4KB–32KB) uses GZIP-6 tier", () => {
+    // ~8KB of repetitive text
+    const data = Buffer.from("the quick brown fox jumps over the lazy dog. ".repeat(180));
+    assert.ok(data.length >= 4096 && data.length < 32768);
+    const compressed = compressSmart(data);
+    // Versioned header + tier tag 0x02 (GZIP-6) at byte 3.
+    assert.equal(compressed[0], 0xec);
+    assert.equal(compressed[3], 0x02, "tag GZIP-6");
+    // Compressed should be smaller
+    assert.ok(compressed.length < data.length, "compressed smaller than raw");
+});
+test("compressSmart: large payload (>32KB) uses BROTLI tier", () => {
+    // ~40KB of repetitive text
+    const data = Buffer.from("this is a long summary of a coding session. ".repeat(900));
+    assert.ok(data.length >= 32768);
+    const compressed = compressSmart(data);
+    // Versioned header + tier tag 0x05 (BROTLI_4) at byte 3.
+    assert.equal(compressed[0], 0xec);
+    assert.equal(compressed[3], 0x05, "tag BROTLI_4");
+    // Compressed should be smaller
+    assert.ok(compressed.length < data.length, "brotli compressed smaller than raw");
+});
+test("compressSmart: small payload (512B–4KB) uses GZIP-1 tier", () => {
+    // ~1.5KB
+    const data = Buffer.from("a moderately sized summary with some repetition. ".repeat(30));
+    assert.ok(data.length >= 512 && data.length < 4096);
+    const compressed = compressSmart(data);
+    assert.equal(compressed[0], 0xec);
+    assert.equal(compressed[3], 0x01, "tag GZIP-1");
+});
+test("decompressSmart roundtrips all tiers", () => {
+    const sizes = [
+        { label: "tiny", gen: () => Buffer.from("small") },
+        { label: "small", gen: () => Buffer.from("x".repeat(600)) },
+        { label: "medium", gen: () => Buffer.from("y".repeat(8000)) },
+        { label: "large", gen: () => Buffer.from("z".repeat(40000)) },
+    ];
+    for (const { label, gen } of sizes) {
+        const original = gen();
+        const compressed = compressSmart(original);
+        const decompressed = decompressSmart(compressed);
+        assert.deepEqual(decompressed, original, `roundtrip failed for ${label} (${original.length}B)`);
+    }
+});
+test("decompressSmart handles legacy untagged gzip files (backward compat)", () => {
+    const data = Buffer.from(JSON.stringify({ legacy: true }));
+    const legacyGzip = gzipSync(data); // no tag byte, starts with 0x1f
+    assert.equal(legacyGzip[0], 0x1f, "gzip magic byte present");
+    const result = decompressSmart(legacyGzip);
+    assert.deepEqual(JSON.parse(result.toString()), { legacy: true });
+});
+test("readGzJson / writeGzJson roundtrip with smart compression", () => {
+    const dir = tmpDir();
+    const path = join(dir, "test.json.gz");
+    const data = [{ id: "a", value: 42 }, { id: "b", value: 99 }];
+    writeGzJson(path, data);
+    const loaded = readGzJson(path, []);
+    assert.deepEqual(loaded, data);
+});
+test("readGzJson reads legacy gzip files written by old code", () => {
+    const dir = tmpDir();
+    const path = join(dir, "legacy.json.gz");
+    const data = { old: true };
+    // Simulate old writeGzJson (plain gzip, no tag)
+    mkdirSync(join(path, ".."), { recursive: true });
+    const buf = gzipSync(Buffer.from(JSON.stringify(data), "utf-8"));
+    writeFileSync(path, buf);
+    const loaded = readGzJson(path, { old: false });
+    assert.deepEqual(loaded, { old: true });
+});
+test("compression tier: GZIP-1 and GZIP-6 tiers produce valid, smaller output", () => {
+    // GZIP-1 tier (~600B input, 512B–4KB band)
+    const small = compressSmart(Buffer.from("compress me ".repeat(200)));
+    assert.equal(small[0], 0xec, "versioned magic");
+    assert.equal(small[3], 0x01, "GZIP-1 tag for small input");
+    assert.ok(small.length < 600, "GZIP-1 output smaller than input");
+    assert.deepEqual(decompressSmart(small), Buffer.from("compress me ".repeat(200)));
+    // GZIP-6 tier (~8KB input, 4KB–32KB band)
+    const big = compressSmart(Buffer.from("compress me ".repeat(1800)));
+    assert.equal(big[0], 0xec, "versioned magic");
+    assert.equal(big[3], 0x02, "GZIP-6 tag for medium input");
+    assert.ok(big.length < Buffer.from("compress me ".repeat(1800)).length, "GZIP-6 output smaller than input");
+    assert.deepEqual(decompressSmart(big), Buffer.from("compress me ".repeat(1800)));
+});
+// ---------------------------------------------------------------------------
+// normalizeSessionId
+// ---------------------------------------------------------------------------
+test("normalizeSessionId adds sess_ prefix when missing", () => {
+    assert.equal(normalizeSessionId("abc"), "sess_abc");
+    assert.equal(normalizeSessionId("sess_abc"), "sess_abc");
+});
+// ---------------------------------------------------------------------------
+// nextCheckpointId
+// ---------------------------------------------------------------------------
+test("nextCheckpointId returns chkpt_001 for new session", () => {
+    const dir = tmpDir();
+    assert.equal(nextCheckpointId("sess_nci1", dir), "chkpt_001");
+});
+test("nextCheckpointId increments highest existing id", () => {
+    const dir = tmpDir();
+    const sid = "sess_nci2";
+    // Write a checkpoints file with two entries to simulate existing state
+    const fakeCheckpoints = [
+        { checkpointId: "chkpt_001", sessionId: "sess_nci2", summary: "a", keyDecisions: [], nextSteps: [], filesModified: [], tokenEstimate: 100, regionHash: "h1", embedding: [], timestamp: 1 },
+        { checkpointId: "chkpt_003", sessionId: "sess_nci2", summary: "b", keyDecisions: [], nextSteps: [], filesModified: [], tokenEstimate: 100, regionHash: "h2", embedding: [], timestamp: 2 },
+    ];
+    writeGzJson(join(dir, "sess_nci2.checkpoints.json.gz"), fakeCheckpoints);
+    assert.equal(nextCheckpointId(sid, dir), "chkpt_004");
+});
+// ---------------------------------------------------------------------------
+// cleanup
+// ---------------------------------------------------------------------------
+test("cleanup", () => {
+    rmSync(baseTmp, { recursive: true, force: true });
+});

package/dist/src/supersede.js

@@ -0,0 +1,68 @@
+/**
+ * supersede.ts — Layer 1 (SUPERSEDE): zero-cost factual pruning.
+ *
+ * If you read server.py, the previous read of server.py is factually obsolete
+ * once you write it. We detect file-read turns that are superseded by a later
+ * write (or a later read) to the same path and mark them for pruning — no
+ * summarization, no token cost. Mirrors memory-mcp MemoryCompactor Stage 1.
+ */
+import { extractFileCandidates } from "./compact.js";
+/** Classify a message's relationship to a file path. */
+function fileOps(msg) {
+    const paths = extractFileCandidates(msg.text);
+    if (paths.length === 0)
+        return [];
+    const low = msg.text.toLowerCase();
+    const isWrite = /\b(write|edit|create|save|append|overwrite|update|patch|modify)\b/.test(low);
+    return paths.map((p) => ({ path: p, op: isWrite ? "write" : "read" }));
+}
+/**
+ * Return the indexes (into `messages`) of file-read turns superseded by a later
+ * operation on the same path. A read is obsolete once a write touches the same
+ * path, or once a newer read of the same path exists (keep only the latest).
+ */
+export function findSuperseded(messages) {
+    // Build, per path, the latest operation index and whether a write occurred.
+    const lastWriteAt = new Map();
+    const lastReadAt = new Map();
+    messages.forEach((m, i) => {
+        for (const { path, op } of fileOps(m)) {
+            if (op === "write")
+                lastWriteAt.set(path, i);
+            else
+                lastReadAt.set(path, i);
+        }
+    });
+    const superseded = new Set();
+    // Reads before a write to the same path are obsolete.
+    messages.forEach((m, i) => {
+        for (const { path, op } of fileOps(m)) {
+            if (op !== "read")
+                continue;
+            const writeAt = lastWriteAt.get(path);
+            if (writeAt !== undefined && i < writeAt)
+                superseded.add(i);
+        }
+    });
+    // For paths with multiple reads and no write, keep only the latest read.
+    const readsByPath = new Map();
+    messages.forEach((m, i) => {
+        for (const { path, op } of fileOps(m)) {
+            if (op !== "read")
+                continue;
+            if (!readsByPath.has(path))
+                readsByPath.set(path, []);
+            readsByPath.get(path).push(i);
+        }
+    });
+    for (const [, idxs] of readsByPath) {
+        if (idxs.length > 1)
+            idxs.slice(0, -1).forEach((i) => superseded.add(i));
+    }
+    return [...superseded].sort((a, b) => a - b);
+}
+/** Convenience: drop superseded messages, preserving order. */
+export function supersede(messages) {
+    const drop = new Set(findSuperseded(messages));
+    return messages.filter((_m, i) => !drop.has(i));
+}

package/dist/src/supersede.test.js

@@ -0,0 +1,36 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { findSuperseded, supersede } from "./supersede.js";
+function msg(role, text) { return { role, text }; }
+test("read superseded by later write to same path is pruned", () => {
+    const messages = [
+        msg("assistant", "read src/server.ts"),
+        msg("user", "now change it"),
+        msg("assistant", "write src/server.ts with the fix"),
+    ];
+    assert.deepEqual(findSuperseded(messages), [0]);
+});
+test("older read superseded by newer read of same path (keep latest)", () => {
+    const messages = [
+        msg("assistant", "read src/a.ts"),
+        msg("assistant", "read src/a.ts again"),
+    ];
+    assert.deepEqual(findSuperseded(messages), [0]);
+});
+test("unrelated reads are kept", () => {
+    const messages = [
+        msg("assistant", "read src/a.ts"),
+        msg("assistant", "read src/b.ts"),
+    ];
+    assert.deepEqual(findSuperseded(messages), []);
+});
+test("supersede() drops the obsolete read and preserves order", () => {
+    const messages = [
+        msg("assistant", "read src/server.ts"),
+        msg("user", "change it"),
+        msg("assistant", "write src/server.ts done"),
+    ];
+    const out = supersede(messages);
+    assert.equal(out.length, 2);
+    assert.equal(out[0].text, "change it");
+});

package/dist/src/tokens.js

@@ -0,0 +1,31 @@
+/**
+ * Token estimation — rough, deterministic, no LLM.
+ *
+ * Ported from claw-code rusty-claude-cli compact.rs estimate_message_tokens
+ * (len/4 + 1 per content block). Good enough to gate compaction and to report
+ * tokens-saved; NOT a substitute for a real tokenizer.
+ */
+/** Estimate tokens for a single text/tool block already as a string. */
+export function estimateBlockTokens(text) {
+    return Math.floor(text.length / 4) + 1;
+}
+/**
+ * Estimate tokens for an EngineMessage. Tool-use/result blocks carry name +
+ * input/output strings, mirrored from the claw-code block accounting.
+ */
+export function estimateMessageTokens(msg) {
+    let t = 0;
+    if (msg.text)
+        t += estimateBlockTokens(msg.text);
+    if (msg.toolName)
+        t += estimateBlockTokens(msg.toolName);
+    if (msg.input)
+        t += estimateBlockTokens(msg.input);
+    if (msg.output)
+        t += estimateBlockTokens(msg.output);
+    return t;
+}
+/** Sum tokens over a list of messages. */
+export function estimateSessionTokens(messages) {
+    return messages.reduce((acc, m) => acc + estimateMessageTokens(m), 0);
+}

package/dist/src/types.js

@@ -0,0 +1,8 @@
+/**
+ * Shared internal types for the pi-mega-compact engine.
+ *
+ * Kept independent of pi's runtime types (src/ is pi-agnostic; the extension
+ * entry in extensions/ adapts between the two). See RESEARCH.md for the pi
+ * AgentMessage contract this must eventually satisfy.
+ */
+export {};

package/dist/src/types.test.js

@@ -0,0 +1,9 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+test("EngineMessage roles are constrained to the pi message contract (no system role)", () => {
+    const roles = ["user", "assistant", "tool", "custom"];
+    // pi Message = user|assistant|tool (+ custom for markers). There is no system role.
+    const probe = "system";
+    assert.equal(roles.includes(probe), false);
+    assert.equal(roles.length, 4);
+});