stageflow 0.40.0 → 0.44.0

package/dist/compaction/pin.js

@@ -0,0 +1,270 @@
+import { scrubSecrets, SECRET_KEY_RE } from "./scrub.js";
+const DEFAULTS = {
+    maxFactsPerMessage: 300,
+    maxFacts: 1500,
+    maxLinesPerMessage: 20_000,
+    maxLineChars: 2000,
+    noiseThreshold: 20,
+};
+const MAX_VALUE_CHARS = 200;
+const MAX_CONTEXT_CHARS = 140;
+const MAX_FACTS_PER_LINE = 12;
+const BASE_SCORE = {
+    error: 100,
+    id: 60,
+    number: 40,
+    url: 35,
+    timestamp: 28,
+    path: 26,
+    other: 18,
+};
+const KIND_ORDER = ["error", "id", "number", "timestamp", "url", "path", "other"];
+const KIND_TITLE = {
+    error: "Errors",
+    id: "Identifiers",
+    number: "Numbers",
+    timestamp: "Timestamps",
+    url: "URLs",
+    path: "Paths",
+    other: "Other",
+};
+const ERROR_RE = /\b(?:errors?|fail(?:ed|s|ure|ures|ing)?|exceptions?|denied|time[d ]?-?outs?|fatal|panic|traceback|refused|unauthori[sz]ed|forbidden|exit(?:ed)?(?: with)? (?:code|status):? ?[1-9]\d*|exit code:? ?[1-9]\d*|status:? ?[45]\d\d|HTTP\/\d(?:\.\d)? [45]\d\d|\b[45]\d\d (?:error|bad|not|internal|service|gateway|forbidden|unauthorized))\b/i;
+const URL_RE = /https?:\/\/[^\s"'<>`)\]},]+/g;
+const ISO_TS_RE = /\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?/g;
+const TIME_RE = /(?<![\d:.])\d{1,2}:\d{2}(?::\d{2})?\s?(?:[AP]M|UTC|GMT)\b|(?<![\d:.\-T])\d{2}:\d{2}:\d{2}(?:\.\d+)?(?![\d:])/gi;
+const UUID_RE = /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi;
+const TICKET_RE = /\b[A-Z][A-Z0-9]{1,9}-\d{1,7}\b/g;
+const TICKET_STOP = new Set(["UTF", "SHA", "ISO", "MD", "AES", "RSA", "HTTP", "TLS", "SSL", "CVE", "RFC", "X", "IPV", "TCP", "UDP", "SSE", "GPT", "COVID"]);
+const PR_RE = /\b(?:PR|pull request|pull|issue|MR)\s*#?\d{1,7}\b|(?<![\w&/#])#\d{2,7}\b/gi;
+const HASH_RE = /\b(?:build|commit|sha\d*|rev|revision|hash|digest|checksum|image|release|tag)\b[^\n]{0,24}?\b([0-9a-f]{7,40})\b/gi;
+const PATH_ABS_RE = /(?<![\w:/.])\/(?:[\w.@+~-]+\/)+[\w.@+~-]+/g;
+const PATH_REL_RE = /(?<![\w:/.@])(?:\.{1,2}\/)?(?:[\w.@+~-]+\/)+[\w.@+~-]*\.[A-Za-z0-9]{1,8}\b/g;
+const KV_EQ_RE = /\b([A-Za-z_][\w.-]{1,40})=("[^"\n]{1,80}"|'[^'\n]{1,80}'|[^\s,;&)}\]"']{1,80})/g;
+const KV_JSON_RE = /"([A-Za-z_][\w.-]{1,40})"\s*:\s*("[^"\n]{1,80}"|-?\d[\d.eE+-]*|true|false|null)/g;
+const KV_COLON_NUM_RE = /\b([A-Za-z_][\w.-]{1,40}):\s+(-?\d[\d,]*(?:\.\d+)?%?)(?![\d:])/g;
+const UNIT_RE = /(?<![\w.:-])\d[\d,]*(?:\.\d+)?\s?(?:ms|sec|secs|seconds|minutes|mins|hours|days|KB|MB|GB|TB|bytes|requests|reqs|errors|failures|users|orders|rows|records|retries|events|messages|items|files|tests|connections|jobs|calls|shards)\b/gi;
+const CURRENCY_RE = /(?<![\w])[$€£]\s?\d[\d,]*(?:\.\d+)?(?:[kKmMbB]\b)?|\b\d[\d,]*(?:\.\d+)?\s?(?:USD|EUR|GBP)\b/g;
+const PERCENT_RE = /(?<![\w.])\d+(?:\.\d+)?%/g;
+const ID_KEY_RE = /(?:^|[_.-])(?:id|uuid|key|ref|no|number|num|sha|hash|rev|build|ticket|incident|order|user|account|customer|request|trace)$/i;
+function messageText(message) {
+    const role = message.role;
+    if (role !== "user" && role !== "toolResult")
+        return undefined;
+    const content = message.content;
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return undefined;
+    const parts = [];
+    for (const block of content) {
+        if (block !== null && typeof block === "object" && block.type === "text") {
+            const text = block.text;
+            if (typeof text === "string")
+                parts.push(text);
+        }
+    }
+    return parts.join("\n");
+}
+function lineTemplate(line) {
+    return line
+        .replace(/\b[0-9a-f]{6,}\b/gi, "#")
+        .replace(/\d+/g, "#")
+        .replace(/"[^"]*"/g, '"#"')
+        .replace(/\s+/g, " ")
+        .trim()
+        .slice(0, 100);
+}
+function singleLine(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+function contextAround(line, index, length) {
+    const flat = line.length > 400 ? line.slice(Math.max(0, index - 200), index + length + 200) : line;
+    const offset = line.length > 400 ? Math.min(index, 200) : index;
+    const start = Math.max(0, offset - 50);
+    const end = Math.min(flat.length, offset + length + 60);
+    let out = singleLine(flat.slice(start, end));
+    if (out.length > MAX_CONTEXT_CHARS)
+        out = out.slice(0, MAX_CONTEXT_CHARS);
+    return out;
+}
+function numericLike(value) {
+    return /^-?\d[\d,]*(?:\.\d+)?(?:[eE][+-]?\d+)?%?$/.test(value);
+}
+function collectCandidates(line, isErrorLine, keyFreq, noiseThreshold) {
+    const out = [];
+    const claimed = [];
+    const free = (start, end) => !claimed.some(([s, e]) => start < e && end > s);
+    const claim = (start, end) => claimed.push([start, end]);
+    const run = (re, on, takeSpan = true) => {
+        re.lastIndex = 0;
+        let m;
+        let guard = 0;
+        while ((m = re.exec(line)) !== null && guard++ < 50) {
+            if (m[0].length === 0) {
+                re.lastIndex++;
+                continue;
+            }
+            const start = m.index;
+            const end = start + m[0].length;
+            if (!free(start, end))
+                continue;
+            const cand = on(m);
+            if (cand) {
+                out.push(cand);
+                if (takeSpan)
+                    claim(start, end);
+            }
+        }
+    };
+    run(URL_RE, (m) => ({ kind: "url", value: m[0].replace(/[.;:!?]+$/, ""), index: m.index }));
+    run(ISO_TS_RE, (m) => ({ kind: "timestamp", value: m[0], index: m.index }));
+    run(UUID_RE, (m) => ({ kind: "id", value: m[0], index: m.index }));
+    run(PR_RE, (m) => ({ kind: "id", value: m[0].replace(/\s+/g, " "), index: m.index }));
+    run(TICKET_RE, (m) => {
+        const prefix = m[0].slice(0, m[0].indexOf("-"));
+        if (TICKET_STOP.has(prefix))
+            return undefined;
+        return { kind: "id", value: m[0], index: m.index };
+    });
+    run(HASH_RE, (m) => {
+        const hex = m[1];
+        if (!/\d/.test(hex) || !/[a-f]/i.test(hex))
+            return undefined;
+        return { kind: "id", value: hex, index: m.index + m[0].length - hex.length };
+    }, false);
+    run(TIME_RE, (m) => ({ kind: "timestamp", value: m[0].trim(), index: m.index }));
+    run(PATH_ABS_RE, (m) => (m[0].length >= 5 ? { kind: "path", value: m[0], index: m.index } : undefined));
+    run(PATH_REL_RE, (m) => ({ kind: "path", value: m[0], index: m.index }));
+    const kv = (key, raw, index) => {
+        if (SECRET_KEY_RE.test(key))
+            return undefined;
+        const value = raw.replace(/^["']|["']$/g, "");
+        if (value.length === 0 || value === "null" || value === "true" || value === "false")
+            return undefined;
+        const common = (keyFreq.get(key) ?? 0) > noiseThreshold;
+        const numeric = numericLike(value);
+        const kind = ID_KEY_RE.test(key) && !numeric ? "id" : ID_KEY_RE.test(key) ? "id" : numeric ? "number" : "other";
+        return { kind, value: `${key}=${value}`, index, weight: common ? -25 : 0 };
+    };
+    run(KV_EQ_RE, (m) => kv(m[1], m[2], m.index));
+    run(KV_JSON_RE, (m) => kv(m[1], m[2], m.index));
+    run(KV_COLON_NUM_RE, (m) => kv(m[1], m[2], m.index));
+    run(CURRENCY_RE, (m) => ({ kind: "number", value: m[0].trim(), index: m.index }));
+    run(PERCENT_RE, (m) => ({ kind: "number", value: m[0], index: m.index }));
+    run(UNIT_RE, (m) => ({ kind: "number", value: m[0].trim(), index: m.index }));
+    if (isErrorLine) {
+        const value = singleLine(line);
+        out.push({ kind: "error", value: value.length > MAX_VALUE_CHARS ? value.slice(0, MAX_VALUE_CHARS) : value, index: 0 });
+    }
+    return out.slice(0, MAX_FACTS_PER_LINE + (isErrorLine ? 1 : 0));
+}
+export function extractPinnedFacts(messages, options = {}) {
+    const cfg = { ...DEFAULTS, ...options };
+    const facts = new Map();
+    let order = 0;
+    messages.forEach((message, messageIndex) => {
+        const text = messageText(message);
+        if (!text)
+            return;
+        const isUser = message.role === "user";
+        const toolName = !isUser && typeof message.toolName === "string" ? message.toolName : undefined;
+        const lines = text.split("\n", cfg.maxLinesPerMessage + 1).slice(0, cfg.maxLinesPerMessage);
+        const templates = new Map();
+        const keyFreq = new Map();
+        const trimmed = [];
+        for (const raw of lines) {
+            const line = raw.length > cfg.maxLineChars ? raw.slice(0, cfg.maxLineChars) : raw;
+            trimmed.push(line);
+            if (line.trim().length === 0)
+                continue;
+            const tpl = lineTemplate(line);
+            templates.set(tpl, (templates.get(tpl) ?? 0) + 1);
+            if (!isUser) {
+                KV_EQ_RE.lastIndex = 0;
+                let m;
+                let g = 0;
+                while ((m = KV_EQ_RE.exec(line)) !== null && g++ < 8)
+                    keyFreq.set(m[1], (keyFreq.get(m[1]) ?? 0) + 1);
+            }
+        }
+        let messageFacts = 0;
+        for (const line of trimmed) {
+            if (messageFacts >= cfg.maxFactsPerMessage)
+                break;
+            if (line.trim().length === 0)
+                continue;
+            const errorLine = ERROR_RE.test(line);
+            const repeats = templates.get(lineTemplate(line)) ?? 1;
+            if (!isUser && !errorLine && repeats > cfg.noiseThreshold)
+                continue;
+            const rarity = errorLine ? 0 : repeats > 1 ? -Math.min(15, Math.log2(repeats) * 3) : 0;
+            for (const cand of collectCandidates(line, errorLine, keyFreq, cfg.noiseThreshold)) {
+                const value = cand.value.length > MAX_VALUE_CHARS ? cand.value.slice(0, MAX_VALUE_CHARS) : cand.value;
+                if (scrubSecrets(value) !== value)
+                    continue;
+                const key = `${cand.kind}\u0000${value}`;
+                const existing = facts.get(key);
+                if (existing) {
+                    existing.count++;
+                    continue;
+                }
+                if (facts.size >= cfg.maxFacts * 2)
+                    continue;
+                const context = scrubSecrets(contextAround(line, cand.index, value.length));
+                const score = BASE_SCORE[cand.kind] + (isUser ? 15 : 0) + rarity + (cand.weight ?? 0);
+                facts.set(key, {
+                    kind: cand.kind,
+                    value,
+                    context,
+                    source: toolName ? { messageIndex, toolName } : { messageIndex },
+                    count: 1,
+                    score,
+                    order: order++,
+                });
+                messageFacts++;
+            }
+        }
+    });
+    const ranked = [...facts.values()].map((fact) => ({
+        ...fact,
+        score: fact.score - Math.min(12, Math.log2(fact.count) * 3),
+    }));
+    ranked.sort((a, b) => b.score - a.score || a.order - b.order);
+    return ranked.slice(0, cfg.maxFacts).map(({ order: _order, ...fact }) => fact);
+}
+export function renderPinned(facts, maxChars) {
+    const title = "## Pinned facts";
+    const omittedLine = (n) => `- (${n} more omitted)`;
+    const base = `${title}\n${omittedLine(facts.length)}`;
+    if (maxChars < base.length)
+        return base.slice(0, Math.max(0, maxChars)).replace(/\n[^\n]*$/, "").slice(0, Math.max(0, maxChars));
+    const chosen = new Map();
+    let used = title.length + 1;
+    let included = 0;
+    const reserve = omittedLine(facts.length).length + 1;
+    for (const fact of facts) {
+        const value = singleLine(scrubSecrets(fact.value));
+        const context = singleLine(scrubSecrets(fact.context));
+        if (value.length === 0)
+            continue;
+        const line = context.length > 0 && context !== value ? `- ${value} — ${context}` : `- ${value}`;
+        const headerCost = chosen.has(fact.kind) ? 0 : `### ${KIND_TITLE[fact.kind]}`.length + 1;
+        const cost = headerCost + line.length + 1;
+        if (used + cost + reserve > maxChars)
+            continue;
+        const bucket = chosen.get(fact.kind) ?? [];
+        bucket.push(line);
+        chosen.set(fact.kind, bucket);
+        used += cost;
+        included++;
+    }
+    const out = [title];
+    for (const kind of KIND_ORDER) {
+        const bucket = chosen.get(kind);
+        if (!bucket)
+            continue;
+        out.push(`### ${KIND_TITLE[kind]}`, ...bucket);
+    }
+    out.push(omittedLine(facts.length - included));
+    return out.join("\n");
+}

package/dist/compaction/scrub.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Everything the compaction code removes before text reaches a summarizer, a pinned fact, a directive or the
+ * working state. Stricter than redactString: it also catches key=value forms, cookies and session ids.
+ * It can over-redact harmless text such as `token_count=5`; callers that store model-written text say so.
+ */
+export declare const SECRET_KEY_RE: RegExp;
+export declare function scrubSecrets(text: string): string;

package/dist/compaction/scrub.js

@@ -0,0 +1,26 @@
+import { redactString } from "../logging/redact.js";
+/**
+ * Everything the compaction code removes before text reaches a summarizer, a pinned fact, a directive or the
+ * working state. Stricter than redactString: it also catches key=value forms, cookies and session ids.
+ * It can over-redact harmless text such as `token_count=5`; callers that store model-written text say so.
+ */
+export const SECRET_KEY_RE = /secret|token|passw|pwd|api[_-]?key|auth|credential|bearer|cookie|private|session|\bsid\b|csrf|xsrf|jwt|signature/i;
+const EXTRA_SECRET_RES = [
+    /\b(?:api[_-]?key|secret|token|password|passwd|authorization|bearer)\b["']?\s*[:=]\s*["']?[^\s"',;]+/gi,
+    /\bBearer\s+[A-Za-z0-9\-._~+/]{8,}=*/gi,
+    /\b(?:Set-)?Cookie:[^\n]*/gi,
+    /\b(?:session|sess|sid|csrf|xsrf|jsessionid|phpsessid)[\w-]*=[^\s;,"']+/gi,
+    /\beyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]{8,}/g,
+    /\bAKIA[0-9A-Z]{16}\b/g,
+    /\bxox[abprs]-[\w-]{8,}/g,
+    /\b(?:sk|pk|rk)[-_](?:live|test|proj|ant)?[-_]?[A-Za-z0-9_-]{16,}/g,
+    /[?&](?:token|key|api_?key|access_token|auth|signature|sig)=[^\s&"']+/gi,
+];
+export function scrubSecrets(text) {
+    let out = redactString(text);
+    for (const re of EXTRA_SECRET_RES) {
+        re.lastIndex = 0;
+        out = out.replace(re, "[redacted]");
+    }
+    return out;
+}

package/dist/compaction/sessionWiring.d.ts

@@ -0,0 +1,22 @@
+import type { InlineExtension } from "@earendil-works/pi-coding-agent";
+export type SessionContextWiring = {
+    /** Inline extensions to register, in order. */
+    extensions: InlineExtension[];
+    /** Tool names to add to the session's allowlist. */
+    toolNames: string[];
+    /** Text for the system prompt; empty when the session has no working state. */
+    guidance: string;
+    /** Tools whose output the tool-output budget must never spill or cap. */
+    budgetExemptNames: string[];
+    /** Tools the agent tool gate lets through without a grant or an approval. */
+    isInternalTool(toolName: string): boolean;
+};
+/**
+ * What a Pi session needs for context management, owned in one place: OpenRouter routing, compaction and
+ * (unless `workingState` is false) the working_state tool with its allowlist entry, prompt guidance and
+ * gate and budget exemptions. Stage sessions, project agents and the operator agent all call this, so the
+ * pieces cannot drift apart.
+ */
+export declare function createSessionContextWiring(options: {
+    workingState: boolean;
+}): SessionContextWiring;

package/dist/compaction/sessionWiring.js

@@ -0,0 +1,26 @@
+import { createOpenRouterRoutingExtension } from "../agent/openRouterRouting.js";
+import { createWorkingStateExtension } from "../tools/workingState.js";
+import { createCompactionExtension } from "./extension.js";
+import { WORKING_STATE_TOOL_NAME } from "./types.js";
+import { WORKING_STATE_GUIDANCE } from "./workingState.js";
+const warn = (message) => console.error(message);
+/**
+ * What a Pi session needs for context management, owned in one place: OpenRouter routing, compaction and
+ * (unless `workingState` is false) the working_state tool with its allowlist entry, prompt guidance and
+ * gate and budget exemptions. Stage sessions, project agents and the operator agent all call this, so the
+ * pieces cannot drift apart.
+ */
+export function createSessionContextWiring(options) {
+    const withState = options.workingState;
+    return {
+        extensions: [
+            { name: "stageflow-openrouter-routing", factory: createOpenRouterRoutingExtension() },
+            { name: "stageflow-compaction", factory: createCompactionExtension({ onWarn: warn }) },
+            ...(withState ? [{ name: "stageflow-working-state", factory: createWorkingStateExtension() }] : []),
+        ],
+        toolNames: withState ? [WORKING_STATE_TOOL_NAME] : [],
+        guidance: withState ? WORKING_STATE_GUIDANCE : "",
+        budgetExemptNames: withState ? [WORKING_STATE_TOOL_NAME] : [],
+        isInternalTool: (toolName) => withState && toolName === WORKING_STATE_TOOL_NAME,
+    };
+}

package/dist/compaction/summarize.d.ts

@@ -0,0 +1,79 @@
+import { type ClearingPolicy } from "./mask.js";
+import { type Directive } from "./directives.js";
+import { type LooseMessage } from "./pin.js";
+import { type ClearedResult, type PinnedFact, type WorkingStateEntry } from "./types.js";
+export declare const COMPACTION_FOCUS: string;
+export declare const SECTIONS_MARKER = "<!-- stageflow:compaction-sections -->";
+export declare const PINNED_RENDER_CHARS = 6000;
+export declare const CLEARED_RENDER_CHARS = 3000;
+export declare const STATE_RENDER_CHARS = 6000;
+export declare const DIRECTIVES_RENDER_CHARS = 5000;
+export declare const PINNED_CARRY_MAX = 400;
+export declare const CLEARED_CARRY_MAX = 200;
+export type StageflowCompactionDetails = {
+    readFiles?: string[];
+    modifiedFiles?: string[];
+    stageflowPinned?: PinnedFact[];
+    stageflowCleared?: ClearedResult[];
+    stageflowDirectives?: Directive[];
+    stageflowPromptVersion?: number;
+};
+/** The sections are always appended last, so the last marker at the start of a line is the real one. */
+export declare function splitSummary(summary: string): {
+    core: string;
+    sections: string;
+};
+/** Details come from the session file, so anything not shaped like ours is dropped rather than trusted. */
+export declare function readStageflowDetails(value: unknown): StageflowCompactionDetails | undefined;
+/** Newest facts first win a tie; capped so repeated compactions cannot grow without bound. */
+export declare function mergePinned(previous: readonly PinnedFact[], fresh: readonly PinnedFact[], max?: number): PinnedFact[];
+export declare function mergeCleared(previous: readonly ClearedResult[], fresh: readonly ClearedResult[], max?: number): ClearedResult[];
+export declare function renderSections(input: {
+    pinned: readonly PinnedFact[];
+    cleared: readonly ClearedResult[];
+    state: readonly WorkingStateEntry[];
+    directives?: readonly Directive[];
+}): string;
+/** What the summarizer reads: message text, thinking and tool-call arguments with secrets removed. */
+export declare function redactMessages<T extends LooseMessage>(messages: readonly T[]): T[];
+export type SummaryInputs = {
+    messages: readonly LooseMessage[];
+    previousDetails?: StageflowCompactionDetails;
+    clearInSummary?: ClearingPolicy;
+};
+export type PreparedSummaryInputs = {
+    messagesForSummarizer: LooseMessage[];
+    pinned: PinnedFact[];
+    cleared: ClearedResult[];
+    directives: Directive[];
+};
+/** Everything deterministic that happens before the model-written summary is requested. */
+export declare function prepareSummaryInputs(input: SummaryInputs): PreparedSummaryInputs;
+export declare function stageflowDetails(piDetails: {
+    readFiles?: string[];
+    modifiedFiles?: string[];
+} | undefined, previous: StageflowCompactionDetails | undefined, pinned: readonly PinnedFact[], cleared: readonly ClearedResult[], directives?: readonly Directive[]): StageflowCompactionDetails;
+export type SummarizeCall = {
+    messages: LooseMessage[];
+    previousSummary: string | undefined;
+};
+export type ComposedSummary<R> = {
+    summary: string;
+    result: R;
+    prepared: PreparedSummaryInputs;
+};
+/**
+ * The whole summary pipeline in one place: pin and redact the input, ask the model for the written part,
+ * scrub it, then append the deterministic sections. The Pi hook and the evaluation harness both call this,
+ * so what the harness measures is what ships.
+ */
+export declare function composeCompactionSummary<R extends {
+    summary: string;
+}>(input: {
+    messages: readonly LooseMessage[];
+    previousSummary?: string;
+    previousDetails?: StageflowCompactionDetails;
+    state: readonly WorkingStateEntry[];
+    clearInSummary?: ClearingPolicy;
+    summarize: (call: SummarizeCall) => Promise<R>;
+}): Promise<ComposedSummary<R>>;

package/dist/compaction/summarize.js

@@ -0,0 +1,171 @@
+import { applyClearing, planClearing, renderClearedIndex } from "./mask.js";
+import { extractDirectives, mergeDirectives, renderDirectives } from "./directives.js";
+import { extractPinnedFacts, renderPinned } from "./pin.js";
+import { scrubSecrets } from "./scrub.js";
+import { COMPACTION_PROMPT_VERSION } from "./types.js";
+import { WorkingStateStore } from "./workingState.js";
+export const COMPACTION_FOCUS = "This is an autonomous agent run, not a chat. Make the summary a working record the agent can continue from without the earlier messages. " +
+    "Copy exact identifiers, numbers, counts, timestamps, versions, paths, URLs and quoted error text verbatim; never round, reword or estimate them. " +
+    "Keep every failure, rejection and dead end explicit, with why it failed, so it is not retried. " +
+    "State what is in flight, which sources or items are already handled and which remain, and the exact next step. " +
+    "Keep decisions made by the operator or user and the deliverable the run must produce, including its required format. " +
+    "Leave out routine output that carries no decision or value.";
+export const SECTIONS_MARKER = "<!-- stageflow:compaction-sections -->";
+export const PINNED_RENDER_CHARS = 6000;
+export const CLEARED_RENDER_CHARS = 3000;
+export const STATE_RENDER_CHARS = 6000;
+export const DIRECTIVES_RENDER_CHARS = 5000;
+export const PINNED_CARRY_MAX = 400;
+export const CLEARED_CARRY_MAX = 200;
+/** The sections are always appended last, so the last marker at the start of a line is the real one. */
+export function splitSummary(summary) {
+    const afterNewline = summary.lastIndexOf(`\n${SECTIONS_MARKER}`);
+    const at = afterNewline >= 0 ? afterNewline + 1 : summary.startsWith(SECTIONS_MARKER) ? 0 : -1;
+    if (at < 0)
+        return { core: summary, sections: "" };
+    return { core: summary.slice(0, at).trimEnd(), sections: summary.slice(at) };
+}
+const FACT_KINDS = new Set(["id", "number", "timestamp", "url", "path", "error", "other"]);
+const DIRECTIVE_SOURCES = new Set(["operator", "user", "rejection"]);
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function strings(value) {
+    return Array.isArray(value) ? value.filter((x) => typeof x === "string") : undefined;
+}
+function validPinned(x) {
+    return isRecord(x) && typeof x.kind === "string" && FACT_KINDS.has(x.kind) && typeof x.value === "string" && typeof x.context === "string" && isRecord(x.source) && typeof x.source.messageIndex === "number";
+}
+function validCleared(x) {
+    return isRecord(x) && typeof x.toolName === "string" && typeof x.toolCallId === "string" && typeof x.recoverHint === "string" && typeof x.bytes === "number" && typeof x.descriptor === "string";
+}
+function validDirective(x) {
+    return isRecord(x) && typeof x.source === "string" && DIRECTIVE_SOURCES.has(x.source) && typeof x.text === "string";
+}
+/** Details come from the session file, so anything not shaped like ours is dropped rather than trusted. */
+export function readStageflowDetails(value) {
+    if (!isRecord(value))
+        return undefined;
+    const readFiles = strings(value.readFiles);
+    const modifiedFiles = strings(value.modifiedFiles);
+    return {
+        ...(readFiles ? { readFiles } : {}),
+        ...(modifiedFiles ? { modifiedFiles } : {}),
+        ...(Array.isArray(value.stageflowPinned) ? { stageflowPinned: value.stageflowPinned.filter(validPinned) } : {}),
+        ...(Array.isArray(value.stageflowCleared) ? { stageflowCleared: value.stageflowCleared.filter(validCleared) } : {}),
+        ...(Array.isArray(value.stageflowDirectives) ? { stageflowDirectives: value.stageflowDirectives.filter(validDirective) } : {}),
+        ...(typeof value.stageflowPromptVersion === "number" ? { stageflowPromptVersion: value.stageflowPromptVersion } : {}),
+    };
+}
+function factKey(f) {
+    return `${f.kind}\u0000${f.value}`;
+}
+/** Newest facts first win a tie; capped so repeated compactions cannot grow without bound. */
+export function mergePinned(previous, fresh, max = PINNED_CARRY_MAX) {
+    const seen = new Set();
+    const out = [];
+    for (const f of [...fresh, ...previous]) {
+        const key = factKey(f);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(f);
+        if (out.length >= max)
+            break;
+    }
+    return out;
+}
+export function mergeCleared(previous, fresh, max = CLEARED_CARRY_MAX) {
+    const seen = new Set();
+    const out = [];
+    for (const c of [...fresh, ...previous]) {
+        if (seen.has(c.toolCallId))
+            continue;
+        seen.add(c.toolCallId);
+        out.push(c);
+        if (out.length >= max)
+            break;
+    }
+    return out;
+}
+export function renderSections(input) {
+    const parts = [];
+    if (input.state.length > 0) {
+        const store = new WorkingStateStore();
+        for (const e of input.state)
+            store.set({ key: e.key, kind: e.kind, value: e.value, updatedAt: e.updatedAt });
+        const rendered = store.render(STATE_RENDER_CHARS);
+        if (rendered)
+            parts.push(rendered);
+    }
+    const directives = renderDirectives(input.directives ?? [], DIRECTIVES_RENDER_CHARS);
+    if (directives)
+        parts.push(directives);
+    if (input.pinned.length > 0)
+        parts.push(renderPinned(input.pinned, PINNED_RENDER_CHARS));
+    if (input.cleared.length > 0)
+        parts.push(renderClearedIndex(input.cleared, CLEARED_RENDER_CHARS));
+    if (parts.length === 0)
+        return "";
+    return `${SECTIONS_MARKER}\n${parts.join("\n\n")}`;
+}
+function scrubValue(value) {
+    if (typeof value === "string")
+        return scrubSecrets(value);
+    if (Array.isArray(value))
+        return value.map(scrubValue);
+    if (value !== null && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, scrubValue(v)]));
+    }
+    return value;
+}
+/** What the summarizer reads: message text, thinking and tool-call arguments with secrets removed. */
+export function redactMessages(messages) {
+    return messages.map((m) => (Array.isArray(m.content) || typeof m.content === "string" ? { ...m, content: scrubValue(m.content) } : m));
+}
+/** Everything deterministic that happens before the model-written summary is requested. */
+export function prepareSummaryInputs(input) {
+    const extracted = extractPinnedFacts(input.messages);
+    const pinned = mergePinned(input.previousDetails?.stageflowPinned ?? [], extracted);
+    const directives = mergeDirectives(input.previousDetails?.stageflowDirectives ?? [], extractDirectives(input.messages));
+    if (!input.clearInSummary) {
+        return { messagesForSummarizer: redactMessages(input.messages), pinned, directives, cleared: [...(input.previousDetails?.stageflowCleared ?? [])] };
+    }
+    const plan = planClearing(input.messages, input.clearInSummary);
+    return {
+        messagesForSummarizer: redactMessages(applyClearing(input.messages, plan)),
+        pinned,
+        directives,
+        cleared: mergeCleared(input.previousDetails?.stageflowCleared ?? [], plan.cleared),
+    };
+}
+export function stageflowDetails(piDetails, previous, pinned, cleared, directives = []) {
+    const union = (a, b) => [...new Set([...(b ?? []), ...(a ?? [])])];
+    return {
+        readFiles: union(piDetails?.readFiles, previous?.readFiles),
+        modifiedFiles: union(piDetails?.modifiedFiles, previous?.modifiedFiles),
+        stageflowPinned: [...pinned],
+        stageflowCleared: [...cleared],
+        stageflowDirectives: [...directives],
+        stageflowPromptVersion: COMPACTION_PROMPT_VERSION,
+    };
+}
+/**
+ * The whole summary pipeline in one place: pin and redact the input, ask the model for the written part,
+ * scrub it, then append the deterministic sections. The Pi hook and the evaluation harness both call this,
+ * so what the harness measures is what ships.
+ */
+export async function composeCompactionSummary(input) {
+    const prepared = prepareSummaryInputs({
+        messages: input.messages,
+        ...(input.previousDetails ? { previousDetails: input.previousDetails } : {}),
+        ...(input.clearInSummary ? { clearInSummary: input.clearInSummary } : {}),
+    });
+    const result = await input.summarize({
+        messages: prepared.messagesForSummarizer,
+        previousSummary: input.previousSummary !== undefined ? splitSummary(input.previousSummary).core : undefined,
+    });
+    const core = scrubSecrets(result.summary).split(SECTIONS_MARKER).join("").trimEnd();
+    const sections = scrubSecrets(renderSections({ pinned: prepared.pinned, cleared: prepared.cleared, state: input.state, directives: prepared.directives }));
+    return { summary: sections ? `${core}\n\n${sections}` : core, result, prepared };
+}

package/dist/compaction/toolPolicy.d.ts

@@ -0,0 +1,3 @@
+export type ClearingClass = "clearable" | "keep";
+/** Fail closed: only provably re-fetchable read results are clearable; everything else (bash included, it can have side effects) is kept. */
+export declare function classifyForClearing(toolName: string, isError: boolean, _details?: unknown): ClearingClass;

package/dist/compaction/toolPolicy.js

@@ -0,0 +1,33 @@
+import { classifyTool } from "../agents/toolRisk.js";
+import { ASK_OPERATOR_TOOL_NAME, EMIT_STAGE_ENVELOPE_TOOL_NAME, WRITE_STAGE_ARTIFACT_TOOL_NAME } from "../tools/stageToolNames.js";
+import { WORKING_STATE_TOOL_NAME } from "./types.js";
+const ALWAYS_KEEP = new Set([
+    ASK_OPERATOR_TOOL_NAME,
+    EMIT_STAGE_ENVELOPE_TOOL_NAME,
+    WRITE_STAGE_ARTIFACT_TOOL_NAME,
+    "write",
+    "edit",
+    WORKING_STATE_TOOL_NAME,
+]);
+const CLEARABLE_BUILTINS = new Set(["read", "grep", "find", "ls"]);
+/** Fail closed: only provably re-fetchable read results are clearable; everything else (bash included, it can have side effects) is kept. */
+export function classifyForClearing(toolName, isError, _details) {
+    if (isError)
+        return "keep";
+    if (typeof toolName !== "string" || toolName.length === 0)
+        return "keep";
+    if (ALWAYS_KEEP.has(toolName))
+        return "keep";
+    if (CLEARABLE_BUILTINS.has(toolName))
+        return "clearable";
+    if (toolName.startsWith("stageflow_")) {
+        const name = toolName.slice("stageflow_".length);
+        return ALWAYS_KEEP.has(name) ? "keep" : classifyTool(undefined, `stageflow.${name}`).readOnly ? "clearable" : "keep";
+    }
+    try {
+        return classifyTool(undefined, toolName).readOnly ? "clearable" : "keep";
+    }
+    catch {
+        return "keep";
+    }
+}