peon-mem 1.0.6 → 1.0.8

package/dist/logger.js

@@ -1,12 +1,32 @@
-import { appendFile, mkdir, readFile } from "node:fs/promises";
+import { appendFile, mkdir, open, rename, stat } from "node:fs/promises";
 import { homedir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
 const DEFAULT_LOG_DIR = join(homedir(), "Library", "Logs", "Peon");
+/**
+ * Reads are bounded to the tail of the log. recent() used to slurp the entire
+ * file and split it, which meant a months-old 100 MB daemon.jsonl blocked the
+ * event loop on every monitor poll (flattening a 100 MB rope + allocating ~400k
+ * strings, only to throw all but `limit` of them away). Same tail-read strategy
+  * the query-embedding cache already uses.
+ *
+ * Sized to comfortably cover the largest real caller (token-stat seeding asks
+ * for 50k entries; entries average ~260 bytes), so bounding reads does not
+ * silently truncate what the daemon rebuilds on boot.
+ */
+const DEFAULT_TAIL_BYTES = 16 * 1024 * 1024;
+/** Rotate before the live file can reach a size that hurts anything. */
+const DEFAULT_MAX_BYTES = 32 * 1024 * 1024;
 export class PeonLogger {
     logFile;
+    maxBytes;
+    tailBytes;
     writeQueue = Promise.resolve();
+    bytesWritten = 0;
+    sizeKnown = false;
     constructor(options = {}) {
         this.logFile = join(options.logDir ?? DEFAULT_LOG_DIR, "daemon.jsonl");
+        this.maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+        this.tailBytes = options.tailBytes ?? DEFAULT_TAIL_BYTES;
     }
     async log(type, fields = {}) {
         const entry = {
@@ -18,10 +38,15 @@ export class PeonLogger {
         await this.enqueueWrite(`${JSON.stringify(entry)}\n`);
         return entry;
     }
+    /**
+     * Newest-first entries from the tail of the log. Cost is bounded by
+     * `tailBytes`, not by the size of the file, so this stays flat as the log
+     * grows. Entries older than the tail window are not visible here — the log
+     * file itself (and its rotated siblings) remain the full record.
+     */
     async recent(limit = 100) {
-        const raw = await readFile(this.logFile, "utf8").catch(() => "");
-        return raw
-            .trim()
+        const text = await this.readTail();
+        return text
             .split("\n")
             .filter(Boolean)
             .slice(-limit)
@@ -31,15 +56,57 @@ export class PeonLogger {
                 return [JSON.parse(line)];
             }
             catch {
+                // Skips both genuinely corrupt lines and the partial first line left
+                // by seeking into the middle of a record.
                 return [];
             }
         });
     }
+    async readTail() {
+        let handle;
+        try {
+            const size = (await stat(this.logFile)).size;
+            const length = Math.min(size, this.tailBytes);
+            if (length === 0)
+                return "";
+            handle = await open(this.logFile, "r");
+            const buffer = Buffer.alloc(length);
+            await handle.read(buffer, 0, length, size - length);
+            const text = buffer.toString("utf8");
+            // A partial leading line is unavoidable when the window starts mid-record.
+            return length < size ? text.slice(text.indexOf("\n") + 1) : text;
+        }
+        catch {
+            return "";
+        }
+        finally {
+            await handle?.close().catch(() => undefined);
+        }
+    }
+    /**
+     * Move the live log aside once it exceeds maxBytes. History is preserved in a
+     * timestamped sibling rather than truncated, so nothing is lost.
+     */
+    async rotateIfNeeded() {
+        if (this.bytesWritten <= this.maxBytes)
+            return;
+        const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+        await rename(this.logFile, `${this.logFile}.${stamp}`).catch(() => undefined);
+        this.bytesWritten = 0;
+    }
     async enqueueWrite(line) {
         const write = async () => {
             try {
-                await mkdir(this.logFile.slice(0, this.logFile.lastIndexOf("/")), { recursive: true });
+                await mkdir(dirname(this.logFile), { recursive: true });
+                // Adopt the on-disk size once, so a daemon restart doesn't forget that
+                // an already-huge log is due for rotation.
+                if (!this.sizeKnown) {
+                    this.bytesWritten = await stat(this.logFile).then((s) => s.size).catch(() => 0);
+                    this.sizeKnown = true;
+                }
+                await this.rotateIfNeeded();
                 await appendFile(this.logFile, line, "utf8");
+                this.bytesWritten += Buffer.byteLength(line, "utf8");
             }
             catch {
                 // Best-effort logging: a log write failure must never crash the daemon

package/dist/memory-store.d.ts

@@ -164,9 +164,13 @@ export declare class PeonMemoryStore {
      * No-op when embeddings are unavailable. supersededBy links to a merged-away id
      * are re-pointed at the surviving record so history stays intact.
      */
-    mergeSimilarActiveRecords(records: MemoryRecord[], threshold?: number): Promise<{
+    mergeSimilarActiveRecords(records: MemoryRecord[], threshold?: number, options?: {
+        maxActive?: number;
+        exhaustive?: boolean;
+    }): Promise<{
         records: MemoryRecord[];
         merged: number;
+        comparisons: number;
     }>;
     readProcessingState(): Promise<ProcessingState>;
     writeProcessingState(state: ProcessingState): Promise<void>;

package/dist/memory-store.js

@@ -33,6 +33,78 @@ async function atomicWrite(path, content) {
     await writeFile(tmp, content, "utf8");
     await rename(tmp, path);
 }
+/**
+ * Ceiling on how many active records the O(n^2) semantic dedup pass will consider.
+ * Override with PEON_DEDUP_MAX_ACTIVE. Set generously enough that ordinary project
+ * brains still dedup, low enough that a very large brain can't stall the daemon.
+ */
+const DEDUP_MAX_ACTIVE = Number(process.env.PEON_DEDUP_MAX_ACTIVE) > 0
+    ? Number(process.env.PEON_DEDUP_MAX_ACTIVE)
+    : 25_000;
+/**
+ * Deterministic random projections for LSH bucketing of dedup candidates.
+ * Seeded so bucketing is reproducible across runs and tests.
+ */
+const DEDUP_BANDS = 8;
+const DEDUP_BITS_PER_BAND = 6;
+/** Yield to the event loop every N records so a long pass can't starve the daemon. */
+const DEDUP_YIELD_EVERY = 200;
+/**
+ * Hard ceiling on candidates examined per record. Bucketing alone only buys a
+ * constant factor when vectors are correlated (buckets fill unevenly), leaving the
+ * pass quadratic. Capping candidates makes the work O(n * k) — genuinely linear in
+ * brain size — at the cost of occasionally missing a merge in a very crowded bucket.
+ * A missed merge leaves a near-duplicate belief; an uncapped pass hangs the daemon.
+ */
+const DEDUP_MAX_CANDIDATES = 128;
+const projectionCache = new Map();
+function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+function projectionsFor(dim) {
+    const cached = projectionCache.get(dim);
+    if (cached)
+        return cached;
+    const rand = mulberry32(0x9e3779b9 ^ dim);
+    const planes = [];
+    for (let p = 0; p < DEDUP_BANDS * DEDUP_BITS_PER_BAND; p += 1) {
+        const plane = new Float32Array(dim);
+        for (let d = 0; d < dim; d += 1)
+            plane[d] = rand() * 2 - 1;
+        planes.push(plane);
+    }
+    projectionCache.set(dim, planes);
+    return planes;
+}
+/**
+ * Band keys for a vector: sign bits of random-hyperplane projections, grouped into
+ * bands. Two vectors with high cosine similarity agree on most bits, so they collide
+ * in at least one band with high probability — letting dedup compare a handful of
+ * plausible candidates instead of every record seen so far.
+ */
+function bandKeys(vector) {
+    const dim = vector.length;
+    const planes = projectionsFor(dim);
+    const keys = [];
+    for (let band = 0; band < DEDUP_BANDS; band += 1) {
+        let bits = "";
+        for (let b = 0; b < DEDUP_BITS_PER_BAND; b += 1) {
+            const plane = planes[band * DEDUP_BITS_PER_BAND + b];
+            let dot = 0;
+            for (let d = 0; d < dim; d += 1)
+                dot += vector[d] * plane[d];
+            bits += dot >= 0 ? "1" : "0";
+        }
+        keys.push(`${band}:${bits}`);
+    }
+    return keys;
+}
 export class PeonMemoryStore {
     projectPath;
     memoryDir;
@@ -612,18 +684,29 @@ export class PeonMemoryStore {
      * No-op when embeddings are unavailable. supersededBy links to a merged-away id
      * are re-pointed at the surviving record so history stays intact.
      */
-    async mergeSimilarActiveRecords(records, threshold = 0.9) {
+    async mergeSimilarActiveRecords(records, threshold = 0.9, options = {}) {
         if (!this.embeddingClient || !this.embeddingStore)
-            return { records, merged: 0 };
+            return { records, merged: 0, comparisons: 0 };
+        // SCALE GUARD. The pass below is O(n^2) pairwise cosine over 1536-dim vectors.
+        // On a 30k-record brain (~6.4k active) that is ~20.5M comparisons / ~31.6B float
+        // ops on the main thread, plus every vector resident as float64 — measured at
+        // 99% CPU and >1.9 GB RSS, which wedged the daemon's event loop entirely.
+        // Above the threshold we skip dedup rather than take the daemon down: a brain
+        // that keeps a few near-duplicates is strictly better than a brain that hangs.
+        // Checked BEFORE sync() so the vector sidecar is never even loaded.
+        const activeCount = records.reduce((n, r) => (r.status === "active" ? n + 1 : n), 0);
+        if (activeCount > (options.maxActive ?? DEDUP_MAX_ACTIVE)) {
+            return { records, merged: 0, comparisons: 0 };
+        }
         let vectorById;
         try {
             vectorById = (await this.embeddingStore.sync(records, this.embeddingClient)).vectorById;
         }
         catch {
-            return { records, merged: 0 };
+            return { records, merged: 0, comparisons: 0 };
         }
         if (vectorById.size === 0)
-            return { records, merged: 0 };
+            return { records, merged: 0, comparisons: 0 };
         const active = records.filter((record) => record.status === "active");
         const passthrough = records.filter((record) => record.status !== "active");
         const kept = [];
@@ -631,22 +714,81 @@ export class PeonMemoryStore {
         const remap = new Map();
         const mergeNow = new Date().toISOString();
         let merged = 0;
+        // Candidate index: band key -> indices into kept[]. Lets each record compare
+        // against a few plausible near-duplicates instead of every record so far,
+        // turning the old O(n^2) scan into roughly linear work.
+        const buckets = new Map();
+        const indexKept = (index, vector, type) => {
+            if (!vector)
+                return;
+            for (const key of bandKeys(vector)) {
+                const full = `${type}|${key}`;
+                const list = buckets.get(full);
+                if (list)
+                    list.push(index);
+                else
+                    buckets.set(full, [index]);
+            }
+        };
+        let comparisons = 0;
+        let processed = 0;
         for (const record of active) {
+            // Long passes must never starve the daemon's event loop the way the old
+            // fully synchronous scan did.
+            processed += 1;
+            if (processed % DEDUP_YIELD_EVERY === 0)
+                await new Promise((resolve) => setImmediate(resolve));
             const vec = vectorById.get(record.id);
             let matchIndex = -1;
             if (vec) {
-                for (let i = 0; i < kept.length; i += 1) {
-                    if (kept[i].type !== record.type)
-                        continue;
-                    const other = vectorById.get(kept[i].id);
-                    if (other && cosineSimilarity(vec, other) >= threshold) {
-                        matchIndex = i;
-                        break;
+                if (options.exhaustive) {
+                    for (let i = 0; i < kept.length; i += 1) {
+                        if (kept[i].type !== record.type)
+                            continue;
+                        const other = vectorById.get(kept[i].id);
+                        if (!other)
+                            continue;
+                        comparisons += 1;
+                        if (cosineSimilarity(vec, other) >= threshold) {
+                            matchIndex = i;
+                            break;
+                        }
+                    }
+                }
+                else {
+                    const seen = new Set();
+                    let examined = 0;
+                    for (const key of bandKeys(vec)) {
+                        const candidates = buckets.get(`${record.type}|${key}`);
+                        if (!candidates)
+                            continue;
+                        // Most recent entries first: a near-duplicate is likeliest among
+                        // recently-seen beliefs, so a capped scan still finds the common case.
+                        for (let c = candidates.length - 1; c >= 0; c -= 1) {
+                            if (examined >= DEDUP_MAX_CANDIDATES)
+                                break;
+                            const i = candidates[c];
+                            if (seen.has(i))
+                                continue;
+                            seen.add(i);
+                            const other = vectorById.get(kept[i].id);
+                            if (!other)
+                                continue;
+                            examined += 1;
+                            comparisons += 1;
+                            if (cosineSimilarity(vec, other) >= threshold) {
+                                matchIndex = i;
+                                break;
+                            }
+                        }
+                        if (matchIndex !== -1 || examined >= DEDUP_MAX_CANDIDATES)
+                            break;
                     }
                 }
             }
             if (matchIndex === -1) {
                 kept.push(record);
+                indexKept(kept.length - 1, vec, record.type);
                 continue;
             }
             const other = kept[matchIndex];
@@ -661,6 +803,10 @@ export class PeonMemoryStore {
                 entities: unique([...record.entities, ...other.entities]),
                 updatedAt: record.updatedAt > other.updatedAt ? record.updatedAt : other.updatedAt
             };
+            // The survivor can be the incoming record, so make its vector findable at
+            // that slot too — otherwise later near-duplicates could miss the bucket.
+            if (canonical.id === record.id)
+                indexKept(matchIndex, vec, record.type);
             remap.set(loser.id, canonical.id);
             // Recoverable-loser rule: don't destroy the merged-away belief — retire it as superseded,
             // linked to the survivor. It leaves active recall but its content stays recoverable and
@@ -682,7 +828,7 @@ export class PeonMemoryStore {
         const fixed = [...passthrough, ...retired].map((record) => record.supersededBy && remap.has(record.supersededBy)
             ? { ...record, supersededBy: resolveRemap(record.supersededBy) }
             : record);
-        return { records: [...kept, ...fixed], merged };
+        return { records: [...kept, ...fixed], merged, comparisons };
     }
     async readProcessingState() {
         const raw = await readFile(join(this.memoryDir, "brain", "processing-state.json"), "utf8").catch(() => "");

package/dist/overview.d.ts

@@ -34,14 +34,26 @@ export interface DuplicatePair {
     bContent: string;
     similarity: number;
 }
-/**
- * Flag near-duplicate ACTIVE beliefs of the same type — a nudge for the user to
- * merge, not an automatic action. Conservative threshold to avoid false alarms.
- */
-export declare function detectDuplicates(records: readonly MemoryRecord[], options?: {
+/** How many record pairs the last duplicate scan actually compared. */
+export declare function duplicateScanStats(): {
+    pairsEvaluated: number;
+};
+export interface DetectDuplicatesOptions {
     threshold?: number;
     limit?: number;
-}): DuplicatePair[];
+    /** Compare every pair (the original scan). Kept for equivalence testing. */
+    exhaustive?: boolean;
+    maxTokenBucket?: number;
+}
+/**
+ * Near-duplicate belief pairs, strongest first.
+ *
+ * Previously this compared every active pair — ~20.4M jaccard computations on a
+ * 6.4k-active brain, to return the top 5 — on both the /overview endpoint and the
+ * consolidation auto-merge path. Candidates are now found through an inverted index
+ * on rare tokens, so the overwhelming majority of pairs are never scored.
+ */
+export declare function detectDuplicates(records: readonly MemoryRecord[], options?: DetectDuplicatesOptions): DuplicatePair[];
 export interface TokenSavings {
     onAvg: number;
     offAvg: number;

package/dist/overview.js

@@ -50,29 +50,107 @@ function jaccard(a, b) {
  * Flag near-duplicate ACTIVE beliefs of the same type — a nudge for the user to
  * merge, not an automatic action. Conservative threshold to avoid false alarms.
  */
+/**
+ * Only the rarest few tokens of a record are used as blocking keys: a pair at
+ * jaccard >= 0.6 shares most of its words, so it almost certainly shares one of
+ * them. Tokens common to a huge share of the brain are useless as keys and are
+ * skipped rather than producing a quadratic bucket.
+ */
+const DUPLICATE_BLOCKING_TOKENS = 12;
+const MAX_TOKEN_BUCKET = Number(process.env.PEON_DUPLICATE_MAX_TOKEN_BUCKET) > 0
+    ? Number(process.env.PEON_DUPLICATE_MAX_TOKEN_BUCKET)
+    : 400;
+let lastDuplicatePairsEvaluated = 0;
+/** How many record pairs the last duplicate scan actually compared. */
+export function duplicateScanStats() {
+    return { pairsEvaluated: lastDuplicatePairsEvaluated };
+}
+/**
+ * Near-duplicate belief pairs, strongest first.
+ *
+ * Previously this compared every active pair — ~20.4M jaccard computations on a
+ * 6.4k-active brain, to return the top 5 — on both the /overview endpoint and the
+ * consolidation auto-merge path. Candidates are now found through an inverted index
+ * on rare tokens, so the overwhelming majority of pairs are never scored.
+ */
 export function detectDuplicates(records, options = {}) {
     const threshold = options.threshold ?? 0.6;
     const limit = options.limit ?? 5;
     const active = records.filter((record) => record.status === "active");
     const sets = active.map((record) => wordSet(record.content));
-    const pairs = [];
-    for (let i = 0; i < active.length; i += 1) {
-        for (let j = i + 1; j < active.length; j += 1) {
-            if (active[i].type !== active[j].type)
+    const found = [];
+    let pairsEvaluated = 0;
+    const score = (i, j) => {
+        if (active[i].type !== active[j].type)
+            return;
+        pairsEvaluated += 1;
+        const similarity = jaccard(sets[i], sets[j]);
+        if (similarity < threshold)
+            return;
+        found.push({
+            i,
+            j,
+            pair: {
+                aId: active[i].id,
+                aContent: active[i].content,
+                bId: active[j].id,
+                bContent: active[j].content,
+                similarity: Math.round(similarity * 100) / 100
+            }
+        });
+    };
+    if (options.exhaustive) {
+        for (let i = 0; i < active.length; i += 1) {
+            for (let j = i + 1; j < active.length; j += 1)
+                score(i, j);
+        }
+    }
+    else {
+        const documentFrequency = new Map();
+        for (const set of sets) {
+            for (const token of set)
+                documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
+        }
+        const buckets = new Map();
+        for (let i = 0; i < active.length; i += 1) {
+            const rarest = [...sets[i]]
+                .sort((a, b) => (documentFrequency.get(a) ?? 0) - (documentFrequency.get(b) ?? 0))
+                .slice(0, DUPLICATE_BLOCKING_TOKENS);
+            for (const token of rarest) {
+                const bucket = buckets.get(token);
+                if (bucket)
+                    bucket.push(i);
+                else
+                    buckets.set(token, [i]);
+            }
+        }
+        const cap = options.maxTokenBucket ?? MAX_TOKEN_BUCKET;
+        const seen = new Set();
+        const width = active.length;
+        for (const bucket of buckets.values()) {
+            if (bucket.length > cap)
                 continue;
-            const similarity = jaccard(sets[i], sets[j]);
-            if (similarity >= threshold) {
-                pairs.push({
-                    aId: active[i].id,
-                    aContent: active[i].content,
-                    bId: active[j].id,
-                    bContent: active[j].content,
-                    similarity: Math.round(similarity * 100) / 100
-                });
+            for (let a = 0; a < bucket.length; a += 1) {
+                for (let b = a + 1; b < bucket.length; b += 1) {
+                    const i = Math.min(bucket[a], bucket[b]);
+                    const j = Math.max(bucket[a], bucket[b]);
+                    const key = i * width + j;
+                    if (seen.has(key))
+                        continue;
+                    seen.add(key);
+                    score(i, j);
+                }
             }
         }
+        // Match the original emission order before the (stable) similarity sort, so
+        // equal-similarity pairs come out in the same order as the exhaustive scan.
+        found.sort((x, y) => x.i - y.i || x.j - y.j);
     }
-    return pairs.sort((left, right) => right.similarity - left.similarity).slice(0, limit);
+    lastDuplicatePairsEvaluated = pairsEvaluated;
+    return found
+        .map((entry) => entry.pair)
+        .sort((left, right) => right.similarity - left.similarity)
+        .slice(0, limit);
 }
 /**
  * Compare Peon-on vs Peon-off session token totals for a project. Returns null

package/dist/processor.d.ts

@@ -88,3 +88,39 @@ export declare class OpenRouterMemoryModelClient implements MemoryModelClient {
     }): Promise<MemoryModelResult>;
 }
 export declare function parseProcessedMemory(content: string): ProcessedMemory;
+/**
+ * Did the model server silently truncate the prompt?
+ *
+ * OpenAI-compatible servers report usage.prompt_tokens: what the model actually read.
+ * Ollama, at its 4096-token default, reports exactly 4096 for a ~17K-token prompt.
+ *
+ * The estimate comes from estimatePromptTokensForTruncation, and the threshold has to
+ * respect how rough it is. For English that estimate is chars/4, which OVER-estimates
+ * (real text runs ~5.5 chars/token), so an untruncated prompt still reports ~0.73 of
+ * it. A truncated one reports ~0.24. Below 0.5 is unambiguous: reaching it without
+ * truncation would take 8+ chars per token. Small prompts are ignored, where
+ * estimation noise is a large share of the total.
+ */
+export declare function detectPromptTruncation(estimatedPromptTokens: number, reportedPromptTokens: number | undefined): boolean;
+/**
+ * Prompt size estimate for detectPromptTruncation ONLY. estimateTokens (chars/4) stays
+ * the cost/reporting estimate; this one exists because chars/4 undercounts token-dense
+ * scripts, which let a truncated CJK prompt pass as untruncated.
+ *
+ * The detector flags reported/estimated < 0.5, so each weight has to sit between two
+ * limits: high enough that a truncated prompt falls below 0.5, and at most ~2x the
+ * MOST efficient tokenizer's rate, or an untruncated prompt falls below 0.5 too. That
+ * false positive is the worse failure: the session is refused on every retry.
+ *
+ * - ASCII, 1/4 per char: unchanged, so English behaves exactly as before (chars/4
+ *   over-counts English ~1.4x; measured untruncated ratio 0.73, truncated 0.24).
+ * - Han, kana, hangul, bopomofo, CJK and fullwidth punctuation, 0.75 per char: efficient
+ *   tokenizers run ~0.45-0.6 tokens per CJK char, giving an untruncated ratio of 0.6-0.8.
+ *   Qwen2.5 on Ollama runs ~0.65, so a truncated CJK prompt now reads well under 0.5.
+ * - Any other non-ASCII, 0.35 per char: Cyrillic, Greek, Arabic, accented Latin and the
+ *   like pack into ~0.22-0.35 tokens per char on large-vocabulary tokenizers. A flat 0.75
+ *   here would read an untruncated Russian log at ~0.31 and block it forever.
+ *
+ * Iterates code points, so an astral character (CJK Extension B, emoji) counts once.
+ */
+export declare function estimatePromptTokensForTruncation(text: string): number;

package/dist/processor.js

@@ -1,5 +1,5 @@
 import { PeonMemoryStore } from "./memory-store.js";
-import { loadPeonConfig } from "./config.js";
+import { llmEnabled, loadPeonConfig } from "./config.js";
 import { createQualityReport } from "./quality.js";
 import { extractDomainEntitiesViaModel } from "./entity-extraction.js";
 export class PeonMemoryProcessor {
@@ -99,7 +99,9 @@ export class PeonMemoryProcessor {
             trigger: input.trigger,
             force: input.force ?? false,
             aiMode: this.config.aiMode,
-            hasApiKey: Boolean(this.config.openRouterApiKey),
+            // A local provider (Ollama) needs no key. Gating on openRouterApiKey here meant a
+            // fully-local setup skipped every automatic consolidation as "missing_api_key".
+            hasApiKey: llmEnabled(this.config),
             hasManualAiResult: Boolean(input.aiResult)
         });
         if (decision.action === "skip") {
@@ -202,6 +204,11 @@ export class OpenRouterMemoryModelClient {
                 // Reserve explicit output room. Without this, OpenRouter applies the provider's default
                 // completion cap, which — paired with the delta cap on the input side — keeps the JSON
                 // reply from truncating mid-object. Env-tunable for very large brains.
+                // Force a JSON-object reply. A hosted model usually obeys "reply with JSON" from
+                // the prompt alone; a local 7B often answers conversationally instead ("It sounds
+                // like..."), which fails the parse and loses the whole consolidation. Ollama and
+                // the OpenAI API both honour this flag, so ask for it rather than trusting prose.
+                response_format: { type: "json_object" },
                 max_tokens: Number(process.env.PEON_CONSOLIDATION_MAX_TOKENS) || 8192
             })
         });
@@ -210,6 +217,19 @@ export class OpenRouterMemoryModelClient {
             throw new Error(`OpenRouter memory processing failed with ${response.status}${body ? `: ${body}` : ""}`);
         }
         const json = (await response.json());
+        // A server whose context window is smaller than this prompt does not error: it
+        // keeps only the TAIL and silently drops the rest — which is the system prompt and
+        // the JSON schema. The model then answers without its instructions, and the result
+        // is empty or wrong. Refuse it rather than mark the session as consolidated.
+        const estimatedPromptTokens = estimatePromptTokensForTruncation(systemPrompt) + estimatePromptTokensForTruncation(userPrompt);
+        const reportedPromptTokens = json.usage?.prompt_tokens;
+        if (detectPromptTruncation(estimatedPromptTokens, reportedPromptTokens)) {
+            throw new Error(`The model server truncated the consolidation prompt: it processed ${reportedPromptTokens} tokens ` +
+                `of roughly ${estimatedPromptTokens} sent. Its context window is too small, so the instructions and ` +
+                `schema were cut off and the result would be empty. The session log was NOT consumed and will be ` +
+                `retried. Fix: give the model a larger context window — for Ollama, create a model with ` +
+                `"PARAMETER num_ctx 32768" (or set OLLAMA_CONTEXT_LENGTH) and point PEON_PROCESSING_MODEL at it.`);
+        }
         const content = json.choices?.[0]?.message?.content;
         if (!content)
             throw new Error("OpenRouter memory processing response did not include content.");
@@ -351,6 +371,69 @@ function isMemoryStatus(value) {
 function estimateTokens(text) {
     return Math.max(1, Math.ceil(text.length / 4));
 }
+/**
+ * Did the model server silently truncate the prompt?
+ *
+ * OpenAI-compatible servers report usage.prompt_tokens: what the model actually read.
+ * Ollama, at its 4096-token default, reports exactly 4096 for a ~17K-token prompt.
+ *
+ * The estimate comes from estimatePromptTokensForTruncation, and the threshold has to
+ * respect how rough it is. For English that estimate is chars/4, which OVER-estimates
+ * (real text runs ~5.5 chars/token), so an untruncated prompt still reports ~0.73 of
+ * it. A truncated one reports ~0.24. Below 0.5 is unambiguous: reaching it without
+ * truncation would take 8+ chars per token. Small prompts are ignored, where
+ * estimation noise is a large share of the total.
+ */
+export function detectPromptTruncation(estimatedPromptTokens, reportedPromptTokens) {
+    if (!reportedPromptTokens || reportedPromptTokens <= 0)
+        return false; // no usage reported: cannot tell
+    if (estimatedPromptTokens - reportedPromptTokens < 1000)
+        return false;
+    return reportedPromptTokens / estimatedPromptTokens < 0.5;
+}
+/** Scripts that tokenize at roughly one token per character or more, not one per word. */
+const TOKEN_DENSE_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Bopomofo}\u3000-\u303F\uFF00-\uFFEF]/u;
+const TOKENS_PER_ASCII_CHAR = 1 / 4;
+const TOKENS_PER_DENSE_CHAR = 0.75;
+const TOKENS_PER_OTHER_CHAR = 0.35;
+/**
+ * Prompt size estimate for detectPromptTruncation ONLY. estimateTokens (chars/4) stays
+ * the cost/reporting estimate; this one exists because chars/4 undercounts token-dense
+ * scripts, which let a truncated CJK prompt pass as untruncated.
+ *
+ * The detector flags reported/estimated < 0.5, so each weight has to sit between two
+ * limits: high enough that a truncated prompt falls below 0.5, and at most ~2x the
+ * MOST efficient tokenizer's rate, or an untruncated prompt falls below 0.5 too. That
+ * false positive is the worse failure: the session is refused on every retry.
+ *
+ * - ASCII, 1/4 per char: unchanged, so English behaves exactly as before (chars/4
+ *   over-counts English ~1.4x; measured untruncated ratio 0.73, truncated 0.24).
+ * - Han, kana, hangul, bopomofo, CJK and fullwidth punctuation, 0.75 per char: efficient
+ *   tokenizers run ~0.45-0.6 tokens per CJK char, giving an untruncated ratio of 0.6-0.8.
+ *   Qwen2.5 on Ollama runs ~0.65, so a truncated CJK prompt now reads well under 0.5.
+ * - Any other non-ASCII, 0.35 per char: Cyrillic, Greek, Arabic, accented Latin and the
+ *   like pack into ~0.22-0.35 tokens per char on large-vocabulary tokenizers. A flat 0.75
+ *   here would read an untruncated Russian log at ~0.31 and block it forever.
+ *
+ * Iterates code points, so an astral character (CJK Extension B, emoji) counts once.
+ */
+export function estimatePromptTokensForTruncation(text) {
+    // Count per class and multiply once: summing 0.35 thousands of times drifts past the
+    // integer and Math.ceil would round it up.
+    let ascii = 0;
+    let dense = 0;
+    let other = 0;
+    for (const char of text) {
+        if ((char.codePointAt(0) ?? 0) < 0x80)
+            ascii += 1;
+        else if (TOKEN_DENSE_SCRIPT.test(char))
+            dense += 1;
+        else
+            other += 1;
+    }
+    const tokens = ascii * TOKENS_PER_ASCII_CHAR + dense * TOKENS_PER_DENSE_CHAR + other * TOKENS_PER_OTHER_CHAR;
+    return Math.max(1, Math.ceil(tokens));
+}
 function estimateTokensByChars(chars) {
     return Math.max(0, Math.ceil(chars / 4));
 }