recallnest 2.6.1 → 3.0.0

package/src/__tests__/verbatim-self-recall.test.ts

@@ -0,0 +1,279 @@
+/**
+ * Verbatim self-recall: can a memory be found using words copied out of its own body?
+ *
+ * ## Where this came from
+ *
+ * On 2026-08-13 a window tried to correct a stored case (`4cb4853e`) and could not reach
+ * it. `explain_memory` was called twice, eleven recall slots in total, with queries taken
+ * verbatim from that case's own text. The target came back zero times. Ranking first in
+ * the same result sets were two short, frequently-read `session_distill` entries.
+ *
+ * The correction was abandoned rather than guessed at — writing without the canonical key
+ * would have produced a second, parallel memory instead of a revision. So the failure was
+ * recorded as an anecdote and nothing more. This file is that anecdote turned into
+ * something that can be run.
+ *
+ * ## What the reproduction shows
+ *
+ * The fixture below is the shape that failed: one long, older, never-read `cases` entry
+ * competing against a dozen short, recent, frequently-read ones, with the long entry
+ * holding by far the best raw content score because the query is lifted from its text.
+ *
+ * Measured against the production `DEFAULT_RETRIEVAL_CONFIG`:
+ *
+ *   raw vector score    target 0.920   distractors 0.565–0.620
+ *   after the chain     target 0.469   distractors 0.47–0.71, target ranked 13 of 13
+ *
+ * The important detail — and it corrects the looser "the threshold cuts candidates down
+ * to single digits" reading this was first filed under — is that **nothing filters the
+ * target out**. Stage counts show `hard_min_score`, `layer_admission` and `noise_filter`
+ * each dropping zero rows; 0.469 clears `hardMinScore` 0.35 comfortably. The entry is
+ * admitted and then ranked last, so at any realistic `limit` it is simply never seen.
+ *
+ * The demotion is cumulative rather than attributable to one stage: importance weight,
+ * length normalisation, time decay and the hotness/frequency boosts each move a long,
+ * old, unread entry down and a short, fresh, popular one up. An ablation confirmed no
+ * single one of them is decisive — restoring any one factor on its own still leaves the
+ * target outside the top 5.
+ *
+ * ## Why these tests assert the failure instead of the fix
+ *
+ * Repairing the ranking chain is a design-level change that is deliberately out of scope
+ * here: it needs a shadow run and a per-row account of what drops out, and the candidate
+ * pool widening people reach for first was already measured and rejected
+ * (`rejected-recall-widen-candidate-pool`, 2026-08-22). So this is a characterisation
+ * test. It pins the current behaviour precisely enough that any change to the chain has
+ * to come past it, and it states the property that ought to hold.
+ *
+ * **If a test in the second block starts failing, that is very likely good news.** It
+ * means self-recall improved. Re-read the numbers, then move the assertion into the first
+ * block as a real invariant.
+ */
+
+import { describe, expect, it } from "bun:test";
+
+import { createRetriever, DEFAULT_RETRIEVAL_CONFIG } from "../retriever.js";
+import { TraceCollector } from "../retrieval-trace.js";
+
+const DAY_MS = 86_400_000;
+
+/** One paragraph of the kind of long case entry that failed — repeated to realistic length. */
+const CASE_PARAGRAPH =
+  "冷启动抑制已证伪：hotness 改成 bonus-only 之后，阈值风险队列从 52.25% 降到 33.12%。" +
+  "互审拦下三处过度断言，shadow 报告的口径与生产分类器不一致。";
+
+const TARGET_TEXT = CASE_PARAGRAPH.repeat(10);
+const TARGET_ID = "4cb4853e-0000-0000-0000-000000000000";
+
+/** Lifted word-for-word out of TARGET_TEXT — this is what "its own phrasing" means. */
+const VERBATIM_QUERY = "冷启动抑制已证伪 阈值风险队列 互审拦下三处过度断言 shadow 报告";
+
+const DISTRACTOR_COUNT = 12;
+
+function buildFixture() {
+  const now = Date.now();
+
+  // The target: long, two months old, never read. Best possible content match.
+  const target = {
+    id: TARGET_ID,
+    text: TARGET_TEXT,
+    vector: [1, 0, 0],
+    category: "cases",
+    scope: "memory",
+    importance: 0.75,
+    timestamp: now - 60 * DAY_MS,
+    metadata: JSON.stringify({
+      evolution: {
+        status: "active",
+        version: 1,
+        accessCount: 0,
+        lastAccessedAt: null,
+        validFrom: now - 60 * DAY_MS,
+        validUntil: null,
+      },
+      boundary: { layer: "durable", authority: "structured-memory", conflictPolicy: "latest-wins" },
+    }),
+  };
+
+  // The distractors: short, fresh, heavily read. Weak content match on this query.
+  const distractors = Array.from({ length: DISTRACTOR_COUNT }, (_, i) => ({
+    id: `dddddddd-${String(i).padStart(4, "0")}-0000-0000-000000000000`,
+    text: `会话提炼片段 ${i}：确认了一个与检索质量相关的方向。`,
+    vector: [1, 0, 0],
+    category: "events",
+    scope: "memory",
+    importance: 0.9,
+    timestamp: now - 2 * DAY_MS,
+    metadata: JSON.stringify({
+      evolution: {
+        status: "active",
+        version: 1,
+        accessCount: 40,
+        lastAccessedAt: now - DAY_MS,
+        validFrom: now - 2 * DAY_MS,
+        validUntil: null,
+      },
+      boundary: { layer: "durable", authority: "structured-memory", conflictPolicy: "latest-wins" },
+    }),
+  }));
+
+  return [
+    { entry: target, score: 0.92 },
+    ...distractors.map((entry, i) => ({ entry, score: 0.62 - i * 0.005 })),
+  ];
+}
+
+function buildRetriever(candidates: ReturnType<typeof buildFixture>) {
+  return createRetriever(
+    {
+      hasFtsSupport: false,
+      async vectorSearch() {
+        return candidates;
+      },
+    } as any,
+    {
+      async embedQuery() {
+        return [1, 0, 0];
+      },
+      async embedPassage() {
+        return [1, 0, 0];
+      },
+    } as any,
+    // Production defaults, minus the reranker (it needs a vendor key and a network call).
+    { ...DEFAULT_RETRIEVAL_CONFIG, rerank: "none" as const },
+  );
+}
+
+async function retrieveWithTrace(limit: number) {
+  const candidates = buildFixture();
+  const retriever = buildRetriever(candidates);
+  const trace = new TraceCollector();
+  const results = await retriever.retrieve({
+    query: VERBATIM_QUERY,
+    scopeFilter: ["memory"],
+    limit,
+    trace,
+  });
+  return { results, stages: trace.finalize(VERBATIM_QUERY, "vector").stages };
+}
+
+// ---------------------------------------------------------------------------
+// Invariants — these must keep holding.
+// ---------------------------------------------------------------------------
+
+describe("verbatim self-recall: what the fixture guarantees", () => {
+  it("gives the target the strictly best raw content score", () => {
+    // Guards the premise. If the fixture ever stops modelling "the query matches this one
+    // best", the tests below would be measuring something else entirely.
+    const candidates = buildFixture();
+    const target = candidates.find((c) => c.entry.id === TARGET_ID);
+    const best = Math.max(...candidates.filter((c) => c.entry.id !== TARGET_ID).map((c) => c.score));
+
+    expect(target?.score).toBe(0.92);
+    expect(target!.score).toBeGreaterThan(best);
+  });
+
+  it("admits the target through every gate — it is not filtered out", async () => {
+    const { stages } = await retrieveWithTrace(DISTRACTOR_COUNT + 1);
+
+    // This is the part that must never regress into an actual filter. Losing a
+    // verbatim-matching memory to a ranking is bad; dropping it at a threshold would be
+    // worse, and would need a different fix.
+    for (const name of ["hard_min_score", "noise_filter"]) {
+      const stage = stages.find((s) => s.name === name);
+      expect(stage, `stage ${name} missing from trace`).toBeDefined();
+      expect(stage?.droppedCount, `stage ${name} dropped rows`).toBe(0);
+    }
+
+    // `layer_admission` is env-gated: applyLayerAdmission returns before touching the
+    // trace when RECALLNEST_LAYER_ADMISSION is off, which is the default. Asserting it
+    // unconditionally would only be asserting the developer's local .env — so check it
+    // when it ran, and rely on the whole-result assertion below when it did not.
+    const admission = stages.find((s) => s.name === "layer_admission");
+    if (admission) expect(admission.droppedCount, "layer admission dropped rows").toBe(0);
+  });
+
+  it("returns every candidate — nothing is removed on the way through", async () => {
+    // The environment-independent form of the claim above: whatever stages ran, the
+    // result set still holds all of them, so no gate removed the target.
+    const { results } = await retrieveWithTrace(DISTRACTOR_COUNT + 1);
+
+    expect(results).toHaveLength(DISTRACTOR_COUNT + 1);
+    expect(results.some((r) => r.entry.id === TARGET_ID)).toBe(true);
+  });
+
+  it("keeps the target's final score above the hard minimum", async () => {
+    const { results } = await retrieveWithTrace(DISTRACTOR_COUNT + 1);
+    const target = results.find((r) => r.entry.id === TARGET_ID);
+
+    expect(target).toBeDefined();
+    expect(target!.score).toBeGreaterThan(DEFAULT_RETRIEVAL_CONFIG.hardMinScore);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// The defect — currently reproduced. A failure here probably means it got better.
+// ---------------------------------------------------------------------------
+
+describe("verbatim self-recall: the reported failure, reproduced", () => {
+  it("does not return the target in a default-sized result window", async () => {
+    // The original report in one line: eleven recall slots across two calls, queries taken
+    // from the entry's own text, target never returned.
+    const { results } = await retrieveWithTrace(5);
+
+    expect(results.some((r) => r.entry.id === TARGET_ID)).toBe(false);
+  });
+
+  it("still does not return it when the window is doubled", async () => {
+    // Widening the window is the obvious first reflex, so it is worth recording that it
+    // does not work here either — the entry is last, not marginal.
+    const { results } = await retrieveWithTrace(10);
+
+    expect(results.some((r) => r.entry.id === TARGET_ID)).toBe(false);
+  });
+
+  it("ranks the best content match last once every candidate is returned", async () => {
+    const { results } = await retrieveWithTrace(DISTRACTOR_COUNT + 1);
+    const rank = results.findIndex((r) => r.entry.id === TARGET_ID);
+
+    expect(results).toHaveLength(DISTRACTOR_COUNT + 1);
+    // Went in first by a wide margin, comes out last: a complete inversion of the
+    // content signal by the weighting stages that follow it.
+    expect(rank).toBe(DISTRACTOR_COUNT);
+  });
+
+  it("shows the inversion is cumulative, not caused by one stage", async () => {
+    // Ablation summary: restoring any single disadvantage on its own — importance,
+    // recency, length, access count — still leaves the target out of the top 5. Recorded
+    // here so nobody re-runs the search for a single culprit.
+    const candidates = buildFixture();
+    const now = Date.now();
+
+    const variants: Array<[string, () => typeof candidates]> = [
+      ["importance raised to match", () =>
+        candidates.map((c) => c.entry.id === TARGET_ID
+          ? { ...c, entry: { ...c.entry, importance: 0.9 } }
+          : c)],
+      ["age reduced to match", () =>
+        candidates.map((c) => c.entry.id === TARGET_ID
+          ? { ...c, entry: { ...c.entry, timestamp: now - 2 * DAY_MS } }
+          : c)],
+      ["text shortened", () =>
+        candidates.map((c) => c.entry.id === TARGET_ID
+          ? { ...c, entry: { ...c.entry, text: CASE_PARAGRAPH } }
+          : c)],
+    ];
+
+    for (const [label, build] of variants) {
+      const results = await buildRetriever(build()).retrieve({
+        query: VERBATIM_QUERY,
+        scopeFilter: ["memory"],
+        limit: 5,
+      });
+      expect(
+        results.some((r) => r.entry.id === TARGET_ID),
+        `${label}: target unexpectedly recalled — the ablation result changed`,
+      ).toBe(false);
+    }
+  });
+});

package/src/__tests__/version-manager.test.ts

@@ -14,6 +14,8 @@ import {
   computeVersionRank,
   deduplicateByVersionGroup,
   createVersionGroup,
+  clearVersionGroupMetadata,
+  planSingletonVersionGroupRepairs,
 } from "../version-manager.js";
 import type { MemoryEntry } from "../store.js";
 
@@ -211,3 +213,36 @@ describe("deduplicateByVersionGroup", () => {
     expect(deduped[0].entry.id).toBe("only");
   });
 });
+
+describe("singleton version-group repair", () => {
+  it("plans only groups with one surviving member", () => {
+    const singleton = makeEntry("solo", { version_group: "vg-single", version_rank: 1 });
+    const pairA = makeEntry("pair-a", { version_group: "vg-pair", version_rank: 2 });
+    const pairB = makeEntry("pair-b", { version_group: "vg-pair", version_rank: 1 });
+    const malformed = makeEntry("bad");
+    malformed.metadata = "{not-json";
+
+    expect(planSingletonVersionGroupRepairs([
+      singleton,
+      pairA,
+      pairB,
+      makeEntry("ungrouped"),
+      malformed,
+    ])).toEqual([{ id: "solo", groupId: "vg-single" }]);
+  });
+
+  it("clears only version-group keys and preserves unrelated metadata", () => {
+    const meta: Record<string, unknown> = {
+      version_group: "vg-single",
+      version_rank: 2.3,
+      version_created: "2026-07-07T00:00:00.000Z",
+      confidence: 0.9,
+      evolution: { status: "active" },
+    };
+
+    expect(clearVersionGroupMetadata(meta, "vg-single")).toEqual({
+      confidence: 0.9,
+      evolution: { status: "active" },
+    });
+  });
+});

package/src/access-tracker.ts

@@ -11,6 +11,7 @@
 
 import type { MemoryStore } from "./store.js";
 import { evaluateTierChange, resolveTier, type MemoryTier } from "./decay-engine.js";
+import { PROCESS_READER_ID, recordRecallHits } from "./recall-ledger.js";
 import { logInfo, logWarn } from "./stderr-log.js";
 
 // ============================================================================
@@ -70,7 +71,10 @@ export class AccessTracker {
     private config: AccessTrackerConfig = DEFAULT_ACCESS_TRACKER_CONFIG,
     readerId?: string,
   ) {
-    this.readerId = readerId ?? `r-${crypto.randomUUID().slice(0, 8)}`;
+    // 默认取进程级身份，不是每个实例各随机一个：components 按 profile 缓存，
+    // 一个会话开两个 profile 就会伪造出两个 reader，既污染 distinctReaderCount
+    // 也让 workflow observation 的 readerId join 对不上（见 recall-ledger.ts 抬头）。
+    this.readerId = readerId ?? PROCESS_READER_ID;
   }
 
   /**
@@ -137,6 +141,11 @@ export class AccessTracker {
 
     const now = Date.now();
 
+    // P0 join key：先记账本再过闸。账本回答「agent 看到过哪些条目」，
+    // 下面的 novelty/cooldown 闸回答「哪些值得强化半衰期」——两个问题，
+    // 用后者过滤前者会把最常被读到的条目系统性排除在 join 之外。
+    recordRecallHits(ids, now);
+
     for (let i = 0; i < ids.length; i++) {
       const id = ids[i];
       const score = scores?.[i];

package/src/activity-counter.ts

@@ -2,6 +2,16 @@
  * HP-3: Activity-driven distill frequency.
  * Tracks write operations since last distill, exposes tier-based thresholds.
  * Complements CC-6 distill-lock: HP-3 decides "when to trigger", CC-6 decides "whether to allow".
+ *
+ * 2026-08-14 并发加固（dream 写爆库根治的一部分）：
+ * - 写路径（increment/reset/resetBatch/prune）全部 async 化，进 `activity-stats` 跨进程锁 ——
+ *   此前 increment（11 个 mcp-server + ingest）与 reset（dream 04:00）裸读改写同一文件，
+ *   互踩窗口内的计数会静默丢失。
+ * - writeStats 改临时文件 + 原子 rename：裸 writeFileSync 期间被并发读到半截 JSON 时，
+ *   readStats 的 catch 会返回空表 —— 一次损坏放大成"整个队列清零"。原子替换后读方
+ *   永远看到完整文件（旧版或新版）。
+ * - 读路径（getWriteCount/listScopesAboveThreshold/getDistillTier）保持同步无锁：原子
+ *   rename 保证读到的文件完整；读到旧一拍的计数对 best-effort 语义无害。
  */
 
 import {
@@ -9,9 +19,11 @@ import {
   readFileSync,
   writeFileSync,
   mkdirSync,
+  renameSync,
 } from "node:fs";
 import { join, dirname } from "node:path";
 import * as envConfig from "./env-config.js";
+import { withLock } from "./distill-lock.js";
 
 // ---------------------------------------------------------------------------
 // Config
@@ -80,7 +92,22 @@ function readStats(statsPath: string): ActivityStats {
 
 function writeStats(statsPath: string, stats: ActivityStats): void {
   mkdirSync(dirname(statsPath), { recursive: true });
-  writeFileSync(statsPath, JSON.stringify(stats, null, 2), "utf-8");
+  // 原子替换：先写同目录临时文件再 rename。并发读方要么看到旧版要么看到新版，
+  // 不会读到半截 JSON（那会被 readStats 的 catch 吞成空表，把整个队列清零）。
+  const tmpPath = `${statsPath}.tmp-${process.pid}`;
+  writeFileSync(tmpPath, JSON.stringify(stats, null, 2), "utf-8");
+  renameSync(tmpPath, statsPath);
+}
+
+/** 写路径共用的锁参数：等待而非跳过（丢计数=丢 dream 触发信号），短超时防卡死写入方。 */
+const ACTIVITY_LOCK_KEY = "activity-stats";
+const ACTIVITY_LOCK_OPTS = { onBusy: "wait" as const, waitTimeoutMs: 2_000, expireMs: 30_000 };
+
+async function withStatsLock<T>(fn: () => T): Promise<T> {
+  const outcome = await withLock(ACTIVITY_LOCK_KEY, async () => fn(), ACTIVITY_LOCK_OPTS);
+  // onBusy:"wait" resolves ran:true or throws on timeout; ran:false unreachable.
+  if (!outcome.ran) throw new Error("activity-stats lock unavailable");
+  return outcome.result;
 }
 
 // ---------------------------------------------------------------------------
@@ -88,16 +115,18 @@ function writeStats(statsPath: string, stats: ActivityStats): void {
 // ---------------------------------------------------------------------------
 
 /** Increment a scope's write counter by n (default 1). Returns the scope's new count. */
-export function incrementWriteCount(
+export async function incrementWriteCount(
   scope: string,
   n = 1,
   cfg?: Partial<ActivityCounterConfig>,
-): number {
+): Promise<number> {
   const { statsPath } = resolveConfig(cfg);
-  const stats = readStats(statsPath);
-  stats.scopes[scope] = (stats.scopes[scope] ?? 0) + n;
-  writeStats(statsPath, stats);
-  return stats.scopes[scope];
+  return withStatsLock(() => {
+    const stats = readStats(statsPath);
+    stats.scopes[scope] = (stats.scopes[scope] ?? 0) + n;
+    writeStats(statsPath, stats);
+    return stats.scopes[scope];
+  });
 }
 
 /** Read a scope's current write count without modifying (0 if never written). */
@@ -107,11 +136,40 @@ export function getWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig
 }
 
 /** Reset one scope's counter after its successful dream/distill (leaves other scopes untouched). */
-export function resetWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig>): void {
+export async function resetWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig>): Promise<void> {
   const { statsPath } = resolveConfig(cfg);
-  const stats = readStats(statsPath);
-  delete stats.scopes[scope];
-  writeStats(statsPath, stats);
+  await withStatsLock(() => {
+    const stats = readStats(statsPath);
+    delete stats.scopes[scope];
+    writeStats(statsPath, stats);
+  });
+}
+
+/**
+ * Remove every scope entry matching `predicate` in one read-modify-write (one lock
+ * acquisition, one atomic file replace). Returns the removed scope names.
+ *
+ * dream --auto 用它把 transcript scope（原文层，不参与巩固——shared-behaviors §5.3）
+ * 从计数文件里整体清扫，含未达标条目：listScopesAboveThreshold 只能看到达标的，
+ * 而未达标的 transcript 条目会让 stats 文件无限膨胀（历史会话只增不减）。
+ */
+export async function pruneWriteCounts(
+  predicate: (scope: string) => boolean,
+  cfg?: Partial<ActivityCounterConfig>,
+): Promise<string[]> {
+  const { statsPath } = resolveConfig(cfg);
+  return withStatsLock(() => {
+    const stats = readStats(statsPath);
+    const removed: string[] = [];
+    for (const scope of Object.keys(stats.scopes)) {
+      if (predicate(scope)) {
+        delete stats.scopes[scope];
+        removed.push(scope);
+      }
+    }
+    if (removed.length > 0) writeStats(statsPath, stats);
+    return removed;
+  });
 }
 
 /** Scopes whose write count is at or above `threshold` — the dream scheduler's work list. */

package/src/age-label.ts

@@ -0,0 +1,29 @@
+/**
+ * Relative age labels for rendered output.
+ *
+ * Why: a model asked to do date arithmetic on `2026-08-03` will sometimes get it
+ * wrong, and "how long ago" is exactly what decides whether a memory is still
+ * worth raising. So every rendered date carries a whole-day age next to it and
+ * nobody downstream has to subtract. (Borrowed from HKUDS/DeepTutor
+ * `memory/recall.py` — "days_ago rather than an ISO timestamp", 2026-08-17.)
+ *
+ * A future stamp (clock skew on an imported record) reads as today, not as a
+ * negative age. Unusable stamps render as "-" instead of a fabricated number.
+ */
+
+const DAY_MS = 86_400_000;
+
+export const AGE_UNKNOWN = "-";
+
+export function formatAgeLabel(timestampMs: number | null | undefined, now: number = Date.now()): string {
+  if (!timestampMs || !Number.isFinite(timestampMs)) return AGE_UNKNOWN;
+  const days = Math.floor((now - timestampMs) / DAY_MS);
+  return `${Math.max(0, days)}d`;
+}
+
+export function formatIsoAgeLabel(iso: string | null | undefined, now: number = Date.now()): string {
+  if (!iso) return AGE_UNKNOWN;
+  const parsed = Date.parse(iso);
+  if (Number.isNaN(parsed)) return AGE_UNKNOWN;
+  return formatAgeLabel(parsed, now);
+}

package/src/audit-log.ts

@@ -21,6 +21,31 @@ export type AuditOperation =
   | "cascade_forget"
   | "reject";
 
+/**
+ * Which revision of a memory a retrieval actually served.
+ *
+ * A `retrieve` row used to record only the query and a hit count, which cannot answer the
+ * question the audit log exists for: when an agent acted on recalled memory, *what did it
+ * read*? Memories are versioned in place — `archiveBeliefVersion` keeps the canonical id
+ * and bumps `evolution.version`, so "memory X was retrieved" is ambiguous across a belief
+ * change. Recording the revision alongside the id makes a past answer reconstructable.
+ *
+ * Field names are short because retrieval is by far the highest-frequency audited
+ * operation and `audit.jsonl` has no rotation.
+ */
+export interface RetrievedRevision {
+  /** First 8 chars of the memory id — enough to join against the store. */
+  id: string;
+  /** `evolution.version` at the moment it was served. */
+  rev: number;
+  /** `evolution.status`: active / superseded / archived / consolidated / pending_review. */
+  st: string;
+  /** `boundary.layer` — durable vs evidence decides how much authority the hit carried. */
+  ly?: string;
+  /** `boundary.authority` — where the content originally came from. */
+  au?: string;
+}
+
 export interface AuditEntry {
   timestamp: string; // ISO 8601
   operation: AuditOperation;
@@ -28,8 +53,18 @@ export interface AuditEntry {
   memoryId?: string;
   actor: string; // "manual" | "agent" | "api" | "system"
   details?: string; // brief context (<=200 chars)
+  /** Revision + provenance of each served result. Capped — see `retrievedTotal`. */
+  retrieved?: RetrievedRevision[];
+  /**
+   * Total results served. Present only when it exceeds `retrieved.length`, so a truncated
+   * list is always visibly truncated rather than silently short.
+   */
+  retrievedTotal?: number;
 }
 
+/** Upper bound on `retrieved` entries per row, keeping line size bounded. */
+export const AUDIT_RETRIEVED_CAP = 10;
+
 export interface AuditLogger {
   log(entry: Omit<AuditEntry, "timestamp">): void;
   /** Read recent entries (newest first) */