recallnest 3.0.1 → 4.0.0

package/src/trigger-store.ts

@@ -0,0 +1,443 @@
+/**
+ * Trigger Store — 写入时预演触发器（借自腾讯 T-Mem，EMNLP 2026，arXiv 2606.15405）。
+ *
+ * 一条记忆存进来的时候，顺手存几句「她以后会怎么问起这件事」（trigger）。
+ * 这些句子单独嵌入、单独建表、只参与召回、永远不进证据：检索时用 query 向量
+ * 对 trigger 表做向量搜索，命中的 trigger 把它的宿主记忆拉进候选池，宿主按
+ * 「所有 trigger 里最高的那条」计分（T-Mem 的 nanmax 归属），再走既有打分链。
+ *
+ * 为什么另开一张表而不是塞进正文：正文向量是几百字的均值，尾巴上的一句问法
+ * 会被稀释（2026-09-22 实测：64df5691 写了「她以后可能会这样问起…」，换套口语
+ * 照样 top-10 不出现）。为什么不在主表加列：主表 12.8 万行，加列要迁移；侧表
+ * 可由主表 metadata.triggers 整表重建（`triggers-backfill --rebuild`），删掉即回退。
+ *
+ * trigger 文本用 embedQuery 嵌入：它本身就是「未来的问法」，与查询同侧任务
+ * （jina-v5 的 retrieval.query），query↔query 的余弦比 query↔passage 稳定。
+ */
+
+import { resolve } from "node:path";
+import type * as LanceDB from "@lancedb/lancedb";
+import { loadLanceDB, escapeSqlLiteral, scopeWhereClause } from "./store.js";
+import { matchesScopeFilter, type ScopeMatchMode } from "./scope-policy.js";
+import { readConsistencyInterval as envReadConsistencyInterval } from "./env-config.js";
+import { logWarn } from "./stderr-log.js";
+import { textOverlapScore, tokenize } from "./multi-vector.js";
+import { redactSecrets } from "./pii-detector.js";
+import { withWriteLock as withCrossProcessLock } from "./distill-lock.js";
+
+export const TRIGGER_TABLE_NAME = "memory_triggers";
+
+/** 每条记忆最多存几条 trigger（T-Mem 每条 item 2–6 个，这里取上限 6）。 */
+export const MAX_TRIGGERS_PER_MEMORY = 6;
+export const MIN_TRIGGER_LENGTH = 2;
+export const MAX_TRIGGER_LENGTH = 200;
+
+export interface TriggerRow {
+  id: string;
+  memory_id: string;
+  scope: string;
+  text: string;
+  vector: number[];
+  created_at: number;
+}
+
+export interface TriggerHit {
+  memoryId: string;
+  /** 命中的那条 trigger 原文（只用于 explain，不进证据） */
+  text: string;
+  /** 余弦相似度（1 - cosine distance） */
+  cosine: number;
+  /** 与主表 vectorSearch 同一映射：1 / (1 + distance)，可直接与向量分比较 */
+  score: number;
+  scope: string;
+  /** query 与 trigger 文本的词面重合（textOverlapScore），只在传了 queryText 时有 */
+  overlap?: number;
+  /** 靠哪道闸放行：hard = 余弦过硬闸；soft = 余弦过软闸 + 词面重合过线 */
+  admittedBy: "hard" | "soft";
+}
+
+export interface TriggerSearchOptions {
+  /** 余弦硬闸（≥ 即放行） */
+  minCosine?: number;
+  /** 余弦软闸：≥ softCosine 且 overlap ≥ minOverlap 也放行；需同时给 queryText */
+  softCosine?: number;
+  minOverlap?: number;
+  /** 软闸只对去停用词后 ≥ 这么多 token 的 query 开（关键词短 query 不走软闸） */
+  minQueryTokens?: number;
+  queryText?: string;
+  scopeMatch?: ScopeMatchMode;
+}
+
+export interface TriggerStoreConfig {
+  dbPath: string;
+  vectorDim: number;
+  readConsistencyInterval?: number;
+  /** 跨进程写锁的锁文件目录；不传用 distill-lock 的默认目录（<dataDir>/locks）。测试指到临时目录。 */
+  lockDir?: string;
+}
+
+/**
+ * 侧表写锁的名字。按库目录区分：同一个库的所有进程共用一把，不同的库（测试的临时库）互不等待。
+ */
+export function triggerWriteLockKey(dbPath: string): string {
+  return `trigger-write:${resolve(dbPath)}`;
+}
+
+export interface TriggerStoreStats {
+  rows: number;
+  memories: number;
+}
+
+/**
+ * 规整 trigger 文本：脱敏、去空白、去重（不分大小写）、丢太短太长的、封顶 6 条。
+ * 脱敏放在这里（2026-10-17）：trigger 原文会进 metadata.triggers、侧表和外部嵌入接口，
+ * 写入、回填、侧表重建都经过这一个函数。
+ */
+export function normalizeTriggerTexts(texts: readonly string[] | undefined | null): string[] {
+  if (!texts || texts.length === 0) return [];
+  const seen = new Set<string>();
+  const out: string[] = [];
+  for (const raw of texts) {
+    if (typeof raw !== "string") continue;
+    const text = redactSecrets(raw, { keepIdentifiers: true }).text.replace(/\s+/g, " ").trim().replace(/[。．.！!？?；;，,、]+$/u, "").trim();
+    if (text.length < MIN_TRIGGER_LENGTH || text.length > MAX_TRIGGER_LENGTH) continue;
+    const key = text.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(text);
+    if (out.length >= MAX_TRIGGERS_PER_MEMORY) break;
+  }
+  return out;
+}
+
+/**
+ * 从既有正文里把「她以后会怎么问」那段抠出来拆成 trigger 候选（回填用，零 LLM）。
+ *
+ * 认的是 §5.3 那条纪律的各种手写变体：「她以后会这么问起：「A」「B」」「她以后可能会这样问起：A、B、C」
+ * 「检索锚点：…」「她会说…」。段落里有引号就取引号内的，没有就按 、；/ 或 拆。
+ * 只认正文里已经写下的话，不生成、不改写——生成是另一条线（qwen-turbo，按 T-Mem 三条禁令），
+ * 且要等批量提炼那批的忠实度有结论再做。
+ */
+export function extractTriggerCandidates(source: string): string[] {
+  if (!source) return [];
+  // 先对整段原文脱敏再拆：按 `/` 拆会把 `https://user:密码@host` 拆成不带 scheme 的
+  // `user:密码@host`，规整时的脱敏就认不出来了（回填会把它嵌入、写进侧表与 metadata）。
+  const text = redactSecrets(source, { keepIdentifiers: true }).text;
+  const lead = /(她以后[^。\n]{0,6}问[起法]?|她(?:大概|可能|应该|以后)?会(?:这么|这样|怎么|这样子)?(?:问|说|提)(?:起)?|检索锚点|以后[^。\n]{0,8}问起|下次[^。\n]{0,6}(?:会|再)[^。\n]{0,4}(?:问|提))/gu;
+  // 取**最后一次**出现：这句纪律写在正文末尾（09-22 实测 98% 在最后 30%），
+  // 正文中段提到「她以后会怎么问」多半是在讨论这条纪律本身（如本项目自己的记忆），不是问法。
+  let m: RegExpExecArray | null = null;
+  for (const hit of text.matchAll(lead)) m = hit;
+  if (!m || m.index === undefined) return [];
+  // 长正文里这句若离末尾太远（后面还拖着六成正文），那是在讨论纪律本身，不是问法
+  if (text.length > 200 && (text.length - m.index) / text.length > 0.6) return [];
+  // 从触发词起到段落结束（下一个空行）为止
+  let segment = text.slice(m.index);
+  const paraEnd = segment.search(/\n\s*\n/);
+  if (paraEnd > 0) segment = segment.slice(0, paraEnd);
+  // 去掉引导语本身（到第一个冒号为止）
+  const colon = segment.search(/[:：]/);
+  const body = colon >= 0 && colon < 40 ? segment.slice(colon + 1) : segment.slice(m[0].length);
+
+  const quoted = [...body.matchAll(/[「“"『]([^」”"』]{2,200})[」”"』]/gu)].map((q) => q[1]);
+  let parts: string[];
+  if (quoted.length > 0) {
+    parts = quoted;
+  } else {
+    parts = body
+      .split(/[、；;／/\n]|，或者|或者|、或|或是/u)
+      .map((p) => p.trim())
+      .filter(Boolean);
+  }
+  const leadOnly = new RegExp(lead.source, "u");
+  const cleaned = parts
+    .map((p) => p.replace(/^(?:比如|例如|像|如)[:：]?\s*/u, "").replace(/^[「“"『]|[」”"』]$/gu, "").trim())
+    // 拆出来的最后一段常常拖着后文（「……这类问题。」），剪掉句号后的尾巴
+    .map((p) => p.split(/[。！？!?]/u)[0].trim())
+    .filter((p) => p.length >= 4 && p.length <= MAX_TRIGGER_LENGTH)
+    // 引导语自己、日期开头的引用、纯路径 —— 都不是问法
+    .filter((p) => !leadOnly.test(p))
+    .filter((p) => !/^\d{4}-\d{2}(-\d{2})?/.test(p))
+    .filter((p) => !(/[\/\\]/.test(p) && !/\s/.test(p)));
+  return normalizeTriggerTexts(cleaned);
+}
+
+export class TriggerStore {
+  private table: LanceDB.Table | null = null;
+  private initPromise: Promise<void> | null = null;
+  private writeChain: Promise<unknown> = Promise.resolve();
+
+  constructor(private readonly config: TriggerStoreConfig) {}
+
+  get dbPath(): string {
+    return this.config.dbPath;
+  }
+
+  private withWriteLock<T>(fn: () => Promise<T>): Promise<T> {
+    const run = this.writeChain.then(fn, fn);
+    this.writeChain = run.catch(() => {});
+    return run;
+  }
+
+  private async ensureInitialized(): Promise<void> {
+    if (this.table) return;
+    if (this.initPromise) return this.initPromise;
+    this.initPromise = this.doInitialize().catch((err) => {
+      this.initPromise = null;
+      throw err;
+    });
+    return this.initPromise;
+  }
+
+  private async doInitialize(): Promise<void> {
+    const lancedb = await loadLanceDB();
+    const db = await lancedb.connect(this.config.dbPath, {
+      readConsistencyInterval:
+        this.config.readConsistencyInterval ?? envReadConsistencyInterval(),
+    });
+
+    let table: LanceDB.Table;
+    try {
+      table = await db.openTable(TRIGGER_TABLE_NAME);
+    } catch {
+      const schemaRow: TriggerRow = {
+        id: "__schema__",
+        memory_id: "__schema__",
+        scope: "__schema__",
+        text: "",
+        vector: new Array(this.config.vectorDim).fill(0),
+        created_at: 0,
+      };
+      try {
+        table = await db.createTable(TRIGGER_TABLE_NAME, [schemaRow]);
+        await table.delete(`id = '__schema__'`);
+      } catch (createErr) {
+        if (String(createErr).includes("already exists")) {
+          table = await db.openTable(TRIGGER_TABLE_NAME);
+        } else {
+          throw createErr;
+        }
+      }
+    }
+    this.table = table;
+  }
+
+  async ready(): Promise<void> {
+    await this.ensureInitialized();
+  }
+
+  /**
+   * 覆盖式写入一条记忆的全部 trigger，同一 memory_id 永远只有一组。
+   * `embed` 由调用方传入（应为 embedQuery 批量版），这里不持有 embedder。
+   * 返回实际写入的条数；texts 规整后为空则只删不加、返回 0。
+   *
+   * 先嵌入、后动表（2026-10-17）：嵌入是网络调用，原先先删旧行再嵌入，嵌入一失败旧 trigger 就全没了。
+   * 现在嵌入失败或条数对不上时直接抛错，表一行不动。
+   *
+   * 动表在跨进程写锁里（`triggerWriteLockKey`，distill-lock 的 withWriteLock）：进程内的写锁管不到
+   * 别的 MCP / CLI 进程。两个进程同时给一条还没有 trigger 的记忆首写，按 id 合并时双方都只是插入、
+   * 不改已有行，Lance 不判冲突，会写出两组（2026-10-17 复审）。嵌入在跨进程锁外面做，锁只罩住读表与一次提交；
+   * 进程内的写锁仍罩住嵌入，同一进程里后发起的重写一定后落表。
+   *
+   * 换新是一次 mergeInsert 提交，合并键是 (id, created_at)：新行的 created_at 在锁内取成比这条记忆
+   * 现有各行都大，所以没有一行旧行对得上，新行全部插入，这条记忆的旧行全部经
+   * whenNotMatchedBySourceDelete（限定本 memory_id）在同一个提交里删掉。只按 id 合并时，旧竞态留下的
+   * 两行同 id 会同时对上一条新行、两行都被更新而不是删掉，重复行永远修不掉（2026-10-17 复审）。
+   */
+  async upsertForMemory(
+    memoryId: string,
+    scope: string,
+    texts: readonly string[],
+    embed: (texts: string[]) => Promise<number[][]>,
+  ): Promise<number> {
+    const cleaned = normalizeTriggerTexts(texts);
+    return this.withWriteLock(async () => {
+      await this.ensureInitialized();
+      const where = `memory_id = '${escapeSqlLiteral(memoryId)}'`;
+      if (cleaned.length === 0) {
+        await this.withSharedLock(async (table) => { await table.delete(where); });
+        return 0;
+      }
+      const vectors = await embed(cleaned);
+      if (vectors.length !== cleaned.length) {
+        throw new Error(`TriggerStore: embed returned ${vectors.length} vectors for ${cleaned.length} texts`);
+      }
+      return this.withSharedLock(async (table) => {
+        const existing = await table.query().where(where).select(["created_at"]).toArray();
+        let createdAt = Date.now();
+        for (const row of existing) {
+          const prev = Number((row as Record<string, unknown>).created_at);
+          if (Number.isFinite(prev) && prev >= createdAt) createdAt = prev + 1;
+        }
+        const rows: TriggerRow[] = cleaned.map((text, i) => ({
+          id: `${memoryId}#${i}`,
+          memory_id: memoryId,
+          scope,
+          text,
+          vector: vectors[i],
+          created_at: createdAt,
+        }));
+        await table.mergeInsert(["id", "created_at"])
+          .whenNotMatchedInsertAll()
+          .whenNotMatchedBySourceDelete({ where })
+          .execute(rows);
+        return rows.length;
+      });
+    });
+  }
+
+  /**
+   * 删掉一条记忆的全部 trigger 行，返回删了几行。
+   * forget 引擎在删主行之前调它（2026-10-05 之前这个方法没有任何调用方，
+   * 记忆删掉后它的问法原文和向量留在侧表里）。
+   * 「数」和「删」都在跨进程写锁里，别的进程经 upsertForMemory 写这条记忆时插不进中间；
+   * 返回值就是删掉的行数。forget 引擎删完主行后会再数一次（countForMemory）。
+   * 侧表还不存在时，这里和 countForMemory 都会像写入路径一样把它建出来（空表）。
+   */
+  async deleteForMemory(memoryId: string): Promise<number> {
+    return this.withWriteLock(async () => {
+      await this.ensureInitialized();
+      const where = `memory_id = '${escapeSqlLiteral(memoryId)}'`;
+      return this.withSharedLock(async (table) => {
+        const rows = await table.countRows(where);
+        if (rows > 0) await table.delete(where);
+        return rows;
+      });
+    });
+  }
+
+  /**
+   * 持跨进程写锁跑一段写表操作。拿到锁之后先把表句柄切到最新版本：
+   * 句柄可能停在等锁之前看到的版本，锁内的读（现有行的 created_at、删前计数）要看到别的进程刚提交的行。
+   */
+  private withSharedLock<T>(fn: (table: LanceDB.Table) => Promise<T>): Promise<T> {
+    // lock-wait: no store-write inside — fn only touches the memory_triggers side table.
+    return withCrossProcessLock(triggerWriteLockKey(this.config.dbPath), async () => {
+      const table = this.table!;
+      await table.checkoutLatest();
+      return fn(table);
+    }, {
+      expireMs: 30_000,
+      waitTimeoutMs: 30_000,
+      ...(this.config.lockDir ? { lockDir: this.config.lockDir } : {}),
+    });
+  }
+
+  /** 这条记忆现在还有几行 trigger（forget 删后回读用）。 */
+  async countForMemory(memoryId: string): Promise<number> {
+    await this.ensureInitialized();
+    return this.table!.countRows(`memory_id = '${escapeSqlLiteral(memoryId)}'`);
+  }
+
+  /**
+   * 用 query 向量搜 trigger，按宿主聚合取最高分（nanmax 归属），再过闸。
+   * 两道闸（2026-09-22 校准，见 env-config triggerGate / triggerSoftGate）：
+   *   hard：cosine ≥ minCosine；
+   *   soft：query 去停用词 ≥ minQueryTokens 个 token，且 cosine ≥ softCosine，且 textOverlapScore(queryText, trigger) ≥ minOverlap。
+   * 为什么要软闸：jina-v5-small 上「腾讯那篇论文的榜我们要不要也去排个名」对
+   * 「别的记忆系统的榜我们要不要也去排个名」余弦只有 0.44，与无关噪声（最高 0.53）分不开，
+   * 但两句共用「榜 / 要不要 / 排个名」——词面是第二根轴，T-Mem 每层也是词法 + 稠密一起用。
+   * scope 过滤下推到 SQL（与主表 vectorSearch 同款，2026-08-16 互审 C1 的教训）。
+   */
+  async search(
+    queryVector: number[],
+    topK: number,
+    scopeFilter?: string[],
+    options: TriggerSearchOptions = {},
+  ): Promise<TriggerHit[]> {
+    await this.ensureInitialized();
+    const minCosine = options.minCosine ?? 0;
+    const minOverlap = options.minOverlap ?? 1;
+    const minQueryTokens = options.minQueryTokens ?? 0;
+    // 软闸的前提：query 像一句话（不是两个关键词）——否则一个共用词就能把词面重合撑高
+    const softEligible = options.softCosine !== undefined
+      && !!options.queryText
+      && tokenize(options.queryText).length >= minQueryTokens;
+    const softCosine = softEligible ? options.softCosine : undefined;
+    const scopeMatch = options.scopeMatch ?? "family";
+    const safeTopK = Math.max(1, Math.min(50, Math.floor(topK)));
+    // 一个宿主可能有多条 trigger 都命中，多取一些再聚合
+    const fetchLimit = Math.min(safeTopK * MAX_TRIGGERS_PER_MEMORY, 200);
+
+    let query = this.table!.vectorSearch(queryVector).distanceType("cosine").limit(fetchLimit);
+    if (scopeFilter && scopeFilter.length > 0) {
+      query = query.where(`(${scopeWhereClause(scopeFilter, scopeMatch)})`);
+    }
+
+    let rows: Array<Record<string, unknown>>;
+    try {
+      rows = await query.toArray();
+    } catch (err) {
+      logWarn("TriggerStore.search failed, continuing without trigger recall:", err);
+      return [];
+    }
+
+    const best = new Map<string, TriggerHit>();
+    for (const row of rows) {
+      const memoryId = String(row.memory_id ?? "");
+      if (!memoryId || memoryId === "__schema__") continue;
+      const rowScope = String(row.scope ?? "");
+      if (!matchesScopeFilter(rowScope, scopeFilter, scopeMatch)) continue;
+      const distance = Number(row._distance ?? 1);
+      const cosine = 1 - distance;
+      const text = String(row.text ?? "");
+      let admittedBy: "hard" | "soft" | null = null;
+      let overlap: number | undefined;
+      if (cosine >= minCosine) {
+        admittedBy = "hard";
+      } else if (softCosine !== undefined && options.queryText && cosine >= softCosine) {
+        overlap = textOverlapScore(options.queryText, text);
+        if (overlap >= minOverlap) admittedBy = "soft";
+      }
+      if (!admittedBy) continue;
+      const hit: TriggerHit = {
+        memoryId,
+        text,
+        cosine,
+        score: 1 / (1 + distance),
+        scope: rowScope,
+        overlap,
+        admittedBy,
+      };
+      const prev = best.get(memoryId);
+      if (!prev || hit.cosine > prev.cosine) best.set(memoryId, hit);
+    }
+    return [...best.values()]
+      .sort((a, b) => b.cosine - a.cosine)
+      .slice(0, safeTopK);
+  }
+
+  /** 列出某 scope 下已有 trigger 的宿主 id（回填时跳过已做的）。 */
+  async listMemoryIds(scopeFilter?: string[]): Promise<Set<string>> {
+    await this.ensureInitialized();
+    let query = this.table!.query().select(["memory_id", "scope"]);
+    if (scopeFilter && scopeFilter.length > 0) {
+      query = query.where(`(${scopeWhereClause(scopeFilter)})`);
+    }
+    const rows = await query.toArray();
+    const ids = new Set<string>();
+    for (const row of rows) {
+      const id = String((row as Record<string, unknown>).memory_id ?? "");
+      if (id && id !== "__schema__") ids.add(id);
+    }
+    return ids;
+  }
+
+  async stats(scopeFilter?: string[]): Promise<TriggerStoreStats> {
+    await this.ensureInitialized();
+    let query = this.table!.query().select(["memory_id", "scope"]);
+    if (scopeFilter && scopeFilter.length > 0) {
+      query = query.where(`(${scopeWhereClause(scopeFilter)})`);
+    }
+    const rows = await query.toArray();
+    const memories = new Set<string>();
+    let count = 0;
+    for (const row of rows) {
+      const id = String((row as Record<string, unknown>).memory_id ?? "");
+      if (!id || id === "__schema__") continue;
+      count += 1;
+      memories.add(id);
+    }
+    return { rows: count, memories: memories.size };
+  }
+}

package/src/ui-server.ts

@@ -5,15 +5,15 @@ import { join, resolve, sep } from "node:path";
 
 import { metaDir } from "./compat.js";
 import type { RetrievalResult } from "./retriever.js";
-import type { MemoryStore } from "./store.js";
 import { distillResults, formatExplainResults, formatSearchResults, selectBriefSeedResults, summarizeResults } from "./memory-output.js";
 import { extractMemoryProvenance } from "./memory-boundaries.js";
-import { archiveDirtyBriefAsset, assetSummaryLine, buildBriefAsset, buildPinAsset, listDirtyBriefAssets, listExportArtifacts, listMemoryAssets, saveBriefAsset, savePinAsset, writeExportArtifact } from "./memory-assets.js";
-import { indexAsset, indexPinnedAsset } from "./asset-sync.js";
+import { archiveDirtyBriefAsset, assetSummaryLine, buildBriefAsset, formatDirtyBriefEmptyMessage, formatDirtyBriefReadErrors, listExportArtifacts, scanDirtyBriefAssets, listMemoryAssets, saveBriefAsset, writeExportArtifact } from "./memory-assets.js";
+import { indexAsset, pinMemory } from "./asset-sync.js";
 import { createComponentResolver, loadConfig, loadDotEnv } from "./runtime-config.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
 import { enforceLocalHttpRequestPolicy, LOCAL_HTTP_HOSTNAME } from "./http-request-policy.js";
 import * as envConfig from "./env-config.js";
+import { HTTP_WRITE_IDLE_TIMEOUT_S } from "./store-write-budget.js";
 const config = (loadDotEnv(), loadConfig());
 const getComponents = createComponentResolver(config);
 
@@ -26,14 +26,6 @@ class UiHttpError extends Error {
   }
 }
 
-function entryToRetrievalResult(entry: Awaited<ReturnType<MemoryStore["get"]>>): RetrievalResult {
-  if (!entry) throw new Error("Memory entry not found.");
-  return {
-    entry,
-    score: entry.importance || 0.7,
-    sources: { fused: { score: entry.importance || 0.7 } },
-  };
-}
 
 async function readJson(request: Request) {
   try {
@@ -112,6 +104,7 @@ function getRetrievalPath(result: RetrievalResult): string {
   if (result.sources.vector) parts.push("vector");
   if (result.sources.bm25) parts.push("bm25");
   if (result.sources.reranked) parts.push("reranked");
+  if (result.sources.trigger) parts.push("trigger");
   return parts.join("+") || "direct";
 }
 
@@ -189,6 +182,9 @@ async function handleSearch(mode: "search" | "explain" | "distill", body: Record
 
 const server = Bun.serve({
   port: readLimit(envConfig.uiPortRaw(), 4317, 1, 65535),
+  // Writes can wait up to 30 s for store-write before they start; Bun's 10 s default would drop
+  // the connection while the handler still commits.
+  idleTimeout: HTTP_WRITE_IDLE_TIMEOUT_S,
   hostname: LOCAL_HTTP_HOSTNAME,
   async fetch(request) {
     try {
@@ -305,13 +301,16 @@ const server = Bun.serve({
       }
 
       if (request.method === "GET" && url.pathname === "/api/dirty-briefs") {
-        const rows = listDirtyBriefAssets();
+        const scan = scanDirtyBriefAssets();
+        const rows = scan.items;
+        const readErrors = formatDirtyBriefReadErrors(scan.readErrors);
         const output = rows.length === 0
-          ? "No dirty briefs found."
-          : rows.map((row) => `${row.id.slice(0, 8)}  ${row.title}  [${row.scope}]  ${row.reasons.join("; ")}`).join("\n");
+          ? formatDirtyBriefEmptyMessage(scan)
+          : rows.map((row) => `${row.id.slice(0, 8)}  ${row.title}  [${row.scope}]  ${row.reasons.join("; ")}`).join("\n") + (readErrors ? `\n\n${readErrors}` : "");
         return Response.json({
           output,
           count: rows.length,
+          readErrors: scan.readErrors,
           items: rows.map((row) => ({
             id: row.id,
             shortId: row.id.slice(0, 8),
@@ -335,15 +334,14 @@ const server = Bun.serve({
         });
         const entry = await store.get(memoryId, scopeSelection.scopeFilter);
         if (!entry) return textResponse(`Memory not found: ${memoryId}`, { status: 404 });
-        await store.update(entry.id, { importance: Math.max(entry.importance || 0.7, 0.95) }, scopeSelection.scopeFilter);
-        const asset = buildPinAsset(entryToRetrievalResult(entry), {
+        const pinned = await pinMemory(store, embedder, entry, scopeSelection.scopeFilter, {
           title: readOptionalString(body.title),
           summary: readOptionalString(body.summary),
           query: readOptionalString(body.query),
           profile: readOptionalString(body.profile) as any || "default",
         });
-        const path = savePinAsset(asset);
-        await indexPinnedAsset(store, embedder, asset);
+        if (!pinned) return textResponse(`Memory not found: ${memoryId}`, { status: 404 });
+        const { asset, path } = pinned;
         return Response.json({
           output: `Pinned ${asset.id.slice(0, 8)}\nMemory: ${entry.id.slice(0, 8)} (${entry.scope})\nPath: ${path}`,
           assetId: asset.id,
@@ -381,9 +379,11 @@ const server = Bun.serve({
       }
 
       if (request.method === "POST" && url.pathname === "/api/clean-dirty-briefs") {
-        const rows = listDirtyBriefAssets();
+        const scan = scanDirtyBriefAssets();
+        const rows = scan.items;
+        const readErrors = formatDirtyBriefReadErrors(scan.readErrors);
         if (rows.length === 0) {
-          return Response.json({ output: "No dirty briefs found.", count: 0, archived: 0, deleted: 0 });
+          return Response.json({ output: formatDirtyBriefEmptyMessage(scan), count: 0, archived: 0, deleted: 0, readErrors: scan.readErrors });
         }
 
         let archived = 0;
@@ -395,10 +395,11 @@ const server = Bun.serve({
         }
 
         return Response.json({
-          output: `Dirty briefs: ${rows.length}\nArchived: ${archived}\nIndex rows deleted: ${deleted}`,
+          output: `Dirty briefs: ${rows.length}\nArchived: ${archived}\nIndex rows deleted: ${deleted}` + (readErrors ? `\n\n${readErrors}` : ""),
           count: rows.length,
           archived,
           deleted,
+          readErrors: scan.readErrors,
         });
       }
 

package/src/version-manager.ts

@@ -117,14 +117,14 @@ export function computeVersionGroupPatch(
  * @param canonical  The stronger entry (higher canonicalScore)
  * @param member     The weaker entry
  * @param scope      Scope filter for updates
- * @returns The version_group ID
+ * @returns The version_group ID, or null when either row was deleted before it could be tagged
  */
 export async function createVersionGroup(
-  store: Pick<MemoryStorePort, "update">,
+  store: Pick<MemoryStorePort, "update" | "patchMetadata">,
   canonical: MemoryEntry,
   member: MemoryEntry,
   scope: string,
-): Promise<string> {
+): Promise<string | null> {
   // Check if canonical already has a version_group
   const canonMeta = parseMetadata(canonical.metadata);
   const existingGroup = typeof canonMeta.version_group === "string" ? canonMeta.version_group : null;
@@ -136,7 +136,10 @@ export async function createVersionGroup(
   canonMeta.version_group = groupId;
   canonMeta.version_rank = canonRank;
   if (!canonMeta.version_created) canonMeta.version_created = now;
-  await store.update(canonical.id, { metadata: JSON.stringify(canonMeta) }, [scope]);
+  // update() returns null when the row was deleted after it was read: a group needs both
+  // rows, so a vanished canonical stops here and the member is left untouched.
+  const canonUpdated = await store.update(canonical.id, { metadata: JSON.stringify(canonMeta) }, [scope]);
+  if (!canonUpdated) return null;
 
   // Update member
   const memberMeta = parseMetadata(member.metadata);
@@ -144,7 +147,18 @@ export async function createVersionGroup(
   memberMeta.version_group = groupId;
   memberMeta.version_rank = memberRank;
   memberMeta.version_created = now;
-  await store.update(member.id, { metadata: JSON.stringify(memberMeta) }, [scope]);
+  const memberUpdated = await store.update(member.id, { metadata: JSON.stringify(memberMeta) }, [scope]);
+  if (!memberUpdated) {
+    // Member vanished: a group created just now would be left with the canonical alone —
+    // take the three group keys back off it. A pre-existing group keeps its other members.
+    // Rolled back through the locked read-modify-write: it re-reads the canonical's current
+    // metadata and clears the three keys only while the group is still the one set here, so
+    // another writer's metadata committed since is kept (the stale canonMeta snapshot is not).
+    if (!existingGroup) {
+      await store.patchMetadata(canonical.id, (meta) => clearVersionGroupMetadata(meta, groupId), [scope]);
+    }
+    return null;
+  }
 
   logInfo(
     `[INFO] Version group ${groupId.slice(0, 8)}: ` +

package/src/write-verifier.ts

@@ -1,6 +1,6 @@
 /**
  * HP-2: Formation Review Pass — post-write verification.
- * After storing a memory, verify: embedding exists, metadata complete, scope set.
+ * After storing a memory, verify: embedding exists, text non-empty, scope and importance set, metadata parses.
  * Non-blocking: failures are logged but never block the write path.
  */
 
@@ -31,7 +31,8 @@ export type VerificationIssue =
   | "missing_vector"
   | "missing_scope"
   | "missing_importance"
-  | "empty_text";
+  | "empty_text"
+  | "corrupt_metadata";
 
 export interface VerificationResult {
   ok: boolean;
@@ -107,19 +108,23 @@ function checkEntry(entry: MemoryEntry | null, id: string): VerificationIssue[]
     issues.push("empty_text");
   }
 
-  // Metadata should contain scope
+  // scope / importance 是行上的列，不在 metadata 里——buildStructuredMetadata 从不写这两个字段。
+  // 原先查 metadata，对每一次真实写入都报 missing_scope + missing_importance（agy-sync 日志
+  // 5,052 条告警里 5,051 条是它，2026-09-24 统计），真正有用的 missing_entry 被埋在里面。
+  if (typeof entry.scope !== "string" || entry.scope.trim().length === 0) {
+    issues.push("missing_scope");
+  }
+  if (typeof entry.importance !== "number" || !Number.isFinite(entry.importance)) {
+    issues.push("missing_importance");
+  }
+
+  // Metadata has to parse: every downstream reader JSON.parse()s it.
   if (entry.metadata) {
     try {
-      const meta = JSON.parse(entry.metadata);
-      if (!meta.scope) issues.push("missing_scope");
-      if (typeof meta.importance !== "number") issues.push("missing_importance");
+      JSON.parse(entry.metadata);
     } catch {
-      issues.push("missing_scope");
-      issues.push("missing_importance");
+      issues.push("corrupt_metadata");
     }
-  } else {
-    issues.push("missing_scope");
-    issues.push("missing_importance");
   }
 
   return issues;