recallnest 2.6.1 → 3.0.0

package/src/store.ts

@@ -9,11 +9,12 @@ import { dirname, join } from "node:path";
 import { logWarn } from "./stderr-log.js";
 import { readConsistencyInterval as envReadConsistencyInterval } from "./env-config.js";
 import type { DurableMemoryCategory } from "./memory-schema.js";
-import { matchesScopeFilter } from "./scope-policy.js";
+import { matchesScopeFilter, type ScopeMatchMode } from "./scope-policy.js";
 import { detectEmotionIfEnabled } from "./emotion-detector.js";
 import { withWriteLock } from "./distill-lock.js";
 import { incrementWriteCount } from "./activity-counter.js";
 import type { MemoryStorePort, MemoryStoreStats, MemoryStoreUpdate } from "./memory-store-port.js";
+import { clearVersionGroupMetadata, planSingletonVersionGroupRepairs } from "./version-manager.js";
 
 // ============================================================================
 // Types
@@ -222,6 +223,27 @@ export function validateStoragePath(dbPath: string): string {
 
 const TABLE_NAME = "memories";
 
+/**
+ * 把 scopeFilter 编译成 SQL 条件。**store 里所有 scope 过滤的唯一出处**
+ * （2026-08-16 收敛，此前同一条规则在 6 个方法里各抄了一遍）。
+ *
+ * `mode` 见 scope-policy.ts 的 ScopeMatchMode：默认 `family` = 历史行为（有冒号精确 /
+ * 无冒号前缀），`exact` = 调用方明说给的是具体 scope 名、一律精确。
+ * 应用层双检 `matchesScopeFilter(row, filter, mode)` 必须传同一个 mode，否则 SQL 收紧了
+ * 应用层又放行（或反之），两层判据打架。
+ */
+function scopeWhereClause(scopeFilter: string[], mode: ScopeMatchMode = "family"): string {
+  return scopeFilter
+    .map(scope => {
+      const safe = escapeSqlLiteral(scope);
+      if (mode === "exact") return `scope = '${safe}'`;
+      return scope.includes(":")
+        ? `scope = '${safe}'`
+        : `scope LIKE '${safe}%'`;
+    })
+    .join(" OR ");
+}
+
 export class MemoryStore implements MemoryStorePort {
   private db: LanceDB.Connection | null = null;
   private table: LanceDB.Table | null = null;
@@ -467,7 +489,7 @@ export class MemoryStore implements MemoryStorePort {
       const statsPath = this.activityStatsPath();
       const countByScope = new Map<string, number>();
       for (const e of toWrite) countByScope.set(e.scope, (countByScope.get(e.scope) ?? 0) + 1);
-      for (const [scope, n] of countByScope) incrementWriteCount(scope, n, { statsPath });
+      for (const [scope, n] of countByScope) await incrementWriteCount(scope, n, { statsPath });
     } catch { /* activity tracking is best-effort */ }
     return toWrite.length;
   }
@@ -497,7 +519,7 @@ export class MemoryStore implements MemoryStorePort {
     }, { expireMs: 30_000 });
     // Per-scope write count drives dream activation. Best-effort — never block a write.
     try {
-      incrementWriteCount(entry.scope, 1, { statsPath: this.activityStatsPath() });
+      await incrementWriteCount(entry.scope, 1, { statsPath: this.activityStatsPath() });
     } catch { /* activity tracking is best-effort */ }
     return entry;
   }
@@ -554,7 +576,13 @@ export class MemoryStore implements MemoryStorePort {
     return res.length > 0;
   }
 
-  async vectorSearch(vector: number[], limit = 5, minScore = 0.3, scopeFilter?: string[]): Promise<MemorySearchResult[]> {
+  async vectorSearch(
+    vector: number[],
+    limit = 5,
+    minScore = 0.3,
+    scopeFilter?: string[],
+    scopeMatch: ScopeMatchMode = "family",
+  ): Promise<MemorySearchResult[]> {
     await this.ensureInitialized();
 
     const safeLimit = clampInt(limit, 1, 20);
@@ -562,19 +590,11 @@ export class MemoryStore implements MemoryStorePort {
 
     let query = this.table!.vectorSearch(vector).distanceType('cosine').limit(fetchLimit);
 
-    // Apply scope filter if provided
-    // Support both exact match ("cc:abc123") and prefix match ("cc")
+    // Apply scope filter if provided. Default mode "family" = 历史行为（含冒号精确 /
+    // 无冒号前缀）；exact 由调用方显式声明。**必须下推到 SQL，不能事后在应用层滤**——
+    // top-k 在收窄前就被兄弟 scope 占满，滤完只剩残缺候选（2026-08-16 互审 C1）。
     if (scopeFilter && scopeFilter.length > 0) {
-      const scopeConditions = scopeFilter
-        .map(scope => {
-          const safe = escapeSqlLiteral(scope);
-          // If scope contains ":", treat as exact match; otherwise prefix match
-          return scope.includes(":")
-            ? `scope = '${safe}'`
-            : `scope LIKE '${safe}%'`;
-        })
-        .join(" OR ");
-      query = query.where(`(${scopeConditions})`);
+      query = query.where(`(${scopeWhereClause(scopeFilter, scopeMatch)})`);
     }
 
     const results = await query.toArray();
@@ -588,8 +608,8 @@ export class MemoryStore implements MemoryStorePort {
 
       const rowScope = (row.scope as string | undefined) ?? "";
 
-      // Double-check scope filter in application layer (prefix-aware)
-      if (!matchesScopeFilter(rowScope, scopeFilter)) {
+      // Double-check scope filter in application layer (same mode as the SQL above)
+      if (!matchesScopeFilter(rowScope, scopeFilter, scopeMatch)) {
         continue;
       }
 
@@ -626,17 +646,9 @@ export class MemoryStore implements MemoryStorePort {
       // Use FTS query type explicitly
       let searchQuery = this.table!.search(query, "fts").limit(safeLimit);
 
-      // Apply scope filter if provided (prefix-aware, same as vectorSearch)
+      // Apply scope filter if provided (family mode, same as vectorSearch's default)
       if (scopeFilter && scopeFilter.length > 0) {
-        const scopeConditions = scopeFilter
-          .map(scope => {
-            const safe = escapeSqlLiteral(scope);
-            return scope.includes(":")
-              ? `scope = '${safe}'`
-              : `scope LIKE '${safe}%'`;
-          })
-          .join(" OR ");
-        searchQuery = searchQuery.where(`(${scopeConditions})`);
+        searchQuery = searchQuery.where(`(${scopeWhereClause(scopeFilter)})`);
       }
 
       const results = await searchQuery.toArray();
@@ -725,7 +737,13 @@ export class MemoryStore implements MemoryStorePort {
     return true;
   }
 
-  async list(scopeFilter?: string[], category?: string, limit = 20, offset = 0): Promise<MemoryEntry[]> {
+  async list(
+    scopeFilter?: string[],
+    category?: string,
+    limit = 20,
+    offset = 0,
+    scopeMatch: ScopeMatchMode = "family",
+  ): Promise<MemoryEntry[]> {
     await this.ensureInitialized();
 
     let query = this.table!.query();
@@ -734,15 +752,9 @@ export class MemoryStore implements MemoryStorePort {
     const conditions: string[] = [];
 
     if (scopeFilter && scopeFilter.length > 0) {
-      const scopeConditions = scopeFilter
-        .map(scope => {
-          const safe = escapeSqlLiteral(scope);
-          return scope.includes(":")
-            ? `scope = '${safe}'`
-            : `scope LIKE '${safe}%'`;
-        })
-        .join(" OR ");
-      conditions.push(`((${scopeConditions}))`);
+      // 收窄必须在 SQL 里：limit 是在这之后下推的，事后过滤等于先截断再筛，
+      // 目标 scope 实际拿到多少行不可控（2026-08-16 互审 C1/K3）。
+      conditions.push(`((${scopeWhereClause(scopeFilter, scopeMatch)}))`);
     }
 
     if (category) {
@@ -801,15 +813,7 @@ export class MemoryStore implements MemoryStorePort {
 
     const conditions: string[] = [];
     if (scopeFilter && scopeFilter.length > 0) {
-      const scopeConditions = scopeFilter
-        .map(scope => {
-          const safe = escapeSqlLiteral(scope);
-          return scope.includes(":")
-            ? `scope = '${safe}'`
-            : `scope LIKE '${safe}%'`;
-        })
-        .join(" OR ");
-      conditions.push(`((${scopeConditions}))`);
+      conditions.push(`((${scopeWhereClause(scopeFilter)}))`);
     }
     if (category) {
       conditions.push(`category = '${escapeSqlLiteral(category)}'`);
@@ -962,21 +966,13 @@ export class MemoryStore implements MemoryStorePort {
     return result;
   }
 
-  async stats(scopeFilter?: string[]): Promise<MemoryStoreStats> {
+  async stats(scopeFilter?: string[], scopeMatch: ScopeMatchMode = "family"): Promise<MemoryStoreStats> {
     await this.ensureInitialized();
 
     let query = this.table!.query();
 
     if (scopeFilter && scopeFilter.length > 0) {
-      const scopeConditions = scopeFilter
-        .map(scope => {
-          const safe = escapeSqlLiteral(scope);
-          return scope.includes(":")
-            ? `scope = '${safe}'`
-            : `scope LIKE '${safe}%'`;
-        })
-        .join(" OR ");
-      query = query.where(`((${scopeConditions}))`);
+      query = query.where(`((${scopeWhereClause(scopeFilter, scopeMatch)}))`);
     }
 
     const results = await query.select(["scope", "category"]).toArray();
@@ -1110,11 +1106,17 @@ export class MemoryStore implements MemoryStorePort {
     // Atomic upsert via mergeInsert — replaces the previous delete+add two-step,
     // which could lose the row entirely on a crash between steps and exposed a
     // window where concurrent reads saw the id momentarily vanish.
-    await this.table!
-      .mergeInsert("id")
-      .whenMatchedUpdateAll()
-      .whenNotMatchedInsertAll()
-      .execute([updated]);
+    //
+    // 2026-08-14: 包进 store-write 锁与 upsert/storeBatch 对齐 —— 此前 update 是唯一
+    // 裸奔的写路径（dream 的 3a/3b/auto-gc 全走它），与其他进程的 mergeInsert 并发时
+    // 靠 LanceDB 乐观重试硬扛，是 commit conflict 的隐性源之一。
+    await withWriteLock("store-write", async () => {
+      await this.table!
+        .mergeInsert("id")
+        .whenMatchedUpdateAll()
+        .whenNotMatchedInsertAll()
+        .execute([updated]);
+    }, { expireMs: 30_000 });
 
     return updated;
   }
@@ -1195,21 +1197,179 @@ export class MemoryStore implements MemoryStorePort {
     return this.metadataPatchQueues.size;
   }
 
+  /**
+   * Batched metadata read-modify-write: one store-write lock acquisition, one
+   * mergeInsert commit per batch of up to 200 rows — versus one commit per row
+   * through update()/patchMetadata().
+   *
+   * 为什么存在（2026-08-14 dream 写爆库根治）：LanceDB 每次写 commit 生成一个新
+   * manifest 版本文件，manifest 大小 ∝ 全表 fragment 数（实测 348-436KB/个）。dream
+   * 的 3a/3b/auto-gc 全部逐条 update，一轮 --auto 写出 ~6600 个 commit ≈ 2GB+ 的
+   * _versions 增量。本方法把"一批相关行的 metadata 变更"折成单次 commit。
+   *
+   * 语义要点：
+   * - patchFn 在 **锁内、以库里最新行为底座** 应用 —— 不接受调用方预先物化的完整
+   *   metadata 串。这是互审 R1 的 P0 修正：3b 持有的是 3a 之前 gather 的旧条目，
+   *   预物化串批量写会把 3a 刚提交的 version_group/sourceMemories 再抹一遍
+   *   （与它要修的 Bug-1 同病，只是从循环内搬到阶段间）。
+   * - patches 按数组序在同一临界区内依次应用：同批的 patchFn 之间可通过调用方
+   *   闭包传递决策（3a 靠它让 canonical 与 members 共享同一个 groupId）。
+   * - 单条 patchFn 抛错或 id 不存在/越权：跳过该条继续（stderr 记一行），不炸整批 ——
+   *   与 dream"单 scope 失败不阻断其他"的分级一致。返回成功写入数。
+   * - 与 patchMetadata 的 per-id 队列并存：本方法自身持 store-write 锁做批内原子，
+   *   跨进程窗口由锁消除；进程内与单条 patchMetadata 的交错等同于现状并发水平。
+   */
+  async patchMetadataBatch(
+    patches: Array<{
+      id: string;
+      patchFn: (meta: Record<string, unknown>, entry: MemoryEntry) => Record<string, unknown>;
+    }>,
+    scopeFilter?: string[],
+  ): Promise<number> {
+    await this.ensureInitialized();
+    if (patches.length === 0) return 0;
+
+    let written = 0;
+    // 分批：避免 OR 链过长（抄 getVectors 的范式，批更保守——每批一个 commit 本身就是收益）
+    const BATCH = 200;
+    for (let i = 0; i < patches.length; i += BATCH) {
+      const batch = patches.slice(i, i + BATCH);
+      written += await withWriteLock("store-write", async () => {
+        // 锁内读最新行
+        const conditions = batch.map(p => `id = '${escapeSqlLiteral(p.id)}'`).join(" OR ");
+        const rows = await this.table!.query().where(conditions).limit(batch.length).toArray();
+        const rowById = new Map<string, Record<string, unknown>>();
+        for (const row of rows) rowById.set(row.id as string, row as Record<string, unknown>);
+
+        const toWrite: MemoryEntry[] = [];
+        for (const { id, patchFn } of batch) {
+          const row = rowById.get(id);
+          if (!row) continue; // 行不存在：跳过（dream 处理途中被并发删除属正常）
+          const rowScope = (row.scope as string | undefined) ?? "";
+          if (!matchesScopeFilter(rowScope, scopeFilter)) continue;
+          let meta: Record<string, unknown>;
+          try {
+            const parsed: unknown = JSON.parse((row.metadata as string) || "{}");
+            meta = parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
+              ? (parsed as Record<string, unknown>)
+              : {};
+          } catch {
+            meta = {};
+          }
+          const entry: MemoryEntry = {
+            id: row.id as string,
+            text: row.text as string,
+            // null 向量防护同 update()：历史病行 → []，metadata-only 批量写不被绊倒
+            vector: row.vector ? Array.from(row.vector as Iterable<number>) : [],
+            category: row.category as MemoryEntry["category"],
+            scope: rowScope,
+            importance: Number(row.importance),
+            timestamp: Number(row.timestamp),
+            metadata: (row.metadata as string) || "{}",
+            language: (row.language as string) || "en",
+            fts_text: (row.fts_text as string) || (row.text as string),
+          };
+          try {
+            const patched = patchFn(meta, entry);
+            toWrite.push({ ...entry, metadata: JSON.stringify(patched) });
+          } catch (err) {
+            console.error(`[patchMetadataBatch] patchFn failed for ${id}, skipping: ${err instanceof Error ? err.message : String(err)}`);
+          }
+        }
+        if (toWrite.length === 0) return 0;
+        await this.table!.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(toWrite);
+        return toWrite.length;
+      }, { expireMs: 30_000 });
+    }
+    return written;
+  }
+
+  /**
+   * Dissolve version groups that have only one surviving member.
+   *
+   * The membership scan and metadata rewrite share the global store-write lock.
+   * Keeping both inside one critical section matters: planning during auto-GC and
+   * patching later could otherwise dismantle a group that gained a new member in
+   * between. Hard deletes that bypass the store may still create a new singleton
+   * after this pass; the next scheduled pass repairs it.
+   */
+  async repairSingletonVersionGroups(): Promise<number> {
+    await this.ensureInitialized();
+
+    return withWriteLock("store-write", async () => {
+      const membershipRows = await this.table!.query()
+        .select(["id", "metadata"])
+        .toArray();
+      const repairs = planSingletonVersionGroupRepairs(
+        membershipRows.map(row => ({
+          id: row.id as string,
+          metadata: (row.metadata as string) || "{}",
+        })),
+      );
+      if (repairs.length === 0) return 0;
+
+      let repaired = 0;
+      const BATCH = 200;
+      for (let i = 0; i < repairs.length; i += BATCH) {
+        const batch = repairs.slice(i, i + BATCH);
+        const expectedById = new Map(batch.map(item => [item.id, item.groupId]));
+        const conditions = batch
+          .map(item => `id = '${escapeSqlLiteral(item.id)}'`)
+          .join(" OR ");
+        const rows = await this.table!.query()
+          .where(conditions)
+          .limit(batch.length)
+          .toArray();
+
+        const toWrite: MemoryEntry[] = [];
+        for (const row of rows) {
+          const id = row.id as string;
+          const expectedGroupId = expectedById.get(id);
+          if (!expectedGroupId) continue;
+          let meta: Record<string, unknown>;
+          try {
+            const parsed: unknown = JSON.parse((row.metadata as string) || "{}");
+            meta = parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
+              ? (parsed as Record<string, unknown>)
+              : {};
+          } catch {
+            continue;
+          }
+          // The group id is rechecked against the locked, current row.
+          if (meta.version_group !== expectedGroupId) continue;
+          clearVersionGroupMetadata(meta, expectedGroupId);
+          toWrite.push({
+            id,
+            text: row.text as string,
+            vector: row.vector ? Array.from(row.vector as Iterable<number>) : [],
+            category: row.category as MemoryEntry["category"],
+            scope: (row.scope as string | undefined) ?? "",
+            importance: Number(row.importance),
+            timestamp: Number(row.timestamp),
+            metadata: JSON.stringify(meta),
+            language: (row.language as string) || "en",
+            fts_text: (row.fts_text as string) || (row.text as string),
+          });
+        }
+        if (toWrite.length === 0) continue;
+        // Matched rows only: a concurrent hard delete must never be resurrected.
+        const result = await this.table!.mergeInsert("id").whenMatchedUpdateAll().execute(toWrite);
+        repaired += result.numUpdatedRows;
+      }
+      return repaired;
+    }, { expireMs: 30 * 60_000 });
+  }
+
   async bulkDelete(scopeFilter: string[], beforeTimestamp?: number): Promise<number> {
     await this.ensureInitialized();
 
     const conditions: string[] = [];
 
     if (scopeFilter.length > 0) {
-      const scopeConditions = scopeFilter
-        .map(scope => {
-          const safe = escapeSqlLiteral(scope);
-          return scope.includes(":")
-            ? `scope = '${safe}'`
-            : `scope LIKE '${safe}%'`;
-        })
-        .join(" OR ");
-      conditions.push(`(${scopeConditions})`);
+      // family 模式（历史行为）。⚠️ 这是**破坏性**操作走前缀：`bulkDelete(["memory"])`
+      // 会连 `memory:pivot` 一起删。当前无生产 caller；要加 caller 请先显式传 exact
+      // （同 forgetByScope 的处理，2026-08-16）。
+      conditions.push(`(${scopeWhereClause(scopeFilter)})`);
     }
 
     if (beforeTimestamp) {