recallnest 2.2.1 → 2.5.4

package/src/source-heartbeat.ts

@@ -0,0 +1,108 @@
+/**
+ * GB-3: Source Heartbeat Tracking
+ *
+ * Every connector ingest writes a heartbeat record to data/source-heartbeat.json.
+ * data_checkup and doctor --ci read heartbeats to detect stale data sources.
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { metaDir } from "./compat.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface SourceHeartbeat {
+  source: string;
+  lastIngest: string;       // ISO 8601
+  recordsIngested: number;
+  errors: string[];
+}
+
+export type HeartbeatFile = Record<string, SourceHeartbeat>;
+
+export interface StaleSource {
+  source: string;
+  lastIngest: string;
+  daysSince: number;
+}
+
+// ---------------------------------------------------------------------------
+// Path
+// ---------------------------------------------------------------------------
+
+const HEARTBEAT_PATH = resolve(metaDir(import.meta), "../data/source-heartbeat.json");
+
+/** Visible for testing — override the default path. */
+export function heartbeatPath(): string {
+  return HEARTBEAT_PATH;
+}
+
+// ---------------------------------------------------------------------------
+// Read / Write
+// ---------------------------------------------------------------------------
+
+export function readHeartbeats(path?: string): HeartbeatFile {
+  const p = path ?? HEARTBEAT_PATH;
+  if (!existsSync(p)) return {};
+  try {
+    return JSON.parse(readFileSync(p, "utf-8")) as HeartbeatFile;
+  } catch {
+    return {};
+  }
+}
+
+export function writeHeartbeat(
+  source: string,
+  recordsIngested: number,
+  errors: string[],
+  path?: string,
+): void {
+  const p = path ?? HEARTBEAT_PATH;
+  const dir = dirname(p);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+  const existing = readHeartbeats(p);
+  existing[source] = {
+    source,
+    lastIngest: new Date().toISOString(),
+    recordsIngested,
+    errors: errors.slice(0, 10), // cap stored errors
+  };
+  writeFileSync(p, JSON.stringify(existing, null, 2) + "\n");
+}
+
+// ---------------------------------------------------------------------------
+// Staleness check
+// ---------------------------------------------------------------------------
+
+const MS_PER_DAY = 86_400_000;
+
+export function checkSourceStaleness(maxDays = 7, path?: string): StaleSource[] {
+  const heartbeats = readHeartbeats(path);
+  const now = Date.now();
+  const stale: StaleSource[] = [];
+
+  for (const hb of Object.values(heartbeats)) {
+    const lastMs = Date.parse(hb.lastIngest);
+    if (Number.isNaN(lastMs)) continue;
+    const daysSince = Math.floor((now - lastMs) / MS_PER_DAY);
+    if (daysSince > maxDays) {
+      stale.push({ source: hb.source, lastIngest: hb.lastIngest, daysSince });
+    }
+  }
+
+  return stale.sort((a, b) => b.daysSince - a.daysSince);
+}
+
+/** Human-readable age string, e.g. "2h ago", "3d ago". */
+export function formatAge(isoDate: string): string {
+  const ms = Date.now() - Date.parse(isoDate);
+  if (ms < 0) return "just now";
+  const hours = Math.floor(ms / 3_600_000);
+  if (hours < 1) return "just now";
+  if (hours < 24) return `${hours}h ago`;
+  const days = Math.floor(hours / 24);
+  return `${days}d ago`;
+}

package/src/store.ts

@@ -5,11 +5,14 @@
 import type * as LanceDB from "@lancedb/lancedb";
 import { createHash, randomUUID } from "node:crypto";
 import { existsSync, accessSync, constants, mkdirSync, realpathSync, lstatSync } from "node:fs";
-import { dirname } from "node:path";
+import { dirname, join } from "node:path";
 import { logWarn } from "./stderr-log.js";
 import type { DurableMemoryCategory } from "./memory-schema.js";
 import { matchesScopeFilter } from "./scope-policy.js";
 import { detectEmotionIfEnabled } from "./emotion-detector.js";
+import { withWriteLock } from "./distill-lock.js";
+import { incrementWriteCount } from "./activity-counter.js";
+import type { MemoryStorePort, MemoryStoreStats, MemoryStoreUpdate } from "./memory-store-port.js";
 
 // ============================================================================
 // Types
@@ -211,11 +214,13 @@ export function validateStoragePath(dbPath: string): string {
 
 const TABLE_NAME = "memories";
 
-export class MemoryStore {
+export class MemoryStore implements MemoryStorePort {
   private db: LanceDB.Connection | null = null;
   private table: LanceDB.Table | null = null;
   private initPromise: Promise<void> | null = null;
   private ftsIndexCreated = false;
+  /** Per-id serial queues for patchMetadata; entries self-remove on settle. */
+  private readonly metadataPatchQueues = new Map<string, Promise<void>>();
 
   constructor(private readonly config: StoreConfig) {}
 
@@ -375,7 +380,10 @@ export class MemoryStore {
     }
 
     try {
-      await this.table!.add([fullEntry]);
+      // P0-2: go through upsert (delete+add under the global store-write lock) rather
+      // than a bare table.add — with the deterministic id above, storing the same
+      // (scope,text) twice now yields one row, not a dup-id pair (fixes the append bug).
+      return await this.upsert(fullEntry);
     } catch (err: any) {
       const code = err.code || "";
       const message = err.message || String(err);
@@ -383,20 +391,21 @@ export class MemoryStore {
         `Failed to store memory in "${this.config.dbPath}": ${code} ${message}`
       );
     }
-    return fullEntry;
   }
 
   /**
    * Batch store multiple entries at once — much faster than individual store() calls.
    * LanceDB handles bulk inserts efficiently with a single index update.
    */
-  async storeBatch(entries: Omit<MemoryEntry, "id" | "timestamp">[]): Promise<number> {
+  async storeBatch(entries: (Omit<MemoryEntry, "id" | "timestamp"> & { id?: string })[]): Promise<number> {
     if (entries.length === 0) return 0;
     await this.ensureInitialized();
 
     const fullEntries: MemoryEntry[] = entries.map(entry => ({
       ...entry,
-      id: deterministicId(entry.scope, entry.text),
+      // P0-2: honor a caller-supplied id (consolidation insight/pattern idempotency),
+      // else the deterministic content id — unified with store() (was: always recompute).
+      id: entry.id ?? deterministicId(entry.scope, entry.text),
       timestamp: Date.now(),
       metadata: entry.metadata || "{}",
       language: entry.language || "en",
@@ -413,8 +422,20 @@ export class MemoryStore {
       return e;
     });
 
+    // P2: collapse in-batch duplicate ids (latest-wins) BEFORE writing. Two entries
+    // with the same (scope,text) → same deterministic id; adding both would create a
+    // fresh dup-id pair in one shot (delete-set was deduped but the add was not).
+    const dedupedById = new Map<string, MemoryEntry>();
+    for (const e of enrichedEntries) dedupedById.set(e.id, e); // later occurrence wins
+    const toWrite = [...dedupedById.values()];
+
     try {
-      await this.table!.add(enrichedEntries);
+      await withWriteLock("store-write", async () => {
+        // P0-1/P0-2/P1: atomic idempotent batch write via mergeInsert (no delete+add
+        // window a crash could tear). toWrite is already in-batch deduped by id above,
+        // so the merge key matches ≤1 existing row (given no pre-existing dup-id rows).
+        await this.table!.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(toWrite);
+      }, { expireMs: 30_000 });
     } catch (err: any) {
       const code = err.code || "";
       const message = err.message || String(err);
@@ -422,7 +443,44 @@ export class MemoryStore {
         `Failed to batch store ${entries.length} memories in "${this.config.dbPath}": ${code} ${message}`
       );
     }
-    return fullEntries.length;
+    // Per-scope write counts drive dream activation. Best-effort — never block writes.
+    try {
+      const statsPath = this.activityStatsPath();
+      const countByScope = new Map<string, number>();
+      for (const e of toWrite) countByScope.set(e.scope, (countByScope.get(e.scope) ?? 0) + 1);
+      for (const [scope, n] of countByScope) incrementWriteCount(scope, n, { statsPath });
+    } catch { /* activity tracking is best-effort */ }
+    return toWrite.length;
+  }
+
+  /**
+   * P0-1/P0-2/P1: Idempotent, crash-atomic write of a fully-formed entry, serialized
+   * across processes.
+   *
+   * Uses mergeInsert("id") (same primitive as update()) so the write is atomic — no
+   * delete+add window that a crash between the two steps could tear, losing the row
+   * (P1). The global `store-write` lock still serializes concurrent merges across the
+   * 11 mcp-server processes so two whenNotMatched inserts can't both fire for the same
+   * id (the P0-1 race). "Same id overwrites" gives content-addressed idempotency to
+   * every store()/storeBatch() caller whose id is deterministicId(scope,text).
+   * Precondition: no pre-existing dup-id rows in the table (run the dedup migration
+   * scripts/cleanup-duplicate-ids.ts first) — else the merge key match is ambiguous.
+   */
+  /** Path to this store's activity-counter stats file, beside the LanceDB dir. */
+  private activityStatsPath(): string {
+    return join(dirname(this.config.dbPath), "activity-stats.json");
+  }
+
+  async upsert(entry: MemoryEntry): Promise<MemoryEntry> {
+    await this.ensureInitialized();
+    await withWriteLock("store-write", async () => {
+      await this.table!.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([entry]);
+    }, { expireMs: 30_000 });
+    // Per-scope write count drives dream activation. Best-effort — never block a write.
+    try {
+      incrementWriteCount(entry.scope, 1, { statsPath: this.activityStatsPath() });
+    } catch { /* activity tracking is best-effort */ }
+    return entry;
   }
 
   /**
@@ -617,11 +675,19 @@ export class MemoryStore {
     if (isFullId) {
       candidates = await this.table!.query().where(`id = '${id}'`).limit(1).toArray();
     } else {
-      // Prefix match: fetch candidates and filter in app layer
-      const all = await this.table!.query().select(["id", "scope"]).limit(1000).toArray();
-      candidates = all.filter((r: any) => (r.id as string).startsWith(id));
+      // v2.5.2 (2026-05-27): Use SQL LIKE for prefix lookup instead of app-layer scan
+      // with limit(1000) — the old `.select().limit(1000).filter()` pattern silently
+      // missed entries beyond row 1000 in 90K+ libraries (Codex trio review finding,
+      // ref: ~/Desktop/codex-v2.5.1-fix-review-20260527.md, store.delete limit 漏洞).
+      // Now mirrors getById prefix behavior: LIKE with limit(2) detects ambiguity.
+      const safeId = escapeSqlLiteral(id);
+      candidates = await this.table!.query()
+        .select(["id", "scope"])
+        .where(`id LIKE '${safeId}%'`)
+        .limit(2)
+        .toArray();
       if (candidates.length > 1) {
-        throw new Error(`Ambiguous prefix "${id}" matches ${candidates.length} memories. Use a longer prefix or full ID.`);
+        throw new Error(`Ambiguous prefix "${id}" matches ${candidates.length}+ memories. Use a longer prefix or full ID.`);
       }
     }
     if (candidates.length === 0) {
@@ -688,6 +754,72 @@ export class MemoryStore {
       .slice(offset, offset + limit);
   }
 
+  /**
+   * True DB-level pagination — where/limit/offset are pushed down to LanceDB,
+   * so unlike list() this never materializes the full match set in JS. Use for
+   * full-corpus maintenance sweeps (auto-gc, promotion scan, lint/checkup).
+   *
+   * Caveats:
+   * - No orderBy support in the LanceDB query API: rows come back in fragment
+   *   scan order, NOT timestamp order. Stable across pages for a read-only
+   *   sweep, but concurrent writes rewrite fragments and may shift rows
+   *   between pages (a missed row is picked up by the next sweep). Do not use
+   *   for user-facing ordered listings — that's what list() is for.
+   * - includeVector loads the embedding column; only request it when the
+   *   consumer actually needs vectors (they dominate row size).
+   */
+  async listPage(opts: {
+    scopeFilter?: string[];
+    category?: string;
+    limit?: number;
+    offset?: number;
+    includeVector?: boolean;
+  } = {}): Promise<MemoryEntry[]> {
+    await this.ensureInitialized();
+    const { scopeFilter, category, limit = 1000, offset = 0, includeVector = false } = opts;
+
+    let query = this.table!.query();
+
+    const conditions: string[] = [];
+    if (scopeFilter && scopeFilter.length > 0) {
+      const scopeConditions = scopeFilter
+        .map(scope => {
+          const safe = escapeSqlLiteral(scope);
+          return scope.includes(":")
+            ? `scope = '${safe}'`
+            : `scope LIKE '${safe}%'`;
+        })
+        .join(" OR ");
+      conditions.push(`((${scopeConditions}))`);
+    }
+    if (category) {
+      conditions.push(`category = '${escapeSqlLiteral(category)}'`);
+    }
+    if (conditions.length > 0) {
+      query = query.where(conditions.join(" AND "));
+    }
+
+    const columns = ["id", "text", "category", "scope", "importance", "timestamp", "metadata"];
+    if (includeVector) columns.push("vector");
+
+    const results = await query
+      .select(columns)
+      .limit(limit)
+      .offset(offset)
+      .toArray();
+
+    return results.map((row): MemoryEntry => ({
+      id: row.id as string,
+      text: row.text as string,
+      vector: includeVector ? Array.from(row.vector as Iterable<number>) : [],
+      category: row.category as MemoryEntry["category"],
+      scope: (row.scope as string | undefined) ?? "",
+      importance: Number(row.importance),
+      timestamp: Number(row.timestamp),
+      metadata: (row.metadata as string) || "{}",
+    }));
+  }
+
   async get(id: string, scopeFilter?: string[]): Promise<MemoryEntry | null> {
     await this.ensureInitialized();
 
@@ -741,18 +873,35 @@ export class MemoryStore {
   }
 
   /**
-   * Get a single entry by exact ID without scope filtering.
-   * Lightweight read used by AccessTracker for metadata updates.
+   * Get a single entry by ID (full UUID) or 8+ hex prefix.
+   * Lightweight read used by AccessTracker for metadata updates and by skill-engine
+   * for outcome recording.
+   *
+   * v2.5.1 (2026-05-27) — 加 prefix lookup support：和 update() / delete() 行为对齐，
+   * 让 agent 拿到 short id（如 store_skill 返回的 8 位前缀、人类粘贴的截断 id）也能
+   * resolve 到完整 entry。歧义 prefix（匹配 >1 条）返回 null 让 caller 处理。
+   * 现有 caller 全部传完整 UUID，向后兼容。
    */
   async getById(id: string): Promise<MemoryEntry | null> {
     await this.ensureInitialized();
+
+    // Support both full UUID and 8+ hex prefix (same pattern as update() / delete()).
+    const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+    const prefixRegex = /^[0-9a-f]{8,}$/i;
+    const isFullId = uuidRegex.test(id);
+    const isPrefix = !isFullId && prefixRegex.test(id);
+    if (!isFullId && !isPrefix) return null;
+
     const safeId = escapeSqlLiteral(id);
+    const whereClause = isFullId ? `id = '${safeId}'` : `id LIKE '${safeId}%'`;
+
     const rows = await this.table!.query()
       .select(["id", "text", "vector", "category", "scope", "importance", "timestamp", "metadata"])
-      .where(`id = '${safeId}'`)
-      .limit(1)
+      .where(whereClause)
+      .limit(2) // detect ambiguous prefix (matches >1)
       .toArray();
     if (rows.length === 0) return null;
+    if (isPrefix && rows.length > 1) return null; // ambiguous prefix
     const row = rows[0];
     return {
       id: row.id as string,
@@ -773,28 +922,28 @@ export class MemoryStore {
    */
   async getVectors(ids: string[]): Promise<Map<string, number[]>> {
     await this.ensureInitialized();
-    if (ids.length === 0) return new Map();
-
-    const conditions = ids.map(id => `id = '${escapeSqlLiteral(id)}'`).join(" OR ");
-    const rows = await this.table!.query()
-      .select(["id", "vector"])
-      .where(conditions)
-      .limit(ids.length)
-      .toArray();
-
     const result = new Map<string, number[]>();
-    for (const row of rows) {
-      const vec = Array.from(row.vector as Iterable<number>);
-      if (vec.length > 0) result.set(row.id as string, vec);
+    if (ids.length === 0) return result;
+
+    // 分批:避免 id 很多时 OR 链过长（如 data-checkup/memory-lint 一次补几千条向量）。
+    const BATCH = 500;
+    for (let i = 0; i < ids.length; i += BATCH) {
+      const batch = ids.slice(i, i + BATCH);
+      const conditions = batch.map(id => `id = '${escapeSqlLiteral(id)}'`).join(" OR ");
+      const rows = await this.table!.query()
+        .select(["id", "vector"])
+        .where(conditions)
+        .limit(batch.length)
+        .toArray();
+      for (const row of rows) {
+        const vec = Array.from(row.vector as Iterable<number>);
+        if (vec.length > 0) result.set(row.id as string, vec);
+      }
     }
     return result;
   }
 
-  async stats(scopeFilter?: string[]): Promise<{
-    totalCount: number;
-    scopeCounts: Record<string, number>;
-    categoryCounts: Record<string, number>
-  }> {
+  async stats(scopeFilter?: string[]): Promise<MemoryStoreStats> {
     await this.ensureInitialized();
 
     let query = this.table!.query();
@@ -874,7 +1023,7 @@ export class MemoryStore {
 
   async update(
     id: string,
-    updates: { text?: string; vector?: number[]; importance?: number; category?: MemoryEntry["category"]; metadata?: string; timestamp?: number; language?: string; fts_text?: string },
+    updates: MemoryStoreUpdate,
     scopeFilter?: string[]
   ): Promise<MemoryEntry | null> {
     await this.ensureInitialized();
@@ -936,14 +1085,94 @@ export class MemoryStore {
       }
     }
 
-    // LanceDB doesn't support in-place update; delete + re-add
-    const resolvedId = escapeSqlLiteral(row.id as string);
-    await this.table!.delete(`id = '${resolvedId}'`);
-    await this.table!.add([updated]);
+    // Atomic upsert via mergeInsert — replaces the previous delete+add two-step,
+    // which could lose the row entirely on a crash between steps and exposed a
+    // window where concurrent reads saw the id momentarily vanish.
+    await this.table!
+      .mergeInsert("id")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([updated]);
 
     return updated;
   }
 
+  /**
+   * Serialized read-modify-write for the metadata column.
+   *
+   * Every direct `getById → mutate metadata → update` round-trip can lose
+   * concurrent increments (last write wins). This method funnels metadata
+   * patches for the same id through a per-id promise queue so each patch
+   * reads the result of the previous one.
+   *
+   * patchFn receives the parsed metadata object plus the freshly-read entry
+   * (for callers that need importance/timestamp context) and returns the new
+   * metadata object. A throwing patchFn abandons that patch — the error
+   * propagates to the caller, later queued patches still run.
+   *
+   * Scope of guarantee: in-process only. Cross-process writers (multiple
+   * mcp-server instances) and the remaining direct update() callers
+   * (consolidation/capture/forget/prospective/conflict engines, auto-gc)
+   * are NOT serialized by this queue. Long-term direction: make this the
+   * only legal metadata write path.
+   */
+  async patchMetadata(
+    id: string,
+    patchFn: (meta: Record<string, unknown>, entry: MemoryEntry) => Record<string, unknown>,
+    scopeFilter?: string[],
+  ): Promise<MemoryEntry | null> {
+    await this.ensureInitialized();
+
+    // Queue key must be the resolved full UUID — otherwise a prefix caller and
+    // a full-id caller targeting the same row would race on separate queues.
+    const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+    let fullId: string;
+    if (uuidRegex.test(id)) {
+      fullId = id;
+    } else {
+      const resolved = await this.getById(id);
+      if (!resolved) return null;
+      fullId = resolved.id;
+    }
+
+    const run = async (): Promise<MemoryEntry | null> => {
+      const entry = await this.getById(fullId);
+      if (!entry) return null;
+      let meta: Record<string, unknown>;
+      try {
+        const parsed: unknown = JSON.parse(entry.metadata || "{}");
+        meta = parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
+          ? (parsed as Record<string, unknown>)
+          : {};
+      } catch {
+        meta = {};
+      }
+      const patched = patchFn(meta, entry);
+      return this.update(fullId, { metadata: JSON.stringify(patched) }, scopeFilter);
+    };
+
+    const prev = this.metadataPatchQueues.get(fullId) ?? Promise.resolve();
+    // Isolate prior failures so one rejected patch can't wedge the queue.
+    const result = prev.then(run, run);
+    // The stored tail settles regardless of outcome; the caller keeps the real
+    // rejection via `result`. Only the current tail removes itself.
+    const tail = result.then(
+      () => undefined,
+      () => undefined,
+    ).then(() => {
+      if (this.metadataPatchQueues.get(fullId) === tail) {
+        this.metadataPatchQueues.delete(fullId);
+      }
+    });
+    this.metadataPatchQueues.set(fullId, tail);
+    return result;
+  }
+
+  /** Number of ids with in-flight metadata patches (test/diagnostic hook). */
+  get pendingMetadataPatchCount(): number {
+    return this.metadataPatchQueues.size;
+  }
+
   async bulkDelete(scopeFilter: string[], beforeTimestamp?: number): Promise<number> {
     await this.ensureInitialized();
 

package/src/tool-output-compressor.ts

@@ -59,7 +59,9 @@ const RULES: CompressionRule[] = [
 
 // Patterns that identify tool output blocks in conversation text.
 // Claude Code / Codex format: command on one line, output follows.
-const TOOL_OUTPUT_BLOCK = /^(?:\$\s+|❯\s+|>\s+)(.+)\n([\s\S]*?)(?=\n(?:\$\s+|❯\s+|>\s+)|\n\n[A-Z]|\n##|\z)/gm;
+// 末项用 (?![\s\S]) 表示「字符串绝对结尾」。不能用 \z（JS 无此锚点，会被当字面字符 z →
+// 在输出里第一个 z 处误截断），也不能用 $（gm 模式下匹配每行尾，惰性 [\s\S]*? 会在第一行就停）。
+const TOOL_OUTPUT_BLOCK = /^(?:\$\s+|❯\s+|>\s+)(.+)\n([\s\S]*?)(?=\n(?:\$\s+|❯\s+|>\s+)|\n\n[A-Z]|\n##|(?![\s\S]))/gm;
 
 // Base64 image data (screenshots, etc.)
 const BASE64_BLOCK = /(?:data:image\/[^;]+;base64,)?[A-Za-z0-9+/]{500,}={0,2}/g;

package/src/ui-server.ts

@@ -12,6 +12,7 @@ import { archiveDirtyBriefAsset, assetSummaryLine, buildBriefAsset, buildPinAsse
 import { indexAsset, indexPinnedAsset } from "./asset-sync.js";
 import { createComponentResolver, loadConfig, loadDotEnv } from "./runtime-config.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
+import * as envConfig from "./env-config.js";
 const config = (loadDotEnv(), loadConfig());
 const getComponents = createComponentResolver(config);
 
@@ -186,7 +187,7 @@ async function handleSearch(mode: "search" | "explain" | "distill", body: Record
 }
 
 const server = Bun.serve({
-  port: readLimit(process.env.RECALLNEST_UI_PORT, 4317, 1, 65535),
+  port: readLimit(envConfig.uiPortRaw(), 4317, 1, 65535),
   async fetch(request) {
     try {
       const url = new URL(request.url);