recallnest 3.0.0 → 4.0.0

package/docs/api-reference.md

@@ -225,6 +225,14 @@ Store a new durable memory entry.
 
 If the same `canonicalKey` is already occupied by a different durable category, RecallNest returns `disposition = "conflict"` plus a `conflictId` instead of silently creating a second durable owner.
 
+`stored` is `true` (status `201`) only when the write landed new content: `disposition` is `stored`, `updated` or `promoted`. Otherwise the status is `200` and `stored` is `false`:
+
+| `disposition` | Meaning | `id` |
+|---|---|---|
+| `deduped` | The content already exists; no new row was written | the existing memory |
+| `conflict` | Canonical key owned by another category; parked in the conflict store | the existing owner |
+| `rejected` | Turned away by admission control; `rejectionReason` says why (`text_too_short`, `noise_detected`, `importance_too_low`, `rate_limited`) | a placeholder, no row exists |
+
 For slot-aware preferences, RecallNest now derives structured keys instead of collapsing everything into a broader same-topic key. Examples:
 
 - atomic brand-item preference: `我喜欢吃麦当劳的麦辣鸡翅` -> `preferences:brand-item:麦当劳:麦辣鸡翅`

package/docs/architecture.md

@@ -47,9 +47,12 @@ See [memory-boundary-contract.md](./memory-boundary-contract.md).
 │                                                              │
 │  ┌─────────────┐  ┌──────────────┐  ┌────────────────────┐  │
 │  │  Retriever   │  │  Classifier  │  │  Query Expander    │  │
-│  │ (hybrid:     │  │ (6 categories│  │  (synonym +        │  │
-│  │  vector +    │  │  auto-assign)│  │   semantic expand) │  │
-│  │  BM25 + RRF) │  │              │  │                    │  │
+│  │ (default:    │  │ (6 categories│  │  (synonym +        │  │
+│  │  vector-only;│  │  auto-assign)│  │   semantic expand) │  │
+│  │  optional    │  │              │  │                    │  │
+│  │  hybrid:     │  │              │  │                    │  │
+│  │  weighted    │  │              │  │                    │  │
+│  │  vector+BM25)│  │              │  │                    │  │
 │  └──────┬───────┘  └──────────────┘  └────────────────────┘  │
 │         │                                                    │
 │  ┌──────┴───────┐  ┌──────────────┐  ┌────────────────────┐  │
@@ -116,13 +119,15 @@ Query                                               Results
     │
     ▼
 ┌──────────────┐    ┌──────────────┐    ┌───────────────┐
-│ Query        │───►│ Hybrid       │───►│ Post-process  │──► Top-K
-│ Expander     │    │ Retrieval    │    │               │   Results
-│ (synonyms)   │    │              │    │ - Decay       │
-└──────────────┘    │ Vector: 0.7  │    │ - Access boost│
-                    │ BM25:   0.3  │    │ - Score floor │
-                    │ RRF merge    │    │ - Dedup       │
-                    └──────────────┘    └───────────────┘
+│ Query        │───►│ Retrieval    │───►│ Post-process  │──► Top-K
+│ Expander     │    │ default:     │    │               │   Results
+│ (synonyms)   │    │  vector-only │    │ - Decay       │
+└──────────────┘    │ hybrid mode: │    │ - Access boost│
+                    │  Vector: 0.7 │    │ - Score floor │
+                    │  BM25:   0.3 │    │ - Dedup       │
+                    │  weighted    │    └───────────────┘
+                    │  fusion      │
+                    └──────────────┘
 ```
 
 ## Key Design Decisions
@@ -131,8 +136,8 @@ Query                                               Results
 |----------|-----------|
 | **LanceDB** (not SQLite/Postgres) | Native vector search, columnar storage, zero-config, single-file DB |
 | **Jina v5** (not OpenAI embeddings) | Task-aware embeddings (query vs passage), better multilingual, 1024-dim sweet spot |
-| **Hybrid retrieval** (vector + BM25) | Vector alone misses keyword matches; BM25 alone misses semantic similarity |
-| **RRF fusion** | Reciprocal Rank Fusion is parameter-free and robust across score distributions |
+| **Hybrid retrieval** (vector + BM25, optional) | Vector alone misses keyword matches; BM25 alone misses semantic similarity. Default `retrieval.mode` is `"vector"` (vector-only search; `DEFAULT_RETRIEVAL_CONFIG` in `src/retriever.ts`, `config.json.example`). Set `retrieval.mode` to `"hybrid"` in `config.json` to add BM25 full-text search; if the store has no full-text index, retrieval stays vector-only |
+| **Weighted score fusion** (hybrid mode only) | Applies only when `retrieval.mode` is `"hybrid"` (default is `"vector"`). Hybrid mode combines vector and BM25 scores as a weighted average (defaults `vectorWeight: 0.7`, `bm25Weight: 0.3`), with a 5% bonus when both legs return the same memory; a memory found by only one leg is scored by that leg alone (BM25-only hits get ×1.15 on short queries). For short queries (≤ 4 tokens) the vector weight is multiplied by 0.7 and the BM25 weight by 1.5, capped at 0.6 (`fuseResults()` in `src/retriever.ts`) |
 | **Weibull decay** (not exponential) | Better models human forgetting: slow start, accelerating fade |
 | **6 categories** (not free-form tags) | Structured enough for filtering and lifecycle rules, simple enough to auto-classify |
 | **HTTP API + MCP** (not just MCP) | MCP is great for CLI tools, but HTTP API works with any language/framework |
@@ -146,7 +151,7 @@ src/
 ├── mcp-server.ts          # MCP server (stdio transport)
 ├── cli.ts                 # CLI entry point (lm command)
 ├── store.ts               # LanceDB storage layer
-├── retriever.ts           # Hybrid retrieval (vector + BM25 + RRF)
+├── retriever.ts           # Hybrid retrieval (vector + BM25, weighted score fusion)
 ├── embedder.ts            # Jina embedding client
 ├── ingest.ts              # Multi-source ingestion pipeline
 ├── chunker.ts             # Text chunking with noise filtering

package/eval/cases-canary.json

@@ -4,7 +4,9 @@
     "query": "mem0 对 recallnest 有没有借鉴作用",
     "profile": "fact-check",
     "limit": 8,
-    "targets": ["a2fe9b62"],
+    "targets": [
+      "a2fe9b62"
+    ],
     "expectTopK": 3,
     "notes": "A 单目标：mem0→recallnest 借鉴审计结论该被召回。"
   },
@@ -13,7 +15,9 @@
     "query": "recallnest 本地两个目录反复踩坑是怎么解决的",
     "profile": "debug",
     "limit": 8,
-    "targets": ["dbcb317a"],
+    "targets": [
+      "dbcb317a"
+    ],
     "expectTopK": 3,
     "notes": "A 单目标：dual-dir 符号链接统一 case（dbcb317a, cases, imp 0.9, baseline rank1）。原 target e40ab64b 系标注错——e40ab64b 实为『RecallNest 演进优先 workflow pattern』，与本 query 无关；dbcb317a 才是『~/recallnest→~/Projects/recallnest symlink 统一』。"
   },
@@ -22,8 +26,13 @@
     "query": "recallnest 的 capture 是纯 heuristic 吗，提取到底用不用 LLM",
     "profile": "fact-check",
     "limit": 8,
-    "targets": ["727da608"],
-    "expectOrder": ["727da608", "a2fe9b62"],
+    "targets": [
+      "727da608"
+    ],
+    "expectOrder": [
+      "727da608",
+      "a2fe9b62"
+    ],
     "expectTopK": 3,
     "notes": "B 新旧对：727da608(cases,imp0.8) 正文明确『修正 a2fe9b62 中 capture 纯heuristic/zero-LLM 的不准确表述』，是本 query 准确答案；a2fe9b62(events,imp0.72,mem0借鉴审计) 含被纠正的旧表述。expectOrder 测『两者同时召回时准确版 727 须排在旧版 a2 前』——纯排名对比(rank 唯一，<与≤等价)，不靠结构化 supersede。校准修正：原 notes『验 confidence/recency/decay』机制声称不成立(二者无 evolution 链/降权链，靠内容质量排序)；且主话题不重合(727=ingest去重史 / a2=mem0审计)，实测 a2 常落选(baseline rank=null)，故本 case 当前主要测 727 召回质量+弱新旧序保护，非真正新旧竞争。理想 B 类应换一对同话题、会同时召回的新旧记忆(P1 后续)。"
   },
@@ -32,8 +41,12 @@
     "query": "工作记忆要不要做 scope 隔离",
     "profile": "default",
     "limit": 8,
-    "targets": ["491c8b30"],
-    "hardNegatives": ["b59e86b2"],
+    "targets": [
+      "491c8b30"
+    ],
+    "hardNegatives": [
+      "b59e86b2"
+    ],
     "expectTopK": 2,
     "notes": "C 干扰项：491c8b30(元决策『工作记忆不做 scope 隔离』,baseline rank1) 应召回。hardNegative b59e86b2(entities,imp0.65,scope=codex:019cfba5) 是 codex 端关于 scope 模型/associative access/隔离的设计讨论片段——词面同话题、别处语境(非 Alice 元决策)，baseline rank2 进了 top。补此 hardNeg 后本 case 在当前裸 retrieve 下会 fail，暴露『跨-scope 旁支讨论被召回进 top』的真问题(契合 P0 scope 主题)。"
   },
@@ -42,7 +55,13 @@
     "query": "Alice 的写作风格偏好是什么",
     "profile": "writing",
     "limit": 6,
-    "expectContentAny": ["口语化", "不端着", "剧评腔", "书面非口语", "可自嘲"],
+    "expectContentAny": [
+      "口语化",
+      "不端着",
+      "剧评腔",
+      "书面非口语",
+      "可自嘲"
+    ],
     "expectTopK": 3,
     "notes": "D 稳定偏好：不钉 id，召回文本命中要点即算对。"
   },
@@ -51,8 +70,87 @@
     "query": "Alice 的配图审美偏好",
     "profile": "default",
     "limit": 6,
-    "expectContentAny": ["手绘", "撞色", "高对比", "静物无人物", "涂鸦"],
+    "expectContentAny": [
+      "手绘",
+      "撞色",
+      "高对比",
+      "静物无人物",
+      "涂鸦"
+    ],
     "expectTopK": 3,
     "notes": "D 稳定偏好：同上。"
+  },
+  {
+    "name": "canary-assoc-onboarding-written",
+    "query": "接新的 AI 是不是又要从头讲一遍",
+    "profile": "default",
+    "scope": "memory:pivot",
+    "limit": 8,
+    "targets": [
+      "5a9affa2"
+    ],
+    "expectTopK": 5,
+    "notes": "assoc 对照组：query 就是正文尾巴写下的『她以后会这么问起』原句。09-22 基线 #1。"
+  },
+  {
+    "name": "canary-assoc-onboarding-zero",
+    "query": "新接的那个 app 答应会遵守约定，结果每次开新对话都当没听过",
+    "profile": "default",
+    "scope": "memory:pivot",
+    "limit": 8,
+    "targets": [
+      "5a9affa2"
+    ],
+    "expectTopK": 5,
+    "notes": "assoc 零词面重合：三层规则结构 / 口头承诺不是机制。09-22 基线 #1（path=vector）。"
+  },
+  {
+    "name": "canary-assoc-rebuild-pool",
+    "query": "要不要把老的那十几万条清一清",
+    "profile": "default",
+    "scope": "memory:pivot",
+    "limit": 8,
+    "targets": [
+      "9f908ad2"
+    ],
+    "expectTopK": 5,
+    "notes": "assoc：08-01 从零建池不修补存量、12.9 万条不删。09-22 基线 #1。"
+  },
+  {
+    "name": "canary-assoc-commit-gate",
+    "query": "记忆写进去之前要不要先让人审一下再入库",
+    "profile": "default",
+    "scope": "memory:pivot",
+    "limit": 8,
+    "targets": [
+      "545562b9",
+      "a81ccffb"
+    ],
+    "expectTopK": 5,
+    "notes": "assoc：提案/提交分离早就有 + deepbrain 否决。09-22 基线 #1 / #5。"
+  },
+  {
+    "name": "canary-assoc-benchmark-tmem",
+    "query": "T-Mem 论文里那个榜单我们能不能也上去比比",
+    "profile": "default",
+    "scope": "memory:pivot",
+    "limit": 8,
+    "targets": [
+      "64df5691"
+    ],
+    "expectTopK": 5,
+    "notes": "assoc 已知坏样本①：07-25 公开 benchmark 否决，正文尾巴已带『她以后可能会这样问起』句。09-22 基线 #7（limit 5 漏）。"
+  },
+  {
+    "name": "canary-assoc-benchmark-tencent",
+    "query": "腾讯那篇论文的榜我们要不要也去排个名",
+    "profile": "default",
+    "scope": "memory:pivot",
+    "limit": 8,
+    "targets": [
+      "64df5691"
+    ],
+    "expectTopK": 5,
+    "notes": "assoc 已知坏样本②：同上目标，措辞换成『榜 / 排名』。09-22 基线 top-10 不出现。"
   }
-]
+]

package/integrations/examples/claude-agent-sdk/memory-agent.ts

@@ -113,6 +113,8 @@ async function handleTool(
       }),
     });
     if (!res.ok) return `Error: ${res.status} ${await res.text()}`;
+    const data = await res.json() as { stored: boolean; disposition: string; id: string; rejectionReason?: string };
+    if (!data.stored) return `Memory not stored (${data.disposition}: ${data.rejectionReason ?? data.id}).`;
     return "Memory stored successfully.";
   }
 

package/integrations/examples/langchain/memory-chain.py

@@ -78,6 +78,10 @@ def store_memory(text: str, category: str = "events") -> str:
         },
     )
     r.raise_for_status()
+    data = r.json()
+    if not data.get("stored"):
+        reason = data.get("rejectionReason") or data.get("id")
+        return f"Memory not stored ({data.get('disposition')}: {reason})."
     return "Memory stored successfully."
 
 

package/integrations/examples/openai-agents-sdk/memory-agent.py

@@ -79,6 +79,10 @@ async def store_memory(text: str, category: str = "events") -> str:
             },
         )
         r.raise_for_status()
+        data = r.json()
+        if not data.get("stored"):
+            reason = data.get("rejectionReason") or data.get("id")
+            return f"Memory not stored ({data.get('disposition')}: {reason})."
         return "Memory stored successfully."
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recallnest",
-  "version": "3.0.0",
+  "version": "4.0.0",
   "description": "RecallNest — MCP-native, local-first memory search and distillation for AI conversations",
   "license": "MIT",
   "type": "module",
@@ -22,7 +22,8 @@
     "eval/continuity/*.json",
     "integrations/",
     "README_CN.md",
-    "src/"
+    "src/",
+    "!src/**/__tests__/**"
   ],
   "scripts": {
     "search": "bun run src/cli.ts search",
@@ -45,6 +46,7 @@
     "eval": "bun run src/eval.ts",
     "eval:continuity": "bun run src/eval.ts --mode continuity --cases eval/continuity/cases.json",
     "eval:canary": "bun run src/eval.ts --mode canary --cases eval/cases-canary.json",
+    "eval:known-item": "bun run eval/known-item.ts run",
     "profiles": "bun run src/cli.ts profiles",
     "doctor": "bun run src/cli.ts doctor",
     "demo": "bun run src/cli.ts demo",
@@ -60,10 +62,17 @@
   "engines": {
     "node": ">=22"
   },
+  "overrides": {
+    "fast-uri": "^3.1.8",
+    "qs": "^6.16.0",
+    "hono": "^4.13.7",
+    "ip-address": "^10.7.1",
+    "proxy-addr": "^2.0.8"
+  },
   "dependencies": {
     "@hono/node-server": "1.19.17",
     "@lancedb/lancedb": "^0.26.0",
-    "@modelcontextprotocol/sdk": "^1.30.0",
+    "@modelcontextprotocol/sdk": "^1.31.0",
     "commander": "^12.0.0",
     "openai": "^7.5.0",
     "tsx": "^4.19.0",

package/src/activity-counter.ts

@@ -104,6 +104,7 @@ const ACTIVITY_LOCK_KEY = "activity-stats";
 const ACTIVITY_LOCK_OPTS = { onBusy: "wait" as const, waitTimeoutMs: 2_000, expireMs: 30_000 };
 
 async function withStatsLock<T>(fn: () => T): Promise<T> {
+  // lock-wait: no store-write inside — fn only reads and rewrites the activity stats file.
   const outcome = await withLock(ACTIVITY_LOCK_KEY, async () => fn(), ACTIVITY_LOCK_OPTS);
   // onBusy:"wait" resolves ran:true or throws on timeout; ran:false unreachable.
   if (!outcome.ran) throw new Error("activity-stats lock unavailable");

package/src/adaptive-retrieval.ts

@@ -74,3 +74,26 @@ export function shouldSkipRetrieval(query: string, minLength?: number): boolean
 
   return false;
 }
+
+/**
+ * 检索入口用的判定：这条检索要不要直接返回空。
+ *
+ * 上面那套规则是「一条聊天消息值不值得自动检索」的判定（省嵌入调用），只该管自动召回。
+ * 人主动发起的查询（MCP search_memory、CLI、HTTP、本地 UI）以 `git` / `docker` / `test` 开头、
+ * 纯数字或 `#22`、不含中文又短于 15 字的短语，都是正常的查法；被闸掉时返回的空结果和
+ * 「库里确实没有」看不出区别。所以：
+ * - `trivialQueryGate` 显式给了就按它；
+ * - 没给时只在 `source === "auto-recall"` 时开闸——`source` 同时管着三处强化写入，
+ *   有些人主动发起的检索（search_memory 的相关 scope 旁路、known-item eval）为了不写库也标成
+ *   auto-recall，它们用 `trivialQueryGate: false` 把闸关掉；
+ * - 空串与纯空白不管哪条路都跳过。
+ */
+export function shouldSkipRetrievalFor(context: {
+  query: string;
+  source?: "manual" | "auto-recall" | "cli";
+  trivialQueryGate?: boolean;
+}): boolean {
+  if (context.query.trim().length === 0) return true;
+  const gate = context.trivialQueryGate ?? context.source === "auto-recall";
+  return gate && shouldSkipRetrieval(context.query);
+}

package/src/api-server.ts

@@ -7,10 +7,13 @@
  */
 
 import { createComponentResolver, loadConfig, loadDotEnv } from "./runtime-config.js";
+import { HTTP_WRITE_IDLE_TIMEOUT_S } from "./store-write-budget.js";
 import { DurableMemoryCategorySchema } from "./memory-schema.js";
 import { persistCaseMemory, persistMemory, persistMemoryBatch, persistWorkflowPattern, promoteMemory } from "./capture-engine.js";
 import { buildSessionCheckpointResult } from "./session-engine.js";
+import { buildStoreHttpResponse } from "./store-receipts.js";
 import { SessionCheckpointStore } from "./session-store.js";
+import { mirrorCheckpointWithDeadline } from "./checkpoint-mirror.js";
 import { composeResumeContext } from "./context-composer.js";
 import { extractMemoryProvenance } from "./memory-boundaries.js";
 import { ConflictStatusSchema } from "./conflict-schema.js";
@@ -242,17 +245,8 @@ async function handleStore(request: Request): Promise<Response> {
   try {
     const { store, embedder } = getComponents();
     const stored = await persistMemory({ store, embedder, conflictStore, auditLogger }, body);
-
-    return jsonResponse({
-      id: stored.id,
-      stored: stored.disposition !== "conflict",
-      disposition: stored.disposition,
-      storedAt: stored.storedAt,
-      category: stored.category,
-      scope: stored.resolvedScope,
-      canonicalKey: stored.canonicalKey,
-      conflictId: stored.conflictId,
-    }, 201);
+    const response = buildStoreHttpResponse(stored);
+    return jsonResponse(response.body, response.status);
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
     return errorResponse(400, message);
@@ -454,6 +448,11 @@ async function handleCheckpoint(request: Request): Promise<Response> {
       ...result,
       record: stored,
     }));
+    // Mirror into the memory table so search can find it; never throws, capped wait.
+    await mirrorCheckpointWithDeadline(() => {
+      const { store, embedder } = getComponents();
+      return { store, embedder };
+    }, stored);
     return jsonResponse(stored, 201);
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
@@ -651,6 +650,9 @@ const port = clampInt(envConfig.apiPortRaw(), 4318, 1, 65535);
 
 const server = Bun.serve({
   port,
+  // Writes can wait up to 30 s for store-write before they start; Bun's 10 s default would drop
+  // the connection while the handler still commits.
+  idleTimeout: HTTP_WRITE_IDLE_TIMEOUT_S,
   hostname: LOCAL_HTTP_HOSTNAME,
   async fetch(request) {
     const policy = await enforceLocalHttpRequestPolicy(request);

package/src/asset-sync.ts

@@ -1,6 +1,7 @@
 import type { Embedder } from "./embedder.js";
-import type { MemoryStore } from "./store.js";
-import type { BriefAsset, MemoryAsset, PinAsset } from "./memory-assets.js";
+import type { MemoryEntry, MemoryStore } from "./store.js";
+import { buildPinAsset, savePinAsset, type BriefAsset, type MemoryAsset, type PinAsset } from "./memory-assets.js";
+import type { RetrievalProfileName } from "./retrieval-profiles.js";
 
 function buildPinnedAssetText(asset: PinAsset): string {
   const tags = asset.tags.join(", ");
@@ -77,3 +78,26 @@ export async function indexAsset(
 }
 
 export const indexPinnedAsset = indexAsset;
+
+/**
+ * pin_memory（MCP 工具、CLI `pin`、本地 UI `/api/pin` 共用）：先把这条记忆的 importance 提到至少 0.95，
+ * update() 返回 null（行在查到之后被删掉了）就返回 null——不写 pin 文件、不建索引副本，由调用方报 not found；
+ * 写成功才用 update() 返回的那一行生成 pin。2026-10-17 起 update() 不再把被删的行插回来，
+ * 原先这里不看返回值，会给一条已经不存在的记忆留下 pin 文件和索引副本。
+ */
+export async function pinMemory(
+  store: MemoryStore,
+  embedder: Embedder,
+  entry: MemoryEntry,
+  scopeFilter: string[] | undefined,
+  options: { title?: string; summary?: string; query?: string; profile?: RetrievalProfileName },
+  save: (asset: PinAsset) => string = savePinAsset,
+): Promise<{ asset: PinAsset; path: string; entry: MemoryEntry } | null> {
+  const updated = await store.update(entry.id, { importance: Math.max(entry.importance || 0.7, 0.95) }, scopeFilter);
+  if (!updated) return null;
+  const score = updated.importance || 0.7;
+  const asset = buildPinAsset({ entry: updated, score, sources: { fused: { score } } }, options);
+  const path = save(asset);
+  await indexAsset(store, embedder, asset);
+  return { asset, path, entry: updated };
+}

package/src/audit-log.ts

@@ -18,6 +18,8 @@ export type AuditOperation =
   | "supersede"
   | "consolidate"
   | "forget"
+  /** A forget that stopped after changing something (see forget-engine.ts). Not a `forget`: the memory may still be there. */
+  | "forget_incomplete"
   | "cascade_forget"
   | "reject";
 

package/src/auto-gc.ts

@@ -22,7 +22,7 @@ import type { RetentionPolicy } from "./retention-policy.js";
 import type { AuditLogger } from "./audit-log.js";
 import * as envConfig from "./env-config.js";
 import { isUsageSignalActive } from "./usage-tracker.js";
-import { acquireLock, releaseLock, getLastDistillTime, stampLock, lockPathForKey } from "./distill-lock.js";
+import { acquireLock, releaseLock, getLastDistillTime, stampLock, clearStamp, lockPathForKey } from "./distill-lock.js";
 import { logInfo } from "./stderr-log.js";
 
 // ---------------------------------------------------------------------------
@@ -315,9 +315,10 @@ export function resetGcTimestamp(): void {
   lastGcTimestamp = 0;
   lastVersionGroupRepairTimestamp = 0;
   try {
-    releaseLock({ lockPath: lockPathForKey("gc-last") });
+    clearStamp({ lockPath: lockPathForKey("gc-last") });
+    clearStamp({ lockPath: lockPathForKey("version-group-repair-last") });
+    // Only releases gc-run if this process holds it; another holder's lock is never removed.
     releaseLock({ lockPath: lockPathForKey("gc-run") });
-    releaseLock({ lockPath: lockPathForKey("version-group-repair-last") });
   } catch {
     /* ignore — nothing to clear */
   }