recallnest 2.5.4 → 2.6.1

package/src/__tests__/ui-server-security.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  enforceLocalHttpRequestPolicy,
+  LOCAL_HTTP_HOSTNAME,
+  LOCAL_JSON_BODY_LIMIT_BYTES,
+} from "../http-request-policy.js";
+
+function uiRequest({
+  method = "GET",
+  host = "127.0.0.1:4317",
+  headers = {},
+  body,
+}: {
+  method?: string;
+  host?: string;
+  headers?: Record<string, string>;
+  body?: string;
+} = {}): Request {
+  return new Request("http://127.0.0.1:4317/api/search", {
+    method,
+    headers: { host, ...headers },
+    body,
+  });
+}
+
+describe("RecallNest UI local HTTP policy", () => {
+  test("binds the UI to the IPv4 loopback interface", () => {
+    expect(LOCAL_HTTP_HOSTNAME).toBe("127.0.0.1");
+  });
+
+  test("allows same-origin browser requests", async () => {
+    const result = await enforceLocalHttpRequestPolicy(uiRequest({
+      method: "POST",
+      headers: {
+        "content-type": "application/json; charset=utf-8",
+        origin: "http://127.0.0.1:4317",
+        "sec-fetch-site": "same-origin",
+      },
+      body: JSON.stringify({ query: "local" }),
+    }));
+
+    expect(result).toEqual({ allowed: true });
+  });
+
+  test("rejects a non-local Host header", async () => {
+    const result = await enforceLocalHttpRequestPolicy(uiRequest({
+      host: "attacker.example",
+    }));
+
+    expect(result).toEqual({ allowed: false, status: 403, message: "Host is not allowed." });
+  });
+
+  test("rejects cross-origin and cross-site browser requests", async () => {
+    const wrongOrigin = await enforceLocalHttpRequestPolicy(uiRequest({
+      headers: { origin: "https://attacker.example" },
+    }));
+    const crossSite = await enforceLocalHttpRequestPolicy(uiRequest({
+      headers: { "sec-fetch-site": "cross-site" },
+    }));
+
+    expect(wrongOrigin).toEqual({ allowed: false, status: 403, message: "Origin is not allowed." });
+    expect(crossSite).toEqual({ allowed: false, status: 403, message: "Cross-site requests are not allowed." });
+  });
+
+  test("rejects non-JSON POST bodies", async () => {
+    const result = await enforceLocalHttpRequestPolicy(uiRequest({
+      method: "POST",
+      headers: { "content-type": "text/plain" },
+      body: JSON.stringify({ query: "local" }),
+    }));
+
+    expect(result).toEqual({
+      allowed: false,
+      status: 415,
+      message: "Content-Type must be application/json.",
+    });
+  });
+
+  test("rejects streamed JSON bodies over the limit", async () => {
+    const result = await enforceLocalHttpRequestPolicy(uiRequest({
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ query: "x".repeat(LOCAL_JSON_BODY_LIMIT_BYTES) }),
+    }));
+
+    expect(result).toEqual({
+      allowed: false,
+      status: 413,
+      message: `Request body exceeds ${LOCAL_JSON_BODY_LIMIT_BYTES} bytes.`,
+    });
+  });
+});

package/src/access-tracker.ts

@@ -38,10 +38,18 @@ export const DEFAULT_ACCESS_TRACKER_CONFIG: AccessTrackerConfig = {
   reinforcementFactor: 0.5,
   maxMultiplier: 3.0,
   accessFreshnessHalfLifeDays: 30,
-  noveltyThreshold: 0.35,
+  // 0 = novelty gate off. 生产实测（2026-07-23）：0.35 要求命中的 cosine ≤0.65 才打点，
+  // 真实相关命中几乎都 >0.65，导致 accessCount 在生产库全空——"最被用到"的记忆
+  // 恰好全被排除，方向与"用进废退"相反。防刷由 cooldown 承担；要恢复旧行为显式传 0.35。
+  noveltyThreshold: 0,
   cooldownMs: 300_000, // 5 minutes
 };
 
+/** readerIds 在 metadata 里的存储上限。distinctReaderCount 到达上限后饱和不再涨。 */
+// simplified: capped array + saturating count instead of Artel's exact memory_reads
+// ledger table — promotion thresholds are ≤4 distinct readers, so saturation at 8 is harmless.
+export const READER_ID_CAP = 8;
+
 // ============================================================================
 // Core
 // ============================================================================
@@ -52,11 +60,36 @@ export class AccessTracker {
   private flushPromise: Promise<void> | null = null;
   /** Cooldown map: entryId → last reinforced timestamp */
   private cooldownMap = new Map<string, number>();
+  /** Distinct-reader identity: one stdio MCP server process ≈ one CC session.
+   *  Used to maintain readerIds / distinctReaderCount on flush (Artel-style read fan-out). */
+  readonly readerId: string;
+  private exitFlushRegistered = false;
 
   constructor(
     private store: MemoryStore,
     private config: AccessTrackerConfig = DEFAULT_ACCESS_TRACKER_CONFIG,
-  ) {}
+    readerId?: string,
+  ) {
+    this.readerId = readerId ?? `r-${crypto.randomUUID().slice(0, 8)}`;
+  }
+
+  /**
+   * Best-effort flush on normal process exit (stdin close → clean shutdown).
+   * Covers the "retrieval happened <flushIntervalMs before exit" window where the
+   * debounce timer would otherwise drop pending deltas. Opt-in so tests creating
+   * many trackers don't leak process listeners. Not wired to SIGTERM on purpose —
+   * an async handler there isn't guaranteed to complete and we don't want to own
+   * the server's exit semantics.
+   */
+  registerExitFlush(): void {
+    if (this.exitFlushRegistered) return;
+    this.exitFlushRegistered = true;
+    process.once("beforeExit", () => {
+      this.flush().catch(err => {
+        logWarn("AccessTracker exit flush failed:", err);
+      });
+    });
+  }
 
   /**
    * Determine whether a retrieval result should trigger access reinforcement.
@@ -173,7 +206,7 @@ export class AccessTracker {
           const importance = entry.importance ?? 0.6;
           const lastAccessedAt = now;
 
-          const currentTier = resolveTier(entry.metadata);
+          const currentTier = resolveTier(entry.metadata, importance);
           const newTier = evaluateTierChange(
             currentTier,
             newCount,
@@ -184,10 +217,27 @@ export class AccessTracker {
             ? { from: currentTier, to: newTier, newCount }
             : null;
 
+          // Distinct-reader bookkeeping (Artel read fan-out, lightweight form):
+          // dedup by readerId in a capped array; count saturates at cap (under-
+          // counting is safer than over-counting for promotion gating).
+          const rawReaders = Array.isArray(meta.readerIds)
+            ? (meta.readerIds as unknown[]).filter((x): x is string => typeof x === "string")
+            : [];
+          let readerIds = rawReaders;
+          let distinctReaderCount = typeof meta.distinctReaderCount === "number"
+            ? meta.distinctReaderCount
+            : rawReaders.length;
+          if (!rawReaders.includes(this.readerId) && rawReaders.length < READER_ID_CAP) {
+            readerIds = [...rawReaders, this.readerId];
+            distinctReaderCount = readerIds.length;
+          }
+
           return {
             ...meta,
             accessCount: newCount,
             lastAccessedAt,
+            readerIds,
+            distinctReaderCount,
             ...(tierChange ? { tier: tierChange.to } : {}),
           };
         });

package/src/adaptive-retrieval.ts

@@ -55,8 +55,19 @@ export function shouldSkipRetrieval(query: string, minLength?: number): boolean
 
   const effectiveMinLength = minLength ?? (hasCJK ? 2 : 15);
 
+  // 【本地分叉点 · 2026-08-08 · 勿在同步上游时覆盖回去】
+  // 单 token（不含空格）= 实体查找意图，与问号一样豁免长度闸。
+  // 上游的 ASCII 阈值 15 是按「英文自然语句」定的（挡 "hello there" 这类闲聊），
+  // 但项目名/工具名几乎都短于 15：cobbler=7 / slidesmith=10 / hippo-wiki=10 / recallnest=10，
+  // 于是「甩一个名字过去」这种最自然的查法一律返回空——库里明明有。
+  // 红绿实测：`cobbler` → No results；`cobbler?` → 100% 命中 12 条，只差一个问号豁免。
+  // 中文不受影响（阈值 2），所以该缺陷只在英文实体名上暴露，一直没被发现。
+  // 安全性：本条排在 SKIP_PATTERNS 之后，hi / ok / ping / test / ls / git 等单词仍被挡住。
+  const isSingleToken = !/\s/.test(trimmed);
+
   // Short non-question messages are skipped; questions (? ？) are always worth checking
   if (trimmed.length < effectiveMinLength &&
+      !isSingleToken &&
       !trimmed.includes('?') && !trimmed.includes('？')) {
     return true;
   }

package/src/api-server.ts

@@ -27,7 +27,9 @@ import { buildManagedCheckpointObservation, buildManagedResumeObservation } from
 import { runAutoRecall } from "./auto-recall.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
 import { runMemoryLint } from "./memory-lint.js";
+import { enforceLocalHttpRequestPolicy, LOCAL_HTTP_HOSTNAME } from "./http-request-policy.js";
 import * as envConfig from "./env-config.js";
+import { PACKAGE_VERSION } from "./version.js";
 
 const config = (loadDotEnv(), loadConfig());
 const getComponents = createComponentResolver(config);
@@ -625,7 +627,7 @@ async function handleHealth(): Promise<Response> {
     const stats = await store.stats();
     return jsonResponse({
       status: "ok",
-      version: "1.0.0",
+      version: PACKAGE_VERSION,
       totalMemories: stats.totalCount,
       timestamp: new Date().toISOString(),
     });
@@ -649,8 +651,13 @@ const port = clampInt(envConfig.apiPortRaw(), 4318, 1, 65535);
 
 const server = Bun.serve({
   port,
-  hostname: "127.0.0.1",
+  hostname: LOCAL_HTTP_HOSTNAME,
   async fetch(request) {
+    const policy = await enforceLocalHttpRequestPolicy(request);
+    if (!policy.allowed) {
+      return errorResponse(policy.status, policy.message);
+    }
+
     const url = new URL(request.url);
     const { pathname } = url;
     const method = request.method;

package/src/auto-recall.ts

@@ -1,6 +1,12 @@
 import { composeResumeContext, type ResumeContextDeps } from "./context-composer.js";
 import { resolveTier } from "./decay-engine.js";
 import type { DurableMemoryCategory } from "./memory-schema.js";
+import {
+  classifyRecallGate,
+  logRecallGateShadow,
+  resolveRecallGateMode,
+  type RecallGateResult,
+} from "./recall-gate.js";
 import { governResults, truncateQuery, type GovernorConfig, type GovernorSession } from "./recall-governor.js";
 import type { RetrievalResult } from "./retriever.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
@@ -33,6 +39,20 @@ export interface AutoRecallResponse {
   searchSkippedReason?: string;
 }
 
+/** Minimal valid ResumeContextResponse for enforce-mode skip-all short-circuits. */
+function emptyResumeContext(): ResumeContextResponse {
+  return {
+    summary: "",
+    stableContext: [],
+    relevantPatterns: [],
+    recentCases: [],
+    injectionHint: "user_attachment",
+    ephemeral: true,
+    responseMode: "default",
+    generatedAt: new Date().toISOString(),
+  };
+}
+
 export async function runAutoRecall(
   deps: ResumeContextDeps,
   request: AutoRecallRequest,
@@ -46,6 +66,34 @@ export async function runAutoRecall(
     throw new Error("message is required");
   }
 
+  // Recall gate (mlp adaptive-retrieval borrow, three-way per Codex review).
+  // Sits BEFORE composeResumeContext — compose itself hits the retriever, so a
+  // gate placed after it would save nothing. Default mode is observe: verdicts
+  // are computed and shadow-logged but behavior is unchanged until real-sample
+  // false-positive rates have been reviewed (shared-behaviors §5).
+  const gateMode = resolveRecallGateMode(request.env);
+  let gateVerdict: RecallGateResult | undefined;
+  if (gateMode !== "off") {
+    gateVerdict = classifyRecallGate(message);
+    logRecallGateShadow({
+      ts: new Date().toISOString(),
+      decision: gateVerdict.decision,
+      ruleId: gateVerdict.ruleId,
+      msgLen: message.length,
+      mode: gateMode,
+      source: request.operation,
+    });
+    if (gateMode === "enforce" && gateVerdict.decision === "skip-all") {
+      return {
+        mode: "resume-only",
+        resolvedScope: undefined,
+        resume: emptyResumeContext(),
+        results: [],
+        searchSkippedReason: `recall-gate: skip-all (${gateVerdict.ruleId})`,
+      };
+    }
+  }
+
   const scopeSelection = resolveScopeSelection({
     scope: request.scope,
     sessionId: request.sessionId,
@@ -67,6 +115,18 @@ export async function runAutoRecall(
     profile: request.profile,
   });
 
+  // Continuity nudges ("继续", "下一步") want the resume context they just got,
+  // not an embedding search over their own two characters.
+  if (gateMode === "enforce" && gateVerdict?.decision === "resume-only") {
+    return {
+      mode: "resume-only",
+      resolvedScope: resume.resolvedScope || resume.latestCheckpoint?.resolvedScope || scopeSelection.resolvedScope,
+      resume,
+      results: [],
+      searchSkippedReason: `recall-gate: resume-only (${gateVerdict.ruleId})`,
+    };
+  }
+
   const resolvedScope = request.allScopes
     ? undefined
     : (resume.resolvedScope || resume.latestCheckpoint?.resolvedScope || scopeSelection.resolvedScope);
@@ -101,7 +161,7 @@ export async function runAutoRecall(
   const promoted: RetrievalResult[] = [];
   const demoted: RetrievalResult[] = [];
   for (const r of rawResults) {
-    const tier = resolveTier(r.entry.metadata);
+    const tier = resolveTier(r.entry.metadata, r.entry.importance);
     if (tier === "core" || tier === "working") {
       promoted.push(r);
     } else {

package/src/belief-history.ts

@@ -0,0 +1,164 @@
+/**
+ * Belief history — copy-on-write archival for belief-changing writes.
+ *
+ * Problem this solves: the latest-wins and conflict-resolution paths used to call
+ * store.update() on the canonical row, replacing text + vector in place. The previous
+ * belief was physically gone, so no retrieval strategy could ever answer
+ * "what did I believe last month?".
+ *
+ * Approach: before overwriting canonical row C, copy C's current content into a new
+ * `superseded` history row H, then update C in place as before. C keeps its id, so the
+ * `deterministicId(scope, canonicalKey)` invariant every canonical lookup depends on is
+ * untouched — history lives beside it rather than replacing it.
+ *
+ * Chain shape (C is the stable canonical id, H1 older than H2):
+ *   H1 --supersededBy--> C     C  --supersedes--> H2
+ *   H2 --supersededBy--> C     H2 --supersedes--> H1
+ * so traceEvolution() walks backward C → H2 → H1 to rebuild the timeline, and forward
+ * from any history row to the belief that is live now.
+ */
+
+import { deterministicId, type MemoryEntry, type MemoryStore } from "./store.js";
+import { parseEvolution, patchEvolution, type EvolutionMetadata } from "./memory-evolution.js";
+import { detectLang, tokenizeFts } from "./language-hook.js";
+import type { Embedder } from "./embedder.js";
+
+/** Namespace for history-row ids — keeps them deterministic (retry-safe) yet distinct from C. */
+const BELIEF_HISTORY_ID_PREFIX = "belief-history:";
+
+export interface BeliefHistoryDeps {
+  store: Pick<MemoryStore, "store"> & Partial<Pick<MemoryStore, "upsert" | "getById">>;
+  embedder: Pick<Embedder, "embedPassage">;
+}
+
+export interface BeliefArchiveResult {
+  /** Id of the newly written history row — becomes the canonical row's `supersedes`. */
+  historyId: string;
+  /** When the archived version stopped being the live belief. */
+  validUntil: number;
+}
+
+/**
+ * Resolve a usable vector for the row being archived.
+ *
+ * store.list() deliberately returns `vector: []` for performance, and that is where the
+ * canonical-match candidates come from — so the entry handed to us usually has no vector.
+ * Re-read the full row first; only re-embed if the store can't give it back.
+ */
+async function resolveVector(deps: BeliefHistoryDeps, current: MemoryEntry): Promise<number[]> {
+  if (current.vector && current.vector.length > 0) return current.vector;
+
+  if (deps.store.getById) {
+    const full = await deps.store.getById(current.id);
+    if (full?.vector && full.vector.length > 0) return full.vector;
+  }
+
+  return await deps.embedder.embedPassage(current.text);
+}
+
+/**
+ * Archive the current content of a canonical memory as a `superseded` history row.
+ *
+ * Throws rather than degrading to a silent overwrite: a belief-changing write that cannot
+ * preserve the old belief is exactly the failure this module exists to prevent, so callers
+ * must not proceed with the overwrite if this rejects.
+ *
+ * Known limitation — archive and overwrite are two writes with no transaction around them.
+ * If the caller's update() fails after this returns, the history row is already committed
+ * while the canonical row still holds that same text, leaving a superseded duplicate of a
+ * belief that is in fact still live. Damage is bounded: nothing is destroyed, and the
+ * deterministic id means a retry reuses the same row rather than piling up copies.
+ */
+export async function archiveBeliefVersion(
+  deps: BeliefHistoryDeps,
+  current: MemoryEntry,
+  options: { now?: number } = {},
+): Promise<BeliefArchiveResult> {
+  const now = options.now ?? Date.now();
+  const currentEvo = parseEvolution(current.metadata, current.timestamp);
+
+  // Text is part of the id, not just (id, timestamp): two belief changes landing in the
+  // same millisecond would otherwise collide and the second archive would overwrite the
+  // first. With the text folded in, a collision means the content is identical anyway.
+  const historyId = deterministicId(
+    current.scope,
+    `${BELIEF_HISTORY_ID_PREFIX}${current.id}@${current.timestamp}\n${current.text}`,
+  );
+  if (historyId === current.id) {
+    throw new Error(`Belief history id collided with canonical id ${current.id}`);
+  }
+
+  const vector = await resolveVector(deps, current);
+  if (vector.length === 0) {
+    throw new Error(`Cannot archive belief ${current.id}: no vector available`);
+  }
+
+  // The history row inherits the old metadata wholesale (canonicalKey, boundary, emotion…)
+  // and only has its evolution block rewritten to close the interval.
+  const historyMetadata = patchEvolution(current.metadata, {
+    status: "superseded",
+    validFrom: currentEvo.validFrom,
+    validUntil: now,
+    supersededBy: current.id,
+    // Point at the version this one replaced, so the backward walk stays a chain.
+    supersedes: currentEvo.supersedes,
+  });
+
+  const historyRow: MemoryEntry = {
+    id: historyId,
+    text: current.text,
+    vector,
+    category: current.category,
+    scope: current.scope,
+    importance: current.importance,
+    // Keep the original timestamp: canonical matching scans the most recent
+    // CANONICAL_SCAN_LIMIT rows, and stamping history rows "now" would let them crowd
+    // live entries out of that window.
+    timestamp: current.timestamp,
+    metadata: historyMetadata,
+    // Derive rather than copy: neither list() nor getById() selects these columns, so
+    // `current.language` / `current.fts_text` are all but guaranteed to be undefined here.
+    // Falling back to the "en" / raw-text defaults would mislabel Chinese memories and
+    // store them unsegmented for FTS — so recompute them the same way live entries do.
+    language: current.language || detectLang(current.text),
+    fts_text: current.fts_text || tokenizeFts(current.text, current.language || detectLang(current.text)),
+  };
+
+  if (deps.store.upsert) {
+    await deps.store.upsert(historyRow);
+  } else {
+    // Fallback for narrower store deps (mocks): store() stamps its own timestamp, which
+    // only costs us the scan-window optimisation above — the history row is still written.
+    await deps.store.store(historyRow);
+  }
+
+  return { historyId, validUntil: now };
+}
+
+/**
+ * Build the evolution block for the canonical row that is about to be overwritten.
+ *
+ * Callers previously passed the incoming metadata straight to store.update(), which wiped
+ * the evolution block along with the text. This merges the incoming metadata with a fresh
+ * evolution block that links back to the archived version.
+ */
+export function buildSupersedingBeliefMetadata(
+  incomingMetadata: string | undefined,
+  previousEvolution: EvolutionMetadata,
+  historyId: string,
+  now: number,
+  evolutionNote?: string | null,
+): string {
+  return patchEvolution(incomingMetadata, {
+    status: "active",
+    version: previousEvolution.version + 1,
+    supersedes: historyId,
+    supersededBy: null,
+    validFrom: now,
+    validUntil: null,
+    // Access stats belong to the canonical key, not to a single version's text.
+    accessCount: previousEvolution.accessCount,
+    lastAccessedAt: previousEvolution.lastAccessedAt,
+    ...(evolutionNote ? { evolutionNote } : {}),
+  });
+}