recallnest 2.5.4 → 2.6.0

package/src/runtime-config.ts

@@ -8,6 +8,7 @@ import { createEmbedder, getVectorDimensions, type EmbeddingConfig } from "./emb
 import { createRetriever, type RetrievalConfig, DEFAULT_RETRIEVAL_CONFIG } from "./retriever.js";
 import { applyRetrievalProfile } from "./retrieval-profiles.js";
 import { AccessTracker } from "./access-tracker.js";
+import { createAuditLogger } from "./audit-log.js";
 import { FrequencyTracker } from "./frequency-tracker.js";
 import { createLLMClient, type LLMClient, type LLMConfig } from "./llm-client.js";
 import { logInfo } from "./stderr-log.js";
@@ -67,7 +68,20 @@ export function loadDotEnv(): void {
 }
 
 export function findConfigPath(): string {
-  if (process.env.LOCAL_MEMORY_CONFIG) return resolve(process.env.LOCAL_MEMORY_CONFIG);
+  if (process.env.LOCAL_MEMORY_CONFIG) {
+    const explicit = resolve(process.env.LOCAL_MEMORY_CONFIG);
+    // Check here rather than letting readFileSync throw a bare ENOENT: scripts
+    // resolve the database at module load, so a stale env var would otherwise
+    // surface as an unexplained crash before any of their own output appears.
+    if (!existsSync(explicit)) {
+      throw new Error(
+        `LOCAL_MEMORY_CONFIG points to a file that does not exist:\n` +
+        `  ${explicit}\n` +
+        `  Fix: correct the env var, or unset it to use the repo's config.json.`
+      );
+    }
+    return explicit;
+  }
 
   const localConfig = resolve(metaDir(import.meta), "../config.json");
   if (existsSync(localConfig)) return localConfig;
@@ -103,8 +117,35 @@ export function expandHome(p: string): string {
   return p;
 }
 
+/**
+ * Single source of truth for "which LanceDB directory do we operate on".
+ *
+ * Every entry point — MCP server, CLI, and maintenance scripts — must resolve
+ * the database through this function. Scripts that hardcode "./data/lancedb"
+ * or read a non-existent config key silently operate on a *different* database
+ * than the server does, so a health check, cleanup, or backfill can report on
+ * an empty directory while the real store sits elsewhere.
+ *
+ * Relative dbPath values resolve against the repo root (not process.cwd()), so
+ * a script's behaviour does not depend on where it was invoked from.
+ *
+ * Pass an already-loaded config where you have one. Calling this with no
+ * argument reads config.json from disk — convenient at a script's top level,
+ * but it means a missing or misconfigured file surfaces here, before the
+ * script's own output. findConfigPath() raises an actionable error for that.
+ */
+export function resolveDbPath(config?: LocalMemoryConfig): string {
+  const cfg = config ?? loadConfig();
+  return resolve(metaDir(import.meta), "..", expandHome(cfg.dbPath));
+}
+
+/** Sidecar directory (frequency-stats.json etc.) that lives beside the database. */
+export function resolveDataDir(config?: LocalMemoryConfig): string {
+  return resolve(resolveDbPath(config), "..");
+}
+
 export function createComponents(config: LocalMemoryConfig, profileName?: string) {
-  const dbPath = resolve(metaDir(import.meta), "..", expandHome(config.dbPath));
+  const dbPath = resolveDbPath(config);
   validateStoragePath(dbPath);
 
   const embeddingConfig: EmbeddingConfig = {
@@ -119,24 +160,48 @@ export function createComponents(config: LocalMemoryConfig, profileName?: string
 
   const embedder = createEmbedder(embeddingConfig);
   const store = new MemoryStore({ dbPath, vectorDim: embedder.dimensions });
-  const baseRetrievalConfig = {
+  const mergedRetrievalConfig = {
     ...DEFAULT_RETRIEVAL_CONFIG,
     ...(config.retrieval || {}),
   };
+  // rerankApiKey 与 embedding.apiKey 同款支持 ${ENV_VAR} 引用。此前 retrieval 段是整体
+  // 展开、不过 resolveEnv，导致这个字段只能写明文——等于逼配置文件存凭证，是 rerank
+  // 一直没被启用的障碍之一。
+  const baseRetrievalConfig = mergedRetrievalConfig.rerankApiKey
+    ? { ...mergedRetrievalConfig, rerankApiKey: resolveEnv(mergedRetrievalConfig.rerankApiKey) }
+    : mergedRetrievalConfig;
   const { profile, config: retrieverConfig } = applyRetrievalProfile(baseRetrievalConfig, profileName);
   const retriever = createRetriever(store, embedder, retrieverConfig);
 
   // Attach access tracker for reinforcement-based decay
   const accessTracker = new AccessTracker(store);
+  accessTracker.registerExitFlush();
   retriever.setAccessTracker(accessTracker);
 
   // P0.2: Attach frequency tracker for hit-count based boosting
-  const dataDir = resolve(metaDir(import.meta), "..", expandHome(config.dbPath), "..");
+  const dataDir = resolveDataDir(config);
   const frequencyTracker = new FrequencyTracker({
     filePath: join(dataDir, "frequency-stats.json"),
   });
   retriever.setFrequencyTracker(frequencyTracker);
 
+  // F-1: 补接 audit logger，让 retrieve 操作真的被记录。
+  //
+  // 2026-08-12 修复：`setAuditLogger`（retriever.ts:749）自定义以来**全仓零调用者**，
+  // 而 retriever.ts:897 的 `this.auditLogger?.log({ operation: "retrieve", ... })`
+  // 早就写好了 —— 可选链让它永远静默 no-op。结果是 audit-log.ts 文件头注释声称
+  // "record every store/update/delete/retrieve operation"、AuditOperation 类型里
+  // 也列着 "retrieve"，而生产 audit.jsonl 里 2841 条记录**零条 retrieve**。
+  //
+  // 同族的 setAccessTracker / setFrequencyTracker 就在上面两行，都接了，唯独这个漏了。
+  // 与同日修的 dream 末尾 resetWriteCount 漏传 scope 完全同型：接线漏一处、运行时
+  // 不报错、静默失效 —— 差别只在那个是漏参数、这个是漏调用。
+  //
+  // simplified: audit.jsonl 无轮转机制（修复时 644K / 2841 行，约 230 字节一行）。
+  // 接上 retrieve 后增速会明显上升，因为检索远比写入频繁。
+  // 升级触发条件：文件超过约 50MB 时加按月轮转，别等到它拖慢 append。
+  retriever.setAuditLogger(createAuditLogger(join(dataDir, "audit.jsonl")));
+
   // Create LLM client if configured (optional, graceful)
   let llm: LLMClient | null = null;
   if (config.llm) {
@@ -150,7 +215,7 @@ export function createComponents(config: LocalMemoryConfig, profileName?: string
 }
 
 export function createStoreOnly(config: LocalMemoryConfig): MemoryStore {
-  const dbPath = resolve(metaDir(import.meta), "..", expandHome(config.dbPath));
+  const dbPath = resolveDbPath(config);
   validateStoragePath(dbPath);
   return new MemoryStore({
     dbPath,

package/src/skill-promotion.ts

@@ -28,6 +28,8 @@ export interface PromotionCandidate {
   confidence: number; // 0-1
   /** A2: zero-LLM verifier verdict (pattern_to_skill only). 失败候选保留披露、排后、不静默丢。 */
   verification?: VerifyResult;
+  /** Read-signal evidence aggregated from source entries（Artel 读驱动升华：被真实读过的候选提权）。 */
+  readEvidence?: { totalAccess: number; distinctReaders: number };
 }
 
 export interface PromotionScanResult {
@@ -266,6 +268,47 @@ export function greedyCluster(
   return clusters;
 }
 
+// ---------------------------------------------------------------------------
+// Read evidence (Artel-style read-driven promotion)
+// ---------------------------------------------------------------------------
+
+/** Max confidence boost from read evidence. Saturates at 4 distinct readers. */
+const READ_BOOST_MAX = 0.15;
+
+/**
+ * Aggregate read signals (accessCount / readerIds / distinctReaderCount, written
+ * by AccessTracker on retrieval) across a candidate's source entries.
+ * distinctReaders = max(union of readerIds, per-entry saturated count) — the
+ * union undercounts once READER_ID_CAP saturates, the per-entry count covers that.
+ */
+export function aggregateReadEvidence(
+  entries: Array<{ metadata?: string }>,
+): { totalAccess: number; distinctReaders: number } {
+  let totalAccess = 0;
+  const readers = new Set<string>();
+  let saturatedMax = 0;
+  for (const e of entries) {
+    try {
+      const meta = JSON.parse(e.metadata || "{}") as Record<string, unknown>;
+      if (typeof meta.accessCount === "number") totalAccess += meta.accessCount;
+      if (Array.isArray(meta.readerIds)) {
+        for (const r of meta.readerIds) if (typeof r === "string") readers.add(r);
+      }
+      if (typeof meta.distinctReaderCount === "number") {
+        saturatedMax = Math.max(saturatedMax, meta.distinctReaderCount);
+      }
+    } catch { /* broken metadata → contributes nothing */ }
+  }
+  return { totalAccess, distinctReaders: Math.max(readers.size, saturatedMax) };
+}
+
+/** Blend read evidence into base confidence: +0..READ_BOOST_MAX, capped at 0.99.
+ *  提权不设门槛（observe-before-enforce：先积累读证据，硬门槛等有生产数据再定）。 */
+export function applyReadBoost(base: number, distinctReaders: number): number {
+  const boost = Math.min(1, distinctReaders / 4) * READ_BOOST_MAX;
+  return Math.min(0.99, base + boost);
+}
+
 // ---------------------------------------------------------------------------
 // Main scan
 // ---------------------------------------------------------------------------
@@ -339,6 +382,7 @@ export async function scanForPromotions(
         const avgSim = computeAverageIntraClusterSimilarity(cluster.members);
 
         caseCandidates++;
+        const caseReadEvidence = aggregateReadEvidence(cluster.members);
         result.candidates.push({
           type: "case_to_pattern",
           sourceEntries: cluster.members.map(m => ({
@@ -348,7 +392,11 @@ export async function scanForPromotions(
           })),
           suggestedName: extractName(cluster.seed.text),
           suggestedDescription: summarizeCluster(cluster.members),
-          confidence: cluster.members.length / (cluster.members.length + 2), // Bayesian smoothing
+          confidence: applyReadBoost(
+            cluster.members.length / (cluster.members.length + 2), // Bayesian smoothing
+            caseReadEvidence.distinctReaders,
+          ),
+          readEvidence: caseReadEvidence,
         });
       }
     }
@@ -394,6 +442,7 @@ export async function scanForPromotions(
     );
 
     patternCandidates++;
+    const patternReadEvidence = aggregateReadEvidence([pattern, ...similarCases.map(h => h.entry)]);
     result.candidates.push({
       type: "pattern_to_skill",
       sourceEntries: [
@@ -407,8 +456,12 @@ export async function scanForPromotions(
       suggestedName: extractName(pattern.text),
       suggestedDescription: `Skill derived from pattern with ${similarCases.length} supporting cases`,
       suggestedImplementation: extractSteps(pattern.text),
-      confidence: similarCases.length / (similarCases.length + 2),
+      confidence: applyReadBoost(
+        similarCases.length / (similarCases.length + 2),
+        patternReadEvidence.distinctReaders,
+      ),
       verification,
+      readEvidence: patternReadEvidence,
     });
   }
 
@@ -455,6 +508,9 @@ export function formatPromotionResult(result: PromotionScanResult): string {
     lines.push(`Confidence: ${(c.confidence * 100).toFixed(1)}%`);
     lines.push(`Description: ${c.suggestedDescription}`);
     lines.push(`Sources: ${c.sourceEntries.length} entries`);
+    if (c.readEvidence) {
+      lines.push(`Read evidence: ${c.readEvidence.totalAccess} total reads, ${c.readEvidence.distinctReaders} distinct reader(s)`);
+    }
     if (c.verification) {
       const v = c.verification;
       if (v.ok) {

package/src/store.ts

@@ -7,6 +7,7 @@ import { createHash, randomUUID } from "node:crypto";
 import { existsSync, accessSync, constants, mkdirSync, realpathSync, lstatSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { logWarn } from "./stderr-log.js";
+import { readConsistencyInterval as envReadConsistencyInterval } from "./env-config.js";
 import type { DurableMemoryCategory } from "./memory-schema.js";
 import { matchesScopeFilter } from "./scope-policy.js";
 import { detectEmotionIfEnabled } from "./emotion-detector.js";
@@ -58,6 +59,13 @@ export interface MemorySearchResult {
 export interface StoreConfig {
   dbPath: string;
   vectorDim: number;
+  /**
+   * LanceDB readConsistencyInterval (seconds): 0 = strong consistency (external
+   * writes visible on every read), positive = bounded staleness window.
+   * Omitted → RECALLNEST_READ_CONSISTENCY_INTERVAL decides (default 0; "off"
+   * restores the legacy unchecked-handle behavior).
+   */
+  readConsistencyInterval?: number;
 }
 
 export type LegacyScopeIssueKind = "missing" | "empty" | "global";
@@ -222,6 +230,9 @@ export class MemoryStore implements MemoryStorePort {
   /** Per-id serial queues for patchMetadata; entries self-remove on settle. */
   private readonly metadataPatchQueues = new Map<string, Promise<void>>();
 
+  /** Set when the init-time schema migration fails; surfaced in store error messages. */
+  private schemaMigrationError: string | null = null;
+
   constructor(private readonly config: StoreConfig) {}
 
   get dbPath(): string {
@@ -248,7 +259,13 @@ export class MemoryStore implements MemoryStorePort {
 
     let db: LanceDB.Connection;
     try {
-      db = await lancedb.connect(this.config.dbPath);
+      // Without a consistency interval a long-lived handle pins its manifest
+      // version — resident MCP/API/UI servers would never see CLI or
+      // sibling-process writes. Explicit config wins so tests can pin behavior.
+      db = await lancedb.connect(this.config.dbPath, {
+        readConsistencyInterval:
+          this.config.readConsistencyInterval ?? envReadConsistencyInterval(),
+      });
     } catch (err: any) {
       const code = err.code || "";
       const message = err.message || String(err);
@@ -266,24 +283,24 @@ export class MemoryStore implements MemoryStorePort {
     try {
       table = await db.openTable(TABLE_NAME);
 
-      // Backward compatibility: add missing columns to existing tables
+      // Backward compatibility: add missing columns to existing tables.
+      // Read the schema definition, not sample rows — a row-sampling check can
+      // never fire on a 0-row table, which deadlocked every write with
+      // "Found field not in schema" on empty pre-language/fts_text tables.
       try {
-        const sample = await table.query().limit(1).toArray();
-        if (sample.length > 0) {
-          if (!("scope" in sample[0])) {
-            logWarn("Adding scope column for backward compatibility with existing data");
-          }
-          if (!("language" in sample[0])) {
-            logWarn("Adding language column for backward compatibility");
-            await table.addColumns([{ name: "language", valueSql: "'en'" }]);
-          }
-          if (!("fts_text" in sample[0])) {
-            logWarn("Adding fts_text column for backward compatibility");
-            await table.addColumns([{ name: "fts_text", valueSql: "text" }]);
-          }
+        const schema = await table.schema();
+        const fields = new Set(schema.fields.map((f) => f.name));
+        const missing: Array<{ name: string; valueSql: string }> = [];
+        if (!fields.has("language")) missing.push({ name: "language", valueSql: "'en'" });
+        if (!fields.has("fts_text")) missing.push({ name: "fts_text", valueSql: "text" });
+        if (missing.length > 0) {
+          logWarn(`Adding missing columns for backward compatibility: ${missing.map((c) => c.name).join(", ")}`);
+          await table.addColumns(missing);
         }
+        this.schemaMigrationError = null;
       } catch (err) {
-        logWarn("Could not check/migrate table schema:", err);
+        this.schemaMigrationError = err instanceof Error ? err.message : String(err);
+        logWarn("Could not check/migrate table schema — subsequent writes may fail:", err);
       }
     } catch (_openErr) {
       // Table doesn't exist yet — create it
@@ -388,7 +405,8 @@ export class MemoryStore implements MemoryStorePort {
       const code = err.code || "";
       const message = err.message || String(err);
       throw new Error(
-        `Failed to store memory in "${this.config.dbPath}": ${code} ${message}`
+        `Failed to store memory in "${this.config.dbPath}": ${code} ${message}` +
+        (this.schemaMigrationError ? `\n  Schema migration failed at init: ${this.schemaMigrationError}` : "")
       );
     }
   }
@@ -440,7 +458,8 @@ export class MemoryStore implements MemoryStorePort {
       const code = err.code || "";
       const message = err.message || String(err);
       throw new Error(
-        `Failed to batch store ${entries.length} memories in "${this.config.dbPath}": ${code} ${message}`
+        `Failed to batch store ${entries.length} memories in "${this.config.dbPath}": ${code} ${message}` +
+        (this.schemaMigrationError ? `\n  Schema migration failed at init: ${this.schemaMigrationError}` : "")
       );
     }
     // Per-scope write counts drive dream activation. Best-effort — never block writes.
@@ -1065,7 +1084,10 @@ export class MemoryStore implements MemoryStorePort {
     const updated: MemoryEntry = {
       id: row.id as string,
       text: updates.text ?? (row.text as string),
-      vector: updates.vector ?? (Array.from(row.vector as Iterable<number>)),
+      // row.vector 可能为 null（历史病行，2026-07-23 dream/auto-gc 生产实锤：
+      // cc:82a919ea 连崩三轮 Array.from(null)）。null → [] 让 metadata-only update
+      // 不再被病行绊倒；空向量本就不可检索，语义不变。
+      vector: updates.vector ?? (row.vector ? Array.from(row.vector as Iterable<number>) : []),
       category: updates.category ?? (row.category as MemoryEntry["category"]),
       scope: rowScope,
       importance: updates.importance ?? Number(row.importance),
@@ -1212,6 +1234,19 @@ export class MemoryStore implements MemoryStorePort {
     return deleteCount;
   }
 
+  /**
+   * Force lazy initialization to complete.
+   *
+   * `hasFtsSupport` is meaningless before init: `ftsIndexCreated` starts false and
+   * only flips once `doInitialize()` has attempted to build the FTS index. Any caller
+   * that *branches* on `hasFtsSupport` must await this first, or it will read the
+   * pre-init false and take the wrong branch. Cheap to call repeatedly —
+   * `ensureInitialized()` returns immediately once the table exists.
+   */
+  async ready(): Promise<void> {
+    await this.ensureInitialized();
+  }
+
   get hasFtsSupport(): boolean {
     return this.ftsIndexCreated;
   }

package/src/term-registry.ts

@@ -272,12 +272,24 @@ export const TASK_RESULT_SPECIFICITY_GROUPS = [
       "managed continuity rules",
       "global instruction file",
     ],
+    // ⚠️ 上面的 resultTerms 是**一次历史事件的专名**（2026 年那轮三端连续性触发验证），
+    // 不是端清单 —— 端接入时不要往里加端名：resultTerms 是「谁被这个组认领」，
+    // 加了 "kimi" 等于断言"任何提到 kimi 的记忆都是那次三端验证的结论"，
+    // 会让大批 kimi 记忆在无关任务下被这个组滤掉。同款判断见 932ec37：
+    // ROADMAP 里 `✅ managed continuity rules installed by setup` 那处也按"历史"原样保留。
+    //
+    // taskTerms 是「什么问法下这条历史仍然相关」，所以只补她真实的问法：
+    // 表里原有 "three-terminal" / "三终端"，但她写的是"三端"/"四端"（"三端"不是"三终端"的子串，
+    // 今天真的匹配不上）。**同样不补端名** —— 补了会让「kimi 挂了」这种无关任务
+    // 也解除过滤、把这条三端历史塞进 context；真正相关的问法已被 触发/接入/规则 覆盖。
     taskTerms: [
       "claude code",
       "codex",
       "gemini cli",
       "three-terminal",
       "三终端",
+      "三端",
+      "四端",
       "trigger",
       "触发",
       "setup",
@@ -462,6 +474,18 @@ export const TASK_RESULT_SPECIFICITY_GROUPS = [
   },
 ];
 
+// 「某个端的验收 / 独立验证视角」这一簇。判据（context-composer-stable-selection.ts:154-157）：
+//   resultTerms 命中记忆正文 && taskTerms 全不命中任务  →  丢弃这条记忆
+// 两个字段方向相反，接入新端时**不要两边对称地加**：
+//   · taskTerms 加词 = 放宽豁免，召回变多 —— 端接入时要补的是这一侧
+//   · resultTerms 加词 = 扩大被滤对象，召回变少 —— 这里**有意不补端名**：
+//     概念词（smoke / integration / 验收 / 验证视角 / 独立验证 / sidecar）本就是端无关的，
+//     已经覆盖了"kimi 的验收视角"这类记忆；而裸词是子串匹配，补进去会把"顺带提了一句 kimi"
+//     的无关偏好整条变成可滤对象，爆炸半径远大于收益。
+//     实证：现有的裸词 "codex" 已经在误认领 —— 记忆 b8731a81（讲规则该放 home 级还是项目级）
+//     只因正文里有 `.codex/AGENTS.md` 这个**路径片段**就被本组claim，
+//     于是"四端的触发规则都装好了吗"这一问反而把最对题的那条滤掉了（shadow 实测）。
+//     再往 resultTerms 加端名 = 把这个已知缺陷复制三份。
 export const PREFERENCE_SPECIFICITY_GROUPS = [
   {
     resultTerms: [
@@ -480,6 +504,17 @@ export const PREFERENCE_SPECIFICITY_GROUPS = [
       "claude code",
       "codex",
       "gemini cli",
+      // 在栈四端 = Claude Code / Codex / Kimi / AGY。AGY 是一个端，
+      // "antigravity" 与 "agy" 是它的两类写法（scope 前缀写 antigravity:，
+      // 她口头和 MEMORY.md 里写 agy），两个都要，不是两个端。
+      "kimi",
+      "antigravity",
+      "agy",
+      // 问端拓扑本身（不点名某一端）时也该解除：shadow 实测
+      // "四端的触发规则都装好了吗"会把 b8731a81（规则该放 home 级还是项目级，
+      // 正文写着"shared-behaviors.md 放三端通用底线"）这条最对题的记忆滤掉。
+      "三端",
+      "四端",
       "smoke",
       "integration",
       "验收",

package/src/ui-server.ts

@@ -12,6 +12,7 @@ import { archiveDirtyBriefAsset, assetSummaryLine, buildBriefAsset, buildPinAsse
 import { indexAsset, indexPinnedAsset } from "./asset-sync.js";
 import { createComponentResolver, loadConfig, loadDotEnv } from "./runtime-config.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
+import { enforceLocalHttpRequestPolicy, LOCAL_HTTP_HOSTNAME } from "./http-request-policy.js";
 import * as envConfig from "./env-config.js";
 const config = (loadDotEnv(), loadConfig());
 const getComponents = createComponentResolver(config);
@@ -188,8 +189,14 @@ async function handleSearch(mode: "search" | "explain" | "distill", body: Record
 
 const server = Bun.serve({
   port: readLimit(envConfig.uiPortRaw(), 4317, 1, 65535),
+  hostname: LOCAL_HTTP_HOSTNAME,
   async fetch(request) {
     try {
+      const policy = await enforceLocalHttpRequestPolicy(request);
+      if (!policy.allowed) {
+        return textResponse(policy.message, { status: policy.status });
+      }
+
       const url = new URL(request.url);
 
       if (request.method === "GET" && (url.pathname === "/" || url.pathname.startsWith("/ui/"))) {

package/src/version.ts

@@ -0,0 +1,19 @@
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { metaDir } from "./compat.js";
+
+type PackageMetadata = {
+  version?: unknown;
+};
+
+const packageMetadata = JSON.parse(
+  readFileSync(resolve(metaDir(import.meta), "../package.json"), "utf8"),
+) as PackageMetadata;
+
+if (typeof packageMetadata.version !== "string" || packageMetadata.version.length === 0) {
+  throw new Error("package.json must contain a non-empty version");
+}
+
+/** The public RecallNest version. package.json is the single source of truth. */
+export const PACKAGE_VERSION = packageMetadata.version;