recallnest 3.0.1 → 4.0.0

package/src/memory-schema.ts

@@ -119,6 +119,16 @@ export const CaseMemorySolutionStepsSchema = normalizedStringListSchema("solutio
 export const CaseMemoryOutcomeSchema = optionalBoundedStringSchema(240);
 export const CaseMemoryToolsSchema = normalizedStringListSchema("tools", 6, 60);
 
+/**
+ * 写入时预演触发器（T-Mem 借鉴，2026-09-22）：这条记忆「她以后会怎么问起」的几句口语问法。
+ * 只参与召回、不进正文、不进证据；单独嵌入存侧表 memory_triggers。
+ * 2–5 条、各说一个角度、别互为变体、别复述正文（T-Mem 三条禁令）。
+ */
+export const MemoryTriggersSchema = z
+  .array(z.string().min(2).max(200))
+  .max(6)
+  .optional();
+
 export const StoreMemoryInputSchema = z.object({
   text: MemoryTextSchema,
   category: DurableMemoryCategorySchema.default("events"),
@@ -129,6 +139,7 @@ export const StoreMemoryInputSchema = z.object({
   canonicalKey: CanonicalKeySchema,
   topicTag: z.string().min(1).max(60).optional(),
   privacyTier: PrivacyTierSchema.default("durable"),
+  triggers: MemoryTriggersSchema,
 });
 
 // Keep in sync with RejectionReason in admission-control.ts

package/src/memory-store-port.ts

@@ -71,5 +71,6 @@ export interface MemoryStorePort {
     minScore?: number,
     scopeFilter?: string[],
     scopeMatch?: ScopeMatchMode,
+    excludeScopes?: string[],
   ): Promise<MemorySearchResult[]>;
 }

package/src/minis-archive.ts

@@ -0,0 +1,151 @@
+/**
+ * Minis 对话入库后的存档。
+ *
+ * 投递目录（config.sources.minis.path，iCloud 桌面上的 minis-outbox/ingest）里的 jsonl
+ * 一旦入库，就挪进 data/minis-archive：
+ * - 投递目录空了 = 都收到了。文件堆在桌面上时分不清哪些处理过（2026-09-11 Alice：「不然我会搞糊涂的」）。
+ * - 存档目录由 Deja 统一树读取（refresh-conversation-truth.py 的 claude 组，分组名 minis），
+ *   原文因此能在 Deja 里找到；recallnest 的 ingest 源清单里没有这个目录，
+ *   所以记忆库只收投递目录那一次，挂 minis: 前缀。
+ *
+ * 先复制、逐字节核对，再删原件，不直接 rename：投递目录是 iCloud 容器，
+ * 把文件 rename 到容器外实测会让它直接消失（2026-08-20 踩过，见 minis-inbox/00-KEEP-收工时导出对话.md）。
+ * 原件删掉后的副本就是存档本身（每日 restic 备份覆盖 ~/recallnest/data）。
+ *
+ * 只挪「台账里记着、大小和 mtime 都没变、且至少有一行可用对话」的文件：
+ * 还没入库的、入库后又被同名覆盖的留给下一轮；格式不合规的留在原处报错。
+ * 注意台账在某个 embedding 批次报错时也会记成已处理（ingestCCTranscripts 的既有行为），
+ * 那种文件同样会被挪走——原文在存档里，要补导就从存档拷回投递目录。
+ *
+ * 2026-10-06 起对话原文默认不再入库（env-config.ts `transcriptIngest`），上面说的「入库」这一步默认不跑：
+ * cli.ts 这时传进来的 `isIngested` 是 `settledCheck()`（文件至少 60 秒没被改过），于是规则变成
+ * 「放稳了、整个文件写完了（`isCompleteJsonl`）、且至少有一行可用对话的就挪」，其余不变
+ * （同名不同内容另存 -2、格式不合规的留在原处报错、先复制核对再删原件）。记忆库不再收 Minis 的对话，
+ * Deja 照旧从存档读。把入库重新打开（RECALLNEST_TRANSCRIPT_INGEST=on）就回到上面那套按台账挪的规则。
+ */
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  utimesSync,
+  writeFileSync,
+} from "node:fs";
+import { extname, join } from "node:path";
+
+export interface MinisArchiveResult {
+  /** 已挪进存档的目标路径 */
+  archived: string[];
+  /** 还没入库（或入库后又变了）的文件名，留在投递目录等下一轮 */
+  pending: string[];
+  /** 出错、原件留在原处的文件名和原因 */
+  errors: string[];
+}
+
+export type IngestedCheck = (filePath: string, size: number, mtimeMs: number) => boolean;
+
+/** 对话入库关着时，文件至少这么久没被改过才挪（毫秒） */
+export const MINIS_SETTLE_MS = 60_000;
+
+/**
+ * 对话入库关着时用的判据：文件已经至少 quietMs 没被改过。
+ * 入库开着的时候，一个文件要先被整个读完、切片、嵌入、记进台账，之后大小和 mtime 都没变才会被挪——
+ * 这段时间等于一个隐含的静置期，正在写的文件过不了。关掉入库后这段时间没有了，这里把它显式补回来：
+ * 刚落地、可能还在写或还在同步的文件留给下一轮（下一轮它的 mtime 自然就够老了）。
+ */
+export function settledCheck(quietMs: number = MINIS_SETTLE_MS, now: () => number = Date.now): IngestedCheck {
+  return (_filePath, _size, mtimeMs) => now() - mtimeMs >= quietMs;
+}
+
+/** 同名但内容不同（同一场对话后来又导出过一版）时另起 -2、-3，两版都留着；内容相同就复用。 */
+function pickTarget(archiveDir: string, name: string, bytes: Buffer): { path: string; alreadyThere: boolean } {
+  const ext = extname(name);
+  const stem = name.slice(0, name.length - ext.length);
+  for (let n = 1; ; n++) {
+    const candidate = join(archiveDir, n === 1 ? name : `${stem}-${n}${ext}`);
+    if (!existsSync(candidate)) return { path: candidate, alreadyThere: false };
+    if (readFileSync(candidate).equals(bytes)) return { path: candidate, alreadyThere: true };
+  }
+}
+
+/**
+ * 一个 jsonl 是不是写完了：以换行结尾，且每个非空行都解析得出 JSON。
+ * 对话入库关着时挪存档之前用它把关——解析对话的函数会跳过残缺行，一个只写了一半的文件照样「有可用的对话行」，
+ * 不看这一条的话，半截文件会被当成完整的收走、原件删掉。读不了（比如 iCloud 占位文件）时照旧抛错，由调用方记进 errors。
+ */
+export function isCompleteJsonl(filePath: string): boolean {
+  const text = readFileSync(filePath, "utf-8");
+  if (!text.endsWith("\n")) return false;
+  for (const line of text.split("\n")) {
+    if (!line.trim()) continue;
+    try { JSON.parse(line); } catch { return false; }
+  }
+  return true;
+}
+
+export function archiveIngestedMinisFiles(
+  dropDir: string,
+  archiveDir: string,
+  isIngested: IngestedCheck,
+  hasConversation: (filePath: string) => boolean = () => true,
+): MinisArchiveResult {
+  const result: MinisArchiveResult = { archived: [], pending: [], errors: [] };
+  if (!existsSync(dropDir)) return result;
+
+  const names = readdirSync(dropDir).filter((f) => f.endsWith(".jsonl")).sort();
+  for (const name of names) {
+    const source = join(dropDir, name);
+    try {
+      const before = statSync(source);
+      if (!before.isFile() || !isIngested(source, before.size, before.mtimeMs)) {
+        result.pending.push(name);
+        continue;
+      }
+
+      // 一行可用对话都没有的文件，台账也会记成已处理（0 chunks）。这种不收走，留在投递目录报错：
+      // 悄悄挪走的话，Minis 导出格式哪天变了，谁都看不见
+      if (!hasConversation(source)) {
+        result.errors.push(`${name}: 没有可用的对话行（格式不合规或还没写完），留在原处`);
+        continue;
+      }
+
+      // iCloud 占位文件（dataless）在这里会抛 EDEADLK，原件不动，下一轮再来
+      const bytes = readFileSync(source);
+      if (bytes.length !== before.size) {
+        result.errors.push(`${name}: 读取时文件在变（${before.size} → ${bytes.length} 字节）`);
+        continue;
+      }
+
+      mkdirSync(archiveDir, { recursive: true });
+      const target = pickTarget(archiveDir, name, bytes);
+      if (!target.alreadyThere) {
+        const partial = `${target.path}.partial`;
+        // 沿用原件权限：Minis 写出来是 600，默认 umask 会把副本放宽成 644
+        writeFileSync(partial, bytes, { mode: before.mode & 0o777 });
+        renameSync(partial, target.path);
+        // 存档沿用原件 mtime，Deja 里显示的时间才是对话时间而不是搬运时间
+        utimesSync(target.path, before.atime, before.mtime);
+        if (!readFileSync(target.path).equals(bytes)) {
+          result.errors.push(`${name}: 存档回读与原件不一致，原件保留`);
+          continue;
+        }
+      }
+
+      // 复制期间原件被同名覆盖（Minis 又导出了一版）：不删，下一轮先把新版入库
+      const after = statSync(source);
+      if (after.size !== before.size || after.mtimeMs !== before.mtimeMs) {
+        result.pending.push(name);
+        continue;
+      }
+
+      unlinkSync(source);
+      result.archived.push(target.path);
+    } catch (err) {
+      result.errors.push(`${name}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  return result;
+}

package/src/noise-filter.ts

@@ -65,6 +65,11 @@ const SHORT_BOILERPLATE_PATTERNS = [
 ];
 const BOILERPLATE_MAX_LENGTH = 10;
 
+// Denial / meta-question patterns describe a whole short utterance ("我没有相关记忆" /
+// "你还记得吗"), so they only apply up to this length. Unanchored substring matching on
+// long text drops real content — see the note in isNoise().
+const UTTERANCE_MAX_LENGTH = 200;
+
 // OpenClaw v3.2+ injected metadata headers (backport from v1.0.29)
 const METADATA_HEADER_PATTERNS = [
   /^Conversation info \(untrusted metadata\)/i,
@@ -150,11 +155,19 @@ export function isNoise(text: string, options: NoiseFilterOptions = {}): boolean
     return true;
   }
 
-  if (opts.filterDenials && DENIAL_PATTERNS.some(p => p.test(trimmed))) {
+  // 否认句 / 问记忆的问句说的是「一整句短话」（助手说没有相关记忆、用户问你还记得吗）。
+  // 不锚定的子串匹配放到长文本上会误伤正文：2026-09-11 改前改后全库对比，有 7 段文档导入块
+  // （983–1,487 字，4 段 open-loops.md、3 段 feedback/reference 规则文件——其中两段讲的恰是
+  // 「"我没找到"不等于"它不存在"」）只因含「我没找到」「我没有数据」被检索层整段丢掉；一条 500 字的
+  // store_memory 因为含「没有记忆实现」被写入闸拒掉。长文本里夹着这类子串，不再算噪声。
+  // （同一作用域另有约 245 段被 bridge 提示词规则判掉，全来自 session-index 回填文件，那组规则
+  //  故意不锚定、不适用长度门槛，不在这里处理。）
+  const isShortUtterance = trimmed.length <= UTTERANCE_MAX_LENGTH;
+  if (opts.filterDenials && isShortUtterance && DENIAL_PATTERNS.some(p => p.test(trimmed))) {
     logInfo(`[INFO] noise-filter: denial pattern matched: "${redactForLog(trimmed)}..."`);
     return true;
   }
-  if (opts.filterMetaQuestions && META_QUESTION_PATTERNS.some(p => p.test(trimmed))) {
+  if (opts.filterMetaQuestions && isShortUtterance && META_QUESTION_PATTERNS.some(p => p.test(trimmed))) {
     logInfo(`[INFO] noise-filter: meta-question filtered: "${redactForLog(trimmed)}..."`);
     return true;
   }

package/src/pii-detector.ts

@@ -48,6 +48,32 @@ interface PIIRule {
    *  true (scan still warns regardless). Keeps snowflake IDs / order numbers
    *  from being destructively rewritten. */
   validate?: (match: string) => boolean;
+  /** With `{ keepIdentifiers: true }`, leave matches this returns true for
+   *  intact — for fields that hold names (keys, tags, triggers), not prose. */
+  isIdentifier?: (match: string) => boolean;
+}
+
+export interface RedactOptions {
+  /**
+   * Skip matches that are ordinary dotted identifiers (`workflow.database.recovery`)
+   * rather than credentials. Off by default: memory text keeps the original rule
+   * set byte for byte, so deterministic ids of already-stored rows still match.
+   */
+  keepIdentifiers?: boolean;
+}
+
+/**
+ * A dot-separated name whose parts are plain words: lowercase, camelCase,
+ * PascalCase or all-caps letters, joined by `_`/`-`, plus short version-style
+ * numbers (`v2`, `2026`). Opaque token segments mix letters and digits or
+ * flip case mid-run, so they fail this.
+ */
+function isOrdinaryDottedIdentifier(match: string): boolean {
+  return match.split(".").every((segment) => {
+    const pieces = segment.split(/[_-]+/).filter(Boolean);
+    return pieces.length > 0 && pieces.every((piece) =>
+      /^(?:[A-Z]?[a-z]+(?:[A-Z][a-z]+)*|[A-Z]+|v?\d{1,4})$/.test(piece));
+  });
 }
 
 /** GB 11643 checksum (ISO 7064 MOD 11-2) for 18-digit Chinese national IDs. */
@@ -157,6 +183,7 @@ const PII_RULES: PIIRule[] = [
     severity: "high",
     pattern: /\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g,
     redact: true,
+    isIdentifier: isOrdinaryDottedIdentifier,
   },
   {
     type: "bearer_token",
@@ -265,7 +292,7 @@ export function scanForPII(text: string): PIIScanResult {
  * Replacement is `[REDACTED:<type>]`; assignment-style rules keep the key
  * name (`password=[REDACTED:password]`) so the memory stays readable.
  */
-export function redactSecrets(text: string): RedactResult {
+export function redactSecrets(text: string, options: RedactOptions = {}): RedactResult {
   let out = text;
   let redacted = 0;
 
@@ -276,6 +303,9 @@ export function redactSecrets(text: string): RedactResult {
       if (rule.validate && !rule.validate(match)) {
         return match; // checksum failed → leave intact (detection-only)
       }
+      if (options.keepIdentifiers && rule.isIdentifier?.(match)) {
+        return match;
+      }
       redacted++;
       if (rule.replacement) return rule.replacement(match);
       if (rule.keepPrefix) {

package/src/pivot-apply.ts

@@ -306,10 +306,9 @@ export async function persistPivotCandidate(
 
   // Post-write verification, synchronous on purpose: a bulk apply needs the
   // verdict inside the ledger row, not on a console later. Deliberately NOT
-  // verifyWrite(): its checkEntry expects scope/importance inside metadata,
-  // which buildStructuredMetadata-shaped rows never carry (a pre-existing
-  // mismatch its async caller silently swallows). This check asserts the claim
-  // that matters instead: the row at entry.id holds THIS candidate's content.
+  // verifyWrite(): that one only checks structural presence (vector, text,
+  // scope, importance, parseable metadata). This check asserts the claim that
+  // matters here: the row at entry.id holds THIS candidate's content.
   let verified = true;
   let verifyIssues: string[] = [];
   if ((disposition === "stored" || disposition === "updated") && deps.store.get) {

package/src/preference-matcher.ts

@@ -15,8 +15,8 @@ import type { MemoryStore, MemoryEntry } from "./store.js";
 import type { Embedder } from "./embedder.js";
 import type { LLMClient } from "./llm-client.js";
 import { logInfo } from "./stderr-log.js";
-import { isActiveMemory, parseEvolution } from "./memory-evolution.js";
-import { archiveBeliefVersion, buildSupersedingBeliefMetadata } from "./belief-history.js";
+import { isActiveMemory, parseEvolution, type EvolutionMetadata } from "./memory-evolution.js";
+import { archiveBeliefVersion, buildSupersedingBeliefMetadata, replaceBeliefVersion } from "./belief-history.js";
 import { detectLang, tokenizeFts } from "./language-hook.js";
 
 // ============================================================================
@@ -46,6 +46,8 @@ export interface PreferenceMatchResult {
   mergeTargetId?: string;
   /** Merged text (when action=merge, LLM-generated) */
   mergedText?: string;
+  /** The existing preference the incoming one duplicates (when action=skip) — reported back to the caller */
+  matchedEntry?: MemoryEntry;
   /** Reason for the decision */
   reason: string;
 }
@@ -61,7 +63,7 @@ export interface PreferenceMatchResult {
  * @param vector   Pre-computed embedding vector for the text
  * @param scope    The scope to search within
  * @param store    Memory store
- * @param llm      LLM client (null = always create)
+ * @param llm      LLM client (null → no LLM: top similarity ≥0.92 skips, otherwise create)
  * @param config   Optional config overrides
  */
 export async function matchPreference(
@@ -96,7 +98,7 @@ export async function matchPreference(
   if (!llm) {
     const topScore = prefMatches[0].score;
     if (topScore >= 0.92) {
-      return { action: "skip", reason: "high-similarity-no-llm" };
+      return { action: "skip", reason: "high-similarity-no-llm", matchedEntry: prefMatches[0].entry };
     }
     return { action: "create", reason: "moderate-similarity-no-llm" };
   }
@@ -110,6 +112,7 @@ export async function matchPreference(
       return {
         action: "skip",
         reason: `llm-skip: ${decision.reason}`,
+        matchedEntry: topMatch.entry,
       };
     }
 
@@ -144,7 +147,7 @@ export async function applyPreferenceMatch(
 ): Promise<{ handled: boolean; entry?: MemoryEntry }> {
   if (result.action === "skip") {
     logInfo(`[INFO] Preference skipped (duplicate): ${result.reason}`);
-    return { handled: true };
+    return result.matchedEntry ? { handled: true, entry: result.matchedEntry } : { handled: true };
   }
 
   if (result.action === "merge" && result.mergeTargetId && result.mergedText) {
@@ -153,31 +156,40 @@ export async function applyPreferenceMatch(
 
     // Merging rewrites the target's text, which is a belief change like any other —
     // archive the version being replaced before overwriting it.
-    const now = Date.now();
-    const previousEvo = parseEvolution(existing.metadata, existing.timestamp);
-    const { historyId } = await archiveBeliefVersion({ store, embedder: deps.embedder }, existing, { now });
-
     // The old code updated `text` alone, leaving vector and fts_text describing the
     // pre-merge wording — so a merged preference stayed unfindable by its own new text.
-    const language = detectLang(result.mergedText);
-    const updated = await store.update(
-      result.mergeTargetId,
-      {
-        text: result.mergedText,
-        vector: await deps.embedder.embedPassage(result.mergedText),
-        metadata: buildSupersedingBeliefMetadata(
-          existing.metadata,
-          previousEvo,
-          historyId,
-          now,
-          `preference-merge:${result.reason}`,
-        ),
-        timestamp: now,
-        language,
-        fts_text: tokenizeFts(result.mergedText, language),
-      },
-      [scope],
-    );
+    const mergedText = result.mergedText;
+    const reason = result.reason;
+    const language = detectLang(mergedText);
+    const vector = await deps.embedder.embedPassage(mergedText);
+    const buildUpdates = (metadata: string | undefined, previousEvo: EvolutionMetadata, historyId: string, now: number) => ({
+      text: mergedText,
+      vector,
+      metadata: buildSupersedingBeliefMetadata(metadata, previousEvo, historyId, now, `preference-merge:${reason}`),
+      timestamp: now,
+      language,
+      fts_text: tokenizeFts(mergedText, language),
+    });
+
+    let updated: MemoryEntry | null;
+    // Real MemoryStore: archive and overwrite under one store-write lock, row re-read inside it
+    // (2026-10-17). Target deleted since the lookup above → nothing written, not handled, and
+    // the caller creates the preference as new — previously a history row was left behind first.
+    if (typeof store.replaceWithArchivedVersion === "function") {
+      const replaced = await replaceBeliefVersion(
+        { store, embedder: deps.embedder },
+        existing,
+        ({ current, previousEvolution, historyId, now }) => buildUpdates(current.metadata, previousEvolution, historyId, now),
+        { scopeFilter: [scope] },
+      );
+      updated = replaced.status === "replaced" ? replaced.entry : null;
+    } else {
+      // Narrow store doubles: the two-write form.
+      const now = Date.now();
+      const previousEvo = parseEvolution(existing.metadata, existing.timestamp);
+      const { historyId } = await archiveBeliefVersion({ store, embedder: deps.embedder }, existing, { now });
+      updated = await store.update(result.mergeTargetId, buildUpdates(existing.metadata, previousEvo, historyId, now), [scope]);
+    }
     if (updated) {
       logInfo(`[INFO] Preference merged into ${result.mergeTargetId.slice(0, 8)}: ${result.reason}`);
       return { handled: true, entry: updated };

package/src/prospective-memory.ts

@@ -180,7 +180,9 @@ export async function fireReminder(
   prospective.firedAt = new Date().toISOString();
   meta.prospective = prospective;
 
-  await store.update(entryId, { metadata: JSON.stringify(meta) }, scopeFilter);
+  // 行在读到之后被删掉（update() 返回 null）就不算触发：不把一条已经不存在的提醒交给调用方
+  const marked = await store.update(entryId, { metadata: JSON.stringify(meta) }, scopeFilter);
+  if (!marked) return null;
 
   logInfo(`[INFO] Reminder fired: "${prospective.trigger}" → ${prospective.action}`);
 
@@ -298,10 +300,11 @@ export async function acceptPredictedReminder(
     (meta.boundary as Record<string, string>).authority = "user";
   }
 
-  await store.update(entryId, {
+  const promoted = await store.update(entryId, {
     metadata: JSON.stringify(meta),
     importance: 0.75, // Upgrade to explicit-level importance
   }, scopeFilter);
+  if (!promoted) return false; // 行已被删：没有接受成
 
   logInfo(`[INFO] Predicted reminder accepted and promoted: ${entryId}`);
   return true;
@@ -330,14 +333,14 @@ export async function demotePredictedReminder(
     prospective.status = "expired";
     prospective.confidence = newConfidence;
     meta.prospective = prospective;
-    await store.update(entryId, { metadata: JSON.stringify(meta) }, scopeFilter);
+    if (!await store.update(entryId, { metadata: JSON.stringify(meta) }, scopeFilter)) return false; // 行已被删
     logInfo(`[INFO] Predicted reminder demoted below threshold and expired: ${entryId}`);
   } else {
     // Still above threshold — just lower confidence
     prospective.confidence = newConfidence;
     prospective.lastSuggestedAt = new Date().toISOString();
     meta.prospective = prospective;
-    await store.update(entryId, { metadata: JSON.stringify(meta) }, scopeFilter);
+    if (!await store.update(entryId, { metadata: JSON.stringify(meta) }, scopeFilter)) return false; // 行已被删
     logInfo(`[INFO] Predicted reminder demoted: ${entryId} (new confidence: ${newConfidence})`);
   }