recallnest 2.3.0 → 2.5.4

package/src/pii-detector.ts

@@ -2,12 +2,21 @@
  * F-3: Lightweight PII detection.
  * Scans text for common sensitive patterns (passwords, tokens, IDs).
  * Returns warnings, does not block writes.
+ *
+ * F-3b (2026-07-10): Deterministic secret redaction on the same rule set.
+ * `redactSecrets` scrubs redactable matches BEFORE text reaches any external
+ * surface (LLM prompts, embedding API, stderr logs, audit trail). Detection
+ * and redaction share one rule table so the two can never drift apart.
+ * Deliberately NOT redacted: phone / email (legitimate memory content, warn
+ * only) and bare long-hex (would false-positive on git commit hashes, which
+ * are high-value memory content; prefixed forms like `secret=<hex>` are
+ * caught by the assignment rules).
  */
 
 export type PIISeverity = "high" | "medium" | "low";
 
 export interface PIIDetection {
-  type: string; // "api_key" | "password" | "id_number" | "email" | "phone" | "credit_card"
+  type: string; // "api_key" | "password" | "id_number" | "email" | "phone" | "credit_card" | ...
   severity: PIISeverity;
   match: string; // the matched text (partially masked)
   position: number; // char offset
@@ -19,42 +28,147 @@ export interface PIIScanResult {
   summary: string; // human-readable summary
 }
 
+export interface RedactResult {
+  text: string; // scrubbed text
+  redacted: number; // number of replacements made
+}
+
 interface PIIRule {
   type: string;
   severity: PIISeverity;
   pattern: RegExp;
+  /** Scrub matches via redactSecrets (detection-only rules leave text intact) */
+  redact: boolean;
+  /** For key=value style rules: keep the key name, scrub only the value */
+  keepPrefix?: RegExp;
+  /** Checksum gate for broad numeric patterns: redact ONLY when this returns
+   *  true (scan still warns regardless). Keeps snowflake IDs / order numbers
+   *  from being destructively rewritten. */
+  validate?: (match: string) => boolean;
+}
+
+/** GB 11643 checksum (ISO 7064 MOD 11-2) for 18-digit Chinese national IDs. */
+function isValidChineseId(id: string): boolean {
+  if (!/^\d{17}[\dXx]$/.test(id)) return false;
+  const weights = [7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2];
+  const checkChars = "10X98765432";
+  let sum = 0;
+  for (let i = 0; i < 17; i++) sum += Number(id[i]) * weights[i];
+  return checkChars[sum % 11] === id[17].toUpperCase();
+}
+
+/** Luhn checksum for card-number candidates (separators stripped first). */
+function isValidLuhn(candidate: string): boolean {
+  const digits = candidate.replace(/[\s-]/g, "");
+  if (!/^\d{16}$/.test(digits)) return false;
+  let sum = 0;
+  for (let i = 0; i < 16; i++) {
+    let d = Number(digits[15 - i]);
+    if (i % 2 === 1) {
+      d *= 2;
+      if (d > 9) d -= 9;
+    }
+    sum += d;
+  }
+  return sum % 10 === 0;
 }
 
 const PII_RULES: PIIRule[] = [
+  // --- Vendor-prefixed token literals (high confidence, full scrub) ---
+  {
+    type: "anthropic_key",
+    severity: "high",
+    pattern: /sk-ant-[A-Za-z0-9_\-]{10,}/g,
+    redact: true,
+  },
+  {
+    type: "github_token",
+    severity: "high",
+    pattern: /gh[pousr]_[A-Za-z0-9]{20,}/g,
+    redact: true,
+  },
+  {
+    type: "slack_token",
+    severity: "high",
+    pattern: /xox[baprs]-[A-Za-z0-9\-]{10,}/g,
+    redact: true,
+  },
+  {
+    type: "aws_access_key",
+    severity: "high",
+    pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
+    redact: true,
+  },
+  {
+    type: "google_api_key",
+    severity: "high",
+    pattern: /\bAIza[A-Za-z0-9_\-]{35}\b/g,
+    redact: true,
+  },
+  {
+    type: "jwt",
+    severity: "high",
+    pattern: /\beyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\b/g,
+    redact: true,
+  },
+  {
+    type: "bearer_token",
+    severity: "high",
+    pattern: /\bBearer\s+[A-Za-z0-9._~+\/\-]{16,}=*/g,
+    redact: true,
+  },
+  {
+    type: "pem_private_key",
+    severity: "high",
+    pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
+    redact: true,
+  },
+  // --- Generic assignment / key styles (keep key name, scrub value) ---
+  // Quoted keys and whitespace around the separator are allowed so JSON/YAML/
+  // config forms ({"api_key": "..."}, password = ...) are caught too.
   {
     type: "api_key",
     severity: "high",
-    pattern: /(?:sk-[A-Za-z0-9_\-]{20,}|(?:api[_-]?key|token|secret)[=:\s]["']?[A-Za-z0-9_\-]{20,})/gi,
+    pattern: /(?:sk-[A-Za-z0-9_\-]{20,}|["']?(?:api[_-]?key|token|secret)["']?\s*[=:]\s*["']?[A-Za-z0-9_\-]{20,})/gi,
+    redact: true,
+    keepPrefix: /^["']?(?:api[_-]?key|token|secret)["']?\s*[=:]\s*["']?/i,
   },
   {
     type: "password",
     severity: "high",
-    pattern: /(?:password|passwd|pwd)[=:\s]["']?[^\s"']{8,}/gi,
+    pattern: /["']?(?:password|passwd|pwd)["']?\s*[=:]\s*["']?[^\s"',;]{8,}/gi,
+    redact: true,
+    keepPrefix: /^["']?(?:password|passwd|pwd)["']?\s*[=:]\s*["']?/i,
   },
+  // --- High-risk personal identifiers ---
+  // Broad numeric patterns scan-warn on shape but only REDACT when the
+  // checksum verifies — otherwise snowflake IDs / order numbers get rewritten.
   {
     type: "id_number",
     severity: "high",
-    pattern: /\d{17}[\dXx]/g,
+    pattern: /(?<!\d)\d{17}[\dXx](?!\d)/g,
+    redact: true,
+    validate: isValidChineseId,
   },
   {
     type: "credit_card",
     severity: "high",
-    pattern: /\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}/g,
+    pattern: /(?<!\d)\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}(?![\d])/g,
+    redact: true,
+    validate: isValidLuhn,
   },
+  // --- Detection-only: legitimate memory content, warn but keep ---
   {
     type: "phone",
     severity: "medium",
     pattern: /1[3-9]\d{9}/g,
+    redact: false,
   },
   {
     type: "email",
     severity: "low",
     pattern: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
+    redact: false,
   },
 ];
 
@@ -96,3 +210,41 @@ export function scanForPII(text: string): PIIScanResult {
     summary,
   };
 }
+
+/**
+ * Deterministically scrub redactable secrets. Pure regex, no LLM call.
+ * Replacement is `[REDACTED:<type>]`; assignment-style rules keep the key
+ * name (`password=[REDACTED:password]`) so the memory stays readable.
+ */
+export function redactSecrets(text: string): RedactResult {
+  let out = text;
+  let redacted = 0;
+
+  for (const rule of PII_RULES) {
+    if (!rule.redact) continue;
+    rule.pattern.lastIndex = 0;
+    out = out.replace(rule.pattern, (match) => {
+      if (rule.validate && !rule.validate(match)) {
+        return match; // checksum failed → leave intact (detection-only)
+      }
+      redacted++;
+      if (rule.keepPrefix) {
+        const prefixMatch = match.match(rule.keepPrefix);
+        if (prefixMatch) {
+          return `${prefixMatch[0]}[REDACTED:${rule.type}]`;
+        }
+      }
+      return `[REDACTED:${rule.type}]`;
+    });
+  }
+
+  return { text: out, redacted };
+}
+
+/**
+ * Redact + truncate for log lines. Truncation happens AFTER redaction so a
+ * secret can never straddle the cut and leak its prefix.
+ */
+export function redactForLog(text: string, max = 60): string {
+  return redactSecrets(text).text.slice(0, max);
+}

package/src/query-expander.ts

@@ -10,8 +10,8 @@
  * v1.3: P0.3 — User-level alias-map (data/alias-map.json) for short query expansion.
  */
 
-import { existsSync, readFileSync } from "fs";
-import { join } from "path";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { dirname, join } from "path";
 import { resolveQueryEntities } from "./entity-resolver.js";
 
 interface SynonymEntry {
@@ -89,9 +89,14 @@ export function resetAliasMapCache(): void {
   aliasMapCache = null;
 }
 
+/** Resolve the alias-map file path currently in effect. */
+export function aliasMapFilePath(): string {
+  return aliasMapPath ?? join(process.cwd(), "data", "alias-map.json");
+}
+
 function loadAliasMap(): AliasEntry[] {
   if (aliasMapCache !== null) return aliasMapCache;
-  const filePath = aliasMapPath ?? join(process.cwd(), "data", "alias-map.json");
+  const filePath = aliasMapFilePath();
   try {
     if (existsSync(filePath)) {
       const raw = JSON.parse(readFileSync(filePath, "utf-8"));
@@ -112,6 +117,75 @@ function loadAliasMap(): AliasEntry[] {
   return aliasMapCache;
 }
 
+// ---------------------------------------------------------------------------
+// User alias management (P1-B manage_alias tool 后端)
+//
+// 这些别名只作用于 BM25 通道(expandQuery 在 hybridRetrieval 内部被调用),
+// 刻意不进 vector embedding——随手加的用户别名若改变语义向量,排查
+// retrieval drift 会非常困难。全通道的 builtin 规则在 src/aliases.ts。
+// ---------------------------------------------------------------------------
+
+/** 写入校验:trigger 最短 2 字符(防高频泛词),expansions 最多 8 个。 */
+export const USER_ALIAS_TRIGGER_MIN = 2;
+export const USER_ALIAS_EXPANSIONS_MAX = 8;
+
+export type AliasUpsertResult =
+  | { ok: true; action: "added" | "updated"; entry: AliasEntry }
+  | { ok: false; reason: string };
+
+/** 新增/更新用户别名(写 alias-map.json;校验后缓存即时失效)。 */
+export function upsertUserAlias(trigger: string, expansions: string[]): AliasUpsertResult {
+  const t = trigger.trim();
+  const exp = [...new Set(expansions.map(e => e.trim()).filter(e => e.length > 0))];
+
+  if (t.length < USER_ALIAS_TRIGGER_MIN) {
+    return { ok: false, reason: `trigger 太短(<${USER_ALIAS_TRIGGER_MIN} 字符)——高频泛词会污染所有查询` };
+  }
+  if (exp.length === 0) {
+    return { ok: false, reason: "expansions 为空" };
+  }
+  if (exp.length > USER_ALIAS_EXPANSIONS_MAX) {
+    return { ok: false, reason: `expansions 超过 ${USER_ALIAS_EXPANSIONS_MAX} 个——过长扩展会冲淡 BM25 原查询词权重` };
+  }
+  if (exp.every(e => e === t)) {
+    return { ok: false, reason: "expansions 与 trigger 相同,无扩展意义" };
+  }
+
+  const rules = loadAliasMap();
+  const idx = rules.findIndex(r => r.trigger === t);
+  const entry: AliasEntry = { trigger: t, expansions: exp };
+  const next = idx >= 0 ? rules.map((r, i) => (i === idx ? entry : r)) : [...rules, entry];
+
+  const path = aliasMapFilePath();
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, JSON.stringify(next, null, 2) + "\n", "utf-8");
+  aliasMapCache = null;
+  return { ok: true, action: idx >= 0 ? "updated" : "added", entry };
+}
+
+/** 删除用户别名;返回是否真的删了。 */
+export function removeUserAlias(trigger: string): boolean {
+  const rules = loadAliasMap();
+  const next = rules.filter(r => r.trigger !== trigger.trim());
+  if (next.length === rules.length) return false;
+  const path = aliasMapFilePath();
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, JSON.stringify(next, null, 2) + "\n", "utf-8");
+  aliasMapCache = null;
+  return true;
+}
+
+/** 列出用户别名规则。 */
+export function listUserAliases(): AliasEntry[] {
+  return [...loadAliasMap()];
+}
+
+/** 检视:query 命中了哪些用户别名(BM25-only 通道)。 */
+export function explainUserAliases(query: string): AliasEntry[] {
+  const lower = query.toLowerCase();
+  return loadAliasMap().filter(e => lower.includes(e.trigger.toLowerCase()));
+}
+
 /**
  * Expand a query by appending synonym terms from the dictionary.
  * Returns the original query with additional terms appended.
@@ -123,7 +197,8 @@ export function expandQuery(query: string): string {
   // Entity resolution pre-pass: normalize aliases before synonym expansion
   const resolved = resolveQueryEntities(query);
   const lower = resolved.toLowerCase();
-  const additions = new Set<string>();
+  const synonymAdditions = new Set<string>();
+  const aliasAdditions = new Set<string>();
 
   // Static synonym map (built-in)
   for (const entry of SYNONYM_MAP) {
@@ -138,7 +213,7 @@ export function expandQuery(query: string): string {
     if (cnMatch || enMatch) {
       for (const exp of entry.expansions) {
         if (!lower.includes(exp.toLowerCase())) {
-          additions.add(exp);
+          synonymAdditions.add(exp);
         }
       }
     }
@@ -151,15 +226,21 @@ export function expandQuery(query: string): string {
     if (lower.includes(triggerLower)) {
       for (const exp of alias.expansions) {
         if (!lower.includes(exp.toLowerCase())) {
-          additions.add(exp);
+          aliasAdditions.add(exp);
         }
       }
     }
   }
 
-  if (additions.size === 0) return resolved;
+  if (aliasAdditions.size === 0 && synonymAdditions.size === 0) return resolved;
 
-  // Cap expansion terms to prevent query bloat
-  const limited = [...additions].slice(0, MAX_EXPANSION_TERMS);
+  // Cap expansion terms to prevent query bloat. User aliases rank first inside
+  // the cap — 用户为这个 trigger 显式配置的 canonical 名,不应被泛化同义词
+  // 挤出 MAX_EXPANSION_TERMS(P1-B:同义词"挂"曾把 alias 的实体名挤掉)。
+  const merged = [
+    ...aliasAdditions,
+    ...[...synonymAdditions].filter(t => !aliasAdditions.has(t)),
+  ];
+  const limited = merged.slice(0, MAX_EXPANSION_TERMS);
   return `${resolved} ${limited.join(" ")}`;
 }

package/src/result-synthesizer.ts

@@ -12,6 +12,7 @@
 
 import type { LLMClient } from "./llm-client.js";
 import { logWarn } from "./stderr-log.js";
+import * as envConfig from "./env-config.js";
 
 // ============================================================================
 // Configuration
@@ -38,7 +39,7 @@ export const DEFAULT_SYNTHESIZER_CONFIG: SynthesizerConfig = {
  * Check if synthesis is enabled via environment variable.
  */
 export function isSynthesisEnabled(): boolean {
-  return process.env.RECALLNEST_SYNTHESIZE === "true";
+  return envConfig.synthesize();
 }
 
 /**

package/src/retention-policy.ts

@@ -7,6 +7,7 @@
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
+import * as envConfig from "./env-config.js";
 
 export interface RetentionPolicy {
   /** Auto-archive memories older than this many days (0 = disabled) */
@@ -29,7 +30,7 @@ function scopeHash(scope: string): string {
 
 function retentionDir(configDir?: string): string {
   return join(
-    configDir ?? (process.env.RECALLNEST_DATA_DIR || "data"),
+    configDir ?? envConfig.dataDir(),
     "retention",
   );
 }