recallnest 1.4.1 → 2.2.0

package/src/__tests__/write-verifier.test.ts

@@ -0,0 +1,122 @@
+import { describe, it, expect } from "bun:test";
+import { verifyWrite } from "../write-verifier.js";
+import type { MemoryEntry } from "../store.js";
+
+function mockStore(entry: MemoryEntry | null) {
+  return {
+    async get(_id: string) {
+      return entry;
+    },
+  };
+}
+
+function makeEntry(overrides: Partial<MemoryEntry> = {}): MemoryEntry {
+  return {
+    id: "test-id",
+    text: "User prefers dark mode",
+    vector: [0.1, 0.2, 0.3],
+    category: "preferences",
+    importance: 0.75,
+    timestamp: Date.now(),
+    metadata: JSON.stringify({ scope: "project:test", importance: 0.75 }),
+    ...overrides,
+  };
+}
+
+describe("HP-2: write-verifier", () => {
+  it("passes for a complete entry", async () => {
+    const result = await verifyWrite(mockStore(makeEntry()), "test-id");
+    expect(result.ok).toBe(true);
+    expect(result.issues).toEqual([]);
+    expect(result.durationMs).toBeGreaterThanOrEqual(0);
+  });
+
+  it("detects missing entry", async () => {
+    const result = await verifyWrite(mockStore(null), "missing-id");
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("missing_entry");
+  });
+
+  it("detects empty vector", async () => {
+    const result = await verifyWrite(
+      mockStore(makeEntry({ vector: [] })),
+      "test-id",
+    );
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("missing_vector");
+  });
+
+  it("detects empty text", async () => {
+    const result = await verifyWrite(
+      mockStore(makeEntry({ text: "  " })),
+      "test-id",
+    );
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("empty_text");
+  });
+
+  it("detects missing scope in metadata", async () => {
+    const result = await verifyWrite(
+      mockStore(makeEntry({ metadata: JSON.stringify({ importance: 0.7 }) })),
+      "test-id",
+    );
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("missing_scope");
+  });
+
+  it("detects missing importance in metadata", async () => {
+    const result = await verifyWrite(
+      mockStore(makeEntry({ metadata: JSON.stringify({ scope: "test" }) })),
+      "test-id",
+    );
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("missing_importance");
+  });
+
+  it("detects corrupt metadata", async () => {
+    const result = await verifyWrite(
+      mockStore(makeEntry({ metadata: "not-json" })),
+      "test-id",
+    );
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("missing_scope");
+  });
+
+  it("respects enabled=false config", async () => {
+    const result = await verifyWrite(mockStore(null), "any", { enabled: false });
+    expect(result.ok).toBe(true);
+    expect(result.durationMs).toBe(0);
+  });
+
+  it("handles store.get timeout gracefully", async () => {
+    const slowStore = {
+      async get() {
+        await new Promise(r => setTimeout(r, 5000));
+        return null;
+      },
+    };
+    const result = await verifyWrite(slowStore, "slow-id", { timeoutMs: 50 });
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("missing_entry");
+  });
+
+  it("handles store.get throwing", async () => {
+    const errStore = {
+      async get() {
+        throw new Error("db error");
+      },
+    };
+    const result = await verifyWrite(errStore, "err-id");
+    expect(result.ok).toBe(false);
+    expect(result.issues).toContain("missing_entry");
+  });
+
+  it("reports multiple issues at once", async () => {
+    const result = await verifyWrite(
+      mockStore(makeEntry({ text: "", vector: [], metadata: "{}" })),
+      "test-id",
+    );
+    expect(result.ok).toBe(false);
+    expect(result.issues.length).toBeGreaterThanOrEqual(3);
+  });
+});

package/src/access-tracker.ts

@@ -95,10 +95,11 @@ export class AccessTracker {
    * Accumulates deltas in memory; flushed to store after debounce.
    *
    * @param ids  Memory entry IDs to record access for
-   * @param scores  Optional parallel array of similarity scores for novelty gating.
+   * @param scores  Optional parallel array of raw similarity scores for novelty gating.
+   *                Entries with an undefined score skip novelty gating.
    *                When provided, only entries passing shouldReinforce() are recorded.
    */
-  recordAccess(ids: string[], scores?: number[]): void {
+  recordAccess(ids: string[], scores?: Array<number | undefined>): void {
     if (ids.length === 0) return;
 
     const now = Date.now();

package/src/activity-counter.ts

@@ -0,0 +1,109 @@
+/**
+ * HP-3: Activity-driven distill frequency.
+ * Tracks write operations since last distill, exposes tier-based thresholds.
+ * Complements CC-6 distill-lock: HP-3 decides "when to trigger", CC-6 decides "whether to allow".
+ */
+
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  mkdirSync,
+} from "node:fs";
+import { join, dirname } from "node:path";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface ActivityCounterConfig {
+  /** Path to the stats file (default: data/activity-stats.json) */
+  statsPath: string;
+  /** Writes needed for light scoring/tagging pass */
+  lightThreshold: number;
+  /** Writes needed for standard distill */
+  standardThreshold: number;
+  /** Writes needed for deep checkpoint */
+  deepThreshold: number;
+}
+
+export const DEFAULT_ACTIVITY_CONFIG: ActivityCounterConfig = {
+  statsPath: join(
+    process.env.RECALLNEST_DATA_DIR || "data",
+    "activity-stats.json",
+  ),
+  lightThreshold: 15,
+  standardThreshold: 50,
+  deepThreshold: 200,
+};
+
+export type DistillTier = "none" | "light" | "standard" | "deep";
+
+// ---------------------------------------------------------------------------
+// Internal
+// ---------------------------------------------------------------------------
+
+interface ActivityStats {
+  writesSinceLastDistill: number;
+  lastResetAt: number;
+}
+
+function resolveConfig(
+  cfg?: Partial<ActivityCounterConfig>,
+): ActivityCounterConfig {
+  return { ...DEFAULT_ACTIVITY_CONFIG, ...cfg };
+}
+
+function readStats(statsPath: string): ActivityStats {
+  if (!existsSync(statsPath)) {
+    return { writesSinceLastDistill: 0, lastResetAt: Date.now() };
+  }
+  try {
+    return JSON.parse(readFileSync(statsPath, "utf-8"));
+  } catch {
+    return { writesSinceLastDistill: 0, lastResetAt: Date.now() };
+  }
+}
+
+function writeStats(statsPath: string, stats: ActivityStats): void {
+  mkdirSync(dirname(statsPath), { recursive: true });
+  writeFileSync(statsPath, JSON.stringify(stats, null, 2), "utf-8");
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/** Increment write counter by n (default 1). Returns new count. */
+export function incrementWriteCount(
+  n = 1,
+  cfg?: Partial<ActivityCounterConfig>,
+): number {
+  const { statsPath } = resolveConfig(cfg);
+  const stats = readStats(statsPath);
+  stats.writesSinceLastDistill += n;
+  writeStats(statsPath, stats);
+  return stats.writesSinceLastDistill;
+}
+
+/** Read current write count without modifying. */
+export function getWriteCount(cfg?: Partial<ActivityCounterConfig>): number {
+  const { statsPath } = resolveConfig(cfg);
+  return readStats(statsPath).writesSinceLastDistill;
+}
+
+/** Reset counter after successful distill. */
+export function resetWriteCount(cfg?: Partial<ActivityCounterConfig>): void {
+  const { statsPath } = resolveConfig(cfg);
+  writeStats(statsPath, { writesSinceLastDistill: 0, lastResetAt: Date.now() });
+}
+
+/** Determine which distill tier the current write count warrants. */
+export function getDistillTier(cfg?: Partial<ActivityCounterConfig>): DistillTier {
+  const config = resolveConfig(cfg);
+  const count = getWriteCount(cfg);
+  if (count >= config.deepThreshold) return "deep";
+  if (count >= config.standardThreshold) return "standard";
+  if (count >= config.lightThreshold) return "light";
+  return "none";
+}

package/src/adaptive-retrieval.ts

@@ -44,15 +44,16 @@ export function shouldSkipRetrieval(query: string, minLength?: number): boolean
   // Force retrieve if query has memory-related intent (BEFORE length check)
   if (FORCE_RETRIEVE_PATTERNS.some(p => p.test(trimmed))) return false;
 
-  // Very short = skip
-  if (trimmed.length < 4) return true;
+  // CJK-aware: Chinese chars carry much more semantic density than ASCII.
+  // 2 Chinese chars ("轮巡") is a meaningful query equivalent to "daily patrol".
+  const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]/.test(trimmed);
+  const absoluteMin = hasCJK ? 2 : 4;
+  if (trimmed.length < absoluteMin) return true;
 
   // Pattern-based skip
   if (SKIP_PATTERNS.some(p => p.test(trimmed))) return true;
 
-  // CJK-aware minimum length: Chinese chars carry more semantic density
-  const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]/.test(trimmed);
-  const effectiveMinLength = minLength ?? (hasCJK ? 6 : 15);
+  const effectiveMinLength = minLength ?? (hasCJK ? 2 : 15);
 
   // Short non-question messages are skipped; questions (? ？) are always worth checking
   if (trimmed.length < effectiveMinLength &&

package/src/admission-control.ts

@@ -0,0 +1,163 @@
+/**
+ * LME-8: Admission Control — 写入准入控制
+ *
+ * 在 capture-engine.ts 的 persistMemory 入口处执行写前门控，
+ * 拦截低质量记忆，避免垃圾进库再过滤。
+ *
+ * 4 层准入检查：
+ *   1. 最短文本长度 — 过短文本拒绝
+ *   2. Noise filter — 噪声模式前置过滤
+ *   3. Importance 下限 — 低 importance 拒绝
+ *   4. Scope 写入频率限制 — 防 flood
+ */
+
+import { isNoise } from "./noise-filter.js";
+import { logInfo } from "./stderr-log.js";
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+export interface AdmissionConfig {
+  /** Minimum text length to accept (default: 10) */
+  minTextLength: number;
+  /** Enable noise-filter pre-check (default: true) */
+  noiseFilterEnabled: boolean;
+  /** Minimum importance to accept (default: 0.2) */
+  minImportance: number;
+  /** Max writes per scope within the rate window (default: 50) */
+  maxWritesPerScope: number;
+  /** Rate limit window in milliseconds (default: 60_000 = 1 minute) */
+  rateLimitWindowMs: number;
+}
+
+const DEFAULT_CONFIG: AdmissionConfig = {
+  minTextLength: 10,
+  noiseFilterEnabled: true,
+  minImportance: 0.2,
+  maxWritesPerScope: 50,
+  rateLimitWindowMs: 60_000,
+};
+
+export function resolveAdmissionConfig(
+  overrides?: Partial<AdmissionConfig>,
+): AdmissionConfig {
+  return { ...DEFAULT_CONFIG, ...overrides };
+}
+
+// ============================================================================
+// Admission Result
+// ============================================================================
+
+export type AdmissionVerdict = "accepted" | "rejected";
+export type RejectionReason =
+  | "text_too_short"
+  | "noise_detected"
+  | "importance_too_low"
+  | "rate_limited";
+
+export interface AdmissionResult {
+  verdict: AdmissionVerdict;
+  reason?: RejectionReason;
+}
+
+const ACCEPTED: AdmissionResult = { verdict: "accepted" };
+
+// ============================================================================
+// Rate Limiter — per-scope write frequency tracking
+// ============================================================================
+
+export class ScopeRateLimiter {
+  private readonly windows = new Map<string, number[]>();
+
+  /**
+   * Record a write for a scope and check if rate limit is exceeded.
+   * Returns true if the write should be allowed.
+   */
+  check(scope: string, config: AdmissionConfig): boolean {
+    const now = Date.now();
+    const cutoff = now - config.rateLimitWindowMs;
+
+    let timestamps = this.windows.get(scope);
+    if (!timestamps) {
+      timestamps = [];
+      this.windows.set(scope, timestamps);
+    }
+
+    // Prune expired timestamps
+    const firstValid = timestamps.findIndex((t) => t >= cutoff);
+    if (firstValid > 0) {
+      timestamps.splice(0, firstValid);
+    } else if (firstValid === -1) {
+      timestamps.length = 0;
+    }
+
+    if (timestamps.length >= config.maxWritesPerScope) {
+      return false;
+    }
+
+    timestamps.push(now);
+    return true;
+  }
+
+  /** Number of scopes currently tracked. */
+  get size(): number {
+    return this.windows.size;
+  }
+
+  /** Clear all tracking state. */
+  clear(): void {
+    this.windows.clear();
+  }
+}
+
+// ============================================================================
+// Public API — admission gate
+// ============================================================================
+
+/**
+ * Check whether a memory write should be admitted.
+ * Call this at the top of persistMemory, before embedding.
+ *
+ * @param text       - Memory text to evaluate
+ * @param importance - Importance score (0–1)
+ * @param scope      - Target scope
+ * @param rateLimiter - Optional shared rate limiter instance
+ * @param config     - Optional config overrides
+ */
+export function checkAdmission(
+  text: string,
+  importance: number,
+  scope: string,
+  rateLimiter?: ScopeRateLimiter,
+  config?: Partial<AdmissionConfig>,
+): AdmissionResult {
+  const cfg = resolveAdmissionConfig(config);
+
+  // Layer 1: minimum text length
+  const trimmed = text.trim();
+  if (trimmed.length < cfg.minTextLength) {
+    logInfo(`[admission-control] rejected: text too short (${trimmed.length} < ${cfg.minTextLength})`);
+    return { verdict: "rejected", reason: "text_too_short" };
+  }
+
+  // Layer 2: noise filter pre-check
+  if (cfg.noiseFilterEnabled && isNoise(trimmed)) {
+    logInfo(`[admission-control] rejected: noise detected`);
+    return { verdict: "rejected", reason: "noise_detected" };
+  }
+
+  // Layer 3: importance floor
+  if (importance < cfg.minImportance) {
+    logInfo(`[admission-control] rejected: importance too low (${importance} < ${cfg.minImportance})`);
+    return { verdict: "rejected", reason: "importance_too_low" };
+  }
+
+  // Layer 4: scope rate limiting
+  if (rateLimiter && !rateLimiter.check(scope, cfg)) {
+    logInfo(`[admission-control] rejected: rate limited for scope "${scope}"`);
+    return { verdict: "rejected", reason: "rate_limited" };
+  }
+
+  return ACCEPTED;
+}

package/src/anchor-generator.ts

@@ -0,0 +1,80 @@
+/**
+ * P0.1 — Anchor Generator
+ *
+ * Generates a short (≤80 chars) retrieval anchor for each memory.
+ * Short anchors bridge the embedding distance gap between brief queries
+ * (e.g. "轮巡") and long stored documents.
+ *
+ * Heuristic-only — no LLM calls, no latency added to writes.
+ */
+
+const MAX_ANCHOR_LENGTH = 80;
+
+/** CJK character detector */
+const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf]/;
+
+/** Extract first meaningful sentence (period, or newline). */
+function firstSentence(text: string): string {
+  // Split on newline first (preserves paragraph structure)
+  const firstLine = text.split(/\n/)[0].trim();
+
+  // Then split on sentence boundaries within that line
+  const match = firstLine.match(/^(.+?)[。！？.!?]/);
+  return match ? match[1].trim() : firstLine;
+}
+
+/** Extract structured title from metadata (patterns, cases). */
+function extractTitle(metadata?: Record<string, unknown>): string | undefined {
+  if (!metadata) return undefined;
+  const wp = metadata.workflowPattern as Record<string, unknown> | undefined;
+  if (wp?.title && typeof wp.title === "string") return wp.title;
+  const cm = metadata.caseMemory as Record<string, unknown> | undefined;
+  if (cm?.title && typeof cm.title === "string") return cm.title;
+  return undefined;
+}
+
+/**
+ * Generate a retrieval anchor (≤80 chars) for a memory entry.
+ *
+ * Priority:
+ * 1. Structured title from metadata (patterns/cases always have one)
+ * 2. First sentence of text (heuristic)
+ *
+ * If the text is already short (≤ MAX_ANCHOR_LENGTH), returns undefined
+ * (no anchor needed — the text itself is a good match target).
+ */
+export function generateAnchor(
+  text: string,
+  metadata?: Record<string, unknown>,
+): string | undefined {
+  // Short text already matches short queries well — skip anchor
+  if (text.length <= MAX_ANCHOR_LENGTH) return undefined;
+
+  // Try structured title first (most precise)
+  const title = extractTitle(metadata);
+  if (title && title.length <= MAX_ANCHOR_LENGTH) {
+    return title;
+  }
+
+  // Fall back to first sentence
+  let anchor = firstSentence(text);
+
+  // Truncate to max length, preserving word/char boundary
+  if (anchor.length > MAX_ANCHOR_LENGTH) {
+    if (CJK_RE.test(anchor)) {
+      // CJK: cut at char boundary
+      anchor = anchor.slice(0, MAX_ANCHOR_LENGTH);
+    } else {
+      // Latin: cut at word boundary
+      anchor = anchor.slice(0, MAX_ANCHOR_LENGTH).replace(/\s+\S*$/, "");
+    }
+  }
+
+  // If anchor is too similar to full text start, it adds no value
+  if (anchor === text.slice(0, anchor.length)) {
+    // Still useful — the shorter embedding will be closer to short queries
+    return anchor;
+  }
+
+  return anchor || undefined;
+}

package/src/api-server.ts

@@ -24,6 +24,7 @@ import { WorkflowObservationStore } from "./workflow-observation-store.js";
 import { buildManagedCheckpointObservation, buildManagedResumeObservation } from "./workflow-observation-managed.js";
 import { runAutoRecall } from "./auto-recall.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
+import { runMemoryLint } from "./memory-lint.js";
 
 const config = (loadDotEnv(), loadConfig());
 const getComponents = createComponentResolver(config);
@@ -589,6 +590,14 @@ async function handleStats(): Promise<Response> {
   });
 }
 
+/** GET /v1/lint — memory quality lint */
+async function handleLint(searchParams: URLSearchParams): Promise<Response> {
+  const { store } = getComponents();
+  const scope = searchParams.get("scope") || undefined;
+  const report = await runMemoryLint({ store, scope });
+  return jsonResponse(report);
+}
+
 /** GET /v1/health — health check */
 async function handleHealth(): Promise<Response> {
   try {
@@ -652,6 +661,7 @@ const server = Bun.serve({
         if (pathname === "/v1/workflow-health") return await handleWorkflowHealth(request);
         if (pathname === "/v1/workflow-evidence") return await handleWorkflowEvidence(request);
         if (pathname === "/v1/stats") return await handleStats();
+        if (pathname === "/v1/lint") return await handleLint(url.searchParams);
         if (pathname === "/v1/health") return await handleHealth();
       }
 

package/src/audit-log.ts

@@ -0,0 +1,122 @@
+/**
+ * F-1: Audit log — record every store/update/delete/retrieve operation.
+ * Logs are written to a separate file (not mixed into memory data).
+ *
+ * Each log entry: { timestamp, operation, scope, memoryId, actor, details }
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+export type AuditOperation =
+  | "store"
+  | "update"
+  | "delete"
+  | "retrieve"
+  | "archive"
+  | "supersede"
+  | "consolidate"
+  | "forget"
+  | "cascade_forget";
+
+export interface AuditEntry {
+  timestamp: string; // ISO 8601
+  operation: AuditOperation;
+  scope?: string;
+  memoryId?: string;
+  actor: string; // "manual" | "agent" | "api" | "system"
+  details?: string; // brief context (<=200 chars)
+}
+
+export interface AuditLogger {
+  log(entry: Omit<AuditEntry, "timestamp">): void;
+  /** Read recent entries (newest first) */
+  getRecent(limit?: number): AuditEntry[];
+  /** Export all entries as JSON array */
+  exportAll(): AuditEntry[];
+  /** Count total entries */
+  count(): number;
+}
+
+function parseLines(raw: string): AuditEntry[] {
+  const entries: AuditEntry[] = [];
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      entries.push(JSON.parse(trimmed) as AuditEntry);
+    } catch {
+      // skip malformed lines
+    }
+  }
+  return entries;
+}
+
+/**
+ * File-based audit logger. Appends JSON lines to a log file.
+ * Non-blocking: write failures are silently ignored (audit must never block operations).
+ */
+export function createAuditLogger(logPath?: string): AuditLogger {
+  const resolvedPath =
+    logPath ??
+    join(process.env.RECALLNEST_DATA_DIR || "data", "audit.jsonl");
+
+  // Ensure directory exists (silent on failure)
+  try {
+    const dir = dirname(resolvedPath);
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
+  } catch {
+    // silent — audit must never block
+  }
+
+  return {
+    log(entry: Omit<AuditEntry, "timestamp">): void {
+      try {
+        const full: AuditEntry = {
+          timestamp: new Date().toISOString(),
+          ...entry,
+        };
+        // Truncate details to 200 chars
+        if (full.details && full.details.length > 200) {
+          full.details = full.details.slice(0, 200);
+        }
+        appendFileSync(resolvedPath, JSON.stringify(full) + "\n", "utf-8");
+      } catch {
+        // silent — audit must never block operations
+      }
+    },
+
+    getRecent(limit = 20): AuditEntry[] {
+      try {
+        if (!existsSync(resolvedPath)) return [];
+        const raw = readFileSync(resolvedPath, "utf-8");
+        const all = parseLines(raw);
+        return all.slice(-limit).reverse();
+      } catch {
+        return [];
+      }
+    },
+
+    exportAll(): AuditEntry[] {
+      try {
+        if (!existsSync(resolvedPath)) return [];
+        const raw = readFileSync(resolvedPath, "utf-8");
+        return parseLines(raw);
+      } catch {
+        return [];
+      }
+    },
+
+    count(): number {
+      try {
+        if (!existsSync(resolvedPath)) return 0;
+        const raw = readFileSync(resolvedPath, "utf-8");
+        return parseLines(raw).length;
+      } catch {
+        return 0;
+      }
+    },
+  };
+}