recallnest 2.2.1 → 2.5.4

package/src/memory-promotion.ts

@@ -0,0 +1,246 @@
+/**
+ * Promote-Scan: auto-promotion pipeline for transcript-downgraded stable facts.
+ *
+ * Transcript ingest downgrades profile/preferences into `events` evidence
+ * (memory-boundaries.ts:resolveIngestBoundary), tagging boundary.downgradedFrom.
+ * Without an automatic promotion path those high-signal facts stay sparse in
+ * durable memory. This module clusters recurring downgraded evidence and, when a
+ * cluster recurs often enough at high enough importance, promotes the seed back
+ * to its original durable category via the injected `promote` callback.
+ *
+ * Idempotency is delegated to promoteMemory -> writeDurableEntry's canonicalKey
+ * dedup, so re-scanning is safe (we do not mutate source evidence — see #19
+ * non-atomic store.update — and rely on dedup instead of a promotedTo marker).
+ */
+
+import type { MemoryEntry, MemoryStore } from "./store.js";
+import type { DurableMemoryCategory, StoredPromotedMemoryRecord } from "./memory-schema.js";
+import { isActiveMemory } from "./memory-evolution.js";
+import { isNoise } from "./noise-filter.js";
+import { extractBoundaryMetadata, parseMetadataObject } from "./memory-boundaries.js";
+import { greedyCluster } from "./skill-promotion.js";
+import { promoteMemory, type PersistMemoryDeps } from "./capture-engine.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/** Categories that transcript ingest downgrades into evidence. */
+type PromotableDowngrade = "profile" | "preferences";
+
+export interface PromoteRequest {
+  memoryId: string;
+  category: DurableMemoryCategory;
+  scope: string;
+  importance: number;
+}
+
+export type PromoteFn = (req: PromoteRequest) => Promise<StoredPromotedMemoryRecord>;
+
+export interface PromoteScanDeps {
+  store: Pick<MemoryStore, "list" | "getVectors">;
+  /** Injected so the scan unit-tests without the full persist pipeline. */
+  promote: PromoteFn;
+}
+
+export interface PromoteScanConfig {
+  /** Min cluster members before a downgraded fact is promoted (default 3). */
+  minOccurrences: number;
+  /** Min average cluster importance to promote (default 0.6). */
+  minImportance: number;
+  /** Cosine similarity threshold for greedy clustering (default 0.82). */
+  clusterThreshold: number;
+  /** Max events fetched from the store (default 2000). */
+  listLimit: number;
+  /** When true (default), find candidates without writing anything. */
+  dryRun: boolean;
+}
+
+export const DEFAULT_PROMOTE_SCAN_CONFIG: PromoteScanConfig = {
+  minOccurrences: 3,
+  minImportance: 0.6,
+  clusterThreshold: 0.82,
+  listLimit: 2000,
+  dryRun: true,
+};
+
+export interface PromoteCandidate {
+  downgradedFrom: PromotableDowngrade;
+  seedId: string;
+  seedText: string;
+  memberIds: string[];
+  occurrences: number;
+  avgImportance: number;
+  /** Filled after a real promotion; null in dryRun mode. */
+  promoted: StoredPromotedMemoryRecord | null;
+}
+
+export interface PromoteScanResult {
+  candidates: PromoteCandidate[];
+  scannedEvidence: number;
+  clusters: number;
+  promoted: number;
+  dryRun: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function hasPromotedToMarker(metadata?: string): boolean {
+  const parsed = parseMetadataObject(metadata);
+  return typeof parsed?.promotedTo === "string" && parsed.promotedTo.trim().length > 0;
+}
+
+function resolveDowngrade(entry: MemoryEntry): PromotableDowngrade | null {
+  const from = extractBoundaryMetadata(entry.metadata)?.downgradedFrom;
+  return from === "profile" || from === "preferences" ? from : null;
+}
+
+function averageImportance(members: MemoryEntry[]): number {
+  if (members.length === 0) return 0;
+  return members.reduce((sum, m) => sum + (m.importance ?? 0), 0) / members.length;
+}
+
+/** Pick the highest-importance member as the promotion seed. */
+function pickSeed(members: MemoryEntry[]): MemoryEntry {
+  return members.reduce((best, m) => (m.importance > best.importance ? m : best), members[0]);
+}
+
+// ---------------------------------------------------------------------------
+// Main scan
+// ---------------------------------------------------------------------------
+
+export async function scanMemoryPromotions(
+  deps: PromoteScanDeps,
+  scope: string,
+  config?: Partial<PromoteScanConfig>,
+): Promise<PromoteScanResult> {
+  const cfg: PromoteScanConfig = { ...DEFAULT_PROMOTE_SCAN_CONFIG, ...config };
+
+  // 1. Load downgraded evidence (events). list() returns empty vectors.
+  const events = await deps.store.list([scope], "events", cfg.listLimit, 0);
+
+  // 2. Keep only active, downgraded-from profile/preferences, not-yet-promoted.
+  const downgraded = events.filter(
+    (e) =>
+      isActiveMemory(e.metadata) &&
+      !hasPromotedToMarker(e.metadata) &&
+      !isNoise(e.text) &&
+      resolveDowngrade(e) !== null,
+  );
+
+  const result: PromoteScanResult = {
+    candidates: [],
+    scannedEvidence: downgraded.length,
+    clusters: 0,
+    promoted: 0,
+    dryRun: cfg.dryRun,
+  };
+
+  if (downgraded.length === 0) return result;
+
+  // 3. Backfill vectors (list omits them for performance) and drop the
+  //    vectorless — they cannot be clustered.
+  const vectorMap = await deps.store.getVectors(downgraded.map((e) => e.id));
+  const withVectors = downgraded
+    .map((e) => ({ ...e, vector: vectorMap.get(e.id) ?? [] }))
+    .filter((e) => e.vector.length > 0);
+
+  // 4. Group by downgraded category — they promote to different targets.
+  const groups = new Map<PromotableDowngrade, MemoryEntry[]>();
+  for (const entry of withVectors) {
+    const from = resolveDowngrade(entry);
+    if (!from) continue;
+    const bucket = groups.get(from);
+    if (bucket) bucket.push(entry);
+    else groups.set(from, [entry]);
+  }
+
+  // 5. Cluster each group; qualify candidates by occurrence + importance.
+  for (const [downgradedFrom, members] of groups) {
+    const clusters = greedyCluster(members, cfg.clusterThreshold);
+    result.clusters += clusters.length;
+
+    for (const cluster of clusters) {
+      if (cluster.members.length < cfg.minOccurrences) continue;
+      const avgImportance = averageImportance(cluster.members);
+      if (avgImportance < cfg.minImportance) continue;
+
+      const seed = pickSeed(cluster.members);
+      const candidate: PromoteCandidate = {
+        downgradedFrom,
+        seedId: seed.id,
+        seedText: seed.text,
+        memberIds: cluster.members.map((m) => m.id),
+        occurrences: cluster.members.length,
+        avgImportance,
+        promoted: null,
+      };
+
+      // 6. Promote (unless dry-run). Dedup is handled by promoteMemory.
+      if (!cfg.dryRun) {
+        candidate.promoted = await deps.promote({
+          memoryId: seed.id,
+          category: downgradedFrom,
+          scope,
+          importance: avgImportance,
+        });
+        result.promoted += 1;
+      }
+
+      result.candidates.push(candidate);
+    }
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Production wiring
+// ---------------------------------------------------------------------------
+
+/** Build scan deps from the persist pipeline, wrapping promoteMemory. */
+export function buildPromoteScanDeps(
+  deps: PersistMemoryDeps & {
+    store: PersistMemoryDeps["store"] & Pick<MemoryStore, "list" | "getVectors">;
+  },
+): PromoteScanDeps {
+  return {
+    store: deps.store,
+    promote: (req) => promoteMemory(deps, req),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+
+export function formatPromoteScanResult(result: PromoteScanResult): string {
+  const lines = [
+    `Promote-scan: ${result.scannedEvidence} downgraded evidence, ${result.clusters} cluster(s) scanned.`,
+  ];
+  if (result.candidates.length === 0) {
+    lines.push("No promotion candidates found.");
+    return lines.join("\n");
+  }
+  lines.push(
+    result.dryRun
+      ? `Found ${result.candidates.length} candidate(s) (dry-run, nothing written):\n`
+      : `Promoted ${result.promoted} of ${result.candidates.length} candidate(s):\n`,
+  );
+  for (const [i, c] of result.candidates.entries()) {
+    lines.push(`### ${i + 1}. [${c.downgradedFrom}] ${c.seedText.split("\n")[0].slice(0, 80)}`);
+    lines.push(`Occurrences: ${c.occurrences} | avg importance: ${c.avgImportance.toFixed(2)}`);
+    lines.push(`Seed: ${c.seedId.slice(0, 8)} | members: ${c.memberIds.length}`);
+    if (c.promoted) {
+      lines.push(
+        c.promoted.disposition === "conflict" && c.promoted.conflictId
+          ? `-> conflict ${c.promoted.conflictId.slice(0, 8)} (manual review)`
+          : `-> durable ${c.promoted.id.slice(0, 8)} (${c.promoted.disposition}, key ${c.promoted.canonicalKey})`,
+      );
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}

package/src/memory-schema.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 
 import { boundedStringSchema, identifierSchema, normalizedStringListSchema, optionalBoundedStringSchema } from "./schema-utils.js";
+import * as envConfig from "./env-config.js";
 
 export const DURABLE_MEMORY_CATEGORIES = [
   "profile",
@@ -130,12 +131,16 @@ export const StoreMemoryInputSchema = z.object({
   privacyTier: PrivacyTierSchema.default("durable"),
 });
 
+// Keep in sync with RejectionReason in admission-control.ts
+export const RejectionReasonSchema = z.enum(["text_too_short", "noise_detected", "importance_too_low", "rate_limited"]);
+
 export const StoredMemoryRecordSchema = StoreMemoryInputSchema.extend({
   id: identifierSchema("id", 128),
   resolvedScope: identifierSchema("resolvedScope", 160),
   storedAt: z.string().datetime(),
   disposition: WriteDispositionSchema.default("stored"),
   conflictId: identifierSchema("conflictId", 128).optional(),
+  rejectionReason: RejectionReasonSchema.optional(),
 });
 
 export const CaptureMemoryItemSchema = z.object({
@@ -181,6 +186,23 @@ export const WorkflowPatternInputSchema = z.object({
   canonicalKey: CanonicalKeySchema,
 });
 
+/** A1 · break-loop 五维（Trellis 借鉴）：debug case 的结构化归因，全可选、向后兼容。 */
+export const DebugFramingSchema = z.object({
+  rootCause: z.enum([
+    "missing_spec",
+    "cross_layer_contract",
+    "change_propagation",
+    "test_coverage_gap",
+    "implicit_assumption",
+  ]).describe("根因归类：缺spec/跨层契约/改动传播失败/测试覆盖缺口/隐式假设"),
+  whyPriorFixFailed: z.string().max(300).optional().describe("为何之前没修对"),
+  defense: z.string().max(300).optional().describe("防御机制（强度：架构>编译期>运行期>测试>review>文档）"),
+  systematicExtension: z.string().max(300).optional().describe("系统性扩展：同类问题还在哪"),
+  knowledgeFix: z.string().max(300).optional().describe("知识固化：沉淀去向"),
+});
+
+export type DebugFraming = z.infer<typeof DebugFramingSchema>;
+
 export const CaseMemoryInputSchema = z.object({
   title: CaseMemoryTitleSchema,
   problem: CaseMemoryProblemSchema,
@@ -193,6 +215,7 @@ export const CaseMemoryInputSchema = z.object({
   source: StoreMemorySourceSchema.default("agent"),
   tags: MemoryTagsSchema,
   canonicalKey: CanonicalKeySchema,
+  debugFraming: DebugFramingSchema.optional(),
 });
 
 export const PromoteMemoryInputSchema = z.object({
@@ -248,9 +271,9 @@ export type PromoteMemoryInput = z.infer<typeof PromoteMemoryInputSchema>;
 export type StoredPromotedMemoryRecord = z.infer<typeof StoredPromotedMemoryRecordSchema>;
 
 export function isEmotionScoringEnabled(): boolean {
-  return process.env.RECALLNEST_EMOTION_SCORING === "true";
+  return envConfig.emotionScoring();
 }
 
 export function isPredictiveMemoryEnabled(): boolean {
-  return process.env.RECALLNEST_PREDICTIVE_MEMORY === "true";
+  return envConfig.predictiveMemory();
 }

package/src/memory-store-port.ts

@@ -0,0 +1,43 @@
+import type { MemoryEntry, MemorySearchResult } from "./store.js";
+
+export interface MemoryStoreStats {
+  totalCount: number;
+  scopeCounts: Record<string, number>;
+  categoryCounts: Record<string, number>;
+}
+
+export type MemoryStoreUpdate = {
+  text?: string;
+  vector?: number[];
+  importance?: number;
+  category?: MemoryEntry["category"];
+  metadata?: string;
+  timestamp?: number;
+  language?: string;
+  fts_text?: string;
+};
+
+export interface MemoryStorePort {
+  stats(scopeFilter?: string[]): Promise<MemoryStoreStats>;
+  list(scopeFilter?: string[], category?: string, limit?: number, offset?: number): Promise<MemoryEntry[]>;
+  /** DB 层真分页(where/limit/offset 下推),全库维护扫描用;list() 是全量拉取再切片。 */
+  listPage(opts?: {
+    scopeFilter?: string[];
+    category?: string;
+    limit?: number;
+    offset?: number;
+    includeVector?: boolean;
+  }): Promise<MemoryEntry[]>;
+  get(id: string, scopeFilter?: string[]): Promise<MemoryEntry | null>;
+  getById(id: string): Promise<MemoryEntry | null>;
+  store(entry: Omit<MemoryEntry, "id" | "timestamp"> & { id?: string }): Promise<MemoryEntry>;
+  update(id: string, updates: MemoryStoreUpdate, scopeFilter?: string[]): Promise<MemoryEntry | null>;
+  /** metadata 读改写的单写通道(per-id 串行队列)。新代码改 metadata 一律走这里,
+   *  不要 getById+update 裸 RMW(并发覆盖)。 */
+  patchMetadata(
+    id: string,
+    patchFn: (meta: Record<string, unknown>, entry: MemoryEntry) => Record<string, unknown>,
+    scopeFilter?: string[],
+  ): Promise<MemoryEntry | null>;
+  vectorSearch(vector: number[], limit?: number, minScore?: number, scopeFilter?: string[]): Promise<MemorySearchResult[]>;
+}

package/src/multi-vector.ts

@@ -16,13 +16,14 @@
  */
 
 import type { Embedder } from "./embedder.js";
+import * as envConfig from "./env-config.js";
 
 // ---------------------------------------------------------------------------
 // Config
 // ---------------------------------------------------------------------------
 
 export function isMultiVectorEnabled(): boolean {
-  return process.env.RECALLNEST_MULTI_VECTOR === "true";
+  return envConfig.multiVector();
 }
 
 // ---------------------------------------------------------------------------

package/src/narrative-schema.ts

@@ -12,13 +12,14 @@
  */
 
 import { z } from "zod";
+import * as envConfig from "./env-config.js";
 
 // ---------------------------------------------------------------------------
 // Feature Flag
 // ---------------------------------------------------------------------------
 
 export function isNarrativeModeEnabled(): boolean {
-  return process.env.RECALLNEST_NARRATIVE_MODE === "true";
+  return envConfig.narrativeMode();
 }
 
 // ---------------------------------------------------------------------------

package/src/noise-filter.ts

@@ -5,6 +5,7 @@
  */
 
 import { logInfo } from "./stderr-log.js";
+import { redactForLog } from "./pii-detector.js";
 
 // Agent-side denial patterns
 const DENIAL_PATTERNS = [
@@ -45,7 +46,7 @@ const META_QUESTION_PATTERNS = [
 
 // Session boilerplate (anchored to prevent false positives on real content)
 const BOILERPLATE_PATTERNS = [
-  /^(hi|hello|hey|good morning|good evening|greetings)/i,
+  /^(hi|hello|hey|good morning|good evening|greetings)\b/i,
   /^fresh session/i,
   /^new session/i,
   /^HEARTBEAT/i,
@@ -81,6 +82,43 @@ const DIAGNOSTIC_ARTIFACT_PATTERNS = [
   /^只回复精确代号/i,
 ];
 
+// CC bridge injected agent prompts: the Telegram bridge writes CC's own system
+// prompts (reply / self-decision templates) as `user` turns, which transcript
+// ingest then mis-extracts as profile/preferences. They are never user content.
+const AGENT_PROMPT_PATTERNS = [
+  /^#{1,3}\s*CC\s+(Reply|Self-Decision)\s+Prompt\b/im,
+  /你不是\s*Alice\s*的数字分身/,
+  /你是\s*CC[。.]?\s*(Alice\s*刚给你发了消息|现在你周期性醒来)/,
+  // Fixed sections of the bridge prompt that survive ingest as standalone chunks
+  // (the long prompt turn gets split, so the title/persona lines may be absent).
+  /你的输出\s*[\r\n]+\s*直接回复\s*Alice/,
+  /节奏要求[：:]\s*[\r\n]+\s*[-*]\s*短话题/,
+];
+
+// CC bridge self-decision / RecallNest distill payloads embed structured JSON
+// (decision logs, conversation history, interaction stats). Long prompts get
+// split by transcript chunking, orphaning JSON field-lines that lost their
+// enclosing braces — these survive ingest looking like preferences. We match
+// the severed-fragment SHAPE, not JSON in general: a complete { } object is left
+// alone (it may be real discussion); only orphaned field-lines (start as "key":
+// with no brace) and dangling structural tokens (leading , } ]) qualify.
+const CONTEXT_JSON_FRAGMENT_PATTERNS = [
+  // bridge/distill proprietary schema keys — ANCHORED to line start (^). These
+  // keys are also RecallNest's own schema fields, so a real memory may MENTION
+  // them inline (e.g. 我们讨论 "distilled_at": 要不要保留); anchoring ensures only
+  // an orphaned line that LEADS with the key is filtered, not such discussion.
+  /^"?(?:next_check_hint|alice_interaction_stats(?:_7d)?|alice_last_cc_hours_ago|engagement_rate|seen_no_reply|distilled_at)"\s*:/,
+  // orphan field-line: whole chunk starts as  "key": <scalar>  with no { wrapper.
+  // SCALAR values only (string/number/bool/null). A `{` or `[` value is left
+  // alone so real config / package.json chunks like `"scripts": {` survive.
+  /^"?[A-Za-z_][\w]*"\s*:\s*(?:"|\d|true\b|false\b|null\b)/,
+  // severed JSON array/object element: a dangling , } ] then a NEWLINE into a
+  // brace/quoted-key. Requiring the newline-then-JSON shape avoids real code
+  // fragments like `} catch (e) {` or `});` (close-token followed by code on
+  // the same line, never a newline into a JSON key).
+  /^[,}\]]+[ \t]*\r?\n\s*[{[]?\s*"[A-Za-z_]/,
+];
+
 export interface NoiseFilterOptions {
   /** Filter agent denial responses (default: true) */
   filterDenials?: boolean;
@@ -88,12 +126,15 @@ export interface NoiseFilterOptions {
   filterMetaQuestions?: boolean;
   /** Filter session boilerplate (default: true) */
   filterBoilerplate?: boolean;
+  /** Filter CC bridge injected agent prompts (default: true) */
+  filterAgentPrompts?: boolean;
 }
 
 const DEFAULT_OPTIONS: Required<NoiseFilterOptions> = {
   filterDenials: true,
   filterMetaQuestions: true,
   filterBoilerplate: true,
+  filterAgentPrompts: true,
 };
 
 /**
@@ -110,21 +151,21 @@ export function isNoise(text: string, options: NoiseFilterOptions = {}): boolean
   }
 
   if (opts.filterDenials && DENIAL_PATTERNS.some(p => p.test(trimmed))) {
-    logInfo(`[INFO] noise-filter: denial pattern matched: "${trimmed.slice(0, 60)}..."`);
+    logInfo(`[INFO] noise-filter: denial pattern matched: "${redactForLog(trimmed)}..."`);
     return true;
   }
   if (opts.filterMetaQuestions && META_QUESTION_PATTERNS.some(p => p.test(trimmed))) {
-    logInfo(`[INFO] noise-filter: meta-question filtered: "${trimmed.slice(0, 60)}..."`);
+    logInfo(`[INFO] noise-filter: meta-question filtered: "${redactForLog(trimmed)}..."`);
     return true;
   }
   if (opts.filterBoilerplate) {
     if (BOILERPLATE_PATTERNS.some(p => p.test(trimmed))) {
-      logInfo(`[INFO] noise-filter: boilerplate filtered: "${trimmed.slice(0, 60)}..."`);
+      logInfo(`[INFO] noise-filter: boilerplate filtered: "${redactForLog(trimmed)}..."`);
       return true;
     }
     if (trimmed.length <= BOILERPLATE_MAX_LENGTH &&
         SHORT_BOILERPLATE_PATTERNS.some(p => p.test(trimmed))) {
-      logInfo(`[INFO] noise-filter: short boilerplate filtered: "${trimmed}"`);
+      logInfo(`[INFO] noise-filter: short boilerplate filtered: "${redactForLog(trimmed)}"`);
       return true;
     }
   }
@@ -135,7 +176,17 @@ export function isNoise(text: string, options: NoiseFilterOptions = {}): boolean
   }
   // Diagnostic artifact noise (synced from upstream v1.1.0-beta.3)
   if (DIAGNOSTIC_ARTIFACT_PATTERNS.some(p => p.test(trimmed))) {
-    logInfo(`[INFO] noise-filter: diagnostic artifact filtered: "${trimmed.slice(0, 60)}..."`);
+    logInfo(`[INFO] noise-filter: diagnostic artifact filtered: "${redactForLog(trimmed)}..."`);
+    return true;
+  }
+  // CC bridge injected agent prompts (reply/self-decision templates leaked as user turns)
+  if (opts.filterAgentPrompts && AGENT_PROMPT_PATTERNS.some(p => p.test(trimmed))) {
+    logInfo(`[INFO] noise-filter: agent prompt filtered: "${redactForLog(trimmed)}..."`);
+    return true;
+  }
+  // Severed bridge/distill context-JSON fragments (orphaned field-lines from chunking)
+  if (opts.filterAgentPrompts && CONTEXT_JSON_FRAGMENT_PATTERNS.some(p => p.test(trimmed))) {
+    logInfo(`[INFO] noise-filter: context JSON fragment filtered: "${redactForLog(trimmed)}..."`);
     return true;
   }