recallnest 2.6.1 → 3.0.1

package/src/dream-pipeline.ts

@@ -15,6 +15,7 @@
  */
 
 import type { MemoryStorePort } from "./memory-store-port.js";
+import type { ScopeMatchMode } from "./scope-policy.js";
 import type { LLMClient } from "./llm-client.js";
 import type { Embedder } from "./embedder.js";
 import { ConsolidationEngine, clusterAndConsolidate, DEFAULT_CONSOLIDATION_CONFIG, isDerivedInsight, type ConsolidationKGSource } from "./consolidation-engine.js";
@@ -45,12 +46,14 @@ export interface DreamConfig {
   extractPatterns: boolean;
   /** Max entries to scan per consolidation run (default: 500) */
   maxEntriesPerRun: number;
-  /** Wall-clock budget for one `dream --auto` sweep, ms (default: 6h).
+  /** Wall-clock budget for one `dream --auto` sweep, ms (default: 2h).
    *  防跨天兜底闸,不是调度器。2026-07-24 那轮跑了 4 天 15 小时(见 cli.ts --auto 分支
    *  与 scripts/dream-consolidation.sh:57 的注释),期间堵死三天调度。
-   *  已知天花板:超预算后剩余 scope 直接丢弃,而每轮都从列表头重来 → 靠后的 scope 会被
-   *  系统性饿死。真解是 session scope TTL(只 dream 近 N 天活跃的),不是把预算调大——
-   *  scope 数 07-10 是 133、07-29 是 475,只增不减,加预算追不上。 */
+   *  2026-08-14: 6h→2h。transcript scope 已整体出队(原文层不参与巩固,cli.ts --auto
+   *  的 pruneWriteCounts(isTranscriptScope)),inline 队列只剩个位数 durable scope,
+   *  典型轮次分钟级;2h 是纯兜底。单 scope 路径(weekly memory / MCP dream tool)不走
+   *  本预算(budget/deadline 只存在于 cli.ts --auto 分支)。env 覆盖:
+   *  RECALLNEST_DREAM_BUDGET_MS。 */
   autoRunBudgetMs: number;
   /** Scopes the `--auto` sweep hands off to a dedicated schedule instead of running inline.
    *  Exact match only — a prefix rule would silently swallow future `memory-*` scopes.
@@ -65,6 +68,23 @@ export interface DreamConfig {
    *  这是缓解不是根治：scope 只增不减仍在（07-10 是 133、07-29 是 475、07-30 是 507），
    *  真解仍是 session scope TTL —— 只 dream 近 N 天活跃的。 */
   autoExcludeScopes: readonly string[];
+  /**
+   * 按策略**永不**巩固的 scope（2026-08-16 加）。与 autoExcludeScopes 是两件事，别混：
+   * 那个是"交给专用 schedule 跑"（deferred，别处会跑），这个是"根本不该被 dream 碰"
+   * （never，没有任何 schedule 会跑）。
+   *
+   * 判定在 runDream 内部而不是 `--auto` 的分拣里 —— CLI、weekly、MCP `dream` 工具
+   * 三条入口都必须受同一条策略约束，放在分拣里等于只拦住了其中一条。
+   *
+   * 首个成员 `memory:pivot`：手写提炼层（判断转折 / 决策 / 偏好 / 踩坑，每条带
+   * canonicalKey + pinned + 来源指针）。2026-08-16 实测它被 dream 碰出两类违例：
+   * 3b 往池里铸了 23 条无 canonicalKey / 无 pinned / 无来源的 LLM 派生行；3a 把 3 条
+   * 手写原件合并进 LLM 复述、复述当了 canonical。**提炼层再合成一层就是"摘要的摘要"**
+   * ——同 gather 不回喂派生 insight 的道理（见 :245 注释）。
+   * 要翻回"允许 3a 去重"，前置条件：派生行永不当 canonical（已修）+ pinned 不失活
+   * （未做，GC/3a 都不认 tag）+ 有 uptake 证据证明去重确有收益。
+   */
+  neverDreamScopes: readonly string[];
   /** GC config for prune phase */
   gc: AutoGcConfig;
 }
@@ -76,8 +96,9 @@ export const DEFAULT_DREAM_CONFIG: DreamConfig = {
   minClusterSize: 3,
   extractPatterns: true,
   maxEntriesPerRun: 500,
-  autoRunBudgetMs: 6 * 60 * 60 * 1000,
+  autoRunBudgetMs: 2 * 60 * 60 * 1000,
   autoExcludeScopes: ["memory"],
+  neverDreamScopes: ["memory:pivot"],
   gc: DEFAULT_AUTO_GC_CONFIG,
 };
 
@@ -146,6 +167,13 @@ export interface DreamResult {
     semanticClustersFound: number;
     insightsGenerated: number;
     patternsExtracted: number;
+    /** 3b 合成里模型判定「这簇没有可提炼结论」的次数（insight + pattern 合计）。
+     *  2026-08-23 起才可能非 0——在此之前合成没有弃权路径，每个合格簇必写一条。
+     *  它要和 insightsGenerated 一起读：0 insight + 高弃权 = 今天确实没料，
+     *  0 insight + 0 弃权 = 合成这一步压根没跑起来，两者完全不同。 */
+    synthesisAbstained: number;
+    /** 3b 合成产物在写库前被校验拦下的次数（提示词回声 / evidence 不合法 / 长度越界）。 */
+    synthesisRejected: number;
     mergedCount: number;
     archivedCount: number;
     /** 3a 去重路径新建的关联数。此前引擎返回了但被 dream 丢弃，导致「只加了关联」
@@ -177,8 +205,14 @@ async function runDreamInner(params: {
   activityStatsPath?: string;
   /** Optional KG evidence source for triple-overlap merging (null/absent = vector-only) */
   kgStore?: ConsolidationKGSource | null;
+  /**
+   * scope 匹配模式（2026-08-16 加）。**调度入口一律传 "exact"** —— 它们手里的是
+   * activity-stats 里的真实 scope 名或运维指定的具体 scope，不是 family selector。
+   * 缺省保持 "family" 只为兼容既有手动调用（`dream --scope cc` 想扫 `cc:*`）。
+   */
+  scopeMatch?: ScopeMatchMode;
 }): Promise<DreamResult> {
-  const { store, llm, embedder, scope, force = false } = params;
+  const { store, llm, embedder, scope, force = false, scopeMatch = "family" } = params;
   const config = { ...DEFAULT_DREAM_CONFIG, ...params.config };
   const statsCfg = params.activityStatsPath ? { statsPath: params.activityStatsPath } : undefined;
   const phases: DreamPhaseResult[] = [];
@@ -192,6 +226,8 @@ async function runDreamInner(params: {
     semanticClustersFound: 0,
     insightsGenerated: 0,
     patternsExtracted: 0,
+    synthesisAbstained: 0,
+    synthesisRejected: 0,
     mergedCount: 0,
     archivedCount: 0,
     relationsAdded: 0,
@@ -199,13 +235,31 @@ async function runDreamInner(params: {
     semanticVectorFed: 0,
   };
 
+  // =========================================================================
+  // Phase 0: Policy gate — scopes that must never be consolidated
+  // =========================================================================
+  // 在写计数门槛**之前**判：策略排除与"今天写得够不够"无关，force 也不该越过它。
+  // 放在 runDream 内部 = CLI / weekly / MCP 三条入口同受约束（互审 C2）。
+  if (config.neverDreamScopes.includes(scope)) {
+    return {
+      ran: false,
+      reason: `policy_excluded (${scope} is in neverDreamScopes)`,
+      output: { kind: "skipped", reason: "policy_excluded", effectsWritten: 0 },
+      phases: [{
+        phase: "orient",
+        detail: `scope ${scope} 按策略不参与巩固（DreamConfig.neverDreamScopes）`,
+      }],
+      stats,
+    };
+  }
+
   // =========================================================================
   // Phase 1: Orient — assess current memory state
   // =========================================================================
   const writeCount = getWriteCount(scope, statsCfg);
   stats.writesSinceLastDream = writeCount;
 
-  const storeStats = await store.stats([scope]);
+  const storeStats = await store.stats([scope], scopeMatch);
   stats.totalMemories = storeStats.totalCount ?? 0;
 
   if (!force && writeCount < config.minWritesForDream) {
@@ -229,7 +283,7 @@ async function runDreamInner(params: {
   // =========================================================================
   // Phase 2: Gather — collect active entries for consolidation
   // =========================================================================
-  const entries = await store.list([scope], undefined, config.maxEntriesPerRun, 0);
+  const entries = await store.list([scope], undefined, config.maxEntriesPerRun, 0, scopeMatch);
   const active = entries.filter(e => isActiveMemory(e.metadata));
   stats.activeMemories = active.length;
 
@@ -282,7 +336,7 @@ async function runDreamInner(params: {
   if (consolidatable.length < config.minClusterSize) {
     phases.push({ phase: "consolidate", detail: "skipped — too few active entries" });
     phases.push({ phase: "prune", detail: "skipped — too few entries for GC" });
-    resetWriteCount(scope, statsCfg);
+    await resetWriteCount(scope, statsCfg);
     return {
       ran: true,
       reason: "completed_early",
@@ -307,6 +361,9 @@ async function runDreamInner(params: {
       ...DEFAULT_CONSOLIDATION_CONFIG,
       clusterThreshold: config.clusterThreshold,
       maxEntriesPerRun: config.maxEntriesPerRun,
+      // 3a 自己重新取数（list + 两处 vectorSearch），gather 收窄了它不会跟着窄 ——
+      // 模式必须显式传进来，漏传就只修了一半（2026-08-16 互审 K3）。
+      scopeMatch,
     }, params.kgStore ?? null);
     const consolidation = await engine.run(scope);
     stats.dedupeClustersFound += consolidation.clustersFound;
@@ -339,6 +396,8 @@ async function runDreamInner(params: {
       stats.clustersFound += clusterResult.clustersFound;
       stats.insightsGenerated = clusterResult.insightsGenerated;
       stats.patternsExtracted = clusterResult.patternsExtracted;
+      stats.synthesisAbstained = clusterResult.synthesisAbstained;
+      stats.synthesisRejected = clusterResult.synthesisRejected;
     }
   }, { onBusy: "skip", expireMs: 600_000 });
 
@@ -351,6 +410,9 @@ async function runDreamInner(params: {
       ? `${stats.dedupeClustersFound} dedupe-clusters (${stats.mergedCount} merged), `
         + `${stats.semanticClustersFound} semantic-clusters -> `
         + `${stats.insightsGenerated} insights, ${stats.patternsExtracted} patterns`
+        // 弃权与校验拦截跟着一起报：没有它们，「合成认真看过但确实没料」和
+        // 「合成根本没跑」在这一行里长得一模一样——这正是本文件反复吃过的亏。
+        + `（弃权 ${stats.synthesisAbstained}，校验拦下 ${stats.synthesisRejected}）`
       : `skipped — another process holds the consolidate lock for scope ${scope}`,
   });
 
@@ -363,7 +425,9 @@ async function runDreamInner(params: {
   phases.push({
     phase: "prune",
     detail: gcResult.triggered
-      ? `${gcResult.archivedCount} entries archived`
+      ? `${gcResult.archivedCount} entries archived, ${gcResult.dissolvedVersionGroups} singleton version groups dissolved`
+      : gcResult.dissolvedVersionGroups > 0
+        ? `${gcResult.dissolvedVersionGroups} singleton version groups dissolved; archive skipped — ${gcResult.reason}`
       : `skipped — ${gcResult.reason}`,
   });
 
@@ -384,7 +448,7 @@ async function runDreamInner(params: {
   // 注意：清对之后，listScopesAboveThreshold 的 FIFO 自组织才真正开始工作 ——
   // 处理完的 scope 被 delete、下次攒够写入重新插入时排到队尾，被跳过的带原计数留在
   // 队首下轮优先。所以这里不需要额外的游标 / checkpoint 文件。
-  if (consolidateOutcome.ran) resetWriteCount(scope, statsCfg);
+  if (consolidateOutcome.ran) await resetWriteCount(scope, statsCfg);
 
   const health = assertDreamSweepHealth({
     llmPresent: llm != null,
@@ -419,6 +483,8 @@ export async function runDream(params: {
   activityStatsPath?: string;
   /** Optional KG evidence source for triple-overlap merging (null/absent = vector-only) */
   kgStore?: ConsolidationKGSource | null;
+  /** 见 runDreamInner 的同名参数：调度入口一律 "exact"，缺省 "family" 只为兼容手动调用。 */
+  scopeMatch?: ScopeMatchMode;
 }): Promise<DreamResult> {
   const outcome = await withLock(
     `dream-${params.scope}`,
@@ -440,6 +506,8 @@ export async function runDream(params: {
       semanticClustersFound: 0,
       insightsGenerated: 0,
       patternsExtracted: 0,
+      synthesisAbstained: 0,
+      synthesisRejected: 0,
       mergedCount: 0,
       archivedCount: 0,
       relationsAdded: 0,
@@ -577,12 +645,21 @@ export function classifyDreamFailure(message: string): DreamFailureKind {
 /** transient 失败占比超过这个比例仍判整轮失败——否则"475 个全被锁挡住"会假绿。 */
 export const DREAM_TRANSIENT_BLOCK_RATIO = 0.2;
 
+/** 比例阈值之外的绝对量豁免：transient 失败不超过这个数就不算整轮失败。
+ *  2026-08-14 transcript 出队后 inline 队列缩到个位数，1/1 = 100% > 20% 让单个
+ *  瞬态锁竞争把整轮判红（当天验证轮实测：一个死进程的残留 store-write 锁 →
+ *  唯一 durable scope 等锁超时 → blocked → launchd 失败通知）。比例阈值的原意是
+ *  「环境性失败要成规模」，1-2 个绝对不算规模——失败的 scope 计数未清、明天自动
+ *  重试，兜底仍在。fatal 不享受豁免（一票否决不变）。 */
+export const DREAM_TRANSIENT_ABSOLUTE_TOLERANCE = 2;
+
 /**
  * 一轮 `dream --auto` 是否该判 blocked。
  *
  * 旧语义是「任意一个 scope 抛异常即 blocked」,在 475 个 scope 的规模下等于
  * 「有一个撞上锁就整轮红」,而红了会让 shell 脚本走重试 → 重跑全量。
- * 新语义:代码/数据缺陷一票否决;环境性失败要成规模才算整轮失败。
+ * 新语义:代码/数据缺陷一票否决;环境性失败要成规模才算整轮失败——
+ * 「规模」同时看比例(> 20%)与绝对量(> 2 个),小队列下单个瞬态不再整轮红。
  */
 export function shouldBlockDreamRun(params: {
   totalScopes: number;
@@ -591,6 +668,7 @@ export function shouldBlockDreamRun(params: {
 }): boolean {
   if (params.fatalFailures > 0) return true;
   if (params.totalScopes <= 0) return false;
+  if (params.transientFailures <= DREAM_TRANSIENT_ABSOLUTE_TOLERANCE) return false;
   return params.transientFailures / params.totalScopes > DREAM_TRANSIENT_BLOCK_RATIO;
 }
 

package/src/embedder.ts

@@ -103,6 +103,24 @@ export interface EmbeddingConfig {
   normalized?: boolean;
   /** Enable automatic chunking for documents exceeding context limits (default: true) */
   chunking?: boolean;
+
+  /**
+   * Per-request timeout (ms) handed to the OpenAI-compatible client.
+   *
+   * **Unset is the default and keeps the historical behavior**: no `timeout` key is
+   * passed to the SDK at all, so the client falls back to its own default (600s in
+   * openai v7). That default is deliberately left alone — bulk embedding of long
+   * documents is legitimately slow, and lowering it globally would turn working
+   * ingests into failures.
+   *
+   * Worth setting on latency-sensitive callers: the SDK default stacks with the SDK's
+   * own 2 retries *and* Embedder's 2 transient retries, so a black-holed endpoint can
+   * hold a single embed call for far longer than any caller expects.
+   *
+   * Non-finite or non-positive values are ignored (treated as unset) rather than
+   * throwing, matching how `env-config.ts` validates its other numeric knobs.
+   */
+  timeoutMs?: number;
 }
 
 // Known embedding model dimensions
@@ -205,9 +223,15 @@ export class Embedder {
     // Enable auto-chunking by default for better handling of long documents
     this._autoChunk = config.chunking !== false;
 
+    const timeoutMs =
+      typeof config.timeoutMs === "number" && Number.isFinite(config.timeoutMs) && config.timeoutMs > 0
+        ? config.timeoutMs
+        : undefined;
+
     this.client = new OpenAI({
       apiKey: resolvedApiKey,
       ...(config.baseURL ? { baseURL: config.baseURL } : {}),
+      ...(timeoutMs !== undefined ? { timeout: timeoutMs } : {}),
     });
 
     this.dimensions = getVectorDimensions(config.model, config.dimensions);
@@ -328,7 +352,15 @@ export class Embedder {
     }
   }
 
-  private async embedSingle(text: string, task?: string): Promise<number[]> {
+  /**
+   * @param allowChunkRetry - False for texts that are already chunks. Without it the
+   *   context-error branch below can re-enter itself forever: `smartChunk` returns the
+   *   input unchanged when it already fits, so re-embedding that chunk reproduces the
+   *   same provider error and chunks again. The regex that gates the branch also matches
+   *   rate-limit wording ("quota exceeded", "Rate limit exceeded"), which turned a 429
+   *   into an unbounded request storm against a paid endpoint.
+   */
+  private async embedSingle(text: string, task?: string, allowChunkRetry = true): Promise<number[]> {
     if (!text || text.trim().length === 0) {
       throw new Error("Cannot embed empty text");
     }
@@ -352,21 +384,28 @@ export class Embedder {
       const errorMsg = error instanceof Error ? error.message : String(error);
       const isContextError = /context|too long|exceed|length/i.test(errorMsg);
 
-      if (isContextError && this._autoChunk) {
+      if (isContextError && this._autoChunk && allowChunkRetry) {
         try {
           logInfo(`Document exceeded context limit (${errorMsg}), attempting chunking...`);
           const chunkResult = smartChunk(text, this._model);
-          
+
           if (chunkResult.chunks.length === 0) {
             throw new Error(`Failed to chunk document: ${errorMsg}`);
           }
 
+          // Terminating condition: the chunker hands short text straight back, so a
+          // single chunk identical to the input means splitting cannot help. Retrying it
+          // would reproduce the same error and recurse.
+          if (chunkResult.chunks.length === 1 && chunkResult.chunks[0] === text) {
+            throw new Error(`Chunking cannot reduce this input: ${errorMsg}`);
+          }
+
           // Embed all chunks in parallel
           logInfo(`Split document into ${chunkResult.chunkCount} chunks for embedding`);
           const chunkEmbeddings = await Promise.all(
             chunkResult.chunks.map(async (chunk, idx) => {
               try {
-                const embedding = await this.embedSingle(chunk, task);
+                const embedding = await this.embedSingle(chunk, task, false);
                 return { embedding };
               } catch (chunkError) {
                 logWarn(`Failed to embed chunk ${idx}:`, chunkError);
@@ -468,7 +507,7 @@ export class Embedder {
 
               // Embed all chunks in parallel, then average.
               const embeddings = await Promise.all(
-                chunkResult.chunks.map((chunk) => this.embedSingle(chunk, task))
+                chunkResult.chunks.map((chunk) => this.embedSingle(chunk, task, false))
               );
 
               const avgEmbedding = embeddings.reduce(

package/src/env-config.ts

@@ -109,6 +109,61 @@ export const readConsistencyInterval = (): number | undefined => {
   return Number.isFinite(n) && n >= 0 ? n : 0;
 };
 
+/**
+ * Wall-clock budget (ms) for one `dream --auto` sweep.
+ *
+ * NOT a pure movement (unlike every accessor above): the original inline read in
+ * cli.ts was `Number(process.env.X ?? DEFAULT)`, which had two silent failure
+ * modes — `""` slipped past `??` and became **0** (every scope skipped, run is a
+ * no-op that still reports ok), and any non-numeric value became **NaN**, making
+ * the `Date.now() > deadline` guard permanently false. The second one silently
+ * reverts the very incident this budget exists to prevent: the 2026-07-24 sweep
+ * ran 4d15h and blocked three days of scheduled runs. Both are corrected here
+ * on purpose; do not "restore" the original parsing.
+ *
+ * Unset / empty / non-finite / <= 0 → caller's fallback.
+ */
+export const dreamBudgetMs = (fallbackMs: number): number => {
+  const raw = process.env.RECALLNEST_DREAM_BUDGET_MS;
+  if (raw === undefined || raw.trim() === "") return fallbackMs;
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? n : fallbackMs;
+};
+
+/**
+ * dream 合成（cluster insight / cross-memory pattern）专用模型，覆盖 config.llm.model。
+ *
+ * 独立于全局模型的理由（2026-08-23）：合成是全库唯一一处「产物质量直接决定库的
+ * 可用性」的 LLM 调用 —— 它写进去的东西会被当成记忆检索出来，而 ingest 侧的
+ * smartExtract 只是给原文打标签，错了还能回原文。三臂实验里模型贡献了约 1/3 的
+ * 改善（同提示词下 32.3% → 45.2%），把这一档单独抬上去，不必让全库调用一起涨价。
+ *
+ * 未设 / 空串 → 用 config.llm.model（即全局默认），保持老行为。
+ */
+export const synthesisModel = (): string | undefined => {
+  const raw = process.env.RECALLNEST_SYNTHESIS_MODEL;
+  const trimmed = raw?.trim();
+  return trimmed ? trimmed : undefined;
+};
+
+/**
+ * Per-request timeout (ms) for the embedding client, overriding `embedding.timeoutMs`
+ * in config.json.
+ *
+ * Exists so the timeout can be set per deployment (the resident MCP server, a one-off
+ * CLI ingest) without editing the shared config file — the same reason
+ * `RECALLNEST_LAYER_ADMISSION` lives here.
+ *
+ * Unset / empty / non-finite / <= 0 → undefined, i.e. fall through to config.json and
+ * ultimately to the SDK default. **Not setting it anywhere keeps the old behavior.**
+ */
+export const embeddingTimeoutMs = (): number | undefined => {
+  const raw = process.env.RECALLNEST_EMBEDDING_TIMEOUT_MS;
+  if (raw === undefined || raw.trim() === "") return undefined;
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? n : undefined;
+};
+
 // --- Raw env values (caller validates / clamps / falls back to config) ---
 
 export const recallModeRaw = (): string | undefined => process.env.RECALLNEST_RECALL_MODE;
@@ -116,3 +171,19 @@ export const recallModeRaw = (): string | undefined => process.env.RECALLNEST_RE
 export const uiPortRaw = (): string | undefined => process.env.RECALLNEST_UI_PORT;
 
 export const apiPortRaw = (): string | undefined => process.env.RECALLNEST_API_PORT;
+
+// --- Gateway (read-only front door; caller validates / clamps) ---
+
+export const gatewayPortRaw = (): string | undefined => process.env.RECALLNEST_GATEWAY_PORT;
+
+export const gatewayHostRaw = (): string | undefined => process.env.RECALLNEST_GATEWAY_HOST;
+
+export const gatewayTokenRaw = (): string | undefined => process.env.RECALLNEST_GATEWAY_TOKEN;
+
+export const gatewayRateMaxRaw = (): string | undefined => process.env.RECALLNEST_GATEWAY_RATE_MAX;
+
+export const gatewayFileRootsRaw = (): string | undefined => process.env.RECALLNEST_GATEWAY_FILE_ROOTS;
+
+export const gatewayRgRaw = (): string | undefined => process.env.RECALLNEST_GATEWAY_RG;
+
+export const apiUrlRaw = (): string | undefined => process.env.RECALLNEST_API_URL;

package/src/forget-engine.ts

@@ -223,8 +223,14 @@ export async function forgetByScope(
   let failedCount = 0;
   let totalCascadeDemoted = 0;
 
-  // Fetch all entries in scope
-  const entries = await store.list([scope], undefined, 5000);
+  // Fetch all entries in scope.
+  //
+  // scopeMatch: "exact" —— **破坏性操作绝不做前缀展开**（2026-08-16 互审 Codex R3 提出）。
+  // 默认 family 语义下 `forgetByScope("memory")` 会连 `memory:pivot` 一起遗忘，而下面
+  // 的 KG 清理走的是精确 `scope = 'memory'`（kg-store.ts:506）—— 记忆行按家族删、KG 只
+  // 清一个 scope，两边还会不一致。当前无生产 caller，正因如此更要在有 caller 之前定死语义：
+  // 要清一整个家族，就显式枚举那些 scope 逐个调用。
+  const entries = await store.list([scope], undefined, 5000, 0, "exact");
 
   for (const entry of entries) {
     const result = await forgetMemory(deps, {