recallnest 3.0.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/CHANGELOG.md +304 -0
  2. package/README.md +251 -258
  3. package/README_CN.md +226 -247
  4. package/assets/ui/app.js +6 -1
  5. package/bin/recallnest.mjs +20 -37
  6. package/docs/api-reference.md +8 -0
  7. package/docs/architecture.md +18 -13
  8. package/eval/cases-canary.json +107 -9
  9. package/integrations/examples/claude-agent-sdk/memory-agent.ts +2 -0
  10. package/integrations/examples/langchain/memory-chain.py +4 -0
  11. package/integrations/examples/openai-agents-sdk/memory-agent.py +4 -0
  12. package/package.json +12 -3
  13. package/src/activity-counter.ts +1 -0
  14. package/src/adaptive-retrieval.ts +23 -0
  15. package/src/api-server.ts +13 -11
  16. package/src/asset-sync.ts +26 -2
  17. package/src/audit-log.ts +2 -0
  18. package/src/auto-gc.ts +4 -3
  19. package/src/belief-history.ts +137 -31
  20. package/src/capture-engine.ts +459 -71
  21. package/src/cascade-forget.ts +7 -3
  22. package/src/checkpoint-mirror.ts +417 -0
  23. package/src/cli.ts +574 -54
  24. package/src/conflict-engine.ts +43 -15
  25. package/src/consolidation-engine.ts +24 -3
  26. package/src/context-collapse-renderer.ts +8 -1
  27. package/src/context-composer-stable.ts +5 -1
  28. package/src/context-composer.ts +11 -2
  29. package/src/distill-lock.ts +246 -74
  30. package/src/dream-pipeline.ts +3 -1
  31. package/src/embedder.ts +24 -0
  32. package/src/env-config.ts +136 -0
  33. package/src/flock.ts +97 -0
  34. package/src/forget-engine.ts +342 -58
  35. package/src/gateway-server.ts +4 -0
  36. package/src/ingest-plan.ts +56 -0
  37. package/src/ingest.ts +294 -121
  38. package/src/kg-contradiction.ts +753 -0
  39. package/src/kg-extractor.ts +17 -0
  40. package/src/kg-store.ts +1 -1
  41. package/src/known-item-eval.ts +455 -0
  42. package/src/llm-client.ts +13 -26
  43. package/src/llm-consolidation.ts +2 -2
  44. package/src/mcp-tool-deps.ts +6 -0
  45. package/src/mcp-tools-advanced.ts +80 -45
  46. package/src/mcp-tools-core.ts +23 -17
  47. package/src/mcp-tools-governance.ts +13 -7
  48. package/src/memory-assets.ts +144 -8
  49. package/src/memory-boundaries.ts +19 -0
  50. package/src/memory-evolution.ts +9 -0
  51. package/src/memory-lint.ts +1 -1
  52. package/src/memory-output.ts +311 -14
  53. package/src/memory-reconcile.ts +1437 -0
  54. package/src/memory-schema.ts +11 -0
  55. package/src/memory-store-port.ts +1 -0
  56. package/src/minis-archive.ts +151 -0
  57. package/src/noise-filter.ts +15 -2
  58. package/src/pii-detector.ts +31 -1
  59. package/src/pivot-apply.ts +55 -5
  60. package/src/pivot-evidence.ts +128 -0
  61. package/src/ppr-traversal.ts +25 -1
  62. package/src/preference-matcher.ts +40 -28
  63. package/src/prospective-memory.ts +7 -4
  64. package/src/retriever.ts +297 -57
  65. package/src/retroactive-boost.ts +3 -2
  66. package/src/runtime-config.ts +15 -1
  67. package/src/scope-policy.ts +1 -0
  68. package/src/session-distiller.ts +6 -2
  69. package/src/session-engine.ts +11 -4
  70. package/src/session-output.ts +20 -1
  71. package/src/session-schema.ts +5 -2
  72. package/src/session-store.ts +59 -12
  73. package/src/skill-promotion.ts +67 -1
  74. package/src/store-receipts.ts +109 -0
  75. package/src/store-write-budget.ts +37 -0
  76. package/src/store.ts +298 -57
  77. package/src/text-fingerprint.ts +25 -0
  78. package/src/trigger-store.ts +443 -0
  79. package/src/ui-server.ts +22 -21
  80. package/src/version-manager.ts +19 -5
  81. package/src/write-verifier.ts +16 -11
  82. package/src/__tests__/access-tracker-novelty.test.ts +0 -324
  83. package/src/__tests__/activity-counter.test.ts +0 -125
  84. package/src/__tests__/adaptive-fidelity.test.ts +0 -102
  85. package/src/__tests__/admission-control.test.ts +0 -206
  86. package/src/__tests__/age-label.test.ts +0 -33
  87. package/src/__tests__/anchor-generator.test.ts +0 -67
  88. package/src/__tests__/api-server-security.test.ts +0 -102
  89. package/src/__tests__/auto-consolidation.test.ts +0 -168
  90. package/src/__tests__/auto-recall.test.ts +0 -131
  91. package/src/__tests__/babel-integration.test.ts +0 -144
  92. package/src/__tests__/batch-dedup.test.ts +0 -65
  93. package/src/__tests__/belief-history.test.ts +0 -220
  94. package/src/__tests__/canary-eval.test.ts +0 -114
  95. package/src/__tests__/capture-engine.test.ts +0 -1328
  96. package/src/__tests__/capture-heuristic.test.ts +0 -171
  97. package/src/__tests__/cascade-forget.test.ts +0 -210
  98. package/src/__tests__/category-min-scores.test.ts +0 -116
  99. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  100. package/src/__tests__/circuit-breaker.test.ts +0 -99
  101. package/src/__tests__/cluster-consolidation.test.ts +0 -1236
  102. package/src/__tests__/confidence-metadata.test.ts +0 -125
  103. package/src/__tests__/confidence-tracker.test.ts +0 -261
  104. package/src/__tests__/conflict-advisor.test.ts +0 -206
  105. package/src/__tests__/conflict-escalation.test.ts +0 -149
  106. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  107. package/src/__tests__/conflict-output.test.ts +0 -85
  108. package/src/__tests__/conflict-store.test.ts +0 -418
  109. package/src/__tests__/connector-v1.test.ts +0 -181
  110. package/src/__tests__/consolidation-engine.test.ts +0 -520
  111. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  112. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  113. package/src/__tests__/context-composer-pins.test.ts +0 -64
  114. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  115. package/src/__tests__/context-reconstructor.test.ts +0 -65
  116. package/src/__tests__/context-renderer.test.ts +0 -91
  117. package/src/__tests__/continuity-eval.test.ts +0 -275
  118. package/src/__tests__/continuity-schema.test.ts +0 -97
  119. package/src/__tests__/contracts.test.ts +0 -262
  120. package/src/__tests__/conversation-importer.test.ts +0 -692
  121. package/src/__tests__/core-recall-eval.test.ts +0 -235
  122. package/src/__tests__/core-summary.test.ts +0 -109
  123. package/src/__tests__/data-checkup.test.ts +0 -304
  124. package/src/__tests__/decay-exemption.test.ts +0 -129
  125. package/src/__tests__/dedup-actions.test.ts +0 -58
  126. package/src/__tests__/dedup-guard.test.ts +0 -39
  127. package/src/__tests__/deterministic-id.test.ts +0 -39
  128. package/src/__tests__/distill-lock.test.ts +0 -305
  129. package/src/__tests__/distill-session-argshape.test.ts +0 -79
  130. package/src/__tests__/doctor.test.ts +0 -319
  131. package/src/__tests__/dream-pipeline.test.ts +0 -820
  132. package/src/__tests__/dream-scope-isolation.test.ts +0 -386
  133. package/src/__tests__/dream-write-batching.test.ts +0 -211
  134. package/src/__tests__/drill-down.test.ts +0 -108
  135. package/src/__tests__/embedder.test.ts +0 -122
  136. package/src/__tests__/emotion-decay.test.ts +0 -46
  137. package/src/__tests__/emotion-detector.test.ts +0 -56
  138. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  139. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  140. package/src/__tests__/entity-resolver.test.ts +0 -58
  141. package/src/__tests__/env-config-invariant.test.ts +0 -80
  142. package/src/__tests__/env-config.test.ts +0 -211
  143. package/src/__tests__/error-signature-boost.test.ts +0 -87
  144. package/src/__tests__/error-signature.test.ts +0 -95
  145. package/src/__tests__/error-taxonomy.test.ts +0 -129
  146. package/src/__tests__/ethics-layer.test.ts +0 -476
  147. package/src/__tests__/event-segmenter.test.ts +0 -153
  148. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  149. package/src/__tests__/frequency-tracker.test.ts +0 -113
  150. package/src/__tests__/freshness.test.ts +0 -186
  151. package/src/__tests__/graph-export.test.ts +0 -489
  152. package/src/__tests__/hotness-blend.test.ts +0 -283
  153. package/src/__tests__/ingest-chunking.test.ts +0 -44
  154. package/src/__tests__/ingest-scope-prefix.test.ts +0 -69
  155. package/src/__tests__/ingest.test.ts +0 -360
  156. package/src/__tests__/interference-detector.test.ts +0 -203
  157. package/src/__tests__/kg-extractor.test.ts +0 -192
  158. package/src/__tests__/kg-store.test.ts +0 -300
  159. package/src/__tests__/llm-client-pivot.test.ts +0 -628
  160. package/src/__tests__/llm-consolidation.test.ts +0 -226
  161. package/src/__tests__/maintenance-locks.test.ts +0 -197
  162. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  163. package/src/__tests__/mcp-related-scopes.test.ts +0 -151
  164. package/src/__tests__/memory-boundaries.test.ts +0 -282
  165. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -362
  166. package/src/__tests__/memory-evolution.test.ts +0 -271
  167. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  168. package/src/__tests__/memory-lint.test.ts +0 -421
  169. package/src/__tests__/memory-output.test.ts +0 -224
  170. package/src/__tests__/memory-promotion.test.ts +0 -315
  171. package/src/__tests__/memory-utility.test.ts +0 -256
  172. package/src/__tests__/migrate-dedup-ids.test.ts +0 -65
  173. package/src/__tests__/multi-vector.test.ts +0 -242
  174. package/src/__tests__/narrative-architecture.test.ts +0 -456
  175. package/src/__tests__/noise-filter.test.ts +0 -205
  176. package/src/__tests__/noise-prototype-bank.test.ts +0 -280
  177. package/src/__tests__/obsidian-connector.test.ts +0 -363
  178. package/src/__tests__/omitted-reporting.test.ts +0 -98
  179. package/src/__tests__/openai-contract.test.ts +0 -513
  180. package/src/__tests__/pending-review.test.ts +0 -77
  181. package/src/__tests__/pii-redact.test.ts +0 -333
  182. package/src/__tests__/pivot-apply.test.ts +0 -373
  183. package/src/__tests__/pivot-distill-supervisor.test.ts +0 -457
  184. package/src/__tests__/pivot-distill.test.ts +0 -1178
  185. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  186. package/src/__tests__/ppr-traversal.test.ts +0 -256
  187. package/src/__tests__/prediction-engine.test.ts +0 -349
  188. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  189. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  190. package/src/__tests__/preference-matcher.test.ts +0 -282
  191. package/src/__tests__/preference-slots.test.ts +0 -205
  192. package/src/__tests__/promotion-distinct-sources.test.ts +0 -162
  193. package/src/__tests__/prospective-memory.test.ts +0 -283
  194. package/src/__tests__/pull-from-macbook-sync-only.test.ts +0 -679
  195. package/src/__tests__/query-entity-detector.test.ts +0 -171
  196. package/src/__tests__/query-expander-alias.test.ts +0 -146
  197. package/src/__tests__/query-expander.test.ts +0 -34
  198. package/src/__tests__/recall-gate.test.ts +0 -266
  199. package/src/__tests__/recall-governor.test.ts +0 -227
  200. package/src/__tests__/recall-mode.test.ts +0 -64
  201. package/src/__tests__/resolve-db-path.test.ts +0 -130
  202. package/src/__tests__/resolve-tier.test.ts +0 -64
  203. package/src/__tests__/result-synthesizer.test.ts +0 -198
  204. package/src/__tests__/resume-context.test.ts +0 -4898
  205. package/src/__tests__/retrieval-eval.test.ts +0 -163
  206. package/src/__tests__/retrieval-trace.test.ts +0 -213
  207. package/src/__tests__/retrieve-audit-revision.test.ts +0 -255
  208. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  209. package/src/__tests__/retriever-lazy-init-dispatch.test.ts +0 -123
  210. package/src/__tests__/retroactive-boost.test.ts +0 -223
  211. package/src/__tests__/salience-scoring.test.ts +0 -106
  212. package/src/__tests__/scope-inventory.test.ts +0 -200
  213. package/src/__tests__/scope-policy.test.ts +0 -69
  214. package/src/__tests__/scope-suggester.test.ts +0 -80
  215. package/src/__tests__/session-checkpoint.test.ts +0 -291
  216. package/src/__tests__/session-distiller.test.ts +0 -640
  217. package/src/__tests__/session-gc.test.ts +0 -215
  218. package/src/__tests__/skill-engine.test.ts +0 -552
  219. package/src/__tests__/skill-promotion.test.ts +0 -654
  220. package/src/__tests__/skill-verifier.test.ts +0 -122
  221. package/src/__tests__/source-heartbeat.test.ts +0 -254
  222. package/src/__tests__/stable-context-length-contract.test.ts +0 -62
  223. package/src/__tests__/store-patch-metadata.test.ts +0 -235
  224. package/src/__tests__/store-read-consistency.test.ts +0 -122
  225. package/src/__tests__/store-schema-migration.test.ts +0 -87
  226. package/src/__tests__/store-scope-audit.test.ts +0 -81
  227. package/src/__tests__/store-upsert.test.ts +0 -201
  228. package/src/__tests__/supersede-chain.test.ts +0 -154
  229. package/src/__tests__/synthesis-contract.test.ts +0 -252
  230. package/src/__tests__/synthesis-promotion.test.ts +0 -369
  231. package/src/__tests__/temporal-parser.test.ts +0 -241
  232. package/src/__tests__/temporal-validity.test.ts +0 -160
  233. package/src/__tests__/term-registry-terminal-roster.test.ts +0 -127
  234. package/src/__tests__/term-registry.test.ts +0 -121
  235. package/src/__tests__/tool-error-surfacing.test.ts +0 -98
  236. package/src/__tests__/tool-output-compressor.test.ts +0 -160
  237. package/src/__tests__/topic-tag.test.ts +0 -113
  238. package/src/__tests__/trusted-memory.test.ts +0 -867
  239. package/src/__tests__/ui-server-security.test.ts +0 -93
  240. package/src/__tests__/usage-tracker.test.ts +0 -287
  241. package/src/__tests__/utility-weight.test.ts +0 -57
  242. package/src/__tests__/verbatim-self-recall.test.ts +0 -279
  243. package/src/__tests__/version-manager.test.ts +0 -248
  244. package/src/__tests__/workflow-observation-skill-outcome.test.ts +0 -352
  245. package/src/__tests__/workflow-observation.test.ts +0 -433
  246. package/src/__tests__/write-verifier.test.ts +0 -122
@@ -0,0 +1,1437 @@
1
+ /**
2
+ * 记忆文件对账:让 memory 源的文档切片与现行记忆文件一一对应。
3
+ *
4
+ * 为什么有这个模块:`ingestMarkdownFiles` 只增不删、每段新文本都要过全库判重,于是记忆文件改过以后
5
+ * 旧版切片永远留在库里被召回,新版又常被自己的旧版判重挡在门外(2026-09-24 实测:活跃的记忆文档切片
6
+ * 68% 对不上任何现行切片;现行文字 687 种没有活跃代表)。
7
+ *
8
+ * 做法:每轮全量规划——记忆目录顶层 *.md 的现行切片(parseMarkdown → redactSecrets → toWellFormed →
9
+ * normalizeDedupText)对 scope "memory"(精确,不含 memory:pivot)的全部行:
10
+ * - 现行文本已有活跃行 → 保留一条,多余的文档切片以 duplicate-text 下架;
11
+ * - 活跃文档切片的文本不在任何现行文件里 → 软下架(evolution.status = "archived" + reconcile 标记,不删行);
12
+ * - 现行文本没有活跃行 → 按 planMemoryReconcile() 第二遍的顺序:合并链完好算有代表 / forget 过的不补回 /
13
+ * 恢复对账下架过的行 / 恢复合并链已断的行 / 被别的机制下架的不推翻 / 否则插入(不过 dedupCheck,
14
+ * 只插库里不存在的 id)。
15
+ *
16
+ * 执行顺序是「插入 → 恢复 → 下架」:先把代表补齐再撤旧的。下架还要等两件事都成立——所在文件的新段落都有了着落、
17
+ * 依赖它的合并链成员已经恢复成功——否则延后到下一轮。
18
+ * 写入时整轮持有三把锁:自己的 memory-reconcile、dream 合并用的 consolidate-memory、GC 用的 gc-run(都是忙就跳过,
19
+ * 持有者进程还活着就绝不抢),所以规划到提交之间 dream / GC 不会插进来拿旧判断写入;其余写方(MCP 写入、访问计数)
20
+ * 由提交时核对兜住。每批写库前先落一条意图日志,写库后再落前后值,崩溃在中间也能撤销。
21
+ *
22
+ * 只在 sources.memory.path 显式配置时使用(auto 路径仍走 ingestMarkdownFiles,行为不变)。
23
+ * 方案、三方互审与上线前的代码单审:sync-bridge AI产出/2026-09-24-recallnest-文档切片不下架/plan.md、mr-plan/。
24
+ */
25
+
26
+ import {
27
+ appendFileSync,
28
+ closeSync,
29
+ existsSync,
30
+ fstatSync,
31
+ mkdirSync,
32
+ openSync,
33
+ readFileSync,
34
+ readSync,
35
+ readdirSync,
36
+ rmSync,
37
+ statSync,
38
+ utimesSync,
39
+ writeFileSync,
40
+ } from "node:fs";
41
+ import { dirname, join } from "node:path";
42
+
43
+ import type { AuditLogger } from "./audit-log.js";
44
+ import { inspectLock, lockPathForKey, withLock } from "./distill-lock.js";
45
+ import type { Embedder } from "./embedder.js";
46
+ import * as envConfig from "./env-config.js";
47
+ import { buildIngestedEntry, generateCoreSummaries, parseMarkdown, smartExtractBatch } from "./ingest.js";
48
+ import type { LLMClient } from "./llm-client.js";
49
+ import { patchEvolutionOnMeta, type EvolutionMetadata } from "./memory-evolution.js";
50
+ import { redactSecrets } from "./pii-detector.js";
51
+ import { deterministicId, type MemoryEntry, type MemoryStore } from "./store.js";
52
+ import { fingerprintNormalized, normalizeDedupText, textFingerprint } from "./text-fingerprint.js";
53
+
54
+ export const MEMORY_DOC_SCOPE = "memory";
55
+ export const DEFAULT_MAX_RETIRE_FLOOR = 300;
56
+ export const DEFAULT_MAX_RETIRE_FRACTION = 0.25;
57
+ export const DIR_SANITY_MIN_RATIO = 0.5;
58
+ /** 同一种护栏原因多久再报一次警(在此之间照常在日志里写 ⚠️,但不以退出码 3 结束) */
59
+ export const GUARD_ALERT_INTERVAL_MS = 24 * 3_600_000;
60
+ /** 锁被一个活着的进程占着超过这么久没更新,就不只是「本轮跳过」,要报警(不然对账会一直静默跳过) */
61
+ export const LOCK_STALE_ALERT_MS = 6 * 3_600_000;
62
+ const INSERT_BATCH = 32;
63
+ const PATCH_BATCH = 200;
64
+ const RECONCILE_NOTE_PREFIX = "memory-reconcile:";
65
+ const LOCK_HEARTBEAT_MS = 60_000;
66
+
67
+ /**
68
+ * 写入时与对账互斥的锁。键名必须与持锁方逐字一致:
69
+ * - `consolidate-memory`:dream 的合并阶段(dream-pipeline.ts `consolidate-${scope}`)与 MCP
70
+ * consolidate_memories(mcp-tools-governance.ts)对 scope "memory" 用的同一把锁;
71
+ * - `gc-run`:auto-gc.ts 全库归档扫描的运行锁。
72
+ * 对账整轮持有它们(忙就跳过本轮),dream / GC 也就不会在对账的规划与提交之间拿旧判断写入。
73
+ */
74
+ export const RECONCILE_LOCK_KEYS = ["memory-reconcile", `consolidate-${MEMORY_DOC_SCOPE}`, "gc-run"] as const;
75
+ /**
76
+ * 对账自己持锁时登记的「预计最长持有」:不设上限,不触发 distill-lock 的超时告警(对账跑多久都正常)。
77
+ * 「持有者进程还活着就绝不抢」现在由内核保证:锁是 flock,活着的持有者谁都拿不走,进程一退出或被杀就自动释放,
78
+ * 跟双方各自传的时限无关(上线前代码单审 C2;PR #27 互审:持锁方的 10 / 30 分钟租期曾被当成可抢的时限)。
79
+ * 活着却长期不动的持有者由 LOCK_STALE_ALERT_MS 报警兜住。
80
+ */
81
+ const NEVER_EXPIRE_MS = Number.MAX_SAFE_INTEGER;
82
+
83
+ export type RetireReason =
84
+ | "not-in-current-files"
85
+ | "duplicate-text"
86
+ | "orphan-file"
87
+ | "sync-conflict-file";
88
+
89
+ export type ReactivateKind = "reconcile-retired" | "broken-consolidation";
90
+
91
+ export interface CurrentChunk {
92
+ file: string;
93
+ idx: number;
94
+ heading: string;
95
+ /** redactSecrets + toWellFormed 之后的原文,也是入库文本 */
96
+ text: string;
97
+ norm: string;
98
+ }
99
+
100
+ export interface ExistingRow {
101
+ id: string;
102
+ scope: string;
103
+ text: string;
104
+ timestamp: number;
105
+ meta: Record<string, unknown>;
106
+ }
107
+
108
+ /** 规划时看到的、提交前要核对的状态 */
109
+ export interface RowState {
110
+ status: string;
111
+ evolutionNote: string | null;
112
+ consolidatedInto: string | null;
113
+ reconcileAction: string | null;
114
+ }
115
+
116
+ /** 撤销要用的改动字段(前值 / 后值) */
117
+ export interface JournalFields {
118
+ status: string;
119
+ validUntil: unknown;
120
+ evolutionNote: string | null;
121
+ consolidatedInto: string | null;
122
+ reconcile: unknown;
123
+ }
124
+
125
+ export interface PlannedRetire {
126
+ id: string;
127
+ file: string | null;
128
+ reason: RetireReason;
129
+ norm: string;
130
+ expect: RowState;
131
+ /** 规划时这一行的改动字段,写进意图日志,崩溃在写库与写结果之间时撤销靠它 */
132
+ before: JournalFields;
133
+ }
134
+
135
+ export interface PlannedReactivate {
136
+ id: string;
137
+ file: string | null;
138
+ kind: ReactivateKind;
139
+ /**
140
+ * 同一段文字的全部合并成员里,合并目标本轮要被下架的那些目标 id。这段文字此刻还靠它们撑着(链还没断),
141
+ * 恢复没成功就不能下架它们(上线前代码单审 N3、第二次单审 F2:不只看被选中的那一行)。
142
+ */
143
+ dependentRetireIds: string[];
144
+ /** 等价于 dependentRetireIds 非空:链是本轮下架才断的(下架被护栏拦下时,broken-consolidation 不恢复) */
145
+ targetRetiredThisRun: boolean;
146
+ norm: string;
147
+ expect: RowState;
148
+ before: JournalFields;
149
+ }
150
+
151
+ export interface PlannedInsert {
152
+ id: string;
153
+ chunk: CurrentChunk;
154
+ /** 这段文字出现在的全部现行文件(同一段可能出现在几个文件里;插入失败时这些文件的旧切片都要等) */
155
+ files: string[];
156
+ /** 这段文字的确定性 id 已被另一段正文占着(见 altInsertId),此时 id 是备用 id,这里记原 id */
157
+ occupiedPrimaryId?: string;
158
+ }
159
+
160
+ export interface MemoryReconcilePlan {
161
+ currentFiles: number;
162
+ currentChunks: number;
163
+ currentDistinct: number;
164
+ existingRows: number;
165
+ activeDocRows: number;
166
+ filesWithActiveDocRows: number;
167
+ keep: number;
168
+ insert: PlannedInsert[];
169
+ retire: PlannedRetire[];
170
+ reactivate: PlannedReactivate[];
171
+ /** 现行文本只剩被合并的行,但它合并进的那一行本轮之后仍活跃 */
172
+ representedByConsolidation: number;
173
+ /** 例外①:forget 过的文本,不补回 */
174
+ exceptionForgotten: number;
175
+ /** 例外②:同文只剩被别的机制(GC 归档、superseded 等)下架的行,不推翻 */
176
+ exceptionOtherInactive: number;
177
+ exceptionOtherInactiveIds: string[];
178
+ /** 插入里有几段的确定性 id 被另一段正文占着、改用备用 id */
179
+ insertAltId: number;
180
+ }
181
+
182
+ // ---------------------------------------------------------------------------
183
+ // 行状态小工具
184
+ // ---------------------------------------------------------------------------
185
+
186
+ function asRecord(value: unknown): Record<string, unknown> | null {
187
+ return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : null;
188
+ }
189
+
190
+ function evolutionOf(meta: Record<string, unknown>): Record<string, unknown> {
191
+ return asRecord(meta.evolution) ?? {};
192
+ }
193
+
194
+ export function rowState(meta: Record<string, unknown>): RowState {
195
+ const evo = evolutionOf(meta);
196
+ const action = asRecord(meta.reconcile)?.action;
197
+ return {
198
+ status: typeof evo.status === "string" ? evo.status : "active",
199
+ evolutionNote: typeof evo.evolutionNote === "string" ? evo.evolutionNote : null,
200
+ consolidatedInto: typeof evo.consolidatedInto === "string" ? evo.consolidatedInto : null,
201
+ reconcileAction: typeof action === "string" ? action : null,
202
+ };
203
+ }
204
+
205
+ export function journalFields(meta: Record<string, unknown>): JournalFields {
206
+ const evo = evolutionOf(meta);
207
+ return {
208
+ status: typeof evo.status === "string" ? evo.status : "active",
209
+ validUntil: evo.validUntil ?? null,
210
+ evolutionNote: typeof evo.evolutionNote === "string" ? evo.evolutionNote : null,
211
+ consolidatedInto: typeof evo.consolidatedInto === "string" ? evo.consolidatedInto : null,
212
+ reconcile: meta.reconcile ?? null,
213
+ };
214
+ }
215
+
216
+ export function isActiveStatus(status: string): boolean {
217
+ return status === "active" || status === "pending_review";
218
+ }
219
+
220
+ export function isMemoryDocSlice(meta: Record<string, unknown>): boolean {
221
+ return meta.source === MEMORY_DOC_SCOPE && asRecord(meta.boundary)?.authority === "document-ingest";
222
+ }
223
+
224
+ export function isSyncConflictFile(name: string): boolean {
225
+ return /sync-conflict/i.test(name);
226
+ }
227
+
228
+ function isReconcileRetired(state: RowState): boolean {
229
+ return state.status === "archived"
230
+ && (state.evolutionNote ?? "").startsWith(RECONCILE_NOTE_PREFIX)
231
+ && state.reconcileAction === "retired";
232
+ }
233
+
234
+ function isForgottenNote(state: RowState): boolean {
235
+ return (state.evolutionNote ?? "").startsWith("forgotten:");
236
+ }
237
+
238
+ function fileOf(meta: Record<string, unknown>): string | null {
239
+ return typeof meta.file === "string" ? meta.file : null;
240
+ }
241
+
242
+ function accessCountOf(meta: Record<string, unknown>): number {
243
+ const n = Number(evolutionOf(meta).accessCount ?? 0);
244
+ return Number.isFinite(n) ? n : 0;
245
+ }
246
+
247
+ /** 多条候选里挑一条:原文与现行切片逐字相同者优先,其次访问多者,再次时间早者,最后按 id 定序。 */
248
+ function pickPreferred(rows: ExistingRow[], exactText: string): ExistingRow {
249
+ return [...rows].sort((a, b) => {
250
+ const ea = a.text === exactText ? 1 : 0;
251
+ const eb = b.text === exactText ? 1 : 0;
252
+ if (ea !== eb) return eb - ea;
253
+ const ca = accessCountOf(a.meta);
254
+ const cb = accessCountOf(b.meta);
255
+ if (ca !== cb) return cb - ca;
256
+ if (a.timestamp !== b.timestamp) return a.timestamp - b.timestamp;
257
+ return a.id < b.id ? -1 : 1;
258
+ })[0];
259
+ }
260
+
261
+ /**
262
+ * 备用插入 id。现行文字的确定性 id 可能已被另一段正文占着(某个子系统原地改写过那一行的正文、id 不变)。
263
+ * 这时插入会撞 id 被跳过、那一行又因正文不在现行文件而下架,这段文字就一个活跃代表都没有。
264
+ * 用固定推导的备用 id 插入:可重跑、下一轮同文已有活跃行就不会再插。
265
+ */
266
+ export function altInsertId(text: string): string {
267
+ return deterministicId(MEMORY_DOC_SCOPE, `${text}\u0000memory-reconcile:alt-id`);
268
+ }
269
+
270
+ function parseMeta(raw: string | undefined): Record<string, unknown> {
271
+ try {
272
+ return asRecord(JSON.parse(raw || "{}")) ?? {};
273
+ } catch {
274
+ return {}; // 坏 metadata 按空处理:isMemoryDocSlice 为 false,永不被下架
275
+ }
276
+ }
277
+
278
+ /** 行上存的前值里再带前值会越套越深:只留一层 */
279
+ function stripPrev(reconcile: unknown): unknown {
280
+ const r = asRecord(reconcile);
281
+ if (!r) return reconcile ?? null;
282
+ const rest: Record<string, unknown> = { ...r };
283
+ delete rest.prev;
284
+ return rest;
285
+ }
286
+
287
+ // ---------------------------------------------------------------------------
288
+ // 规划(纯函数)
289
+ // ---------------------------------------------------------------------------
290
+
291
+ export interface ForgetSet {
292
+ /** forget 事件的 memoryId */
293
+ ids: Set<string>;
294
+ /** forget 事件记下的归一文本指纹(2026-09-24 起 forget 引擎在 details 里写 `norm=<指纹>`;更早的事件没有) */
295
+ norms: Set<string>;
296
+ }
297
+
298
+ export interface PlanInput {
299
+ /** 现行切片(已跳过 sync-conflict 文件) */
300
+ current: CurrentChunk[];
301
+ /** 现行记忆目录顶层 *.md 文件名(含 sync-conflict 文件,用来区分「文件已删」与「冲突副本」) */
302
+ currentFileNames: Set<string>;
303
+ /** scope 恰为 "memory" 的全部行(任何来源、任何状态) */
304
+ existing: ExistingRow[];
305
+ /** 审计日志里 scope "memory" 的 forget 事件 id */
306
+ forgottenIds: Set<string>;
307
+ /** 同上事件里的归一文本指纹 */
308
+ forgottenNorms?: Set<string>;
309
+ /** existing 之外、被 consolidatedInto 指到的行的状态(比如合并进了别的 scope) */
310
+ externalStatus?: Map<string, string>;
311
+ }
312
+
313
+ export function planMemoryReconcile(input: PlanInput): MemoryReconcilePlan {
314
+ const { current, currentFileNames, existing, forgottenIds } = input;
315
+ const forgottenNorms = input.forgottenNorms ?? new Set<string>();
316
+ const externalStatus = input.externalStatus ?? new Map<string, string>();
317
+
318
+ const curByNorm = new Map<string, CurrentChunk>();
319
+ const filesByNorm = new Map<string, Set<string>>();
320
+ for (const chunk of current) {
321
+ if (!curByNorm.has(chunk.norm)) curByNorm.set(chunk.norm, chunk);
322
+ const files = filesByNorm.get(chunk.norm) ?? new Set<string>();
323
+ files.add(chunk.file);
324
+ filesByNorm.set(chunk.norm, files);
325
+ }
326
+
327
+ const rowsByNorm = new Map<string, ExistingRow[]>();
328
+ const existingIds = new Set(existing.map((r) => r.id));
329
+ const statusById = new Map<string, string>();
330
+ const normById = new Map<string, string>();
331
+ const activeDocFiles = new Set<string>();
332
+ let activeDocRows = 0;
333
+ for (const row of existing) {
334
+ const norm = normalizeDedupText(row.text);
335
+ normById.set(row.id, norm);
336
+ const list = rowsByNorm.get(norm) ?? [];
337
+ list.push(row);
338
+ rowsByNorm.set(norm, list);
339
+ const state = rowState(row.meta);
340
+ statusById.set(row.id, state.status);
341
+ if (isActiveStatus(state.status) && isMemoryDocSlice(row.meta)) {
342
+ activeDocRows++;
343
+ const file = fileOf(row.meta);
344
+ if (file) activeDocFiles.add(file);
345
+ }
346
+ }
347
+
348
+ const retire: PlannedRetire[] = [];
349
+ const retireIds = new Set<string>();
350
+ const addRetire = (row: ExistingRow, reason: RetireReason, norm: string) => {
351
+ if (retireIds.has(row.id)) return;
352
+ retireIds.add(row.id);
353
+ retire.push({ id: row.id, file: fileOf(row.meta), reason, norm, expect: rowState(row.meta), before: journalFields(row.meta) });
354
+ };
355
+
356
+ // 第一遍:现行文本的活跃行——留一条,多余的文档切片下架
357
+ let keep = 0;
358
+ for (const [norm, chunk] of curByNorm) {
359
+ const active = (rowsByNorm.get(norm) ?? []).filter((r) => isActiveStatus(rowState(r.meta).status));
360
+ if (active.length === 0) continue;
361
+ keep++;
362
+ const keeper = pickPreferred(active, chunk.text);
363
+ for (const row of active) {
364
+ if (row.id !== keeper.id && isMemoryDocSlice(row.meta)) addRetire(row, "duplicate-text", norm);
365
+ }
366
+ }
367
+ // 第一遍(续):文本已不在任何现行文件里的活跃文档切片
368
+ for (const row of existing) {
369
+ const norm = normById.get(row.id)!;
370
+ if (curByNorm.has(norm)) continue;
371
+ if (!isMemoryDocSlice(row.meta) || !isActiveStatus(rowState(row.meta).status)) continue;
372
+ const file = fileOf(row.meta);
373
+ const reason: RetireReason = file && isSyncConflictFile(file)
374
+ ? "sync-conflict-file"
375
+ : file && !currentFileNames.has(file)
376
+ ? "orphan-file"
377
+ : "not-in-current-files";
378
+ addRetire(row, reason, norm);
379
+ }
380
+
381
+ // 第二遍:没有活跃行的现行文本
382
+ const insert: PlannedInsert[] = [];
383
+ const reactivate: PlannedReactivate[] = [];
384
+ let representedByConsolidation = 0;
385
+ let exceptionForgotten = 0;
386
+ let exceptionOtherInactive = 0;
387
+ const exceptionOtherInactiveIds: string[] = [];
388
+ let insertAltId = 0;
389
+
390
+ const targetStaysActive = (targetId: string | null): boolean => {
391
+ if (!targetId || retireIds.has(targetId)) return false;
392
+ const status = statusById.get(targetId) ?? externalStatus.get(targetId);
393
+ return status !== undefined && isActiveStatus(status);
394
+ };
395
+
396
+ for (const [norm, chunk] of curByNorm) {
397
+ const rows = rowsByNorm.get(norm) ?? [];
398
+ if (rows.some((r) => isActiveStatus(rowState(r.meta).status))) continue;
399
+
400
+ // 1. 合并链完好:合并目标本轮之后仍活跃 → 已有代表
401
+ const consolidated = rows.filter((r) => rowState(r.meta).status === "consolidated");
402
+ if (consolidated.some((r) => targetStaysActive(rowState(r.meta).consolidatedInto))) {
403
+ representedByConsolidation++;
404
+ continue;
405
+ }
406
+ // 2. forget 过:审计里有这段文字的确定性 id / 备用 id / 归一指纹,或还没删掉的同文行带 forgotten: 标记
407
+ // → 不补回,也不恢复同文的别的行(forget 删的是一行,要忘的是这段文字)
408
+ const id = deterministicId(MEMORY_DOC_SCOPE, chunk.text);
409
+ const altId = altInsertId(chunk.text);
410
+ if (forgottenIds.has(id) || forgottenIds.has(altId) || forgottenNorms.has(fingerprintNormalized(norm))
411
+ || rows.some((r) => isForgottenNote(rowState(r.meta)))) {
412
+ exceptionForgotten++;
413
+ continue;
414
+ }
415
+ // 同文组全部合并成员里、本轮要被下架的合并目标:恢复没成功时它们要延后下架
416
+ const dependentRetireIds = [...new Set(consolidated
417
+ .map((r) => rowState(r.meta).consolidatedInto)
418
+ .filter((t): t is string => t !== null && retireIds.has(t)))];
419
+ // 3. 对账下架过的 → 恢复
420
+ const retiredByUs = rows.filter((r) => isMemoryDocSlice(r.meta) && isReconcileRetired(rowState(r.meta)));
421
+ if (retiredByUs.length > 0) {
422
+ const row = pickPreferred(retiredByUs, chunk.text);
423
+ reactivate.push({ id: row.id, file: fileOf(row.meta), kind: "reconcile-retired", dependentRetireIds, targetRetiredThisRun: dependentRetireIds.length > 0, norm, expect: rowState(row.meta), before: journalFields(row.meta) });
424
+ continue;
425
+ }
426
+ // 4. 合并链已断(目标不活跃、不存在,或本轮要被下架)→ 恢复被合并的那一行
427
+ const brokenConsolidation = consolidated.filter((r) => isMemoryDocSlice(r.meta));
428
+ if (brokenConsolidation.length > 0) {
429
+ const row = pickPreferred(brokenConsolidation, chunk.text);
430
+ reactivate.push({
431
+ id: row.id,
432
+ file: fileOf(row.meta),
433
+ kind: "broken-consolidation",
434
+ dependentRetireIds,
435
+ targetRetiredThisRun: dependentRetireIds.length > 0,
436
+ norm,
437
+ expect: rowState(row.meta),
438
+ before: journalFields(row.meta),
439
+ });
440
+ continue;
441
+ }
442
+ // 5. 只剩被别的机制下架的行 → 不推翻,计数
443
+ if (rows.length > 0) {
444
+ exceptionOtherInactive++;
445
+ exceptionOtherInactiveIds.push(...rows.map((r) => r.id));
446
+ continue;
447
+ }
448
+ // 6. 库里没有 → 插入。走到这里说明 scope 内没有同文行,确定性 id 若已存在,必是被另一段正文占着 → 用备用 id
449
+ const files = [...(filesByNorm.get(norm) ?? new Set([chunk.file]))].sort();
450
+ if (existingIds.has(id)) {
451
+ insert.push({ id: altId, chunk, files, occupiedPrimaryId: id });
452
+ insertAltId++;
453
+ } else {
454
+ insert.push({ id, chunk, files });
455
+ }
456
+ }
457
+
458
+ return {
459
+ currentFiles: [...currentFileNames].filter((f) => !isSyncConflictFile(f)).length,
460
+ currentChunks: current.length,
461
+ currentDistinct: curByNorm.size,
462
+ existingRows: existing.length,
463
+ activeDocRows,
464
+ filesWithActiveDocRows: activeDocFiles.size,
465
+ keep,
466
+ insert,
467
+ retire,
468
+ reactivate,
469
+ representedByConsolidation,
470
+ exceptionForgotten,
471
+ exceptionOtherInactive,
472
+ exceptionOtherInactiveIds,
473
+ insertAltId,
474
+ };
475
+ }
476
+
477
+ // ---------------------------------------------------------------------------
478
+ // 收集输入
479
+ // ---------------------------------------------------------------------------
480
+
481
+ export function collectCurrentChunks(memDir: string): { chunks: CurrentChunk[]; fileNames: Set<string> } {
482
+ const fileNames = new Set(readdirSync(memDir).filter((f) => f.endsWith(".md")));
483
+ const chunks: CurrentChunk[] = [];
484
+ for (const file of [...fileNames].sort()) {
485
+ if (isSyncConflictFile(file)) continue;
486
+ const sections = parseMarkdown(join(memDir, file));
487
+ sections.forEach((section, idx) => {
488
+ // F-3b:与 ingestMarkdownFiles 一样先脱敏再入库,比对也用脱敏后的文本。
489
+ // toWellFormed:切片器按 UTF-16 下标切、带重叠回退,会把 emoji 等四字节字符切成半截;库存储时把半截换成
490
+ // U+FFFD,id 的哈希编码也同样替换,于是「id 相同、正文差一个字」——不先换掉,这段文字每轮都认不出自己在库里的那一行
491
+ // (2026-09-24 快照彩排:open-loops.md 一段开头是半截字符,按原文比对永远缺代表,所在文件的旧切片就永远下不掉)。
492
+ const text = redactSecrets(section.text).text.toWellFormed();
493
+ chunks.push({ file, idx, heading: section.heading, text, norm: normalizeDedupText(text) });
494
+ });
495
+ }
496
+ return { chunks, fileNames };
497
+ }
498
+
499
+ export async function loadMemoryScopeRows(store: MemoryStore): Promise<ExistingRow[]> {
500
+ const entries = await store.listPage({
501
+ scopeFilter: [MEMORY_DOC_SCOPE],
502
+ scopeMatch: "exact",
503
+ limit: 10_000_000,
504
+ });
505
+ return entries
506
+ .filter((entry) => entry.scope === MEMORY_DOC_SCOPE)
507
+ .map((entry) => ({ id: entry.id, scope: entry.scope, text: entry.text, timestamp: entry.timestamp, meta: parseMeta(entry.metadata) }));
508
+ }
509
+
510
+ const FORGET_NORM_RE = /(?:^|\s)norm=([0-9a-f]{16})(?:\s|$)/;
511
+
512
+ function collectForgets(chunk: string, into: ForgetSet): void {
513
+ for (const line of chunk.split("\n")) {
514
+ if (!line.includes("\"forget\"")) continue;
515
+ try {
516
+ const e = asRecord(JSON.parse(line));
517
+ if (!e || e.operation !== "forget" || e.scope !== MEMORY_DOC_SCOPE) continue;
518
+ if (typeof e.memoryId === "string") into.ids.add(e.memoryId);
519
+ const m = typeof e.details === "string" ? FORGET_NORM_RE.exec(e.details) : null;
520
+ if (m) into.norms.add(m[1]);
521
+ } catch { /* 跳过坏行 */ }
522
+ }
523
+ }
524
+
525
+ /** 这段文字是否已被 forget:行自己的 id、这段文字的确定性 id 或备用 id、归一指纹,任一在 forget 记录里都算 */
526
+ function isTextForgotten(forgets: ForgetSet, text: string, rowId?: string): boolean {
527
+ return (rowId !== undefined && forgets.ids.has(rowId))
528
+ || forgets.ids.has(deterministicId(MEMORY_DOC_SCOPE, text))
529
+ || forgets.ids.has(altInsertId(text))
530
+ || forgets.norms.has(textFingerprint(text));
531
+ }
532
+
533
+ /** 审计日志里 scope "memory" 的 forget 事件。文件不在就是空集(识别失效,不报错)。 */
534
+ export function loadForgetSet(auditPath: string): ForgetSet {
535
+ const set: ForgetSet = { ids: new Set(), norms: new Set() };
536
+ if (existsSync(auditPath)) collectForgets(readFileSync(auditPath, "utf-8"), set);
537
+ return set;
538
+ }
539
+
540
+ /**
541
+ * 跟着 audit.jsonl 往后读:每批插入写库前一刻刷新一次,规划之后才发生的 forget 也挡得住(互审 C3)。
542
+ * 只读新追加的字节;文件变短(被轮转 / 截断)就从头重读。
543
+ */
544
+ export class ForgetWatcher {
545
+ private offset = 0;
546
+ private carry = "";
547
+ readonly set: ForgetSet = { ids: new Set(), norms: new Set() };
548
+
549
+ constructor(private readonly auditPath: string) {
550
+ this.refresh();
551
+ }
552
+
553
+ refresh(): ForgetSet {
554
+ if (!existsSync(this.auditPath)) return this.set;
555
+ const fd = openSync(this.auditPath, "r");
556
+ try {
557
+ const size = fstatSync(fd).size;
558
+ if (size < this.offset) {
559
+ this.offset = 0;
560
+ this.carry = "";
561
+ }
562
+ if (size === this.offset) return this.set;
563
+ const buf = Buffer.alloc(size - this.offset);
564
+ readSync(fd, buf, 0, buf.length, this.offset);
565
+ this.offset = size;
566
+ const text = this.carry + buf.toString("utf-8");
567
+ const lastNl = text.lastIndexOf("\n");
568
+ this.carry = lastNl >= 0 ? text.slice(lastNl + 1) : text;
569
+ if (lastNl >= 0) collectForgets(text.slice(0, lastNl), this.set);
570
+ } finally {
571
+ closeSync(fd);
572
+ }
573
+ return this.set;
574
+ }
575
+
576
+ /** 这段文字(按它的入库 id 与归一文本)是否已被 forget */
577
+ isForgotten(id: string, norm: string): boolean {
578
+ return this.set.ids.has(id) || this.set.norms.has(fingerprintNormalized(norm));
579
+ }
580
+ }
581
+
582
+ // ---------------------------------------------------------------------------
583
+ // 执行
584
+ // ---------------------------------------------------------------------------
585
+
586
+ function stableStringify(value: unknown): string {
587
+ if (value === null || value === undefined || typeof value !== "object") return JSON.stringify(value ?? null);
588
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
589
+ const obj = value as Record<string, unknown>;
590
+ return `{${Object.keys(obj).sort().map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(",")}}`;
591
+ }
592
+
593
+ class Journal {
594
+ constructor(private readonly path: string | null) {
595
+ if (path) mkdirSync(dirname(path), { recursive: true });
596
+ }
597
+ /** 追加失败直接抛错:意图日志写不进去就不许写库 */
598
+ write(entries: object[]): void {
599
+ if (!this.path || entries.length === 0) return;
600
+ appendFileSync(this.path, entries.map((e) => JSON.stringify(e)).join("\n") + "\n");
601
+ }
602
+ }
603
+
604
+ export interface ApplyDeps {
605
+ store: MemoryStore;
606
+ embedder: Pick<Embedder, "embedBatchPassage">;
607
+ llm: LLMClient | null;
608
+ auditLogger?: Pick<AuditLogger, "log"> | null;
609
+ }
610
+
611
+ export interface ApplyResult {
612
+ inserted: number;
613
+ insertSkippedExisting: number;
614
+ insertSkippedForgotten: number;
615
+ /** 没插进去的(嵌入失败、空向量、批次出错、id 被另一段文字占着)——它们所在的文件本轮不下架 */
616
+ insertPending: number;
617
+ insertErrors: string[];
618
+ reactivatedReconcileRetired: number;
619
+ reactivatedBrokenConsolidation: number;
620
+ reactivateSkippedStateChanged: number;
621
+ /** 规划之后这段文字被 forget 了:不恢复(第二次代码单审 F1) */
622
+ reactivateSkippedForgotten: number;
623
+ retired: number;
624
+ /** 所在文件还有没插进去的段落,本轮先不下 */
625
+ retireDeferredPendingInsert: number;
626
+ /** 依赖它的合并链成员本轮没恢复成功,先不下 */
627
+ retireDeferredDependency: number;
628
+ retireSkippedStateChanged: number;
629
+ }
630
+
631
+ export interface ApplyContext {
632
+ runId: string;
633
+ journalPath: string | null;
634
+ /** 下架是否被护栏(上限)拦下;拦下时只插入与恢复(因目标本轮下架而断的合并链也不恢复) */
635
+ retireAllowed: boolean;
636
+ forgetWatcher?: ForgetWatcher | null;
637
+ now?: () => number;
638
+ }
639
+
640
+ interface InsertCandidate {
641
+ item: PlannedInsert;
642
+ vector: number[];
643
+ extraction: Awaited<ReturnType<typeof smartExtractBatch>>[number];
644
+ summary: string | null;
645
+ }
646
+
647
+ async function applyInserts(
648
+ deps: ApplyDeps,
649
+ inserts: PlannedInsert[],
650
+ ctx: ApplyContext & { isoNow: string; journal: Journal },
651
+ result: ApplyResult,
652
+ ): Promise<Set<string>> {
653
+ /** 已确认有着落的插入项(插进去了 / 库里已有同文 / forget 过不再插) */
654
+ const settled = new Set<string>();
655
+ for (let i = 0; i < inserts.length; i += INSERT_BATCH) {
656
+ const batch = inserts.slice(i, i + INSERT_BATCH);
657
+ try {
658
+ const vectors = await deps.embedder.embedBatchPassage(batch.map((item) => item.chunk.text));
659
+ const embedded: Array<{ item: PlannedInsert; vector: number[] }> = [];
660
+ const emptyFiles = new Set<string>();
661
+ batch.forEach((item, j) => {
662
+ const vector = vectors[j];
663
+ if (Array.isArray(vector) && vector.length > 0) embedded.push({ item, vector });
664
+ else emptyFiles.add(item.chunk.file);
665
+ });
666
+ if (emptyFiles.size > 0) {
667
+ result.insertErrors.push(`${batch.length - embedded.length} 段嵌入为空(${[...emptyFiles].join("、")})`);
668
+ }
669
+ if (embedded.length === 0) continue;
670
+ const texts = embedded.map(({ item }) => item.chunk.text);
671
+ const extractions = await smartExtractBatch(texts, deps.llm);
672
+ const summaries = await generateCoreSummaries(texts, deps.llm);
673
+ // 规划之后才 forget 的也不插(互审 C3):嵌入与 LLM 抽取都做完、写库前一刻再读一次审计日志
674
+ ctx.forgetWatcher?.refresh();
675
+ const toInsert: InsertCandidate[] = [];
676
+ embedded.forEach(({ item, vector }, j) => {
677
+ if (ctx.forgetWatcher?.isForgotten(item.id, item.chunk.norm)) {
678
+ result.insertSkippedForgotten++;
679
+ settled.add(item.id);
680
+ return;
681
+ }
682
+ toInsert.push({ item, vector, extraction: extractions[j], summary: summaries[j] });
683
+ });
684
+ if (toInsert.length === 0) continue;
685
+ const entries = toInsert.map(({ item, vector, extraction, summary }) => {
686
+ const built = buildIngestedEntry({
687
+ source: MEMORY_DOC_SCOPE,
688
+ scope: MEMORY_DOC_SCOPE,
689
+ text: item.chunk.text,
690
+ vector,
691
+ extraction,
692
+ file: item.chunk.file,
693
+ heading: item.chunk.heading,
694
+ coreSummary: summary,
695
+ });
696
+ const meta = parseMeta(built.metadata);
697
+ meta.reconcile = { v: 1, action: "inserted", at: ctx.isoNow, run: ctx.runId };
698
+ return { ...built, category: built.category as MemoryEntry["category"], id: item.id, metadata: JSON.stringify(meta) };
699
+ });
700
+ // 写库前先落意图(含插入后应有的字段与正文指纹):崩溃在写库与写结果之间时,撤销照样能核对这一行之后有没有被改过
701
+ ctx.journal.write([{
702
+ run: ctx.runId,
703
+ phase: "intent",
704
+ action: "insert",
705
+ items: entries.map((e) => ({ id: e.id, after: journalFields(parseMeta(e.metadata)), textFp: textFingerprint(e.text) })),
706
+ }]);
707
+ const res = await deps.store.insertIfAbsent(entries);
708
+ result.inserted += res.inserted.length;
709
+ for (const id of res.inserted) settled.add(id);
710
+ const itemById = new Map(toInsert.map(({ item }) => [item.id, item]));
711
+ // 撞 id 被跳过:只有占着这个 id 的行正是同一段文字(别的写方刚插了同文),才算有了着落;
712
+ // 正文不同说明 id 被另一段文字占着,这段文字仍没有代表——记为没插进去(所在文件本轮不下架、要人看)
713
+ const skippedLines: object[] = [];
714
+ for (const id of res.skippedExisting) {
715
+ const item = itemById.get(id)!;
716
+ const holder = await deps.store.getById(id);
717
+ const sameText = !!holder && normalizeDedupText(holder.text) === item.chunk.norm;
718
+ if (sameText) {
719
+ result.insertSkippedExisting++;
720
+ settled.add(id);
721
+ } else {
722
+ result.insertErrors.push(`id ${id.slice(0, 8)} 已被另一段文字占着,未插入(${item.chunk.file})`);
723
+ }
724
+ // 没插进去也落一条结果:撤销据此知道这个 id 上的行不是本轮写的,不去动它
725
+ skippedLines.push({ run: ctx.runId, action: "insert", id, file: item.chunk.file, inserted: false, skipped: sameText ? "same-text-exists" : "id-occupied" });
726
+ }
727
+ ctx.journal.write(skippedLines);
728
+ const entryById = new Map(entries.map((e) => [e.id, e]));
729
+ ctx.journal.write(res.inserted.map((id) => {
730
+ const item = itemById.get(id)!;
731
+ const entry = entryById.get(id)!;
732
+ return {
733
+ run: ctx.runId,
734
+ action: "insert",
735
+ id,
736
+ file: item.chunk.file,
737
+ inserted: true,
738
+ after: journalFields(parseMeta(entry.metadata)),
739
+ textFp: textFingerprint(entry.text),
740
+ ...(item.occupiedPrimaryId ? { occupiedPrimaryId: item.occupiedPrimaryId } : {}),
741
+ };
742
+ }));
743
+ } catch (err) {
744
+ result.insertErrors.push(err instanceof Error ? err.message : String(err));
745
+ }
746
+ }
747
+ result.insertPending = inserts.filter((item) => !settled.has(item.id)).length;
748
+ return settled;
749
+ }
750
+
751
+ interface PatchItem {
752
+ id: string;
753
+ file: string | null;
754
+ norm: string;
755
+ expect: RowState;
756
+ before: JournalFields;
757
+ }
758
+
759
+ async function applyPatches<T extends PatchItem>(
760
+ deps: ApplyDeps,
761
+ planned: T[],
762
+ ctx: ApplyContext & { journal: Journal },
763
+ check: (item: T, meta: Record<string, unknown>, text: string, scope: string) => boolean,
764
+ mutate: (item: T, meta: Record<string, unknown>) => void,
765
+ describe: (item: T) => { action: string; detail: string; auditOp: "archive" | "update" },
766
+ ): Promise<{ applied: T[]; skipped: T[] }> {
767
+ const applied: T[] = [];
768
+ const skipped: T[] = [];
769
+ for (let i = 0; i < planned.length; i += PATCH_BATCH) {
770
+ const batch = planned.slice(i, i + PATCH_BATCH);
771
+ const byId = new Map(batch.map((item) => [item.id, item]));
772
+ const pending: object[] = [];
773
+ const batchApplied: T[] = [];
774
+ const seen = new Set<string>();
775
+ // 写库前先落意图:崩溃在写库与写结果之间时,撤销据此知道这些行可能被本轮写过,再按行上的本轮标记
776
+ // (提交时在写锁内记下的真实前值)核对与还原;这里的 before 是规划时的值,只供人工排查
777
+ ctx.journal.write([{
778
+ run: ctx.runId,
779
+ phase: "intent",
780
+ action: describe(batch[0]).action,
781
+ items: batch.map((item) => ({ id: item.id, before: item.before })),
782
+ }]);
783
+ await deps.store.patchMetadataBatch(batch.map((item) => ({
784
+ id: item.id,
785
+ patchFn: (meta, entry) => {
786
+ const planItem = byId.get(entry.id)!;
787
+ seen.add(entry.id);
788
+ if (!check(planItem, meta, entry.text, entry.scope)) {
789
+ skipped.push(planItem);
790
+ return meta;
791
+ }
792
+ const before = journalFields(meta);
793
+ mutate(planItem, meta);
794
+ const d = describe(planItem);
795
+ pending.push({ run: ctx.runId, action: d.action, id: entry.id, file: planItem.file, detail: d.detail, before, after: journalFields(meta) });
796
+ batchApplied.push(planItem);
797
+ return meta;
798
+ },
799
+ })));
800
+ for (const item of batch) if (!seen.has(item.id)) skipped.push(item); // 行已不在库
801
+ ctx.journal.write(pending);
802
+ applied.push(...batchApplied);
803
+ for (const item of batchApplied) {
804
+ const d = describe(item);
805
+ try {
806
+ deps.auditLogger?.log({ operation: d.auditOp, scope: MEMORY_DOC_SCOPE, memoryId: item.id, actor: "system", details: `${RECONCILE_NOTE_PREFIX} ${d.detail}` });
807
+ } catch { /* 审计失败不影响对账 */ }
808
+ }
809
+ }
810
+ return { applied, skipped };
811
+ }
812
+
813
+ /** 执行一份计划:插入 → 恢复 → 下架。调用方负责持锁。 */
814
+ export async function applyMemoryReconcile(
815
+ deps: ApplyDeps,
816
+ plan: MemoryReconcilePlan,
817
+ ctx: ApplyContext,
818
+ ): Promise<ApplyResult> {
819
+ const nowMs = (ctx.now ?? Date.now)();
820
+ const isoNow = new Date(nowMs).toISOString();
821
+ const journal = new Journal(ctx.journalPath);
822
+ const result: ApplyResult = {
823
+ inserted: 0,
824
+ insertSkippedExisting: 0,
825
+ insertSkippedForgotten: 0,
826
+ insertPending: 0,
827
+ insertErrors: [],
828
+ reactivatedReconcileRetired: 0,
829
+ reactivatedBrokenConsolidation: 0,
830
+ reactivateSkippedStateChanged: 0,
831
+ reactivateSkippedForgotten: 0,
832
+ retired: 0,
833
+ retireDeferredPendingInsert: 0,
834
+ retireDeferredDependency: 0,
835
+ retireSkippedStateChanged: 0,
836
+ };
837
+
838
+ // 1. 插入
839
+ const settled = await applyInserts(deps, plan.insert, { ...ctx, isoNow, journal }, result);
840
+ const pendingFiles = new Set(plan.insert.filter((item) => !settled.has(item.id)).flatMap((item) => item.files));
841
+
842
+ // 2. 恢复(在下架之前:先有代表再撤旧的)。因目标本轮下架而断的合并链,只在下架照常进行时才恢复。
843
+ // 恢复前再读一次审计日志:规划之后被 forget 的文字(删的可能是同文的另一行)不恢复(第二次代码单审 F1)。
844
+ ctx.forgetWatcher?.refresh();
845
+ const deferRetireIds = new Set<string>();
846
+ const toReactivate: PlannedReactivate[] = [];
847
+ for (const item of plan.reactivate) {
848
+ if (item.kind === "broken-consolidation" && item.targetRetiredThisRun && !ctx.retireAllowed) continue;
849
+ if (ctx.forgetWatcher?.isForgotten(item.id, item.norm)) {
850
+ result.reactivateSkippedForgotten++;
851
+ for (const t of item.dependentRetireIds) deferRetireIds.add(t);
852
+ continue;
853
+ }
854
+ toReactivate.push(item);
855
+ }
856
+ if (toReactivate.length > 0) {
857
+ const { applied, skipped } = await applyPatches(
858
+ deps,
859
+ toReactivate,
860
+ { ...ctx, journal },
861
+ (item, meta, text, scope) => {
862
+ const state = rowState(meta);
863
+ if (scope !== MEMORY_DOC_SCOPE || !isMemoryDocSlice(meta) || normalizeDedupText(text) !== item.norm) return false;
864
+ if (item.kind === "reconcile-retired") return isReconcileRetired(state);
865
+ return state.status === "consolidated" && state.consolidatedInto === item.expect.consolidatedInto;
866
+ },
867
+ (item, meta) => {
868
+ const prev = journalFields(meta);
869
+ patchEvolutionOnMeta(meta, {
870
+ status: "active",
871
+ validUntil: null,
872
+ evolutionNote: null,
873
+ ...(item.kind === "broken-consolidation" ? { consolidatedInto: null } : {}),
874
+ });
875
+ meta.reconcile = { v: 1, action: "reactivated", kind: item.kind, at: isoNow, run: ctx.runId, prev: { ...prev, reconcile: stripPrev(prev.reconcile) } };
876
+ },
877
+ (item) => ({ action: "reactivate", detail: item.kind, auditOp: "update" }),
878
+ );
879
+ result.reactivatedReconcileRetired = applied.filter((i) => i.kind === "reconcile-retired").length;
880
+ result.reactivatedBrokenConsolidation = applied.filter((i) => i.kind === "broken-consolidation").length;
881
+ result.reactivateSkippedStateChanged = skipped.length;
882
+ // 恢复没成功:这段文字此刻还靠那些本轮要下架的合并目标撑着,先别下(上线前代码单审 N3、第二次 F2)
883
+ for (const item of skipped) for (const t of item.dependentRetireIds) deferRetireIds.add(t);
884
+ }
885
+
886
+ // 3. 下架:重复文本随时可下(同文的保留行仍活跃);其余要等所在文件的新段落都有了着落;依赖未满足的都延后
887
+ if (ctx.retireAllowed && plan.retire.length > 0) {
888
+ const ready: PlannedRetire[] = [];
889
+ for (const item of plan.retire) {
890
+ if (deferRetireIds.has(item.id)) {
891
+ result.retireDeferredDependency++;
892
+ } else if (item.reason !== "duplicate-text" && item.file && pendingFiles.has(item.file)) {
893
+ result.retireDeferredPendingInsert++;
894
+ } else {
895
+ ready.push(item);
896
+ }
897
+ }
898
+ const { applied, skipped } = await applyPatches(
899
+ deps,
900
+ ready,
901
+ { ...ctx, journal },
902
+ (item, meta, text, scope) => scope === MEMORY_DOC_SCOPE
903
+ && isMemoryDocSlice(meta)
904
+ && normalizeDedupText(text) === item.norm
905
+ && rowState(meta).status === item.expect.status,
906
+ (item, meta) => {
907
+ const prev = journalFields(meta);
908
+ patchEvolutionOnMeta(meta, { status: "archived", validUntil: nowMs, evolutionNote: `${RECONCILE_NOTE_PREFIX} ${item.reason}` });
909
+ meta.reconcile = { v: 1, action: "retired", reason: item.reason, at: isoNow, run: ctx.runId, file: item.file, prev: { ...prev, reconcile: stripPrev(prev.reconcile) } };
910
+ },
911
+ (item) => ({ action: "retire", detail: item.reason, auditOp: "archive" }),
912
+ );
913
+ result.retired = applied.length;
914
+ result.retireSkippedStateChanged = skipped.length;
915
+ }
916
+
917
+ journal.write([{ run: ctx.runId, summary: result }]);
918
+ return result;
919
+ }
920
+
921
+ // ---------------------------------------------------------------------------
922
+ // 编排:持锁 → 收集 → 规划 → 护栏 → 执行
923
+ // ---------------------------------------------------------------------------
924
+
925
+ export interface ReconcileOptions extends ApplyDeps {
926
+ memDir: string;
927
+ /**
928
+ * 数据目录:锁(<dataDir>/locks)、审计(<dataDir>/audit.jsonl)、对账日志与护栏状态都在这里。
929
+ * 必须是其他进程用的同一个目录(resolve(envConfig.dataDir())),CLI 会核对它与库所在目录一致。
930
+ */
931
+ dataDir: string;
932
+ /** sources.memory.path 是否显式配置;auto 不允许对账 */
933
+ explicitPath: boolean;
934
+ apply: boolean;
935
+ /** 覆盖默认的单轮下架上限 */
936
+ maxRetire?: number;
937
+ now?: () => number;
938
+ }
939
+
940
+ export type GuardReason = "dir-sanity" | "cap" | "insert-failed" | "lock-stale";
941
+
942
+ export interface ReconcileOutcome {
943
+ ran: boolean;
944
+ mode: "dry-run" | "apply";
945
+ skippedReason?: string;
946
+ runId: string;
947
+ plan: MemoryReconcilePlan | null;
948
+ maxRetire: number;
949
+ /** 护栏:dir-sanity 整轮不写;cap 只拦下架;insert-failed 有段落没插进去;lock-stale 锁被活着的进程久占 */
950
+ guard: { reason: GuardReason; detail: string } | null;
951
+ applied: ApplyResult | null;
952
+ journalPath: string | null;
953
+ /** 要报警(护栏触发且距上次同因报警超过 24 小时)——ingest 据此以退出码 3 结束 */
954
+ alert: boolean;
955
+ }
956
+
957
+ export function resolveMaxRetire(activeDocRows: number, override?: number): number {
958
+ if (override !== undefined && Number.isFinite(override) && override >= 0) return Math.floor(override);
959
+ return envConfig.memoryReconcileMaxRetire()
960
+ ?? Math.max(DEFAULT_MAX_RETIRE_FLOOR, Math.ceil(activeDocRows * DEFAULT_MAX_RETIRE_FRACTION));
961
+ }
962
+
963
+ export function guardStatePath(dataDir: string): string {
964
+ return join(dataDir, "memory-reconcile-guard.json");
965
+ }
966
+
967
+ /**
968
+ * 护栏报警去抖:同一原因 24 小时内只报一次(第三轮 Kimi / Agy:护栏持续触发时每天最多 7 次报警会被当噪音)。
969
+ * 没有护栏触发时清掉状态,下次触发立刻报。
970
+ */
971
+ export function decideGuardAlert(dataDir: string, guard: ReconcileOutcome["guard"], nowMs: number): boolean {
972
+ const path = guardStatePath(dataDir);
973
+ if (!guard) {
974
+ if (existsSync(path)) rmSync(path, { force: true });
975
+ return false;
976
+ }
977
+ let last: Record<string, unknown> = {};
978
+ try {
979
+ last = asRecord(JSON.parse(readFileSync(path, "utf-8"))) ?? {};
980
+ } catch { /* 没有状态 = 没报过 */ }
981
+ const lastAlertAt = typeof last.lastAlertAt === "number" ? last.lastAlertAt : null;
982
+ const alert = last.reason !== guard.reason || lastAlertAt === null || nowMs - lastAlertAt >= GUARD_ALERT_INTERVAL_MS;
983
+ try {
984
+ mkdirSync(dataDir, { recursive: true });
985
+ writeFileSync(path, JSON.stringify({
986
+ reason: guard.reason,
987
+ detail: guard.detail,
988
+ lastAlertAt: alert ? nowMs : lastAlertAt,
989
+ lastSeenAt: nowMs,
990
+ }) + "\n");
991
+ } catch { /* 写不了状态就每次都报,宁可吵 */ }
992
+ return alert;
993
+ }
994
+
995
+ type LockedOutcome<T> = { ran: true; result: T } | { ran: false; busy: string };
996
+
997
+ async function withLocks<T>(keys: readonly string[], fn: () => Promise<T>): Promise<LockedOutcome<T>> {
998
+ if (keys.length === 0) return { ran: true, result: await fn() };
999
+ const [head, ...rest] = keys;
1000
+ let inner: LockedOutcome<T> = { ran: false, busy: head };
1001
+ const outcome = await withLock(head, async () => {
1002
+ inner = await withLocks(rest, fn);
1003
+ }, { onBusy: "skip", expireMs: NEVER_EXPIRE_MS });
1004
+ return outcome.ran ? inner : { ran: false, busy: head };
1005
+ }
1006
+
1007
+ /** 刷新所持锁文件的 mtime。锁不会因 mtime 旧被抢(flock),这里只让 inspectLock 报的「多久没更新」反映对账还在动。 */
1008
+ function touchLocks(keys: readonly string[]): void {
1009
+ const now = new Date();
1010
+ for (const key of keys) {
1011
+ try {
1012
+ utimesSync(lockPathForKey(key), now, now);
1013
+ } catch { /* 锁文件不在就算了 */ }
1014
+ }
1015
+ }
1016
+
1017
+ /** 拿齐三把锁、定期续期,再跑 fn;拿不齐就返回被谁占着 */
1018
+ async function underReconcileLocks<T>(fn: () => Promise<T>): Promise<LockedOutcome<T>> {
1019
+ return withLocks(RECONCILE_LOCK_KEYS, async () => {
1020
+ const heartbeat = setInterval(() => touchLocks(RECONCILE_LOCK_KEYS), LOCK_HEARTBEAT_MS);
1021
+ heartbeat.unref?.();
1022
+ try {
1023
+ return await fn();
1024
+ } finally {
1025
+ clearInterval(heartbeat);
1026
+ }
1027
+ });
1028
+ }
1029
+
1030
+ function busyDescription(key: string): string {
1031
+ return key === "memory-reconcile" ? "另一轮对账或撤销" : key === "gc-run" ? "GC" : "dream 合并";
1032
+ }
1033
+
1034
+ export async function reconcileMemoryDocuments(opts: ReconcileOptions): Promise<ReconcileOutcome> {
1035
+ const now = opts.now ?? Date.now;
1036
+ const runId = `mr-${new Date(now()).toISOString().replace(/[:.]/g, "-")}`;
1037
+ if (!opts.explicitPath) {
1038
+ throw new Error("memory 对账需要显式配置 sources.memory.path(当前为 auto);auto 路径请继续用 ingestMarkdownFiles");
1039
+ }
1040
+ if (!existsSync(opts.memDir) || !statSync(opts.memDir).isDirectory()) {
1041
+ throw new Error(`记忆目录不存在:${opts.memDir}`);
1042
+ }
1043
+
1044
+ const run = async (): Promise<ReconcileOutcome> => {
1045
+ const forgetWatcher = new ForgetWatcher(join(opts.dataDir, "audit.jsonl"));
1046
+ const { chunks, fileNames } = collectCurrentChunks(opts.memDir);
1047
+ const existing = await loadMemoryScopeRows(opts.store);
1048
+
1049
+ // 合并目标不在 scope "memory" 里的,单独查一次状态
1050
+ const known = new Set(existing.map((r) => r.id));
1051
+ const externalStatus = new Map<string, string>();
1052
+ for (const row of existing) {
1053
+ const target = rowState(row.meta).consolidatedInto;
1054
+ if (!target || known.has(target) || externalStatus.has(target)) continue;
1055
+ const entry = await opts.store.getById(target);
1056
+ if (entry) externalStatus.set(target, rowState(parseMeta(entry.metadata)).status);
1057
+ }
1058
+
1059
+ const plan = planMemoryReconcile({
1060
+ current: chunks,
1061
+ currentFileNames: fileNames,
1062
+ existing,
1063
+ forgottenIds: new Set(forgetWatcher.set.ids),
1064
+ forgottenNorms: new Set(forgetWatcher.set.norms),
1065
+ externalStatus,
1066
+ });
1067
+ const maxRetire = resolveMaxRetire(plan.activeDocRows, opts.maxRetire);
1068
+ let guard: ReconcileOutcome["guard"] = null;
1069
+ if (plan.filesWithActiveDocRows > 0 && plan.currentFiles < plan.filesWithActiveDocRows * DIR_SANITY_MIN_RATIO) {
1070
+ guard = { reason: "dir-sanity", detail: `记忆目录现行文件 ${plan.currentFiles} 个,不到有活跃文档切片的文件数 ${plan.filesWithActiveDocRows} 的一半,本轮不写入` };
1071
+ } else if (plan.retire.length > maxRetire) {
1072
+ guard = { reason: "cap", detail: `计划下架 ${plan.retire.length} 条,超过单轮上限 ${maxRetire},本轮只插入与恢复` };
1073
+ }
1074
+
1075
+ if (!opts.apply) {
1076
+ return { ran: true, mode: "dry-run", runId, plan, maxRetire, guard, applied: null, journalPath: null, alert: false };
1077
+ }
1078
+ if (guard?.reason === "dir-sanity") {
1079
+ return { ran: true, mode: "apply", runId, plan, maxRetire, guard, applied: null, journalPath: null, alert: decideGuardAlert(opts.dataDir, guard, now()) };
1080
+ }
1081
+
1082
+ const journalPath = join(opts.dataDir, "reconcile-journals", `memory-reconcile-${runId.slice(3)}.jsonl`);
1083
+ const applied = await applyMemoryReconcile(
1084
+ { store: opts.store, embedder: opts.embedder, llm: opts.llm, auditLogger: opts.auditLogger },
1085
+ plan,
1086
+ { runId, journalPath, retireAllowed: guard === null, forgetWatcher, now },
1087
+ );
1088
+ if (guard === null && applied.insertErrors.length > 0) {
1089
+ guard = { reason: "insert-failed", detail: `${applied.insertPending} 段没插进去(所在文件本轮不下架):${applied.insertErrors.slice(0, 3).join(";")}` };
1090
+ }
1091
+ return { ran: true, mode: "apply", runId, plan, maxRetire, guard, applied, journalPath, alert: decideGuardAlert(opts.dataDir, guard, now()) };
1092
+ };
1093
+
1094
+ // 只看计划不写库:不持锁,任何时候都能跑
1095
+ if (!opts.apply) return run();
1096
+
1097
+ const outcome = await underReconcileLocks(run);
1098
+ if (!outcome.ran) {
1099
+ const holder = inspectLock(outcome.busy);
1100
+ const stale = holder.exists && holder.pidAlive && (holder.ageMs ?? 0) >= LOCK_STALE_ALERT_MS;
1101
+ const skippedReason = `${busyDescription(outcome.busy)}正在运行(锁 ${outcome.busy} 被进程 ${holder.pid ?? "?"} 占着${holder.ageMs !== null ? `、${Math.round(holder.ageMs / 60_000)} 分钟没更新` : ""}),本轮跳过`;
1102
+ const guard: ReconcileOutcome["guard"] = stale
1103
+ ? { reason: "lock-stale", detail: `超过 ${Math.round(LOCK_STALE_ALERT_MS / 3_600_000)} 小时没更新,对账会一直跳过,看一下那个进程` }
1104
+ : null;
1105
+ return {
1106
+ ran: false,
1107
+ mode: "apply",
1108
+ skippedReason,
1109
+ runId,
1110
+ plan: null,
1111
+ maxRetire: 0,
1112
+ guard,
1113
+ applied: null,
1114
+ journalPath: null,
1115
+ alert: stale ? decideGuardAlert(opts.dataDir, guard, now()) : false,
1116
+ };
1117
+ }
1118
+ return outcome.result;
1119
+ }
1120
+
1121
+ export function formatReconcileSummary(outcome: ReconcileOutcome): string {
1122
+ if (!outcome.ran || !outcome.plan) {
1123
+ let line = `对账跳过:${outcome.skippedReason ?? "未知原因"}`;
1124
+ if (outcome.guard) line += `。⚠️ 护栏(${outcome.guard.reason}):${outcome.guard.detail}${outcome.alert ? "" : "(同一原因 24 小时内已报过警,这次只记日志)"}`;
1125
+ return line;
1126
+ }
1127
+ const p = outcome.plan;
1128
+ const a = outcome.applied;
1129
+ const reactA = p.reactivate.filter((r) => r.kind === "reconcile-retired").length;
1130
+ const reactB = p.reactivate.filter((r) => r.kind === "broken-consolidation").length;
1131
+ const parts = [
1132
+ `现行 ${p.currentChunks} 段 / ${p.currentFiles} 个文件`,
1133
+ `保留 ${p.keep}`,
1134
+ a ? `插入 ${a.inserted}` : `计划插入 ${p.insert.length}`,
1135
+ ...(p.insertAltId ? [`其中 ${p.insertAltId} 段的原 id 被另一段正文占着、用备用 id`] : []),
1136
+ a ? `恢复 ${a.reactivatedReconcileRetired}+${a.reactivatedBrokenConsolidation}` : `计划恢复 ${reactA}+${reactB}`,
1137
+ a ? `下架 ${a.retired}` : `计划下架 ${p.retire.length}`,
1138
+ `合并链完好 ${p.representedByConsolidation}`,
1139
+ `例外 forget ${p.exceptionForgotten} / 其他下架 ${p.exceptionOtherInactive}`,
1140
+ ];
1141
+ if (a) {
1142
+ const extra: string[] = [];
1143
+ if (a.insertSkippedExisting) extra.push(`插入时已存在同文 ${a.insertSkippedExisting}`);
1144
+ if (a.insertSkippedForgotten) extra.push(`插入前刚被 forget ${a.insertSkippedForgotten}`);
1145
+ if (a.reactivateSkippedForgotten) extra.push(`恢复前刚被 forget ${a.reactivateSkippedForgotten}`);
1146
+ if (a.retireDeferredPendingInsert) extra.push(`待插入文件的下架延后 ${a.retireDeferredPendingInsert}`);
1147
+ if (a.retireDeferredDependency) extra.push(`合并链成员没恢复成的下架延后 ${a.retireDeferredDependency}`);
1148
+ const changed = a.retireSkippedStateChanged + a.reactivateSkippedStateChanged;
1149
+ if (changed) extra.push(`提交时状态已变跳过 ${changed}`);
1150
+ if (extra.length) parts.push(extra.join(","));
1151
+ }
1152
+ let line = parts.join(",");
1153
+ if (outcome.guard) {
1154
+ line += `。⚠️ 护栏(${outcome.guard.reason}):${outcome.guard.detail}`;
1155
+ if (outcome.mode === "apply" && !outcome.alert) line += "(同一原因 24 小时内已报过警,这次只记日志)";
1156
+ }
1157
+ return line;
1158
+ }
1159
+
1160
+ // ---------------------------------------------------------------------------
1161
+ // 撤销
1162
+ // ---------------------------------------------------------------------------
1163
+
1164
+ export interface UndoResult {
1165
+ restored: number;
1166
+ insertsRetired: number;
1167
+ skippedConflict: string[];
1168
+ /**
1169
+ * 撤了会造出断链而没撤的:合并链成员的恢复(它原来指向的那一行现在不活跃、或本次撤销会把它撤成不活跃),
1170
+ * 以及此刻有合并成员指向的插入行 / 恢复行
1171
+ */
1172
+ skippedDependency: string[];
1173
+ /** 撤回去会让一段已被 forget 的文字重新活跃:保留非活跃 */
1174
+ skippedForgotten: string[];
1175
+ /** 日志里解析不了的行(比如写到一半崩溃留下的截断尾行) */
1176
+ unparsedLines: number;
1177
+ undoJournalPath: string;
1178
+ }
1179
+
1180
+ interface UndoRecord {
1181
+ run: string;
1182
+ action: "insert" | "retire" | "reactivate";
1183
+ id: string;
1184
+ /** 结果行的实际前值(只有意图行时不用它,改用行上 reconcile.prev 存的提交时前值) */
1185
+ before?: JournalFields;
1186
+ /** 结果行的实际后值;插入的意图行也带(插入前就知道) */
1187
+ after?: JournalFields;
1188
+ textFp?: string;
1189
+ /** 只有意图、没有结果(崩溃在写库与写结果之间) */
1190
+ intentOnly: boolean;
1191
+ }
1192
+
1193
+ const UNDO_ACTIONS: ReadonlySet<string> = new Set(["insert", "retire", "reactivate"]);
1194
+
1195
+ function asJournalFields(value: unknown): JournalFields | undefined {
1196
+ const r = asRecord(value);
1197
+ return r ? (r as unknown as JournalFields) : undefined;
1198
+ }
1199
+
1200
+ function readUndoRecords(journalPath: string): { records: UndoRecord[]; unparsed: number } {
1201
+ // 按「动作 + id」分开记:同一个 id 可能既是没插成的插入目标、又是本轮的下架对象(备用 id 被一行别的正文占着、
1202
+ // 那一行又因文件已删而下架),混在一起会让插入的「没写」吞掉下架的意图(第三次代码单审 F6)
1203
+ const key = (action: string, id: string) => `${action}\u0000${id}`;
1204
+ const results = new Map<string, UndoRecord>();
1205
+ const intents = new Map<string, UndoRecord>();
1206
+ /** 插入时撞 id 被跳过的:这个 id 上的行不是本轮插入的 */
1207
+ const insertNotWritten = new Set<string>();
1208
+ let unparsed = 0;
1209
+ for (const line of readFileSync(journalPath, "utf-8").split("\n")) {
1210
+ if (!line.trim()) continue;
1211
+ let rec: Record<string, unknown> | null;
1212
+ try {
1213
+ rec = asRecord(JSON.parse(line));
1214
+ } catch {
1215
+ unparsed++;
1216
+ continue;
1217
+ }
1218
+ if (!rec || typeof rec.run !== "string" || typeof rec.action !== "string" || !UNDO_ACTIONS.has(rec.action)) continue;
1219
+ const run = rec.run;
1220
+ const action = rec.action as UndoRecord["action"];
1221
+ if (rec.phase === "intent") {
1222
+ if (Array.isArray(rec.items)) {
1223
+ for (const it of rec.items) {
1224
+ const item = asRecord(it);
1225
+ if (!item || typeof item.id !== "string") continue;
1226
+ intents.set(key(action, item.id), {
1227
+ run,
1228
+ action,
1229
+ id: item.id,
1230
+ after: action === "insert" ? asJournalFields(item.after) : undefined,
1231
+ textFp: typeof item.textFp === "string" ? item.textFp : undefined,
1232
+ intentOnly: true,
1233
+ });
1234
+ }
1235
+ }
1236
+ continue;
1237
+ }
1238
+ if (typeof rec.id !== "string") continue;
1239
+ if (action === "insert" && rec.inserted === false) {
1240
+ insertNotWritten.add(rec.id);
1241
+ continue;
1242
+ }
1243
+ results.set(key(action, rec.id), {
1244
+ run,
1245
+ action,
1246
+ id: rec.id,
1247
+ before: asJournalFields(rec.before),
1248
+ after: asJournalFields(rec.after),
1249
+ textFp: typeof rec.textFp === "string" ? rec.textFp : undefined,
1250
+ intentOnly: false,
1251
+ });
1252
+ }
1253
+ for (const [k, rec] of intents) {
1254
+ if (results.has(k) || (rec.action === "insert" && insertNotWritten.has(rec.id))) continue;
1255
+ results.set(k, rec);
1256
+ }
1257
+ return { records: [...results.values()], unparsed };
1258
+ }
1259
+
1260
+ /**
1261
+ * 只有意图行时,从行上的本轮标记推出「当时写成了什么样」与「写之前是什么样」(reconcile.prev 是提交时在写锁内记下的真实前值)。
1262
+ * 推不出来(标记不是本轮这个动作、没有前值)就返回 null,调用方按冲突跳过。
1263
+ */
1264
+ function expectedFromMarker(meta: Record<string, unknown>, rec: UndoRecord): { before: JournalFields; after: JournalFields } | null {
1265
+ const marker = asRecord(meta.reconcile);
1266
+ const expectedAction = rec.action === "retire" ? "retired" : "reactivated";
1267
+ if (!marker || marker.run !== rec.run || marker.action !== expectedAction) return null;
1268
+ const prev = asJournalFields(marker.prev);
1269
+ if (!prev) return null;
1270
+ if (rec.action === "retire") {
1271
+ const at = typeof marker.at === "string" ? Date.parse(marker.at) : NaN;
1272
+ if (!Number.isFinite(at) || typeof marker.reason !== "string") return null;
1273
+ return {
1274
+ before: prev,
1275
+ after: { status: "archived", validUntil: at, evolutionNote: `${RECONCILE_NOTE_PREFIX} ${marker.reason}`, consolidatedInto: prev.consolidatedInto, reconcile: marker },
1276
+ };
1277
+ }
1278
+ return {
1279
+ before: prev,
1280
+ after: {
1281
+ status: "active",
1282
+ validUntil: null,
1283
+ evolutionNote: null,
1284
+ consolidatedInto: marker.kind === "broken-consolidation" ? null : prev.consolidatedInto,
1285
+ reconcile: marker,
1286
+ },
1287
+ };
1288
+ }
1289
+
1290
+ /**
1291
+ * 按对账日志逐条撤销,整轮持有与对账相同的三把锁。
1292
+ * - 下架、恢复:只在「当前值 == 当时写成的样子」时写回写之前的值。有结果行用结果行;只有意图行的,用行上本轮标记里
1293
+ * 存的提交时前值,并按标记推出当时写成的样子来核对——之后被改过就跳过,绝不拿规划时的旧值覆盖。
1294
+ * - 合并链成员的恢复:它原来指向的那一行必须此刻活跃、而且不在本次撤销要撤成不活跃的名单里,否则保留成员并列出。
1295
+ * - 插入与恢复的撤销会让行失活:此刻已有合并成员指向它的(对账之后 dream 合并进来的),保留并列出。
1296
+ * - 撤回去会让一行重新活跃的(撤下架),先查审计里的 forget:这段文字已被 forget 就保留非活跃并列出。
1297
+ * - 插入:当前字段与正文指纹都等于插入时的样子才改成 archived(undo-insert),不删行;之后被改过的保留。
1298
+ * 后做的先撤:下架 → 恢复 → 插入。撤销之后要么把 sources.memory.path 改回 auto,要么暂停导入——
1299
+ * 否则下一轮对账会按现行文件重新对一遍。
1300
+ */
1301
+ export async function undoMemoryReconcile(
1302
+ deps: { store: MemoryStore; auditLogger?: Pick<AuditLogger, "log"> | null },
1303
+ journalPath: string,
1304
+ opts: { now?: () => number; auditPath?: string } = {},
1305
+ ): Promise<UndoResult> {
1306
+ const outcome = await underReconcileLocks(() => undoUnlocked(deps, journalPath, opts));
1307
+ if (!outcome.ran) {
1308
+ throw new Error(`撤销没执行:${busyDescription(outcome.busy)}正在运行(锁 ${outcome.busy} 被占用),稍后再试`);
1309
+ }
1310
+ return outcome.result;
1311
+ }
1312
+
1313
+ async function undoUnlocked(
1314
+ deps: { store: MemoryStore; auditLogger?: Pick<AuditLogger, "log"> | null },
1315
+ journalPath: string,
1316
+ opts: { now?: () => number; auditPath?: string },
1317
+ ): Promise<UndoResult> {
1318
+ const { records, unparsed } = readUndoRecords(journalPath);
1319
+ const forgets = opts.auditPath ? loadForgetSet(opts.auditPath) : null;
1320
+ const nowMs = (opts.now ?? Date.now)();
1321
+ const isoNow = new Date(nowMs).toISOString();
1322
+ const undoRun = `undo-${isoNow.replace(/[:.]/g, "-")}`;
1323
+ const undoJournalPath = `${journalPath.replace(/\.jsonl$/, "")}.${undoRun}.jsonl`;
1324
+ const journal = new Journal(undoJournalPath);
1325
+ const result: UndoResult = {
1326
+ restored: 0,
1327
+ insertsRetired: 0,
1328
+ skippedConflict: [],
1329
+ skippedDependency: [],
1330
+ skippedForgotten: [],
1331
+ unparsedLines: unparsed,
1332
+ undoJournalPath,
1333
+ };
1334
+
1335
+ // 本次撤销会撤成不活跃的行:插入的行(改成 archived)与恢复过的行(撤回原来的非活跃状态)。
1336
+ // 合并链成员原来指向的若在其中,撤成员就会和它一起失活——按撤销完成后的样子判依赖(第二次代码单审 F3)。
1337
+ const willDeactivate = new Set(records.filter((r) => r.action === "insert" || r.action === "reactivate").map((r) => r.id));
1338
+ // 反过来:此刻已有合并成员指向的行(比如对账之后 dream 把别的行合并进了本轮插入或恢复的行),撤成不活跃也会造出断链
1339
+ const targetsInUse = new Set<string>();
1340
+ if (willDeactivate.size > 0) {
1341
+ for (const row of await loadMemoryScopeRows(deps.store)) {
1342
+ const st = rowState(row.meta);
1343
+ if (st.status === "consolidated" && st.consolidatedInto) targetsInUse.add(st.consolidatedInto);
1344
+ }
1345
+ }
1346
+
1347
+ const restore = (meta: Record<string, unknown>, before: JournalFields) => {
1348
+ patchEvolutionOnMeta(meta, {
1349
+ status: before.status as EvolutionMetadata["status"],
1350
+ validUntil: typeof before.validUntil === "number" ? before.validUntil : null,
1351
+ evolutionNote: before.evolutionNote,
1352
+ consolidatedInto: before.consolidatedInto,
1353
+ });
1354
+ if (before.reconcile === null || before.reconcile === undefined) delete meta.reconcile;
1355
+ else meta.reconcile = before.reconcile;
1356
+ };
1357
+
1358
+ const phases: Array<UndoRecord["action"]> = ["retire", "reactivate", "insert"];
1359
+ for (const phase of phases) {
1360
+ const list = records.filter((r) => r.action === phase);
1361
+ for (let i = 0; i < list.length; i += PATCH_BATCH) {
1362
+ const batch = list.slice(i, i + PATCH_BATCH);
1363
+ const dependencyOk = new Set<string>();
1364
+ if (phase === "reactivate") {
1365
+ for (const rec of batch) {
1366
+ const current = await deps.store.getById(rec.id);
1367
+ const before = rec.intentOnly
1368
+ ? (current ? expectedFromMarker(parseMeta(current.metadata), rec)?.before : undefined)
1369
+ : rec.before;
1370
+ const target = before?.status === "consolidated" ? before.consolidatedInto : null;
1371
+ if (!target) {
1372
+ dependencyOk.add(rec.id);
1373
+ continue;
1374
+ }
1375
+ if (willDeactivate.has(target)) continue;
1376
+ const holder = await deps.store.getById(target);
1377
+ if (holder && isActiveStatus(rowState(parseMeta(holder.metadata)).status)) dependencyOk.add(rec.id);
1378
+ }
1379
+ }
1380
+ const byId = new Map(batch.map((r) => [r.id, r]));
1381
+ const pending: object[] = [];
1382
+ const seen = new Set<string>();
1383
+ await deps.store.patchMetadataBatch(batch.map((rec) => ({
1384
+ id: rec.id,
1385
+ patchFn: (meta, entry) => {
1386
+ const r = byId.get(entry.id)!;
1387
+ seen.add(entry.id);
1388
+ const current = journalFields(meta);
1389
+ if (r.action !== "retire" && targetsInUse.has(entry.id)) {
1390
+ result.skippedDependency.push(r.id);
1391
+ return meta;
1392
+ }
1393
+ if (r.action === "insert") {
1394
+ const untouched = !!r.after
1395
+ && stableStringify(current) === stableStringify(r.after)
1396
+ && !!r.textFp && r.textFp === textFingerprint(entry.text);
1397
+ if (!untouched) {
1398
+ result.skippedConflict.push(r.id);
1399
+ return meta;
1400
+ }
1401
+ patchEvolutionOnMeta(meta, { status: "archived", validUntil: nowMs, evolutionNote: `${RECONCILE_NOTE_PREFIX} undo-insert` });
1402
+ meta.reconcile = { v: 1, action: "retired", reason: "undo-insert", at: isoNow, run: undoRun };
1403
+ pending.push({ run: undoRun, action: "undo-insert", id: r.id, before: current, after: journalFields(meta) });
1404
+ result.insertsRetired++;
1405
+ return meta;
1406
+ }
1407
+ if (r.action === "reactivate" && !dependencyOk.has(r.id)) {
1408
+ result.skippedDependency.push(r.id);
1409
+ return meta;
1410
+ }
1411
+ const expected = r.intentOnly
1412
+ ? expectedFromMarker(meta, r)
1413
+ : (r.before && r.after ? { before: r.before, after: r.after } : null);
1414
+ if (!expected || stableStringify(current) !== stableStringify(expected.after)) {
1415
+ result.skippedConflict.push(r.id);
1416
+ return meta;
1417
+ }
1418
+ // 撤回去会重新活跃(撤下架):这段文字已被 forget 就不撤
1419
+ if (isActiveStatus(expected.before.status) && forgets && isTextForgotten(forgets, entry.text, entry.id)) {
1420
+ result.skippedForgotten.push(r.id);
1421
+ return meta;
1422
+ }
1423
+ restore(meta, expected.before);
1424
+ pending.push({ run: undoRun, action: `undo-${r.action}`, id: r.id, before: current, after: journalFields(meta) });
1425
+ result.restored++;
1426
+ return meta;
1427
+ },
1428
+ })));
1429
+ for (const rec of batch) if (!seen.has(rec.id)) result.skippedConflict.push(rec.id);
1430
+ journal.write(pending);
1431
+ }
1432
+ }
1433
+ try {
1434
+ deps.auditLogger?.log({ operation: "update", scope: MEMORY_DOC_SCOPE, actor: "system", details: `${RECONCILE_NOTE_PREFIX} undo restored=${result.restored} insertsRetired=${result.insertsRetired} conflicts=${result.skippedConflict.length} dependency=${result.skippedDependency.length} forgotten=${result.skippedForgotten.length}` });
1435
+ } catch { /* 审计失败不影响撤销 */ }
1436
+ return result;
1437
+ }