recallnest 2.2.1 → 2.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/CHANGELOG.md +150 -0
  2. package/README.md +46 -35
  3. package/README_CN.md +41 -26
  4. package/assets/ui/app.js +1031 -0
  5. package/assets/ui/index.html +224 -0
  6. package/assets/ui/styles.css +781 -0
  7. package/config.json.example +4 -1
  8. package/connectors/examples/email-adapter.ts +73 -0
  9. package/connectors/examples/logs-adapter.ts +103 -0
  10. package/connectors/examples/rss-adapter.ts +95 -0
  11. package/docs/api-reference.md +1291 -0
  12. package/docs/architecture.md +168 -0
  13. package/docs/connector-spec.md +126 -0
  14. package/eval/cases-canary.json +58 -0
  15. package/eval/cases-scope-robustness.json +278 -0
  16. package/eval/cases.json +117 -20
  17. package/package.json +22 -1
  18. package/src/__tests__/access-tracker-novelty.test.ts +238 -0
  19. package/src/__tests__/activity-counter.test.ts +125 -0
  20. package/src/__tests__/adaptive-fidelity.test.ts +102 -0
  21. package/src/__tests__/admission-control.test.ts +206 -0
  22. package/src/__tests__/anchor-generator.test.ts +67 -0
  23. package/src/__tests__/auto-consolidation.test.ts +168 -0
  24. package/src/__tests__/auto-recall.test.ts +131 -0
  25. package/src/__tests__/babel-integration.test.ts +144 -0
  26. package/src/__tests__/batch-dedup.test.ts +65 -0
  27. package/src/__tests__/canary-eval.test.ts +114 -0
  28. package/src/__tests__/capture-engine.test.ts +1238 -0
  29. package/src/__tests__/capture-heuristic.test.ts +171 -0
  30. package/src/__tests__/cascade-forget.test.ts +210 -0
  31. package/src/__tests__/category-min-scores.test.ts +33 -0
  32. package/src/__tests__/cc8-cc1-essential-context.test.ts +298 -0
  33. package/src/__tests__/circuit-breaker.test.ts +99 -0
  34. package/src/__tests__/cluster-consolidation.test.ts +782 -0
  35. package/src/__tests__/confidence-metadata.test.ts +125 -0
  36. package/src/__tests__/confidence-tracker.test.ts +261 -0
  37. package/src/__tests__/conflict-advisor.test.ts +206 -0
  38. package/src/__tests__/conflict-escalation.test.ts +149 -0
  39. package/src/__tests__/conflict-lifecycle.test.ts +83 -0
  40. package/src/__tests__/conflict-output.test.ts +85 -0
  41. package/src/__tests__/conflict-store.test.ts +374 -0
  42. package/src/__tests__/connector-v1.test.ts +181 -0
  43. package/src/__tests__/consolidation-engine.test.ts +333 -0
  44. package/src/__tests__/constructive-retrieval.test.ts +509 -0
  45. package/src/__tests__/context-collapse-renderer.test.ts +214 -0
  46. package/src/__tests__/context-composer-pins.test.ts +64 -0
  47. package/src/__tests__/context-composer-task-selection.test.ts +24 -0
  48. package/src/__tests__/context-reconstructor.test.ts +65 -0
  49. package/src/__tests__/context-renderer.test.ts +91 -0
  50. package/src/__tests__/continuity-eval.test.ts +275 -0
  51. package/src/__tests__/continuity-schema.test.ts +97 -0
  52. package/src/__tests__/contracts.test.ts +216 -0
  53. package/src/__tests__/conversation-importer.test.ts +692 -0
  54. package/src/__tests__/core-recall-eval.test.ts +235 -0
  55. package/src/__tests__/core-summary.test.ts +109 -0
  56. package/src/__tests__/data-checkup.test.ts +276 -0
  57. package/src/__tests__/decay-exemption.test.ts +84 -0
  58. package/src/__tests__/dedup-actions.test.ts +58 -0
  59. package/src/__tests__/dedup-guard.test.ts +39 -0
  60. package/src/__tests__/deterministic-id.test.ts +39 -0
  61. package/src/__tests__/distill-lock.test.ts +305 -0
  62. package/src/__tests__/distill-session-argshape.test.ts +79 -0
  63. package/src/__tests__/doctor.test.ts +319 -0
  64. package/src/__tests__/dream-pipeline.test.ts +321 -0
  65. package/src/__tests__/drill-down.test.ts +108 -0
  66. package/src/__tests__/embedder.test.ts +122 -0
  67. package/src/__tests__/emotion-decay.test.ts +46 -0
  68. package/src/__tests__/emotion-detector.test.ts +56 -0
  69. package/src/__tests__/emotion-phase1-integration.test.ts +153 -0
  70. package/src/__tests__/emotion-retrieval.test.ts +56 -0
  71. package/src/__tests__/entity-resolver.test.ts +58 -0
  72. package/src/__tests__/env-config.test.ts +151 -0
  73. package/src/__tests__/error-signature-boost.test.ts +87 -0
  74. package/src/__tests__/error-signature.test.ts +95 -0
  75. package/src/__tests__/error-taxonomy.test.ts +129 -0
  76. package/src/__tests__/ethics-layer.test.ts +476 -0
  77. package/src/__tests__/event-segmenter.test.ts +153 -0
  78. package/src/__tests__/feature-flags-integration.test.ts +54 -0
  79. package/src/__tests__/frequency-tracker.test.ts +113 -0
  80. package/src/__tests__/graph-export.test.ts +489 -0
  81. package/src/__tests__/hotness-blend.test.ts +76 -0
  82. package/src/__tests__/ingest-chunking.test.ts +44 -0
  83. package/src/__tests__/ingest.test.ts +360 -0
  84. package/src/__tests__/interference-detector.test.ts +203 -0
  85. package/src/__tests__/kg-extractor.test.ts +192 -0
  86. package/src/__tests__/kg-store.test.ts +300 -0
  87. package/src/__tests__/llm-consolidation.test.ts +226 -0
  88. package/src/__tests__/maintenance-locks.test.ts +174 -0
  89. package/src/__tests__/mcp-efficiency.test.ts +288 -0
  90. package/src/__tests__/mcp-related-scopes.test.ts +151 -0
  91. package/src/__tests__/memory-boundaries.test.ts +250 -0
  92. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +345 -0
  93. package/src/__tests__/memory-evolution.test.ts +271 -0
  94. package/src/__tests__/memory-health-rebalance.test.ts +144 -0
  95. package/src/__tests__/memory-lint.test.ts +421 -0
  96. package/src/__tests__/memory-output.test.ts +215 -0
  97. package/src/__tests__/memory-promotion.test.ts +315 -0
  98. package/src/__tests__/migrate-dedup-ids.test.ts +65 -0
  99. package/src/__tests__/multi-vector.test.ts +242 -0
  100. package/src/__tests__/narrative-architecture.test.ts +456 -0
  101. package/src/__tests__/noise-filter.test.ts +205 -0
  102. package/src/__tests__/obsidian-connector.test.ts +363 -0
  103. package/src/__tests__/omitted-reporting.test.ts +98 -0
  104. package/src/__tests__/pending-review.test.ts +77 -0
  105. package/src/__tests__/pii-redact.test.ts +314 -0
  106. package/src/__tests__/post-retrieval-filter.test.ts +178 -0
  107. package/src/__tests__/ppr-traversal.test.ts +256 -0
  108. package/src/__tests__/prediction-engine.test.ts +349 -0
  109. package/src/__tests__/predictive-prospective-memory.test.ts +416 -0
  110. package/src/__tests__/preference-guard-smoke.test.ts +211 -0
  111. package/src/__tests__/preference-matcher.test.ts +230 -0
  112. package/src/__tests__/preference-slots.test.ts +205 -0
  113. package/src/__tests__/prospective-memory.test.ts +283 -0
  114. package/src/__tests__/query-entity-detector.test.ts +171 -0
  115. package/src/__tests__/query-expander-alias.test.ts +146 -0
  116. package/src/__tests__/query-expander.test.ts +34 -0
  117. package/src/__tests__/recall-governor.test.ts +227 -0
  118. package/src/__tests__/recall-mode.test.ts +64 -0
  119. package/src/__tests__/result-synthesizer.test.ts +198 -0
  120. package/src/__tests__/resume-context.test.ts +4898 -0
  121. package/src/__tests__/retrieval-trace.test.ts +213 -0
  122. package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
  123. package/src/__tests__/retroactive-boost.test.ts +223 -0
  124. package/src/__tests__/salience-scoring.test.ts +106 -0
  125. package/src/__tests__/scope-inventory.test.ts +200 -0
  126. package/src/__tests__/scope-policy.test.ts +69 -0
  127. package/src/__tests__/scope-suggester.test.ts +80 -0
  128. package/src/__tests__/session-checkpoint.test.ts +291 -0
  129. package/src/__tests__/session-distiller.test.ts +640 -0
  130. package/src/__tests__/session-gc.test.ts +215 -0
  131. package/src/__tests__/skill-engine.test.ts +552 -0
  132. package/src/__tests__/skill-promotion.test.ts +601 -0
  133. package/src/__tests__/skill-verifier.test.ts +122 -0
  134. package/src/__tests__/source-heartbeat.test.ts +250 -0
  135. package/src/__tests__/store-patch-metadata.test.ts +205 -0
  136. package/src/__tests__/store-scope-audit.test.ts +81 -0
  137. package/src/__tests__/store-upsert.test.ts +184 -0
  138. package/src/__tests__/supersede-chain.test.ts +154 -0
  139. package/src/__tests__/temporal-parser.test.ts +241 -0
  140. package/src/__tests__/temporal-validity.test.ts +160 -0
  141. package/src/__tests__/term-registry.test.ts +121 -0
  142. package/src/__tests__/tool-error-surfacing.test.ts +98 -0
  143. package/src/__tests__/tool-output-compressor.test.ts +160 -0
  144. package/src/__tests__/topic-tag.test.ts +113 -0
  145. package/src/__tests__/trusted-memory.test.ts +839 -0
  146. package/src/__tests__/usage-tracker.test.ts +287 -0
  147. package/src/__tests__/version-manager.test.ts +213 -0
  148. package/src/__tests__/workflow-observation-skill-outcome.test.ts +352 -0
  149. package/src/__tests__/workflow-observation.test.ts +433 -0
  150. package/src/__tests__/write-verifier.test.ts +122 -0
  151. package/src/access-tracker.ts +38 -32
  152. package/src/activity-counter.ts +46 -18
  153. package/src/aliases.ts +133 -0
  154. package/src/api-server.ts +27 -7
  155. package/src/audit-log.ts +4 -2
  156. package/src/auto-consolidation.ts +8 -3
  157. package/src/auto-gc.ts +85 -21
  158. package/src/capture-engine.ts +54 -21
  159. package/src/cli.ts +199 -37
  160. package/src/confidence-tracker.ts +36 -18
  161. package/src/connector-types.ts +57 -0
  162. package/src/consolidation-engine.ts +109 -11
  163. package/src/context-composer.ts +8 -1
  164. package/src/conversation-importer.ts +25 -1
  165. package/src/data-checkup.ts +126 -7
  166. package/src/distill-lock.ts +194 -15
  167. package/src/doctor.ts +38 -2
  168. package/src/dream-pipeline.ts +118 -34
  169. package/src/env-config.ts +77 -0
  170. package/src/error-signature.ts +153 -0
  171. package/src/error-taxonomy.ts +115 -0
  172. package/src/eval.ts +251 -6
  173. package/src/idempotency.ts +8 -0
  174. package/src/ingest.ts +234 -10
  175. package/src/kg-extractor.ts +6 -2
  176. package/src/kg-store.ts +236 -44
  177. package/src/llm-consolidation.ts +2 -1
  178. package/src/mcp-server.ts +85 -1799
  179. package/src/mcp-tool-deps.ts +48 -0
  180. package/src/mcp-tools-advanced.ts +941 -0
  181. package/src/mcp-tools-core.ts +633 -0
  182. package/src/mcp-tools-governance.ts +475 -0
  183. package/src/memory-boundaries.ts +3 -1
  184. package/src/memory-evolution.ts +73 -1
  185. package/src/memory-lint.ts +183 -10
  186. package/src/memory-output.ts +114 -3
  187. package/src/memory-promotion.ts +246 -0
  188. package/src/memory-schema.ts +25 -2
  189. package/src/memory-store-port.ts +43 -0
  190. package/src/multi-vector.ts +2 -1
  191. package/src/narrative-schema.ts +2 -1
  192. package/src/noise-filter.ts +57 -6
  193. package/src/obsidian-connector.ts +282 -0
  194. package/src/pii-detector.ts +157 -5
  195. package/src/query-expander.ts +90 -9
  196. package/src/result-synthesizer.ts +2 -1
  197. package/src/retention-policy.ts +2 -1
  198. package/src/retriever.ts +259 -139
  199. package/src/runtime-config.ts +8 -1
  200. package/src/scope-suggester.ts +113 -0
  201. package/src/session-distiller.ts +23 -2
  202. package/src/session-engine.ts +2 -1
  203. package/src/session-schema.ts +3 -2
  204. package/src/session-store.ts +5 -0
  205. package/src/skill-engine.ts +78 -0
  206. package/src/skill-promotion.ts +247 -47
  207. package/src/skill-schema.ts +16 -3
  208. package/src/skill-verifier.ts +152 -0
  209. package/src/source-heartbeat.ts +108 -0
  210. package/src/store.ts +267 -38
  211. package/src/tool-output-compressor.ts +3 -1
  212. package/src/ui-server.ts +2 -1
  213. package/src/usage-tracker.ts +242 -0
  214. package/src/version-manager.ts +3 -2
  215. package/src/workflow-observation-engine.ts +2 -1
  216. package/src/workflow-observation-schema.ts +2 -1
  217. package/src/workflow-observation-store.ts +6 -2
  218. package/.env +0 -4
  219. package/CLAUDE.md +0 -46
  220. package/config.json +0 -55
  221. package/daily-recall.sh +0 -110
  222. package/logs/daily-recall.error.log +0 -0
  223. package/logs/daily-recall.log +0 -30
  224. package/logs/ingest-2026-04-11.log +0 -110
  225. package/logs/ingest-2026-04-12.log +0 -90
  226. package/logs/launchagent-stderr.log +0 -0
  227. package/logs/launchagent-stdout.log +0 -0
@@ -18,7 +18,9 @@ import {
18
18
  utimesSync,
19
19
  existsSync,
20
20
  } from "node:fs";
21
+ import { createHash, randomUUID } from "node:crypto";
21
22
  import { join, dirname } from "node:path";
23
+ import * as envConfig from "./env-config.js";
22
24
 
23
25
  // ---------------------------------------------------------------------------
24
26
  // Config
@@ -34,7 +36,7 @@ export interface DistillLockConfig {
34
36
  }
35
37
 
36
38
  export const DEFAULT_DISTILL_LOCK_CONFIG: DistillLockConfig = {
37
- lockPath: join(process.env.RECALLNEST_DATA_DIR || "data", "distill.lock"),
39
+ lockPath: join(envConfig.dataDir(), "distill.lock"),
38
40
  expireMs: 3_600_000,
39
41
  minCheckpoints: 3,
40
42
  };
@@ -56,6 +58,17 @@ function isPidAlive(pid: number): boolean {
56
58
  }
57
59
  }
58
60
 
61
+ // P2: in-process record of the ownership token we wrote into each lock file we hold.
62
+ // releaseLock only removes a lock whose on-disk token still matches — so if our lock
63
+ // expired mid-run and another process legitimately reclaimed it, our finally-release
64
+ // won't delete the new owner's lock (which would break mutual exclusion).
65
+ const heldTokens = new Map<string, string>();
66
+
67
+ /** Parse the PID prefix from a lock file body (`<pid>` legacy or `<pid>:<token>`). */
68
+ function parsePid(content: string): number {
69
+ return parseInt((content.split(":")[0] ?? "").trim(), 10);
70
+ }
71
+
59
72
  // ---------------------------------------------------------------------------
60
73
  // Public API
61
74
  // ---------------------------------------------------------------------------
@@ -66,29 +79,75 @@ function isPidAlive(pid: number): boolean {
66
79
  */
67
80
  export function acquireLock(cfg?: Partial<DistillLockConfig>): boolean {
68
81
  const { lockPath, expireMs } = resolveConfig(cfg);
82
+ mkdirSync(dirname(lockPath), { recursive: true });
69
83
 
70
- if (existsSync(lockPath)) {
71
- const existingPid = parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
72
- const mtime = statSync(lockPath).mtimeMs;
73
- const expired = Date.now() - mtime > expireMs;
84
+ // Fast path: atomic exclusive create. On POSIX the O_EXCL ("wx") open wins the
85
+ // cross-process race — only one of N processes creating the same lock succeeds.
86
+ // This closes the check-then-write TOCTOU the previous existsSync+writeFileSync
87
+ // form had (two processes could both see "no lock" then both write); the P0-1
88
+ // store-write lock relies on acquisition being truly exclusive.
89
+ if (tryExclusiveCreate(lockPath)) return true;
74
90
 
75
- if (!isNaN(existingPid) && isPidAlive(existingPid) && !expired) {
76
- return false;
77
- }
78
- // PID dead or lock expired — overwrite
91
+ // Lock file exists — decide whether it is reclaimable (dead holder or expired).
92
+ let existingPid = NaN;
93
+ let mtime = 0;
94
+ try {
95
+ existingPid = parsePid(readFileSync(lockPath, "utf-8"));
96
+ mtime = statSync(lockPath).mtimeMs;
97
+ } catch {
98
+ // Holder released between our failed create and this read — retry the create.
99
+ return tryExclusiveCreate(lockPath);
100
+ }
101
+ const expired = Date.now() - mtime > expireMs;
102
+ if (!isNaN(existingPid) && isPidAlive(existingPid) && !expired) {
103
+ return false; // held by a live process, not expired
79
104
  }
80
105
 
81
- mkdirSync(dirname(lockPath), { recursive: true });
82
- writeFileSync(lockPath, String(process.pid), "utf-8");
83
- return true;
106
+ // Reclaim a stale lock: remove it, then re-create atomically. If a competing
107
+ // process reclaims first, our O_EXCL create fails and we concede the race.
108
+ try {
109
+ unlinkSync(lockPath);
110
+ } catch {
111
+ /* already gone — fall through to the atomic create */
112
+ }
113
+ return tryExclusiveCreate(lockPath);
114
+ }
115
+
116
+ /**
117
+ * Atomic O_EXCL create of the lock file. Writes a unique ownership token (`<pid>:<uuid>`)
118
+ * and records it so releaseLock can verify we still own the lock. false on EEXIST.
119
+ */
120
+ function tryExclusiveCreate(lockPath: string): boolean {
121
+ const token = `${process.pid}:${randomUUID()}`;
122
+ try {
123
+ writeFileSync(lockPath, token, { encoding: "utf-8", flag: "wx" });
124
+ heldTokens.set(lockPath, token);
125
+ return true;
126
+ } catch (err) {
127
+ if ((err as { code?: unknown }).code === "EEXIST") return false;
128
+ throw err;
129
+ }
84
130
  }
85
131
 
86
- /** Release the distill lock by removing the lock file. */
132
+ /**
133
+ * Release the distill lock. If we hold a known ownership token for this path, only
134
+ * remove the file when its on-disk token still matches ours — otherwise our lock was
135
+ * expired and reclaimed by another process, and deleting would evict the new owner.
136
+ * When we hold no token (e.g. a direct caller/test-written lock) fall back to removing.
137
+ */
87
138
  export function releaseLock(cfg?: Partial<DistillLockConfig>): void {
88
139
  const { lockPath } = resolveConfig(cfg);
89
- if (existsSync(lockPath)) {
90
- unlinkSync(lockPath);
140
+ const ourToken = heldTokens.get(lockPath);
141
+ heldTokens.delete(lockPath);
142
+ if (!existsSync(lockPath)) return;
143
+ if (ourToken !== undefined) {
144
+ try {
145
+ if (readFileSync(lockPath, "utf-8").trim() !== ourToken) return; // reclaimed by another owner
146
+ } catch {
147
+ return;
148
+ }
91
149
  }
150
+ unlinkSync(lockPath);
92
151
  }
93
152
 
94
153
  /**
@@ -120,3 +179,123 @@ export function getLastDistillTime(cfg?: Partial<DistillLockConfig>): number {
120
179
  if (!existsSync(lockPath)) return 0;
121
180
  return statSync(lockPath).mtimeMs;
122
181
  }
182
+
183
+ // ---------------------------------------------------------------------------
184
+ // P0-1: Named cross-process locks (store-write + maintenance families)
185
+ //
186
+ // distill-lock was dead code (0 src imports). P0-1 wires it up: one global
187
+ // `store-write` lock serializes every `memories`-table write across the 11
188
+ // mcp-server processes, and scope-keyed maintenance locks (dream/consolidate/
189
+ // distill) + a global gc lock stop redundant concurrent maintenance runs.
190
+ // ---------------------------------------------------------------------------
191
+
192
+ /** Base directory for named locks. Defaults to <dataDir>/locks; overridable in tests. */
193
+ export function locksDir(lockDir?: string): string {
194
+ return lockDir ?? join(envConfig.dataDir(), "locks");
195
+ }
196
+
197
+ /**
198
+ * Resolve a lock key (e.g. "store-write", "dream-<scope>", "gc-global") to a safe
199
+ * lock-file path. The key is sanitized for the filesystem and suffixed with a short
200
+ * hash of the raw key so distinct keys never collide after sanitization/truncation.
201
+ */
202
+ export function lockPathForKey(key: string, lockDir?: string): string {
203
+ const safe = key.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
204
+ const hash = createHash("sha256").update(key).digest("hex").slice(0, 8);
205
+ return join(locksDir(lockDir), `${safe}-${hash}.lock`);
206
+ }
207
+
208
+ export interface WithLockOptions {
209
+ /** Stale-lock reclaim window in ms (default 600_000 = 10min). */
210
+ expireMs?: number;
211
+ /** Override the base lock directory (tests). */
212
+ lockDir?: string;
213
+ /** Behavior when another live process holds the lock (default "skip"). */
214
+ onBusy?: "skip" | "wait";
215
+ /** For onBusy="wait": total time budget before throwing (default 10_000). */
216
+ waitTimeoutMs?: number;
217
+ /** For onBusy="wait": poll interval in ms (default 50). */
218
+ pollMs?: number;
219
+ }
220
+
221
+ export type WithLockOutcome<T> =
222
+ | { ran: true; result: T }
223
+ | { ran: false; reason: "locked_by_another_process" };
224
+
225
+ const sleep = (ms: number): Promise<void> =>
226
+ new Promise((resolve) => setTimeout(resolve, ms));
227
+
228
+ /**
229
+ * Run `fn` while holding a named cross-process lock, releasing it afterwards
230
+ * (even on throw). Maintenance-type writes use onBusy:"skip" — for a personal
231
+ * store, skipping a redundant dream/consolidate/gc run beats queueing it.
232
+ */
233
+ export async function withLock<T>(
234
+ key: string,
235
+ fn: () => Promise<T>,
236
+ opts: WithLockOptions = {},
237
+ ): Promise<WithLockOutcome<T>> {
238
+ const lockPath = lockPathForKey(key, opts.lockDir);
239
+ const lockCfg = { lockPath, expireMs: opts.expireMs ?? 600_000 };
240
+
241
+ if (!acquireLock(lockCfg)) {
242
+ if ((opts.onBusy ?? "skip") === "skip") {
243
+ return { ran: false, reason: "locked_by_another_process" };
244
+ }
245
+ // wait: poll until acquired or the budget is exhausted.
246
+ const timeoutMs = opts.waitTimeoutMs ?? 10_000;
247
+ const deadline = Date.now() + timeoutMs;
248
+ const pollMs = opts.pollMs ?? 50;
249
+ let acquired = false;
250
+ while (Date.now() < deadline) {
251
+ await sleep(pollMs);
252
+ if (acquireLock(lockCfg)) {
253
+ acquired = true;
254
+ break;
255
+ }
256
+ }
257
+ if (!acquired) {
258
+ throw new Error(`lock '${key}' timed out after ${timeoutMs}ms (held by another process)`);
259
+ }
260
+ }
261
+
262
+ try {
263
+ return { ran: true, result: await fn() };
264
+ } finally {
265
+ releaseLock(lockCfg);
266
+ }
267
+ }
268
+
269
+ /**
270
+ * Run `fn` under a named write lock, waiting for the lock rather than skipping —
271
+ * losing a memory write would be data loss. Throws if the lock can't be acquired
272
+ * within the timeout. Returns fn's result directly. Used by store.upsert/storeBatch
273
+ * so all `memories`-table writes serialize across the 11 mcp-server processes.
274
+ */
275
+ export async function withWriteLock<T>(
276
+ key: string,
277
+ fn: () => Promise<T>,
278
+ opts: Omit<WithLockOptions, "onBusy"> = {},
279
+ ): Promise<T> {
280
+ const outcome = await withLock(key, fn, { ...opts, onBusy: "wait" });
281
+ // onBusy:"wait" resolves to {ran:true} or throws; ran:false is unreachable here.
282
+ if (!outcome.ran) {
283
+ throw new Error(`write lock '${key}' unavailable`);
284
+ }
285
+ return outcome.result;
286
+ }
287
+
288
+ /**
289
+ * Stamp a lock file's mtime to now without releasing it, marking "last completed".
290
+ * gc (auto-gc.ts) uses this to persist a cross-process throttle timestamp in the
291
+ * lock file's mtime (getLastDistillTime pattern) so the 24h throttle holds across
292
+ * processes rather than each process tracking its own module-level timestamp.
293
+ */
294
+ export function stampLock(cfg?: Partial<DistillLockConfig>): void {
295
+ const { lockPath } = resolveConfig(cfg);
296
+ mkdirSync(dirname(lockPath), { recursive: true });
297
+ // Create-or-overwrite so the mtime becomes now. Unlike releaseLock this leaves the
298
+ // file in place — it is a persistent "last run" marker (gc's cross-process throttle),
299
+ // read back via getLastDistillTime.
300
+ writeFileSync(lockPath, String(process.pid), "utf-8");
301
+ }
package/src/doctor.ts CHANGED
@@ -27,6 +27,7 @@ import {
27
27
  } from "./memory-schema.js";
28
28
  import { collectScopeInventory, type ScopeInventoryReport } from "./scope-inventory.js";
29
29
  import { MemoryStore, type MemoryEntry, validateStoragePath } from "./store.js";
30
+ import { readHeartbeats, formatAge } from "./source-heartbeat.js";
30
31
 
31
32
  interface CheckResult {
32
33
  name: string;
@@ -61,6 +62,13 @@ function warn(name: string, message: string, fix?: string): CheckResult {
61
62
  return { name, status: "warn", message, fix };
62
63
  }
63
64
 
65
+ export function describeSecretPresence(
66
+ value: string | undefined,
67
+ placeholder: string,
68
+ ): string | null {
69
+ return value && value !== placeholder ? "set" : null;
70
+ }
71
+
64
72
  function summarizeScopeInventoryLayers(report: ScopeInventoryReport): string {
65
73
  return report.layers
66
74
  .filter((layer) => layer.anomalyCount > 0 || layer.invalidCount > 0)
@@ -322,8 +330,8 @@ export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckRe
322
330
  // 3. .env + JINA_API_KEY
323
331
  loadDotEnv();
324
332
  const jinaKey = process.env.JINA_API_KEY;
325
- if (jinaKey && jinaKey !== "your_jina_api_key_here") {
326
- results.push(pass("JINA_API_KEY", `set (${jinaKey.slice(0, 8)}...)`));
333
+ if (describeSecretPresence(jinaKey, "your_jina_api_key_here")) {
334
+ results.push(pass("JINA_API_KEY", "set"));
327
335
  } else if (options.ci) {
328
336
  results.push(warn(
329
337
  "JINA_API_KEY",
@@ -508,6 +516,34 @@ export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckRe
508
516
  ));
509
517
  }
510
518
 
519
+ // GB-3: Source heartbeat summary
520
+ try {
521
+ const heartbeats = readHeartbeats();
522
+ const sources = Object.values(heartbeats);
523
+ if (sources.length === 0) {
524
+ results.push(pass("Source heartbeats", "no connector sources tracked yet"));
525
+ } else {
526
+ const summary = sources
527
+ .map((s) => `${s.source}: ${formatAge(s.lastIngest)}`)
528
+ .join(", ");
529
+ const staleCount = sources.filter((s) => {
530
+ const daysSince = (Date.now() - Date.parse(s.lastIngest)) / 86_400_000;
531
+ return daysSince > 7;
532
+ }).length;
533
+ if (staleCount > 0) {
534
+ results.push(warn(
535
+ "Source heartbeats",
536
+ `${summary} (${staleCount} stale >7d)`,
537
+ "lm ingest --connector <file>",
538
+ ));
539
+ } else {
540
+ results.push(pass("Source heartbeats", summary));
541
+ }
542
+ }
543
+ } catch {
544
+ results.push(warn("Source heartbeats", "unable to read heartbeat file"));
545
+ }
546
+
511
547
  return results;
512
548
  }
513
549
 
@@ -14,13 +14,15 @@
14
14
  * LLM-driven consolidation for semantic-level maintenance.
15
15
  */
16
16
 
17
- import type { MemoryStore } from "./store.js";
17
+ import type { MemoryStorePort } from "./memory-store-port.js";
18
18
  import type { LLMClient } from "./llm-client.js";
19
19
  import type { Embedder } from "./embedder.js";
20
- import { ConsolidationEngine, clusterAndConsolidate, DEFAULT_CONSOLIDATION_CONFIG } from "./consolidation-engine.js";
20
+ import { ConsolidationEngine, clusterAndConsolidate, DEFAULT_CONSOLIDATION_CONFIG, type ConsolidationKGSource } from "./consolidation-engine.js";
21
21
  import { maybeRunGc, type GcResult, type AutoGcConfig, DEFAULT_AUTO_GC_CONFIG } from "./auto-gc.js";
22
22
  import { getWriteCount, resetWriteCount } from "./activity-counter.js";
23
+ import { deriveUsageStatus, getUsageMetadata, isUsageSignalActive } from "./usage-tracker.js";
23
24
  import { isActiveMemory } from "./memory-evolution.js";
25
+ import { withLock } from "./distill-lock.js";
24
26
 
25
27
  // ---------------------------------------------------------------------------
26
28
  // Types
@@ -75,17 +77,22 @@ export interface DreamResult {
75
77
  // Pipeline
76
78
  // ---------------------------------------------------------------------------
77
79
 
78
- export async function runDream(params: {
79
- store: MemoryStore;
80
+ async function runDreamInner(params: {
81
+ store: MemoryStorePort;
80
82
  llm: LLMClient | null;
81
83
  embedder: Pick<Embedder, "embedPassage">;
82
84
  scope: string;
83
85
  config?: Partial<DreamConfig>;
84
86
  /** Skip the minimum-writes gate (force run) */
85
87
  force?: boolean;
88
+ /** Override the activity-counter stats file (defaults to <dataDir>/activity-stats.json). */
89
+ activityStatsPath?: string;
90
+ /** Optional KG evidence source for triple-overlap merging (null/absent = vector-only) */
91
+ kgStore?: ConsolidationKGSource | null;
86
92
  }): Promise<DreamResult> {
87
93
  const { store, llm, embedder, scope, force = false } = params;
88
94
  const config = { ...DEFAULT_DREAM_CONFIG, ...params.config };
95
+ const statsCfg = params.activityStatsPath ? { statsPath: params.activityStatsPath } : undefined;
89
96
  const phases: DreamPhaseResult[] = [];
90
97
 
91
98
  const stats = {
@@ -102,7 +109,7 @@ export async function runDream(params: {
102
109
  // =========================================================================
103
110
  // Phase 1: Orient — assess current memory state
104
111
  // =========================================================================
105
- const writeCount = getWriteCount();
112
+ const writeCount = getWriteCount(scope, statsCfg);
106
113
  stats.writesSinceLastDream = writeCount;
107
114
 
108
115
  const storeStats = await store.stats([scope]);
@@ -132,15 +139,41 @@ export async function runDream(params: {
132
139
  const active = entries.filter(e => isActiveMemory(e.metadata));
133
140
  stats.activeMemories = active.length;
134
141
 
142
+ // P0 B-1 观察:离线持久化 usageStatus 快照。写入路径不写 status(写入时
143
+ // useCount 必 >0,永远写不出 cold),所以 cold 只能由离线批量 derive 产生。
144
+ // 持久化值仅供 data_checkup/RIF 等观察视图——任何决策路径必须现场
145
+ // deriveUsageStatus(快照会 stale:之后 useCount/accessCount 继续变化)。
146
+ // best-effort:走 patchMetadata 单写通道,失败不阻断 dream。
147
+ // use 信号未采集时整段跳过——否则持久化的全是假 cold(useCount 恒零)。
148
+ let usageSnapshots = 0;
149
+ for (const entry of isUsageSignalActive() ? active : []) {
150
+ const status = deriveUsageStatus(entry);
151
+ const current = getUsageMetadata(entry).usageStatus;
152
+ if (status === current) continue;
153
+ // 默认态(unused 且从无 usage 对象)不写,避免给全库无谓写放大
154
+ if (status === "unused" && current === undefined) continue;
155
+ try {
156
+ await store.patchMetadata(entry.id, meta => {
157
+ const usage = meta.usage && typeof meta.usage === "object" && !Array.isArray(meta.usage)
158
+ ? (meta.usage as Record<string, unknown>)
159
+ : {};
160
+ meta.usage = { ...usage, usageStatus: status };
161
+ return meta;
162
+ });
163
+ usageSnapshots++;
164
+ } catch { /* observation snapshot must never block dream */ }
165
+ }
166
+
135
167
  phases.push({
136
168
  phase: "gather",
137
- detail: `${active.length} active entries gathered from ${entries.length} total`,
169
+ detail: `${active.length} active entries gathered from ${entries.length} total`
170
+ + (usageSnapshots > 0 ? `; usage snapshot: ${usageSnapshots} statuses persisted` : ""),
138
171
  });
139
172
 
140
173
  if (active.length < config.minClusterSize) {
141
174
  phases.push({ phase: "consolidate", detail: "skipped — too few active entries" });
142
175
  phases.push({ phase: "prune", detail: "skipped — too few entries for GC" });
143
- resetWriteCount();
176
+ resetWriteCount(scope, statsCfg);
144
177
  return { ran: true, reason: "completed_early", phases, stats };
145
178
  }
146
179
 
@@ -148,36 +181,44 @@ export async function runDream(params: {
148
181
  // Phase 3: Consolidate — cluster, merge, generate insights + patterns
149
182
  // =========================================================================
150
183
 
151
- // 3a: Deterministic consolidation (merge near-duplicates, link clusters)
152
- const engine = new ConsolidationEngine(store, {
153
- ...DEFAULT_CONSOLIDATION_CONFIG,
154
- clusterThreshold: config.clusterThreshold,
155
- maxEntriesPerRun: config.maxEntriesPerRun,
156
- });
157
- const consolidation = await engine.run(scope);
158
- stats.clustersFound += consolidation.clustersFound;
159
- stats.mergedCount += consolidation.mergedCount;
160
-
161
- // 3b: LLM-driven cluster consolidation (insights + patterns) — requires LLM
162
- if (llm) {
163
- const clusterResult = await clusterAndConsolidate({
164
- entries: active,
165
- embedder,
166
- llm,
167
- store,
168
- scope,
169
- minClusterSize: config.minClusterSize,
170
- clusterThreshold: config.clusterThreshold - 0.07, // slightly lower for semantic clustering
171
- extractPatterns: config.extractPatterns,
172
- });
173
- stats.clustersFound += clusterResult.clustersFound;
174
- stats.insightsGenerated = clusterResult.insightsGenerated;
175
- stats.patternsExtracted = clusterResult.patternsExtracted;
176
- }
184
+ // P2 (codex): run the consolidation phase under the scope's consolidate lock so a
185
+ // standalone consolidate_memories on the same scope can't run concurrently with the
186
+ // dream's own consolidation. Nested inside the dream lock (dream → consolidate →
187
+ // store-write), consistent with the one-way lock order.
188
+ const consolidateOutcome = await withLock(`consolidate-${scope}`, async () => {
189
+ // 3a: Deterministic consolidation (merge near-duplicates, link clusters)
190
+ const engine = new ConsolidationEngine(store, {
191
+ ...DEFAULT_CONSOLIDATION_CONFIG,
192
+ clusterThreshold: config.clusterThreshold,
193
+ maxEntriesPerRun: config.maxEntriesPerRun,
194
+ }, params.kgStore ?? null);
195
+ const consolidation = await engine.run(scope);
196
+ stats.clustersFound += consolidation.clustersFound;
197
+ stats.mergedCount += consolidation.mergedCount;
198
+
199
+ // 3b: LLM-driven cluster consolidation (insights + patterns) — requires LLM
200
+ if (llm) {
201
+ const clusterResult = await clusterAndConsolidate({
202
+ entries: active,
203
+ embedder,
204
+ llm,
205
+ store,
206
+ scope,
207
+ minClusterSize: config.minClusterSize,
208
+ clusterThreshold: config.clusterThreshold - 0.07, // slightly lower for semantic clustering
209
+ extractPatterns: config.extractPatterns,
210
+ });
211
+ stats.clustersFound += clusterResult.clustersFound;
212
+ stats.insightsGenerated = clusterResult.insightsGenerated;
213
+ stats.patternsExtracted = clusterResult.patternsExtracted;
214
+ }
215
+ }, { onBusy: "skip", expireMs: 600_000 });
177
216
 
178
217
  phases.push({
179
218
  phase: "consolidate",
180
- detail: `${stats.clustersFound} clusters, ${stats.mergedCount} merged, ${stats.insightsGenerated} insights, ${stats.patternsExtracted} patterns`,
219
+ detail: consolidateOutcome.ran
220
+ ? `${stats.clustersFound} clusters, ${stats.mergedCount} merged, ${stats.insightsGenerated} insights, ${stats.patternsExtracted} patterns`
221
+ : `skipped — another process holds the consolidate lock for scope ${scope}`,
181
222
  });
182
223
 
183
224
  // =========================================================================
@@ -199,6 +240,49 @@ export async function runDream(params: {
199
240
  return { ran: true, phases, stats };
200
241
  }
201
242
 
243
+ /**
244
+ * P0-1: run dream under a per-scope cross-process lock. Dream internally does
245
+ * consolidation + gc, so the lock sits at this outermost entry — no nested maintenance
246
+ * locks. If another of the 11 mcp-server processes holds the scope's dream lock, skip
247
+ * rather than queue: a redundant dream on a personal store is pure waste, and the next
248
+ * writer's dream picks up whatever this run skipped.
249
+ */
250
+ export async function runDream(params: {
251
+ store: MemoryStorePort;
252
+ llm: LLMClient | null;
253
+ embedder: Pick<Embedder, "embedPassage">;
254
+ scope: string;
255
+ config?: Partial<DreamConfig>;
256
+ /** Skip the minimum-writes gate (force run) */
257
+ force?: boolean;
258
+ /** Override the activity-counter stats file (defaults to <dataDir>/activity-stats.json). */
259
+ activityStatsPath?: string;
260
+ /** Optional KG evidence source for triple-overlap merging (null/absent = vector-only) */
261
+ kgStore?: ConsolidationKGSource | null;
262
+ }): Promise<DreamResult> {
263
+ const outcome = await withLock(
264
+ `dream-${params.scope}`,
265
+ () => runDreamInner(params),
266
+ { onBusy: "skip", expireMs: 600_000 },
267
+ );
268
+ if (outcome.ran) return outcome.result;
269
+ return {
270
+ ran: false,
271
+ reason: "locked_by_another_process",
272
+ phases: [{ phase: "orient", detail: `another process holds the dream lock for scope ${params.scope}` }],
273
+ stats: {
274
+ totalMemories: 0,
275
+ activeMemories: 0,
276
+ writesSinceLastDream: 0,
277
+ clustersFound: 0,
278
+ insightsGenerated: 0,
279
+ patternsExtracted: 0,
280
+ mergedCount: 0,
281
+ archivedCount: 0,
282
+ },
283
+ };
284
+ }
285
+
202
286
  // ---------------------------------------------------------------------------
203
287
  // Formatting
204
288
  // ---------------------------------------------------------------------------
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Centralized RECALLNEST_* environment variable accessors (P3-C config 收口).
3
+ *
4
+ * Single source of truth for every fixed `RECALLNEST_*` env flag read across `src/`:
5
+ * the env NAME, its default, and its parsing all live here. Consumers call these
6
+ * accessors instead of reading `process.env` directly.
7
+ *
8
+ * This is a PURE-MOVEMENT refactor — each accessor preserves the exact original
9
+ * parsing. Do not "improve" them; the following invariants are load-bearing:
10
+ *
11
+ * - Boolean flags use strict `=== "true"` — no trim, no case-folding, no truthy
12
+ * coercion. `"True"` / `" true "` / `"1"` must NOT enable a flag.
13
+ * - String defaults use `||`, NOT `??` — an empty string falls through to the
14
+ * default (matches the original inline reads).
15
+ * - Raw-value accessors (recall mode, ports) return the UNPARSED env value so the
16
+ * caller keeps its existing validation / clamping / config-fallback logic. In
17
+ * particular `recallModeRaw()` must not default to "summary" here, or it would
18
+ * bypass the `config.recallMode` fallback in resolveRecallMode().
19
+ * - `mcpTier()` keeps only the type assertion with NO runtime validation — an
20
+ * illegal non-empty value is preserved as-is (downstream shouldRegisterTool()
21
+ * relies on this), so do not coerce unknown values back to the default.
22
+ *
23
+ * Accessors are FUNCTIONS (lazy): they read `process.env` at call time. Never
24
+ * freeze them into module-load constants — 100+ tests toggle `process.env` at
25
+ * runtime. Consumers must substitute in place and never relocate a read across
26
+ * the `loadDotEnv()` boundary or into a different evaluation phase: five reads are
27
+ * intentionally module-init-time eager (mcp-server tier, api/ui ports,
28
+ * activity-counter / distill-lock data dir) and must stay eager.
29
+ *
30
+ * Intentionally NOT centralized here:
31
+ * - scope-policy.ts's RECALLNEST_DEFAULT_SCOPE / _SCOPE / _PROJECT_SCOPE /
32
+ * _SESSION_ID — an injectable `options.env || process.env` policy entry that is
33
+ * already consolidated in that module with caller-injection semantics.
34
+ * - store.ts's `RECALLNEST_NS` — a local hash-namespace constant, not an env var.
35
+ * - config-template `${VAR}` expansion in runtime-config / embedder / llm-client.
36
+ */
37
+
38
+ // --- Boolean feature flags (strict === "true") ---
39
+
40
+ export const multiVector = (): boolean => process.env.RECALLNEST_MULTI_VECTOR === "true";
41
+
42
+ export const emotionScoring = (): boolean => process.env.RECALLNEST_EMOTION_SCORING === "true";
43
+
44
+ export const predictiveMemory = (): boolean => process.env.RECALLNEST_PREDICTIVE_MEMORY === "true";
45
+
46
+ export const synthesize = (): boolean => process.env.RECALLNEST_SYNTHESIZE === "true";
47
+
48
+ export const llmConsolidation = (): boolean => process.env.RECALLNEST_LLM_CONSOLIDATION === "true";
49
+
50
+ export const constructiveRetrieval = (): boolean =>
51
+ process.env.RECALLNEST_CONSTRUCTIVE_RETRIEVAL === "true";
52
+
53
+ export const narrativeMode = (): boolean => process.env.RECALLNEST_NARRATIVE_MODE === "true";
54
+
55
+ export const kgMode = (): boolean => process.env.RECALLNEST_KG_MODE === "true";
56
+
57
+ export const coreSummary = (): boolean => process.env.RECALLNEST_CORE_SUMMARY === "true";
58
+
59
+ export const errorSignatureBoost = (): boolean =>
60
+ process.env.RECALLNEST_ERROR_SIGNATURE_BOOST === "true";
61
+
62
+ export const usageDecay = (): boolean => process.env.RECALLNEST_USAGE_DECAY === "true";
63
+
64
+ // --- String settings with `||` default (empty string falls through) ---
65
+
66
+ export const dataDir = (): string => process.env.RECALLNEST_DATA_DIR || "data";
67
+
68
+ export const mcpTier = (): "core" | "advanced" | "full" =>
69
+ (process.env.RECALLNEST_MCP_TIER || "advanced") as "core" | "advanced" | "full";
70
+
71
+ // --- Raw env values (caller validates / clamps / falls back to config) ---
72
+
73
+ export const recallModeRaw = (): string | undefined => process.env.RECALLNEST_RECALL_MODE;
74
+
75
+ export const uiPortRaw = (): string | undefined => process.env.RECALLNEST_UI_PORT;
76
+
77
+ export const apiPortRaw = (): string | undefined => process.env.RECALLNEST_API_PORT;