recallnest 2.3.0 → 2.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/CHANGELOG.md +150 -0
  2. package/README.md +17 -31
  3. package/README_CN.md +12 -22
  4. package/assets/ui/app.js +1031 -0
  5. package/assets/ui/index.html +224 -0
  6. package/assets/ui/styles.css +781 -0
  7. package/config.json.example +4 -1
  8. package/docs/api-reference.md +1291 -0
  9. package/docs/architecture.md +168 -0
  10. package/docs/connector-spec.md +126 -0
  11. package/eval/cases-canary.json +58 -0
  12. package/eval/cases-scope-robustness.json +278 -0
  13. package/eval/cases.json +117 -20
  14. package/package.json +22 -1
  15. package/src/__tests__/access-tracker-novelty.test.ts +238 -0
  16. package/src/__tests__/activity-counter.test.ts +125 -0
  17. package/src/__tests__/adaptive-fidelity.test.ts +102 -0
  18. package/src/__tests__/admission-control.test.ts +206 -0
  19. package/src/__tests__/anchor-generator.test.ts +67 -0
  20. package/src/__tests__/auto-consolidation.test.ts +168 -0
  21. package/src/__tests__/auto-recall.test.ts +131 -0
  22. package/src/__tests__/babel-integration.test.ts +144 -0
  23. package/src/__tests__/batch-dedup.test.ts +65 -0
  24. package/src/__tests__/canary-eval.test.ts +114 -0
  25. package/src/__tests__/capture-engine.test.ts +1238 -0
  26. package/src/__tests__/capture-heuristic.test.ts +171 -0
  27. package/src/__tests__/cascade-forget.test.ts +210 -0
  28. package/src/__tests__/category-min-scores.test.ts +33 -0
  29. package/src/__tests__/cc8-cc1-essential-context.test.ts +298 -0
  30. package/src/__tests__/circuit-breaker.test.ts +99 -0
  31. package/src/__tests__/cluster-consolidation.test.ts +782 -0
  32. package/src/__tests__/confidence-metadata.test.ts +125 -0
  33. package/src/__tests__/confidence-tracker.test.ts +261 -0
  34. package/src/__tests__/conflict-advisor.test.ts +206 -0
  35. package/src/__tests__/conflict-escalation.test.ts +149 -0
  36. package/src/__tests__/conflict-lifecycle.test.ts +83 -0
  37. package/src/__tests__/conflict-output.test.ts +85 -0
  38. package/src/__tests__/conflict-store.test.ts +374 -0
  39. package/src/__tests__/connector-v1.test.ts +181 -0
  40. package/src/__tests__/consolidation-engine.test.ts +333 -0
  41. package/src/__tests__/constructive-retrieval.test.ts +509 -0
  42. package/src/__tests__/context-collapse-renderer.test.ts +214 -0
  43. package/src/__tests__/context-composer-pins.test.ts +64 -0
  44. package/src/__tests__/context-composer-task-selection.test.ts +24 -0
  45. package/src/__tests__/context-reconstructor.test.ts +65 -0
  46. package/src/__tests__/context-renderer.test.ts +91 -0
  47. package/src/__tests__/continuity-eval.test.ts +275 -0
  48. package/src/__tests__/continuity-schema.test.ts +97 -0
  49. package/src/__tests__/contracts.test.ts +216 -0
  50. package/src/__tests__/conversation-importer.test.ts +692 -0
  51. package/src/__tests__/core-recall-eval.test.ts +235 -0
  52. package/src/__tests__/core-summary.test.ts +109 -0
  53. package/src/__tests__/data-checkup.test.ts +276 -0
  54. package/src/__tests__/decay-exemption.test.ts +84 -0
  55. package/src/__tests__/dedup-actions.test.ts +58 -0
  56. package/src/__tests__/dedup-guard.test.ts +39 -0
  57. package/src/__tests__/deterministic-id.test.ts +39 -0
  58. package/src/__tests__/distill-lock.test.ts +305 -0
  59. package/src/__tests__/distill-session-argshape.test.ts +79 -0
  60. package/src/__tests__/doctor.test.ts +319 -0
  61. package/src/__tests__/dream-pipeline.test.ts +321 -0
  62. package/src/__tests__/drill-down.test.ts +108 -0
  63. package/src/__tests__/embedder.test.ts +122 -0
  64. package/src/__tests__/emotion-decay.test.ts +46 -0
  65. package/src/__tests__/emotion-detector.test.ts +56 -0
  66. package/src/__tests__/emotion-phase1-integration.test.ts +153 -0
  67. package/src/__tests__/emotion-retrieval.test.ts +56 -0
  68. package/src/__tests__/entity-resolver.test.ts +58 -0
  69. package/src/__tests__/env-config.test.ts +151 -0
  70. package/src/__tests__/error-signature-boost.test.ts +87 -0
  71. package/src/__tests__/error-signature.test.ts +95 -0
  72. package/src/__tests__/error-taxonomy.test.ts +129 -0
  73. package/src/__tests__/ethics-layer.test.ts +476 -0
  74. package/src/__tests__/event-segmenter.test.ts +153 -0
  75. package/src/__tests__/feature-flags-integration.test.ts +54 -0
  76. package/src/__tests__/frequency-tracker.test.ts +113 -0
  77. package/src/__tests__/graph-export.test.ts +489 -0
  78. package/src/__tests__/hotness-blend.test.ts +76 -0
  79. package/src/__tests__/ingest-chunking.test.ts +44 -0
  80. package/src/__tests__/ingest.test.ts +360 -0
  81. package/src/__tests__/interference-detector.test.ts +203 -0
  82. package/src/__tests__/kg-extractor.test.ts +192 -0
  83. package/src/__tests__/kg-store.test.ts +300 -0
  84. package/src/__tests__/llm-consolidation.test.ts +226 -0
  85. package/src/__tests__/maintenance-locks.test.ts +174 -0
  86. package/src/__tests__/mcp-efficiency.test.ts +288 -0
  87. package/src/__tests__/mcp-related-scopes.test.ts +151 -0
  88. package/src/__tests__/memory-boundaries.test.ts +250 -0
  89. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +345 -0
  90. package/src/__tests__/memory-evolution.test.ts +271 -0
  91. package/src/__tests__/memory-health-rebalance.test.ts +144 -0
  92. package/src/__tests__/memory-lint.test.ts +421 -0
  93. package/src/__tests__/memory-output.test.ts +215 -0
  94. package/src/__tests__/memory-promotion.test.ts +315 -0
  95. package/src/__tests__/migrate-dedup-ids.test.ts +65 -0
  96. package/src/__tests__/multi-vector.test.ts +242 -0
  97. package/src/__tests__/narrative-architecture.test.ts +456 -0
  98. package/src/__tests__/noise-filter.test.ts +205 -0
  99. package/src/__tests__/obsidian-connector.test.ts +363 -0
  100. package/src/__tests__/omitted-reporting.test.ts +98 -0
  101. package/src/__tests__/pending-review.test.ts +77 -0
  102. package/src/__tests__/pii-redact.test.ts +314 -0
  103. package/src/__tests__/post-retrieval-filter.test.ts +178 -0
  104. package/src/__tests__/ppr-traversal.test.ts +256 -0
  105. package/src/__tests__/prediction-engine.test.ts +349 -0
  106. package/src/__tests__/predictive-prospective-memory.test.ts +416 -0
  107. package/src/__tests__/preference-guard-smoke.test.ts +211 -0
  108. package/src/__tests__/preference-matcher.test.ts +230 -0
  109. package/src/__tests__/preference-slots.test.ts +205 -0
  110. package/src/__tests__/prospective-memory.test.ts +283 -0
  111. package/src/__tests__/query-entity-detector.test.ts +171 -0
  112. package/src/__tests__/query-expander-alias.test.ts +146 -0
  113. package/src/__tests__/query-expander.test.ts +34 -0
  114. package/src/__tests__/recall-governor.test.ts +227 -0
  115. package/src/__tests__/recall-mode.test.ts +64 -0
  116. package/src/__tests__/result-synthesizer.test.ts +198 -0
  117. package/src/__tests__/resume-context.test.ts +4898 -0
  118. package/src/__tests__/retrieval-trace.test.ts +213 -0
  119. package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
  120. package/src/__tests__/retroactive-boost.test.ts +223 -0
  121. package/src/__tests__/salience-scoring.test.ts +106 -0
  122. package/src/__tests__/scope-inventory.test.ts +200 -0
  123. package/src/__tests__/scope-policy.test.ts +69 -0
  124. package/src/__tests__/scope-suggester.test.ts +80 -0
  125. package/src/__tests__/session-checkpoint.test.ts +291 -0
  126. package/src/__tests__/session-distiller.test.ts +640 -0
  127. package/src/__tests__/session-gc.test.ts +215 -0
  128. package/src/__tests__/skill-engine.test.ts +552 -0
  129. package/src/__tests__/skill-promotion.test.ts +601 -0
  130. package/src/__tests__/skill-verifier.test.ts +122 -0
  131. package/src/__tests__/source-heartbeat.test.ts +250 -0
  132. package/src/__tests__/store-patch-metadata.test.ts +205 -0
  133. package/src/__tests__/store-scope-audit.test.ts +81 -0
  134. package/src/__tests__/store-upsert.test.ts +184 -0
  135. package/src/__tests__/supersede-chain.test.ts +154 -0
  136. package/src/__tests__/temporal-parser.test.ts +241 -0
  137. package/src/__tests__/temporal-validity.test.ts +160 -0
  138. package/src/__tests__/term-registry.test.ts +121 -0
  139. package/src/__tests__/tool-error-surfacing.test.ts +98 -0
  140. package/src/__tests__/tool-output-compressor.test.ts +160 -0
  141. package/src/__tests__/topic-tag.test.ts +113 -0
  142. package/src/__tests__/trusted-memory.test.ts +839 -0
  143. package/src/__tests__/usage-tracker.test.ts +287 -0
  144. package/src/__tests__/version-manager.test.ts +213 -0
  145. package/src/__tests__/workflow-observation-skill-outcome.test.ts +352 -0
  146. package/src/__tests__/workflow-observation.test.ts +433 -0
  147. package/src/__tests__/write-verifier.test.ts +122 -0
  148. package/src/access-tracker.ts +38 -32
  149. package/src/activity-counter.ts +46 -18
  150. package/src/aliases.ts +133 -0
  151. package/src/api-server.ts +27 -7
  152. package/src/audit-log.ts +4 -2
  153. package/src/auto-consolidation.ts +8 -3
  154. package/src/auto-gc.ts +85 -21
  155. package/src/capture-engine.ts +54 -21
  156. package/src/cli.ts +169 -37
  157. package/src/confidence-tracker.ts +36 -18
  158. package/src/consolidation-engine.ts +109 -11
  159. package/src/context-composer.ts +8 -1
  160. package/src/data-checkup.ts +95 -6
  161. package/src/distill-lock.ts +194 -15
  162. package/src/doctor.ts +9 -2
  163. package/src/dream-pipeline.ts +118 -34
  164. package/src/env-config.ts +77 -0
  165. package/src/error-signature.ts +153 -0
  166. package/src/error-taxonomy.ts +115 -0
  167. package/src/eval.ts +251 -6
  168. package/src/idempotency.ts +8 -0
  169. package/src/ingest.ts +31 -10
  170. package/src/kg-extractor.ts +6 -2
  171. package/src/kg-store.ts +236 -44
  172. package/src/llm-consolidation.ts +2 -1
  173. package/src/mcp-server.ts +85 -1799
  174. package/src/mcp-tool-deps.ts +48 -0
  175. package/src/mcp-tools-advanced.ts +941 -0
  176. package/src/mcp-tools-core.ts +633 -0
  177. package/src/mcp-tools-governance.ts +475 -0
  178. package/src/memory-boundaries.ts +3 -1
  179. package/src/memory-evolution.ts +73 -1
  180. package/src/memory-lint.ts +183 -10
  181. package/src/memory-output.ts +114 -3
  182. package/src/memory-promotion.ts +246 -0
  183. package/src/memory-schema.ts +25 -2
  184. package/src/memory-store-port.ts +43 -0
  185. package/src/multi-vector.ts +2 -1
  186. package/src/narrative-schema.ts +2 -1
  187. package/src/noise-filter.ts +57 -6
  188. package/src/pii-detector.ts +157 -5
  189. package/src/query-expander.ts +90 -9
  190. package/src/result-synthesizer.ts +2 -1
  191. package/src/retention-policy.ts +2 -1
  192. package/src/retriever.ts +259 -139
  193. package/src/runtime-config.ts +8 -1
  194. package/src/scope-suggester.ts +113 -0
  195. package/src/session-distiller.ts +23 -2
  196. package/src/session-engine.ts +2 -1
  197. package/src/session-schema.ts +1 -0
  198. package/src/session-store.ts +5 -0
  199. package/src/skill-engine.ts +78 -0
  200. package/src/skill-promotion.ts +247 -47
  201. package/src/skill-schema.ts +16 -3
  202. package/src/skill-verifier.ts +152 -0
  203. package/src/store.ts +267 -38
  204. package/src/tool-output-compressor.ts +3 -1
  205. package/src/ui-server.ts +2 -1
  206. package/src/usage-tracker.ts +242 -0
  207. package/src/version-manager.ts +3 -2
  208. package/src/workflow-observation-engine.ts +2 -1
  209. package/src/workflow-observation-schema.ts +2 -1
  210. package/src/workflow-observation-store.ts +6 -2
  211. package/.env +0 -4
  212. package/CLAUDE.md +0 -46
  213. package/config.json +0 -55
  214. package/daily-recall.sh +0 -110
  215. package/logs/daily-recall.error.log +0 -0
  216. package/logs/daily-recall.log +0 -60
  217. package/logs/ingest-2026-04-11.log +0 -110
  218. package/logs/ingest-2026-04-12.log +0 -90
  219. package/logs/ingest-2026-04-13.log +0 -87
  220. package/logs/launchagent-stderr.log +0 -0
  221. package/logs/launchagent-stdout.log +0 -0
@@ -11,6 +11,7 @@ import {
11
11
  mkdirSync,
12
12
  } from "node:fs";
13
13
  import { join, dirname } from "node:path";
14
+ import * as envConfig from "./env-config.js";
14
15
 
15
16
  // ---------------------------------------------------------------------------
16
17
  // Config
@@ -29,7 +30,7 @@ export interface ActivityCounterConfig {
29
30
 
30
31
  export const DEFAULT_ACTIVITY_CONFIG: ActivityCounterConfig = {
31
32
  statsPath: join(
32
- process.env.RECALLNEST_DATA_DIR || "data",
33
+ envConfig.dataDir(),
33
34
  "activity-stats.json",
34
35
  ),
35
36
  lightThreshold: 15,
@@ -44,8 +45,8 @@ export type DistillTier = "none" | "light" | "standard" | "deep";
44
45
  // ---------------------------------------------------------------------------
45
46
 
46
47
  interface ActivityStats {
47
- writesSinceLastDistill: number;
48
- lastResetAt: number;
48
+ /** Per-scope write counts since that scope's last dream/distill reset. */
49
+ scopes: Record<string, number>;
49
50
  }
50
51
 
51
52
  function resolveConfig(
@@ -56,12 +57,24 @@ function resolveConfig(
56
57
 
57
58
  function readStats(statsPath: string): ActivityStats {
58
59
  if (!existsSync(statsPath)) {
59
- return { writesSinceLastDistill: 0, lastResetAt: Date.now() };
60
+ return { scopes: {} };
60
61
  }
61
62
  try {
62
- return JSON.parse(readFileSync(statsPath, "utf-8"));
63
+ const parsed: unknown = JSON.parse(readFileSync(statsPath, "utf-8"));
64
+ // Robust against the legacy global format ({writesSinceLastDistill}) — no migration,
65
+ // just treat anything without a `scopes` map as empty (counts restart per scope).
66
+ if (
67
+ parsed !== null &&
68
+ typeof parsed === "object" &&
69
+ "scopes" in parsed &&
70
+ typeof (parsed as { scopes?: unknown }).scopes === "object" &&
71
+ (parsed as { scopes?: unknown }).scopes !== null
72
+ ) {
73
+ return { scopes: (parsed as { scopes: Record<string, number> }).scopes };
74
+ }
75
+ return { scopes: {} };
63
76
  } catch {
64
- return { writesSinceLastDistill: 0, lastResetAt: Date.now() };
77
+ return { scopes: {} };
65
78
  }
66
79
  }
67
80
 
@@ -74,34 +87,49 @@ function writeStats(statsPath: string, stats: ActivityStats): void {
74
87
  // Public API
75
88
  // ---------------------------------------------------------------------------
76
89
 
77
- /** Increment write counter by n (default 1). Returns new count. */
90
+ /** Increment a scope's write counter by n (default 1). Returns the scope's new count. */
78
91
  export function incrementWriteCount(
92
+ scope: string,
79
93
  n = 1,
80
94
  cfg?: Partial<ActivityCounterConfig>,
81
95
  ): number {
82
96
  const { statsPath } = resolveConfig(cfg);
83
97
  const stats = readStats(statsPath);
84
- stats.writesSinceLastDistill += n;
98
+ stats.scopes[scope] = (stats.scopes[scope] ?? 0) + n;
85
99
  writeStats(statsPath, stats);
86
- return stats.writesSinceLastDistill;
100
+ return stats.scopes[scope];
87
101
  }
88
102
 
89
- /** Read current write count without modifying. */
90
- export function getWriteCount(cfg?: Partial<ActivityCounterConfig>): number {
103
+ /** Read a scope's current write count without modifying (0 if never written). */
104
+ export function getWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig>): number {
91
105
  const { statsPath } = resolveConfig(cfg);
92
- return readStats(statsPath).writesSinceLastDistill;
106
+ return readStats(statsPath).scopes[scope] ?? 0;
93
107
  }
94
108
 
95
- /** Reset counter after successful distill. */
96
- export function resetWriteCount(cfg?: Partial<ActivityCounterConfig>): void {
109
+ /** Reset one scope's counter after its successful dream/distill (leaves other scopes untouched). */
110
+ export function resetWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig>): void {
97
111
  const { statsPath } = resolveConfig(cfg);
98
- writeStats(statsPath, { writesSinceLastDistill: 0, lastResetAt: Date.now() });
112
+ const stats = readStats(statsPath);
113
+ delete stats.scopes[scope];
114
+ writeStats(statsPath, stats);
115
+ }
116
+
117
+ /** Scopes whose write count is at or above `threshold` — the dream scheduler's work list. */
118
+ export function listScopesAboveThreshold(
119
+ threshold: number,
120
+ cfg?: Partial<ActivityCounterConfig>,
121
+ ): string[] {
122
+ const { statsPath } = resolveConfig(cfg);
123
+ const stats = readStats(statsPath);
124
+ return Object.entries(stats.scopes)
125
+ .filter(([, count]) => count >= threshold)
126
+ .map(([scope]) => scope);
99
127
  }
100
128
 
101
- /** Determine which distill tier the current write count warrants. */
102
- export function getDistillTier(cfg?: Partial<ActivityCounterConfig>): DistillTier {
129
+ /** Determine which distill tier a scope's write count warrants. */
130
+ export function getDistillTier(scope: string, cfg?: Partial<ActivityCounterConfig>): DistillTier {
103
131
  const config = resolveConfig(cfg);
104
- const count = getWriteCount(cfg);
132
+ const count = getWriteCount(scope, cfg);
105
133
  if (count >= config.deepThreshold) return "deep";
106
134
  if (count >= config.standardThreshold) return "standard";
107
135
  if (count >= config.lightThreshold) return "light";
package/src/aliases.ts ADDED
@@ -0,0 +1,133 @@
1
+ /**
2
+ * Alias dictionary for query rewriting.
3
+ *
4
+ * Manually maintained — do NOT use LLM expansion (risks precision pollution).
5
+ * Each rule appends entity tokens to the query when the pattern matches,
6
+ * giving BM25/vector search more anchor points without losing the original wording.
7
+ *
8
+ * Why expand (append) rather than replace:
9
+ * - Preserves the user's exact wording for vector embedding fidelity.
10
+ * - BM25 gets extra keyword anchors to find the right entity.
11
+ * - Reversible via inspecting the expanded vs. original query.
12
+ *
13
+ * Add new rules sparingly. Each entry should solve a real failure
14
+ * documented in FAILURES.md, not a hypothetical.
15
+ */
16
+
17
+ export interface AliasRule {
18
+ pattern: RegExp;
19
+ expansion: string;
20
+ comment?: string;
21
+ }
22
+
23
+ export const ALIAS_RULES: AliasRule[] = [
24
+ // Entity aliases — colloquial → canonical name
25
+ {
26
+ pattern: /(?:我的)?记忆项目/,
27
+ expansion: "recallnest",
28
+ comment: "FAILURES 2026-04-25: '我的记忆项目' → recallnest",
29
+ },
30
+ {
31
+ pattern: /我的桥/,
32
+ expansion: "telegram-ai-bridge",
33
+ comment: "Alice 用'我的桥'指 telegram-ai-bridge",
34
+ },
35
+
36
+ // Operations vocabulary — operator wording → canonical entities
37
+ {
38
+ pattern: /aws\s*(?:bot|ssh|podcast)/i,
39
+ expansion: "aws-bot hermes-aws aws-podcast-daily",
40
+ comment: "FAILURES 2026-03-06: aws bot/ssh/podcast → aws-bot hermes-aws entities",
41
+ },
42
+ {
43
+ pattern: /bot\s*(?:挂|崩|不能用|跑不起|失联|down)/i,
44
+ expansion: "bot-doctor hermes docker launchd 排查 重启",
45
+ comment: "Operator bot-crash debugging vocabulary",
46
+ },
47
+
48
+ // Cross-window / continuity hints — these strengthen recall of session state.
49
+ // Pattern 必须明确出现"隔壁/另一窗口/session"才扩;纯"之前说过"太宽会污染
50
+ // 普通话题(如 fuzzy_ai_feelings 的 "之前聊过 AI 感受" 被错误归类成 cross-window)。
51
+ {
52
+ pattern: /(?:刚刚|刚才|之前)(?:.{0,10})(?:隔壁|另一个|其他)(?:.{0,10})(?:窗口|session)/,
53
+ expansion: "checkpoint session resume_context 跨窗口",
54
+ comment: "FAILURES 2026-05-10: cross-window real-time sync hint",
55
+ },
56
+
57
+ // Tool provenance — common ambiguous tool references
58
+ {
59
+ pattern: /taobao.{0,5}(?:mcp|官方)/i,
60
+ expansion: "MCP server provenance source",
61
+ comment: "FAILURES 2026-05-01: taobao MCP official-vs-third-party",
62
+ },
63
+
64
+ // RecallNest self-introspection — capability queries
65
+ {
66
+ pattern: /recallnest.{0,15}(?:有哪些|工具|能做|功能|tool)/i,
67
+ expansion: "search_memory store_memory resume_context checkpoint_session MCP",
68
+ comment: "FAILURES 2026-04-20: RN self-introspection capability digest",
69
+ },
70
+ {
71
+ pattern: /recallnest.{0,10}(?:ingest|跑了|跑过|状态|telemetry)/i,
72
+ expansion: "import_conversations weekly-distill ingest telemetry 成功 失败",
73
+ comment: "FAILURES 2026-04-20: RN ingest telemetry queries",
74
+ },
75
+
76
+ // Personal-life recall — subscription / friends provenance
77
+ {
78
+ pattern: /(?:chatgpt|订阅).{0,10}(?:谁|怎么|哪里|帮|顶)/i,
79
+ expansion: "ChatGPT Plus 群友 加拿大 堂姐 澳洲 代订 aliceljyalice",
80
+ comment: "FAILURES 2026-05-12: ChatGPT subscription provenance chain",
81
+ },
82
+ ];
83
+
84
+ /** 超过此长度的 query 不做别名扩展——长 query 是粘贴文档场景,扩展只添噪声 (P1-B)。 */
85
+ export const ALIAS_QUERY_MAX_LENGTH = 100;
86
+
87
+ /**
88
+ * Expand a query by appending alias tokens for matching patterns.
89
+ * Returns the original query unchanged if no rules match.
90
+ *
91
+ * 注意分工:本模块只承载**静态 builtin 规则**,作用于 retrieve() 全通道
92
+ * (vector + BM25)。**用户级** alias(data/alias-map.json)由
93
+ * query-expander.ts 加载,只作用于 BM25 通道(hybridRetrieval 内部)——
94
+ * 用户别名不进 vector embedding,避免随手加的别名造成语义向量漂移。
95
+ *
96
+ * Example:
97
+ * expandQueryWithAliases("帮我看我的记忆项目")
98
+ * → "帮我看我的记忆项目 recallnest"
99
+ * expandQueryWithAliases("aws ssh 怎么连")
100
+ * → "aws ssh 怎么连 aws-bot hermes-aws aws-podcast-daily"
101
+ */
102
+ export function expandQueryWithAliases(query: string): string {
103
+ if (!query || typeof query !== "string") return query;
104
+ if (query.length > ALIAS_QUERY_MAX_LENGTH) return query;
105
+
106
+ const additions: string[] = [];
107
+ const seen = new Set<string>();
108
+
109
+ for (const rule of ALIAS_RULES) {
110
+ if (rule.pattern.test(query)) {
111
+ for (const token of rule.expansion.split(/\s+/)) {
112
+ if (token && !seen.has(token) && !query.includes(token)) {
113
+ additions.push(token);
114
+ seen.add(token);
115
+ }
116
+ }
117
+ }
118
+ }
119
+
120
+ return additions.length > 0 ? `${query} ${additions.join(" ")}` : query;
121
+ }
122
+
123
+ /**
124
+ * Inspection helper — return which builtin rules matched a query, for debugging.
125
+ * 用户级规则(BM25-only)的检视走 query-expander.ts 的 explainUserAliases。
126
+ */
127
+ export function explainAliasExpansion(query: string): Array<{ pattern: string; expansion: string; comment?: string }> {
128
+ return ALIAS_RULES.filter((rule) => rule.pattern.test(query)).map((rule) => ({
129
+ pattern: rule.pattern.source,
130
+ expansion: rule.expansion,
131
+ comment: rule.comment,
132
+ }));
133
+ }
package/src/api-server.ts CHANGED
@@ -17,19 +17,23 @@ import { ConflictStatusSchema } from "./conflict-schema.js";
17
17
  import { resolveConflictCandidate } from "./conflict-engine.js";
18
18
  import { escalateConflicts } from "./conflict-escalation.js";
19
19
  import { ConflictCandidateStore } from "./conflict-store.js";
20
+ import { createAuditLogger } from "./audit-log.js";
20
21
  import { parseConflictAttention, summarizeConflictLifecycle } from "./conflict-lifecycle.js";
21
22
  import { buildConflictAuditSummary, clusterConflicts, summarizeConflictAdvice } from "./conflict-advisor.js";
22
23
  import { buildWorkflowEvidence, buildWorkflowObservationRecord, inspectWorkflowDashboard, inspectWorkflowHealth } from "./workflow-observation-engine.js";
24
+ import { recordSkillOutcome } from "./skill-engine.js";
23
25
  import { WorkflowObservationStore } from "./workflow-observation-store.js";
24
26
  import { buildManagedCheckpointObservation, buildManagedResumeObservation } from "./workflow-observation-managed.js";
25
27
  import { runAutoRecall } from "./auto-recall.js";
26
28
  import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
27
29
  import { runMemoryLint } from "./memory-lint.js";
30
+ import * as envConfig from "./env-config.js";
28
31
 
29
32
  const config = (loadDotEnv(), loadConfig());
30
33
  const getComponents = createComponentResolver(config);
31
34
  const checkpointStore = new SessionCheckpointStore();
32
35
  const conflictStore = new ConflictCandidateStore();
36
+ const auditLogger = createAuditLogger();
33
37
  const workflowObservationStore = new WorkflowObservationStore();
34
38
 
35
39
  // ============================================================================
@@ -235,7 +239,7 @@ async function handleStore(request: Request): Promise<Response> {
235
239
  const body = await readJson(request);
236
240
  try {
237
241
  const { store, embedder } = getComponents();
238
- const stored = await persistMemory({ store, embedder, conflictStore }, body);
242
+ const stored = await persistMemory({ store, embedder, conflictStore, auditLogger }, body);
239
243
 
240
244
  return jsonResponse({
241
245
  id: stored.id,
@@ -284,7 +288,7 @@ async function handlePattern(request: Request): Promise<Response> {
284
288
  const body = await readJson(request);
285
289
  try {
286
290
  const { store, embedder } = getComponents();
287
- const stored = await persistWorkflowPattern({ store, embedder, conflictStore }, body);
291
+ const stored = await persistWorkflowPattern({ store, embedder, conflictStore, auditLogger }, body);
288
292
  return jsonResponse({
289
293
  id: stored.id,
290
294
  stored: stored.disposition !== "conflict",
@@ -308,7 +312,7 @@ async function handleCase(request: Request): Promise<Response> {
308
312
  const body = await readJson(request);
309
313
  try {
310
314
  const { store, embedder } = getComponents();
311
- const stored = await persistCaseMemory({ store, embedder, conflictStore }, body);
315
+ const stored = await persistCaseMemory({ store, embedder, conflictStore, auditLogger }, body);
312
316
  return jsonResponse({
313
317
  id: stored.id,
314
318
  stored: stored.disposition !== "conflict",
@@ -332,7 +336,7 @@ async function handlePromote(request: Request): Promise<Response> {
332
336
  const body = await readJson(request);
333
337
  try {
334
338
  const { store, embedder } = getComponents();
335
- const stored = await promoteMemory({ store, embedder, conflictStore }, body);
339
+ const stored = await promoteMemory({ store, embedder, conflictStore, auditLogger }, body);
336
340
  return jsonResponse({
337
341
  id: stored.id,
338
342
  stored: stored.disposition !== "conflict",
@@ -455,13 +459,29 @@ async function handleCheckpoint(request: Request): Promise<Response> {
455
459
  }
456
460
  }
457
461
 
458
- /** POST /v1/workflow-observe — persist a workflow observation outside durable memory */
462
+ /** POST /v1/workflow-observe — persist a workflow observation outside durable memory; bumps skill counters when `skillId` is present */
459
463
  async function handleWorkflowObserve(request: Request): Promise<Response> {
460
464
  const body = await readJson(request);
461
465
  try {
462
466
  const record = buildWorkflowObservationRecord(body);
463
467
  const stored = await workflowObservationStore.save(record);
464
- return jsonResponse(stored, 201);
468
+
469
+ let skillOutcome: unknown = undefined;
470
+ if (record.skillId) {
471
+ try {
472
+ const { store } = getComponents();
473
+ const result = await recordSkillOutcome(store, record.skillId, record.outcome);
474
+ skillOutcome = result;
475
+ if (!result.updated) {
476
+ console.warn(`[api /v1/workflow-observe] skillId=${record.skillId} update skipped: ${result.reason}`);
477
+ }
478
+ } catch (err) {
479
+ console.warn(`[api /v1/workflow-observe] recordSkillOutcome threw for skillId=${record.skillId}:`, err);
480
+ skillOutcome = { updated: false, reason: "exception" };
481
+ }
482
+ }
483
+
484
+ return jsonResponse({ observation: stored, ...(skillOutcome !== undefined ? { skillOutcome } : {}) }, 201);
465
485
  } catch (err) {
466
486
  const message = err instanceof Error ? err.message : String(err);
467
487
  return errorResponse(400, message);
@@ -625,7 +645,7 @@ async function handleHealth(): Promise<Response> {
625
645
  // Server
626
646
  // ============================================================================
627
647
 
628
- const port = clampInt(process.env.RECALLNEST_API_PORT, 4318, 1, 65535);
648
+ const port = clampInt(envConfig.apiPortRaw(), 4318, 1, 65535);
629
649
 
630
650
  const server = Bun.serve({
631
651
  port,
package/src/audit-log.ts CHANGED
@@ -7,6 +7,7 @@
7
7
 
8
8
  import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
9
9
  import { dirname, join } from "node:path";
10
+ import * as envConfig from "./env-config.js";
10
11
 
11
12
  export type AuditOperation =
12
13
  | "store"
@@ -17,7 +18,8 @@ export type AuditOperation =
17
18
  | "supersede"
18
19
  | "consolidate"
19
20
  | "forget"
20
- | "cascade_forget";
21
+ | "cascade_forget"
22
+ | "reject";
21
23
 
22
24
  export interface AuditEntry {
23
25
  timestamp: string; // ISO 8601
@@ -59,7 +61,7 @@ function parseLines(raw: string): AuditEntry[] {
59
61
  export function createAuditLogger(logPath?: string): AuditLogger {
60
62
  const resolvedPath =
61
63
  logPath ??
62
- join(process.env.RECALLNEST_DATA_DIR || "data", "audit.jsonl");
64
+ join(envConfig.dataDir(), "audit.jsonl");
63
65
 
64
66
  // Ensure directory exists (silent on failure)
65
67
  try {
@@ -10,7 +10,8 @@
10
10
  * reason is operational hygiene.
11
11
  */
12
12
 
13
- import type { MemoryStore } from "./store.js";
13
+ import type { MemoryEntry } from "./store.js";
14
+ import type { MemoryStorePort } from "./memory-store-port.js";
14
15
  import { ConsolidationEngine, type ConsolidationConfig, type ConsolidationResult, DEFAULT_CONSOLIDATION_CONFIG } from "./consolidation-engine.js";
15
16
  import type { LLMClient } from "./llm-client.js";
16
17
  import { isLLMConsolidationEnabled, evaluateCluster, executeMergeDecisions } from "./llm-consolidation.js";
@@ -54,16 +55,20 @@ export interface AutoConsolidationResult {
54
55
  /**
55
56
  * Check conditions and run consolidation if thresholds are met.
56
57
  * Returns immediately if conditions not met (no-op).
58
+ *
59
+ * @deprecated Prefer dream-pipeline.ts for full maintenance. This module is
60
+ * kept for compatibility with callers that only need count/time-gated
61
+ * consolidation.
57
62
  */
58
63
  export async function maybeConsolidate(
59
- store: MemoryStore,
64
+ store: MemoryStorePort,
60
65
  scope: string,
61
66
  config: AutoConsolidationConfig = DEFAULT_AUTO_CONSOLIDATION_CONFIG,
62
67
  /** Tier 3.7: Optional LLM client for semantic consolidation decisions */
63
68
  llm?: LLMClient | null,
64
69
  ): Promise<AutoConsolidationResult> {
65
70
  const stats = await store.stats([scope]);
66
- const currentCount = stats.total ?? 0;
71
+ const currentCount = stats.totalCount ?? 0;
67
72
 
68
73
  // Condition 1: enough new memories since last run
69
74
  const newSinceLastRun = currentCount - lastRunMemoryCount;
package/src/auto-gc.ts CHANGED
@@ -9,16 +9,20 @@
9
9
  * instead of raw age/importance heuristics. Archive-first, delete-never.
10
10
  */
11
11
 
12
- import type { MemoryStore } from "./store.js";
12
+ import type { MemoryStorePort } from "./memory-store-port.js";
13
13
  import {
14
14
  parseEvolution,
15
15
  computeDecayScore,
16
+ computeUsageAdjustedDecayScore,
16
17
  buildArchivedMetadata,
17
18
  isActiveMemory,
18
19
  } from "./memory-evolution.js";
19
20
  import { loadRetentionPolicy, shouldArchiveByPolicy } from "./retention-policy.js";
20
21
  import type { RetentionPolicy } from "./retention-policy.js";
21
22
  import type { AuditLogger } from "./audit-log.js";
23
+ import * as envConfig from "./env-config.js";
24
+ import { isUsageSignalActive } from "./usage-tracker.js";
25
+ import { acquireLock, releaseLock, getLastDistillTime, stampLock, lockPathForKey } from "./distill-lock.js";
22
26
 
23
27
  // ---------------------------------------------------------------------------
24
28
  // Config
@@ -73,48 +77,89 @@ export interface GcResult {
73
77
  * - Not pinned (importance >= 0.95 is considered pinned)
74
78
  */
75
79
  export async function maybeRunGc(
76
- store: MemoryStore,
80
+ store: MemoryStorePort,
77
81
  config: AutoGcConfig = DEFAULT_AUTO_GC_CONFIG,
78
82
  retentionConfigDir?: string,
79
83
  auditLogger?: AuditLogger,
80
84
  ): Promise<GcResult> {
81
85
  const stats = await store.stats();
82
- const totalMemories = stats.total ?? 0;
86
+ const totalMemories = stats.totalCount ?? 0;
83
87
 
84
88
  // Condition 1: enough memories to warrant GC
85
89
  if (totalMemories < config.minMemoryCount) {
86
90
  return { triggered: false, reason: "below_memory_threshold", archivedCount: 0, totalChecked: 0 };
87
91
  }
88
92
 
89
- // Condition 2: enough time since last GC
90
- const hoursSinceLastGc = (Date.now() - lastGcTimestamp) / 3_600_000;
93
+ // Condition 2: enough time since last GC. P0-1: the throttle is cross-process now —
94
+ // take the max of this process's in-memory timestamp and the shared gc-last stamp
95
+ // file's mtime. The old module-level var alone let each of the 11 mcp-server
96
+ // processes run gc on its own independent 24h clock.
97
+ const gcStampPath = lockPathForKey("gc-last");
98
+ const lastGc = Math.max(lastGcTimestamp, getLastDistillTime({ lockPath: gcStampPath }));
99
+ const hoursSinceLastGc = (Date.now() - lastGc) / 3_600_000;
91
100
  if (hoursSinceLastGc < config.minHoursSinceLastGc) {
92
101
  return { triggered: false, reason: "too_soon", archivedCount: 0, totalChecked: 0 };
93
102
  }
94
103
 
104
+ // Concurrency guard: only one process scans the full corpus at a time. A process that
105
+ // passed the throttle in the same instant loses the O_EXCL race here and skips.
106
+ const gcRunLock = { lockPath: lockPathForKey("gc-run"), expireMs: 30 * 60_000 };
107
+ if (!acquireLock(gcRunLock)) {
108
+ return { triggered: false, reason: "locked_by_another_process", archivedCount: 0, totalChecked: 0 };
109
+ }
110
+
95
111
  // All conditions met — run GC
96
- lastGcTimestamp = Date.now();
112
+ lastGcTimestamp = Date.now(); // in-process immediate throttle
97
113
 
114
+ try {
115
+ const { archivedCount, totalChecked } = await runGcScan(store, config, retentionConfigDir, auditLogger);
116
+ // Stamp completion into the shared throttle marker (create-or-touch → mtime = now)
117
+ // so the other processes observe "last run" via getLastDistillTime.
118
+ stampLock({ lockPath: gcStampPath });
119
+ return { triggered: true, archivedCount, totalChecked };
120
+ } finally {
121
+ releaseLock(gcRunLock);
122
+ }
123
+ }
124
+
125
+ /**
126
+ * The full-corpus archive scan — the body of a triggered GC run. Extracted from
127
+ * maybeRunGc so the P0-1 lock/throttle wiring stays readable; the scan logic itself
128
+ * is unchanged.
129
+ */
130
+ async function runGcScan(
131
+ store: MemoryStorePort,
132
+ config: AutoGcConfig,
133
+ retentionConfigDir: string | undefined,
134
+ auditLogger: AuditLogger | undefined,
135
+ ): Promise<{ archivedCount: number; totalChecked: number }> {
98
136
  const now = Date.now();
99
137
  let archivedCount = 0;
100
138
  let totalChecked = 0;
101
139
 
102
- // Scan memories by listing them
103
- const entries = await store.list({ limit: 5000 });
104
- totalChecked = entries.length;
140
+ // Full-corpus scan, page by page. The previous implementation listed only
141
+ // the newest 5000 rows (list() sorts by timestamp desc before slicing) —
142
+ // old low-value memories, the very target of GC, never entered the scan
143
+ // window on a 99K+ corpus. Two passes keep memory at one page instead of
144
+ // the whole corpus; pass 2 mutates rows (archive), which may shift fragment
145
+ // scan order — a row missed this run is caught by the next one.
146
+ const GC_PAGE_SIZE = 2000;
105
147
 
106
- // Pre-compute per-scope active memory counts and cache retention policies.
107
- // This avoids repeated file reads inside the loop.
148
+ // Pass 1: per-scope active memory counts for retention policies.
108
149
  const activeCounts = new Map<string, number>();
109
150
  const policyCache = new Map<string, RetentionPolicy>();
110
151
  const scopesSeen = new Set<string>();
111
152
 
112
- for (const entry of entries) {
113
- const scope = entry.scope ?? "";
114
- scopesSeen.add(scope);
115
- if (isActiveMemory(entry.metadata)) {
116
- activeCounts.set(scope, (activeCounts.get(scope) ?? 0) + 1);
153
+ for (let offset = 0; ; offset += GC_PAGE_SIZE) {
154
+ const page = await store.listPage({ limit: GC_PAGE_SIZE, offset });
155
+ for (const entry of page) {
156
+ const scope = entry.scope ?? "";
157
+ scopesSeen.add(scope);
158
+ if (isActiveMemory(entry.metadata)) {
159
+ activeCounts.set(scope, (activeCounts.get(scope) ?? 0) + 1);
160
+ }
117
161
  }
162
+ if (page.length < GC_PAGE_SIZE) break;
118
163
  }
119
164
 
120
165
  // Batch-load retention policies for all scopes (one file read per scope)
@@ -122,8 +167,15 @@ export async function maybeRunGc(
122
167
  policyCache.set(scope, loadRetentionPolicy(scope, retentionConfigDir));
123
168
  }
124
169
 
125
- for (const entry of entries) {
126
- if (archivedCount >= config.maxArchivePerRun) break;
170
+ // Pass 2: archive sweep.
171
+ const useUsageAdjustedDecay = envConfig.usageDecay() && isUsageSignalActive();
172
+ outer:
173
+ for (let offset = 0; ; offset += GC_PAGE_SIZE) {
174
+ const page = await store.listPage({ limit: GC_PAGE_SIZE, offset });
175
+ totalChecked += page.length;
176
+
177
+ for (const entry of page) {
178
+ if (archivedCount >= config.maxArchivePerRun) break outer;
127
179
 
128
180
  // Only archive active memories
129
181
  if (!isActiveMemory(entry.metadata)) continue;
@@ -139,7 +191,9 @@ export async function maybeRunGc(
139
191
  let shouldArchive = false;
140
192
  if (ageDays >= config.minAgeDays) {
141
193
  const evo = parseEvolution(entry.metadata, entry.timestamp);
142
- let decayScore = computeDecayScore(evo, importance, now, entry.metadata);
194
+ let decayScore = useUsageAdjustedDecay
195
+ ? computeUsageAdjustedDecayScore(evo, importance, now, entry.metadata)
196
+ : computeDecayScore(evo, importance, now, entry.metadata);
143
197
  // F3: Expired memories (validUntil < now) get 2x decay acceleration
144
198
  if (evo.validUntil != null && evo.validUntil < now) {
145
199
  decayScore *= 0.5; // Halve the score → archives faster
@@ -182,12 +236,22 @@ export async function maybeRunGc(
182
236
  }
183
237
  }
184
238
 
185
- return { triggered: true, archivedCount, totalChecked };
239
+ if (page.length < GC_PAGE_SIZE) break;
240
+ }
241
+
242
+ return { archivedCount, totalChecked };
186
243
  }
187
244
 
188
245
  /**
189
- * Reset last GC timestamp (for testing).
246
+ * Reset last GC timestamp (for testing). Clears both the in-memory timestamp and the
247
+ * persisted cross-process throttle marker / run lock so tests start from a clean slate.
190
248
  */
191
249
  export function resetGcTimestamp(): void {
192
250
  lastGcTimestamp = 0;
251
+ try {
252
+ releaseLock({ lockPath: lockPathForKey("gc-last") });
253
+ releaseLock({ lockPath: lockPathForKey("gc-run") });
254
+ } catch {
255
+ /* ignore — nothing to clear */
256
+ }
193
257
  }