recallnest 2.6.1 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/CHANGELOG.md +292 -0
  2. package/README.md +280 -195
  3. package/README_CN.md +266 -199
  4. package/config.json.example +8 -1
  5. package/docs/api-reference.md +1 -1
  6. package/integrations/README.md +1 -1
  7. package/integrations/agy/README.md +73 -0
  8. package/integrations/agy/setup.sh +75 -0
  9. package/integrations/setup-all.sh +1 -1
  10. package/package.json +7 -5
  11. package/src/access-tracker.ts +10 -1
  12. package/src/activity-counter.ts +69 -11
  13. package/src/age-label.ts +29 -0
  14. package/src/audit-log.ts +35 -0
  15. package/src/auto-gc.ts +97 -30
  16. package/src/capture-engine.ts +26 -4
  17. package/src/cli.ts +224 -10
  18. package/src/consolidation-engine.ts +317 -123
  19. package/src/context-composer-stable.ts +5 -1
  20. package/src/data-checkup.ts +17 -3
  21. package/src/dream-pipeline.ts +90 -12
  22. package/src/embedder.ts +44 -5
  23. package/src/env-config.ts +71 -0
  24. package/src/forget-engine.ts +8 -2
  25. package/src/gateway-server.ts +272 -0
  26. package/src/ingest.ts +248 -22
  27. package/src/kg-extractor.ts +17 -0
  28. package/src/llm-client.ts +86 -25
  29. package/src/mcp-server.ts +2 -1
  30. package/src/mcp-tools-advanced.ts +7 -2
  31. package/src/mcp-tools-core.ts +11 -1
  32. package/src/mcp-tools-governance.ts +45 -3
  33. package/src/memory-evolution.ts +16 -0
  34. package/src/memory-output.ts +47 -8
  35. package/src/memory-promotion.ts +280 -0
  36. package/src/memory-store-port.ts +32 -3
  37. package/src/memory-utility.ts +314 -0
  38. package/src/pivot-apply.ts +52 -1
  39. package/src/pivot-evidence.ts +128 -0
  40. package/src/ppr-traversal.ts +25 -1
  41. package/src/recall-ledger.ts +76 -0
  42. package/src/retriever.ts +82 -9
  43. package/src/runtime-config.ts +9 -0
  44. package/src/scope-policy.ts +22 -1
  45. package/src/session-engine.ts +11 -4
  46. package/src/session-output.ts +13 -4
  47. package/src/session-store.ts +2 -3
  48. package/src/skill-promotion.ts +218 -1
  49. package/src/store.ts +231 -71
  50. package/src/synthesis-contract.ts +286 -0
  51. package/src/version-manager.ts +83 -0
  52. package/src/workflow-observation-schema.ts +26 -0
  53. package/integrations/gemini-cli/README.md +0 -57
  54. package/integrations/gemini-cli/setup.sh +0 -64
  55. package/src/__tests__/access-tracker-novelty.test.ts +0 -304
  56. package/src/__tests__/activity-counter.test.ts +0 -125
  57. package/src/__tests__/adaptive-fidelity.test.ts +0 -102
  58. package/src/__tests__/admission-control.test.ts +0 -206
  59. package/src/__tests__/anchor-generator.test.ts +0 -67
  60. package/src/__tests__/api-server-security.test.ts +0 -102
  61. package/src/__tests__/auto-consolidation.test.ts +0 -168
  62. package/src/__tests__/auto-recall.test.ts +0 -131
  63. package/src/__tests__/babel-integration.test.ts +0 -144
  64. package/src/__tests__/batch-dedup.test.ts +0 -65
  65. package/src/__tests__/belief-history.test.ts +0 -220
  66. package/src/__tests__/canary-eval.test.ts +0 -114
  67. package/src/__tests__/capture-engine.test.ts +0 -1328
  68. package/src/__tests__/capture-heuristic.test.ts +0 -171
  69. package/src/__tests__/cascade-forget.test.ts +0 -210
  70. package/src/__tests__/category-min-scores.test.ts +0 -116
  71. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  72. package/src/__tests__/circuit-breaker.test.ts +0 -99
  73. package/src/__tests__/cluster-consolidation.test.ts +0 -945
  74. package/src/__tests__/confidence-metadata.test.ts +0 -125
  75. package/src/__tests__/confidence-tracker.test.ts +0 -261
  76. package/src/__tests__/conflict-advisor.test.ts +0 -206
  77. package/src/__tests__/conflict-escalation.test.ts +0 -149
  78. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  79. package/src/__tests__/conflict-output.test.ts +0 -85
  80. package/src/__tests__/conflict-store.test.ts +0 -418
  81. package/src/__tests__/connector-v1.test.ts +0 -181
  82. package/src/__tests__/consolidation-engine.test.ts +0 -430
  83. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  84. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  85. package/src/__tests__/context-composer-pins.test.ts +0 -64
  86. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  87. package/src/__tests__/context-reconstructor.test.ts +0 -65
  88. package/src/__tests__/context-renderer.test.ts +0 -91
  89. package/src/__tests__/continuity-eval.test.ts +0 -275
  90. package/src/__tests__/continuity-schema.test.ts +0 -97
  91. package/src/__tests__/contracts.test.ts +0 -262
  92. package/src/__tests__/conversation-importer.test.ts +0 -692
  93. package/src/__tests__/core-recall-eval.test.ts +0 -235
  94. package/src/__tests__/core-summary.test.ts +0 -109
  95. package/src/__tests__/data-checkup.test.ts +0 -276
  96. package/src/__tests__/decay-exemption.test.ts +0 -129
  97. package/src/__tests__/dedup-actions.test.ts +0 -58
  98. package/src/__tests__/dedup-guard.test.ts +0 -39
  99. package/src/__tests__/deterministic-id.test.ts +0 -39
  100. package/src/__tests__/distill-lock.test.ts +0 -305
  101. package/src/__tests__/distill-session-argshape.test.ts +0 -79
  102. package/src/__tests__/doctor.test.ts +0 -319
  103. package/src/__tests__/dream-pipeline.test.ts +0 -780
  104. package/src/__tests__/drill-down.test.ts +0 -108
  105. package/src/__tests__/embedder.test.ts +0 -122
  106. package/src/__tests__/emotion-decay.test.ts +0 -46
  107. package/src/__tests__/emotion-detector.test.ts +0 -56
  108. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  109. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  110. package/src/__tests__/entity-resolver.test.ts +0 -58
  111. package/src/__tests__/env-config.test.ts +0 -151
  112. package/src/__tests__/error-signature-boost.test.ts +0 -87
  113. package/src/__tests__/error-signature.test.ts +0 -95
  114. package/src/__tests__/error-taxonomy.test.ts +0 -129
  115. package/src/__tests__/ethics-layer.test.ts +0 -476
  116. package/src/__tests__/event-segmenter.test.ts +0 -153
  117. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  118. package/src/__tests__/frequency-tracker.test.ts +0 -113
  119. package/src/__tests__/freshness.test.ts +0 -186
  120. package/src/__tests__/graph-export.test.ts +0 -489
  121. package/src/__tests__/hotness-blend.test.ts +0 -283
  122. package/src/__tests__/ingest-chunking.test.ts +0 -44
  123. package/src/__tests__/ingest.test.ts +0 -360
  124. package/src/__tests__/interference-detector.test.ts +0 -203
  125. package/src/__tests__/kg-extractor.test.ts +0 -192
  126. package/src/__tests__/kg-store.test.ts +0 -300
  127. package/src/__tests__/llm-client-pivot.test.ts +0 -628
  128. package/src/__tests__/llm-consolidation.test.ts +0 -226
  129. package/src/__tests__/maintenance-locks.test.ts +0 -189
  130. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  131. package/src/__tests__/mcp-related-scopes.test.ts +0 -151
  132. package/src/__tests__/memory-boundaries.test.ts +0 -282
  133. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -345
  134. package/src/__tests__/memory-evolution.test.ts +0 -271
  135. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  136. package/src/__tests__/memory-lint.test.ts +0 -421
  137. package/src/__tests__/memory-output.test.ts +0 -215
  138. package/src/__tests__/memory-promotion.test.ts +0 -315
  139. package/src/__tests__/migrate-dedup-ids.test.ts +0 -65
  140. package/src/__tests__/multi-vector.test.ts +0 -242
  141. package/src/__tests__/narrative-architecture.test.ts +0 -456
  142. package/src/__tests__/noise-filter.test.ts +0 -205
  143. package/src/__tests__/noise-prototype-bank.test.ts +0 -280
  144. package/src/__tests__/obsidian-connector.test.ts +0 -363
  145. package/src/__tests__/omitted-reporting.test.ts +0 -98
  146. package/src/__tests__/pending-review.test.ts +0 -77
  147. package/src/__tests__/pii-redact.test.ts +0 -333
  148. package/src/__tests__/pivot-apply.test.ts +0 -373
  149. package/src/__tests__/pivot-distill-supervisor.test.ts +0 -457
  150. package/src/__tests__/pivot-distill.test.ts +0 -1178
  151. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  152. package/src/__tests__/ppr-traversal.test.ts +0 -256
  153. package/src/__tests__/prediction-engine.test.ts +0 -349
  154. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  155. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  156. package/src/__tests__/preference-matcher.test.ts +0 -282
  157. package/src/__tests__/preference-slots.test.ts +0 -205
  158. package/src/__tests__/prospective-memory.test.ts +0 -283
  159. package/src/__tests__/pull-from-macbook-sync-only.test.ts +0 -679
  160. package/src/__tests__/query-entity-detector.test.ts +0 -171
  161. package/src/__tests__/query-expander-alias.test.ts +0 -146
  162. package/src/__tests__/query-expander.test.ts +0 -34
  163. package/src/__tests__/recall-gate.test.ts +0 -266
  164. package/src/__tests__/recall-governor.test.ts +0 -227
  165. package/src/__tests__/recall-mode.test.ts +0 -64
  166. package/src/__tests__/resolve-db-path.test.ts +0 -130
  167. package/src/__tests__/resolve-tier.test.ts +0 -64
  168. package/src/__tests__/result-synthesizer.test.ts +0 -198
  169. package/src/__tests__/resume-context.test.ts +0 -4898
  170. package/src/__tests__/retrieval-eval.test.ts +0 -163
  171. package/src/__tests__/retrieval-trace.test.ts +0 -213
  172. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  173. package/src/__tests__/retriever-lazy-init-dispatch.test.ts +0 -123
  174. package/src/__tests__/retroactive-boost.test.ts +0 -223
  175. package/src/__tests__/salience-scoring.test.ts +0 -106
  176. package/src/__tests__/scope-inventory.test.ts +0 -200
  177. package/src/__tests__/scope-policy.test.ts +0 -69
  178. package/src/__tests__/scope-suggester.test.ts +0 -80
  179. package/src/__tests__/session-checkpoint.test.ts +0 -291
  180. package/src/__tests__/session-distiller.test.ts +0 -640
  181. package/src/__tests__/session-gc.test.ts +0 -215
  182. package/src/__tests__/skill-engine.test.ts +0 -552
  183. package/src/__tests__/skill-promotion.test.ts +0 -654
  184. package/src/__tests__/skill-verifier.test.ts +0 -122
  185. package/src/__tests__/source-heartbeat.test.ts +0 -250
  186. package/src/__tests__/stable-context-length-contract.test.ts +0 -62
  187. package/src/__tests__/store-patch-metadata.test.ts +0 -205
  188. package/src/__tests__/store-read-consistency.test.ts +0 -122
  189. package/src/__tests__/store-schema-migration.test.ts +0 -87
  190. package/src/__tests__/store-scope-audit.test.ts +0 -81
  191. package/src/__tests__/store-upsert.test.ts +0 -201
  192. package/src/__tests__/supersede-chain.test.ts +0 -154
  193. package/src/__tests__/temporal-parser.test.ts +0 -241
  194. package/src/__tests__/temporal-validity.test.ts +0 -160
  195. package/src/__tests__/term-registry-terminal-roster.test.ts +0 -127
  196. package/src/__tests__/term-registry.test.ts +0 -121
  197. package/src/__tests__/tool-error-surfacing.test.ts +0 -98
  198. package/src/__tests__/tool-output-compressor.test.ts +0 -160
  199. package/src/__tests__/topic-tag.test.ts +0 -113
  200. package/src/__tests__/trusted-memory.test.ts +0 -839
  201. package/src/__tests__/ui-server-security.test.ts +0 -93
  202. package/src/__tests__/usage-tracker.test.ts +0 -287
  203. package/src/__tests__/version-manager.test.ts +0 -213
  204. package/src/__tests__/workflow-observation-skill-outcome.test.ts +0 -352
  205. package/src/__tests__/workflow-observation.test.ts +0 -433
  206. package/src/__tests__/write-verifier.test.ts +0 -122
  207. /package/integrations/{gemini-cli/gemini-md-snippet.md → agy/agy-md-snippet.md} +0 -0
@@ -0,0 +1,73 @@
1
+ # Antigravity CLI (agy) Integration
2
+
3
+ > agy 接入指南:一键配 MCP + continuity 规则,让 agy 在新窗口里主动恢复稳定上下文。
4
+
5
+ > **Note on naming.** Google retired the standalone `gemini` CLI for personal
6
+ > accounts on 2026-06-18; the [Antigravity CLI](https://antigravity.google)
7
+ > (`agy`) is its successor. This integration targets `agy`. If you are on a
8
+ > Code Assist Standard/Enterprise plan and still run the legacy CLI, point the
9
+ > same MCP entry at whatever config file your build reads.
10
+
11
+ ## Quick Start
12
+
13
+ ```bash
14
+ bash integrations/agy/setup.sh
15
+ ```
16
+
17
+ ## What It Does
18
+
19
+ - Adds RecallNest as an MCP server in `~/.gemini/config/mcp_config.json`
20
+ - Installs a managed RecallNest block in `~/.gemini/config/GEMINI.md`
21
+
22
+ ## Continuity Rules
23
+
24
+ The managed block comes from [agy-md-snippet.md](agy-md-snippet.md) and tells agy to:
25
+
26
+ - call `resume_context` at the start of fresh windows or continuity-sensitive tasks
27
+ - run lightweight `search_memory` on task pivots inside the same project before repo exploration drifts
28
+ - reuse known `scope` / `sessionId` and the resolved scope returned by `resume_context` in follow-up recall calls
29
+ - treat recalled or startup-hook repo state as unverified until this window explicitly checks the repo
30
+ - save `checkpoint_session` before leaving resumable work
31
+ - do not inspect repo state just to enrich a close-window checkpoint unless the user explicitly asked for repo state
32
+ - do not write unverified repo-state claims into `checkpoint_session`
33
+ - capture durable facts with `store_memory` and reusable workflows with `store_workflow_pattern`
34
+
35
+ ## Shared Index
36
+
37
+ agy shares the same LanceDB index as Claude Code, Codex and Kimi. Memories ingested from any source are searchable by all of them.
38
+
39
+ ## Manual Setup
40
+
41
+ Add to `~/.gemini/config/mcp_config.json`:
42
+
43
+ ```json
44
+ {
45
+ "mcpServers": {
46
+ "recallnest": {
47
+ "type": "stdio",
48
+ "command": "bun",
49
+ "args": ["run", "RECALLNEST_PATH/src/mcp-server.ts"]
50
+ }
51
+ }
52
+ }
53
+ ```
54
+
55
+ Replace `RECALLNEST_PATH` with your actual path.
56
+
57
+ Then copy [agy-md-snippet.md](agy-md-snippet.md) into `~/.gemini/config/GEMINI.md` if you are not using `setup.sh`.
58
+
59
+ ## Tool Permissions
60
+
61
+ agy gates tool calls separately from MCP registration. In non-interactive mode
62
+ (`agy -p`) it cannot prompt, so permission-requiring tools are auto-denied and
63
+ the agent silently returns nothing. If RecallNest tools appear registered but
64
+ never fire, set `toolPermission` in `~/.gemini/antigravity-cli/settings.json`,
65
+ or pass `--dangerously-skip-permissions` for a single run.
66
+
67
+ ## Verify
68
+
69
+ ```bash
70
+ agy -p "resume my context for RecallNest continuity work" --print-timeout 180s </dev/null
71
+ ```
72
+
73
+ If `resume_context` or `search_memory` is called, you're set.
@@ -0,0 +1,75 @@
1
+ #!/usr/bin/env bash
2
+ # RecallNest — Antigravity CLI (agy) MCP setup (idempotent)
3
+ # Usage: bash integrations/agy/setup.sh
4
+ #
5
+ # Google retired the standalone Gemini CLI for personal accounts on 2026-06-18.
6
+ # Its successor is the Antigravity CLI (`agy`), which reads MCP servers from
7
+ # ~/.gemini/config/mcp_config.json — not the old ~/.gemini/settings.json.
8
+
9
+ set -euo pipefail
10
+
11
+ AGY_CONFIG_DIR="$HOME/.gemini/config"
12
+ AGY_MCP_CONFIG="$AGY_CONFIG_DIR/mcp_config.json"
13
+ AGY_MD="$AGY_CONFIG_DIR/GEMINI.md"
14
+ RECALLNEST_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
15
+ MCP_ENTRY="$RECALLNEST_DIR/src/mcp-server.ts"
16
+ RULES_SNIPPET="$RECALLNEST_DIR/integrations/agy/agy-md-snippet.md"
17
+ RULES_HELPER="$RECALLNEST_DIR/integrations/shared/install-managed-block.sh"
18
+
19
+ if [[ ! -f "$MCP_ENTRY" ]]; then
20
+ echo "ERROR: $MCP_ENTRY not found. Run this from the recallnest repo root."
21
+ exit 1
22
+ fi
23
+
24
+ if [[ ! -f "$RULES_HELPER" ]]; then
25
+ echo "ERROR: $RULES_HELPER not found."
26
+ exit 1
27
+ fi
28
+
29
+ if ! command -v agy >/dev/null 2>&1; then
30
+ echo "WARNING: 'agy' not found on PATH. Install the Antigravity CLI first:"
31
+ echo " https://antigravity.google"
32
+ echo "Continuing anyway — the config will be written for when agy is installed."
33
+ echo ""
34
+ fi
35
+
36
+ mkdir -p "$AGY_CONFIG_DIR"
37
+ if [[ ! -f "$AGY_MCP_CONFIG" ]]; then
38
+ echo '{"mcpServers":{}}' > "$AGY_MCP_CONFIG"
39
+ echo "Created $AGY_MCP_CONFIG"
40
+ fi
41
+
42
+ # Check if already configured
43
+ if bun -e "
44
+ const c = JSON.parse(require('fs').readFileSync('$AGY_MCP_CONFIG','utf8'));
45
+ process.exit(c.mcpServers?.recallnest ? 0 : 1);
46
+ " 2>/dev/null; then
47
+ echo "RecallNest MCP already configured in $AGY_MCP_CONFIG — skipping."
48
+ else
49
+ bun -e "
50
+ const fs = require('fs');
51
+ const c = JSON.parse(fs.readFileSync('$AGY_MCP_CONFIG','utf8'));
52
+ if (!c.mcpServers) c.mcpServers = {};
53
+ c.mcpServers.recallnest = {
54
+ type: 'stdio',
55
+ command: 'bun',
56
+ args: ['run', '$MCP_ENTRY']
57
+ };
58
+ fs.writeFileSync('$AGY_MCP_CONFIG', JSON.stringify(c, null, 2) + '\n');
59
+ "
60
+ echo "Added RecallNest MCP to $AGY_MCP_CONFIG"
61
+ fi
62
+
63
+ . "$RULES_HELPER"
64
+ install_managed_markdown_block "$RULES_SNIPPET" "$AGY_MD" "recallnest-continuity"
65
+ echo "Installed RecallNest continuity rules in $AGY_MD"
66
+
67
+ echo ""
68
+ echo "Setup complete. Restart agy to activate, then verify with:"
69
+ echo " agy -p 'List your available MCP tools' --print-timeout 180s </dev/null"
70
+ echo ""
71
+ echo "Continuity baseline seed:"
72
+ echo " (cd \"$RECALLNEST_DIR\" && bun run seed:continuity)"
73
+ echo ""
74
+ echo "Managed snippet source:"
75
+ echo " $RULES_SNIPPET"
@@ -19,7 +19,7 @@ bash "$SCRIPT_DIR/codex/setup.sh"
19
19
  echo ""
20
20
 
21
21
  echo "--- Gemini CLI ---"
22
- bash "$SCRIPT_DIR/gemini-cli/setup.sh"
22
+ bash "$SCRIPT_DIR/agy/setup.sh"
23
23
  echo ""
24
24
 
25
25
  echo "======================================"
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "recallnest",
3
- "version": "2.6.1",
3
+ "version": "3.0.1",
4
4
  "description": "RecallNest — MCP-native, local-first memory search and distillation for AI conversations",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -22,7 +22,8 @@
22
22
  "eval/continuity/*.json",
23
23
  "integrations/",
24
24
  "README_CN.md",
25
- "src/"
25
+ "src/",
26
+ "!src/**/__tests__/**"
26
27
  ],
27
28
  "scripts": {
28
29
  "search": "bun run src/cli.ts search",
@@ -54,17 +55,18 @@
54
55
  "seed:continuity": "bun run seed:patterns && bun run seed:cases && bun run seed:memories",
55
56
  "smoke:claude-continuity": "bash integrations/claude-code/smoke-continuity.sh",
56
57
  "ui": "bun run src/ui-server.ts",
57
- "api": "bun run src/api-server.ts"
58
+ "api": "bun run src/api-server.ts",
59
+ "gateway": "bun run src/gateway-server.ts"
58
60
  },
59
61
  "engines": {
60
- "node": ">=18.14.1"
62
+ "node": ">=22"
61
63
  },
62
64
  "dependencies": {
63
65
  "@hono/node-server": "1.19.17",
64
66
  "@lancedb/lancedb": "^0.26.0",
65
67
  "@modelcontextprotocol/sdk": "^1.30.0",
66
68
  "commander": "^12.0.0",
67
- "openai": "^4.0.0",
69
+ "openai": "^7.5.0",
68
70
  "tsx": "^4.19.0",
69
71
  "zod": "^3.25.76"
70
72
  },
@@ -11,6 +11,7 @@
11
11
 
12
12
  import type { MemoryStore } from "./store.js";
13
13
  import { evaluateTierChange, resolveTier, type MemoryTier } from "./decay-engine.js";
14
+ import { PROCESS_READER_ID, recordRecallHits } from "./recall-ledger.js";
14
15
  import { logInfo, logWarn } from "./stderr-log.js";
15
16
 
16
17
  // ============================================================================
@@ -70,7 +71,10 @@ export class AccessTracker {
70
71
  private config: AccessTrackerConfig = DEFAULT_ACCESS_TRACKER_CONFIG,
71
72
  readerId?: string,
72
73
  ) {
73
- this.readerId = readerId ?? `r-${crypto.randomUUID().slice(0, 8)}`;
74
+ // 默认取进程级身份,不是每个实例各随机一个:components 按 profile 缓存,
75
+ // 一个会话开两个 profile 就会伪造出两个 reader,既污染 distinctReaderCount
76
+ // 也让 workflow observation 的 readerId join 对不上(见 recall-ledger.ts 抬头)。
77
+ this.readerId = readerId ?? PROCESS_READER_ID;
74
78
  }
75
79
 
76
80
  /**
@@ -137,6 +141,11 @@ export class AccessTracker {
137
141
 
138
142
  const now = Date.now();
139
143
 
144
+ // P0 join key:先记账本再过闸。账本回答「agent 看到过哪些条目」,
145
+ // 下面的 novelty/cooldown 闸回答「哪些值得强化半衰期」——两个问题,
146
+ // 用后者过滤前者会把最常被读到的条目系统性排除在 join 之外。
147
+ recordRecallHits(ids, now);
148
+
140
149
  for (let i = 0; i < ids.length; i++) {
141
150
  const id = ids[i];
142
151
  const score = scores?.[i];
@@ -2,6 +2,16 @@
2
2
  * HP-3: Activity-driven distill frequency.
3
3
  * Tracks write operations since last distill, exposes tier-based thresholds.
4
4
  * Complements CC-6 distill-lock: HP-3 decides "when to trigger", CC-6 decides "whether to allow".
5
+ *
6
+ * 2026-08-14 并发加固(dream 写爆库根治的一部分):
7
+ * - 写路径(increment/reset/resetBatch/prune)全部 async 化,进 `activity-stats` 跨进程锁 ——
8
+ * 此前 increment(11 个 mcp-server + ingest)与 reset(dream 04:00)裸读改写同一文件,
9
+ * 互踩窗口内的计数会静默丢失。
10
+ * - writeStats 改临时文件 + 原子 rename:裸 writeFileSync 期间被并发读到半截 JSON 时,
11
+ * readStats 的 catch 会返回空表 —— 一次损坏放大成"整个队列清零"。原子替换后读方
12
+ * 永远看到完整文件(旧版或新版)。
13
+ * - 读路径(getWriteCount/listScopesAboveThreshold/getDistillTier)保持同步无锁:原子
14
+ * rename 保证读到的文件完整;读到旧一拍的计数对 best-effort 语义无害。
5
15
  */
6
16
 
7
17
  import {
@@ -9,9 +19,11 @@ import {
9
19
  readFileSync,
10
20
  writeFileSync,
11
21
  mkdirSync,
22
+ renameSync,
12
23
  } from "node:fs";
13
24
  import { join, dirname } from "node:path";
14
25
  import * as envConfig from "./env-config.js";
26
+ import { withLock } from "./distill-lock.js";
15
27
 
16
28
  // ---------------------------------------------------------------------------
17
29
  // Config
@@ -80,7 +92,22 @@ function readStats(statsPath: string): ActivityStats {
80
92
 
81
93
  function writeStats(statsPath: string, stats: ActivityStats): void {
82
94
  mkdirSync(dirname(statsPath), { recursive: true });
83
- writeFileSync(statsPath, JSON.stringify(stats, null, 2), "utf-8");
95
+ // 原子替换:先写同目录临时文件再 rename。并发读方要么看到旧版要么看到新版,
96
+ // 不会读到半截 JSON(那会被 readStats 的 catch 吞成空表,把整个队列清零)。
97
+ const tmpPath = `${statsPath}.tmp-${process.pid}`;
98
+ writeFileSync(tmpPath, JSON.stringify(stats, null, 2), "utf-8");
99
+ renameSync(tmpPath, statsPath);
100
+ }
101
+
102
+ /** 写路径共用的锁参数:等待而非跳过(丢计数=丢 dream 触发信号),短超时防卡死写入方。 */
103
+ const ACTIVITY_LOCK_KEY = "activity-stats";
104
+ const ACTIVITY_LOCK_OPTS = { onBusy: "wait" as const, waitTimeoutMs: 2_000, expireMs: 30_000 };
105
+
106
+ async function withStatsLock<T>(fn: () => T): Promise<T> {
107
+ const outcome = await withLock(ACTIVITY_LOCK_KEY, async () => fn(), ACTIVITY_LOCK_OPTS);
108
+ // onBusy:"wait" resolves ran:true or throws on timeout; ran:false unreachable.
109
+ if (!outcome.ran) throw new Error("activity-stats lock unavailable");
110
+ return outcome.result;
84
111
  }
85
112
 
86
113
  // ---------------------------------------------------------------------------
@@ -88,16 +115,18 @@ function writeStats(statsPath: string, stats: ActivityStats): void {
88
115
  // ---------------------------------------------------------------------------
89
116
 
90
117
  /** Increment a scope's write counter by n (default 1). Returns the scope's new count. */
91
- export function incrementWriteCount(
118
+ export async function incrementWriteCount(
92
119
  scope: string,
93
120
  n = 1,
94
121
  cfg?: Partial<ActivityCounterConfig>,
95
- ): number {
122
+ ): Promise<number> {
96
123
  const { statsPath } = resolveConfig(cfg);
97
- const stats = readStats(statsPath);
98
- stats.scopes[scope] = (stats.scopes[scope] ?? 0) + n;
99
- writeStats(statsPath, stats);
100
- return stats.scopes[scope];
124
+ return withStatsLock(() => {
125
+ const stats = readStats(statsPath);
126
+ stats.scopes[scope] = (stats.scopes[scope] ?? 0) + n;
127
+ writeStats(statsPath, stats);
128
+ return stats.scopes[scope];
129
+ });
101
130
  }
102
131
 
103
132
  /** Read a scope's current write count without modifying (0 if never written). */
@@ -107,11 +136,40 @@ export function getWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig
107
136
  }
108
137
 
109
138
  /** Reset one scope's counter after its successful dream/distill (leaves other scopes untouched). */
110
- export function resetWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig>): void {
139
+ export async function resetWriteCount(scope: string, cfg?: Partial<ActivityCounterConfig>): Promise<void> {
111
140
  const { statsPath } = resolveConfig(cfg);
112
- const stats = readStats(statsPath);
113
- delete stats.scopes[scope];
114
- writeStats(statsPath, stats);
141
+ await withStatsLock(() => {
142
+ const stats = readStats(statsPath);
143
+ delete stats.scopes[scope];
144
+ writeStats(statsPath, stats);
145
+ });
146
+ }
147
+
148
+ /**
149
+ * Remove every scope entry matching `predicate` in one read-modify-write (one lock
150
+ * acquisition, one atomic file replace). Returns the removed scope names.
151
+ *
152
+ * dream --auto 用它把 transcript scope(原文层,不参与巩固——shared-behaviors §5.3)
153
+ * 从计数文件里整体清扫,含未达标条目:listScopesAboveThreshold 只能看到达标的,
154
+ * 而未达标的 transcript 条目会让 stats 文件无限膨胀(历史会话只增不减)。
155
+ */
156
+ export async function pruneWriteCounts(
157
+ predicate: (scope: string) => boolean,
158
+ cfg?: Partial<ActivityCounterConfig>,
159
+ ): Promise<string[]> {
160
+ const { statsPath } = resolveConfig(cfg);
161
+ return withStatsLock(() => {
162
+ const stats = readStats(statsPath);
163
+ const removed: string[] = [];
164
+ for (const scope of Object.keys(stats.scopes)) {
165
+ if (predicate(scope)) {
166
+ delete stats.scopes[scope];
167
+ removed.push(scope);
168
+ }
169
+ }
170
+ if (removed.length > 0) writeStats(statsPath, stats);
171
+ return removed;
172
+ });
115
173
  }
116
174
 
117
175
  /** Scopes whose write count is at or above `threshold` — the dream scheduler's work list. */
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Relative age labels for rendered output.
3
+ *
4
+ * Why: a model asked to do date arithmetic on `2026-08-03` will sometimes get it
5
+ * wrong, and "how long ago" is exactly what decides whether a memory is still
6
+ * worth raising. So every rendered date carries a whole-day age next to it and
7
+ * nobody downstream has to subtract. (Borrowed from HKUDS/DeepTutor
8
+ * `memory/recall.py` — "days_ago rather than an ISO timestamp", 2026-08-17.)
9
+ *
10
+ * A future stamp (clock skew on an imported record) reads as today, not as a
11
+ * negative age. Unusable stamps render as "-" instead of a fabricated number.
12
+ */
13
+
14
+ const DAY_MS = 86_400_000;
15
+
16
+ export const AGE_UNKNOWN = "-";
17
+
18
+ export function formatAgeLabel(timestampMs: number | null | undefined, now: number = Date.now()): string {
19
+ if (!timestampMs || !Number.isFinite(timestampMs)) return AGE_UNKNOWN;
20
+ const days = Math.floor((now - timestampMs) / DAY_MS);
21
+ return `${Math.max(0, days)}d`;
22
+ }
23
+
24
+ export function formatIsoAgeLabel(iso: string | null | undefined, now: number = Date.now()): string {
25
+ if (!iso) return AGE_UNKNOWN;
26
+ const parsed = Date.parse(iso);
27
+ if (Number.isNaN(parsed)) return AGE_UNKNOWN;
28
+ return formatAgeLabel(parsed, now);
29
+ }
package/src/audit-log.ts CHANGED
@@ -21,6 +21,31 @@ export type AuditOperation =
21
21
  | "cascade_forget"
22
22
  | "reject";
23
23
 
24
+ /**
25
+ * Which revision of a memory a retrieval actually served.
26
+ *
27
+ * A `retrieve` row used to record only the query and a hit count, which cannot answer the
28
+ * question the audit log exists for: when an agent acted on recalled memory, *what did it
29
+ * read*? Memories are versioned in place — `archiveBeliefVersion` keeps the canonical id
30
+ * and bumps `evolution.version`, so "memory X was retrieved" is ambiguous across a belief
31
+ * change. Recording the revision alongside the id makes a past answer reconstructable.
32
+ *
33
+ * Field names are short because retrieval is by far the highest-frequency audited
34
+ * operation and `audit.jsonl` has no rotation.
35
+ */
36
+ export interface RetrievedRevision {
37
+ /** First 8 chars of the memory id — enough to join against the store. */
38
+ id: string;
39
+ /** `evolution.version` at the moment it was served. */
40
+ rev: number;
41
+ /** `evolution.status`: active / superseded / archived / consolidated / pending_review. */
42
+ st: string;
43
+ /** `boundary.layer` — durable vs evidence decides how much authority the hit carried. */
44
+ ly?: string;
45
+ /** `boundary.authority` — where the content originally came from. */
46
+ au?: string;
47
+ }
48
+
24
49
  export interface AuditEntry {
25
50
  timestamp: string; // ISO 8601
26
51
  operation: AuditOperation;
@@ -28,8 +53,18 @@ export interface AuditEntry {
28
53
  memoryId?: string;
29
54
  actor: string; // "manual" | "agent" | "api" | "system"
30
55
  details?: string; // brief context (<=200 chars)
56
+ /** Revision + provenance of each served result. Capped — see `retrievedTotal`. */
57
+ retrieved?: RetrievedRevision[];
58
+ /**
59
+ * Total results served. Present only when it exceeds `retrieved.length`, so a truncated
60
+ * list is always visibly truncated rather than silently short.
61
+ */
62
+ retrievedTotal?: number;
31
63
  }
32
64
 
65
+ /** Upper bound on `retrieved` entries per row, keeping line size bounded. */
66
+ export const AUDIT_RETRIEVED_CAP = 10;
67
+
33
68
  export interface AuditLogger {
34
69
  log(entry: Omit<AuditEntry, "timestamp">): void;
35
70
  /** Read recent entries (newest first) */
package/src/auto-gc.ts CHANGED
@@ -14,7 +14,7 @@ import {
14
14
  parseEvolution,
15
15
  computeDecayScore,
16
16
  computeUsageAdjustedDecayScore,
17
- buildArchivedMetadata,
17
+ patchEvolutionOnMeta,
18
18
  isActiveMemory,
19
19
  } from "./memory-evolution.js";
20
20
  import { loadRetentionPolicy, shouldArchiveByPolicy } from "./retention-policy.js";
@@ -23,6 +23,7 @@ import type { AuditLogger } from "./audit-log.js";
23
23
  import * as envConfig from "./env-config.js";
24
24
  import { isUsageSignalActive } from "./usage-tracker.js";
25
25
  import { acquireLock, releaseLock, getLastDistillTime, stampLock, lockPathForKey } from "./distill-lock.js";
26
+ import { logInfo } from "./stderr-log.js";
26
27
 
27
28
  // ---------------------------------------------------------------------------
28
29
  // Config
@@ -54,6 +55,7 @@ export const DEFAULT_AUTO_GC_CONFIG: AutoGcConfig = {
54
55
  // ---------------------------------------------------------------------------
55
56
 
56
57
  let lastGcTimestamp = 0;
58
+ let lastVersionGroupRepairTimestamp = 0;
57
59
 
58
60
  // ---------------------------------------------------------------------------
59
61
  // Core
@@ -64,6 +66,7 @@ export interface GcResult {
64
66
  reason?: string;
65
67
  archivedCount: number;
66
68
  totalChecked: number;
69
+ dissolvedVersionGroups: number;
67
70
  }
68
71
 
69
72
  /**
@@ -85,38 +88,70 @@ export async function maybeRunGc(
85
88
  const stats = await store.stats();
86
89
  const totalMemories = stats.totalCount ?? 0;
87
90
 
88
- // Condition 1: enough memories to warrant GC
89
- if (totalMemories < config.minMemoryCount) {
90
- return { triggered: false, reason: "below_memory_threshold", archivedCount: 0, totalChecked: 0 };
91
- }
91
+ // Singleton version-group repair has its own persistent throttle. It must not be
92
+ // hidden behind minMemoryCount: a small store can lose one member of a pair too.
93
+ // Reuse the configured maintenance interval, but keep a separate timestamp so a
94
+ // repair-only pass does not postpone a later archive pass (or vice versa).
95
+ const repairStampPath = lockPathForKey("version-group-repair-last");
96
+ const lastRepair = Math.max(
97
+ lastVersionGroupRepairTimestamp,
98
+ getLastDistillTime({ lockPath: repairStampPath }),
99
+ );
100
+ const repairDue = totalMemories > 0
101
+ && (Date.now() - lastRepair) / 3_600_000 >= config.minHoursSinceLastGc;
92
102
 
93
- // Condition 2: enough time since last GC. P0-1: the throttle is cross-process now —
94
- // take the max of this process's in-memory timestamp and the shared gc-last stamp
95
- // file's mtime. The old module-level var alone let each of the 11 mcp-server
96
- // processes run gc on its own independent 24h clock.
103
+ // Archive GC retains its existing count and time gates. P0-1: the time throttle
104
+ // is cross-process, using the shared gc-last marker in addition to process state.
97
105
  const gcStampPath = lockPathForKey("gc-last");
98
106
  const lastGc = Math.max(lastGcTimestamp, getLastDistillTime({ lockPath: gcStampPath }));
99
107
  const hoursSinceLastGc = (Date.now() - lastGc) / 3_600_000;
100
- if (hoursSinceLastGc < config.minHoursSinceLastGc) {
101
- return { triggered: false, reason: "too_soon", archivedCount: 0, totalChecked: 0 };
108
+ const archiveSkipReason = totalMemories < config.minMemoryCount
109
+ ? "below_memory_threshold"
110
+ : hoursSinceLastGc < config.minHoursSinceLastGc
111
+ ? "too_soon"
112
+ : undefined;
113
+
114
+ // Nothing is due: preserve the old fast no-op path without taking a run lock.
115
+ if (!repairDue && archiveSkipReason) {
116
+ return { triggered: false, reason: archiveSkipReason, archivedCount: 0, totalChecked: 0, dissolvedVersionGroups: 0 };
102
117
  }
103
118
 
104
119
  // Concurrency guard: only one process scans the full corpus at a time. A process that
105
120
  // passed the throttle in the same instant loses the O_EXCL race here and skips.
106
121
  const gcRunLock = { lockPath: lockPathForKey("gc-run"), expireMs: 30 * 60_000 };
107
122
  if (!acquireLock(gcRunLock)) {
108
- return { triggered: false, reason: "locked_by_another_process", archivedCount: 0, totalChecked: 0 };
123
+ return { triggered: false, reason: "locked_by_another_process", archivedCount: 0, totalChecked: 0, dissolvedVersionGroups: 0 };
109
124
  }
110
125
 
111
- // All conditions met — run GC
112
- lastGcTimestamp = Date.now(); // in-process immediate throttle
113
-
114
126
  try {
127
+ let dissolvedVersionGroups = 0;
128
+ if (repairDue) {
129
+ dissolvedVersionGroups = await store.repairSingletonVersionGroups();
130
+ lastVersionGroupRepairTimestamp = Date.now();
131
+ stampLock({ lockPath: repairStampPath });
132
+ if (dissolvedVersionGroups > 0) {
133
+ logInfo(`[INFO] Auto-GC dissolved ${dissolvedVersionGroups} singleton version group(s)`);
134
+ }
135
+ }
136
+
137
+ // A repair-only pass is still useful maintenance, but `triggered` continues to
138
+ // mean that the archive scan ran so existing callers keep their old semantics.
139
+ if (archiveSkipReason) {
140
+ return {
141
+ triggered: false,
142
+ reason: archiveSkipReason,
143
+ archivedCount: 0,
144
+ totalChecked: 0,
145
+ dissolvedVersionGroups,
146
+ };
147
+ }
148
+
149
+ lastGcTimestamp = Date.now(); // in-process immediate throttle
115
150
  const { archivedCount, totalChecked } = await runGcScan(store, config, retentionConfigDir, auditLogger);
116
151
  // Stamp completion into the shared throttle marker (create-or-touch → mtime = now)
117
152
  // so the other processes observe "last run" via getLastDistillTime.
118
153
  stampLock({ lockPath: gcStampPath });
119
- return { triggered: true, archivedCount, totalChecked };
154
+ return { triggered: true, archivedCount, totalChecked, dissolvedVersionGroups };
120
155
  } finally {
121
156
  releaseLock(gcRunLock);
122
157
  }
@@ -169,13 +204,47 @@ async function runGcScan(
169
204
 
170
205
  // Pass 2: archive sweep.
171
206
  const useUsageAdjustedDecay = envConfig.usageDecay() && isUsageSignalActive();
207
+
208
+ // 2026-08-14 批量化(dream 写爆库根治):候选先收集,页末统一一次 patchMetadataBatch
209
+ // —— 原逐条 update 每行一个 LanceDB commit(每天最多 100 个 manifest),现在页数级。
210
+ // activeCounts 仍在"选中"时即刻预递减,保持旧路径"逐行归档、后续行的 retention 判断
211
+ // 看到已减的 activeCount"语义(互审 C6)。selected 与 archivedCount 分开:前者控
212
+ // maxArchivePerRun 上限(选中即占额度),后者按批量写的实际写入数累计。
213
+ let selected = 0;
214
+ let pendingArchive: Array<{ id: string; ageDays: number }> = [];
215
+ const flushArchiveBatch = async (): Promise<void> => {
216
+ if (pendingArchive.length === 0) return;
217
+ const batch = pendingArchive;
218
+ pendingArchive = [];
219
+ const written = await store.patchMetadataBatch(
220
+ batch.map(({ id }) => ({
221
+ id,
222
+ patchFn: (meta: Record<string, unknown>) => patchEvolutionOnMeta(meta, { status: "archived" }),
223
+ })),
224
+ );
225
+ archivedCount += written;
226
+ // F-1: Audit log — record archive operations (silent on failure)。按提交批记录:
227
+ // 批内个别行被并发删除时 audit 会多记一条,best-effort 语义可接受(旧路径里
228
+ // update 失败是直接炸掉整个 GC,连 audit 都到不了)。
229
+ for (const { id, ageDays } of batch) {
230
+ try {
231
+ auditLogger?.log({
232
+ operation: "archive",
233
+ memoryId: id,
234
+ actor: "system",
235
+ details: `auto-gc: age=${Math.floor(ageDays)}d`,
236
+ });
237
+ } catch { /* Audit must never block GC */ }
238
+ }
239
+ };
240
+
172
241
  outer:
173
242
  for (let offset = 0; ; offset += GC_PAGE_SIZE) {
174
243
  const page = await store.listPage({ limit: GC_PAGE_SIZE, offset });
175
244
  totalChecked += page.length;
176
245
 
177
246
  for (const entry of page) {
178
- if (archivedCount >= config.maxArchivePerRun) break outer;
247
+ if (selected >= config.maxArchivePerRun) break outer;
179
248
 
180
249
  // Only archive active memories
181
250
  if (!isActiveMemory(entry.metadata)) continue;
@@ -215,19 +284,10 @@ async function runGcScan(
215
284
  }
216
285
 
217
286
  if (shouldArchive) {
218
- const archivedMeta = buildArchivedMetadata(entry.metadata);
219
- await store.update(entry.id, { metadata: archivedMeta });
220
- archivedCount++;
221
- // F-1: Audit log — record archive operation (silent on failure)
222
- try {
223
- auditLogger?.log({
224
- operation: "archive",
225
- memoryId: entry.id,
226
- actor: "system",
227
- details: `auto-gc: age=${Math.floor(ageDays)}d`,
228
- });
229
- } catch { /* Audit must never block GC */ }
230
- // Decrement active count for the scope (memory is no longer active)
287
+ pendingArchive.push({ id: entry.id, ageDays });
288
+ selected++;
289
+ // Decrement active count for the scope(预递减:选中即视为不再 active,
290
+ // 同页后续行的 retention 判断与旧逐行路径看到同样的计数)
231
291
  const scope = entry.scope ?? "";
232
292
  const prev = activeCounts.get(scope) ?? 0;
233
293
  if (prev > 0) {
@@ -236,9 +296,14 @@ async function runGcScan(
236
296
  }
237
297
  }
238
298
 
299
+ // 页末提交本页候选(含 break outer 前最后一页扫过的部分)
300
+ await flushArchiveBatch();
239
301
  if (page.length < GC_PAGE_SIZE) break;
240
302
  }
241
303
 
304
+ // break outer 跳出时 pendingArchive 可能还有未提交的候选 —— 上限命中不弃单(互审 C6)
305
+ await flushArchiveBatch();
306
+
242
307
  return { archivedCount, totalChecked };
243
308
  }
244
309
 
@@ -248,9 +313,11 @@ async function runGcScan(
248
313
  */
249
314
  export function resetGcTimestamp(): void {
250
315
  lastGcTimestamp = 0;
316
+ lastVersionGroupRepairTimestamp = 0;
251
317
  try {
252
318
  releaseLock({ lockPath: lockPathForKey("gc-last") });
253
319
  releaseLock({ lockPath: lockPathForKey("gc-run") });
320
+ releaseLock({ lockPath: lockPathForKey("version-group-repair-last") });
254
321
  } catch {
255
322
  /* ignore — nothing to clear */
256
323
  }