recallnest 2.3.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. package/CHANGELOG.md +187 -0
  2. package/README.md +49 -38
  3. package/README_CN.md +46 -31
  4. package/assets/ui/app.js +1031 -0
  5. package/assets/ui/index.html +224 -0
  6. package/assets/ui/styles.css +781 -0
  7. package/config.json.example +4 -1
  8. package/docs/api-reference.md +1291 -0
  9. package/docs/architecture.md +168 -0
  10. package/docs/connector-spec.md +126 -0
  11. package/eval/cases-canary.json +58 -0
  12. package/eval/cases-scope-robustness.json +278 -0
  13. package/eval/cases.json +117 -20
  14. package/package.json +25 -4
  15. package/src/__tests__/access-tracker-novelty.test.ts +304 -0
  16. package/src/__tests__/activity-counter.test.ts +125 -0
  17. package/src/__tests__/adaptive-fidelity.test.ts +102 -0
  18. package/src/__tests__/admission-control.test.ts +206 -0
  19. package/src/__tests__/anchor-generator.test.ts +67 -0
  20. package/src/__tests__/api-server-security.test.ts +102 -0
  21. package/src/__tests__/auto-consolidation.test.ts +168 -0
  22. package/src/__tests__/auto-recall.test.ts +131 -0
  23. package/src/__tests__/babel-integration.test.ts +144 -0
  24. package/src/__tests__/batch-dedup.test.ts +65 -0
  25. package/src/__tests__/belief-history.test.ts +220 -0
  26. package/src/__tests__/canary-eval.test.ts +114 -0
  27. package/src/__tests__/capture-engine.test.ts +1328 -0
  28. package/src/__tests__/capture-heuristic.test.ts +171 -0
  29. package/src/__tests__/cascade-forget.test.ts +210 -0
  30. package/src/__tests__/category-min-scores.test.ts +116 -0
  31. package/src/__tests__/cc8-cc1-essential-context.test.ts +298 -0
  32. package/src/__tests__/circuit-breaker.test.ts +99 -0
  33. package/src/__tests__/cluster-consolidation.test.ts +945 -0
  34. package/src/__tests__/confidence-metadata.test.ts +125 -0
  35. package/src/__tests__/confidence-tracker.test.ts +261 -0
  36. package/src/__tests__/conflict-advisor.test.ts +206 -0
  37. package/src/__tests__/conflict-escalation.test.ts +149 -0
  38. package/src/__tests__/conflict-lifecycle.test.ts +83 -0
  39. package/src/__tests__/conflict-output.test.ts +85 -0
  40. package/src/__tests__/conflict-store.test.ts +418 -0
  41. package/src/__tests__/connector-v1.test.ts +181 -0
  42. package/src/__tests__/consolidation-engine.test.ts +430 -0
  43. package/src/__tests__/constructive-retrieval.test.ts +509 -0
  44. package/src/__tests__/context-collapse-renderer.test.ts +214 -0
  45. package/src/__tests__/context-composer-pins.test.ts +64 -0
  46. package/src/__tests__/context-composer-task-selection.test.ts +24 -0
  47. package/src/__tests__/context-reconstructor.test.ts +65 -0
  48. package/src/__tests__/context-renderer.test.ts +91 -0
  49. package/src/__tests__/continuity-eval.test.ts +275 -0
  50. package/src/__tests__/continuity-schema.test.ts +97 -0
  51. package/src/__tests__/contracts.test.ts +262 -0
  52. package/src/__tests__/conversation-importer.test.ts +692 -0
  53. package/src/__tests__/core-recall-eval.test.ts +235 -0
  54. package/src/__tests__/core-summary.test.ts +109 -0
  55. package/src/__tests__/data-checkup.test.ts +276 -0
  56. package/src/__tests__/decay-exemption.test.ts +129 -0
  57. package/src/__tests__/dedup-actions.test.ts +58 -0
  58. package/src/__tests__/dedup-guard.test.ts +39 -0
  59. package/src/__tests__/deterministic-id.test.ts +39 -0
  60. package/src/__tests__/distill-lock.test.ts +305 -0
  61. package/src/__tests__/distill-session-argshape.test.ts +79 -0
  62. package/src/__tests__/doctor.test.ts +319 -0
  63. package/src/__tests__/dream-pipeline.test.ts +780 -0
  64. package/src/__tests__/drill-down.test.ts +108 -0
  65. package/src/__tests__/embedder.test.ts +122 -0
  66. package/src/__tests__/emotion-decay.test.ts +46 -0
  67. package/src/__tests__/emotion-detector.test.ts +56 -0
  68. package/src/__tests__/emotion-phase1-integration.test.ts +153 -0
  69. package/src/__tests__/emotion-retrieval.test.ts +56 -0
  70. package/src/__tests__/entity-resolver.test.ts +58 -0
  71. package/src/__tests__/env-config.test.ts +151 -0
  72. package/src/__tests__/error-signature-boost.test.ts +87 -0
  73. package/src/__tests__/error-signature.test.ts +95 -0
  74. package/src/__tests__/error-taxonomy.test.ts +129 -0
  75. package/src/__tests__/ethics-layer.test.ts +476 -0
  76. package/src/__tests__/event-segmenter.test.ts +153 -0
  77. package/src/__tests__/feature-flags-integration.test.ts +54 -0
  78. package/src/__tests__/frequency-tracker.test.ts +113 -0
  79. package/src/__tests__/freshness.test.ts +186 -0
  80. package/src/__tests__/graph-export.test.ts +489 -0
  81. package/src/__tests__/hotness-blend.test.ts +283 -0
  82. package/src/__tests__/ingest-chunking.test.ts +44 -0
  83. package/src/__tests__/ingest.test.ts +360 -0
  84. package/src/__tests__/interference-detector.test.ts +203 -0
  85. package/src/__tests__/kg-extractor.test.ts +192 -0
  86. package/src/__tests__/kg-store.test.ts +300 -0
  87. package/src/__tests__/llm-client-pivot.test.ts +628 -0
  88. package/src/__tests__/llm-consolidation.test.ts +226 -0
  89. package/src/__tests__/maintenance-locks.test.ts +189 -0
  90. package/src/__tests__/mcp-efficiency.test.ts +288 -0
  91. package/src/__tests__/mcp-related-scopes.test.ts +151 -0
  92. package/src/__tests__/memory-boundaries.test.ts +282 -0
  93. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +345 -0
  94. package/src/__tests__/memory-evolution.test.ts +271 -0
  95. package/src/__tests__/memory-health-rebalance.test.ts +144 -0
  96. package/src/__tests__/memory-lint.test.ts +421 -0
  97. package/src/__tests__/memory-output.test.ts +215 -0
  98. package/src/__tests__/memory-promotion.test.ts +315 -0
  99. package/src/__tests__/migrate-dedup-ids.test.ts +65 -0
  100. package/src/__tests__/multi-vector.test.ts +242 -0
  101. package/src/__tests__/narrative-architecture.test.ts +456 -0
  102. package/src/__tests__/noise-filter.test.ts +205 -0
  103. package/src/__tests__/noise-prototype-bank.test.ts +280 -0
  104. package/src/__tests__/obsidian-connector.test.ts +363 -0
  105. package/src/__tests__/omitted-reporting.test.ts +98 -0
  106. package/src/__tests__/pending-review.test.ts +77 -0
  107. package/src/__tests__/pii-redact.test.ts +333 -0
  108. package/src/__tests__/pivot-apply.test.ts +373 -0
  109. package/src/__tests__/pivot-distill-supervisor.test.ts +457 -0
  110. package/src/__tests__/pivot-distill.test.ts +1178 -0
  111. package/src/__tests__/post-retrieval-filter.test.ts +178 -0
  112. package/src/__tests__/ppr-traversal.test.ts +256 -0
  113. package/src/__tests__/prediction-engine.test.ts +349 -0
  114. package/src/__tests__/predictive-prospective-memory.test.ts +416 -0
  115. package/src/__tests__/preference-guard-smoke.test.ts +211 -0
  116. package/src/__tests__/preference-matcher.test.ts +282 -0
  117. package/src/__tests__/preference-slots.test.ts +205 -0
  118. package/src/__tests__/prospective-memory.test.ts +283 -0
  119. package/src/__tests__/pull-from-macbook-sync-only.test.ts +679 -0
  120. package/src/__tests__/query-entity-detector.test.ts +171 -0
  121. package/src/__tests__/query-expander-alias.test.ts +146 -0
  122. package/src/__tests__/query-expander.test.ts +34 -0
  123. package/src/__tests__/recall-gate.test.ts +266 -0
  124. package/src/__tests__/recall-governor.test.ts +227 -0
  125. package/src/__tests__/recall-mode.test.ts +64 -0
  126. package/src/__tests__/resolve-db-path.test.ts +130 -0
  127. package/src/__tests__/resolve-tier.test.ts +64 -0
  128. package/src/__tests__/result-synthesizer.test.ts +198 -0
  129. package/src/__tests__/resume-context.test.ts +4898 -0
  130. package/src/__tests__/retrieval-eval.test.ts +163 -0
  131. package/src/__tests__/retrieval-trace.test.ts +213 -0
  132. package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
  133. package/src/__tests__/retriever-lazy-init-dispatch.test.ts +123 -0
  134. package/src/__tests__/retroactive-boost.test.ts +223 -0
  135. package/src/__tests__/salience-scoring.test.ts +106 -0
  136. package/src/__tests__/scope-inventory.test.ts +200 -0
  137. package/src/__tests__/scope-policy.test.ts +69 -0
  138. package/src/__tests__/scope-suggester.test.ts +80 -0
  139. package/src/__tests__/session-checkpoint.test.ts +291 -0
  140. package/src/__tests__/session-distiller.test.ts +640 -0
  141. package/src/__tests__/session-gc.test.ts +215 -0
  142. package/src/__tests__/skill-engine.test.ts +552 -0
  143. package/src/__tests__/skill-promotion.test.ts +654 -0
  144. package/src/__tests__/skill-verifier.test.ts +122 -0
  145. package/src/__tests__/source-heartbeat.test.ts +250 -0
  146. package/src/__tests__/stable-context-length-contract.test.ts +62 -0
  147. package/src/__tests__/store-patch-metadata.test.ts +205 -0
  148. package/src/__tests__/store-read-consistency.test.ts +122 -0
  149. package/src/__tests__/store-schema-migration.test.ts +87 -0
  150. package/src/__tests__/store-scope-audit.test.ts +81 -0
  151. package/src/__tests__/store-upsert.test.ts +201 -0
  152. package/src/__tests__/supersede-chain.test.ts +154 -0
  153. package/src/__tests__/temporal-parser.test.ts +241 -0
  154. package/src/__tests__/temporal-validity.test.ts +160 -0
  155. package/src/__tests__/term-registry-terminal-roster.test.ts +127 -0
  156. package/src/__tests__/term-registry.test.ts +121 -0
  157. package/src/__tests__/tool-error-surfacing.test.ts +98 -0
  158. package/src/__tests__/tool-output-compressor.test.ts +160 -0
  159. package/src/__tests__/topic-tag.test.ts +113 -0
  160. package/src/__tests__/trusted-memory.test.ts +839 -0
  161. package/src/__tests__/ui-server-security.test.ts +93 -0
  162. package/src/__tests__/usage-tracker.test.ts +287 -0
  163. package/src/__tests__/version-manager.test.ts +213 -0
  164. package/src/__tests__/workflow-observation-skill-outcome.test.ts +352 -0
  165. package/src/__tests__/workflow-observation.test.ts +433 -0
  166. package/src/__tests__/write-verifier.test.ts +122 -0
  167. package/src/access-tracker.ts +90 -34
  168. package/src/activity-counter.ts +46 -18
  169. package/src/adaptive-retrieval.ts +11 -0
  170. package/src/aliases.ts +133 -0
  171. package/src/api-server.ts +36 -9
  172. package/src/audit-log.ts +4 -2
  173. package/src/auto-consolidation.ts +8 -3
  174. package/src/auto-gc.ts +85 -21
  175. package/src/auto-recall.ts +61 -1
  176. package/src/belief-history.ts +164 -0
  177. package/src/capture-engine.ts +142 -32
  178. package/src/cli.ts +268 -41
  179. package/src/confidence-tracker.ts +36 -18
  180. package/src/conflict-engine.ts +24 -3
  181. package/src/consolidation-engine.ts +255 -23
  182. package/src/context-composer-stable-selection.ts +7 -1
  183. package/src/context-composer.ts +8 -1
  184. package/src/data-checkup.ts +96 -7
  185. package/src/decay-engine.ts +59 -7
  186. package/src/distill-lock.ts +194 -15
  187. package/src/doctor.ts +13 -4
  188. package/src/dream-pipeline.ts +473 -41
  189. package/src/env-config.ts +118 -0
  190. package/src/error-signature.ts +153 -0
  191. package/src/error-taxonomy.ts +115 -0
  192. package/src/eval.ts +402 -14
  193. package/src/freshness.ts +192 -0
  194. package/src/http-request-policy.ts +148 -0
  195. package/src/idempotency.ts +8 -0
  196. package/src/ingest.ts +308 -12
  197. package/src/kg-extractor.ts +13 -3
  198. package/src/kg-store.ts +236 -44
  199. package/src/llm-client.ts +106 -11
  200. package/src/llm-consolidation.ts +2 -1
  201. package/src/mcp-server.ts +87 -1800
  202. package/src/mcp-tool-deps.ts +48 -0
  203. package/src/mcp-tools-advanced.ts +954 -0
  204. package/src/mcp-tools-core.ts +637 -0
  205. package/src/mcp-tools-governance.ts +475 -0
  206. package/src/memory-boundaries.ts +26 -3
  207. package/src/memory-evolution.ts +73 -1
  208. package/src/memory-health-rebalance.ts +4 -0
  209. package/src/memory-lint.ts +183 -10
  210. package/src/memory-output.ts +129 -4
  211. package/src/memory-promotion.ts +246 -0
  212. package/src/memory-schema.ts +25 -2
  213. package/src/memory-store-port.ts +46 -0
  214. package/src/multi-vector.ts +2 -1
  215. package/src/narrative-schema.ts +2 -1
  216. package/src/noise-filter.ts +57 -6
  217. package/src/noise-prototype-bank.ts +408 -0
  218. package/src/pii-detector.ts +207 -5
  219. package/src/pivot-apply.ts +399 -0
  220. package/src/preference-matcher.ts +37 -3
  221. package/src/query-expander.ts +90 -9
  222. package/src/recall-gate.ts +305 -0
  223. package/src/result-synthesizer.ts +2 -1
  224. package/src/retention-policy.ts +2 -1
  225. package/src/retrieval-profiles.ts +46 -4
  226. package/src/retriever.ts +354 -157
  227. package/src/retroactive-boost.ts +6 -2
  228. package/src/runtime-config.ts +78 -6
  229. package/src/scope-suggester.ts +113 -0
  230. package/src/session-distiller.ts +23 -2
  231. package/src/session-engine.ts +2 -1
  232. package/src/session-schema.ts +1 -0
  233. package/src/session-store.ts +5 -0
  234. package/src/skill-engine.ts +78 -0
  235. package/src/skill-promotion.ts +304 -48
  236. package/src/skill-schema.ts +16 -3
  237. package/src/skill-verifier.ts +152 -0
  238. package/src/store.ts +321 -57
  239. package/src/term-registry.ts +35 -0
  240. package/src/tool-output-compressor.ts +3 -1
  241. package/src/ui-server.ts +9 -1
  242. package/src/usage-tracker.ts +242 -0
  243. package/src/version-manager.ts +3 -2
  244. package/src/version.ts +19 -0
  245. package/src/workflow-observation-engine.ts +2 -1
  246. package/src/workflow-observation-schema.ts +2 -1
  247. package/src/workflow-observation-store.ts +6 -2
  248. package/.env +0 -4
  249. package/CLAUDE.md +0 -46
  250. package/config.json +0 -55
  251. package/daily-recall.sh +0 -110
  252. package/logs/daily-recall.error.log +0 -0
  253. package/logs/daily-recall.log +0 -60
  254. package/logs/ingest-2026-04-11.log +0 -110
  255. package/logs/ingest-2026-04-12.log +0 -90
  256. package/logs/ingest-2026-04-13.log +0 -87
  257. package/logs/launchagent-stderr.log +0 -0
  258. package/logs/launchagent-stdout.log +0 -0
@@ -0,0 +1,168 @@
1
+ # Architecture
2
+
3
+ > 架构总览:RecallNest 的分层设计,从接入层到存储层。
4
+
5
+ 边界说明:
6
+
7
+ - structured capture 写 durable memory
8
+ - checkpoint store 写 session state
9
+ - raw ingest 写 evidence
10
+
11
+ See [memory-boundary-contract.md](./memory-boundary-contract.md).
12
+
13
+ ## System Architecture
14
+
15
+ ```
16
+ ┌─────────────────────────────────────────────────────────────┐
17
+ │ Client Layer │
18
+ ├──────────┬──────────┬──────────┬──────────┬─────────────────┤
19
+ │ Claude │ Gemini │ Codex │ Custom │ curl / any │
20
+ │ Code │ CLI │ │ Agents │ HTTP client │
21
+ └────┬─────┴────┬─────┴────┬─────┴────┬─────┴────┬────────────┘
22
+ │ │ │ │ │
23
+ └──── MCP (stdio) ───┘ └── HTTP ──┘
24
+ │ │
25
+ ▼ ▼
26
+ ┌─────────────────────────────────────────────────────────────┐
27
+ │ Integration Layer │
28
+ ├────────────────────────┬────────────────────────────────────┤
29
+ │ MCP Server │ HTTP API Server │
30
+ │ (mcp-server.ts) │ (api-server.ts) │
31
+ │ │ port 4318 │
32
+ │ Tools: │ │
33
+ │ - search_memory │ Endpoints: │
34
+ │ - memory_stats │ - POST /v1/recall │
35
+ │ - brief_memory │ - POST /v1/store │
36
+ │ - distill_memory │ - POST /v1/search │
37
+ │ - pin_memory │ - GET /v1/stats │
38
+ │ - explain_memory │ - GET /v1/health │
39
+ │ - export_memory │ - POST /v1/consolidate │
40
+ │ │ - GET /v1/gaps │
41
+ └────────────┬───────────┴──────────────┬─────────────────────┘
42
+ │ │
43
+ ▼ ▼
44
+ ┌─────────────────────────────────────────────────────────────┐
45
+ │ Core Engine │
46
+ ├─────────────────────────────────────────────────────────────┤
47
+ │ │
48
+ │ ┌─────────────┐ ┌──────────────┐ ┌────────────────────┐ │
49
+ │ │ Retriever │ │ Classifier │ │ Query Expander │ │
50
+ │ │ (hybrid: │ │ (6 categories│ │ (synonym + │ │
51
+ │ │ vector + │ │ auto-assign)│ │ semantic expand) │ │
52
+ │ │ BM25 + RRF) │ │ │ │ │ │
53
+ │ └──────┬───────┘ └──────────────┘ └────────────────────┘ │
54
+ │ │ │
55
+ │ ┌──────┴───────┐ ┌──────────────┐ ┌────────────────────┐ │
56
+ │ │ Decay │ │ Access │ │ Noise Filter │ │
57
+ │ │ Engine │ │ Tracker │ │ (relevance │ │
58
+ │ │ (Weibull) │ │ (use it or │ │ threshold) │ │
59
+ │ │ │ │ lose it) │ │ │ │
60
+ │ └──────────────┘ └──────────────┘ └────────────────────┘ │
61
+ │ │
62
+ │ ┌──────────────┐ ┌──────────────┐ │
63
+ │ │ Consolidator │ │ Gap Detector │ ← Self-Evolution │
64
+ │ │ (merge/dedup)│ │ (find blind │ │
65
+ │ │ │ │ spots) │ │
66
+ │ └──────────────┘ └──────────────┘ │
67
+ │ │
68
+ └──────────────────────────┬──────────────────────────────────┘
69
+ │
70
+ ▼
71
+ ┌─────────────────────────────────────────────────────────────┐
72
+ │ Storage Layer │
73
+ ├─────────────────────────┬───────────────────────────────────┤
74
+ │ LanceDB │ Jina Embeddings │
75
+ │ (vector + columnar) │ (v5, 1024-dim) │
76
+ │ │ │
77
+ │ Tables: │ Task-aware: │
78
+ │ - memories (main) │ - retrieval.query (for searches) │
79
+ │ - access_log │ - retrieval.passage (for docs) │
80
+ │ - search_log │ │
81
+ └─────────────────────────┴───────────────────────────────────┘
82
+ ```
83
+
84
+ ## Data Flow
85
+
86
+ ### Ingestion Pipeline
87
+
88
+ ```
89
+ Source Files Processing Storage
90
+ ───────────── ────────── ───────
91
+
92
+ CC transcripts ─┐
93
+ │ ┌──────────────────┐ ┌───────────┐
94
+ Codex sessions ─┼──►│ Chunker │──►│ Embedder │──► LanceDB
95
+ │ │ (split by turn, │ │ (Jina v5) │
96
+ Gemini chats ───┤ │ noise filter) │ └───────────┘
97
+ │ └──────────────────┘
98
+ Memory .md ─────┘ │
99
+ ▼
100
+ ┌──────────────┐
101
+ │ Classifier │
102
+ │ (6 categories│
103
+ │ + tier) │
104
+ └──────────────┘
105
+ ```
106
+
107
+ > 入库流程:多源文件 → 分块 → 过滤噪音 → 分类 → 嵌入向量 → 存入 LanceDB
108
+
109
+ ### Search Pipeline
110
+
111
+ ```
112
+ Query Results
113
+ ───── ───────
114
+
115
+ "Docker debugging"
116
+ │
117
+ ▼
118
+ ┌──────────────┐ ┌──────────────┐ ┌───────────────┐
119
+ │ Query │───►│ Hybrid │───►│ Post-process │──► Top-K
120
+ │ Expander │ │ Retrieval │ │ │ Results
121
+ │ (synonyms) │ │ │ │ - Decay │
122
+ └──────────────┘ │ Vector: 0.7 │ │ - Access boost│
123
+ │ BM25: 0.3 │ │ - Score floor │
124
+ │ RRF merge │ │ - Dedup │
125
+ └──────────────┘ └───────────────┘
126
+ ```
127
+
128
+ ## Key Design Decisions
129
+
130
+ | Decision | Rationale |
131
+ |----------|-----------|
132
+ | **LanceDB** (not SQLite/Postgres) | Native vector search, columnar storage, zero-config, single-file DB |
133
+ | **Jina v5** (not OpenAI embeddings) | Task-aware embeddings (query vs passage), better multilingual, 1024-dim sweet spot |
134
+ | **Hybrid retrieval** (vector + BM25) | Vector alone misses keyword matches; BM25 alone misses semantic similarity |
135
+ | **RRF fusion** | Reciprocal Rank Fusion is parameter-free and robust across score distributions |
136
+ | **Weibull decay** (not exponential) | Better models human forgetting: slow start, accelerating fade |
137
+ | **6 categories** (not free-form tags) | Structured enough for filtering and lifecycle rules, simple enough to auto-classify |
138
+ | **HTTP API + MCP** (not just MCP) | MCP is great for CLI tools, but HTTP API works with any language/framework |
139
+ | **Bun runtime** | Fast startup, native TypeScript, good for CLI tools and local servers |
140
+
141
+ ## File Map
142
+
143
+ ```
144
+ src/
145
+ ├── api-server.ts # HTTP API server (port 4318)
146
+ ├── mcp-server.ts # MCP server (stdio transport)
147
+ ├── cli.ts # CLI entry point (lm command)
148
+ ├── store.ts # LanceDB storage layer
149
+ ├── retriever.ts # Hybrid retrieval (vector + BM25 + RRF)
150
+ ├── embedder.ts # Jina embedding client
151
+ ├── ingest.ts # Multi-source ingestion pipeline
152
+ ├── chunker.ts # Text chunking with noise filtering
153
+ ├── decay-engine.ts # Weibull time decay
154
+ ├── access-tracker.ts # "Use it or lose it" tracking
155
+ ├── noise-filter.ts # Low-quality content filter
156
+ ├── query-expander.ts # Query expansion (synonyms)
157
+ ├── retrieval-profiles.ts # 4 search profiles (precision/balanced/exploratory/recent)
158
+ ├── memory-output.ts # Output formatting
159
+ ├── memory-assets.ts # Brief/pin/export asset management
160
+ ├── asset-sync.ts # Asset indexing
161
+ ├── runtime-config.ts # Config loading
162
+ ├── llm-client.ts # LLM client for smart extraction
163
+ ├── doctor.ts # Health check diagnostics
164
+ ├── adaptive-retrieval.ts # Adaptive retrieval strategies
165
+ ├── stderr-log.ts # Logging
166
+ ├── tracker.ts # Ingestion tracker (incremental)
167
+ └── __tests__/ # Test suite
168
+ ```
@@ -0,0 +1,126 @@
1
+ # RecallNest Connector Specification (connector-v1)
2
+
3
+ ## Overview
4
+
5
+ The connector-v1 format is the standard interface for feeding external data into RecallNest. Any external script, service, or integration can produce a JSON file in this format, and RecallNest will consume it through its existing ingest pipeline (dedup, embedding, smart extraction, tiered storage).
6
+
7
+ **Design philosophy**: RecallNest's core engine stays lean. External connectors handle data fetching and normalization; RecallNest handles memory quality (dedup, decay, governance, retrieval).
8
+
9
+ ## JSON Schema
10
+
11
+ ```json
12
+ {
13
+ "version": "connector-v1",
14
+ "source": "obsidian",
15
+ "scope": "vault:my-notes",
16
+ "batchId": "batch-2026-04-13-001",
17
+ "producedAt": "2026-04-13T12:00:00Z",
18
+ "records": [
19
+ {
20
+ "id": "obsidian:notes/meeting-2026-04-13.md",
21
+ "text": "Meeting with team about Q2 goals...",
22
+ "title": "Q2 Planning Meeting",
23
+ "categoryHint": "events",
24
+ "importanceHint": 0.8,
25
+ "tags": ["meeting", "q2", "link:Team-Goals"],
26
+ "timestamp": "2026-04-13T10:00:00Z",
27
+ "contentHash": "a1b2c3d4e5f67890",
28
+ "sourceMetadata": {
29
+ "filePath": "notes/meeting-2026-04-13.md",
30
+ "frontmatter": { "tags": ["meeting"] }
31
+ }
32
+ }
33
+ ]
34
+ }
35
+ ```
36
+
37
+ ## Field Reference
38
+
39
+ ### Top-level
40
+
41
+ | Field | Type | Required | Description |
42
+ |-------|------|----------|-------------|
43
+ | `version` | `"connector-v1"` | Yes | Fixed version string for format detection |
44
+ | `source` | string | Yes | Data source identifier (e.g. `"obsidian"`, `"email"`, `"rss"`) |
45
+ | `scope` | string | Yes | Target scope for all records (e.g. `"vault:my-notes"`, `"project:research"`) |
46
+ | `batchId` | string | No | Batch identifier for idempotent retry |
47
+ | `producedAt` | string (ISO 8601) | Yes | When this payload was generated |
48
+ | `records` | ConnectorRecord[] | Yes | Array of records to ingest |
49
+
50
+ ### ConnectorRecord
51
+
52
+ | Field | Type | Required | Description |
53
+ |-------|------|----------|-------------|
54
+ | `id` | string | Yes | Unique ID within the source (used for incremental tracking) |
55
+ | `text` | string | Yes | Main content to be stored and embedded |
56
+ | `title` | string | No | Title or heading (prepended as `[title]` in stored text) |
57
+ | `categoryHint` | string | No | Suggested category: `preferences`, `events`, `knowledge`, `facts`, `decisions` |
58
+ | `importanceHint` | number (0-1) | No | Suggested importance score (default: extracted by LLM or 0.7) |
59
+ | `tags` | string[] | No | Tags stored in metadata as `connectorTags` |
60
+ | `timestamp` | string (ISO 8601) | No | Original creation/modification time |
61
+ | `contentHash` | string | No | Content hash for incremental sync (skip if unchanged) |
62
+ | `sourceMetadata` | object | No | Arbitrary metadata from the source (stored as-is in entry metadata) |
63
+
64
+ ## Ingest Behavior
65
+
66
+ 1. **Format detection**: `detectFormat()` identifies connector-v1 by checking `version === "connector-v1"` and `Array.isArray(records)`. Takes highest priority among object-type formats.
67
+
68
+ 2. **Incremental sync**: If `contentHash` is present, RecallNest stores it in entry metadata. Future versions may use this for pre-embedding dedup.
69
+
70
+ 3. **Dedup**: Each record goes through the standard two-stage dedup pipeline (vector pre-filter + optional LLM semantic decision).
71
+
72
+ 4. **Category/importance**: If `categoryHint` or `importanceHint` are provided, they override the LLM-extracted values. Otherwise, smart extraction determines them.
73
+
74
+ 5. **Tags**: `tags` are stored in `metadata.connectorTags` (separate from RecallNest's internal tags to avoid namespace collision).
75
+
76
+ 6. **Scope**: All records share the top-level `scope`. To ingest records into different scopes, produce separate connector-v1 payloads.
77
+
78
+ ## Writing a Connector
79
+
80
+ A connector is any script that produces a connector-v1 JSON file. Minimal example in TypeScript:
81
+
82
+ ```typescript
83
+ import { writeFileSync } from "node:fs";
84
+
85
+ const output = {
86
+ version: "connector-v1",
87
+ source: "my-app",
88
+ scope: "project:my-app",
89
+ producedAt: new Date().toISOString(),
90
+ records: [
91
+ {
92
+ id: "note-001",
93
+ text: "User prefers dark mode and compact layout",
94
+ title: "UI Preferences",
95
+ categoryHint: "preferences",
96
+ importanceHint: 0.9,
97
+ tags: ["ui", "preferences"],
98
+ },
99
+ ],
100
+ };
101
+
102
+ writeFileSync("my-connector-output.json", JSON.stringify(output, null, 2));
103
+ ```
104
+
105
+ Then feed it to RecallNest:
106
+
107
+ ```bash
108
+ # Via CLI
109
+ recallnest ingest --connector my-connector-output.json
110
+
111
+ # Via MCP tool
112
+ # Use import_conversations with format="connector-v1" and content=<file contents>
113
+ ```
114
+
115
+ ## Built-in Connectors
116
+
117
+ | Connector | Source | Description |
118
+ |-----------|--------|-------------|
119
+ | `obsidian-connector` | `obsidian` | Scans an Obsidian vault, extracts frontmatter + wikilinks, maps folder structure to tags |
120
+
121
+ ## Versioning
122
+
123
+ The `version` field enables future format evolution. RecallNest will:
124
+ - Accept `"connector-v1"` indefinitely (no breaking changes within v1)
125
+ - Future v2 will be a separate format, not a replacement
126
+ - Unknown versions are rejected with a clear error message
@@ -0,0 +1,58 @@
1
+ [
2
+ {
3
+ "name": "canary-A-mem0-borrow",
4
+ "query": "mem0 对 recallnest 有没有借鉴作用",
5
+ "profile": "fact-check",
6
+ "limit": 8,
7
+ "targets": ["a2fe9b62"],
8
+ "expectTopK": 3,
9
+ "notes": "A 单目标:mem0→recallnest 借鉴审计结论该被召回。"
10
+ },
11
+ {
12
+ "name": "canary-A-dual-dir-fix",
13
+ "query": "recallnest 本地两个目录反复踩坑是怎么解决的",
14
+ "profile": "debug",
15
+ "limit": 8,
16
+ "targets": ["dbcb317a"],
17
+ "expectTopK": 3,
18
+ "notes": "A 单目标:dual-dir 符号链接统一 case(dbcb317a, cases, imp 0.9, baseline rank1)。原 target e40ab64b 系标注错——e40ab64b 实为『RecallNest 演进优先 workflow pattern』,与本 query 无关;dbcb317a 才是『~/recallnest→~/Projects/recallnest symlink 统一』。"
19
+ },
20
+ {
21
+ "name": "canary-B-capture-heuristic-evolution",
22
+ "query": "recallnest 的 capture 是纯 heuristic 吗,提取到底用不用 LLM",
23
+ "profile": "fact-check",
24
+ "limit": 8,
25
+ "targets": ["727da608"],
26
+ "expectOrder": ["727da608", "a2fe9b62"],
27
+ "expectTopK": 3,
28
+ "notes": "B 新旧对:727da608(cases,imp0.8) 正文明确『修正 a2fe9b62 中 capture 纯heuristic/zero-LLM 的不准确表述』,是本 query 准确答案;a2fe9b62(events,imp0.72,mem0借鉴审计) 含被纠正的旧表述。expectOrder 测『两者同时召回时准确版 727 须排在旧版 a2 前』——纯排名对比(rank 唯一,<与≤等价),不靠结构化 supersede。校准修正:原 notes『验 confidence/recency/decay』机制声称不成立(二者无 evolution 链/降权链,靠内容质量排序);且主话题不重合(727=ingest去重史 / a2=mem0审计),实测 a2 常落选(baseline rank=null),故本 case 当前主要测 727 召回质量+弱新旧序保护,非真正新旧竞争。理想 B 类应换一对同话题、会同时召回的新旧记忆(P1 后续)。"
29
+ },
30
+ {
31
+ "name": "canary-C-scope-isolation-noise",
32
+ "query": "工作记忆要不要做 scope 隔离",
33
+ "profile": "default",
34
+ "limit": 8,
35
+ "targets": ["491c8b30"],
36
+ "hardNegatives": ["b59e86b2"],
37
+ "expectTopK": 2,
38
+ "notes": "C 干扰项:491c8b30(元决策『工作记忆不做 scope 隔离』,baseline rank1) 应召回。hardNegative b59e86b2(entities,imp0.65,scope=codex:019cfba5) 是 codex 端关于 scope 模型/associative access/隔离的设计讨论片段——词面同话题、别处语境(非 Alice 元决策),baseline rank2 进了 top。补此 hardNeg 后本 case 在当前裸 retrieve 下会 fail,暴露『跨-scope 旁支讨论被召回进 top』的真问题(契合 P0 scope 主题)。"
39
+ },
40
+ {
41
+ "name": "canary-D-writing-style",
42
+ "query": "Alice 的写作风格偏好是什么",
43
+ "profile": "writing",
44
+ "limit": 6,
45
+ "expectContentAny": ["口语化", "不端着", "剧评腔", "书面非口语", "可自嘲"],
46
+ "expectTopK": 3,
47
+ "notes": "D 稳定偏好:不钉 id,召回文本命中要点即算对。"
48
+ },
49
+ {
50
+ "name": "canary-D-visual-style",
51
+ "query": "Alice 的配图审美偏好",
52
+ "profile": "default",
53
+ "limit": 6,
54
+ "expectContentAny": ["手绘", "撞色", "高对比", "静物无人物", "涂鸦"],
55
+ "expectTopK": 3,
56
+ "notes": "D 稳定偏好:同上。"
57
+ }
58
+ ]
@@ -0,0 +1,278 @@
1
+ [
2
+ {
3
+ "name": "scope-727da608-v1-WITH",
4
+ "query": "recallnest 的 capture 用不用 LLM,是不是纯 heuristic",
5
+ "profile": "fact-check",
6
+ "limit": 8,
7
+ "targets": [
8
+ "727da608"
9
+ ],
10
+ "expectTopK": 3,
11
+ "scope": "project:recallnest",
12
+ "notes": "P1.1 scope 影响 — 目标 727da608 变体1 带 scope=project:recallnest"
13
+ },
14
+ {
15
+ "name": "scope-727da608-v1-WITHOUT",
16
+ "query": "recallnest 的 capture 用不用 LLM,是不是纯 heuristic",
17
+ "profile": "fact-check",
18
+ "limit": 8,
19
+ "targets": [
20
+ "727da608"
21
+ ],
22
+ "expectTopK": 3,
23
+ "notes": "P1.1 scope 影响 — 目标 727da608 变体1 不带 scope(裸 retrieve)"
24
+ },
25
+ {
26
+ "name": "scope-727da608-v2-WITH",
27
+ "query": "ingest 去重为什么会把携带新信息的内容误删",
28
+ "profile": "fact-check",
29
+ "limit": 8,
30
+ "targets": [
31
+ "727da608"
32
+ ],
33
+ "expectTopK": 3,
34
+ "scope": "project:recallnest",
35
+ "notes": "P1.1 scope 影响 — 目标 727da608 变体2 带 scope=project:recallnest"
36
+ },
37
+ {
38
+ "name": "scope-727da608-v2-WITHOUT",
39
+ "query": "ingest 去重为什么会把携带新信息的内容误删",
40
+ "profile": "fact-check",
41
+ "limit": 8,
42
+ "targets": [
43
+ "727da608"
44
+ ],
45
+ "expectTopK": 3,
46
+ "notes": "P1.1 scope 影响 — 目标 727da608 变体2 不带 scope(裸 retrieve)"
47
+ },
48
+ {
49
+ "name": "scope-727da608-v3-WITH",
50
+ "query": "recallnest 早期 ingest 排除太厉害是什么原因、后来怎么修的",
51
+ "profile": "fact-check",
52
+ "limit": 8,
53
+ "targets": [
54
+ "727da608"
55
+ ],
56
+ "expectTopK": 3,
57
+ "scope": "project:recallnest",
58
+ "notes": "P1.1 scope 影响 — 目标 727da608 变体3 带 scope=project:recallnest"
59
+ },
60
+ {
61
+ "name": "scope-727da608-v3-WITHOUT",
62
+ "query": "recallnest 早期 ingest 排除太厉害是什么原因、后来怎么修的",
63
+ "profile": "fact-check",
64
+ "limit": 8,
65
+ "targets": [
66
+ "727da608"
67
+ ],
68
+ "expectTopK": 3,
69
+ "notes": "P1.1 scope 影响 — 目标 727da608 变体3 不带 scope(裸 retrieve)"
70
+ },
71
+ {
72
+ "name": "scope-a2fe9b62-v1-WITH",
73
+ "query": "mem0 对 recallnest 有没有借鉴价值",
74
+ "profile": "fact-check",
75
+ "limit": 8,
76
+ "targets": [
77
+ "a2fe9b62"
78
+ ],
79
+ "expectTopK": 3,
80
+ "scope": "project:recallnest",
81
+ "notes": "P1.1 scope 影响 — 目标 a2fe9b62 变体1 带 scope=project:recallnest"
82
+ },
83
+ {
84
+ "name": "scope-a2fe9b62-v1-WITHOUT",
85
+ "query": "mem0 对 recallnest 有没有借鉴价值",
86
+ "profile": "fact-check",
87
+ "limit": 8,
88
+ "targets": [
89
+ "a2fe9b62"
90
+ ],
91
+ "expectTopK": 3,
92
+ "notes": "P1.1 scope 影响 — 目标 a2fe9b62 变体1 不带 scope(裸 retrieve)"
93
+ },
94
+ {
95
+ "name": "scope-a2fe9b62-v2-WITH",
96
+ "query": "mem0 的招牌设计 recallnest 有没有独立实现",
97
+ "profile": "fact-check",
98
+ "limit": 8,
99
+ "targets": [
100
+ "a2fe9b62"
101
+ ],
102
+ "expectTopK": 3,
103
+ "scope": "project:recallnest",
104
+ "notes": "P1.1 scope 影响 — 目标 a2fe9b62 变体2 带 scope=project:recallnest"
105
+ },
106
+ {
107
+ "name": "scope-a2fe9b62-v2-WITHOUT",
108
+ "query": "mem0 的招牌设计 recallnest 有没有独立实现",
109
+ "profile": "fact-check",
110
+ "limit": 8,
111
+ "targets": [
112
+ "a2fe9b62"
113
+ ],
114
+ "expectTopK": 3,
115
+ "notes": "P1.1 scope 影响 — 目标 a2fe9b62 变体2 不带 scope(裸 retrieve)"
116
+ },
117
+ {
118
+ "name": "scope-a2fe9b62-v3-WITH",
119
+ "query": "mem0 借鉴审计的核心结论是什么",
120
+ "profile": "fact-check",
121
+ "limit": 8,
122
+ "targets": [
123
+ "a2fe9b62"
124
+ ],
125
+ "expectTopK": 3,
126
+ "scope": "project:recallnest",
127
+ "notes": "P1.1 scope 影响 — 目标 a2fe9b62 变体3 带 scope=project:recallnest"
128
+ },
129
+ {
130
+ "name": "scope-a2fe9b62-v3-WITHOUT",
131
+ "query": "mem0 借鉴审计的核心结论是什么",
132
+ "profile": "fact-check",
133
+ "limit": 8,
134
+ "targets": [
135
+ "a2fe9b62"
136
+ ],
137
+ "expectTopK": 3,
138
+ "notes": "P1.1 scope 影响 — 目标 a2fe9b62 变体3 不带 scope(裸 retrieve)"
139
+ },
140
+ {
141
+ "name": "scope-491c8b30-v1-WITH",
142
+ "query": "工作记忆要不要做 scope 隔离",
143
+ "profile": "default",
144
+ "limit": 8,
145
+ "targets": [
146
+ "491c8b30"
147
+ ],
148
+ "expectTopK": 3,
149
+ "scope": "project:recallnest",
150
+ "notes": "P1.1 scope 影响 — 目标 491c8b30 变体1 带 scope=project:recallnest"
151
+ },
152
+ {
153
+ "name": "scope-491c8b30-v1-WITHOUT",
154
+ "query": "工作记忆要不要做 scope 隔离",
155
+ "profile": "default",
156
+ "limit": 8,
157
+ "targets": [
158
+ "491c8b30"
159
+ ],
160
+ "expectTopK": 3,
161
+ "notes": "P1.1 scope 影响 — 目标 491c8b30 变体1 不带 scope(裸 retrieve)"
162
+ },
163
+ {
164
+ "name": "scope-491c8b30-v2-WITH",
165
+ "query": "单位/工作记忆该不该 forget 清理",
166
+ "profile": "default",
167
+ "limit": 8,
168
+ "targets": [
169
+ "491c8b30"
170
+ ],
171
+ "expectTopK": 3,
172
+ "scope": "project:recallnest",
173
+ "notes": "P1.1 scope 影响 — 目标 491c8b30 变体2 带 scope=project:recallnest"
174
+ },
175
+ {
176
+ "name": "scope-491c8b30-v2-WITHOUT",
177
+ "query": "单位/工作记忆该不该 forget 清理",
178
+ "profile": "default",
179
+ "limit": 8,
180
+ "targets": [
181
+ "491c8b30"
182
+ ],
183
+ "expectTopK": 3,
184
+ "notes": "P1.1 scope 影响 — 目标 491c8b30 变体2 不带 scope(裸 retrieve)"
185
+ },
186
+ {
187
+ "name": "scope-491c8b30-v3-WITH",
188
+ "query": "Alice 对工作记忆隔离是怎么决定的",
189
+ "profile": "default",
190
+ "limit": 8,
191
+ "targets": [
192
+ "491c8b30"
193
+ ],
194
+ "expectTopK": 3,
195
+ "scope": "project:recallnest",
196
+ "notes": "P1.1 scope 影响 — 目标 491c8b30 变体3 带 scope=project:recallnest"
197
+ },
198
+ {
199
+ "name": "scope-491c8b30-v3-WITHOUT",
200
+ "query": "Alice 对工作记忆隔离是怎么决定的",
201
+ "profile": "default",
202
+ "limit": 8,
203
+ "targets": [
204
+ "491c8b30"
205
+ ],
206
+ "expectTopK": 3,
207
+ "notes": "P1.1 scope 影响 — 目标 491c8b30 变体3 不带 scope(裸 retrieve)"
208
+ },
209
+ {
210
+ "name": "scope-dbcb317a-v1-WITH",
211
+ "query": "recallnest 本地两个目录反复踩坑怎么解决的",
212
+ "profile": "debug",
213
+ "limit": 8,
214
+ "targets": [
215
+ "dbcb317a"
216
+ ],
217
+ "expectTopK": 3,
218
+ "scope": "project:recallnest",
219
+ "notes": "P1.1 scope 影响 — 目标 dbcb317a 变体1 带 scope=project:recallnest"
220
+ },
221
+ {
222
+ "name": "scope-dbcb317a-v1-WITHOUT",
223
+ "query": "recallnest 本地两个目录反复踩坑怎么解决的",
224
+ "profile": "debug",
225
+ "limit": 8,
226
+ "targets": [
227
+ "dbcb317a"
228
+ ],
229
+ "expectTopK": 3,
230
+ "notes": "P1.1 scope 影响 — 目标 dbcb317a 变体1 不带 scope(裸 retrieve)"
231
+ },
232
+ {
233
+ "name": "scope-dbcb317a-v2-WITH",
234
+ "query": "~/recallnest 和 ~/Projects/recallnest 是什么关系",
235
+ "profile": "debug",
236
+ "limit": 8,
237
+ "targets": [
238
+ "dbcb317a"
239
+ ],
240
+ "expectTopK": 3,
241
+ "scope": "project:recallnest",
242
+ "notes": "P1.1 scope 影响 — 目标 dbcb317a 变体2 带 scope=project:recallnest"
243
+ },
244
+ {
245
+ "name": "scope-dbcb317a-v2-WITHOUT",
246
+ "query": "~/recallnest 和 ~/Projects/recallnest 是什么关系",
247
+ "profile": "debug",
248
+ "limit": 8,
249
+ "targets": [
250
+ "dbcb317a"
251
+ ],
252
+ "expectTopK": 3,
253
+ "notes": "P1.1 scope 影响 — 目标 dbcb317a 变体2 不带 scope(裸 retrieve)"
254
+ },
255
+ {
256
+ "name": "scope-dbcb317a-v3-WITH",
257
+ "query": "recallnest 用符号链接统一目录是怎么回事",
258
+ "profile": "debug",
259
+ "limit": 8,
260
+ "targets": [
261
+ "dbcb317a"
262
+ ],
263
+ "expectTopK": 3,
264
+ "scope": "project:recallnest",
265
+ "notes": "P1.1 scope 影响 — 目标 dbcb317a 变体3 带 scope=project:recallnest"
266
+ },
267
+ {
268
+ "name": "scope-dbcb317a-v3-WITHOUT",
269
+ "query": "recallnest 用符号链接统一目录是怎么回事",
270
+ "profile": "debug",
271
+ "limit": 8,
272
+ "targets": [
273
+ "dbcb317a"
274
+ ],
275
+ "expectTopK": 3,
276
+ "notes": "P1.1 scope 影响 — 目标 dbcb317a 变体3 不带 scope(裸 retrieve)"
277
+ }
278
+ ]