recallnest 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. package/.env +4 -0
  2. package/CLAUDE.md +1 -1
  3. package/README.md +30 -5
  4. package/README_CN.md +30 -5
  5. package/config.json +55 -0
  6. package/connectors/examples/email-adapter.ts +73 -0
  7. package/connectors/examples/logs-adapter.ts +103 -0
  8. package/connectors/examples/rss-adapter.ts +95 -0
  9. package/logs/daily-recall.error.log +0 -0
  10. package/logs/daily-recall.log +60 -0
  11. package/logs/ingest-2026-04-11.log +110 -0
  12. package/logs/ingest-2026-04-12.log +90 -0
  13. package/logs/ingest-2026-04-13.log +87 -0
  14. package/logs/launchagent-stderr.log +0 -0
  15. package/logs/launchagent-stdout.log +0 -0
  16. package/package.json +1 -1
  17. package/src/cli.ts +30 -0
  18. package/src/connector-types.ts +57 -0
  19. package/src/conversation-importer.ts +25 -1
  20. package/src/data-checkup.ts +31 -1
  21. package/src/doctor.ts +29 -0
  22. package/src/ingest.ts +203 -0
  23. package/src/mcp-server.ts +2 -2
  24. package/src/obsidian-connector.ts +282 -0
  25. package/src/session-schema.ts +2 -2
  26. package/src/source-heartbeat.ts +108 -0
  27. package/.claude-plugin/CLAUDE.md +0 -29
  28. package/.claude-plugin/marketplace.json +0 -18
  29. package/.dockerignore +0 -11
  30. package/.github/workflows/ci.yml +0 -28
  31. package/.mcp.json +0 -9
  32. package/CHANGELOG.md +0 -51
  33. package/DEMO.md +0 -94
  34. package/Dockerfile +0 -20
  35. package/EVAL.md +0 -89
  36. package/FAILURES.md +0 -75
  37. package/OPERATIONS.md +0 -59
  38. package/ROADMAP.md +0 -245
  39. package/SKILL.md +0 -250
  40. package/assets/dashboard.png +0 -0
  41. package/assets/knowledge-graph.png +0 -0
  42. package/assets/recallnest-workbench.png +0 -0
  43. package/assets/screenshots/ui-full.png +0 -0
  44. package/assets/screenshots/ui-overview.png +0 -0
  45. package/assets/screenshots/ui-search.png +0 -0
  46. package/assets/ui/app.js +0 -1031
  47. package/assets/ui/index.html +0 -224
  48. package/assets/ui/styles.css +0 -781
  49. package/assets/wechat-qr.jpg +0 -0
  50. package/bun.lock +0 -337
  51. package/daily-reflections/2026-03-05.md +0 -15
  52. package/daily-reflections/2026-03-06.md +0 -15
  53. package/daily-reflections/2026-03-07.md +0 -7
  54. package/daily-reflections/2026-03-08.md +0 -34
  55. package/daily-reflections/2026-03-09.md +0 -35
  56. package/daily-reflections/2026-03-10.md +0 -36
  57. package/daily-reflections/2026-03-11.md +0 -35
  58. package/daily-reflections/2026-03-12.md +0 -36
  59. package/daily-reflections/2026-03-13.md +0 -35
  60. package/daily-reflections/2026-03-14.md +0 -35
  61. package/daily-reflections/2026-03-15.md +0 -35
  62. package/daily-reflections/2026-03-16.md +0 -41
  63. package/daily-reflections/2026-03-17.md +0 -10
  64. package/daily-reflections/2026-03-18.md +0 -10
  65. package/daily-reflections/2026-03-19.md +0 -10
  66. package/daily-reflections/2026-03-20.md +0 -10
  67. package/daily-reflections/2026-03-21.md +0 -10
  68. package/daily-reflections/2026-03-22.md +0 -10
  69. package/daily-reflections/2026-03-23.md +0 -10
  70. package/daily-reflections/2026-03-24.md +0 -10
  71. package/daily-reflections/2026-03-25.md +0 -32
  72. package/daily-reflections/2026-03-26.md +0 -37
  73. package/daily-reflections/2026-03-27.md +0 -35
  74. package/daily-reflections/2026-03-28.md +0 -35
  75. package/daily-reflections/2026-03-29.md +0 -35
  76. package/daily-reflections/2026-03-30.md +0 -36
  77. package/daily-reflections/2026-03-31.md +0 -35
  78. package/daily-reflections/2026-04-01.md +0 -35
  79. package/daily-reflections/2026-04-02.md +0 -35
  80. package/daily-reflections/2026-04-03.md +0 -37
  81. package/daily-reflections/2026-04-04.md +0 -37
  82. package/daily-reflections/2026-04-05.md +0 -35
  83. package/daily-reflections/2026-04-06.md +0 -36
  84. package/daily-reflections/2026-04-07.md +0 -35
  85. package/daily-reflections/2026-04-08.md +0 -35
  86. package/daily-reflections/2026-04-12.md +0 -35
  87. package/data/alias-map.example.json +0 -22
  88. package/docs/api-reference.md +0 -1291
  89. package/docs/architecture.md +0 -168
  90. package/docs/memory-boundary-contract.md +0 -226
  91. package/docs/memory-categories.md +0 -119
  92. package/docs/pipeline-audit-ultramemory.md +0 -177
  93. package/docs/plans/2026-03-08-query-expansion.md +0 -270
  94. package/docs/plans/2026-03-16-continuity-layer.md +0 -196
  95. package/docs/plans/2026-04-01-recall-boost.md +0 -100
  96. package/docs/plans/mcp-tiering-plan.md +0 -126
  97. package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
  98. package/docs/self-evolution.md +0 -198
  99. package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
  100. package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
  101. package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
  102. package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
  103. package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
  104. package/eval/continuity/results-2026-03-16.md +0 -117
  105. package/eval/continuity/results-2026-03-19.md +0 -453
  106. package/eval/continuity/runbook.md +0 -110
  107. package/eval/continuity/seed-cases.md +0 -54
  108. package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
  109. package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
  110. package/eval/reports/2026-03-06-baseline.md +0 -82
  111. package/eval/reports/query-expansion.md +0 -138
  112. package/glama.json +0 -6
  113. package/lm +0 -19
  114. package/plan-philosophy-upgrades.md +0 -363
  115. package/plan-product-upgrades.md +0 -155
  116. package/plan-v1.2-distributable.md +0 -105
  117. package/plan.md +0 -140
  118. package/research-memory-evolution.md +0 -440
  119. package/research-rl-agent-memory.md +0 -330
  120. package/scripts/archive-distilled-junk.ts +0 -116
  121. package/scripts/backfill-anchors.ts +0 -130
  122. package/scripts/backfill-l0l1l2.ts +0 -98
  123. package/scripts/blind-scan.ts +0 -588
  124. package/scripts/cleanup-step1-exact-dups.ts +0 -117
  125. package/scripts/cleanup-step2-template-dups.ts +0 -113
  126. package/scripts/cleanup-step3-stale-content.ts +0 -149
  127. package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
  128. package/scripts/delete-distilled-junk.ts +0 -88
  129. package/scripts/diagnose-vector-search.ts +0 -132
  130. package/scripts/distill-facts.ts +0 -305
  131. package/scripts/health-check.ts +0 -343
  132. package/scripts/incremental-ingest.sh +0 -71
  133. package/scripts/migrate-language.ts +0 -75
  134. package/scripts/mlp-health-check.ts +0 -382
  135. package/scripts/pull-claude-desktop.ts +0 -515
  136. package/scripts/re-ingest-cc.sh +0 -66
  137. package/scripts/rebalance-memory-health.ts +0 -181
  138. package/scripts/run-desktop-ingest.sh +0 -11
  139. package/scripts/start-server.sh +0 -12
  140. package/scripts/verify-emotion-decay.ts +0 -148
  141. package/scripts/viz-dashboard.html +0 -423
  142. package/scripts/viz-export.ts +0 -703
  143. package/src/__tests__/access-tracker-novelty.test.ts +0 -228
  144. package/src/__tests__/activity-counter.test.ts +0 -96
  145. package/src/__tests__/admission-control.test.ts +0 -206
  146. package/src/__tests__/anchor-generator.test.ts +0 -67
  147. package/src/__tests__/auto-consolidation.test.ts +0 -168
  148. package/src/__tests__/auto-recall.test.ts +0 -131
  149. package/src/__tests__/babel-integration.test.ts +0 -139
  150. package/src/__tests__/batch-dedup.test.ts +0 -65
  151. package/src/__tests__/capture-engine.test.ts +0 -1144
  152. package/src/__tests__/capture-heuristic.test.ts +0 -171
  153. package/src/__tests__/cascade-forget.test.ts +0 -210
  154. package/src/__tests__/category-min-scores.test.ts +0 -33
  155. package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
  156. package/src/__tests__/circuit-breaker.test.ts +0 -99
  157. package/src/__tests__/cluster-consolidation.test.ts +0 -712
  158. package/src/__tests__/confidence-metadata.test.ts +0 -125
  159. package/src/__tests__/confidence-tracker.test.ts +0 -249
  160. package/src/__tests__/conflict-advisor.test.ts +0 -206
  161. package/src/__tests__/conflict-escalation.test.ts +0 -149
  162. package/src/__tests__/conflict-lifecycle.test.ts +0 -83
  163. package/src/__tests__/conflict-output.test.ts +0 -85
  164. package/src/__tests__/conflict-store.test.ts +0 -374
  165. package/src/__tests__/consolidation-engine.test.ts +0 -181
  166. package/src/__tests__/constructive-retrieval.test.ts +0 -509
  167. package/src/__tests__/context-collapse-renderer.test.ts +0 -214
  168. package/src/__tests__/context-composer-pins.test.ts +0 -64
  169. package/src/__tests__/context-composer-task-selection.test.ts +0 -24
  170. package/src/__tests__/context-reconstructor.test.ts +0 -65
  171. package/src/__tests__/context-renderer.test.ts +0 -91
  172. package/src/__tests__/continuity-eval.test.ts +0 -275
  173. package/src/__tests__/continuity-schema.test.ts +0 -97
  174. package/src/__tests__/conversation-importer.test.ts +0 -692
  175. package/src/__tests__/core-recall-eval.test.ts +0 -235
  176. package/src/__tests__/core-summary.test.ts +0 -109
  177. package/src/__tests__/data-checkup.test.ts +0 -225
  178. package/src/__tests__/decay-exemption.test.ts +0 -84
  179. package/src/__tests__/dedup-actions.test.ts +0 -58
  180. package/src/__tests__/dedup-guard.test.ts +0 -39
  181. package/src/__tests__/deterministic-id.test.ts +0 -39
  182. package/src/__tests__/distill-lock.test.ts +0 -176
  183. package/src/__tests__/doctor.test.ts +0 -303
  184. package/src/__tests__/dream-pipeline.test.ts +0 -232
  185. package/src/__tests__/drill-down.test.ts +0 -108
  186. package/src/__tests__/embedder.test.ts +0 -122
  187. package/src/__tests__/emotion-decay.test.ts +0 -46
  188. package/src/__tests__/emotion-detector.test.ts +0 -56
  189. package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
  190. package/src/__tests__/emotion-retrieval.test.ts +0 -56
  191. package/src/__tests__/entity-resolver.test.ts +0 -58
  192. package/src/__tests__/ethics-layer.test.ts +0 -476
  193. package/src/__tests__/event-segmenter.test.ts +0 -153
  194. package/src/__tests__/feature-flags-integration.test.ts +0 -54
  195. package/src/__tests__/frequency-tracker.test.ts +0 -113
  196. package/src/__tests__/graph-export.test.ts +0 -489
  197. package/src/__tests__/hotness-blend.test.ts +0 -76
  198. package/src/__tests__/ingest.test.ts +0 -360
  199. package/src/__tests__/interference-detector.test.ts +0 -203
  200. package/src/__tests__/kg-extractor.test.ts +0 -184
  201. package/src/__tests__/kg-store.test.ts +0 -198
  202. package/src/__tests__/llm-consolidation.test.ts +0 -226
  203. package/src/__tests__/mcp-efficiency.test.ts +0 -288
  204. package/src/__tests__/memory-boundaries.test.ts +0 -237
  205. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
  206. package/src/__tests__/memory-evolution.test.ts +0 -271
  207. package/src/__tests__/memory-health-rebalance.test.ts +0 -144
  208. package/src/__tests__/memory-lint.test.ts +0 -271
  209. package/src/__tests__/memory-output.test.ts +0 -215
  210. package/src/__tests__/multi-vector.test.ts +0 -242
  211. package/src/__tests__/narrative-architecture.test.ts +0 -456
  212. package/src/__tests__/pending-review.test.ts +0 -77
  213. package/src/__tests__/post-retrieval-filter.test.ts +0 -178
  214. package/src/__tests__/ppr-traversal.test.ts +0 -256
  215. package/src/__tests__/prediction-engine.test.ts +0 -349
  216. package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
  217. package/src/__tests__/preference-guard-smoke.test.ts +0 -211
  218. package/src/__tests__/preference-matcher.test.ts +0 -230
  219. package/src/__tests__/preference-slots.test.ts +0 -205
  220. package/src/__tests__/prospective-memory.test.ts +0 -283
  221. package/src/__tests__/query-entity-detector.test.ts +0 -171
  222. package/src/__tests__/query-expander-alias.test.ts +0 -89
  223. package/src/__tests__/query-expander.test.ts +0 -34
  224. package/src/__tests__/recall-governor.test.ts +0 -227
  225. package/src/__tests__/recall-mode.test.ts +0 -64
  226. package/src/__tests__/result-synthesizer.test.ts +0 -198
  227. package/src/__tests__/resume-context.test.ts +0 -4898
  228. package/src/__tests__/retrieval-trace.test.ts +0 -213
  229. package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
  230. package/src/__tests__/retroactive-boost.test.ts +0 -223
  231. package/src/__tests__/salience-scoring.test.ts +0 -106
  232. package/src/__tests__/scope-inventory.test.ts +0 -200
  233. package/src/__tests__/scope-policy.test.ts +0 -69
  234. package/src/__tests__/session-checkpoint.test.ts +0 -259
  235. package/src/__tests__/session-distiller.test.ts +0 -611
  236. package/src/__tests__/session-gc.test.ts +0 -215
  237. package/src/__tests__/skill-engine.test.ts +0 -546
  238. package/src/__tests__/skill-promotion.test.ts +0 -349
  239. package/src/__tests__/store-scope-audit.test.ts +0 -81
  240. package/src/__tests__/supersede-chain.test.ts +0 -154
  241. package/src/__tests__/temporal-parser.test.ts +0 -241
  242. package/src/__tests__/temporal-validity.test.ts +0 -160
  243. package/src/__tests__/term-registry.test.ts +0 -121
  244. package/src/__tests__/tool-output-compressor.test.ts +0 -133
  245. package/src/__tests__/topic-tag.test.ts +0 -113
  246. package/src/__tests__/trusted-memory.test.ts +0 -828
  247. package/src/__tests__/version-manager.test.ts +0 -213
  248. package/src/__tests__/workflow-observation.test.ts +0 -400
  249. package/src/__tests__/write-verifier.test.ts +0 -122
package/src/mcp-server.ts CHANGED
@@ -1819,11 +1819,11 @@ registerTool(
1819
1819
 
1820
1820
  registerTool(
1821
1821
  "import_conversations",
1822
- "Import a conversation file (Claude Code JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, or plaintext) into memory. Auto-detects format or use explicit format parameter. Messages are normalized and stored via the standard persistMemory pipeline.",
1822
+ "Import a conversation file (Claude Code JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, plaintext, or connector-v1 JSON) into memory. Auto-detects format or use explicit format parameter. Messages are normalized and stored via the standard persistMemory pipeline. For connector-v1 format, use the standard ConnectorOutputV1 schema (see docs/connector-spec.md).",
1823
1823
  {
1824
1824
  content: z.string().min(1).max(500_000).describe("Raw file content to import"),
1825
1825
  scope: z.string().min(1).max(160).describe("Target scope for imported memories, e.g. 'project:myapp'"),
1826
- format: z.enum(["auto", "claude-code", "claude-ai", "chatgpt", "slack", "plaintext"]).default("auto").describe("Conversation format. Use 'auto' to detect automatically."),
1826
+ format: z.enum(["auto", "claude-code", "claude-ai", "chatgpt", "slack", "plaintext", "connector-v1"]).default("auto").describe("Conversation format. Use 'auto' to detect automatically. 'connector-v1' for standard connector output."),
1827
1827
  },
1828
1828
  async ({ content, scope, format }) => {
1829
1829
  const { detectFormat, normalizeConversation, ingestNormalizedMessages } = await import("./conversation-importer.js");
@@ -0,0 +1,282 @@
1
+ /**
2
+ * Obsidian Vault Connector (GB-1)
3
+ *
4
+ * Scans an Obsidian vault directory and produces a ConnectorOutputV1 payload.
5
+ * Recognizes:
6
+ * - .obsidian/ directory (vault detection)
7
+ * - YAML frontmatter (tags, aliases, arbitrary metadata)
8
+ * - [[wikilinks]] (extracted as entity associations via link: tags)
9
+ * - Directory structure (folder: tags for scope/topic mapping)
10
+ * - Content hash (SHA-256 prefix) for incremental sync
11
+ */
12
+
13
+ import { createHash } from "node:crypto";
14
+ import { readFileSync, readdirSync, existsSync, statSync } from "node:fs";
15
+ import { join, relative, basename, dirname } from "node:path";
16
+ import type { ConnectorOutputV1, ConnectorRecord } from "./connector-types.js";
17
+
18
+ // ============================================================================
19
+ // Vault Detection
20
+ // ============================================================================
21
+
22
+ /** Check if a directory is an Obsidian vault (contains .obsidian/ folder) */
23
+ export function isObsidianVault(dirPath: string): boolean {
24
+ try {
25
+ return existsSync(join(dirPath, ".obsidian"));
26
+ } catch {
27
+ return false;
28
+ }
29
+ }
30
+
31
+ // ============================================================================
32
+ // YAML Frontmatter Parser
33
+ // ============================================================================
34
+
35
+ const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)(?:\r?\n)?---/;
36
+
37
+ export interface ObsidianFrontmatter {
38
+ tags?: string[];
39
+ aliases?: string[];
40
+ [key: string]: unknown;
41
+ }
42
+
43
+ /**
44
+ * Parse YAML frontmatter from markdown content.
45
+ * Lightweight parser — handles:
46
+ * - key: value (strings)
47
+ * - key: [inline, list]
48
+ * - key:\n - list\n - items
49
+ * Does NOT handle nested objects, multi-line strings, or anchors.
50
+ * Returns null frontmatter on parse failure (never throws).
51
+ */
52
+ export function parseFrontmatter(content: string): {
53
+ frontmatter: ObsidianFrontmatter | null;
54
+ body: string;
55
+ } {
56
+ const match = content.match(FRONTMATTER_RE);
57
+ if (!match) return { frontmatter: null, body: content };
58
+
59
+ const yamlBlock = match[1];
60
+ const body = content.slice(match[0].length).trimStart();
61
+
62
+ try {
63
+ const fm: ObsidianFrontmatter = {};
64
+ let currentKey: string | null = null;
65
+ let listAccum: string[] = [];
66
+
67
+ function flushList(): void {
68
+ if (currentKey && listAccum.length > 0) {
69
+ fm[currentKey] = listAccum;
70
+ listAccum = [];
71
+ }
72
+ currentKey = null;
73
+ }
74
+
75
+ for (const line of yamlBlock.split("\n")) {
76
+ // List item: " - value"
77
+ const listItem = line.match(/^\s+-\s+(.+)/);
78
+ if (listItem && currentKey) {
79
+ listAccum.push(listItem[1].trim().replace(/^['"]|['"]$/g, ""));
80
+ continue;
81
+ }
82
+
83
+ // Key-value: "key: value"
84
+ const kv = line.match(/^([\w][\w-]*):\s*(.*)/);
85
+ if (kv) {
86
+ flushList();
87
+ const [, key, rawVal] = kv;
88
+ const val = rawVal.trim();
89
+
90
+ if (val === "" || val === "[]") {
91
+ // Empty value or empty inline list — start list accumulator
92
+ currentKey = key;
93
+ listAccum = [];
94
+ } else if (val.startsWith("[") && val.endsWith("]")) {
95
+ // Inline list: [a, b, c]
96
+ fm[key] = val
97
+ .slice(1, -1)
98
+ .split(",")
99
+ .map((s) => s.trim().replace(/^['"]|['"]$/g, ""))
100
+ .filter(Boolean);
101
+ } else {
102
+ // Simple string value
103
+ fm[key] = val.replace(/^['"]|['"]$/g, "");
104
+ }
105
+ }
106
+ }
107
+ flushList();
108
+
109
+ return {
110
+ frontmatter: Object.keys(fm).length > 0 ? fm : null,
111
+ body,
112
+ };
113
+ } catch {
114
+ // Parse failure — return body without frontmatter
115
+ return { frontmatter: null, body };
116
+ }
117
+ }
118
+
119
+ // ============================================================================
120
+ // Wikilink Extraction
121
+ // ============================================================================
122
+
123
+ const WIKILINK_PATTERN = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
124
+
125
+ export interface WikiLink {
126
+ /** Link target (file name without extension) */
127
+ target: string;
128
+ /** Optional display text (after |) */
129
+ displayText?: string;
130
+ }
131
+
132
+ /**
133
+ * Extract [[wikilinks]] from markdown text.
134
+ * Handles [[target]] and [[target|display text]].
135
+ * Does NOT handle embed syntax ![[embed]] (those are skipped).
136
+ */
137
+ export function extractWikiLinks(text: string): WikiLink[] {
138
+ const links: WikiLink[] = [];
139
+ // Create a fresh regex to avoid stale lastIndex
140
+ const re = new RegExp(WIKILINK_PATTERN.source, WIKILINK_PATTERN.flags);
141
+ let match: RegExpExecArray | null;
142
+
143
+ while ((match = re.exec(text)) !== null) {
144
+ // Skip embed syntax: character before [[ is !
145
+ if (match.index > 0 && text[match.index - 1] === "!") continue;
146
+
147
+ links.push({
148
+ target: match[1].trim(),
149
+ ...(match[2] ? { displayText: match[2].trim() } : {}),
150
+ });
151
+ }
152
+
153
+ return links;
154
+ }
155
+
156
+ // ============================================================================
157
+ // Content Hash
158
+ // ============================================================================
159
+
160
+ /** SHA-256 first 16 hex chars — compact but collision-safe for incremental sync */
161
+ export function contentHash(text: string): string {
162
+ return createHash("sha256").update(text).digest("hex").slice(0, 16);
163
+ }
164
+
165
+ // ============================================================================
166
+ // Vault Scanner
167
+ // ============================================================================
168
+
169
+ export interface ScanVaultOptions {
170
+ /** Scope prefix, defaults to "vault:<vault-folder-name>" */
171
+ scopePrefix?: string;
172
+ /** Directories to exclude (defaults: .obsidian, .trash, .git, node_modules) */
173
+ excludeDirs?: string[];
174
+ /** Minimum file content length to include (default: 30 chars) */
175
+ minContentLength?: number;
176
+ }
177
+
178
+ /**
179
+ * Scan an Obsidian vault and produce a ConnectorOutputV1 payload.
180
+ *
181
+ * - Recursively walks .md files (skipping excluded directories)
182
+ * - Parses YAML frontmatter -> tags
183
+ * - Extracts [[wikilinks]] -> link:<target> tags
184
+ * - Maps folder structure -> folder:<path> tags
185
+ * - Computes SHA-256 content hash for incremental sync
186
+ */
187
+ export function scanVault(
188
+ vaultPath: string,
189
+ options?: ScanVaultOptions,
190
+ ): ConnectorOutputV1 {
191
+ const vaultName = basename(vaultPath);
192
+ const scopePrefix = options?.scopePrefix ?? `vault:${vaultName}`;
193
+ const defaultExclude = [".obsidian", ".trash", ".git", "node_modules"];
194
+ const excludeDirs = new Set(options?.excludeDirs ?? defaultExclude);
195
+ const minLen = options?.minContentLength ?? 30;
196
+
197
+ const records: ConnectorRecord[] = [];
198
+
199
+ function walk(dir: string): void {
200
+ let entries: ReturnType<typeof readdirSync>;
201
+ try {
202
+ entries = readdirSync(dir, { withFileTypes: true });
203
+ } catch {
204
+ return; // Permission denied or broken symlink — skip
205
+ }
206
+
207
+ for (const entry of entries) {
208
+ if (excludeDirs.has(entry.name)) continue;
209
+
210
+ const full = join(dir, entry.name);
211
+
212
+ if (entry.isDirectory()) {
213
+ walk(full);
214
+ continue;
215
+ }
216
+
217
+ if (!entry.name.endsWith(".md")) continue;
218
+
219
+ let raw: string;
220
+ try {
221
+ raw = readFileSync(full, "utf-8");
222
+ } catch {
223
+ continue; // Unreadable file — skip
224
+ }
225
+
226
+ if (raw.trim().length < minLen) continue;
227
+
228
+ const { frontmatter, body } = parseFrontmatter(raw);
229
+ const wikiLinks = extractWikiLinks(body);
230
+ const relPath = relative(vaultPath, full);
231
+ const folder = dirname(relPath);
232
+
233
+ // --- Build tags ---
234
+ const tags: string[] = [];
235
+
236
+ // Frontmatter tags
237
+ if (frontmatter?.tags) {
238
+ const fmTags = Array.isArray(frontmatter.tags)
239
+ ? frontmatter.tags
240
+ : [String(frontmatter.tags)];
241
+ for (const t of fmTags) {
242
+ tags.push(String(t));
243
+ }
244
+ }
245
+
246
+ // Wikilink entity associations
247
+ for (const link of wikiLinks) {
248
+ tags.push(`link:${link.target}`);
249
+ }
250
+
251
+ // Folder path
252
+ if (folder !== ".") {
253
+ tags.push(`folder:${folder}`);
254
+ }
255
+
256
+ records.push({
257
+ id: `obsidian:${relPath}`,
258
+ text: body,
259
+ title: basename(entry.name, ".md"),
260
+ tags,
261
+ contentHash: contentHash(raw),
262
+ timestamp: new Date(statSync(full).mtimeMs).toISOString(),
263
+ sourceMetadata: {
264
+ vaultPath,
265
+ relativePath: relPath,
266
+ ...(frontmatter ? { frontmatter } : {}),
267
+ ...(wikiLinks.length > 0 ? { wikiLinks } : {}),
268
+ },
269
+ });
270
+ }
271
+ }
272
+
273
+ walk(vaultPath);
274
+
275
+ return {
276
+ version: "connector-v1",
277
+ source: "obsidian",
278
+ scope: scopePrefix,
279
+ producedAt: new Date().toISOString(),
280
+ records,
281
+ };
282
+ }
@@ -66,9 +66,9 @@ export const ResumeContextResponseSchema = z.object({
66
66
  essentialContext: EssentialContextSchema.optional(),
67
67
  latestCheckpoint: ResumeCheckpointSummarySchema.optional(),
68
68
  /** CC-1: Hint for where to inject recalled context in the prompt. */
69
- injectionHint: z.enum(["system_prompt", "user_attachment"]).default("user_attachment").optional(),
69
+ injectionHint: z.enum(["system_prompt", "user_attachment"]).optional().default("user_attachment"),
70
70
  /** Upstream #345: Mark recalled context as ephemeral — host should discard on compaction, not persist to transcript. */
71
- ephemeral: z.boolean().default(true).optional(),
71
+ ephemeral: z.boolean().optional().default(true),
72
72
  responseMode: ResumeResponseModeSchema.default("default"),
73
73
  responseGuidance: optionalBoundedStringSchema(400),
74
74
  /** Constructive retrieval: LLM-synthesized reconstruction of context. */
@@ -0,0 +1,108 @@
1
+ /**
2
+ * GB-3: Source Heartbeat Tracking
3
+ *
4
+ * Every connector ingest writes a heartbeat record to data/source-heartbeat.json.
5
+ * data_checkup and doctor --ci read heartbeats to detect stale data sources.
6
+ */
7
+
8
+ import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
9
+ import { resolve, dirname } from "node:path";
10
+ import { metaDir } from "./compat.js";
11
+
12
+ // ---------------------------------------------------------------------------
13
+ // Types
14
+ // ---------------------------------------------------------------------------
15
+
16
+ export interface SourceHeartbeat {
17
+ source: string;
18
+ lastIngest: string; // ISO 8601
19
+ recordsIngested: number;
20
+ errors: string[];
21
+ }
22
+
23
+ export type HeartbeatFile = Record<string, SourceHeartbeat>;
24
+
25
+ export interface StaleSource {
26
+ source: string;
27
+ lastIngest: string;
28
+ daysSince: number;
29
+ }
30
+
31
+ // ---------------------------------------------------------------------------
32
+ // Path
33
+ // ---------------------------------------------------------------------------
34
+
35
+ const HEARTBEAT_PATH = resolve(metaDir(import.meta), "../data/source-heartbeat.json");
36
+
37
+ /** Visible for testing — override the default path. */
38
+ export function heartbeatPath(): string {
39
+ return HEARTBEAT_PATH;
40
+ }
41
+
42
+ // ---------------------------------------------------------------------------
43
+ // Read / Write
44
+ // ---------------------------------------------------------------------------
45
+
46
+ export function readHeartbeats(path?: string): HeartbeatFile {
47
+ const p = path ?? HEARTBEAT_PATH;
48
+ if (!existsSync(p)) return {};
49
+ try {
50
+ return JSON.parse(readFileSync(p, "utf-8")) as HeartbeatFile;
51
+ } catch {
52
+ return {};
53
+ }
54
+ }
55
+
56
+ export function writeHeartbeat(
57
+ source: string,
58
+ recordsIngested: number,
59
+ errors: string[],
60
+ path?: string,
61
+ ): void {
62
+ const p = path ?? HEARTBEAT_PATH;
63
+ const dir = dirname(p);
64
+ if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
65
+
66
+ const existing = readHeartbeats(p);
67
+ existing[source] = {
68
+ source,
69
+ lastIngest: new Date().toISOString(),
70
+ recordsIngested,
71
+ errors: errors.slice(0, 10), // cap stored errors
72
+ };
73
+ writeFileSync(p, JSON.stringify(existing, null, 2) + "\n");
74
+ }
75
+
76
+ // ---------------------------------------------------------------------------
77
+ // Staleness check
78
+ // ---------------------------------------------------------------------------
79
+
80
+ const MS_PER_DAY = 86_400_000;
81
+
82
+ export function checkSourceStaleness(maxDays = 7, path?: string): StaleSource[] {
83
+ const heartbeats = readHeartbeats(path);
84
+ const now = Date.now();
85
+ const stale: StaleSource[] = [];
86
+
87
+ for (const hb of Object.values(heartbeats)) {
88
+ const lastMs = Date.parse(hb.lastIngest);
89
+ if (Number.isNaN(lastMs)) continue;
90
+ const daysSince = Math.floor((now - lastMs) / MS_PER_DAY);
91
+ if (daysSince > maxDays) {
92
+ stale.push({ source: hb.source, lastIngest: hb.lastIngest, daysSince });
93
+ }
94
+ }
95
+
96
+ return stale.sort((a, b) => b.daysSince - a.daysSince);
97
+ }
98
+
99
+ /** Human-readable age string, e.g. "2h ago", "3d ago". */
100
+ export function formatAge(isoDate: string): string {
101
+ const ms = Date.now() - Date.parse(isoDate);
102
+ if (ms < 0) return "just now";
103
+ const hours = Math.floor(ms / 3_600_000);
104
+ if (hours < 1) return "just now";
105
+ if (hours < 24) return `${hours}h ago`;
106
+ const days = Math.floor(hours / 24);
107
+ return `${days}d ago`;
108
+ }
@@ -1,29 +0,0 @@
1
- # RecallNest Memory Plugin
2
-
3
- RecallNest is a local-first shared memory layer backed by LanceDB. It stores and recalls context across sessions, terminals, and agents (Claude Code, Codex, Gemini CLI).
4
-
5
- **Prerequisite**: Requires [Bun](https://bun.sh) runtime. On first start, dependencies install automatically.
6
-
7
- ## Tool Quick Reference
8
-
9
- | When | Tool | Purpose |
10
- |------|------|---------|
11
- | Session opens, user says continue/接着/上个窗口 | `resume_context` | Load stable context, latest checkpoint, recent cases |
12
- | Before answering about prior work / known facts | `search_memory` | Targeted fact/pattern recall (2-3 key nouns) |
13
- | After solving something significant | `store_memory` or `store_case` | Persist durable knowledge |
14
- | Before closing a window with unfinished work | `checkpoint_session` | Save current state for next session |
15
- | After discovering a reusable workflow | `store_workflow_pattern` | Persist as durable pattern |
16
- | Memory feels stale or conflicted | `consolidate_memories` | Merge near-duplicates (dry-run by default) |
17
-
18
- ## Continuity Rules
19
-
20
- - If the user says `continue`, `继续`, `接着`, `刚才`, `上个窗口`, `不要让我重复前情`, or asks where you stopped / what to do next — call `resume_context` before any Bash, Read, or repo exploration.
21
- - If the user starts a concrete task in an active project or asks for prior implementation detail — do lightweight RecallNest recovery before repo exploration even without an explicit `continue`.
22
- - Do not substitute `git status`, `git log`, or reading local docs for `resume_context`. Local inspection can validate current code state only **after** continuity has been recovered.
23
- - When `scope` or `sessionId` is known, pass it into recall tools instead of relying on global recall.
24
- - If `resume_context` returns a resolved scope or checkpoint scope, reuse that same scope in follow-up `search_memory`, `brief_memory`, `pin_memory` calls.
25
- - If recalled context mentions repo state (modified files, pending push), treat it as **handoff context only** — do not restate as current state without verifying in this window.
26
- - Before leaving a window with resumable work, call `checkpoint_session`. Do not include repo state claims unless you actually inspected the repo this window.
27
- - Use `store_memory` for durable profile, preference, entity, or case knowledge.
28
- - Use `store_workflow_pattern` for reusable multi-step workflows.
29
- - If the user explicitly corrects a continuity miss, fix the workflow and call `workflow_observe` with `outcome: corrected`.
@@ -1,18 +0,0 @@
1
- {
2
- "name": "AliceLJY",
3
- "owner": {
4
- "name": "Alice Jin Ye"
5
- },
6
- "metadata": {
7
- "description": "RecallNest — shared memory layer for Claude Code, Codex, and Gemini CLI",
8
- "homepage": "https://github.com/AliceLJY/recallnest"
9
- },
10
- "plugins": [
11
- {
12
- "name": "recallnest",
13
- "version": "1.4.1",
14
- "source": "./",
15
- "description": "Local-first memory system: store, search, and recall context across sessions and terminals. Powered by LanceDB with hybrid retrieval."
16
- }
17
- ]
18
- }
package/.dockerignore DELETED
@@ -1,11 +0,0 @@
1
- node_modules/
2
- data/*
3
- !data/alias-map.example.json
4
- logs/
5
- .env
6
- .git/
7
- *.md
8
- src/__tests__/
9
- eval/
10
- integrations/
11
- scripts/
@@ -1,28 +0,0 @@
1
- name: CI
2
-
3
- on:
4
- push:
5
- branches: [main]
6
- pull_request:
7
- branches: [main]
8
-
9
- jobs:
10
- check:
11
- runs-on: ubuntu-latest
12
- steps:
13
- - uses: actions/checkout@v4
14
-
15
- - uses: oven-sh/setup-bun@v2
16
- with:
17
- bun-version: latest
18
-
19
- - run: bun install
20
-
21
- - name: Setup config
22
- run: cp config.json.example config.json
23
-
24
- - name: Doctor (CI mode)
25
- run: bun run src/cli.ts doctor --ci
26
-
27
- - name: TypeScript check
28
- run: bun build src/cli.ts --target bun --no-bundle --outdir /tmp/check 2>&1 || true
package/.mcp.json DELETED
@@ -1,9 +0,0 @@
1
- {
2
- "mcpServers": {
3
- "recallnest": {
4
- "type": "stdio",
5
- "command": "bash",
6
- "args": ["${CLAUDE_PLUGIN_ROOT}/scripts/start-server.sh"]
7
- }
8
- }
9
- }
package/CHANGELOG.md DELETED
@@ -1,51 +0,0 @@
1
- # Changelog
2
-
3
- ## v1.3.1 — Upstream Sync (2026-03-12)
4
-
5
- Synced with [CortexReach/memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) master (v1.1.0-beta.6+).
6
-
7
- ### Changed
8
-
9
- - **Retriever**: Added `source` field to `RetrievalContext` — access reinforcement now only fires on manual retrieval, preventing auto-recall from strengthening noise memories (synced from upstream beta.2 design).
10
- - **Noise filter**: Added Chinese meta-question patterns (`你记得`, `记不记得`, `还记得…吗`, `上次…说`, `之前…提到`) and diagnostic artifact filter (synced from upstream beta.3).
11
- - **README**: Updated upstream credit link from `win4r/memory-lancedb-pro` to `CortexReach/memory-lancedb-pro`, added CortexReach team acknowledgement.
12
-
13
- ## v1.2.0 — First Distributable Release (2026-03-08)
14
-
15
- The goal of this release: a new user can go from `git clone` to first search result in 15 minutes.
16
-
17
- ### New
18
-
19
- - **`lm doctor`** — one-command pre-flight check for Bun, config, API key, data directory, transcript paths, and index health. Supports `--ci` mode for GitHub Actions.
20
- - **`lm demo`** — run sample queries to see RecallNest in action before writing your own.
21
- - **`config.json.example`** — ships with absolute `~/.recallnest/data/lancedb` path. New users copy this instead of editing the tracked config.
22
- - **GitHub Actions CI** — runs `doctor --ci` and TypeScript check on every push.
23
- - **Ingest pre-validation** — embedding API is tested before processing any files. Invalid Jina key now fails fast with a clear message instead of crashing mid-ingest.
24
-
25
- ### Changed
26
-
27
- - **README rewritten** — added Prerequisites table (Bun + Jina key), 5-step quickstart with expected output, Troubleshooting section.
28
- - **Gemini support marked "coming soon"** — README, config example, and doctor all honestly reflect that Gemini CLI sessions are encrypted protobuf and not yet parseable. The `lm ingest` command prints a clear skip message instead of silently failing.
29
- - **Config path robustness** — default `dbPath` changed from relative `./data/lancedb` to absolute `~/.recallnest/data/lancedb` in config example. Auto-detect failure messages now include the user's actual home path.
30
- - **`config.json` untracked** — added to `.gitignore` so user config is not overwritten by `git pull`.
31
-
32
- ### Fixed
33
-
34
- - Auto-detect hint in `doctor` now shows a real example path based on the current user's home directory.
35
- - `findConfigPath()` error message now suggests `cp config.json.example config.json` when the example file exists.
36
-
37
- ## v1.1.0 — Hybrid Retrieval + MCP + UI (2026-02)
38
-
39
- - Hybrid retrieval: LanceDB vector + BM25 keyword search with configurable weights
40
- - Retrieval profiles: `default`, `writing`, `debug`, `fact-check`
41
- - MCP server with 9 tools: search, explain, distill, brief, pin, list assets/pins, export, stats
42
- - Local web workbench UI at `http://localhost:4317`
43
- - Multi-source ingest: Claude Code transcripts, Codex sessions, Gemini sessions, markdown notes
44
- - Asset system: pin, brief, export with re-indexing
45
- - Time-aware scoring with configurable decay
46
-
47
- ## v1.0.0 — Initial Release (2026-01)
48
-
49
- - Basic vector search over Claude Code transcripts
50
- - LanceDB storage with Jina embeddings
51
- - CLI interface
package/DEMO.md DELETED
@@ -1,94 +0,0 @@
1
- # RecallNest 30-Second Demo
2
-
3
- Use this script when you want to explain RecallNest fast on GitHub, X, Telegram, or in a live screen recording.
4
-
5
- ## Demo Goal
6
-
7
- Show three things in one pass:
8
-
9
- | What to prove | What the viewer should understand |
10
- |------|------|
11
- | Recall | old AI conversations can be found quickly |
12
- | Explainability | hits are not opaque |
13
- | Reusability | useful context can become a long-lived asset |
14
-
15
- ## Fast Setup
16
-
17
- ```bash
18
- cd /Users/anxianjingya/recallnest
19
- bun run src/ui-server.ts
20
- ```
21
-
22
- Open:
23
-
24
- ```text
25
- http://localhost:4317
26
- ```
27
-
28
- ## Recommended Demo Query
29
-
30
- Use:
31
-
32
- ```text
33
- telegram bridge
34
- ```
35
-
36
- Reason:
37
-
38
- | Why this query works | Result |
39
- |------|------|
40
- | already appears in the current dataset | likely to return hits immediately |
41
- | has operational context | good for showing explainability |
42
- | already has assets | good for showing reuse |
43
-
44
- ## 30-Second Talk Track
45
-
46
- ### Version A
47
-
48
- ```text
49
- This is RecallNest, a local-first memory workbench for AI conversations.
50
- It turns Claude Code, Codex, Gemini, and notes into a recall layer.
51
- I can search an old topic, inspect why it matched, then pin useful context back into memory so it can be recalled again later.
52
- The same memory layer is exposed through MCP, so an agent can use it and I can still verify it in the UI.
53
- ```
54
-
55
- ### Version B
56
-
57
- ```text
58
- RecallNest is not just transcript search.
59
- It lets me search, explain, distill, and reuse past AI conversations.
60
- The key idea is that useful hits do not disappear again: I can turn them into assets and feed them back into recall.
61
- ```
62
-
63
- ## Demo Click Path
64
-
65
- | Step | UI action | What to say |
66
- |------|------|------|
67
- | 1 | show the loaded `telegram bridge` query | "This starts from a real transcript topic." |
68
- | 2 | click `Search` | "I can recall the relevant conversation history." |
69
- | 3 | click `Details` on one result | "The hit is inspectable, not a black box." |
70
- | 4 | click `Pin` or `Brief` | "Useful context becomes a reusable asset." |
71
- | 5 | switch to `Assets` | "Assets stay visible and can be recalled later." |
72
-
73
- ## Screenshot Rules
74
-
75
- | Rule | Why |
76
- |------|------|
77
- | keep the browser crop tight | the product, not Chrome, should dominate the frame |
78
- | show real hits in the result area | empty states spread poorly |
79
- | keep `Trace Output` collapsed unless you are explaining it | it distracts from the main product story |
80
- | prefer one strong screenshot over many average ones | easier to share and understand |
81
-
82
- ## If The UI Opens Empty
83
-
84
- Check:
85
-
86
- 1. the server is running
87
- 2. the current dataset was ingested
88
- 3. the page has been refreshed after startup
89
-
90
- If needed, re-run:
91
-
92
- ```bash
93
- bun run src/ui-server.ts
94
- ```
package/Dockerfile DELETED
@@ -1,20 +0,0 @@
1
- FROM oven/bun:1-alpine AS base
2
- WORKDIR /app
3
-
4
- # Install dependencies
5
- COPY package.json ./
6
- RUN bun install --frozen-lockfile 2>/dev/null || bun install
7
-
8
- # Copy source
9
- COPY src/ src/
10
- COPY data/alias-map.example.json data/alias-map.example.json
11
-
12
- # Default environment
13
- ENV RECALLNEST_API_PORT=4318
14
- ENV RECALLNEST_UI_PORT=4317
15
-
16
- EXPOSE 4318 4317
17
-
18
- # MCP stdio mode (default) — Glama introspection connects here
19
- # Override with CMD ["bun","run","src/api-server.ts"] for HTTP mode
20
- CMD ["bun", "run", "src/mcp-server.ts"]