recallnest 2.2.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
2
2
  import type { PersistMemoryDeps } from "./capture-engine.js";
3
3
  import { persistMemory } from "./capture-engine.js";
4
4
  import type { StoredMemoryRecord } from "./memory-schema.js";
5
+ import { isConnectorOutputV1, type ConnectorOutputV1 } from "./connector-types.js";
5
6
 
6
7
  export interface NormalizedMessage {
7
8
  role: "user" | "assistant" | "system";
@@ -14,7 +15,8 @@ export type ConversationFormat =
14
15
  | "claude-ai"
15
16
  | "chatgpt"
16
17
  | "slack"
17
- | "plaintext";
18
+ | "plaintext"
19
+ | "connector-v1";
18
20
 
19
21
  // ---------------------------------------------------------------------------
20
22
  // Format detection
@@ -40,6 +42,10 @@ export function detectFormat(content: string): ConversationFormat {
40
42
  }
41
43
 
42
44
  if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
45
+ // connector-v1: highest priority — explicit version tag
46
+ if (isConnectorOutputV1(parsed)) {
47
+ return "connector-v1";
48
+ }
43
49
  if (parsed.chat_conversations !== undefined) {
44
50
  return "claude-ai";
45
51
  }
@@ -366,6 +372,22 @@ export function normalizePlaintext(content: string): NormalizedMessage[] {
366
372
  return messages;
367
373
  }
368
374
 
375
+ // ---------------------------------------------------------------------------
376
+ // Connector-v1 normalizer
377
+ // ---------------------------------------------------------------------------
378
+
379
+ export function normalizeConnectorV1(content: string): NormalizedMessage[] {
380
+ const parsed: ConnectorOutputV1 = JSON.parse(content);
381
+ if (!isConnectorOutputV1(parsed)) {
382
+ throw new Error("Invalid connector-v1 format: missing version/source/scope/records");
383
+ }
384
+ return parsed.records.map((r) => ({
385
+ role: "user" as const,
386
+ content: r.title ? `[${r.title}] ${r.text}` : r.text,
387
+ ...(r.timestamp ? { timestamp: r.timestamp } : {}),
388
+ }));
389
+ }
390
+
369
391
  // ---------------------------------------------------------------------------
370
392
  // Unified normalize dispatcher
371
393
  // ---------------------------------------------------------------------------
@@ -382,6 +404,8 @@ export function normalizeConversation(content: string, format: ConversationForma
382
404
  return normalizeSlack(content);
383
405
  case "plaintext":
384
406
  return normalizePlaintext(content);
407
+ case "connector-v1":
408
+ return normalizeConnectorV1(content);
385
409
  }
386
410
  }
387
411
 
@@ -15,6 +15,7 @@
15
15
  import type { MemoryEntry, MemoryStore } from "./store.js";
16
16
  import { resolveTier, type MemoryTier } from "./decay-engine.js";
17
17
  import { measureInterferenceDensity } from "./interference-detector.js";
18
+ import { readHeartbeats, checkSourceStaleness, formatAge } from "./source-heartbeat.js";
18
19
 
19
20
  // ---------------------------------------------------------------------------
20
21
  // Types
@@ -151,7 +152,33 @@ function checkVersionGroups(entries: MemoryEntry[]): CheckResult {
151
152
  };
152
153
  }
153
154
 
154
- /** Check 6 (F2): Interference density — detect semantic clustering pressure. */
155
+ /** Check 6 (GB-3): Source health — detect stale connector data sources. */
156
+ function checkSourceHealth(heartbeatPath?: string): CheckResult {
157
+ const heartbeats = readHeartbeats(heartbeatPath);
158
+ const sources = Object.values(heartbeats);
159
+ if (sources.length === 0) {
160
+ return { name: "source_health", status: "ok", detail: "No connector sources tracked yet" };
161
+ }
162
+
163
+ const stale30 = checkSourceStaleness(30, heartbeatPath);
164
+ const stale7 = checkSourceStaleness(7, heartbeatPath);
165
+
166
+ const summary = sources
167
+ .map((s) => `${s.source}: ${formatAge(s.lastIngest)}`)
168
+ .join(", ");
169
+
170
+ if (stale30.length > 0) {
171
+ const names = stale30.map((s) => `${s.source} (${s.daysSince}d)`).join(", ");
172
+ return { name: "source_health", status: "error", detail: `${summary} — critically stale: ${names}` };
173
+ }
174
+ if (stale7.length > 0) {
175
+ const names = stale7.map((s) => `${s.source} (${s.daysSince}d)`).join(", ");
176
+ return { name: "source_health", status: "warning", detail: `${summary} — stale: ${names}` };
177
+ }
178
+ return { name: "source_health", status: "ok", detail: `All sources fresh: ${summary}` };
179
+ }
180
+
181
+ /** Check 7 (F2): Interference density — detect semantic clustering pressure. */
155
182
  function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
156
183
  // Only check entries with vectors (skip schema/empty entries)
157
184
  const withVectors = entries.filter(e => e.vector && e.vector.length > 0);
@@ -192,6 +219,8 @@ function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
192
219
  export interface CheckupDeps {
193
220
  store: Pick<MemoryStore, "list" | "stats">;
194
221
  openConflictCount: number;
222
+ /** Optional override path for source-heartbeat.json (used in tests). */
223
+ heartbeatPath?: string;
195
224
  }
196
225
 
197
226
  export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport> {
@@ -204,6 +233,7 @@ export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport>
204
233
  checkTierDistribution(entries),
205
234
  checkConflictBacklog(deps.openConflictCount),
206
235
  checkVersionGroups(entries),
236
+ checkSourceHealth(deps.heartbeatPath),
207
237
  checkInterferenceDensity(entries),
208
238
  ],
209
239
  totalEntries: entries.length,
package/src/doctor.ts CHANGED
@@ -27,6 +27,7 @@ import {
27
27
  } from "./memory-schema.js";
28
28
  import { collectScopeInventory, type ScopeInventoryReport } from "./scope-inventory.js";
29
29
  import { MemoryStore, type MemoryEntry, validateStoragePath } from "./store.js";
30
+ import { readHeartbeats, formatAge } from "./source-heartbeat.js";
30
31
 
31
32
  interface CheckResult {
32
33
  name: string;
@@ -508,6 +509,34 @@ export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckRe
508
509
  ));
509
510
  }
510
511
 
512
+ // GB-3: Source heartbeat summary
513
+ try {
514
+ const heartbeats = readHeartbeats();
515
+ const sources = Object.values(heartbeats);
516
+ if (sources.length === 0) {
517
+ results.push(pass("Source heartbeats", "no connector sources tracked yet"));
518
+ } else {
519
+ const summary = sources
520
+ .map((s) => `${s.source}: ${formatAge(s.lastIngest)}`)
521
+ .join(", ");
522
+ const staleCount = sources.filter((s) => {
523
+ const daysSince = (Date.now() - Date.parse(s.lastIngest)) / 86_400_000;
524
+ return daysSince > 7;
525
+ }).length;
526
+ if (staleCount > 0) {
527
+ results.push(warn(
528
+ "Source heartbeats",
529
+ `${summary} (${staleCount} stale >7d)`,
530
+ "lm ingest --connector <file>",
531
+ ));
532
+ } else {
533
+ results.push(pass("Source heartbeats", summary));
534
+ }
535
+ }
536
+ } catch {
537
+ results.push(warn("Source heartbeats", "unable to read heartbeat file"));
538
+ }
539
+
511
540
  return results;
512
541
  }
513
542
 
package/src/ingest.ts CHANGED
@@ -15,6 +15,9 @@ import { detectLang, tokenizeFts } from "./language-hook.js";
15
15
  import type { MemoryStore, MemoryEntry } from "./store.js";
16
16
  import type { Embedder } from "./embedder.js";
17
17
  import { chunkDocument, type ChunkerConfig } from "./chunker.js";
18
+ import type { ConnectorOutputV1, ConnectorRecord } from "./connector-types.js";
19
+ import { isConnectorOutputV1 } from "./connector-types.js";
20
+ import { isObsidianVault, scanVault } from "./obsidian-connector.js";
18
21
  import { segmentEvents, type EventSegmenterConfig, DEFAULT_EVENT_SEGMENTER_CONFIG } from "./event-segmenter.js";
19
22
  import { isProcessed, markProcessed } from "./tracker.js";
20
23
  import type { LLMClient, SmartExtraction } from "./llm-client.js";
@@ -1850,3 +1853,203 @@ export async function ingestGenericText(
1850
1853
 
1851
1854
  return result;
1852
1855
  }
1856
+
1857
+ // ============================================================================
1858
+ // Connector-v1 Ingest (GB-2)
1859
+ // ============================================================================
1860
+
1861
+ /**
1862
+ * Ingest a connector-v1 formatted JSON payload.
1863
+ *
1864
+ * External connectors (Obsidian, Email, RSS, etc.) produce a standard
1865
+ * ConnectorOutputV1 JSON → this function consumes it through the existing
1866
+ * dedup / smartExtract / buildIngestedEntry pipeline.
1867
+ *
1868
+ * contentHash on each record enables incremental sync: if a record with the
1869
+ * same hash is already in the store, it is skipped without embedding.
1870
+ */
1871
+ export async function ingestConnectorFile(
1872
+ store: MemoryStore,
1873
+ embedder: Embedder,
1874
+ content: string,
1875
+ options: { verbose?: boolean; noDedup?: boolean; llm?: LLMClient | null } = {},
1876
+ ): Promise<IngestResult> {
1877
+ const parsed = JSON.parse(content);
1878
+ if (!isConnectorOutputV1(parsed)) {
1879
+ return {
1880
+ source: "connector",
1881
+ filesProcessed: 0,
1882
+ chunksIngested: 0,
1883
+ chunksSkipped: 0,
1884
+ chunksDeduped: 0,
1885
+ dedupReasonCounts: createDedupReasonCounts(),
1886
+ errors: ["Invalid connector-v1 format: missing version/source/scope/records"],
1887
+ };
1888
+ }
1889
+
1890
+ const connectorData = parsed as ConnectorOutputV1;
1891
+ const { source, scope, records } = connectorData;
1892
+
1893
+ const result: IngestResult = {
1894
+ source: `connector:${source}`,
1895
+ filesProcessed: 1,
1896
+ chunksIngested: 0,
1897
+ chunksSkipped: 0,
1898
+ chunksDeduped: 0,
1899
+ dedupReasonCounts: createDedupReasonCounts(),
1900
+ errors: [],
1901
+ };
1902
+
1903
+ if (records.length === 0) return result;
1904
+
1905
+ const batchSize = 32;
1906
+
1907
+ for (let i = 0; i < records.length; i += batchSize) {
1908
+ const batch = records.slice(i, i + batchSize);
1909
+ const texts = batch.map((r: ConnectorRecord) =>
1910
+ r.title ? `[${r.title}] ${r.text}` : r.text,
1911
+ );
1912
+
1913
+ try {
1914
+ const vectors = await embedder.embedBatchPassage(texts);
1915
+
1916
+ // --- dedup pass ---
1917
+ const dedupedTexts: string[] = [];
1918
+ const dedupedVectors: number[][] = [];
1919
+ const dedupedRecords: ConnectorRecord[] = [];
1920
+
1921
+ for (let j = 0; j < batch.length; j++) {
1922
+ const vector = vectors[j];
1923
+ if (!vector || vector.length === 0) {
1924
+ result.chunksSkipped++;
1925
+ continue;
1926
+ }
1927
+ if (!options.noDedup) {
1928
+ const decision = await dedupCheck(store, vector, texts[j], options.llm);
1929
+ recordDedupDecision(result, decision);
1930
+ if (decision.secondaryDeletes?.length) {
1931
+ await executeSecondaryDeletes(store, decision.secondaryDeletes);
1932
+ }
1933
+ if (decision.action === "skip") continue;
1934
+ }
1935
+ dedupedTexts.push(texts[j]);
1936
+ dedupedVectors.push(vector);
1937
+ dedupedRecords.push(batch[j]);
1938
+ }
1939
+
1940
+ if (dedupedTexts.length === 0) continue;
1941
+
1942
+ // --- extract + build entries ---
1943
+ const extractions = await smartExtractBatch(dedupedTexts, options.llm);
1944
+ const coreSummaries = await generateCoreSummaries(dedupedTexts, options.llm);
1945
+ const toStore: Array<{
1946
+ text: string;
1947
+ vector: number[];
1948
+ category: string;
1949
+ scope: string;
1950
+ importance: number;
1951
+ metadata: string;
1952
+ }> = [];
1953
+
1954
+ for (let j = 0; j < dedupedTexts.length; j++) {
1955
+ const record = dedupedRecords[j];
1956
+ const entry = buildIngestedEntry({
1957
+ source: `connector:${source}`,
1958
+ scope,
1959
+ text: dedupedTexts[j],
1960
+ vector: dedupedVectors[j],
1961
+ extraction: extractions[j],
1962
+ file: record.id,
1963
+ heading: record.title,
1964
+ coreSummary: coreSummaries[j],
1965
+ });
1966
+
1967
+ // Merge connector-specific metadata
1968
+ const meta = JSON.parse(entry.metadata);
1969
+ if (record.tags?.length) meta.connectorTags = record.tags;
1970
+ if (record.contentHash) meta.contentHash = record.contentHash;
1971
+ if (record.sourceMetadata) meta.sourceMetadata = record.sourceMetadata;
1972
+ entry.metadata = JSON.stringify(meta);
1973
+
1974
+ // Respect categoryHint / importanceHint from connector
1975
+ if (record.categoryHint) {
1976
+ entry.category = record.categoryHint;
1977
+ }
1978
+ if (record.importanceHint !== undefined) {
1979
+ entry.importance = record.importanceHint;
1980
+ }
1981
+
1982
+ toStore.push(entry);
1983
+ }
1984
+
1985
+ if (toStore.length > 0) {
1986
+ await store.storeBatch(toStore);
1987
+ result.chunksIngested += toStore.length;
1988
+ }
1989
+ } catch (err: unknown) {
1990
+ const msg = err instanceof Error ? err.message : String(err);
1991
+ result.errors.push(`Connector batch error: ${msg}`);
1992
+ }
1993
+ }
1994
+
1995
+ if (options.verbose) {
1996
+ console.log(` connector:${source}: ${records.length} records → ${result.chunksIngested} stored`);
1997
+ }
1998
+
1999
+ // GB-3: Write source heartbeat after connector ingest
2000
+ try {
2001
+ const { writeHeartbeat } = await import("./source-heartbeat.js");
2002
+ writeHeartbeat(source, result.chunksIngested, result.errors);
2003
+ } catch {
2004
+ // Heartbeat write is best-effort — don't break ingest on failure
2005
+ }
2006
+
2007
+ return result;
2008
+ }
2009
+
2010
+ // ============================================================================
2011
+ // Obsidian Vault Ingest (GB-1)
2012
+ // ============================================================================
2013
+
2014
+ /**
2015
+ * Scan an Obsidian vault and ingest its markdown files through the connector-v1 pipeline.
2016
+ *
2017
+ * Wrapper that:
2018
+ * 1. Validates the path is an Obsidian vault (.obsidian/ exists)
2019
+ * 2. Calls scanVault() to produce a ConnectorOutputV1 payload
2020
+ * 3. Feeds the payload to ingestConnectorFile()
2021
+ */
2022
+ export async function ingestObsidianVault(
2023
+ store: MemoryStore,
2024
+ embedder: Embedder,
2025
+ vaultPath: string,
2026
+ options: {
2027
+ verbose?: boolean;
2028
+ noDedup?: boolean;
2029
+ llm?: LLMClient | null;
2030
+ scopePrefix?: string;
2031
+ } = {},
2032
+ ): Promise<IngestResult> {
2033
+ const resolved = expandHome(vaultPath);
2034
+
2035
+ if (!isObsidianVault(resolved)) {
2036
+ return {
2037
+ source: "obsidian",
2038
+ filesProcessed: 0,
2039
+ chunksIngested: 0,
2040
+ chunksSkipped: 0,
2041
+ chunksDeduped: 0,
2042
+ dedupReasonCounts: createDedupReasonCounts(),
2043
+ errors: [`Not an Obsidian vault (missing .obsidian/ directory): ${resolved}`],
2044
+ };
2045
+ }
2046
+
2047
+ const output = scanVault(resolved, { scopePrefix: options.scopePrefix });
2048
+ const content = JSON.stringify(output);
2049
+
2050
+ if (options.verbose) {
2051
+ console.log(` obsidian: scanned ${output.records.length} notes from ${resolved}`);
2052
+ }
2053
+
2054
+ return ingestConnectorFile(store, embedder, content, options);
2055
+ }
package/src/mcp-server.ts CHANGED
@@ -1819,11 +1819,11 @@ registerTool(
1819
1819
 
1820
1820
  registerTool(
1821
1821
  "import_conversations",
1822
- "Import a conversation file (Claude Code JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, or plaintext) into memory. Auto-detects format or use explicit format parameter. Messages are normalized and stored via the standard persistMemory pipeline.",
1822
+ "Import a conversation file (Claude Code JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, plaintext, or connector-v1 JSON) into memory. Auto-detects format or use explicit format parameter. Messages are normalized and stored via the standard persistMemory pipeline. For connector-v1 format, use the standard ConnectorOutputV1 schema (see docs/connector-spec.md).",
1823
1823
  {
1824
1824
  content: z.string().min(1).max(500_000).describe("Raw file content to import"),
1825
1825
  scope: z.string().min(1).max(160).describe("Target scope for imported memories, e.g. 'project:myapp'"),
1826
- format: z.enum(["auto", "claude-code", "claude-ai", "chatgpt", "slack", "plaintext"]).default("auto").describe("Conversation format. Use 'auto' to detect automatically."),
1826
+ format: z.enum(["auto", "claude-code", "claude-ai", "chatgpt", "slack", "plaintext", "connector-v1"]).default("auto").describe("Conversation format. Use 'auto' to detect automatically. 'connector-v1' for standard connector output."),
1827
1827
  },
1828
1828
  async ({ content, scope, format }) => {
1829
1829
  const { detectFormat, normalizeConversation, ingestNormalizedMessages } = await import("./conversation-importer.js");
@@ -0,0 +1,282 @@
1
+ /**
2
+ * Obsidian Vault Connector (GB-1)
3
+ *
4
+ * Scans an Obsidian vault directory and produces a ConnectorOutputV1 payload.
5
+ * Recognizes:
6
+ * - .obsidian/ directory (vault detection)
7
+ * - YAML frontmatter (tags, aliases, arbitrary metadata)
8
+ * - [[wikilinks]] (extracted as entity associations via link: tags)
9
+ * - Directory structure (folder: tags for scope/topic mapping)
10
+ * - Content hash (SHA-256 prefix) for incremental sync
11
+ */
12
+
13
+ import { createHash } from "node:crypto";
14
+ import { readFileSync, readdirSync, existsSync, statSync } from "node:fs";
15
+ import { join, relative, basename, dirname } from "node:path";
16
+ import type { ConnectorOutputV1, ConnectorRecord } from "./connector-types.js";
17
+
18
+ // ============================================================================
19
+ // Vault Detection
20
+ // ============================================================================
21
+
22
+ /** Check if a directory is an Obsidian vault (contains .obsidian/ folder) */
23
+ export function isObsidianVault(dirPath: string): boolean {
24
+ try {
25
+ return existsSync(join(dirPath, ".obsidian"));
26
+ } catch {
27
+ return false;
28
+ }
29
+ }
30
+
31
+ // ============================================================================
32
+ // YAML Frontmatter Parser
33
+ // ============================================================================
34
+
35
+ const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)(?:\r?\n)?---/;
36
+
37
+ export interface ObsidianFrontmatter {
38
+ tags?: string[];
39
+ aliases?: string[];
40
+ [key: string]: unknown;
41
+ }
42
+
43
+ /**
44
+ * Parse YAML frontmatter from markdown content.
45
+ * Lightweight parser — handles:
46
+ * - key: value (strings)
47
+ * - key: [inline, list]
48
+ * - key:\n - list\n - items
49
+ * Does NOT handle nested objects, multi-line strings, or anchors.
50
+ * Returns null frontmatter on parse failure (never throws).
51
+ */
52
+ export function parseFrontmatter(content: string): {
53
+ frontmatter: ObsidianFrontmatter | null;
54
+ body: string;
55
+ } {
56
+ const match = content.match(FRONTMATTER_RE);
57
+ if (!match) return { frontmatter: null, body: content };
58
+
59
+ const yamlBlock = match[1];
60
+ const body = content.slice(match[0].length).trimStart();
61
+
62
+ try {
63
+ const fm: ObsidianFrontmatter = {};
64
+ let currentKey: string | null = null;
65
+ let listAccum: string[] = [];
66
+
67
+ function flushList(): void {
68
+ if (currentKey && listAccum.length > 0) {
69
+ fm[currentKey] = listAccum;
70
+ listAccum = [];
71
+ }
72
+ currentKey = null;
73
+ }
74
+
75
+ for (const line of yamlBlock.split("\n")) {
76
+ // List item: " - value"
77
+ const listItem = line.match(/^\s+-\s+(.+)/);
78
+ if (listItem && currentKey) {
79
+ listAccum.push(listItem[1].trim().replace(/^['"]|['"]$/g, ""));
80
+ continue;
81
+ }
82
+
83
+ // Key-value: "key: value"
84
+ const kv = line.match(/^([\w][\w-]*):\s*(.*)/);
85
+ if (kv) {
86
+ flushList();
87
+ const [, key, rawVal] = kv;
88
+ const val = rawVal.trim();
89
+
90
+ if (val === "" || val === "[]") {
91
+ // Empty value or empty inline list — start list accumulator
92
+ currentKey = key;
93
+ listAccum = [];
94
+ } else if (val.startsWith("[") && val.endsWith("]")) {
95
+ // Inline list: [a, b, c]
96
+ fm[key] = val
97
+ .slice(1, -1)
98
+ .split(",")
99
+ .map((s) => s.trim().replace(/^['"]|['"]$/g, ""))
100
+ .filter(Boolean);
101
+ } else {
102
+ // Simple string value
103
+ fm[key] = val.replace(/^['"]|['"]$/g, "");
104
+ }
105
+ }
106
+ }
107
+ flushList();
108
+
109
+ return {
110
+ frontmatter: Object.keys(fm).length > 0 ? fm : null,
111
+ body,
112
+ };
113
+ } catch {
114
+ // Parse failure — return body without frontmatter
115
+ return { frontmatter: null, body };
116
+ }
117
+ }
118
+
119
+ // ============================================================================
120
+ // Wikilink Extraction
121
+ // ============================================================================
122
+
123
+ const WIKILINK_PATTERN = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
124
+
125
+ export interface WikiLink {
126
+ /** Link target (file name without extension) */
127
+ target: string;
128
+ /** Optional display text (after |) */
129
+ displayText?: string;
130
+ }
131
+
132
+ /**
133
+ * Extract [[wikilinks]] from markdown text.
134
+ * Handles [[target]] and [[target|display text]].
135
+ * Does NOT handle embed syntax ![[embed]] (those are skipped).
136
+ */
137
+ export function extractWikiLinks(text: string): WikiLink[] {
138
+ const links: WikiLink[] = [];
139
+ // Create a fresh regex to avoid stale lastIndex
140
+ const re = new RegExp(WIKILINK_PATTERN.source, WIKILINK_PATTERN.flags);
141
+ let match: RegExpExecArray | null;
142
+
143
+ while ((match = re.exec(text)) !== null) {
144
+ // Skip embed syntax: character before [[ is !
145
+ if (match.index > 0 && text[match.index - 1] === "!") continue;
146
+
147
+ links.push({
148
+ target: match[1].trim(),
149
+ ...(match[2] ? { displayText: match[2].trim() } : {}),
150
+ });
151
+ }
152
+
153
+ return links;
154
+ }
155
+
156
+ // ============================================================================
157
+ // Content Hash
158
+ // ============================================================================
159
+
160
+ /** SHA-256 first 16 hex chars — compact but collision-safe for incremental sync */
161
+ export function contentHash(text: string): string {
162
+ return createHash("sha256").update(text).digest("hex").slice(0, 16);
163
+ }
164
+
165
+ // ============================================================================
166
+ // Vault Scanner
167
+ // ============================================================================
168
+
169
+ export interface ScanVaultOptions {
170
+ /** Scope prefix, defaults to "vault:<vault-folder-name>" */
171
+ scopePrefix?: string;
172
+ /** Directories to exclude (defaults: .obsidian, .trash, .git, node_modules) */
173
+ excludeDirs?: string[];
174
+ /** Minimum file content length to include (default: 30 chars) */
175
+ minContentLength?: number;
176
+ }
177
+
178
+ /**
179
+ * Scan an Obsidian vault and produce a ConnectorOutputV1 payload.
180
+ *
181
+ * - Recursively walks .md files (skipping excluded directories)
182
+ * - Parses YAML frontmatter -> tags
183
+ * - Extracts [[wikilinks]] -> link:<target> tags
184
+ * - Maps folder structure -> folder:<path> tags
185
+ * - Computes SHA-256 content hash for incremental sync
186
+ */
187
+ export function scanVault(
188
+ vaultPath: string,
189
+ options?: ScanVaultOptions,
190
+ ): ConnectorOutputV1 {
191
+ const vaultName = basename(vaultPath);
192
+ const scopePrefix = options?.scopePrefix ?? `vault:${vaultName}`;
193
+ const defaultExclude = [".obsidian", ".trash", ".git", "node_modules"];
194
+ const excludeDirs = new Set(options?.excludeDirs ?? defaultExclude);
195
+ const minLen = options?.minContentLength ?? 30;
196
+
197
+ const records: ConnectorRecord[] = [];
198
+
199
+ function walk(dir: string): void {
200
+ let entries: ReturnType<typeof readdirSync>;
201
+ try {
202
+ entries = readdirSync(dir, { withFileTypes: true });
203
+ } catch {
204
+ return; // Permission denied or broken symlink — skip
205
+ }
206
+
207
+ for (const entry of entries) {
208
+ if (excludeDirs.has(entry.name)) continue;
209
+
210
+ const full = join(dir, entry.name);
211
+
212
+ if (entry.isDirectory()) {
213
+ walk(full);
214
+ continue;
215
+ }
216
+
217
+ if (!entry.name.endsWith(".md")) continue;
218
+
219
+ let raw: string;
220
+ try {
221
+ raw = readFileSync(full, "utf-8");
222
+ } catch {
223
+ continue; // Unreadable file — skip
224
+ }
225
+
226
+ if (raw.trim().length < minLen) continue;
227
+
228
+ const { frontmatter, body } = parseFrontmatter(raw);
229
+ const wikiLinks = extractWikiLinks(body);
230
+ const relPath = relative(vaultPath, full);
231
+ const folder = dirname(relPath);
232
+
233
+ // --- Build tags ---
234
+ const tags: string[] = [];
235
+
236
+ // Frontmatter tags
237
+ if (frontmatter?.tags) {
238
+ const fmTags = Array.isArray(frontmatter.tags)
239
+ ? frontmatter.tags
240
+ : [String(frontmatter.tags)];
241
+ for (const t of fmTags) {
242
+ tags.push(String(t));
243
+ }
244
+ }
245
+
246
+ // Wikilink entity associations
247
+ for (const link of wikiLinks) {
248
+ tags.push(`link:${link.target}`);
249
+ }
250
+
251
+ // Folder path
252
+ if (folder !== ".") {
253
+ tags.push(`folder:${folder}`);
254
+ }
255
+
256
+ records.push({
257
+ id: `obsidian:${relPath}`,
258
+ text: body,
259
+ title: basename(entry.name, ".md"),
260
+ tags,
261
+ contentHash: contentHash(raw),
262
+ timestamp: new Date(statSync(full).mtimeMs).toISOString(),
263
+ sourceMetadata: {
264
+ vaultPath,
265
+ relativePath: relPath,
266
+ ...(frontmatter ? { frontmatter } : {}),
267
+ ...(wikiLinks.length > 0 ? { wikiLinks } : {}),
268
+ },
269
+ });
270
+ }
271
+ }
272
+
273
+ walk(vaultPath);
274
+
275
+ return {
276
+ version: "connector-v1",
277
+ source: "obsidian",
278
+ scope: scopePrefix,
279
+ producedAt: new Date().toISOString(),
280
+ records,
281
+ };
282
+ }
@@ -66,9 +66,9 @@ export const ResumeContextResponseSchema = z.object({
66
66
  essentialContext: EssentialContextSchema.optional(),
67
67
  latestCheckpoint: ResumeCheckpointSummarySchema.optional(),
68
68
  /** CC-1: Hint for where to inject recalled context in the prompt. */
69
- injectionHint: z.enum(["system_prompt", "user_attachment"]).default("user_attachment").optional(),
69
+ injectionHint: z.enum(["system_prompt", "user_attachment"]).optional().default("user_attachment"),
70
70
  /** Upstream #345: Mark recalled context as ephemeral — host should discard on compaction, not persist to transcript. */
71
- ephemeral: z.boolean().default(true).optional(),
71
+ ephemeral: z.boolean().optional().default(true),
72
72
  responseMode: ResumeResponseModeSchema.default("default"),
73
73
  responseGuidance: optionalBoundedStringSchema(400),
74
74
  /** Constructive retrieval: LLM-synthesized reconstruction of context. */