recallnest 2.2.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1 -1
- package/README.md +30 -5
- package/README_CN.md +30 -5
- package/connectors/examples/email-adapter.ts +73 -0
- package/connectors/examples/logs-adapter.ts +103 -0
- package/connectors/examples/rss-adapter.ts +95 -0
- package/logs/daily-recall.log +30 -0
- package/logs/ingest-2026-04-13.log +87 -0
- package/package.json +1 -1
- package/src/cli.ts +30 -0
- package/src/connector-types.ts +57 -0
- package/src/conversation-importer.ts +25 -1
- package/src/data-checkup.ts +31 -1
- package/src/doctor.ts +29 -0
- package/src/ingest.ts +203 -0
- package/src/mcp-server.ts +2 -2
- package/src/obsidian-connector.ts +282 -0
- package/src/session-schema.ts +2 -2
- package/src/source-heartbeat.ts +108 -0
|
@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
|
|
|
2
2
|
import type { PersistMemoryDeps } from "./capture-engine.js";
|
|
3
3
|
import { persistMemory } from "./capture-engine.js";
|
|
4
4
|
import type { StoredMemoryRecord } from "./memory-schema.js";
|
|
5
|
+
import { isConnectorOutputV1, type ConnectorOutputV1 } from "./connector-types.js";
|
|
5
6
|
|
|
6
7
|
export interface NormalizedMessage {
|
|
7
8
|
role: "user" | "assistant" | "system";
|
|
@@ -14,7 +15,8 @@ export type ConversationFormat =
|
|
|
14
15
|
| "claude-ai"
|
|
15
16
|
| "chatgpt"
|
|
16
17
|
| "slack"
|
|
17
|
-
| "plaintext"
|
|
18
|
+
| "plaintext"
|
|
19
|
+
| "connector-v1";
|
|
18
20
|
|
|
19
21
|
// ---------------------------------------------------------------------------
|
|
20
22
|
// Format detection
|
|
@@ -40,6 +42,10 @@ export function detectFormat(content: string): ConversationFormat {
|
|
|
40
42
|
}
|
|
41
43
|
|
|
42
44
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
45
|
+
// connector-v1: highest priority — explicit version tag
|
|
46
|
+
if (isConnectorOutputV1(parsed)) {
|
|
47
|
+
return "connector-v1";
|
|
48
|
+
}
|
|
43
49
|
if (parsed.chat_conversations !== undefined) {
|
|
44
50
|
return "claude-ai";
|
|
45
51
|
}
|
|
@@ -366,6 +372,22 @@ export function normalizePlaintext(content: string): NormalizedMessage[] {
|
|
|
366
372
|
return messages;
|
|
367
373
|
}
|
|
368
374
|
|
|
375
|
+
// ---------------------------------------------------------------------------
|
|
376
|
+
// Connector-v1 normalizer
|
|
377
|
+
// ---------------------------------------------------------------------------
|
|
378
|
+
|
|
379
|
+
export function normalizeConnectorV1(content: string): NormalizedMessage[] {
|
|
380
|
+
const parsed: ConnectorOutputV1 = JSON.parse(content);
|
|
381
|
+
if (!isConnectorOutputV1(parsed)) {
|
|
382
|
+
throw new Error("Invalid connector-v1 format: missing version/source/scope/records");
|
|
383
|
+
}
|
|
384
|
+
return parsed.records.map((r) => ({
|
|
385
|
+
role: "user" as const,
|
|
386
|
+
content: r.title ? `[${r.title}] ${r.text}` : r.text,
|
|
387
|
+
...(r.timestamp ? { timestamp: r.timestamp } : {}),
|
|
388
|
+
}));
|
|
389
|
+
}
|
|
390
|
+
|
|
369
391
|
// ---------------------------------------------------------------------------
|
|
370
392
|
// Unified normalize dispatcher
|
|
371
393
|
// ---------------------------------------------------------------------------
|
|
@@ -382,6 +404,8 @@ export function normalizeConversation(content: string, format: ConversationForma
|
|
|
382
404
|
return normalizeSlack(content);
|
|
383
405
|
case "plaintext":
|
|
384
406
|
return normalizePlaintext(content);
|
|
407
|
+
case "connector-v1":
|
|
408
|
+
return normalizeConnectorV1(content);
|
|
385
409
|
}
|
|
386
410
|
}
|
|
387
411
|
|
package/src/data-checkup.ts
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import type { MemoryEntry, MemoryStore } from "./store.js";
|
|
16
16
|
import { resolveTier, type MemoryTier } from "./decay-engine.js";
|
|
17
17
|
import { measureInterferenceDensity } from "./interference-detector.js";
|
|
18
|
+
import { readHeartbeats, checkSourceStaleness, formatAge } from "./source-heartbeat.js";
|
|
18
19
|
|
|
19
20
|
// ---------------------------------------------------------------------------
|
|
20
21
|
// Types
|
|
@@ -151,7 +152,33 @@ function checkVersionGroups(entries: MemoryEntry[]): CheckResult {
|
|
|
151
152
|
};
|
|
152
153
|
}
|
|
153
154
|
|
|
154
|
-
/** Check 6 (
|
|
155
|
+
/** Check 6 (GB-3): Source health — detect stale connector data sources. */
|
|
156
|
+
function checkSourceHealth(heartbeatPath?: string): CheckResult {
|
|
157
|
+
const heartbeats = readHeartbeats(heartbeatPath);
|
|
158
|
+
const sources = Object.values(heartbeats);
|
|
159
|
+
if (sources.length === 0) {
|
|
160
|
+
return { name: "source_health", status: "ok", detail: "No connector sources tracked yet" };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const stale30 = checkSourceStaleness(30, heartbeatPath);
|
|
164
|
+
const stale7 = checkSourceStaleness(7, heartbeatPath);
|
|
165
|
+
|
|
166
|
+
const summary = sources
|
|
167
|
+
.map((s) => `${s.source}: ${formatAge(s.lastIngest)}`)
|
|
168
|
+
.join(", ");
|
|
169
|
+
|
|
170
|
+
if (stale30.length > 0) {
|
|
171
|
+
const names = stale30.map((s) => `${s.source} (${s.daysSince}d)`).join(", ");
|
|
172
|
+
return { name: "source_health", status: "error", detail: `${summary} — critically stale: ${names}` };
|
|
173
|
+
}
|
|
174
|
+
if (stale7.length > 0) {
|
|
175
|
+
const names = stale7.map((s) => `${s.source} (${s.daysSince}d)`).join(", ");
|
|
176
|
+
return { name: "source_health", status: "warning", detail: `${summary} — stale: ${names}` };
|
|
177
|
+
}
|
|
178
|
+
return { name: "source_health", status: "ok", detail: `All sources fresh: ${summary}` };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Check 7 (F2): Interference density — detect semantic clustering pressure. */
|
|
155
182
|
function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
|
|
156
183
|
// Only check entries with vectors (skip schema/empty entries)
|
|
157
184
|
const withVectors = entries.filter(e => e.vector && e.vector.length > 0);
|
|
@@ -192,6 +219,8 @@ function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
|
|
|
192
219
|
export interface CheckupDeps {
|
|
193
220
|
store: Pick<MemoryStore, "list" | "stats">;
|
|
194
221
|
openConflictCount: number;
|
|
222
|
+
/** Optional override path for source-heartbeat.json (used in tests). */
|
|
223
|
+
heartbeatPath?: string;
|
|
195
224
|
}
|
|
196
225
|
|
|
197
226
|
export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport> {
|
|
@@ -204,6 +233,7 @@ export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport>
|
|
|
204
233
|
checkTierDistribution(entries),
|
|
205
234
|
checkConflictBacklog(deps.openConflictCount),
|
|
206
235
|
checkVersionGroups(entries),
|
|
236
|
+
checkSourceHealth(deps.heartbeatPath),
|
|
207
237
|
checkInterferenceDensity(entries),
|
|
208
238
|
],
|
|
209
239
|
totalEntries: entries.length,
|
package/src/doctor.ts
CHANGED
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
} from "./memory-schema.js";
|
|
28
28
|
import { collectScopeInventory, type ScopeInventoryReport } from "./scope-inventory.js";
|
|
29
29
|
import { MemoryStore, type MemoryEntry, validateStoragePath } from "./store.js";
|
|
30
|
+
import { readHeartbeats, formatAge } from "./source-heartbeat.js";
|
|
30
31
|
|
|
31
32
|
interface CheckResult {
|
|
32
33
|
name: string;
|
|
@@ -508,6 +509,34 @@ export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckRe
|
|
|
508
509
|
));
|
|
509
510
|
}
|
|
510
511
|
|
|
512
|
+
// GB-3: Source heartbeat summary
|
|
513
|
+
try {
|
|
514
|
+
const heartbeats = readHeartbeats();
|
|
515
|
+
const sources = Object.values(heartbeats);
|
|
516
|
+
if (sources.length === 0) {
|
|
517
|
+
results.push(pass("Source heartbeats", "no connector sources tracked yet"));
|
|
518
|
+
} else {
|
|
519
|
+
const summary = sources
|
|
520
|
+
.map((s) => `${s.source}: ${formatAge(s.lastIngest)}`)
|
|
521
|
+
.join(", ");
|
|
522
|
+
const staleCount = sources.filter((s) => {
|
|
523
|
+
const daysSince = (Date.now() - Date.parse(s.lastIngest)) / 86_400_000;
|
|
524
|
+
return daysSince > 7;
|
|
525
|
+
}).length;
|
|
526
|
+
if (staleCount > 0) {
|
|
527
|
+
results.push(warn(
|
|
528
|
+
"Source heartbeats",
|
|
529
|
+
`${summary} (${staleCount} stale >7d)`,
|
|
530
|
+
"lm ingest --connector <file>",
|
|
531
|
+
));
|
|
532
|
+
} else {
|
|
533
|
+
results.push(pass("Source heartbeats", summary));
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
} catch {
|
|
537
|
+
results.push(warn("Source heartbeats", "unable to read heartbeat file"));
|
|
538
|
+
}
|
|
539
|
+
|
|
511
540
|
return results;
|
|
512
541
|
}
|
|
513
542
|
|
package/src/ingest.ts
CHANGED
|
@@ -15,6 +15,9 @@ import { detectLang, tokenizeFts } from "./language-hook.js";
|
|
|
15
15
|
import type { MemoryStore, MemoryEntry } from "./store.js";
|
|
16
16
|
import type { Embedder } from "./embedder.js";
|
|
17
17
|
import { chunkDocument, type ChunkerConfig } from "./chunker.js";
|
|
18
|
+
import type { ConnectorOutputV1, ConnectorRecord } from "./connector-types.js";
|
|
19
|
+
import { isConnectorOutputV1 } from "./connector-types.js";
|
|
20
|
+
import { isObsidianVault, scanVault } from "./obsidian-connector.js";
|
|
18
21
|
import { segmentEvents, type EventSegmenterConfig, DEFAULT_EVENT_SEGMENTER_CONFIG } from "./event-segmenter.js";
|
|
19
22
|
import { isProcessed, markProcessed } from "./tracker.js";
|
|
20
23
|
import type { LLMClient, SmartExtraction } from "./llm-client.js";
|
|
@@ -1850,3 +1853,203 @@ export async function ingestGenericText(
|
|
|
1850
1853
|
|
|
1851
1854
|
return result;
|
|
1852
1855
|
}
|
|
1856
|
+
|
|
1857
|
+
// ============================================================================
|
|
1858
|
+
// Connector-v1 Ingest (GB-2)
|
|
1859
|
+
// ============================================================================
|
|
1860
|
+
|
|
1861
|
+
/**
|
|
1862
|
+
* Ingest a connector-v1 formatted JSON payload.
|
|
1863
|
+
*
|
|
1864
|
+
* External connectors (Obsidian, Email, RSS, etc.) produce a standard
|
|
1865
|
+
* ConnectorOutputV1 JSON → this function consumes it through the existing
|
|
1866
|
+
* dedup / smartExtract / buildIngestedEntry pipeline.
|
|
1867
|
+
*
|
|
1868
|
+
* contentHash on each record enables incremental sync: if a record with the
|
|
1869
|
+
* same hash is already in the store, it is skipped without embedding.
|
|
1870
|
+
*/
|
|
1871
|
+
export async function ingestConnectorFile(
|
|
1872
|
+
store: MemoryStore,
|
|
1873
|
+
embedder: Embedder,
|
|
1874
|
+
content: string,
|
|
1875
|
+
options: { verbose?: boolean; noDedup?: boolean; llm?: LLMClient | null } = {},
|
|
1876
|
+
): Promise<IngestResult> {
|
|
1877
|
+
const parsed = JSON.parse(content);
|
|
1878
|
+
if (!isConnectorOutputV1(parsed)) {
|
|
1879
|
+
return {
|
|
1880
|
+
source: "connector",
|
|
1881
|
+
filesProcessed: 0,
|
|
1882
|
+
chunksIngested: 0,
|
|
1883
|
+
chunksSkipped: 0,
|
|
1884
|
+
chunksDeduped: 0,
|
|
1885
|
+
dedupReasonCounts: createDedupReasonCounts(),
|
|
1886
|
+
errors: ["Invalid connector-v1 format: missing version/source/scope/records"],
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
const connectorData = parsed as ConnectorOutputV1;
|
|
1891
|
+
const { source, scope, records } = connectorData;
|
|
1892
|
+
|
|
1893
|
+
const result: IngestResult = {
|
|
1894
|
+
source: `connector:${source}`,
|
|
1895
|
+
filesProcessed: 1,
|
|
1896
|
+
chunksIngested: 0,
|
|
1897
|
+
chunksSkipped: 0,
|
|
1898
|
+
chunksDeduped: 0,
|
|
1899
|
+
dedupReasonCounts: createDedupReasonCounts(),
|
|
1900
|
+
errors: [],
|
|
1901
|
+
};
|
|
1902
|
+
|
|
1903
|
+
if (records.length === 0) return result;
|
|
1904
|
+
|
|
1905
|
+
const batchSize = 32;
|
|
1906
|
+
|
|
1907
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
1908
|
+
const batch = records.slice(i, i + batchSize);
|
|
1909
|
+
const texts = batch.map((r: ConnectorRecord) =>
|
|
1910
|
+
r.title ? `[${r.title}] ${r.text}` : r.text,
|
|
1911
|
+
);
|
|
1912
|
+
|
|
1913
|
+
try {
|
|
1914
|
+
const vectors = await embedder.embedBatchPassage(texts);
|
|
1915
|
+
|
|
1916
|
+
// --- dedup pass ---
|
|
1917
|
+
const dedupedTexts: string[] = [];
|
|
1918
|
+
const dedupedVectors: number[][] = [];
|
|
1919
|
+
const dedupedRecords: ConnectorRecord[] = [];
|
|
1920
|
+
|
|
1921
|
+
for (let j = 0; j < batch.length; j++) {
|
|
1922
|
+
const vector = vectors[j];
|
|
1923
|
+
if (!vector || vector.length === 0) {
|
|
1924
|
+
result.chunksSkipped++;
|
|
1925
|
+
continue;
|
|
1926
|
+
}
|
|
1927
|
+
if (!options.noDedup) {
|
|
1928
|
+
const decision = await dedupCheck(store, vector, texts[j], options.llm);
|
|
1929
|
+
recordDedupDecision(result, decision);
|
|
1930
|
+
if (decision.secondaryDeletes?.length) {
|
|
1931
|
+
await executeSecondaryDeletes(store, decision.secondaryDeletes);
|
|
1932
|
+
}
|
|
1933
|
+
if (decision.action === "skip") continue;
|
|
1934
|
+
}
|
|
1935
|
+
dedupedTexts.push(texts[j]);
|
|
1936
|
+
dedupedVectors.push(vector);
|
|
1937
|
+
dedupedRecords.push(batch[j]);
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
if (dedupedTexts.length === 0) continue;
|
|
1941
|
+
|
|
1942
|
+
// --- extract + build entries ---
|
|
1943
|
+
const extractions = await smartExtractBatch(dedupedTexts, options.llm);
|
|
1944
|
+
const coreSummaries = await generateCoreSummaries(dedupedTexts, options.llm);
|
|
1945
|
+
const toStore: Array<{
|
|
1946
|
+
text: string;
|
|
1947
|
+
vector: number[];
|
|
1948
|
+
category: string;
|
|
1949
|
+
scope: string;
|
|
1950
|
+
importance: number;
|
|
1951
|
+
metadata: string;
|
|
1952
|
+
}> = [];
|
|
1953
|
+
|
|
1954
|
+
for (let j = 0; j < dedupedTexts.length; j++) {
|
|
1955
|
+
const record = dedupedRecords[j];
|
|
1956
|
+
const entry = buildIngestedEntry({
|
|
1957
|
+
source: `connector:${source}`,
|
|
1958
|
+
scope,
|
|
1959
|
+
text: dedupedTexts[j],
|
|
1960
|
+
vector: dedupedVectors[j],
|
|
1961
|
+
extraction: extractions[j],
|
|
1962
|
+
file: record.id,
|
|
1963
|
+
heading: record.title,
|
|
1964
|
+
coreSummary: coreSummaries[j],
|
|
1965
|
+
});
|
|
1966
|
+
|
|
1967
|
+
// Merge connector-specific metadata
|
|
1968
|
+
const meta = JSON.parse(entry.metadata);
|
|
1969
|
+
if (record.tags?.length) meta.connectorTags = record.tags;
|
|
1970
|
+
if (record.contentHash) meta.contentHash = record.contentHash;
|
|
1971
|
+
if (record.sourceMetadata) meta.sourceMetadata = record.sourceMetadata;
|
|
1972
|
+
entry.metadata = JSON.stringify(meta);
|
|
1973
|
+
|
|
1974
|
+
// Respect categoryHint / importanceHint from connector
|
|
1975
|
+
if (record.categoryHint) {
|
|
1976
|
+
entry.category = record.categoryHint;
|
|
1977
|
+
}
|
|
1978
|
+
if (record.importanceHint !== undefined) {
|
|
1979
|
+
entry.importance = record.importanceHint;
|
|
1980
|
+
}
|
|
1981
|
+
|
|
1982
|
+
toStore.push(entry);
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
if (toStore.length > 0) {
|
|
1986
|
+
await store.storeBatch(toStore);
|
|
1987
|
+
result.chunksIngested += toStore.length;
|
|
1988
|
+
}
|
|
1989
|
+
} catch (err: unknown) {
|
|
1990
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1991
|
+
result.errors.push(`Connector batch error: ${msg}`);
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
if (options.verbose) {
|
|
1996
|
+
console.log(` connector:${source}: ${records.length} records → ${result.chunksIngested} stored`);
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
// GB-3: Write source heartbeat after connector ingest
|
|
2000
|
+
try {
|
|
2001
|
+
const { writeHeartbeat } = await import("./source-heartbeat.js");
|
|
2002
|
+
writeHeartbeat(source, result.chunksIngested, result.errors);
|
|
2003
|
+
} catch {
|
|
2004
|
+
// Heartbeat write is best-effort — don't break ingest on failure
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
return result;
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
// ============================================================================
|
|
2011
|
+
// Obsidian Vault Ingest (GB-1)
|
|
2012
|
+
// ============================================================================
|
|
2013
|
+
|
|
2014
|
+
/**
|
|
2015
|
+
* Scan an Obsidian vault and ingest its markdown files through the connector-v1 pipeline.
|
|
2016
|
+
*
|
|
2017
|
+
* Wrapper that:
|
|
2018
|
+
* 1. Validates the path is an Obsidian vault (.obsidian/ exists)
|
|
2019
|
+
* 2. Calls scanVault() to produce a ConnectorOutputV1 payload
|
|
2020
|
+
* 3. Feeds the payload to ingestConnectorFile()
|
|
2021
|
+
*/
|
|
2022
|
+
export async function ingestObsidianVault(
|
|
2023
|
+
store: MemoryStore,
|
|
2024
|
+
embedder: Embedder,
|
|
2025
|
+
vaultPath: string,
|
|
2026
|
+
options: {
|
|
2027
|
+
verbose?: boolean;
|
|
2028
|
+
noDedup?: boolean;
|
|
2029
|
+
llm?: LLMClient | null;
|
|
2030
|
+
scopePrefix?: string;
|
|
2031
|
+
} = {},
|
|
2032
|
+
): Promise<IngestResult> {
|
|
2033
|
+
const resolved = expandHome(vaultPath);
|
|
2034
|
+
|
|
2035
|
+
if (!isObsidianVault(resolved)) {
|
|
2036
|
+
return {
|
|
2037
|
+
source: "obsidian",
|
|
2038
|
+
filesProcessed: 0,
|
|
2039
|
+
chunksIngested: 0,
|
|
2040
|
+
chunksSkipped: 0,
|
|
2041
|
+
chunksDeduped: 0,
|
|
2042
|
+
dedupReasonCounts: createDedupReasonCounts(),
|
|
2043
|
+
errors: [`Not an Obsidian vault (missing .obsidian/ directory): ${resolved}`],
|
|
2044
|
+
};
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
const output = scanVault(resolved, { scopePrefix: options.scopePrefix });
|
|
2048
|
+
const content = JSON.stringify(output);
|
|
2049
|
+
|
|
2050
|
+
if (options.verbose) {
|
|
2051
|
+
console.log(` obsidian: scanned ${output.records.length} notes from ${resolved}`);
|
|
2052
|
+
}
|
|
2053
|
+
|
|
2054
|
+
return ingestConnectorFile(store, embedder, content, options);
|
|
2055
|
+
}
|
package/src/mcp-server.ts
CHANGED
|
@@ -1819,11 +1819,11 @@ registerTool(
|
|
|
1819
1819
|
|
|
1820
1820
|
registerTool(
|
|
1821
1821
|
"import_conversations",
|
|
1822
|
-
"Import a conversation file (Claude Code JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, or
|
|
1822
|
+
"Import a conversation file (Claude Code JSONL, Claude.ai JSON, ChatGPT JSON, Slack JSON, plaintext, or connector-v1 JSON) into memory. Auto-detects format or use explicit format parameter. Messages are normalized and stored via the standard persistMemory pipeline. For connector-v1 format, use the standard ConnectorOutputV1 schema (see docs/connector-spec.md).",
|
|
1823
1823
|
{
|
|
1824
1824
|
content: z.string().min(1).max(500_000).describe("Raw file content to import"),
|
|
1825
1825
|
scope: z.string().min(1).max(160).describe("Target scope for imported memories, e.g. 'project:myapp'"),
|
|
1826
|
-
format: z.enum(["auto", "claude-code", "claude-ai", "chatgpt", "slack", "plaintext"]).default("auto").describe("Conversation format. Use 'auto' to detect automatically."),
|
|
1826
|
+
format: z.enum(["auto", "claude-code", "claude-ai", "chatgpt", "slack", "plaintext", "connector-v1"]).default("auto").describe("Conversation format. Use 'auto' to detect automatically. 'connector-v1' for standard connector output."),
|
|
1827
1827
|
},
|
|
1828
1828
|
async ({ content, scope, format }) => {
|
|
1829
1829
|
const { detectFormat, normalizeConversation, ingestNormalizedMessages } = await import("./conversation-importer.js");
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Obsidian Vault Connector (GB-1)
|
|
3
|
+
*
|
|
4
|
+
* Scans an Obsidian vault directory and produces a ConnectorOutputV1 payload.
|
|
5
|
+
* Recognizes:
|
|
6
|
+
* - .obsidian/ directory (vault detection)
|
|
7
|
+
* - YAML frontmatter (tags, aliases, arbitrary metadata)
|
|
8
|
+
* - [[wikilinks]] (extracted as entity associations via link: tags)
|
|
9
|
+
* - Directory structure (folder: tags for scope/topic mapping)
|
|
10
|
+
* - Content hash (SHA-256 prefix) for incremental sync
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
14
|
+
import { readFileSync, readdirSync, existsSync, statSync } from "node:fs";
|
|
15
|
+
import { join, relative, basename, dirname } from "node:path";
|
|
16
|
+
import type { ConnectorOutputV1, ConnectorRecord } from "./connector-types.js";
|
|
17
|
+
|
|
18
|
+
// ============================================================================
|
|
19
|
+
// Vault Detection
|
|
20
|
+
// ============================================================================
|
|
21
|
+
|
|
22
|
+
/** Check if a directory is an Obsidian vault (contains .obsidian/ folder) */
|
|
23
|
+
export function isObsidianVault(dirPath: string): boolean {
|
|
24
|
+
try {
|
|
25
|
+
return existsSync(join(dirPath, ".obsidian"));
|
|
26
|
+
} catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// ============================================================================
|
|
32
|
+
// YAML Frontmatter Parser
|
|
33
|
+
// ============================================================================
|
|
34
|
+
|
|
35
|
+
const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)(?:\r?\n)?---/;
|
|
36
|
+
|
|
37
|
+
export interface ObsidianFrontmatter {
|
|
38
|
+
tags?: string[];
|
|
39
|
+
aliases?: string[];
|
|
40
|
+
[key: string]: unknown;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Parse YAML frontmatter from markdown content.
|
|
45
|
+
* Lightweight parser — handles:
|
|
46
|
+
* - key: value (strings)
|
|
47
|
+
* - key: [inline, list]
|
|
48
|
+
* - key:\n - list\n - items
|
|
49
|
+
* Does NOT handle nested objects, multi-line strings, or anchors.
|
|
50
|
+
* Returns null frontmatter on parse failure (never throws).
|
|
51
|
+
*/
|
|
52
|
+
export function parseFrontmatter(content: string): {
|
|
53
|
+
frontmatter: ObsidianFrontmatter | null;
|
|
54
|
+
body: string;
|
|
55
|
+
} {
|
|
56
|
+
const match = content.match(FRONTMATTER_RE);
|
|
57
|
+
if (!match) return { frontmatter: null, body: content };
|
|
58
|
+
|
|
59
|
+
const yamlBlock = match[1];
|
|
60
|
+
const body = content.slice(match[0].length).trimStart();
|
|
61
|
+
|
|
62
|
+
try {
|
|
63
|
+
const fm: ObsidianFrontmatter = {};
|
|
64
|
+
let currentKey: string | null = null;
|
|
65
|
+
let listAccum: string[] = [];
|
|
66
|
+
|
|
67
|
+
function flushList(): void {
|
|
68
|
+
if (currentKey && listAccum.length > 0) {
|
|
69
|
+
fm[currentKey] = listAccum;
|
|
70
|
+
listAccum = [];
|
|
71
|
+
}
|
|
72
|
+
currentKey = null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
for (const line of yamlBlock.split("\n")) {
|
|
76
|
+
// List item: " - value"
|
|
77
|
+
const listItem = line.match(/^\s+-\s+(.+)/);
|
|
78
|
+
if (listItem && currentKey) {
|
|
79
|
+
listAccum.push(listItem[1].trim().replace(/^['"]|['"]$/g, ""));
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Key-value: "key: value"
|
|
84
|
+
const kv = line.match(/^([\w][\w-]*):\s*(.*)/);
|
|
85
|
+
if (kv) {
|
|
86
|
+
flushList();
|
|
87
|
+
const [, key, rawVal] = kv;
|
|
88
|
+
const val = rawVal.trim();
|
|
89
|
+
|
|
90
|
+
if (val === "" || val === "[]") {
|
|
91
|
+
// Empty value or empty inline list — start list accumulator
|
|
92
|
+
currentKey = key;
|
|
93
|
+
listAccum = [];
|
|
94
|
+
} else if (val.startsWith("[") && val.endsWith("]")) {
|
|
95
|
+
// Inline list: [a, b, c]
|
|
96
|
+
fm[key] = val
|
|
97
|
+
.slice(1, -1)
|
|
98
|
+
.split(",")
|
|
99
|
+
.map((s) => s.trim().replace(/^['"]|['"]$/g, ""))
|
|
100
|
+
.filter(Boolean);
|
|
101
|
+
} else {
|
|
102
|
+
// Simple string value
|
|
103
|
+
fm[key] = val.replace(/^['"]|['"]$/g, "");
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
flushList();
|
|
108
|
+
|
|
109
|
+
return {
|
|
110
|
+
frontmatter: Object.keys(fm).length > 0 ? fm : null,
|
|
111
|
+
body,
|
|
112
|
+
};
|
|
113
|
+
} catch {
|
|
114
|
+
// Parse failure — return body without frontmatter
|
|
115
|
+
return { frontmatter: null, body };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// ============================================================================
|
|
120
|
+
// Wikilink Extraction
|
|
121
|
+
// ============================================================================
|
|
122
|
+
|
|
123
|
+
const WIKILINK_PATTERN = /\[\[([^\]|]+)(?:\|([^\]]+))?\]\]/g;
|
|
124
|
+
|
|
125
|
+
export interface WikiLink {
|
|
126
|
+
/** Link target (file name without extension) */
|
|
127
|
+
target: string;
|
|
128
|
+
/** Optional display text (after |) */
|
|
129
|
+
displayText?: string;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Extract [[wikilinks]] from markdown text.
|
|
134
|
+
* Handles [[target]] and [[target|display text]].
|
|
135
|
+
* Does NOT handle embed syntax ![[embed]] (those are skipped).
|
|
136
|
+
*/
|
|
137
|
+
export function extractWikiLinks(text: string): WikiLink[] {
|
|
138
|
+
const links: WikiLink[] = [];
|
|
139
|
+
// Create a fresh regex to avoid stale lastIndex
|
|
140
|
+
const re = new RegExp(WIKILINK_PATTERN.source, WIKILINK_PATTERN.flags);
|
|
141
|
+
let match: RegExpExecArray | null;
|
|
142
|
+
|
|
143
|
+
while ((match = re.exec(text)) !== null) {
|
|
144
|
+
// Skip embed syntax: character before [[ is !
|
|
145
|
+
if (match.index > 0 && text[match.index - 1] === "!") continue;
|
|
146
|
+
|
|
147
|
+
links.push({
|
|
148
|
+
target: match[1].trim(),
|
|
149
|
+
...(match[2] ? { displayText: match[2].trim() } : {}),
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return links;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// ============================================================================
|
|
157
|
+
// Content Hash
|
|
158
|
+
// ============================================================================
|
|
159
|
+
|
|
160
|
+
/** SHA-256 first 16 hex chars — compact but collision-safe for incremental sync */
|
|
161
|
+
export function contentHash(text: string): string {
|
|
162
|
+
return createHash("sha256").update(text).digest("hex").slice(0, 16);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// ============================================================================
|
|
166
|
+
// Vault Scanner
|
|
167
|
+
// ============================================================================
|
|
168
|
+
|
|
169
|
+
export interface ScanVaultOptions {
|
|
170
|
+
/** Scope prefix, defaults to "vault:<vault-folder-name>" */
|
|
171
|
+
scopePrefix?: string;
|
|
172
|
+
/** Directories to exclude (defaults: .obsidian, .trash, .git, node_modules) */
|
|
173
|
+
excludeDirs?: string[];
|
|
174
|
+
/** Minimum file content length to include (default: 30 chars) */
|
|
175
|
+
minContentLength?: number;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Scan an Obsidian vault and produce a ConnectorOutputV1 payload.
|
|
180
|
+
*
|
|
181
|
+
* - Recursively walks .md files (skipping excluded directories)
|
|
182
|
+
* - Parses YAML frontmatter -> tags
|
|
183
|
+
* - Extracts [[wikilinks]] -> link:<target> tags
|
|
184
|
+
* - Maps folder structure -> folder:<path> tags
|
|
185
|
+
* - Computes SHA-256 content hash for incremental sync
|
|
186
|
+
*/
|
|
187
|
+
export function scanVault(
|
|
188
|
+
vaultPath: string,
|
|
189
|
+
options?: ScanVaultOptions,
|
|
190
|
+
): ConnectorOutputV1 {
|
|
191
|
+
const vaultName = basename(vaultPath);
|
|
192
|
+
const scopePrefix = options?.scopePrefix ?? `vault:${vaultName}`;
|
|
193
|
+
const defaultExclude = [".obsidian", ".trash", ".git", "node_modules"];
|
|
194
|
+
const excludeDirs = new Set(options?.excludeDirs ?? defaultExclude);
|
|
195
|
+
const minLen = options?.minContentLength ?? 30;
|
|
196
|
+
|
|
197
|
+
const records: ConnectorRecord[] = [];
|
|
198
|
+
|
|
199
|
+
function walk(dir: string): void {
|
|
200
|
+
let entries: ReturnType<typeof readdirSync>;
|
|
201
|
+
try {
|
|
202
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
203
|
+
} catch {
|
|
204
|
+
return; // Permission denied or broken symlink — skip
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
for (const entry of entries) {
|
|
208
|
+
if (excludeDirs.has(entry.name)) continue;
|
|
209
|
+
|
|
210
|
+
const full = join(dir, entry.name);
|
|
211
|
+
|
|
212
|
+
if (entry.isDirectory()) {
|
|
213
|
+
walk(full);
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (!entry.name.endsWith(".md")) continue;
|
|
218
|
+
|
|
219
|
+
let raw: string;
|
|
220
|
+
try {
|
|
221
|
+
raw = readFileSync(full, "utf-8");
|
|
222
|
+
} catch {
|
|
223
|
+
continue; // Unreadable file — skip
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (raw.trim().length < minLen) continue;
|
|
227
|
+
|
|
228
|
+
const { frontmatter, body } = parseFrontmatter(raw);
|
|
229
|
+
const wikiLinks = extractWikiLinks(body);
|
|
230
|
+
const relPath = relative(vaultPath, full);
|
|
231
|
+
const folder = dirname(relPath);
|
|
232
|
+
|
|
233
|
+
// --- Build tags ---
|
|
234
|
+
const tags: string[] = [];
|
|
235
|
+
|
|
236
|
+
// Frontmatter tags
|
|
237
|
+
if (frontmatter?.tags) {
|
|
238
|
+
const fmTags = Array.isArray(frontmatter.tags)
|
|
239
|
+
? frontmatter.tags
|
|
240
|
+
: [String(frontmatter.tags)];
|
|
241
|
+
for (const t of fmTags) {
|
|
242
|
+
tags.push(String(t));
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Wikilink entity associations
|
|
247
|
+
for (const link of wikiLinks) {
|
|
248
|
+
tags.push(`link:${link.target}`);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// Folder path
|
|
252
|
+
if (folder !== ".") {
|
|
253
|
+
tags.push(`folder:${folder}`);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
records.push({
|
|
257
|
+
id: `obsidian:${relPath}`,
|
|
258
|
+
text: body,
|
|
259
|
+
title: basename(entry.name, ".md"),
|
|
260
|
+
tags,
|
|
261
|
+
contentHash: contentHash(raw),
|
|
262
|
+
timestamp: new Date(statSync(full).mtimeMs).toISOString(),
|
|
263
|
+
sourceMetadata: {
|
|
264
|
+
vaultPath,
|
|
265
|
+
relativePath: relPath,
|
|
266
|
+
...(frontmatter ? { frontmatter } : {}),
|
|
267
|
+
...(wikiLinks.length > 0 ? { wikiLinks } : {}),
|
|
268
|
+
},
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
walk(vaultPath);
|
|
274
|
+
|
|
275
|
+
return {
|
|
276
|
+
version: "connector-v1",
|
|
277
|
+
source: "obsidian",
|
|
278
|
+
scope: scopePrefix,
|
|
279
|
+
producedAt: new Date().toISOString(),
|
|
280
|
+
records,
|
|
281
|
+
};
|
|
282
|
+
}
|
package/src/session-schema.ts
CHANGED
|
@@ -66,9 +66,9 @@ export const ResumeContextResponseSchema = z.object({
|
|
|
66
66
|
essentialContext: EssentialContextSchema.optional(),
|
|
67
67
|
latestCheckpoint: ResumeCheckpointSummarySchema.optional(),
|
|
68
68
|
/** CC-1: Hint for where to inject recalled context in the prompt. */
|
|
69
|
-
injectionHint: z.enum(["system_prompt", "user_attachment"]).default("user_attachment")
|
|
69
|
+
injectionHint: z.enum(["system_prompt", "user_attachment"]).optional().default("user_attachment"),
|
|
70
70
|
/** Upstream #345: Mark recalled context as ephemeral — host should discard on compaction, not persist to transcript. */
|
|
71
|
-
ephemeral: z.boolean().
|
|
71
|
+
ephemeral: z.boolean().optional().default(true),
|
|
72
72
|
responseMode: ResumeResponseModeSchema.default("default"),
|
|
73
73
|
responseGuidance: optionalBoundedStringSchema(400),
|
|
74
74
|
/** Constructive retrieval: LLM-synthesized reconstruction of context. */
|