seekstone 0.13.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/chunk-RRLYVK2X.js +18 -0
- package/dist/chunk-RRLYVK2X.js.map +1 -0
- package/dist/fetch-model-7TGOJMZL.js +81 -0
- package/dist/fetch-model-7TGOJMZL.js.map +1 -0
- package/dist/index.js +882 -57
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
resolveSemanticConfig
|
|
4
|
+
} from "./chunk-RRLYVK2X.js";
|
|
2
5
|
|
|
3
6
|
// src/index.ts
|
|
7
|
+
import { homedir } from "os";
|
|
4
8
|
import { resolve as resolve2 } from "path";
|
|
5
9
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
6
10
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
@@ -17,6 +21,7 @@ function parseCliIntent(argv) {
|
|
|
17
21
|
}
|
|
18
22
|
return "init";
|
|
19
23
|
}
|
|
24
|
+
if (argv[0] === "fetch-model") return "fetch-model";
|
|
20
25
|
for (const arg of argv) {
|
|
21
26
|
if (arg === "--version" || arg === "-v") return "version";
|
|
22
27
|
if (arg === "--help" || arg === "-h") return "help";
|
|
@@ -33,10 +38,11 @@ Seekstone runs as a Model Context Protocol stdio server. It is normally
|
|
|
33
38
|
launched by an MCP client (Claude Desktop, Claude Code, Cursor, \u2026), not run by hand.
|
|
34
39
|
|
|
35
40
|
Usage:
|
|
36
|
-
seekstone
|
|
37
|
-
seekstone init
|
|
38
|
-
seekstone
|
|
39
|
-
seekstone --
|
|
41
|
+
seekstone Start the MCP server (reads from stdin/stdout)
|
|
42
|
+
seekstone init Validate a vault and print/patch the client MCP config
|
|
43
|
+
seekstone fetch-model Download the local embedding model for semantic search
|
|
44
|
+
seekstone --version Print the version and exit
|
|
45
|
+
seekstone --help Print this help and exit
|
|
40
46
|
|
|
41
47
|
"seekstone init" options:
|
|
42
48
|
${initOptionLines}
|
|
@@ -48,6 +54,9 @@ Optional environment:
|
|
|
48
54
|
SEEKSTONE_LOG_LEVEL error | warn | info (default) | debug
|
|
49
55
|
SEEKSTONE_LOG_FILE Absolute path; append JSON-line logs here
|
|
50
56
|
SEEKSTONE_WATCH_POLL Set to 1 to poll for changes (network drives, WSL)
|
|
57
|
+
SEEKSTONE_SEMANTIC Set to 1 to enable semantic search (needs fetch-model first)
|
|
58
|
+
SEEKSTONE_MODEL_PATH Override the embedding-model directory
|
|
59
|
+
SEEKSTONE_CACHE_DIR Override the cache root (default ~/.cache/seekstone)
|
|
51
60
|
|
|
52
61
|
Add to Claude Code:
|
|
53
62
|
claude mcp add seekstone --env SEEKSTONE_VAULT=/path/to/vault -- npx -y seekstone
|
|
@@ -370,8 +379,39 @@ function resolveLink(target, notes2) {
|
|
|
370
379
|
|
|
371
380
|
// src/tools/search.ts
|
|
372
381
|
import { z as z2 } from "zod";
|
|
382
|
+
|
|
383
|
+
// src/semantic/excerpt.ts
|
|
384
|
+
function chunkExcerpt(body, span, terms, maxLen) {
|
|
385
|
+
const normalized = body.includes("\r") ? body.replace(/\r\n/g, "\n") : body;
|
|
386
|
+
const start = Math.min(span.start, normalized.length);
|
|
387
|
+
const end = Math.min(Math.max(span.end, start), normalized.length);
|
|
388
|
+
const text = start === end ? normalized : normalized.slice(start, end);
|
|
389
|
+
return extractExcerpt(text, terms, maxLen);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// src/semantic/route.ts
|
|
393
|
+
var MAX_ROUTE_WORDS = 6;
|
|
394
|
+
function queryWords(query) {
|
|
395
|
+
return query.toLowerCase().split(/\W+/).filter(Boolean);
|
|
396
|
+
}
|
|
397
|
+
function routeToLexical(query, topPaths, topN = 3) {
|
|
398
|
+
const words = new Set(queryWords(query));
|
|
399
|
+
if (words.size === 0) return false;
|
|
400
|
+
return topPaths.slice(0, topN).some((path2) => {
|
|
401
|
+
const base = (path2.split(/[\\/]/).pop() ?? "").replace(/\.md$/, "").toLowerCase();
|
|
402
|
+
const parts = new Set(base.split(/\W+/).filter(Boolean));
|
|
403
|
+
return parts.size === words.size && [...parts].every((p) => words.has(p));
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// src/tools/search.ts
|
|
373
408
|
var SearchInput = z2.object({
|
|
374
409
|
query: z2.string().min(1).describe("Search query. Supports fuzzy matching and prefix search."),
|
|
410
|
+
// Optional (not .default()) so the inferred type keeps existing direct
|
|
411
|
+
// callers compiling; search() itself defaults it to 'lexical'.
|
|
412
|
+
mode: z2.enum(["lexical", "semantic", "hybrid"]).optional().describe(
|
|
413
|
+
"lexical = keyword search (default). semantic = meaning-based search over local embeddings (requires SEEKSTONE_SEMANTIC=1 and a fetched model). hybrid = exact-title lookups go lexical, everything else semantic."
|
|
414
|
+
),
|
|
375
415
|
limit: z2.number().int().min(1).max(50).default(10).describe("Maximum number of results to return."),
|
|
376
416
|
folder: z2.string().optional().describe("Restrict results to notes under this vault-relative folder prefix."),
|
|
377
417
|
tag: z2.string().optional().describe("Restrict results to notes containing this tag."),
|
|
@@ -379,44 +419,113 @@ var SearchInput = z2.object({
|
|
|
379
419
|
"Max characters of match context per hit (default 120). Lower trims payload; higher gives more context."
|
|
380
420
|
)
|
|
381
421
|
});
|
|
422
|
+
var SEMANTIC_CANDIDATES = 50;
|
|
382
423
|
function basenameNoExt(path2) {
|
|
383
424
|
const base = path2.slice(path2.lastIndexOf("/") + 1);
|
|
384
425
|
const dot = base.lastIndexOf(".");
|
|
385
426
|
return dot > 0 ? base.slice(0, dot) : base;
|
|
386
427
|
}
|
|
387
428
|
function search(ctx2, input) {
|
|
429
|
+
const mode = input.mode ?? "lexical";
|
|
430
|
+
if (mode === "lexical") return lexicalSearch(ctx2, input);
|
|
431
|
+
const semantic = requireSemantic(ctx2);
|
|
432
|
+
if (mode === "semantic") return semanticSearch(ctx2, semantic, input);
|
|
433
|
+
const words = queryWords(input.query);
|
|
434
|
+
if (words.length > 0 && words.length <= MAX_ROUTE_WORDS) {
|
|
435
|
+
const lexHits = lexicalSearch(ctx2, input);
|
|
436
|
+
if (routeToLexical(
|
|
437
|
+
input.query,
|
|
438
|
+
lexHits.map((h) => h.path)
|
|
439
|
+
))
|
|
440
|
+
return lexHits;
|
|
441
|
+
}
|
|
442
|
+
return semanticSearch(ctx2, semantic, input);
|
|
443
|
+
}
|
|
444
|
+
function requireSemantic(ctx2) {
|
|
445
|
+
if (!ctx2.semantic) {
|
|
446
|
+
throw new Error(
|
|
447
|
+
JSON.stringify({
|
|
448
|
+
error: "semantic_unavailable",
|
|
449
|
+
hint: 'Semantic search is not enabled. Set SEEKSTONE_SEMANTIC=1 in the server env and run `npx -y seekstone fetch-model` once to download the local embedding model, then restart the session. Meanwhile, mode: "lexical" works.'
|
|
450
|
+
})
|
|
451
|
+
);
|
|
452
|
+
}
|
|
453
|
+
const progress = ctx2.semantic.progress;
|
|
454
|
+
if (progress.state === "building") {
|
|
455
|
+
throw new Error(
|
|
456
|
+
JSON.stringify({
|
|
457
|
+
error: "semantic_building",
|
|
458
|
+
done: progress.done,
|
|
459
|
+
total: progress.total,
|
|
460
|
+
hint: 'The semantic index is still embedding this vault (first run only \u2014 later runs load a cache). Retry shortly, or use mode: "lexical".'
|
|
461
|
+
})
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
return ctx2.semantic;
|
|
465
|
+
}
|
|
466
|
+
function searchTerms(query) {
|
|
467
|
+
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 1);
|
|
468
|
+
}
|
|
469
|
+
function lexicalSearch(ctx2, input) {
|
|
388
470
|
const results = ctx2.index.search(input.query, {
|
|
389
471
|
boost: { title: 3, tags: 2, body: 1 },
|
|
390
472
|
fuzzy: 0.2,
|
|
391
473
|
prefix: true
|
|
392
474
|
});
|
|
393
|
-
const terms = input.query
|
|
475
|
+
const terms = searchTerms(input.query);
|
|
394
476
|
const hits = [];
|
|
395
477
|
for (const r of results) {
|
|
396
|
-
|
|
397
|
-
if (
|
|
398
|
-
const note2 = ctx2.notes.get(r.id);
|
|
399
|
-
if (!note2) continue;
|
|
400
|
-
const noteTags2 = note2.tags.split(" ");
|
|
401
|
-
if (!noteTags2.some((t) => t === input.tag || t === input.tag?.replace(/^#/, ""))) continue;
|
|
402
|
-
}
|
|
403
|
-
const note = ctx2.notes.get(r.id);
|
|
404
|
-
const body = note?.body ?? "";
|
|
405
|
-
const noteTags = note?.tags ? note.tags.split(" ").filter(Boolean) : [];
|
|
406
|
-
const title = r.title;
|
|
478
|
+
const note = filterNote(ctx2, input, r.id);
|
|
479
|
+
if (!note) continue;
|
|
407
480
|
const hit = {
|
|
408
481
|
path: r.id,
|
|
409
482
|
// Round the raw MiniSearch score: 2 decimals preserve relevance gaps without 17-digit float tax.
|
|
410
483
|
score: Math.round(r.score * 100) / 100,
|
|
411
|
-
excerpt: extractExcerpt(body, terms, input.excerptLength ?? 120)
|
|
484
|
+
excerpt: extractExcerpt(note.body, terms, input.excerptLength ?? 120)
|
|
412
485
|
};
|
|
413
|
-
|
|
414
|
-
if (noteTags.length > 0) hit.tags = noteTags;
|
|
486
|
+
decorate(hit, r.title, note.tags);
|
|
415
487
|
hits.push(hit);
|
|
416
488
|
if (hits.length >= input.limit) break;
|
|
417
489
|
}
|
|
418
490
|
return hits;
|
|
419
491
|
}
|
|
492
|
+
function semanticSearch(ctx2, semantic, input) {
|
|
493
|
+
const queryVec = semantic.embedQuery(input.query);
|
|
494
|
+
const candidates = semantic.store.topNotes(queryVec, SEMANTIC_CANDIDATES);
|
|
495
|
+
const terms = searchTerms(input.query);
|
|
496
|
+
const hits = [];
|
|
497
|
+
for (const c of candidates) {
|
|
498
|
+
const note = filterNote(ctx2, input, c.path);
|
|
499
|
+
if (!note) continue;
|
|
500
|
+
const hit = {
|
|
501
|
+
path: c.path,
|
|
502
|
+
// Cosine similarity ∈ [-1, 1]: 3 decimals keep the ranking legible.
|
|
503
|
+
score: Math.round(c.score * 1e3) / 1e3,
|
|
504
|
+
excerpt: chunkExcerpt(note.body, c, terms, input.excerptLength ?? 120)
|
|
505
|
+
};
|
|
506
|
+
decorate(hit, note.title, note.tags);
|
|
507
|
+
hits.push(hit);
|
|
508
|
+
if (hits.length >= input.limit) break;
|
|
509
|
+
}
|
|
510
|
+
return hits;
|
|
511
|
+
}
|
|
512
|
+
function filterNote(ctx2, input, path2) {
|
|
513
|
+
if (input.folder && !path2.startsWith(input.folder)) return void 0;
|
|
514
|
+
const note = ctx2.notes.get(path2);
|
|
515
|
+
if (!note) return void 0;
|
|
516
|
+
if (input.tag) {
|
|
517
|
+
const noteTags = note.tags.split(" ");
|
|
518
|
+
if (!noteTags.some((t) => t === input.tag || t === input.tag?.replace(/^#/, ""))) {
|
|
519
|
+
return void 0;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
return note;
|
|
523
|
+
}
|
|
524
|
+
function decorate(hit, title, tags) {
|
|
525
|
+
if (title && title !== basenameNoExt(hit.path)) hit.title = title;
|
|
526
|
+
const noteTags = tags ? tags.split(" ").filter(Boolean) : [];
|
|
527
|
+
if (noteTags.length > 0) hit.tags = noteTags;
|
|
528
|
+
}
|
|
420
529
|
|
|
421
530
|
// src/tools/context_pack.ts
|
|
422
531
|
var ContextPackInput = z3.object({
|
|
@@ -701,7 +810,7 @@ ${stringifyYaml(input.frontmatter, { lineWidth: 0 })}---
|
|
|
701
810
|
}
|
|
702
811
|
|
|
703
812
|
// src/tools/delete_note.ts
|
|
704
|
-
import { access as access2, mkdir as mkdir2, rename as rename2, rm } from "fs/promises";
|
|
813
|
+
import { access as access2, mkdir as mkdir2, readFile as readFile3, rename as rename2, rm } from "fs/promises";
|
|
705
814
|
import { basename as basename2, extname as extname2, join } from "path";
|
|
706
815
|
import { z as z5 } from "zod";
|
|
707
816
|
|
|
@@ -742,6 +851,9 @@ var DeleteNoteInput = z5.object({
|
|
|
742
851
|
path: z5.string().describe("Vault-relative path of the note to delete."),
|
|
743
852
|
permanent: z5.boolean().default(false).describe(
|
|
744
853
|
"Permanently remove the note instead of moving it to the vault .trash/ folder. Defaults to false \u2014 deletes are recoverable."
|
|
854
|
+
),
|
|
855
|
+
prevHash: z5.string().optional().describe(
|
|
856
|
+
"Optional compare-and-swap guard: the contentHash from a prior read. Fails with hash_conflict if the note changed since \u2014 so you never delete content you have not seen."
|
|
745
857
|
)
|
|
746
858
|
});
|
|
747
859
|
async function freeTrashPath(vaultRoot2, noteBase) {
|
|
@@ -761,11 +873,12 @@ async function deleteNote(ctx2, rawInput) {
|
|
|
761
873
|
const input = DeleteNoteInput.parse(rawInput);
|
|
762
874
|
const absPath = resolveVaultPath(ctx2.vaultRoot, input.path);
|
|
763
875
|
assertWritable(ctx2.policy, input.path);
|
|
876
|
+
const raw = await readFile3(absPath, "utf8");
|
|
877
|
+
if (input.prevHash !== void 0) assertHashMatch(raw, input.prevHash, input.path);
|
|
764
878
|
let trashedTo;
|
|
765
879
|
if (input.permanent) {
|
|
766
880
|
await rm(absPath);
|
|
767
881
|
} else {
|
|
768
|
-
await access2(absPath);
|
|
769
882
|
await mkdir2(join(ctx2.vaultRoot, ".trash"), { recursive: true });
|
|
770
883
|
trashedTo = await freeTrashPath(ctx2.vaultRoot, basename2(input.path));
|
|
771
884
|
await rename2(absPath, join(ctx2.vaultRoot, trashedTo));
|
|
@@ -775,7 +888,11 @@ async function deleteNote(ctx2, rawInput) {
|
|
|
775
888
|
ctx2.index.discard(input.path);
|
|
776
889
|
ctx2.notes.delete(input.path);
|
|
777
890
|
}
|
|
778
|
-
const result = {
|
|
891
|
+
const result = {
|
|
892
|
+
path: input.path,
|
|
893
|
+
permanent: input.permanent,
|
|
894
|
+
contentHash: contentHash(raw)
|
|
895
|
+
};
|
|
779
896
|
if (trashedTo !== void 0) result.trashedTo = trashedTo;
|
|
780
897
|
return result;
|
|
781
898
|
}
|
|
@@ -903,7 +1020,7 @@ function listTags(ctx2, input) {
|
|
|
903
1020
|
}
|
|
904
1021
|
|
|
905
1022
|
// src/tools/move_note.ts
|
|
906
|
-
import { access as access3, mkdir as mkdir3, rename as rename3 } from "fs/promises";
|
|
1023
|
+
import { access as access3, mkdir as mkdir3, readFile as readFile4, rename as rename3 } from "fs/promises";
|
|
907
1024
|
import { dirname as dirname2, join as join2 } from "path";
|
|
908
1025
|
import { z as z10 } from "zod";
|
|
909
1026
|
|
|
@@ -1006,6 +1123,9 @@ var MoveNoteInput = z10.object({
|
|
|
1006
1123
|
overwrite: z10.boolean().default(false).describe("Overwrite the destination if it exists. Defaults to false."),
|
|
1007
1124
|
rewriteLinks: z10.boolean().default(true).describe(
|
|
1008
1125
|
"Rewrite wikilinks and markdown links in other notes that point at the moved note, so nothing breaks. Defaults to true; pass false to move the file only."
|
|
1126
|
+
),
|
|
1127
|
+
prevHash: z10.string().optional().describe(
|
|
1128
|
+
"Optional compare-and-swap guard on the source note: the contentHash from a prior read. Fails with hash_conflict if the note changed since."
|
|
1009
1129
|
)
|
|
1010
1130
|
});
|
|
1011
1131
|
async function moveNote(ctx2, rawInput) {
|
|
@@ -1014,11 +1134,13 @@ async function moveNote(ctx2, rawInput) {
|
|
|
1014
1134
|
const absTo = resolveVaultPath(ctx2.vaultRoot, input.to);
|
|
1015
1135
|
assertWritable(ctx2.policy, input.from);
|
|
1016
1136
|
assertWritable(ctx2.policy, input.to);
|
|
1137
|
+
let raw;
|
|
1017
1138
|
try {
|
|
1018
|
-
await
|
|
1139
|
+
raw = await readFile4(absFrom, "utf8");
|
|
1019
1140
|
} catch {
|
|
1020
1141
|
throw new Error(`Note not found: ${input.from}`);
|
|
1021
1142
|
}
|
|
1143
|
+
if (input.prevHash !== void 0) assertHashMatch(raw, input.prevHash, input.from);
|
|
1022
1144
|
if (!input.overwrite) {
|
|
1023
1145
|
try {
|
|
1024
1146
|
await access3(absTo);
|
|
@@ -1041,7 +1163,6 @@ async function moveNote(ctx2, rawInput) {
|
|
|
1041
1163
|
}
|
|
1042
1164
|
await mkdir3(dirname2(absTo), { recursive: true });
|
|
1043
1165
|
await rename3(absFrom, absTo);
|
|
1044
|
-
const raw = ctx2.notes.get(input.from)?.raw ?? "";
|
|
1045
1166
|
removeNoteBacklinks(ctx2, input.from);
|
|
1046
1167
|
if (ctx2.notes.has(input.from)) {
|
|
1047
1168
|
ctx2.index.discard(input.from);
|
|
@@ -1079,13 +1200,19 @@ async function moveNote(ctx2, rawInput) {
|
|
|
1079
1200
|
}
|
|
1080
1201
|
}
|
|
1081
1202
|
ctx2.backlinks.delete(input.from);
|
|
1082
|
-
const result = {
|
|
1203
|
+
const result = {
|
|
1204
|
+
from: input.from,
|
|
1205
|
+
to: input.to,
|
|
1206
|
+
notesRewritten,
|
|
1207
|
+
linksRewritten,
|
|
1208
|
+
contentHash: contentHash(raw)
|
|
1209
|
+
};
|
|
1083
1210
|
if (skipped.length > 0) result.skipped = skipped;
|
|
1084
1211
|
return result;
|
|
1085
1212
|
}
|
|
1086
1213
|
|
|
1087
1214
|
// src/tools/outline_note.ts
|
|
1088
|
-
import { readFile as
|
|
1215
|
+
import { readFile as readFile5 } from "fs/promises";
|
|
1089
1216
|
|
|
1090
1217
|
// ../core/src/outline.ts
|
|
1091
1218
|
var HEADING_RE = /^(#{1,6})\s+(.+)/;
|
|
@@ -1143,7 +1270,7 @@ var OutlineNoteInput = z11.object({
|
|
|
1143
1270
|
});
|
|
1144
1271
|
async function outlineNote(ctx2, input) {
|
|
1145
1272
|
const absPath = resolveVaultPath(ctx2.vaultRoot, input.path);
|
|
1146
|
-
const raw = await
|
|
1273
|
+
const raw = await readFile5(absPath, "utf8");
|
|
1147
1274
|
return buildOutline(raw, {
|
|
1148
1275
|
includeBlocks: input.includeBlocks,
|
|
1149
1276
|
includeSizes: input.includeSizes
|
|
@@ -1151,7 +1278,7 @@ async function outlineNote(ctx2, input) {
|
|
|
1151
1278
|
}
|
|
1152
1279
|
|
|
1153
1280
|
// src/tools/patch_frontmatter.ts
|
|
1154
|
-
import { readFile as
|
|
1281
|
+
import { readFile as readFile6 } from "fs/promises";
|
|
1155
1282
|
import { parseDocument as parseDocument2 } from "yaml";
|
|
1156
1283
|
import { z as z12 } from "zod";
|
|
1157
1284
|
var PatchFrontmatterInput = z12.object({
|
|
@@ -1166,7 +1293,7 @@ var PatchFrontmatterInput = z12.object({
|
|
|
1166
1293
|
async function patchFrontmatter(ctx2, input) {
|
|
1167
1294
|
const absPath = resolveVaultPath(ctx2.vaultRoot, input.path);
|
|
1168
1295
|
assertWritable(ctx2.policy, input.path);
|
|
1169
|
-
const original = await
|
|
1296
|
+
const original = await readFile6(absPath, "utf8");
|
|
1170
1297
|
if (input.prevHash !== void 0) assertHashMatch(original, input.prevHash, input.path);
|
|
1171
1298
|
const fm = parseFrontmatter(original);
|
|
1172
1299
|
const keysChanged = [];
|
|
@@ -1226,7 +1353,7 @@ ${original}`;
|
|
|
1226
1353
|
}
|
|
1227
1354
|
|
|
1228
1355
|
// src/tools/patch_note.ts
|
|
1229
|
-
import { readFile as
|
|
1356
|
+
import { readFile as readFile7 } from "fs/promises";
|
|
1230
1357
|
import { z as z13 } from "zod";
|
|
1231
1358
|
var PatchNoteInput = z13.object({
|
|
1232
1359
|
path: z13.string().describe("Vault-relative path to the note."),
|
|
@@ -1257,7 +1384,7 @@ function lineStart(raw, offset) {
|
|
|
1257
1384
|
async function patchNote(ctx2, input) {
|
|
1258
1385
|
const absPath = resolveVaultPath(ctx2.vaultRoot, input.path);
|
|
1259
1386
|
assertWritable(ctx2.policy, input.path);
|
|
1260
|
-
const raw = await
|
|
1387
|
+
const raw = await readFile7(absPath, "utf8");
|
|
1261
1388
|
if (input.prevHash !== void 0) assertHashMatch(raw, input.prevHash, input.path);
|
|
1262
1389
|
const fm = parseFrontmatter(raw);
|
|
1263
1390
|
const originalFmRegion = raw.slice(0, fm.bodyStart);
|
|
@@ -1357,7 +1484,7 @@ ${input.content}${contentNl}`;
|
|
|
1357
1484
|
};
|
|
1358
1485
|
}
|
|
1359
1486
|
async function verifyFrontmatter(absPath, originalFmRegion) {
|
|
1360
|
-
const written = await
|
|
1487
|
+
const written = await readFile7(absPath, "utf8");
|
|
1361
1488
|
if (written.slice(0, originalFmRegion.length) !== originalFmRegion) {
|
|
1362
1489
|
throw new Error("Write-safety violation: frontmatter region changed unexpectedly");
|
|
1363
1490
|
}
|
|
@@ -1372,7 +1499,7 @@ function updateCache(ctx2, path2, newContent, bodyStart) {
|
|
|
1372
1499
|
}
|
|
1373
1500
|
|
|
1374
1501
|
// src/tools/periodic_note.ts
|
|
1375
|
-
import { access as access4, mkdir as mkdir4, readFile as
|
|
1502
|
+
import { access as access4, mkdir as mkdir4, readFile as readFile8 } from "fs/promises";
|
|
1376
1503
|
import { dirname as dirname3, join as join3 } from "path";
|
|
1377
1504
|
import { z as z14 } from "zod";
|
|
1378
1505
|
var PERIOD = z14.enum(["daily", "weekly", "monthly", "quarterly", "yearly"]);
|
|
@@ -1385,7 +1512,7 @@ var DEFAULTS = {
|
|
|
1385
1512
|
};
|
|
1386
1513
|
async function readJson(absPath) {
|
|
1387
1514
|
try {
|
|
1388
|
-
return JSON.parse(await
|
|
1515
|
+
return JSON.parse(await readFile8(absPath, "utf8"));
|
|
1389
1516
|
} catch {
|
|
1390
1517
|
return null;
|
|
1391
1518
|
}
|
|
@@ -1471,7 +1598,7 @@ async function getPeriodicNote(ctx2, rawInput) {
|
|
|
1471
1598
|
ctx2.vaultRoot,
|
|
1472
1599
|
cfg.template.endsWith(".md") ? cfg.template : `${cfg.template}.md`
|
|
1473
1600
|
);
|
|
1474
|
-
body = await
|
|
1601
|
+
body = await readFile8(templateAbs, "utf8");
|
|
1475
1602
|
} catch {
|
|
1476
1603
|
}
|
|
1477
1604
|
}
|
|
@@ -1498,7 +1625,7 @@ async function appendPeriodicNote(ctx2, rawInput) {
|
|
|
1498
1625
|
assertWritable(ctx2.policy, path2);
|
|
1499
1626
|
let original = "";
|
|
1500
1627
|
try {
|
|
1501
|
-
original = await
|
|
1628
|
+
original = await readFile8(abs, "utf8");
|
|
1502
1629
|
if (input.prevHash !== void 0) assertHashMatch(original, input.prevHash, path2);
|
|
1503
1630
|
} catch (err) {
|
|
1504
1631
|
if (err.code !== "ENOENT") throw err;
|
|
@@ -1671,7 +1798,7 @@ function queryNotes(ctx2, input) {
|
|
|
1671
1798
|
}
|
|
1672
1799
|
|
|
1673
1800
|
// src/tools/read_note.ts
|
|
1674
|
-
import { readFile as
|
|
1801
|
+
import { readFile as readFile9 } from "fs/promises";
|
|
1675
1802
|
import { z as z16 } from "zod";
|
|
1676
1803
|
var ReadNoteInput = z16.object({
|
|
1677
1804
|
path: z16.string().describe('Vault-relative path to the note, e.g. "Daily Notes/2026-05-29.md".'),
|
|
@@ -1689,7 +1816,7 @@ var ReadNoteInput = z16.object({
|
|
|
1689
1816
|
});
|
|
1690
1817
|
async function readNote(ctx2, input) {
|
|
1691
1818
|
const absPath = resolveVaultPath(ctx2.vaultRoot, input.path);
|
|
1692
|
-
const raw = await
|
|
1819
|
+
const raw = await readFile9(absPath, "utf8");
|
|
1693
1820
|
const noteBytes = Buffer.byteLength(raw, "utf8");
|
|
1694
1821
|
const hash = contentHash(raw);
|
|
1695
1822
|
if (input.section === void 0 && input.block === void 0 && input.lines === void 0) {
|
|
@@ -1777,7 +1904,7 @@ async function readNote(ctx2, input) {
|
|
|
1777
1904
|
}
|
|
1778
1905
|
|
|
1779
1906
|
// src/tools/rename_heading.ts
|
|
1780
|
-
import { readFile as
|
|
1907
|
+
import { readFile as readFile10 } from "fs/promises";
|
|
1781
1908
|
import { join as join4 } from "path";
|
|
1782
1909
|
import { z as z17 } from "zod";
|
|
1783
1910
|
var RenameHeadingInput = z17.object({
|
|
@@ -1818,7 +1945,7 @@ async function renameHeading(ctx2, rawInput) {
|
|
|
1818
1945
|
assertWritable(ctx2.policy, input.path);
|
|
1819
1946
|
let raw;
|
|
1820
1947
|
try {
|
|
1821
|
-
raw = await
|
|
1948
|
+
raw = await readFile10(absPath, "utf8");
|
|
1822
1949
|
} catch {
|
|
1823
1950
|
throw new Error(`Note not found: ${input.path}`);
|
|
1824
1951
|
}
|
|
@@ -1854,7 +1981,7 @@ async function renameHeading(ctx2, rawInput) {
|
|
|
1854
1981
|
newRaw = newRaw.slice(0, fm.bodyStart) + selfRewrite.content;
|
|
1855
1982
|
const originalFmRegion = raw.slice(0, fm.bodyStart);
|
|
1856
1983
|
await atomicWrite(absPath, newRaw);
|
|
1857
|
-
const written = await
|
|
1984
|
+
const written = await readFile10(absPath, "utf8");
|
|
1858
1985
|
if (written.slice(0, originalFmRegion.length) !== originalFmRegion) {
|
|
1859
1986
|
throw new Error("Write-safety violation: frontmatter region changed unexpectedly");
|
|
1860
1987
|
}
|
|
@@ -1900,7 +2027,7 @@ async function renameHeading(ctx2, rawInput) {
|
|
|
1900
2027
|
}
|
|
1901
2028
|
|
|
1902
2029
|
// src/tools/replace_in_note.ts
|
|
1903
|
-
import { readFile as
|
|
2030
|
+
import { readFile as readFile11 } from "fs/promises";
|
|
1904
2031
|
import { z as z18 } from "zod";
|
|
1905
2032
|
var ReplaceInNoteInput = z18.object({
|
|
1906
2033
|
path: z18.string().describe("Vault-relative path to the note."),
|
|
@@ -1937,7 +2064,7 @@ async function replaceInNote(ctx2, input) {
|
|
|
1937
2064
|
} catch (err) {
|
|
1938
2065
|
throw new Error(`Invalid regex: ${err instanceof Error ? err.message : String(err)}`);
|
|
1939
2066
|
}
|
|
1940
|
-
const raw = await
|
|
2067
|
+
const raw = await readFile11(absPath, "utf8");
|
|
1941
2068
|
if (input.prevHash !== void 0) assertHashMatch(raw, input.prevHash, input.path);
|
|
1942
2069
|
const fm = parseFrontmatter(raw);
|
|
1943
2070
|
const originalFmRegion = raw.slice(0, fm.bodyStart);
|
|
@@ -1954,7 +2081,10 @@ async function replaceInNote(ctx2, input) {
|
|
|
1954
2081
|
path: input.path,
|
|
1955
2082
|
replacements: toReplace.length,
|
|
1956
2083
|
matches: matches2,
|
|
1957
|
-
frontmatterUnchanged: true
|
|
2084
|
+
frontmatterUnchanged: true,
|
|
2085
|
+
// Nothing was written — hash the content as read, so chained edits can
|
|
2086
|
+
// proceed from a dry run without a re-read.
|
|
2087
|
+
contentHash: contentHash(raw)
|
|
1958
2088
|
};
|
|
1959
2089
|
}
|
|
1960
2090
|
const singleRe = new RegExp(re.source, re.flags.replace("g", ""));
|
|
@@ -1969,7 +2099,7 @@ async function replaceInNote(ctx2, input) {
|
|
|
1969
2099
|
newBody += body.slice(lastIndex);
|
|
1970
2100
|
const newContent = `${originalFmRegion}${newBody}`;
|
|
1971
2101
|
await atomicWrite(absPath, newContent);
|
|
1972
|
-
const written = await
|
|
2102
|
+
const written = await readFile11(absPath, "utf8");
|
|
1973
2103
|
if (written.slice(0, originalFmRegion.length) !== originalFmRegion) {
|
|
1974
2104
|
throw new Error("Write-safety violation: frontmatter region changed unexpectedly");
|
|
1975
2105
|
}
|
|
@@ -2212,7 +2342,7 @@ async function run(ctx2, name, args) {
|
|
|
2212
2342
|
}
|
|
2213
2343
|
|
|
2214
2344
|
// src/index/build.ts
|
|
2215
|
-
import { readFile as
|
|
2345
|
+
import { readFile as readFile12 } from "fs/promises";
|
|
2216
2346
|
import { basename as basename3, extname as extname4 } from "path";
|
|
2217
2347
|
|
|
2218
2348
|
// ../core/src/walk.ts
|
|
@@ -2294,7 +2424,7 @@ async function buildIndex(vaultRoot2) {
|
|
|
2294
2424
|
await Promise.all(
|
|
2295
2425
|
noteEntries.map(async (entry) => {
|
|
2296
2426
|
try {
|
|
2297
|
-
const raw = await
|
|
2427
|
+
const raw = await readFile12(entry.absPath, "utf8");
|
|
2298
2428
|
const fm = parseFrontmatter(raw);
|
|
2299
2429
|
const inlineTags = extractInlineTags(fm.body);
|
|
2300
2430
|
const fmTags = frontmatterTags(fm.data);
|
|
@@ -2347,7 +2477,7 @@ function buildBacklinks(notes2) {
|
|
|
2347
2477
|
|
|
2348
2478
|
// src/init.ts
|
|
2349
2479
|
import { execFileSync } from "child_process";
|
|
2350
|
-
import { access as access5, copyFile, mkdir as mkdir5, readdir, readFile as
|
|
2480
|
+
import { access as access5, copyFile, mkdir as mkdir5, readdir, readFile as readFile13, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
2351
2481
|
import path, { dirname as dirname4, join as join6 } from "path";
|
|
2352
2482
|
function parseInitArgs(argv) {
|
|
2353
2483
|
const opts = { write: false, client: "desktop" };
|
|
@@ -2491,7 +2621,7 @@ async function runInit(opts, deps) {
|
|
|
2491
2621
|
]
|
|
2492
2622
|
};
|
|
2493
2623
|
}
|
|
2494
|
-
const rf = deps.readFile ??
|
|
2624
|
+
const rf = deps.readFile ?? readFile13;
|
|
2495
2625
|
let vaultPath = opts.vault ?? deps.env.SEEKSTONE_VAULT;
|
|
2496
2626
|
if (!vaultPath) {
|
|
2497
2627
|
const registryPath = obsidianRegistryPath(deps.platform, {
|
|
@@ -2698,7 +2828,7 @@ function createLogger(opts = {}) {
|
|
|
2698
2828
|
);
|
|
2699
2829
|
}
|
|
2700
2830
|
}
|
|
2701
|
-
function
|
|
2831
|
+
function writeFile4(record) {
|
|
2702
2832
|
if (!fileOk || filePath === void 0) return;
|
|
2703
2833
|
const line = `${JSON.stringify(record)}
|
|
2704
2834
|
`;
|
|
@@ -2720,7 +2850,7 @@ function createLogger(opts = {}) {
|
|
|
2720
2850
|
const f = fields ?? {};
|
|
2721
2851
|
stderr(`${lvl.toUpperCase().padEnd(5)} ${ts} ${msg}${fmtFields(f)}
|
|
2722
2852
|
`);
|
|
2723
|
-
|
|
2853
|
+
writeFile4({ t: ts, level: lvl, msg, ...f });
|
|
2724
2854
|
}
|
|
2725
2855
|
return {
|
|
2726
2856
|
level,
|
|
@@ -2751,6 +2881,664 @@ function installProcessGuards(log2) {
|
|
|
2751
2881
|
};
|
|
2752
2882
|
}
|
|
2753
2883
|
|
|
2884
|
+
// ../core/src/embed/chunk.ts
|
|
2885
|
+
var MIN_WINDOW_WORDS = 250;
|
|
2886
|
+
var MAX_WINDOW_WORDS = 400;
|
|
2887
|
+
function chunkNote(title, body) {
|
|
2888
|
+
const normalized = body.replace(/\r\n/g, "\n");
|
|
2889
|
+
const paragraphs = splitParagraphs(normalized).flatMap(splitOversized);
|
|
2890
|
+
if (paragraphs.length === 0) return [{ text: title, words: 0, start: 0, end: 0 }];
|
|
2891
|
+
const chunks = [];
|
|
2892
|
+
let window = [];
|
|
2893
|
+
let windowWords = 0;
|
|
2894
|
+
const flush = () => {
|
|
2895
|
+
const first = window[0];
|
|
2896
|
+
const last = window[window.length - 1];
|
|
2897
|
+
if (!first || !last) return;
|
|
2898
|
+
chunks.push({
|
|
2899
|
+
text: `${title}
|
|
2900
|
+
|
|
2901
|
+
${window.map((p) => p.text).join("\n\n")}`,
|
|
2902
|
+
words: windowWords,
|
|
2903
|
+
start: first.start,
|
|
2904
|
+
end: last.end
|
|
2905
|
+
});
|
|
2906
|
+
window = [];
|
|
2907
|
+
windowWords = 0;
|
|
2908
|
+
};
|
|
2909
|
+
for (const para of paragraphs) {
|
|
2910
|
+
if (windowWords >= MIN_WINDOW_WORDS || windowWords + para.words > MAX_WINDOW_WORDS) flush();
|
|
2911
|
+
window.push(para);
|
|
2912
|
+
windowWords += para.words;
|
|
2913
|
+
}
|
|
2914
|
+
flush();
|
|
2915
|
+
return chunks;
|
|
2916
|
+
}
|
|
2917
|
+
function splitParagraphs(normalized) {
|
|
2918
|
+
const out = [];
|
|
2919
|
+
const push = (rawStart, raw) => {
|
|
2920
|
+
const text = raw.trim();
|
|
2921
|
+
if (text.length === 0) return;
|
|
2922
|
+
const start = rawStart + (raw.length - raw.trimStart().length);
|
|
2923
|
+
out.push({ text, words: countWords(text), start, end: start + text.length });
|
|
2924
|
+
};
|
|
2925
|
+
const sep4 = /\n[ \t]*\n+/g;
|
|
2926
|
+
let prev = 0;
|
|
2927
|
+
for (const m of normalized.matchAll(sep4)) {
|
|
2928
|
+
push(prev, normalized.slice(prev, m.index));
|
|
2929
|
+
prev = m.index + m[0].length;
|
|
2930
|
+
}
|
|
2931
|
+
push(prev, normalized.slice(prev));
|
|
2932
|
+
return out;
|
|
2933
|
+
}
|
|
2934
|
+
function splitOversized(para) {
|
|
2935
|
+
if (para.words <= MAX_WINDOW_WORDS) return [para];
|
|
2936
|
+
const words = [...para.text.matchAll(/\S+/g)];
|
|
2937
|
+
const pieces = [];
|
|
2938
|
+
for (let i = 0; i < words.length; i += MAX_WINDOW_WORDS) {
|
|
2939
|
+
const slice = words.slice(i, i + MAX_WINDOW_WORDS);
|
|
2940
|
+
const first = slice[0];
|
|
2941
|
+
const last = slice[slice.length - 1];
|
|
2942
|
+
if (!first || !last) continue;
|
|
2943
|
+
pieces.push({
|
|
2944
|
+
text: slice.map((w) => w[0]).join(" "),
|
|
2945
|
+
words: slice.length,
|
|
2946
|
+
start: para.start + first.index,
|
|
2947
|
+
end: para.start + last.index + last[0].length
|
|
2948
|
+
});
|
|
2949
|
+
}
|
|
2950
|
+
return pieces;
|
|
2951
|
+
}
|
|
2952
|
+
function countWords(text) {
|
|
2953
|
+
const m = text.match(/\S+/g);
|
|
2954
|
+
return m ? m.length : 0;
|
|
2955
|
+
}
|
|
2956
|
+
|
|
2957
|
+
// ../core/src/embed/model2vec.ts
|
|
2958
|
+
import { readFile as readFile14 } from "fs/promises";
|
|
2959
|
+
import { basename as basename4, join as join7 } from "path";
|
|
2960
|
+
|
|
2961
|
+
// ../core/src/embed/safetensors.ts
|
|
2962
|
+
function readSafetensors(buf) {
|
|
2963
|
+
if (buf.byteLength < 8) {
|
|
2964
|
+
throw new Error(`safetensors: buffer too small (${buf.byteLength} bytes) to hold a header`);
|
|
2965
|
+
}
|
|
2966
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
2967
|
+
const headerLen = Number(view.getBigUint64(0, true));
|
|
2968
|
+
if (8 + headerLen > buf.byteLength) {
|
|
2969
|
+
throw new Error(
|
|
2970
|
+
`safetensors: header length ${headerLen} exceeds buffer (${buf.byteLength} bytes)`
|
|
2971
|
+
);
|
|
2972
|
+
}
|
|
2973
|
+
const header = JSON.parse(new TextDecoder().decode(buf.subarray(8, 8 + headerLen)));
|
|
2974
|
+
const dataStart = 8 + headerLen;
|
|
2975
|
+
const out = /* @__PURE__ */ new Map();
|
|
2976
|
+
const unsupported = [];
|
|
2977
|
+
for (const [name, meta] of Object.entries(header)) {
|
|
2978
|
+
if (name === "__metadata__") continue;
|
|
2979
|
+
if (meta.dtype !== "F32") {
|
|
2980
|
+
unsupported.push(`${name} (${meta.dtype})`);
|
|
2981
|
+
continue;
|
|
2982
|
+
}
|
|
2983
|
+
const [begin, end] = meta.data_offsets;
|
|
2984
|
+
const byteLen = end - begin;
|
|
2985
|
+
const elems = meta.shape.reduce((a, b) => a * b, 1);
|
|
2986
|
+
if (byteLen !== elems * 4) {
|
|
2987
|
+
throw new Error(
|
|
2988
|
+
`safetensors: tensor "${name}" data span ${byteLen} B does not match shape [${meta.shape.join(", ")}]`
|
|
2989
|
+
);
|
|
2990
|
+
}
|
|
2991
|
+
if (dataStart + end > buf.byteLength) {
|
|
2992
|
+
throw new Error(`safetensors: tensor "${name}" extends past the end of the buffer`);
|
|
2993
|
+
}
|
|
2994
|
+
const absStart = buf.byteOffset + dataStart + begin;
|
|
2995
|
+
let data;
|
|
2996
|
+
if (absStart % 4 === 0) {
|
|
2997
|
+
data = new Float32Array(buf.buffer, absStart, elems);
|
|
2998
|
+
} else {
|
|
2999
|
+
const copy = new Uint8Array(byteLen);
|
|
3000
|
+
copy.set(buf.subarray(dataStart + begin, dataStart + end));
|
|
3001
|
+
data = new Float32Array(copy.buffer);
|
|
3002
|
+
}
|
|
3003
|
+
out.set(name, { dtype: meta.dtype, shape: meta.shape, data });
|
|
3004
|
+
}
|
|
3005
|
+
if (unsupported.length > 0) {
|
|
3006
|
+
throw new Error(`safetensors: only F32 tensors are supported; found ${unsupported.join(", ")}`);
|
|
3007
|
+
}
|
|
3008
|
+
return out;
|
|
3009
|
+
}
|
|
3010
|
+
|
|
3011
|
+
// ../core/src/embed/tokenizer.ts
|
|
3012
|
+
function loadWordPieceTokenizer(tokenizerJson) {
|
|
3013
|
+
const tj = tokenizerJson;
|
|
3014
|
+
const model = tj?.model;
|
|
3015
|
+
if (model?.type !== "WordPiece" || !model.vocab) {
|
|
3016
|
+
throw new Error(
|
|
3017
|
+
`tokenizer: unsupported model type "${model?.type ?? "missing"}" (only WordPiece is implemented)`
|
|
3018
|
+
);
|
|
3019
|
+
}
|
|
3020
|
+
const vocab = new Map(Object.entries(model.vocab));
|
|
3021
|
+
const unkToken = model.unk_token ?? "[UNK]";
|
|
3022
|
+
const unkId = vocab.get(unkToken);
|
|
3023
|
+
if (unkId === void 0) {
|
|
3024
|
+
throw new Error(`tokenizer: unk_token "${unkToken}" is not in the vocab`);
|
|
3025
|
+
}
|
|
3026
|
+
const prefix = model.continuing_subword_prefix ?? "##";
|
|
3027
|
+
const maxChars = model.max_input_chars_per_word ?? 100;
|
|
3028
|
+
const norm = tj.normalizer ?? {};
|
|
3029
|
+
const cleanText = norm.clean_text ?? true;
|
|
3030
|
+
const handleCjk = norm.handle_chinese_chars ?? true;
|
|
3031
|
+
const lowercase = norm.lowercase ?? true;
|
|
3032
|
+
const stripAccents = norm.strip_accents ?? lowercase;
|
|
3033
|
+
const specialIds = new Set(
|
|
3034
|
+
(tj.added_tokens ?? []).filter((t) => t.special).map((t) => t.id)
|
|
3035
|
+
);
|
|
3036
|
+
function normalize(text) {
|
|
3037
|
+
let out = cleanText ? doCleanText(text) : text;
|
|
3038
|
+
if (handleCjk) out = padCjkChars(out);
|
|
3039
|
+
if (stripAccents) out = out.normalize("NFD").replace(new RegExp("\\p{Mn}", "gu"), "");
|
|
3040
|
+
if (lowercase) out = out.toLowerCase();
|
|
3041
|
+
return out;
|
|
3042
|
+
}
|
|
3043
|
+
function wordPiece(word) {
|
|
3044
|
+
const chars = [...word];
|
|
3045
|
+
if (chars.length > maxChars) return [unkId];
|
|
3046
|
+
const ids = [];
|
|
3047
|
+
let start = 0;
|
|
3048
|
+
while (start < chars.length) {
|
|
3049
|
+
let end = chars.length;
|
|
3050
|
+
let id;
|
|
3051
|
+
while (start < end) {
|
|
3052
|
+
const piece = (start > 0 ? prefix : "") + chars.slice(start, end).join("");
|
|
3053
|
+
const found = vocab.get(piece);
|
|
3054
|
+
if (found !== void 0) {
|
|
3055
|
+
id = found;
|
|
3056
|
+
break;
|
|
3057
|
+
}
|
|
3058
|
+
end--;
|
|
3059
|
+
}
|
|
3060
|
+
if (id === void 0) return [unkId];
|
|
3061
|
+
ids.push(id);
|
|
3062
|
+
start = end;
|
|
3063
|
+
}
|
|
3064
|
+
return ids;
|
|
3065
|
+
}
|
|
3066
|
+
return {
|
|
3067
|
+
vocabSize: vocab.size,
|
|
3068
|
+
unkId,
|
|
3069
|
+
specialIds,
|
|
3070
|
+
encode(text) {
|
|
3071
|
+
const ids = [];
|
|
3072
|
+
for (const word of preTokenize(normalize(text))) {
|
|
3073
|
+
for (const id of wordPiece(word)) ids.push(id);
|
|
3074
|
+
}
|
|
3075
|
+
return ids;
|
|
3076
|
+
}
|
|
3077
|
+
};
|
|
3078
|
+
}
|
|
3079
|
+
function preTokenize(text) {
|
|
3080
|
+
const words = [];
|
|
3081
|
+
let cur = "";
|
|
3082
|
+
for (const ch of text) {
|
|
3083
|
+
const cp = ch.codePointAt(0);
|
|
3084
|
+
if (isWhitespace(ch, cp)) {
|
|
3085
|
+
if (cur) {
|
|
3086
|
+
words.push(cur);
|
|
3087
|
+
cur = "";
|
|
3088
|
+
}
|
|
3089
|
+
} else if (isPunctuation(ch, cp)) {
|
|
3090
|
+
if (cur) {
|
|
3091
|
+
words.push(cur);
|
|
3092
|
+
cur = "";
|
|
3093
|
+
}
|
|
3094
|
+
words.push(ch);
|
|
3095
|
+
} else {
|
|
3096
|
+
cur += ch;
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
if (cur) words.push(cur);
|
|
3100
|
+
return words;
|
|
3101
|
+
}
|
|
3102
|
+
function doCleanText(text) {
|
|
3103
|
+
let out = "";
|
|
3104
|
+
for (const ch of text) {
|
|
3105
|
+
const cp = ch.codePointAt(0);
|
|
3106
|
+
if (cp === 0 || cp === 65533 || isControl(ch, cp)) continue;
|
|
3107
|
+
out += isWhitespace(ch, cp) ? " " : ch;
|
|
3108
|
+
}
|
|
3109
|
+
return out;
|
|
3110
|
+
}
|
|
3111
|
+
function padCjkChars(text) {
|
|
3112
|
+
let out = "";
|
|
3113
|
+
for (const ch of text) {
|
|
3114
|
+
out += isCjk(ch.codePointAt(0)) ? ` ${ch} ` : ch;
|
|
3115
|
+
}
|
|
3116
|
+
return out;
|
|
3117
|
+
}
|
|
3118
|
+
function isWhitespace(ch, cp) {
|
|
3119
|
+
if (cp === 32 || cp === 9 || cp === 10 || cp === 13) return true;
|
|
3120
|
+
return new RegExp("\\p{Zs}", "u").test(ch);
|
|
3121
|
+
}
|
|
3122
|
+
function isControl(ch, cp) {
|
|
3123
|
+
if (cp === 9 || cp === 10 || cp === 13) return false;
|
|
3124
|
+
return new RegExp("\\p{Cc}|\\p{Cf}", "u").test(ch);
|
|
3125
|
+
}
|
|
3126
|
+
function isPunctuation(ch, cp) {
|
|
3127
|
+
if (cp >= 33 && cp <= 47 || cp >= 58 && cp <= 64 || cp >= 91 && cp <= 96 || cp >= 123 && cp <= 126) {
|
|
3128
|
+
return true;
|
|
3129
|
+
}
|
|
3130
|
+
return new RegExp("\\p{P}", "u").test(ch);
|
|
3131
|
+
}
|
|
3132
|
+
function isCjk(cp) {
|
|
3133
|
+
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
3134
|
+
}
|
|
3135
|
+
|
|
3136
|
+
// ../core/src/embed/model2vec.ts
|
|
3137
|
+
var TENSOR_NAME = "embeddings";
|
|
3138
|
+
async function loadModel2Vec(modelDir) {
|
|
3139
|
+
const [configRaw, tokenizerRaw, weights] = await Promise.all([
|
|
3140
|
+
readFile14(join7(modelDir, "config.json"), "utf8"),
|
|
3141
|
+
readFile14(join7(modelDir, "tokenizer.json"), "utf8"),
|
|
3142
|
+
readFile14(join7(modelDir, "model.safetensors"))
|
|
3143
|
+
]);
|
|
3144
|
+
const config = JSON.parse(configRaw);
|
|
3145
|
+
const tokenizer = loadWordPieceTokenizer(JSON.parse(tokenizerRaw));
|
|
3146
|
+
const tensors = readSafetensors(weights);
|
|
3147
|
+
const tensor = tensors.get(TENSOR_NAME) ?? (tensors.size === 1 ? [...tensors.values()][0] : void 0);
|
|
3148
|
+
if (!tensor) {
|
|
3149
|
+
throw new Error(
|
|
3150
|
+
`model2vec: no "${TENSOR_NAME}" tensor in ${modelDir}; found: ${[...tensors.keys()].join(", ")}`
|
|
3151
|
+
);
|
|
3152
|
+
}
|
|
3153
|
+
if (tensor.shape.length !== 2) {
|
|
3154
|
+
throw new Error(`model2vec: expected a 2-D embedding matrix, got [${tensor.shape.join(", ")}]`);
|
|
3155
|
+
}
|
|
3156
|
+
const [rows, dim] = tensor.shape;
|
|
3157
|
+
if (rows !== tokenizer.vocabSize) {
|
|
3158
|
+
throw new Error(
|
|
3159
|
+
`model2vec: embedding rows (${rows}) do not match tokenizer vocab size (${tokenizer.vocabSize})`
|
|
3160
|
+
);
|
|
3161
|
+
}
|
|
3162
|
+
if (config.hidden_dim !== void 0 && config.hidden_dim !== dim) {
|
|
3163
|
+
throw new Error(
|
|
3164
|
+
`model2vec: config hidden_dim ${config.hidden_dim} does not match tensor dim ${dim}`
|
|
3165
|
+
);
|
|
3166
|
+
}
|
|
3167
|
+
const table = tensor.data;
|
|
3168
|
+
return {
|
|
3169
|
+
id: basename4(modelDir),
|
|
3170
|
+
dim,
|
|
3171
|
+
embed(text) {
|
|
3172
|
+
const out = new Float32Array(dim);
|
|
3173
|
+
let pooled = 0;
|
|
3174
|
+
for (const id of tokenizer.encode(text)) {
|
|
3175
|
+
if (tokenizer.specialIds.has(id) && id !== tokenizer.unkId) continue;
|
|
3176
|
+
const base = id * dim;
|
|
3177
|
+
for (let j = 0; j < dim; j++) {
|
|
3178
|
+
out[j] = out[j] + table[base + j];
|
|
3179
|
+
}
|
|
3180
|
+
pooled++;
|
|
3181
|
+
}
|
|
3182
|
+
if (pooled === 0) return out;
|
|
3183
|
+
let sumSquares = 0;
|
|
3184
|
+
for (let j = 0; j < dim; j++) {
|
|
3185
|
+
const mean = out[j] / pooled;
|
|
3186
|
+
out[j] = mean;
|
|
3187
|
+
sumSquares += mean * mean;
|
|
3188
|
+
}
|
|
3189
|
+
const norm = Math.sqrt(sumSquares);
|
|
3190
|
+
if (norm > 0) {
|
|
3191
|
+
for (let j = 0; j < dim; j++) out[j] = out[j] / norm;
|
|
3192
|
+
}
|
|
3193
|
+
return out;
|
|
3194
|
+
}
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
// src/semantic/cache.ts
|
|
3199
|
+
import { createHash as createHash2 } from "crypto";
|
|
3200
|
+
import { mkdir as mkdir6, readFile as readFile15, rename as rename4, writeFile as writeFile3 } from "fs/promises";
|
|
3201
|
+
import { join as join8 } from "path";
|
|
3202
|
+
var CACHE_VERSION = 2;
|
|
3203
|
+
function cachePathsFor(cacheDir, vaultRoot2, modelId) {
|
|
3204
|
+
const vaultKey = createHash2("sha256").update(vaultRoot2).digest("hex").slice(0, 16);
|
|
3205
|
+
const dir = join8(cacheDir, "embeddings", vaultKey);
|
|
3206
|
+
return { dir, bin: join8(dir, `${modelId}.bin`), manifest: join8(dir, `${modelId}.json`) };
|
|
3207
|
+
}
|
|
3208
|
+
async function loadCache(paths, modelId, dim) {
|
|
3209
|
+
try {
|
|
3210
|
+
const manifest = JSON.parse(await readFile15(paths.manifest, "utf8"));
|
|
3211
|
+
if (manifest.version !== CACHE_VERSION || manifest.modelId !== modelId || manifest.dim !== dim) {
|
|
3212
|
+
return void 0;
|
|
3213
|
+
}
|
|
3214
|
+
const buf = await readFile15(paths.bin);
|
|
3215
|
+
const totalChunks = manifest.entries.reduce((a, e) => a + e.chunks, 0);
|
|
3216
|
+
const vectorBytes = totalChunks * dim * 4;
|
|
3217
|
+
const spanBytes = totalChunks * 2 * 4;
|
|
3218
|
+
if (buf.byteLength !== vectorBytes + spanBytes) return void 0;
|
|
3219
|
+
const aligned = new Uint8Array(buf.byteLength);
|
|
3220
|
+
aligned.set(buf);
|
|
3221
|
+
const floats = new Float32Array(aligned.buffer, 0, totalChunks * dim);
|
|
3222
|
+
const spans = new Uint32Array(aligned.buffer, vectorBytes, totalChunks * 2);
|
|
3223
|
+
const vectors = /* @__PURE__ */ new Map();
|
|
3224
|
+
const hashes = /* @__PURE__ */ new Map();
|
|
3225
|
+
let chunkOffset = 0;
|
|
3226
|
+
for (const e of manifest.entries) {
|
|
3227
|
+
if (e.chunks <= 0) return void 0;
|
|
3228
|
+
vectors.set(e.path, {
|
|
3229
|
+
packed: floats.subarray(chunkOffset * dim, (chunkOffset + e.chunks) * dim),
|
|
3230
|
+
spans: spans.subarray(chunkOffset * 2, (chunkOffset + e.chunks) * 2)
|
|
3231
|
+
});
|
|
3232
|
+
hashes.set(e.path, e.contentHash);
|
|
3233
|
+
chunkOffset += e.chunks;
|
|
3234
|
+
}
|
|
3235
|
+
return { vectors, hashes };
|
|
3236
|
+
} catch {
|
|
3237
|
+
return void 0;
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
async function saveCache(paths, modelId, dim, notes2, hashes) {
|
|
3241
|
+
const entries = [];
|
|
3242
|
+
const kept = [];
|
|
3243
|
+
let totalChunks = 0;
|
|
3244
|
+
for (const [path2, note] of notes2) {
|
|
3245
|
+
const contentHash2 = hashes.get(path2);
|
|
3246
|
+
if (!contentHash2) continue;
|
|
3247
|
+
const chunks = note.packed.length / dim;
|
|
3248
|
+
entries.push({ path: path2, contentHash: contentHash2, chunks });
|
|
3249
|
+
kept.push(note);
|
|
3250
|
+
totalChunks += chunks;
|
|
3251
|
+
}
|
|
3252
|
+
const bin = new Uint8Array(totalChunks * dim * 4 + totalChunks * 2 * 4);
|
|
3253
|
+
const floats = new Float32Array(bin.buffer, 0, totalChunks * dim);
|
|
3254
|
+
const spans = new Uint32Array(bin.buffer, totalChunks * dim * 4, totalChunks * 2);
|
|
3255
|
+
let floatOffset = 0;
|
|
3256
|
+
let spanOffset = 0;
|
|
3257
|
+
for (const note of kept) {
|
|
3258
|
+
floats.set(note.packed, floatOffset);
|
|
3259
|
+
spans.set(note.spans, spanOffset);
|
|
3260
|
+
floatOffset += note.packed.length;
|
|
3261
|
+
spanOffset += note.spans.length;
|
|
3262
|
+
}
|
|
3263
|
+
const manifest = { version: CACHE_VERSION, modelId, dim, entries };
|
|
3264
|
+
await mkdir6(paths.dir, { recursive: true });
|
|
3265
|
+
await atomicWriteBytes(paths.bin, bin);
|
|
3266
|
+
await atomicWriteBytes(paths.manifest, new TextEncoder().encode(JSON.stringify(manifest)));
|
|
3267
|
+
}
|
|
3268
|
+
async function atomicWriteBytes(absPath, bytes) {
|
|
3269
|
+
const tmpPath = `${absPath}.seekstone-cache-tmp`;
|
|
3270
|
+
await writeFile3(tmpPath, bytes);
|
|
3271
|
+
await rename4(tmpPath, absPath);
|
|
3272
|
+
}
|
|
3273
|
+
|
|
3274
|
+
// src/semantic/store.ts
|
|
3275
|
+
var SemanticStore = class {
|
|
3276
|
+
dim;
|
|
3277
|
+
notes = /* @__PURE__ */ new Map();
|
|
3278
|
+
chunks = 0;
|
|
3279
|
+
constructor(dim) {
|
|
3280
|
+
if (!Number.isInteger(dim) || dim <= 0) {
|
|
3281
|
+
throw new Error(`semantic store: invalid dim ${dim}`);
|
|
3282
|
+
}
|
|
3283
|
+
this.dim = dim;
|
|
3284
|
+
}
|
|
3285
|
+
/** Total chunk vectors across all notes. */
|
|
3286
|
+
get chunkCount() {
|
|
3287
|
+
return this.chunks;
|
|
3288
|
+
}
|
|
3289
|
+
get noteCount() {
|
|
3290
|
+
return this.notes.size;
|
|
3291
|
+
}
|
|
3292
|
+
/** Replace (or insert) a note's packed chunk vectors and spans. */
|
|
3293
|
+
setNote(path2, packed, spans) {
|
|
3294
|
+
if (packed.length === 0 || packed.length % this.dim !== 0) {
|
|
3295
|
+
throw new Error(
|
|
3296
|
+
`semantic store: packed length ${packed.length} is not a positive multiple of dim ${this.dim} (${path2})`
|
|
3297
|
+
);
|
|
3298
|
+
}
|
|
3299
|
+
const n = packed.length / this.dim;
|
|
3300
|
+
if (spans.length !== n * 2) {
|
|
3301
|
+
throw new Error(
|
|
3302
|
+
`semantic store: spans length ${spans.length} does not match ${n} chunks (${path2})`
|
|
3303
|
+
);
|
|
3304
|
+
}
|
|
3305
|
+
const prev = this.notes.get(path2);
|
|
3306
|
+
if (prev) this.chunks -= prev.packed.length / this.dim;
|
|
3307
|
+
this.notes.set(path2, { packed, spans });
|
|
3308
|
+
this.chunks += n;
|
|
3309
|
+
}
|
|
3310
|
+
removeNote(path2) {
|
|
3311
|
+
const prev = this.notes.get(path2);
|
|
3312
|
+
if (!prev) return;
|
|
3313
|
+
this.chunks -= prev.packed.length / this.dim;
|
|
3314
|
+
this.notes.delete(path2);
|
|
3315
|
+
}
|
|
3316
|
+
getNote(path2) {
|
|
3317
|
+
return this.notes.get(path2);
|
|
3318
|
+
}
|
|
3319
|
+
/** Iterate (path, note vectors) — cache serialization uses this. */
|
|
3320
|
+
entries() {
|
|
3321
|
+
return this.notes.entries();
|
|
3322
|
+
}
|
|
3323
|
+
/** Top `k` notes by max-pooled chunk cosine, ties broken path-ascending. */
|
|
3324
|
+
topNotes(query, k) {
|
|
3325
|
+
if (query.length !== this.dim) {
|
|
3326
|
+
throw new Error(`semantic store: query dim ${query.length} does not match ${this.dim}`);
|
|
3327
|
+
}
|
|
3328
|
+
const dim = this.dim;
|
|
3329
|
+
const hits = [];
|
|
3330
|
+
for (const [path2, { packed, spans }] of this.notes) {
|
|
3331
|
+
let best = Number.NEGATIVE_INFINITY;
|
|
3332
|
+
let bestChunk = 0;
|
|
3333
|
+
const n = packed.length / dim;
|
|
3334
|
+
for (let c = 0; c < n; c++) {
|
|
3335
|
+
const base = c * dim;
|
|
3336
|
+
let score = 0;
|
|
3337
|
+
for (let j = 0; j < dim; j++) {
|
|
3338
|
+
score += packed[base + j] * query[j];
|
|
3339
|
+
}
|
|
3340
|
+
if (score > best) {
|
|
3341
|
+
best = score;
|
|
3342
|
+
bestChunk = c;
|
|
3343
|
+
}
|
|
3344
|
+
}
|
|
3345
|
+
hits.push({
|
|
3346
|
+
path: path2,
|
|
3347
|
+
score: best,
|
|
3348
|
+
chunkIndex: bestChunk,
|
|
3349
|
+
start: spans[bestChunk * 2],
|
|
3350
|
+
end: spans[bestChunk * 2 + 1]
|
|
3351
|
+
});
|
|
3352
|
+
}
|
|
3353
|
+
return hits.sort((a, b) => b.score - a.score || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)).slice(0, Math.max(0, k));
|
|
3354
|
+
}
|
|
3355
|
+
};
|
|
3356
|
+
|
|
3357
|
+
// src/semantic/state.ts
|
|
3358
|
+
var Semantic = class _Semantic {
|
|
3359
|
+
embedder;
|
|
3360
|
+
store;
|
|
3361
|
+
progress;
|
|
3362
|
+
ctx;
|
|
3363
|
+
paths;
|
|
3364
|
+
log;
|
|
3365
|
+
debounceMs;
|
|
3366
|
+
saveDebounceMs;
|
|
3367
|
+
yieldEvery;
|
|
3368
|
+
hashes = /* @__PURE__ */ new Map();
|
|
3369
|
+
pending = /* @__PURE__ */ new Map();
|
|
3370
|
+
saveTimer;
|
|
3371
|
+
stopped = false;
|
|
3372
|
+
constructor(embedder, ctx2, cfg, deps) {
|
|
3373
|
+
this.embedder = embedder;
|
|
3374
|
+
this.store = new SemanticStore(embedder.dim);
|
|
3375
|
+
this.ctx = ctx2;
|
|
3376
|
+
this.paths = cachePathsFor(cfg.cacheDir, ctx2.vaultRoot, embedder.id);
|
|
3377
|
+
this.log = deps.log;
|
|
3378
|
+
this.debounceMs = deps.debounceMs ?? 300;
|
|
3379
|
+
this.saveDebounceMs = deps.saveDebounceMs ?? 5e3;
|
|
3380
|
+
this.yieldEvery = deps.yieldEvery ?? 8;
|
|
3381
|
+
this.progress = { state: "building", done: 0, total: ctx2.notes.size };
|
|
3382
|
+
}
|
|
3383
|
+
/**
|
|
3384
|
+
* Load the model and kick off the background index build. Rejects with an
|
|
3385
|
+
* actionable message when the model files are missing — the user enabled
|
|
3386
|
+
* the feature explicitly, so a broken setup should fail loudly at boot.
|
|
3387
|
+
*/
|
|
3388
|
+
static async start(ctx2, cfg, deps = {}) {
|
|
3389
|
+
let embedder;
|
|
3390
|
+
try {
|
|
3391
|
+
embedder = await (deps.loadModel ?? loadModel2Vec)(cfg.modelDir);
|
|
3392
|
+
} catch (err) {
|
|
3393
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
3394
|
+
throw new Error(
|
|
3395
|
+
`semantic search: could not load the embedding model from ${cfg.modelDir} \u2014 run \`npx -y seekstone fetch-model\` to download it, or point SEEKSTONE_MODEL_PATH at a Model2Vec model directory (${reason})`
|
|
3396
|
+
);
|
|
3397
|
+
}
|
|
3398
|
+
const semantic = new _Semantic(embedder, ctx2, cfg, deps);
|
|
3399
|
+
void semantic.build();
|
|
3400
|
+
return semantic;
|
|
3401
|
+
}
|
|
3402
|
+
/** Resolves when the initial build has finished (tests and progress checks). */
|
|
3403
|
+
buildDone = Promise.resolve();
|
|
3404
|
+
ready() {
|
|
3405
|
+
return this.buildDone;
|
|
3406
|
+
}
|
|
3407
|
+
build() {
|
|
3408
|
+
this.buildDone = this.buildInner().catch((err) => {
|
|
3409
|
+
this.log?.error("semantic build failed", {
|
|
3410
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3411
|
+
});
|
|
3412
|
+
});
|
|
3413
|
+
return this.buildDone;
|
|
3414
|
+
}
|
|
3415
|
+
async buildInner() {
|
|
3416
|
+
const t0 = performance.now();
|
|
3417
|
+
const cached = await loadCache(this.paths, this.embedder.id, this.embedder.dim);
|
|
3418
|
+
const ids = [...this.ctx.notes.keys()];
|
|
3419
|
+
this.progress = { state: "building", done: 0, total: ids.length };
|
|
3420
|
+
let reused = 0;
|
|
3421
|
+
let sinceYield = 0;
|
|
3422
|
+
for (const id of ids) {
|
|
3423
|
+
if (this.stopped) return;
|
|
3424
|
+
if (!this.hashes.has(id)) {
|
|
3425
|
+
const note = this.ctx.notes.get(id);
|
|
3426
|
+
if (note) {
|
|
3427
|
+
const current = contentHash(note.raw);
|
|
3428
|
+
const cachedVecs = cached?.hashes.get(id) === current ? cached.vectors.get(id) : void 0;
|
|
3429
|
+
if (cachedVecs !== void 0 && cachedVecs.packed.length > 0) {
|
|
3430
|
+
this.store.setNote(id, cachedVecs.packed, cachedVecs.spans);
|
|
3431
|
+
reused++;
|
|
3432
|
+
} else {
|
|
3433
|
+
const { packed, spans } = this.embedNote(note);
|
|
3434
|
+
this.store.setNote(id, packed, spans);
|
|
3435
|
+
if (++sinceYield >= this.yieldEvery) {
|
|
3436
|
+
sinceYield = 0;
|
|
3437
|
+
await new Promise((r) => setImmediate(r));
|
|
3438
|
+
}
|
|
3439
|
+
}
|
|
3440
|
+
this.hashes.set(id, current);
|
|
3441
|
+
}
|
|
3442
|
+
}
|
|
3443
|
+
if (this.progress.state === "building") this.progress.done++;
|
|
3444
|
+
}
|
|
3445
|
+
this.progress = { state: "ready" };
|
|
3446
|
+
this.log?.info("semantic index ready", {
|
|
3447
|
+
notes: this.store.noteCount,
|
|
3448
|
+
chunks: this.store.chunkCount,
|
|
3449
|
+
reusedFromCache: reused,
|
|
3450
|
+
buildMs: Math.round(performance.now() - t0)
|
|
3451
|
+
});
|
|
3452
|
+
if (reused < this.store.noteCount) await this.save();
|
|
3453
|
+
}
|
|
3454
|
+
embedNote(note) {
|
|
3455
|
+
const chunks = chunkNote(note.title, note.body);
|
|
3456
|
+
const dim = this.embedder.dim;
|
|
3457
|
+
const packed = new Float32Array(chunks.length * dim);
|
|
3458
|
+
const spans = new Uint32Array(chunks.length * 2);
|
|
3459
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
3460
|
+
const chunk = chunks[i];
|
|
3461
|
+
packed.set(this.embedder.embed(chunk.text), i * dim);
|
|
3462
|
+
spans[i * 2] = chunk.start;
|
|
3463
|
+
spans[i * 2 + 1] = chunk.end;
|
|
3464
|
+
}
|
|
3465
|
+
return { packed, spans };
|
|
3466
|
+
}
|
|
3467
|
+
embedQuery(query) {
|
|
3468
|
+
return this.embedder.embed(query);
|
|
3469
|
+
}
|
|
3470
|
+
/** Watcher hook: a note was added or changed (ctx.notes is already fresh). */
|
|
3471
|
+
noteChanged(path2) {
|
|
3472
|
+
if (this.stopped) return;
|
|
3473
|
+
const existing = this.pending.get(path2);
|
|
3474
|
+
if (existing) clearTimeout(existing);
|
|
3475
|
+
const timer = setTimeout(() => {
|
|
3476
|
+
this.pending.delete(path2);
|
|
3477
|
+
this.reembed(path2);
|
|
3478
|
+
}, this.debounceMs);
|
|
3479
|
+
timer.unref?.();
|
|
3480
|
+
this.pending.set(path2, timer);
|
|
3481
|
+
}
|
|
3482
|
+
/** Watcher hook: a note was deleted. */
|
|
3483
|
+
noteRemoved(path2) {
|
|
3484
|
+
const existing = this.pending.get(path2);
|
|
3485
|
+
if (existing) {
|
|
3486
|
+
clearTimeout(existing);
|
|
3487
|
+
this.pending.delete(path2);
|
|
3488
|
+
}
|
|
3489
|
+
if (this.hashes.has(path2) || this.store.getNote(path2)) {
|
|
3490
|
+
this.store.removeNote(path2);
|
|
3491
|
+
this.hashes.delete(path2);
|
|
3492
|
+
this.scheduleSave();
|
|
3493
|
+
}
|
|
3494
|
+
}
|
|
3495
|
+
reembed(path2) {
|
|
3496
|
+
const note = this.ctx.notes.get(path2);
|
|
3497
|
+
if (!note) return;
|
|
3498
|
+
const current = contentHash(note.raw);
|
|
3499
|
+
const prior = this.hashes.get(path2);
|
|
3500
|
+
if (prior === current) return;
|
|
3501
|
+
const { packed, spans } = this.embedNote(note);
|
|
3502
|
+
this.store.setNote(path2, packed, spans);
|
|
3503
|
+
this.hashes.set(path2, current);
|
|
3504
|
+
this.log?.debug("semantic re-embed", { path: path2 });
|
|
3505
|
+
this.scheduleSave();
|
|
3506
|
+
}
|
|
3507
|
+
scheduleSave() {
|
|
3508
|
+
if (this.saveTimer) clearTimeout(this.saveTimer);
|
|
3509
|
+
this.saveTimer = setTimeout(() => {
|
|
3510
|
+
this.saveTimer = void 0;
|
|
3511
|
+
void this.save();
|
|
3512
|
+
}, this.saveDebounceMs);
|
|
3513
|
+
this.saveTimer.unref?.();
|
|
3514
|
+
}
|
|
3515
|
+
async save() {
|
|
3516
|
+
if (this.stopped) return;
|
|
3517
|
+
try {
|
|
3518
|
+
await saveCache(
|
|
3519
|
+
this.paths,
|
|
3520
|
+
this.embedder.id,
|
|
3521
|
+
this.embedder.dim,
|
|
3522
|
+
this.store.entries(),
|
|
3523
|
+
this.hashes
|
|
3524
|
+
);
|
|
3525
|
+
this.log?.debug("semantic cache saved", { notes: this.store.noteCount });
|
|
3526
|
+
} catch (err) {
|
|
3527
|
+
this.log?.warn("semantic cache save failed", {
|
|
3528
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3529
|
+
});
|
|
3530
|
+
}
|
|
3531
|
+
}
|
|
3532
|
+
/** Tear down timers (tests; the process otherwise runs for the session). */
|
|
3533
|
+
stop() {
|
|
3534
|
+
this.stopped = true;
|
|
3535
|
+
for (const timer of this.pending.values()) clearTimeout(timer);
|
|
3536
|
+
this.pending.clear();
|
|
3537
|
+
if (this.saveTimer) clearTimeout(this.saveTimer);
|
|
3538
|
+
this.saveTimer = void 0;
|
|
3539
|
+
}
|
|
3540
|
+
};
|
|
3541
|
+
|
|
2754
3542
|
// src/tool-list.ts
|
|
2755
3543
|
var READ_ONLY_ANNOTATIONS = {
|
|
2756
3544
|
readOnlyHint: true,
|
|
@@ -2778,11 +3566,16 @@ var ALL_TOOLS = [
|
|
|
2778
3566
|
{
|
|
2779
3567
|
name: "search",
|
|
2780
3568
|
annotations: READ_ONLY_ANNOTATIONS,
|
|
2781
|
-
description:
|
|
3569
|
+
description: 'Full-text search across the vault. Returns ranked excerpts (~200 chars) \u2014 not full notes \u2014 to minimise context usage. Supports fuzzy matching and prefix search; with SEEKSTONE_SEMANTIC=1, mode "semantic"/"hybrid" searches by meaning via a local embedding model.',
|
|
2782
3570
|
inputSchema: {
|
|
2783
3571
|
type: "object",
|
|
2784
3572
|
properties: {
|
|
2785
3573
|
query: { type: "string", description: "Search query." },
|
|
3574
|
+
mode: {
|
|
3575
|
+
type: "string",
|
|
3576
|
+
enum: ["lexical", "semantic", "hybrid"],
|
|
3577
|
+
description: "lexical = keyword search (default). semantic = meaning-based search (requires SEEKSTONE_SEMANTIC=1). hybrid = exact-title lookups go lexical, everything else semantic."
|
|
3578
|
+
},
|
|
2786
3579
|
limit: { type: "number", description: "Max results (1\u201350, default 10)." },
|
|
2787
3580
|
folder: { type: "string", description: "Restrict to a vault-relative folder prefix." },
|
|
2788
3581
|
tag: { type: "string", description: "Restrict to notes with this tag." }
|
|
@@ -2981,6 +3774,10 @@ var ALL_TOOLS = [
|
|
|
2981
3774
|
permanent: {
|
|
2982
3775
|
type: "boolean",
|
|
2983
3776
|
description: "Permanently remove instead of moving to .trash/. Defaults to false."
|
|
3777
|
+
},
|
|
3778
|
+
prevHash: {
|
|
3779
|
+
type: "string",
|
|
3780
|
+
description: "Optional compare-and-swap guard: the contentHash from a prior read. Fails with hash_conflict if the note changed since \u2014 so you never delete content you have not seen."
|
|
2984
3781
|
}
|
|
2985
3782
|
},
|
|
2986
3783
|
required: ["path"]
|
|
@@ -3002,6 +3799,10 @@ var ALL_TOOLS = [
|
|
|
3002
3799
|
rewriteLinks: {
|
|
3003
3800
|
type: "boolean",
|
|
3004
3801
|
description: "Rewrite references in other notes to follow the move. Defaults to true; pass false to move the file only."
|
|
3802
|
+
},
|
|
3803
|
+
prevHash: {
|
|
3804
|
+
type: "string",
|
|
3805
|
+
description: "Optional compare-and-swap guard on the source note: the contentHash from a prior read. Fails with hash_conflict if the note changed since."
|
|
3005
3806
|
}
|
|
3006
3807
|
},
|
|
3007
3808
|
required: ["from", "to"]
|
|
@@ -3259,8 +4060,8 @@ function visibleTools(policy2) {
|
|
|
3259
4060
|
|
|
3260
4061
|
// src/watcher.ts
|
|
3261
4062
|
import { realpathSync } from "fs";
|
|
3262
|
-
import { readFile as
|
|
3263
|
-
import { join as
|
|
4063
|
+
import { readFile as readFile16 } from "fs/promises";
|
|
4064
|
+
import { join as join9, relative as relative2, sep as sep3 } from "path";
|
|
3264
4065
|
import chokidar from "chokidar";
|
|
3265
4066
|
function startWatcher(ctx2, log2, opts) {
|
|
3266
4067
|
const usePolling = opts?.usePolling ?? process.env.SEEKSTONE_WATCH_POLL === "1";
|
|
@@ -3272,10 +4073,11 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
3272
4073
|
const toRel = (abs) => relative2(vaultRoot2, abs).split(sep3).join("/");
|
|
3273
4074
|
async function upsert(relPath, op) {
|
|
3274
4075
|
try {
|
|
3275
|
-
const raw = await
|
|
4076
|
+
const raw = await readFile16(join9(vaultRoot2, relPath), "utf8");
|
|
3276
4077
|
removeNoteBacklinks(ctx2, relPath);
|
|
3277
4078
|
upsertDoc(ctx2, buildDoc(relPath, raw));
|
|
3278
4079
|
addNoteBacklinks(ctx2, relPath, raw);
|
|
4080
|
+
ctx2.semantic?.noteChanged(relPath);
|
|
3279
4081
|
log2?.debug("index updated", { path: relPath, op });
|
|
3280
4082
|
} catch {
|
|
3281
4083
|
}
|
|
@@ -3285,6 +4087,7 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
3285
4087
|
removeNoteBacklinks(ctx2, relPath);
|
|
3286
4088
|
ctx2.index.discard(relPath);
|
|
3287
4089
|
ctx2.notes.delete(relPath);
|
|
4090
|
+
ctx2.semantic?.noteRemoved(relPath);
|
|
3288
4091
|
log2?.debug("index removed", { path: relPath, op: "delete" });
|
|
3289
4092
|
}
|
|
3290
4093
|
}
|
|
@@ -3324,7 +4127,7 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
3324
4127
|
}
|
|
3325
4128
|
|
|
3326
4129
|
// src/index.ts
|
|
3327
|
-
var VERSION = true ? "0.
|
|
4130
|
+
var VERSION = true ? "0.15.0" : "0.0.0-dev";
|
|
3328
4131
|
var intent = parseCliIntent(process.argv.slice(2));
|
|
3329
4132
|
if (intent === "version") {
|
|
3330
4133
|
process.stdout.write(`${VERSION}
|
|
@@ -3352,6 +4155,14 @@ if (intent === "init") {
|
|
|
3352
4155
|
`);
|
|
3353
4156
|
process.exit(result.exitCode);
|
|
3354
4157
|
}
|
|
4158
|
+
if (intent === "fetch-model") {
|
|
4159
|
+
const { runFetchModel } = await import("./fetch-model-7TGOJMZL.js");
|
|
4160
|
+
const { homedir: homedir2 } = await import("os");
|
|
4161
|
+
const result = await runFetchModel({ env: process.env, homedir: homedir2() });
|
|
4162
|
+
process.stdout.write(`${result.output.join("\n")}
|
|
4163
|
+
`);
|
|
4164
|
+
process.exit(result.exitCode);
|
|
4165
|
+
}
|
|
3355
4166
|
var log = createLogger();
|
|
3356
4167
|
installProcessGuards(log);
|
|
3357
4168
|
var rawVaultRoot = process.env.SEEKSTONE_VAULT;
|
|
@@ -3367,6 +4178,20 @@ log.info("building index", { vault: vaultRoot });
|
|
|
3367
4178
|
var { index, notes, backlinks, buildMs } = await buildIndex(vaultRoot);
|
|
3368
4179
|
log.info("index ready", { notes: notes.size, buildMs });
|
|
3369
4180
|
var ctx = { vaultRoot, index, notes, backlinks, policy };
|
|
4181
|
+
var semanticCfg = resolveSemanticConfig(process.env, homedir());
|
|
4182
|
+
if (semanticCfg) {
|
|
4183
|
+
try {
|
|
4184
|
+
ctx.semantic = await Semantic.start(ctx, semanticCfg, { log });
|
|
4185
|
+
log.info("semantic search enabled", {
|
|
4186
|
+
model: ctx.semantic.embedder.id,
|
|
4187
|
+
dim: ctx.semantic.embedder.dim,
|
|
4188
|
+
notes: notes.size
|
|
4189
|
+
});
|
|
4190
|
+
} catch (err) {
|
|
4191
|
+
log.error(err instanceof Error ? err.message : String(err));
|
|
4192
|
+
process.exit(1);
|
|
4193
|
+
}
|
|
4194
|
+
}
|
|
3370
4195
|
var watcher = startWatcher(ctx, log);
|
|
3371
4196
|
process.on("exit", () => void watcher.stop());
|
|
3372
4197
|
var server = new Server({ name: "seekstone", version: VERSION }, { capabilities: { tools: {} } });
|