seekstone 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/chunk-RRLYVK2X.js +18 -0
- package/dist/chunk-RRLYVK2X.js.map +1 -0
- package/dist/fetch-model-7TGOJMZL.js +81 -0
- package/dist/fetch-model-7TGOJMZL.js.map +1 -0
- package/dist/index.js +822 -26
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
resolveSemanticConfig
|
|
4
|
+
} from "./chunk-RRLYVK2X.js";
|
|
2
5
|
|
|
3
6
|
// src/index.ts
|
|
7
|
+
import { homedir } from "os";
|
|
4
8
|
import { resolve as resolve2 } from "path";
|
|
5
9
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
6
10
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
@@ -17,6 +21,7 @@ function parseCliIntent(argv) {
|
|
|
17
21
|
}
|
|
18
22
|
return "init";
|
|
19
23
|
}
|
|
24
|
+
if (argv[0] === "fetch-model") return "fetch-model";
|
|
20
25
|
for (const arg of argv) {
|
|
21
26
|
if (arg === "--version" || arg === "-v") return "version";
|
|
22
27
|
if (arg === "--help" || arg === "-h") return "help";
|
|
@@ -33,10 +38,11 @@ Seekstone runs as a Model Context Protocol stdio server. It is normally
|
|
|
33
38
|
launched by an MCP client (Claude Desktop, Claude Code, Cursor, \u2026), not run by hand.
|
|
34
39
|
|
|
35
40
|
Usage:
|
|
36
|
-
seekstone
|
|
37
|
-
seekstone init
|
|
38
|
-
seekstone
|
|
39
|
-
seekstone --
|
|
41
|
+
seekstone Start the MCP server (reads from stdin/stdout)
|
|
42
|
+
seekstone init Validate a vault and print/patch the client MCP config
|
|
43
|
+
seekstone fetch-model Download the local embedding model for semantic search
|
|
44
|
+
seekstone --version Print the version and exit
|
|
45
|
+
seekstone --help Print this help and exit
|
|
40
46
|
|
|
41
47
|
"seekstone init" options:
|
|
42
48
|
${initOptionLines}
|
|
@@ -48,6 +54,9 @@ Optional environment:
|
|
|
48
54
|
SEEKSTONE_LOG_LEVEL error | warn | info (default) | debug
|
|
49
55
|
SEEKSTONE_LOG_FILE Absolute path; append JSON-line logs here
|
|
50
56
|
SEEKSTONE_WATCH_POLL Set to 1 to poll for changes (network drives, WSL)
|
|
57
|
+
SEEKSTONE_SEMANTIC Set to 1 to enable semantic search (needs fetch-model first)
|
|
58
|
+
SEEKSTONE_MODEL_PATH Override the embedding-model directory
|
|
59
|
+
SEEKSTONE_CACHE_DIR Override the cache root (default ~/.cache/seekstone)
|
|
51
60
|
|
|
52
61
|
Add to Claude Code:
|
|
53
62
|
claude mcp add seekstone --env SEEKSTONE_VAULT=/path/to/vault -- npx -y seekstone
|
|
@@ -370,8 +379,39 @@ function resolveLink(target, notes2) {
|
|
|
370
379
|
|
|
371
380
|
// src/tools/search.ts
|
|
372
381
|
import { z as z2 } from "zod";
|
|
382
|
+
|
|
383
|
+
// src/semantic/excerpt.ts
|
|
384
|
+
function chunkExcerpt(body, span, terms, maxLen) {
|
|
385
|
+
const normalized = body.includes("\r") ? body.replace(/\r\n/g, "\n") : body;
|
|
386
|
+
const start = Math.min(span.start, normalized.length);
|
|
387
|
+
const end = Math.min(Math.max(span.end, start), normalized.length);
|
|
388
|
+
const text = start === end ? normalized : normalized.slice(start, end);
|
|
389
|
+
return extractExcerpt(text, terms, maxLen);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// src/semantic/route.ts
|
|
393
|
+
var MAX_ROUTE_WORDS = 6;
|
|
394
|
+
function queryWords(query) {
|
|
395
|
+
return query.toLowerCase().split(/\W+/).filter(Boolean);
|
|
396
|
+
}
|
|
397
|
+
function routeToLexical(query, topPaths, topN = 3) {
|
|
398
|
+
const words = new Set(queryWords(query));
|
|
399
|
+
if (words.size === 0) return false;
|
|
400
|
+
return topPaths.slice(0, topN).some((path2) => {
|
|
401
|
+
const base = (path2.split(/[\\/]/).pop() ?? "").replace(/\.md$/, "").toLowerCase();
|
|
402
|
+
const parts = new Set(base.split(/\W+/).filter(Boolean));
|
|
403
|
+
return parts.size === words.size && [...parts].every((p) => words.has(p));
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// src/tools/search.ts
|
|
373
408
|
var SearchInput = z2.object({
|
|
374
409
|
query: z2.string().min(1).describe("Search query. Supports fuzzy matching and prefix search."),
|
|
410
|
+
// Optional (not .default()) so the inferred type keeps existing direct
|
|
411
|
+
// callers compiling; search() itself defaults it to 'lexical'.
|
|
412
|
+
mode: z2.enum(["lexical", "semantic", "hybrid"]).optional().describe(
|
|
413
|
+
"lexical = keyword search (default). semantic = meaning-based search over local embeddings (requires SEEKSTONE_SEMANTIC=1 and a fetched model). hybrid = exact-title lookups go lexical, everything else semantic."
|
|
414
|
+
),
|
|
375
415
|
limit: z2.number().int().min(1).max(50).default(10).describe("Maximum number of results to return."),
|
|
376
416
|
folder: z2.string().optional().describe("Restrict results to notes under this vault-relative folder prefix."),
|
|
377
417
|
tag: z2.string().optional().describe("Restrict results to notes containing this tag."),
|
|
@@ -379,44 +419,113 @@ var SearchInput = z2.object({
|
|
|
379
419
|
"Max characters of match context per hit (default 120). Lower trims payload; higher gives more context."
|
|
380
420
|
)
|
|
381
421
|
});
|
|
422
|
+
var SEMANTIC_CANDIDATES = 50;
|
|
382
423
|
function basenameNoExt(path2) {
|
|
383
424
|
const base = path2.slice(path2.lastIndexOf("/") + 1);
|
|
384
425
|
const dot = base.lastIndexOf(".");
|
|
385
426
|
return dot > 0 ? base.slice(0, dot) : base;
|
|
386
427
|
}
|
|
387
428
|
function search(ctx2, input) {
|
|
429
|
+
const mode = input.mode ?? "lexical";
|
|
430
|
+
if (mode === "lexical") return lexicalSearch(ctx2, input);
|
|
431
|
+
const semantic = requireSemantic(ctx2);
|
|
432
|
+
if (mode === "semantic") return semanticSearch(ctx2, semantic, input);
|
|
433
|
+
const words = queryWords(input.query);
|
|
434
|
+
if (words.length > 0 && words.length <= MAX_ROUTE_WORDS) {
|
|
435
|
+
const lexHits = lexicalSearch(ctx2, input);
|
|
436
|
+
if (routeToLexical(
|
|
437
|
+
input.query,
|
|
438
|
+
lexHits.map((h) => h.path)
|
|
439
|
+
))
|
|
440
|
+
return lexHits;
|
|
441
|
+
}
|
|
442
|
+
return semanticSearch(ctx2, semantic, input);
|
|
443
|
+
}
|
|
444
|
+
function requireSemantic(ctx2) {
|
|
445
|
+
if (!ctx2.semantic) {
|
|
446
|
+
throw new Error(
|
|
447
|
+
JSON.stringify({
|
|
448
|
+
error: "semantic_unavailable",
|
|
449
|
+
hint: 'Semantic search is not enabled. Set SEEKSTONE_SEMANTIC=1 in the server env and run `npx -y seekstone fetch-model` once to download the local embedding model, then restart the session. Meanwhile, mode: "lexical" works.'
|
|
450
|
+
})
|
|
451
|
+
);
|
|
452
|
+
}
|
|
453
|
+
const progress = ctx2.semantic.progress;
|
|
454
|
+
if (progress.state === "building") {
|
|
455
|
+
throw new Error(
|
|
456
|
+
JSON.stringify({
|
|
457
|
+
error: "semantic_building",
|
|
458
|
+
done: progress.done,
|
|
459
|
+
total: progress.total,
|
|
460
|
+
hint: 'The semantic index is still embedding this vault (first run only \u2014 later runs load a cache). Retry shortly, or use mode: "lexical".'
|
|
461
|
+
})
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
return ctx2.semantic;
|
|
465
|
+
}
|
|
466
|
+
function searchTerms(query) {
|
|
467
|
+
return query.toLowerCase().split(/\s+/).filter((t) => t.length > 1);
|
|
468
|
+
}
|
|
469
|
+
function lexicalSearch(ctx2, input) {
|
|
388
470
|
const results = ctx2.index.search(input.query, {
|
|
389
471
|
boost: { title: 3, tags: 2, body: 1 },
|
|
390
472
|
fuzzy: 0.2,
|
|
391
473
|
prefix: true
|
|
392
474
|
});
|
|
393
|
-
const terms = input.query
|
|
475
|
+
const terms = searchTerms(input.query);
|
|
394
476
|
const hits = [];
|
|
395
477
|
for (const r of results) {
|
|
396
|
-
|
|
397
|
-
if (
|
|
398
|
-
const note2 = ctx2.notes.get(r.id);
|
|
399
|
-
if (!note2) continue;
|
|
400
|
-
const noteTags2 = note2.tags.split(" ");
|
|
401
|
-
if (!noteTags2.some((t) => t === input.tag || t === input.tag?.replace(/^#/, ""))) continue;
|
|
402
|
-
}
|
|
403
|
-
const note = ctx2.notes.get(r.id);
|
|
404
|
-
const body = note?.body ?? "";
|
|
405
|
-
const noteTags = note?.tags ? note.tags.split(" ").filter(Boolean) : [];
|
|
406
|
-
const title = r.title;
|
|
478
|
+
const note = filterNote(ctx2, input, r.id);
|
|
479
|
+
if (!note) continue;
|
|
407
480
|
const hit = {
|
|
408
481
|
path: r.id,
|
|
409
482
|
// Round the raw MiniSearch score: 2 decimals preserve relevance gaps without 17-digit float tax.
|
|
410
483
|
score: Math.round(r.score * 100) / 100,
|
|
411
|
-
excerpt: extractExcerpt(body, terms, input.excerptLength ?? 120)
|
|
484
|
+
excerpt: extractExcerpt(note.body, terms, input.excerptLength ?? 120)
|
|
485
|
+
};
|
|
486
|
+
decorate(hit, r.title, note.tags);
|
|
487
|
+
hits.push(hit);
|
|
488
|
+
if (hits.length >= input.limit) break;
|
|
489
|
+
}
|
|
490
|
+
return hits;
|
|
491
|
+
}
|
|
492
|
+
function semanticSearch(ctx2, semantic, input) {
|
|
493
|
+
const queryVec = semantic.embedQuery(input.query);
|
|
494
|
+
const candidates = semantic.store.topNotes(queryVec, SEMANTIC_CANDIDATES);
|
|
495
|
+
const terms = searchTerms(input.query);
|
|
496
|
+
const hits = [];
|
|
497
|
+
for (const c of candidates) {
|
|
498
|
+
const note = filterNote(ctx2, input, c.path);
|
|
499
|
+
if (!note) continue;
|
|
500
|
+
const hit = {
|
|
501
|
+
path: c.path,
|
|
502
|
+
// Cosine similarity ∈ [-1, 1]: 3 decimals keep the ranking legible.
|
|
503
|
+
score: Math.round(c.score * 1e3) / 1e3,
|
|
504
|
+
excerpt: chunkExcerpt(note.body, c, terms, input.excerptLength ?? 120)
|
|
412
505
|
};
|
|
413
|
-
|
|
414
|
-
if (noteTags.length > 0) hit.tags = noteTags;
|
|
506
|
+
decorate(hit, note.title, note.tags);
|
|
415
507
|
hits.push(hit);
|
|
416
508
|
if (hits.length >= input.limit) break;
|
|
417
509
|
}
|
|
418
510
|
return hits;
|
|
419
511
|
}
|
|
512
|
+
function filterNote(ctx2, input, path2) {
|
|
513
|
+
if (input.folder && !path2.startsWith(input.folder)) return void 0;
|
|
514
|
+
const note = ctx2.notes.get(path2);
|
|
515
|
+
if (!note) return void 0;
|
|
516
|
+
if (input.tag) {
|
|
517
|
+
const noteTags = note.tags.split(" ");
|
|
518
|
+
if (!noteTags.some((t) => t === input.tag || t === input.tag?.replace(/^#/, ""))) {
|
|
519
|
+
return void 0;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
return note;
|
|
523
|
+
}
|
|
524
|
+
function decorate(hit, title, tags) {
|
|
525
|
+
if (title && title !== basenameNoExt(hit.path)) hit.title = title;
|
|
526
|
+
const noteTags = tags ? tags.split(" ").filter(Boolean) : [];
|
|
527
|
+
if (noteTags.length > 0) hit.tags = noteTags;
|
|
528
|
+
}
|
|
420
529
|
|
|
421
530
|
// src/tools/context_pack.ts
|
|
422
531
|
var ContextPackInput = z3.object({
|
|
@@ -2719,7 +2828,7 @@ function createLogger(opts = {}) {
|
|
|
2719
2828
|
);
|
|
2720
2829
|
}
|
|
2721
2830
|
}
|
|
2722
|
-
function
|
|
2831
|
+
function writeFile4(record) {
|
|
2723
2832
|
if (!fileOk || filePath === void 0) return;
|
|
2724
2833
|
const line = `${JSON.stringify(record)}
|
|
2725
2834
|
`;
|
|
@@ -2741,7 +2850,7 @@ function createLogger(opts = {}) {
|
|
|
2741
2850
|
const f = fields ?? {};
|
|
2742
2851
|
stderr(`${lvl.toUpperCase().padEnd(5)} ${ts} ${msg}${fmtFields(f)}
|
|
2743
2852
|
`);
|
|
2744
|
-
|
|
2853
|
+
writeFile4({ t: ts, level: lvl, msg, ...f });
|
|
2745
2854
|
}
|
|
2746
2855
|
return {
|
|
2747
2856
|
level,
|
|
@@ -2772,6 +2881,664 @@ function installProcessGuards(log2) {
|
|
|
2772
2881
|
};
|
|
2773
2882
|
}
|
|
2774
2883
|
|
|
2884
|
+
// ../core/src/embed/chunk.ts
|
|
2885
|
+
var MIN_WINDOW_WORDS = 250;
|
|
2886
|
+
var MAX_WINDOW_WORDS = 400;
|
|
2887
|
+
function chunkNote(title, body) {
|
|
2888
|
+
const normalized = body.replace(/\r\n/g, "\n");
|
|
2889
|
+
const paragraphs = splitParagraphs(normalized).flatMap(splitOversized);
|
|
2890
|
+
if (paragraphs.length === 0) return [{ text: title, words: 0, start: 0, end: 0 }];
|
|
2891
|
+
const chunks = [];
|
|
2892
|
+
let window = [];
|
|
2893
|
+
let windowWords = 0;
|
|
2894
|
+
const flush = () => {
|
|
2895
|
+
const first = window[0];
|
|
2896
|
+
const last = window[window.length - 1];
|
|
2897
|
+
if (!first || !last) return;
|
|
2898
|
+
chunks.push({
|
|
2899
|
+
text: `${title}
|
|
2900
|
+
|
|
2901
|
+
${window.map((p) => p.text).join("\n\n")}`,
|
|
2902
|
+
words: windowWords,
|
|
2903
|
+
start: first.start,
|
|
2904
|
+
end: last.end
|
|
2905
|
+
});
|
|
2906
|
+
window = [];
|
|
2907
|
+
windowWords = 0;
|
|
2908
|
+
};
|
|
2909
|
+
for (const para of paragraphs) {
|
|
2910
|
+
if (windowWords >= MIN_WINDOW_WORDS || windowWords + para.words > MAX_WINDOW_WORDS) flush();
|
|
2911
|
+
window.push(para);
|
|
2912
|
+
windowWords += para.words;
|
|
2913
|
+
}
|
|
2914
|
+
flush();
|
|
2915
|
+
return chunks;
|
|
2916
|
+
}
|
|
2917
|
+
function splitParagraphs(normalized) {
|
|
2918
|
+
const out = [];
|
|
2919
|
+
const push = (rawStart, raw) => {
|
|
2920
|
+
const text = raw.trim();
|
|
2921
|
+
if (text.length === 0) return;
|
|
2922
|
+
const start = rawStart + (raw.length - raw.trimStart().length);
|
|
2923
|
+
out.push({ text, words: countWords(text), start, end: start + text.length });
|
|
2924
|
+
};
|
|
2925
|
+
const sep4 = /\n[ \t]*\n+/g;
|
|
2926
|
+
let prev = 0;
|
|
2927
|
+
for (const m of normalized.matchAll(sep4)) {
|
|
2928
|
+
push(prev, normalized.slice(prev, m.index));
|
|
2929
|
+
prev = m.index + m[0].length;
|
|
2930
|
+
}
|
|
2931
|
+
push(prev, normalized.slice(prev));
|
|
2932
|
+
return out;
|
|
2933
|
+
}
|
|
2934
|
+
function splitOversized(para) {
|
|
2935
|
+
if (para.words <= MAX_WINDOW_WORDS) return [para];
|
|
2936
|
+
const words = [...para.text.matchAll(/\S+/g)];
|
|
2937
|
+
const pieces = [];
|
|
2938
|
+
for (let i = 0; i < words.length; i += MAX_WINDOW_WORDS) {
|
|
2939
|
+
const slice = words.slice(i, i + MAX_WINDOW_WORDS);
|
|
2940
|
+
const first = slice[0];
|
|
2941
|
+
const last = slice[slice.length - 1];
|
|
2942
|
+
if (!first || !last) continue;
|
|
2943
|
+
pieces.push({
|
|
2944
|
+
text: slice.map((w) => w[0]).join(" "),
|
|
2945
|
+
words: slice.length,
|
|
2946
|
+
start: para.start + first.index,
|
|
2947
|
+
end: para.start + last.index + last[0].length
|
|
2948
|
+
});
|
|
2949
|
+
}
|
|
2950
|
+
return pieces;
|
|
2951
|
+
}
|
|
2952
|
+
function countWords(text) {
|
|
2953
|
+
const m = text.match(/\S+/g);
|
|
2954
|
+
return m ? m.length : 0;
|
|
2955
|
+
}
|
|
2956
|
+
|
|
2957
|
+
// ../core/src/embed/model2vec.ts
|
|
2958
|
+
import { readFile as readFile14 } from "fs/promises";
|
|
2959
|
+
import { basename as basename4, join as join7 } from "path";
|
|
2960
|
+
|
|
2961
|
+
// ../core/src/embed/safetensors.ts
|
|
2962
|
+
function readSafetensors(buf) {
|
|
2963
|
+
if (buf.byteLength < 8) {
|
|
2964
|
+
throw new Error(`safetensors: buffer too small (${buf.byteLength} bytes) to hold a header`);
|
|
2965
|
+
}
|
|
2966
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
2967
|
+
const headerLen = Number(view.getBigUint64(0, true));
|
|
2968
|
+
if (8 + headerLen > buf.byteLength) {
|
|
2969
|
+
throw new Error(
|
|
2970
|
+
`safetensors: header length ${headerLen} exceeds buffer (${buf.byteLength} bytes)`
|
|
2971
|
+
);
|
|
2972
|
+
}
|
|
2973
|
+
const header = JSON.parse(new TextDecoder().decode(buf.subarray(8, 8 + headerLen)));
|
|
2974
|
+
const dataStart = 8 + headerLen;
|
|
2975
|
+
const out = /* @__PURE__ */ new Map();
|
|
2976
|
+
const unsupported = [];
|
|
2977
|
+
for (const [name, meta] of Object.entries(header)) {
|
|
2978
|
+
if (name === "__metadata__") continue;
|
|
2979
|
+
if (meta.dtype !== "F32") {
|
|
2980
|
+
unsupported.push(`${name} (${meta.dtype})`);
|
|
2981
|
+
continue;
|
|
2982
|
+
}
|
|
2983
|
+
const [begin, end] = meta.data_offsets;
|
|
2984
|
+
const byteLen = end - begin;
|
|
2985
|
+
const elems = meta.shape.reduce((a, b) => a * b, 1);
|
|
2986
|
+
if (byteLen !== elems * 4) {
|
|
2987
|
+
throw new Error(
|
|
2988
|
+
`safetensors: tensor "${name}" data span ${byteLen} B does not match shape [${meta.shape.join(", ")}]`
|
|
2989
|
+
);
|
|
2990
|
+
}
|
|
2991
|
+
if (dataStart + end > buf.byteLength) {
|
|
2992
|
+
throw new Error(`safetensors: tensor "${name}" extends past the end of the buffer`);
|
|
2993
|
+
}
|
|
2994
|
+
const absStart = buf.byteOffset + dataStart + begin;
|
|
2995
|
+
let data;
|
|
2996
|
+
if (absStart % 4 === 0) {
|
|
2997
|
+
data = new Float32Array(buf.buffer, absStart, elems);
|
|
2998
|
+
} else {
|
|
2999
|
+
const copy = new Uint8Array(byteLen);
|
|
3000
|
+
copy.set(buf.subarray(dataStart + begin, dataStart + end));
|
|
3001
|
+
data = new Float32Array(copy.buffer);
|
|
3002
|
+
}
|
|
3003
|
+
out.set(name, { dtype: meta.dtype, shape: meta.shape, data });
|
|
3004
|
+
}
|
|
3005
|
+
if (unsupported.length > 0) {
|
|
3006
|
+
throw new Error(`safetensors: only F32 tensors are supported; found ${unsupported.join(", ")}`);
|
|
3007
|
+
}
|
|
3008
|
+
return out;
|
|
3009
|
+
}
|
|
3010
|
+
|
|
3011
|
+
// ../core/src/embed/tokenizer.ts
|
|
3012
|
+
function loadWordPieceTokenizer(tokenizerJson) {
|
|
3013
|
+
const tj = tokenizerJson;
|
|
3014
|
+
const model = tj?.model;
|
|
3015
|
+
if (model?.type !== "WordPiece" || !model.vocab) {
|
|
3016
|
+
throw new Error(
|
|
3017
|
+
`tokenizer: unsupported model type "${model?.type ?? "missing"}" (only WordPiece is implemented)`
|
|
3018
|
+
);
|
|
3019
|
+
}
|
|
3020
|
+
const vocab = new Map(Object.entries(model.vocab));
|
|
3021
|
+
const unkToken = model.unk_token ?? "[UNK]";
|
|
3022
|
+
const unkId = vocab.get(unkToken);
|
|
3023
|
+
if (unkId === void 0) {
|
|
3024
|
+
throw new Error(`tokenizer: unk_token "${unkToken}" is not in the vocab`);
|
|
3025
|
+
}
|
|
3026
|
+
const prefix = model.continuing_subword_prefix ?? "##";
|
|
3027
|
+
const maxChars = model.max_input_chars_per_word ?? 100;
|
|
3028
|
+
const norm = tj.normalizer ?? {};
|
|
3029
|
+
const cleanText = norm.clean_text ?? true;
|
|
3030
|
+
const handleCjk = norm.handle_chinese_chars ?? true;
|
|
3031
|
+
const lowercase = norm.lowercase ?? true;
|
|
3032
|
+
const stripAccents = norm.strip_accents ?? lowercase;
|
|
3033
|
+
const specialIds = new Set(
|
|
3034
|
+
(tj.added_tokens ?? []).filter((t) => t.special).map((t) => t.id)
|
|
3035
|
+
);
|
|
3036
|
+
function normalize(text) {
|
|
3037
|
+
let out = cleanText ? doCleanText(text) : text;
|
|
3038
|
+
if (handleCjk) out = padCjkChars(out);
|
|
3039
|
+
if (stripAccents) out = out.normalize("NFD").replace(new RegExp("\\p{Mn}", "gu"), "");
|
|
3040
|
+
if (lowercase) out = out.toLowerCase();
|
|
3041
|
+
return out;
|
|
3042
|
+
}
|
|
3043
|
+
function wordPiece(word) {
|
|
3044
|
+
const chars = [...word];
|
|
3045
|
+
if (chars.length > maxChars) return [unkId];
|
|
3046
|
+
const ids = [];
|
|
3047
|
+
let start = 0;
|
|
3048
|
+
while (start < chars.length) {
|
|
3049
|
+
let end = chars.length;
|
|
3050
|
+
let id;
|
|
3051
|
+
while (start < end) {
|
|
3052
|
+
const piece = (start > 0 ? prefix : "") + chars.slice(start, end).join("");
|
|
3053
|
+
const found = vocab.get(piece);
|
|
3054
|
+
if (found !== void 0) {
|
|
3055
|
+
id = found;
|
|
3056
|
+
break;
|
|
3057
|
+
}
|
|
3058
|
+
end--;
|
|
3059
|
+
}
|
|
3060
|
+
if (id === void 0) return [unkId];
|
|
3061
|
+
ids.push(id);
|
|
3062
|
+
start = end;
|
|
3063
|
+
}
|
|
3064
|
+
return ids;
|
|
3065
|
+
}
|
|
3066
|
+
return {
|
|
3067
|
+
vocabSize: vocab.size,
|
|
3068
|
+
unkId,
|
|
3069
|
+
specialIds,
|
|
3070
|
+
encode(text) {
|
|
3071
|
+
const ids = [];
|
|
3072
|
+
for (const word of preTokenize(normalize(text))) {
|
|
3073
|
+
for (const id of wordPiece(word)) ids.push(id);
|
|
3074
|
+
}
|
|
3075
|
+
return ids;
|
|
3076
|
+
}
|
|
3077
|
+
};
|
|
3078
|
+
}
|
|
3079
|
+
function preTokenize(text) {
|
|
3080
|
+
const words = [];
|
|
3081
|
+
let cur = "";
|
|
3082
|
+
for (const ch of text) {
|
|
3083
|
+
const cp = ch.codePointAt(0);
|
|
3084
|
+
if (isWhitespace(ch, cp)) {
|
|
3085
|
+
if (cur) {
|
|
3086
|
+
words.push(cur);
|
|
3087
|
+
cur = "";
|
|
3088
|
+
}
|
|
3089
|
+
} else if (isPunctuation(ch, cp)) {
|
|
3090
|
+
if (cur) {
|
|
3091
|
+
words.push(cur);
|
|
3092
|
+
cur = "";
|
|
3093
|
+
}
|
|
3094
|
+
words.push(ch);
|
|
3095
|
+
} else {
|
|
3096
|
+
cur += ch;
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
if (cur) words.push(cur);
|
|
3100
|
+
return words;
|
|
3101
|
+
}
|
|
3102
|
+
function doCleanText(text) {
|
|
3103
|
+
let out = "";
|
|
3104
|
+
for (const ch of text) {
|
|
3105
|
+
const cp = ch.codePointAt(0);
|
|
3106
|
+
if (cp === 0 || cp === 65533 || isControl(ch, cp)) continue;
|
|
3107
|
+
out += isWhitespace(ch, cp) ? " " : ch;
|
|
3108
|
+
}
|
|
3109
|
+
return out;
|
|
3110
|
+
}
|
|
3111
|
+
function padCjkChars(text) {
|
|
3112
|
+
let out = "";
|
|
3113
|
+
for (const ch of text) {
|
|
3114
|
+
out += isCjk(ch.codePointAt(0)) ? ` ${ch} ` : ch;
|
|
3115
|
+
}
|
|
3116
|
+
return out;
|
|
3117
|
+
}
|
|
3118
|
+
function isWhitespace(ch, cp) {
|
|
3119
|
+
if (cp === 32 || cp === 9 || cp === 10 || cp === 13) return true;
|
|
3120
|
+
return new RegExp("\\p{Zs}", "u").test(ch);
|
|
3121
|
+
}
|
|
3122
|
+
function isControl(ch, cp) {
|
|
3123
|
+
if (cp === 9 || cp === 10 || cp === 13) return false;
|
|
3124
|
+
return new RegExp("\\p{Cc}|\\p{Cf}", "u").test(ch);
|
|
3125
|
+
}
|
|
3126
|
+
function isPunctuation(ch, cp) {
|
|
3127
|
+
if (cp >= 33 && cp <= 47 || cp >= 58 && cp <= 64 || cp >= 91 && cp <= 96 || cp >= 123 && cp <= 126) {
|
|
3128
|
+
return true;
|
|
3129
|
+
}
|
|
3130
|
+
return new RegExp("\\p{P}", "u").test(ch);
|
|
3131
|
+
}
|
|
3132
|
+
function isCjk(cp) {
|
|
3133
|
+
return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
|
|
3134
|
+
}
|
|
3135
|
+
|
|
3136
|
+
// ../core/src/embed/model2vec.ts
|
|
3137
|
+
var TENSOR_NAME = "embeddings";
|
|
3138
|
+
async function loadModel2Vec(modelDir) {
|
|
3139
|
+
const [configRaw, tokenizerRaw, weights] = await Promise.all([
|
|
3140
|
+
readFile14(join7(modelDir, "config.json"), "utf8"),
|
|
3141
|
+
readFile14(join7(modelDir, "tokenizer.json"), "utf8"),
|
|
3142
|
+
readFile14(join7(modelDir, "model.safetensors"))
|
|
3143
|
+
]);
|
|
3144
|
+
const config = JSON.parse(configRaw);
|
|
3145
|
+
const tokenizer = loadWordPieceTokenizer(JSON.parse(tokenizerRaw));
|
|
3146
|
+
const tensors = readSafetensors(weights);
|
|
3147
|
+
const tensor = tensors.get(TENSOR_NAME) ?? (tensors.size === 1 ? [...tensors.values()][0] : void 0);
|
|
3148
|
+
if (!tensor) {
|
|
3149
|
+
throw new Error(
|
|
3150
|
+
`model2vec: no "${TENSOR_NAME}" tensor in ${modelDir}; found: ${[...tensors.keys()].join(", ")}`
|
|
3151
|
+
);
|
|
3152
|
+
}
|
|
3153
|
+
if (tensor.shape.length !== 2) {
|
|
3154
|
+
throw new Error(`model2vec: expected a 2-D embedding matrix, got [${tensor.shape.join(", ")}]`);
|
|
3155
|
+
}
|
|
3156
|
+
const [rows, dim] = tensor.shape;
|
|
3157
|
+
if (rows !== tokenizer.vocabSize) {
|
|
3158
|
+
throw new Error(
|
|
3159
|
+
`model2vec: embedding rows (${rows}) do not match tokenizer vocab size (${tokenizer.vocabSize})`
|
|
3160
|
+
);
|
|
3161
|
+
}
|
|
3162
|
+
if (config.hidden_dim !== void 0 && config.hidden_dim !== dim) {
|
|
3163
|
+
throw new Error(
|
|
3164
|
+
`model2vec: config hidden_dim ${config.hidden_dim} does not match tensor dim ${dim}`
|
|
3165
|
+
);
|
|
3166
|
+
}
|
|
3167
|
+
const table = tensor.data;
|
|
3168
|
+
return {
|
|
3169
|
+
id: basename4(modelDir),
|
|
3170
|
+
dim,
|
|
3171
|
+
embed(text) {
|
|
3172
|
+
const out = new Float32Array(dim);
|
|
3173
|
+
let pooled = 0;
|
|
3174
|
+
for (const id of tokenizer.encode(text)) {
|
|
3175
|
+
if (tokenizer.specialIds.has(id) && id !== tokenizer.unkId) continue;
|
|
3176
|
+
const base = id * dim;
|
|
3177
|
+
for (let j = 0; j < dim; j++) {
|
|
3178
|
+
out[j] = out[j] + table[base + j];
|
|
3179
|
+
}
|
|
3180
|
+
pooled++;
|
|
3181
|
+
}
|
|
3182
|
+
if (pooled === 0) return out;
|
|
3183
|
+
let sumSquares = 0;
|
|
3184
|
+
for (let j = 0; j < dim; j++) {
|
|
3185
|
+
const mean = out[j] / pooled;
|
|
3186
|
+
out[j] = mean;
|
|
3187
|
+
sumSquares += mean * mean;
|
|
3188
|
+
}
|
|
3189
|
+
const norm = Math.sqrt(sumSquares);
|
|
3190
|
+
if (norm > 0) {
|
|
3191
|
+
for (let j = 0; j < dim; j++) out[j] = out[j] / norm;
|
|
3192
|
+
}
|
|
3193
|
+
return out;
|
|
3194
|
+
}
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
// src/semantic/cache.ts
|
|
3199
|
+
import { createHash as createHash2 } from "crypto";
|
|
3200
|
+
import { mkdir as mkdir6, readFile as readFile15, rename as rename4, writeFile as writeFile3 } from "fs/promises";
|
|
3201
|
+
import { join as join8 } from "path";
|
|
3202
|
+
var CACHE_VERSION = 2;
|
|
3203
|
+
function cachePathsFor(cacheDir, vaultRoot2, modelId) {
|
|
3204
|
+
const vaultKey = createHash2("sha256").update(vaultRoot2).digest("hex").slice(0, 16);
|
|
3205
|
+
const dir = join8(cacheDir, "embeddings", vaultKey);
|
|
3206
|
+
return { dir, bin: join8(dir, `${modelId}.bin`), manifest: join8(dir, `${modelId}.json`) };
|
|
3207
|
+
}
|
|
3208
|
+
async function loadCache(paths, modelId, dim) {
|
|
3209
|
+
try {
|
|
3210
|
+
const manifest = JSON.parse(await readFile15(paths.manifest, "utf8"));
|
|
3211
|
+
if (manifest.version !== CACHE_VERSION || manifest.modelId !== modelId || manifest.dim !== dim) {
|
|
3212
|
+
return void 0;
|
|
3213
|
+
}
|
|
3214
|
+
const buf = await readFile15(paths.bin);
|
|
3215
|
+
const totalChunks = manifest.entries.reduce((a, e) => a + e.chunks, 0);
|
|
3216
|
+
const vectorBytes = totalChunks * dim * 4;
|
|
3217
|
+
const spanBytes = totalChunks * 2 * 4;
|
|
3218
|
+
if (buf.byteLength !== vectorBytes + spanBytes) return void 0;
|
|
3219
|
+
const aligned = new Uint8Array(buf.byteLength);
|
|
3220
|
+
aligned.set(buf);
|
|
3221
|
+
const floats = new Float32Array(aligned.buffer, 0, totalChunks * dim);
|
|
3222
|
+
const spans = new Uint32Array(aligned.buffer, vectorBytes, totalChunks * 2);
|
|
3223
|
+
const vectors = /* @__PURE__ */ new Map();
|
|
3224
|
+
const hashes = /* @__PURE__ */ new Map();
|
|
3225
|
+
let chunkOffset = 0;
|
|
3226
|
+
for (const e of manifest.entries) {
|
|
3227
|
+
if (e.chunks <= 0) return void 0;
|
|
3228
|
+
vectors.set(e.path, {
|
|
3229
|
+
packed: floats.subarray(chunkOffset * dim, (chunkOffset + e.chunks) * dim),
|
|
3230
|
+
spans: spans.subarray(chunkOffset * 2, (chunkOffset + e.chunks) * 2)
|
|
3231
|
+
});
|
|
3232
|
+
hashes.set(e.path, e.contentHash);
|
|
3233
|
+
chunkOffset += e.chunks;
|
|
3234
|
+
}
|
|
3235
|
+
return { vectors, hashes };
|
|
3236
|
+
} catch {
|
|
3237
|
+
return void 0;
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
async function saveCache(paths, modelId, dim, notes2, hashes) {
|
|
3241
|
+
const entries = [];
|
|
3242
|
+
const kept = [];
|
|
3243
|
+
let totalChunks = 0;
|
|
3244
|
+
for (const [path2, note] of notes2) {
|
|
3245
|
+
const contentHash2 = hashes.get(path2);
|
|
3246
|
+
if (!contentHash2) continue;
|
|
3247
|
+
const chunks = note.packed.length / dim;
|
|
3248
|
+
entries.push({ path: path2, contentHash: contentHash2, chunks });
|
|
3249
|
+
kept.push(note);
|
|
3250
|
+
totalChunks += chunks;
|
|
3251
|
+
}
|
|
3252
|
+
const bin = new Uint8Array(totalChunks * dim * 4 + totalChunks * 2 * 4);
|
|
3253
|
+
const floats = new Float32Array(bin.buffer, 0, totalChunks * dim);
|
|
3254
|
+
const spans = new Uint32Array(bin.buffer, totalChunks * dim * 4, totalChunks * 2);
|
|
3255
|
+
let floatOffset = 0;
|
|
3256
|
+
let spanOffset = 0;
|
|
3257
|
+
for (const note of kept) {
|
|
3258
|
+
floats.set(note.packed, floatOffset);
|
|
3259
|
+
spans.set(note.spans, spanOffset);
|
|
3260
|
+
floatOffset += note.packed.length;
|
|
3261
|
+
spanOffset += note.spans.length;
|
|
3262
|
+
}
|
|
3263
|
+
const manifest = { version: CACHE_VERSION, modelId, dim, entries };
|
|
3264
|
+
await mkdir6(paths.dir, { recursive: true });
|
|
3265
|
+
await atomicWriteBytes(paths.bin, bin);
|
|
3266
|
+
await atomicWriteBytes(paths.manifest, new TextEncoder().encode(JSON.stringify(manifest)));
|
|
3267
|
+
}
|
|
3268
|
+
async function atomicWriteBytes(absPath, bytes) {
|
|
3269
|
+
const tmpPath = `${absPath}.seekstone-cache-tmp`;
|
|
3270
|
+
await writeFile3(tmpPath, bytes);
|
|
3271
|
+
await rename4(tmpPath, absPath);
|
|
3272
|
+
}
|
|
3273
|
+
|
|
3274
|
+
// src/semantic/store.ts
|
|
3275
|
+
var SemanticStore = class {
|
|
3276
|
+
dim;
|
|
3277
|
+
notes = /* @__PURE__ */ new Map();
|
|
3278
|
+
chunks = 0;
|
|
3279
|
+
constructor(dim) {
|
|
3280
|
+
if (!Number.isInteger(dim) || dim <= 0) {
|
|
3281
|
+
throw new Error(`semantic store: invalid dim ${dim}`);
|
|
3282
|
+
}
|
|
3283
|
+
this.dim = dim;
|
|
3284
|
+
}
|
|
3285
|
+
/** Total chunk vectors across all notes. */
|
|
3286
|
+
get chunkCount() {
|
|
3287
|
+
return this.chunks;
|
|
3288
|
+
}
|
|
3289
|
+
get noteCount() {
|
|
3290
|
+
return this.notes.size;
|
|
3291
|
+
}
|
|
3292
|
+
/** Replace (or insert) a note's packed chunk vectors and spans. */
|
|
3293
|
+
setNote(path2, packed, spans) {
|
|
3294
|
+
if (packed.length === 0 || packed.length % this.dim !== 0) {
|
|
3295
|
+
throw new Error(
|
|
3296
|
+
`semantic store: packed length ${packed.length} is not a positive multiple of dim ${this.dim} (${path2})`
|
|
3297
|
+
);
|
|
3298
|
+
}
|
|
3299
|
+
const n = packed.length / this.dim;
|
|
3300
|
+
if (spans.length !== n * 2) {
|
|
3301
|
+
throw new Error(
|
|
3302
|
+
`semantic store: spans length ${spans.length} does not match ${n} chunks (${path2})`
|
|
3303
|
+
);
|
|
3304
|
+
}
|
|
3305
|
+
const prev = this.notes.get(path2);
|
|
3306
|
+
if (prev) this.chunks -= prev.packed.length / this.dim;
|
|
3307
|
+
this.notes.set(path2, { packed, spans });
|
|
3308
|
+
this.chunks += n;
|
|
3309
|
+
}
|
|
3310
|
+
removeNote(path2) {
|
|
3311
|
+
const prev = this.notes.get(path2);
|
|
3312
|
+
if (!prev) return;
|
|
3313
|
+
this.chunks -= prev.packed.length / this.dim;
|
|
3314
|
+
this.notes.delete(path2);
|
|
3315
|
+
}
|
|
3316
|
+
getNote(path2) {
|
|
3317
|
+
return this.notes.get(path2);
|
|
3318
|
+
}
|
|
3319
|
+
/** Iterate (path, note vectors) — cache serialization uses this. */
|
|
3320
|
+
entries() {
|
|
3321
|
+
return this.notes.entries();
|
|
3322
|
+
}
|
|
3323
|
+
/** Top `k` notes by max-pooled chunk cosine, ties broken path-ascending. */
|
|
3324
|
+
topNotes(query, k) {
|
|
3325
|
+
if (query.length !== this.dim) {
|
|
3326
|
+
throw new Error(`semantic store: query dim ${query.length} does not match ${this.dim}`);
|
|
3327
|
+
}
|
|
3328
|
+
const dim = this.dim;
|
|
3329
|
+
const hits = [];
|
|
3330
|
+
for (const [path2, { packed, spans }] of this.notes) {
|
|
3331
|
+
let best = Number.NEGATIVE_INFINITY;
|
|
3332
|
+
let bestChunk = 0;
|
|
3333
|
+
const n = packed.length / dim;
|
|
3334
|
+
for (let c = 0; c < n; c++) {
|
|
3335
|
+
const base = c * dim;
|
|
3336
|
+
let score = 0;
|
|
3337
|
+
for (let j = 0; j < dim; j++) {
|
|
3338
|
+
score += packed[base + j] * query[j];
|
|
3339
|
+
}
|
|
3340
|
+
if (score > best) {
|
|
3341
|
+
best = score;
|
|
3342
|
+
bestChunk = c;
|
|
3343
|
+
}
|
|
3344
|
+
}
|
|
3345
|
+
hits.push({
|
|
3346
|
+
path: path2,
|
|
3347
|
+
score: best,
|
|
3348
|
+
chunkIndex: bestChunk,
|
|
3349
|
+
start: spans[bestChunk * 2],
|
|
3350
|
+
end: spans[bestChunk * 2 + 1]
|
|
3351
|
+
});
|
|
3352
|
+
}
|
|
3353
|
+
return hits.sort((a, b) => b.score - a.score || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)).slice(0, Math.max(0, k));
|
|
3354
|
+
}
|
|
3355
|
+
};
|
|
3356
|
+
|
|
3357
|
+
// src/semantic/state.ts
|
|
3358
|
+
var Semantic = class _Semantic {
|
|
3359
|
+
embedder;
|
|
3360
|
+
store;
|
|
3361
|
+
progress;
|
|
3362
|
+
ctx;
|
|
3363
|
+
paths;
|
|
3364
|
+
log;
|
|
3365
|
+
debounceMs;
|
|
3366
|
+
saveDebounceMs;
|
|
3367
|
+
yieldEvery;
|
|
3368
|
+
hashes = /* @__PURE__ */ new Map();
|
|
3369
|
+
pending = /* @__PURE__ */ new Map();
|
|
3370
|
+
saveTimer;
|
|
3371
|
+
stopped = false;
|
|
3372
|
+
constructor(embedder, ctx2, cfg, deps) {
|
|
3373
|
+
this.embedder = embedder;
|
|
3374
|
+
this.store = new SemanticStore(embedder.dim);
|
|
3375
|
+
this.ctx = ctx2;
|
|
3376
|
+
this.paths = cachePathsFor(cfg.cacheDir, ctx2.vaultRoot, embedder.id);
|
|
3377
|
+
this.log = deps.log;
|
|
3378
|
+
this.debounceMs = deps.debounceMs ?? 300;
|
|
3379
|
+
this.saveDebounceMs = deps.saveDebounceMs ?? 5e3;
|
|
3380
|
+
this.yieldEvery = deps.yieldEvery ?? 8;
|
|
3381
|
+
this.progress = { state: "building", done: 0, total: ctx2.notes.size };
|
|
3382
|
+
}
|
|
3383
|
+
/**
|
|
3384
|
+
* Load the model and kick off the background index build. Rejects with an
|
|
3385
|
+
* actionable message when the model files are missing — the user enabled
|
|
3386
|
+
* the feature explicitly, so a broken setup should fail loudly at boot.
|
|
3387
|
+
*/
|
|
3388
|
+
static async start(ctx2, cfg, deps = {}) {
|
|
3389
|
+
let embedder;
|
|
3390
|
+
try {
|
|
3391
|
+
embedder = await (deps.loadModel ?? loadModel2Vec)(cfg.modelDir);
|
|
3392
|
+
} catch (err) {
|
|
3393
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
3394
|
+
throw new Error(
|
|
3395
|
+
`semantic search: could not load the embedding model from ${cfg.modelDir} \u2014 run \`npx -y seekstone fetch-model\` to download it, or point SEEKSTONE_MODEL_PATH at a Model2Vec model directory (${reason})`
|
|
3396
|
+
);
|
|
3397
|
+
}
|
|
3398
|
+
const semantic = new _Semantic(embedder, ctx2, cfg, deps);
|
|
3399
|
+
void semantic.build();
|
|
3400
|
+
return semantic;
|
|
3401
|
+
}
|
|
3402
|
+
/** Resolves when the initial build has finished (tests and progress checks). */
|
|
3403
|
+
buildDone = Promise.resolve();
|
|
3404
|
+
ready() {
|
|
3405
|
+
return this.buildDone;
|
|
3406
|
+
}
|
|
3407
|
+
build() {
|
|
3408
|
+
this.buildDone = this.buildInner().catch((err) => {
|
|
3409
|
+
this.log?.error("semantic build failed", {
|
|
3410
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3411
|
+
});
|
|
3412
|
+
});
|
|
3413
|
+
return this.buildDone;
|
|
3414
|
+
}
|
|
3415
|
+
async buildInner() {
|
|
3416
|
+
const t0 = performance.now();
|
|
3417
|
+
const cached = await loadCache(this.paths, this.embedder.id, this.embedder.dim);
|
|
3418
|
+
const ids = [...this.ctx.notes.keys()];
|
|
3419
|
+
this.progress = { state: "building", done: 0, total: ids.length };
|
|
3420
|
+
let reused = 0;
|
|
3421
|
+
let sinceYield = 0;
|
|
3422
|
+
for (const id of ids) {
|
|
3423
|
+
if (this.stopped) return;
|
|
3424
|
+
if (!this.hashes.has(id)) {
|
|
3425
|
+
const note = this.ctx.notes.get(id);
|
|
3426
|
+
if (note) {
|
|
3427
|
+
const current = contentHash(note.raw);
|
|
3428
|
+
const cachedVecs = cached?.hashes.get(id) === current ? cached.vectors.get(id) : void 0;
|
|
3429
|
+
if (cachedVecs !== void 0 && cachedVecs.packed.length > 0) {
|
|
3430
|
+
this.store.setNote(id, cachedVecs.packed, cachedVecs.spans);
|
|
3431
|
+
reused++;
|
|
3432
|
+
} else {
|
|
3433
|
+
const { packed, spans } = this.embedNote(note);
|
|
3434
|
+
this.store.setNote(id, packed, spans);
|
|
3435
|
+
if (++sinceYield >= this.yieldEvery) {
|
|
3436
|
+
sinceYield = 0;
|
|
3437
|
+
await new Promise((r) => setImmediate(r));
|
|
3438
|
+
}
|
|
3439
|
+
}
|
|
3440
|
+
this.hashes.set(id, current);
|
|
3441
|
+
}
|
|
3442
|
+
}
|
|
3443
|
+
if (this.progress.state === "building") this.progress.done++;
|
|
3444
|
+
}
|
|
3445
|
+
this.progress = { state: "ready" };
|
|
3446
|
+
this.log?.info("semantic index ready", {
|
|
3447
|
+
notes: this.store.noteCount,
|
|
3448
|
+
chunks: this.store.chunkCount,
|
|
3449
|
+
reusedFromCache: reused,
|
|
3450
|
+
buildMs: Math.round(performance.now() - t0)
|
|
3451
|
+
});
|
|
3452
|
+
if (reused < this.store.noteCount) await this.save();
|
|
3453
|
+
}
|
|
3454
|
+
embedNote(note) {
|
|
3455
|
+
const chunks = chunkNote(note.title, note.body);
|
|
3456
|
+
const dim = this.embedder.dim;
|
|
3457
|
+
const packed = new Float32Array(chunks.length * dim);
|
|
3458
|
+
const spans = new Uint32Array(chunks.length * 2);
|
|
3459
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
3460
|
+
const chunk = chunks[i];
|
|
3461
|
+
packed.set(this.embedder.embed(chunk.text), i * dim);
|
|
3462
|
+
spans[i * 2] = chunk.start;
|
|
3463
|
+
spans[i * 2 + 1] = chunk.end;
|
|
3464
|
+
}
|
|
3465
|
+
return { packed, spans };
|
|
3466
|
+
}
|
|
3467
|
+
embedQuery(query) {
|
|
3468
|
+
return this.embedder.embed(query);
|
|
3469
|
+
}
|
|
3470
|
+
/** Watcher hook: a note was added or changed (ctx.notes is already fresh). */
|
|
3471
|
+
noteChanged(path2) {
|
|
3472
|
+
if (this.stopped) return;
|
|
3473
|
+
const existing = this.pending.get(path2);
|
|
3474
|
+
if (existing) clearTimeout(existing);
|
|
3475
|
+
const timer = setTimeout(() => {
|
|
3476
|
+
this.pending.delete(path2);
|
|
3477
|
+
this.reembed(path2);
|
|
3478
|
+
}, this.debounceMs);
|
|
3479
|
+
timer.unref?.();
|
|
3480
|
+
this.pending.set(path2, timer);
|
|
3481
|
+
}
|
|
3482
|
+
/** Watcher hook: a note was deleted. */
|
|
3483
|
+
noteRemoved(path2) {
|
|
3484
|
+
const existing = this.pending.get(path2);
|
|
3485
|
+
if (existing) {
|
|
3486
|
+
clearTimeout(existing);
|
|
3487
|
+
this.pending.delete(path2);
|
|
3488
|
+
}
|
|
3489
|
+
if (this.hashes.has(path2) || this.store.getNote(path2)) {
|
|
3490
|
+
this.store.removeNote(path2);
|
|
3491
|
+
this.hashes.delete(path2);
|
|
3492
|
+
this.scheduleSave();
|
|
3493
|
+
}
|
|
3494
|
+
}
|
|
3495
|
+
reembed(path2) {
|
|
3496
|
+
const note = this.ctx.notes.get(path2);
|
|
3497
|
+
if (!note) return;
|
|
3498
|
+
const current = contentHash(note.raw);
|
|
3499
|
+
const prior = this.hashes.get(path2);
|
|
3500
|
+
if (prior === current) return;
|
|
3501
|
+
const { packed, spans } = this.embedNote(note);
|
|
3502
|
+
this.store.setNote(path2, packed, spans);
|
|
3503
|
+
this.hashes.set(path2, current);
|
|
3504
|
+
this.log?.debug("semantic re-embed", { path: path2 });
|
|
3505
|
+
this.scheduleSave();
|
|
3506
|
+
}
|
|
3507
|
+
scheduleSave() {
|
|
3508
|
+
if (this.saveTimer) clearTimeout(this.saveTimer);
|
|
3509
|
+
this.saveTimer = setTimeout(() => {
|
|
3510
|
+
this.saveTimer = void 0;
|
|
3511
|
+
void this.save();
|
|
3512
|
+
}, this.saveDebounceMs);
|
|
3513
|
+
this.saveTimer.unref?.();
|
|
3514
|
+
}
|
|
3515
|
+
async save() {
|
|
3516
|
+
if (this.stopped) return;
|
|
3517
|
+
try {
|
|
3518
|
+
await saveCache(
|
|
3519
|
+
this.paths,
|
|
3520
|
+
this.embedder.id,
|
|
3521
|
+
this.embedder.dim,
|
|
3522
|
+
this.store.entries(),
|
|
3523
|
+
this.hashes
|
|
3524
|
+
);
|
|
3525
|
+
this.log?.debug("semantic cache saved", { notes: this.store.noteCount });
|
|
3526
|
+
} catch (err) {
|
|
3527
|
+
this.log?.warn("semantic cache save failed", {
|
|
3528
|
+
error: err instanceof Error ? err.message : String(err)
|
|
3529
|
+
});
|
|
3530
|
+
}
|
|
3531
|
+
}
|
|
3532
|
+
/** Tear down timers (tests; the process otherwise runs for the session). */
|
|
3533
|
+
stop() {
|
|
3534
|
+
this.stopped = true;
|
|
3535
|
+
for (const timer of this.pending.values()) clearTimeout(timer);
|
|
3536
|
+
this.pending.clear();
|
|
3537
|
+
if (this.saveTimer) clearTimeout(this.saveTimer);
|
|
3538
|
+
this.saveTimer = void 0;
|
|
3539
|
+
}
|
|
3540
|
+
};
|
|
3541
|
+
|
|
2775
3542
|
// src/tool-list.ts
|
|
2776
3543
|
var READ_ONLY_ANNOTATIONS = {
|
|
2777
3544
|
readOnlyHint: true,
|
|
@@ -2799,11 +3566,16 @@ var ALL_TOOLS = [
|
|
|
2799
3566
|
{
|
|
2800
3567
|
name: "search",
|
|
2801
3568
|
annotations: READ_ONLY_ANNOTATIONS,
|
|
2802
|
-
description:
|
|
3569
|
+
description: 'Full-text search across the vault. Returns ranked excerpts (~200 chars) \u2014 not full notes \u2014 to minimise context usage. Supports fuzzy matching and prefix search; with SEEKSTONE_SEMANTIC=1, mode "semantic"/"hybrid" searches by meaning via a local embedding model.',
|
|
2803
3570
|
inputSchema: {
|
|
2804
3571
|
type: "object",
|
|
2805
3572
|
properties: {
|
|
2806
3573
|
query: { type: "string", description: "Search query." },
|
|
3574
|
+
mode: {
|
|
3575
|
+
type: "string",
|
|
3576
|
+
enum: ["lexical", "semantic", "hybrid"],
|
|
3577
|
+
description: "lexical = keyword search (default). semantic = meaning-based search (requires SEEKSTONE_SEMANTIC=1). hybrid = exact-title lookups go lexical, everything else semantic."
|
|
3578
|
+
},
|
|
2807
3579
|
limit: { type: "number", description: "Max results (1\u201350, default 10)." },
|
|
2808
3580
|
folder: { type: "string", description: "Restrict to a vault-relative folder prefix." },
|
|
2809
3581
|
tag: { type: "string", description: "Restrict to notes with this tag." }
|
|
@@ -3288,8 +4060,8 @@ function visibleTools(policy2) {
|
|
|
3288
4060
|
|
|
3289
4061
|
// src/watcher.ts
|
|
3290
4062
|
import { realpathSync } from "fs";
|
|
3291
|
-
import { readFile as
|
|
3292
|
-
import { join as
|
|
4063
|
+
import { readFile as readFile16 } from "fs/promises";
|
|
4064
|
+
import { join as join9, relative as relative2, sep as sep3 } from "path";
|
|
3293
4065
|
import chokidar from "chokidar";
|
|
3294
4066
|
function startWatcher(ctx2, log2, opts) {
|
|
3295
4067
|
const usePolling = opts?.usePolling ?? process.env.SEEKSTONE_WATCH_POLL === "1";
|
|
@@ -3301,10 +4073,11 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
3301
4073
|
const toRel = (abs) => relative2(vaultRoot2, abs).split(sep3).join("/");
|
|
3302
4074
|
async function upsert(relPath, op) {
|
|
3303
4075
|
try {
|
|
3304
|
-
const raw = await
|
|
4076
|
+
const raw = await readFile16(join9(vaultRoot2, relPath), "utf8");
|
|
3305
4077
|
removeNoteBacklinks(ctx2, relPath);
|
|
3306
4078
|
upsertDoc(ctx2, buildDoc(relPath, raw));
|
|
3307
4079
|
addNoteBacklinks(ctx2, relPath, raw);
|
|
4080
|
+
ctx2.semantic?.noteChanged(relPath);
|
|
3308
4081
|
log2?.debug("index updated", { path: relPath, op });
|
|
3309
4082
|
} catch {
|
|
3310
4083
|
}
|
|
@@ -3314,6 +4087,7 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
3314
4087
|
removeNoteBacklinks(ctx2, relPath);
|
|
3315
4088
|
ctx2.index.discard(relPath);
|
|
3316
4089
|
ctx2.notes.delete(relPath);
|
|
4090
|
+
ctx2.semantic?.noteRemoved(relPath);
|
|
3317
4091
|
log2?.debug("index removed", { path: relPath, op: "delete" });
|
|
3318
4092
|
}
|
|
3319
4093
|
}
|
|
@@ -3353,7 +4127,7 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
3353
4127
|
}
|
|
3354
4128
|
|
|
3355
4129
|
// src/index.ts
|
|
3356
|
-
var VERSION = true ? "0.
|
|
4130
|
+
var VERSION = true ? "0.15.0" : "0.0.0-dev";
|
|
3357
4131
|
var intent = parseCliIntent(process.argv.slice(2));
|
|
3358
4132
|
if (intent === "version") {
|
|
3359
4133
|
process.stdout.write(`${VERSION}
|
|
@@ -3381,6 +4155,14 @@ if (intent === "init") {
|
|
|
3381
4155
|
`);
|
|
3382
4156
|
process.exit(result.exitCode);
|
|
3383
4157
|
}
|
|
4158
|
+
if (intent === "fetch-model") {
|
|
4159
|
+
const { runFetchModel } = await import("./fetch-model-7TGOJMZL.js");
|
|
4160
|
+
const { homedir: homedir2 } = await import("os");
|
|
4161
|
+
const result = await runFetchModel({ env: process.env, homedir: homedir2() });
|
|
4162
|
+
process.stdout.write(`${result.output.join("\n")}
|
|
4163
|
+
`);
|
|
4164
|
+
process.exit(result.exitCode);
|
|
4165
|
+
}
|
|
3384
4166
|
var log = createLogger();
|
|
3385
4167
|
installProcessGuards(log);
|
|
3386
4168
|
var rawVaultRoot = process.env.SEEKSTONE_VAULT;
|
|
@@ -3396,6 +4178,20 @@ log.info("building index", { vault: vaultRoot });
|
|
|
3396
4178
|
var { index, notes, backlinks, buildMs } = await buildIndex(vaultRoot);
|
|
3397
4179
|
log.info("index ready", { notes: notes.size, buildMs });
|
|
3398
4180
|
var ctx = { vaultRoot, index, notes, backlinks, policy };
|
|
4181
|
+
var semanticCfg = resolveSemanticConfig(process.env, homedir());
|
|
4182
|
+
if (semanticCfg) {
|
|
4183
|
+
try {
|
|
4184
|
+
ctx.semantic = await Semantic.start(ctx, semanticCfg, { log });
|
|
4185
|
+
log.info("semantic search enabled", {
|
|
4186
|
+
model: ctx.semantic.embedder.id,
|
|
4187
|
+
dim: ctx.semantic.embedder.dim,
|
|
4188
|
+
notes: notes.size
|
|
4189
|
+
});
|
|
4190
|
+
} catch (err) {
|
|
4191
|
+
log.error(err instanceof Error ? err.message : String(err));
|
|
4192
|
+
process.exit(1);
|
|
4193
|
+
}
|
|
4194
|
+
}
|
|
3399
4195
|
var watcher = startWatcher(ctx, log);
|
|
3400
4196
|
process.on("exit", () => void watcher.stop());
|
|
3401
4197
|
var server = new Server({ name: "seekstone", version: VERSION }, { capabilities: { tools: {} } });
|