seekstone 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,6 +1,10 @@
1
1
  #!/usr/bin/env node
2
+ import {
3
+ resolveSemanticConfig
4
+ } from "./chunk-RRLYVK2X.js";
2
5
 
3
6
  // src/index.ts
7
+ import { homedir } from "os";
4
8
  import { resolve as resolve2 } from "path";
5
9
  import { Server } from "@modelcontextprotocol/sdk/server/index.js";
6
10
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
@@ -17,6 +21,7 @@ function parseCliIntent(argv) {
17
21
  }
18
22
  return "init";
19
23
  }
24
+ if (argv[0] === "fetch-model") return "fetch-model";
20
25
  for (const arg of argv) {
21
26
  if (arg === "--version" || arg === "-v") return "version";
22
27
  if (arg === "--help" || arg === "-h") return "help";
@@ -33,10 +38,11 @@ Seekstone runs as a Model Context Protocol stdio server. It is normally
33
38
  launched by an MCP client (Claude Desktop, Claude Code, Cursor, \u2026), not run by hand.
34
39
 
35
40
  Usage:
36
- seekstone Start the MCP server (reads from stdin/stdout)
37
- seekstone init Validate a vault and print/patch the client MCP config
38
- seekstone --version Print the version and exit
39
- seekstone --help Print this help and exit
41
+ seekstone Start the MCP server (reads from stdin/stdout)
42
+ seekstone init Validate a vault and print/patch the client MCP config
43
+ seekstone fetch-model Download the local embedding model for semantic search
44
+ seekstone --version Print the version and exit
45
+ seekstone --help Print this help and exit
40
46
 
41
47
  "seekstone init" options:
42
48
  ${initOptionLines}
@@ -48,6 +54,9 @@ Optional environment:
48
54
  SEEKSTONE_LOG_LEVEL error | warn | info (default) | debug
49
55
  SEEKSTONE_LOG_FILE Absolute path; append JSON-line logs here
50
56
  SEEKSTONE_WATCH_POLL Set to 1 to poll for changes (network drives, WSL)
57
+ SEEKSTONE_SEMANTIC Set to 1 to enable semantic search (needs fetch-model first)
58
+ SEEKSTONE_MODEL_PATH Override the embedding-model directory
59
+ SEEKSTONE_CACHE_DIR Override the cache root (default ~/.cache/seekstone)
51
60
 
52
61
  Add to Claude Code:
53
62
  claude mcp add seekstone --env SEEKSTONE_VAULT=/path/to/vault -- npx -y seekstone
@@ -370,8 +379,39 @@ function resolveLink(target, notes2) {
370
379
 
371
380
  // src/tools/search.ts
372
381
  import { z as z2 } from "zod";
382
+
383
+ // src/semantic/excerpt.ts
384
+ function chunkExcerpt(body, span, terms, maxLen) {
385
+ const normalized = body.includes("\r") ? body.replace(/\r\n/g, "\n") : body;
386
+ const start = Math.min(span.start, normalized.length);
387
+ const end = Math.min(Math.max(span.end, start), normalized.length);
388
+ const text = start === end ? normalized : normalized.slice(start, end);
389
+ return extractExcerpt(text, terms, maxLen);
390
+ }
391
+
392
+ // src/semantic/route.ts
393
+ var MAX_ROUTE_WORDS = 6;
394
+ function queryWords(query) {
395
+ return query.toLowerCase().split(/\W+/).filter(Boolean);
396
+ }
397
+ function routeToLexical(query, topPaths, topN = 3) {
398
+ const words = new Set(queryWords(query));
399
+ if (words.size === 0) return false;
400
+ return topPaths.slice(0, topN).some((path2) => {
401
+ const base = (path2.split(/[\\/]/).pop() ?? "").replace(/\.md$/, "").toLowerCase();
402
+ const parts = new Set(base.split(/\W+/).filter(Boolean));
403
+ return parts.size === words.size && [...parts].every((p) => words.has(p));
404
+ });
405
+ }
406
+
407
+ // src/tools/search.ts
373
408
  var SearchInput = z2.object({
374
409
  query: z2.string().min(1).describe("Search query. Supports fuzzy matching and prefix search."),
410
+ // Optional (not .default()) so the inferred type keeps existing direct
411
+ // callers compiling; search() itself defaults it to 'lexical'.
412
+ mode: z2.enum(["lexical", "semantic", "hybrid"]).optional().describe(
413
+ "lexical = keyword search (default). semantic = meaning-based search over local embeddings (requires SEEKSTONE_SEMANTIC=1 and a fetched model). hybrid = exact-title lookups go lexical, everything else semantic."
414
+ ),
375
415
  limit: z2.number().int().min(1).max(50).default(10).describe("Maximum number of results to return."),
376
416
  folder: z2.string().optional().describe("Restrict results to notes under this vault-relative folder prefix."),
377
417
  tag: z2.string().optional().describe("Restrict results to notes containing this tag."),
@@ -379,44 +419,113 @@ var SearchInput = z2.object({
379
419
  "Max characters of match context per hit (default 120). Lower trims payload; higher gives more context."
380
420
  )
381
421
  });
422
+ var SEMANTIC_CANDIDATES = 50;
382
423
  function basenameNoExt(path2) {
383
424
  const base = path2.slice(path2.lastIndexOf("/") + 1);
384
425
  const dot = base.lastIndexOf(".");
385
426
  return dot > 0 ? base.slice(0, dot) : base;
386
427
  }
387
428
  function search(ctx2, input) {
429
+ const mode = input.mode ?? "lexical";
430
+ if (mode === "lexical") return lexicalSearch(ctx2, input);
431
+ const semantic = requireSemantic(ctx2);
432
+ if (mode === "semantic") return semanticSearch(ctx2, semantic, input);
433
+ const words = queryWords(input.query);
434
+ if (words.length > 0 && words.length <= MAX_ROUTE_WORDS) {
435
+ const lexHits = lexicalSearch(ctx2, input);
436
+ if (routeToLexical(
437
+ input.query,
438
+ lexHits.map((h) => h.path)
439
+ ))
440
+ return lexHits;
441
+ }
442
+ return semanticSearch(ctx2, semantic, input);
443
+ }
444
+ function requireSemantic(ctx2) {
445
+ if (!ctx2.semantic) {
446
+ throw new Error(
447
+ JSON.stringify({
448
+ error: "semantic_unavailable",
449
+ hint: 'Semantic search is not enabled. Set SEEKSTONE_SEMANTIC=1 in the server env and run `npx -y seekstone fetch-model` once to download the local embedding model, then restart the session. Meanwhile, mode: "lexical" works.'
450
+ })
451
+ );
452
+ }
453
+ const progress = ctx2.semantic.progress;
454
+ if (progress.state === "building") {
455
+ throw new Error(
456
+ JSON.stringify({
457
+ error: "semantic_building",
458
+ done: progress.done,
459
+ total: progress.total,
460
+ hint: 'The semantic index is still embedding this vault (first run only \u2014 later runs load a cache). Retry shortly, or use mode: "lexical".'
461
+ })
462
+ );
463
+ }
464
+ return ctx2.semantic;
465
+ }
466
+ function searchTerms(query) {
467
+ return query.toLowerCase().split(/\s+/).filter((t) => t.length > 1);
468
+ }
469
+ function lexicalSearch(ctx2, input) {
388
470
  const results = ctx2.index.search(input.query, {
389
471
  boost: { title: 3, tags: 2, body: 1 },
390
472
  fuzzy: 0.2,
391
473
  prefix: true
392
474
  });
393
- const terms = input.query.toLowerCase().split(/\s+/).filter((t) => t.length > 1);
475
+ const terms = searchTerms(input.query);
394
476
  const hits = [];
395
477
  for (const r of results) {
396
- if (input.folder && !r.id.startsWith(input.folder)) continue;
397
- if (input.tag) {
398
- const note2 = ctx2.notes.get(r.id);
399
- if (!note2) continue;
400
- const noteTags2 = note2.tags.split(" ");
401
- if (!noteTags2.some((t) => t === input.tag || t === input.tag?.replace(/^#/, ""))) continue;
402
- }
403
- const note = ctx2.notes.get(r.id);
404
- const body = note?.body ?? "";
405
- const noteTags = note?.tags ? note.tags.split(" ").filter(Boolean) : [];
406
- const title = r.title;
478
+ const note = filterNote(ctx2, input, r.id);
479
+ if (!note) continue;
407
480
  const hit = {
408
481
  path: r.id,
409
482
  // Round the raw MiniSearch score: 2 decimals preserve relevance gaps without 17-digit float tax.
410
483
  score: Math.round(r.score * 100) / 100,
411
- excerpt: extractExcerpt(body, terms, input.excerptLength ?? 120)
484
+ excerpt: extractExcerpt(note.body, terms, input.excerptLength ?? 120)
485
+ };
486
+ decorate(hit, r.title, note.tags);
487
+ hits.push(hit);
488
+ if (hits.length >= input.limit) break;
489
+ }
490
+ return hits;
491
+ }
492
+ function semanticSearch(ctx2, semantic, input) {
493
+ const queryVec = semantic.embedQuery(input.query);
494
+ const candidates = semantic.store.topNotes(queryVec, SEMANTIC_CANDIDATES);
495
+ const terms = searchTerms(input.query);
496
+ const hits = [];
497
+ for (const c of candidates) {
498
+ const note = filterNote(ctx2, input, c.path);
499
+ if (!note) continue;
500
+ const hit = {
501
+ path: c.path,
502
+ // Cosine similarity ∈ [-1, 1]: 3 decimals keep the ranking legible.
503
+ score: Math.round(c.score * 1e3) / 1e3,
504
+ excerpt: chunkExcerpt(note.body, c, terms, input.excerptLength ?? 120)
412
505
  };
413
- if (title && title !== basenameNoExt(r.id)) hit.title = title;
414
- if (noteTags.length > 0) hit.tags = noteTags;
506
+ decorate(hit, note.title, note.tags);
415
507
  hits.push(hit);
416
508
  if (hits.length >= input.limit) break;
417
509
  }
418
510
  return hits;
419
511
  }
512
+ function filterNote(ctx2, input, path2) {
513
+ if (input.folder && !path2.startsWith(input.folder)) return void 0;
514
+ const note = ctx2.notes.get(path2);
515
+ if (!note) return void 0;
516
+ if (input.tag) {
517
+ const noteTags = note.tags.split(" ");
518
+ if (!noteTags.some((t) => t === input.tag || t === input.tag?.replace(/^#/, ""))) {
519
+ return void 0;
520
+ }
521
+ }
522
+ return note;
523
+ }
524
+ function decorate(hit, title, tags) {
525
+ if (title && title !== basenameNoExt(hit.path)) hit.title = title;
526
+ const noteTags = tags ? tags.split(" ").filter(Boolean) : [];
527
+ if (noteTags.length > 0) hit.tags = noteTags;
528
+ }
420
529
 
421
530
  // src/tools/context_pack.ts
422
531
  var ContextPackInput = z3.object({
@@ -2719,7 +2828,7 @@ function createLogger(opts = {}) {
2719
2828
  );
2720
2829
  }
2721
2830
  }
2722
- function writeFile3(record) {
2831
+ function writeFile4(record) {
2723
2832
  if (!fileOk || filePath === void 0) return;
2724
2833
  const line = `${JSON.stringify(record)}
2725
2834
  `;
@@ -2741,7 +2850,7 @@ function createLogger(opts = {}) {
2741
2850
  const f = fields ?? {};
2742
2851
  stderr(`${lvl.toUpperCase().padEnd(5)} ${ts} ${msg}${fmtFields(f)}
2743
2852
  `);
2744
- writeFile3({ t: ts, level: lvl, msg, ...f });
2853
+ writeFile4({ t: ts, level: lvl, msg, ...f });
2745
2854
  }
2746
2855
  return {
2747
2856
  level,
@@ -2772,6 +2881,664 @@ function installProcessGuards(log2) {
2772
2881
  };
2773
2882
  }
2774
2883
 
2884
+ // ../core/src/embed/chunk.ts
2885
+ var MIN_WINDOW_WORDS = 250;
2886
+ var MAX_WINDOW_WORDS = 400;
2887
+ function chunkNote(title, body) {
2888
+ const normalized = body.replace(/\r\n/g, "\n");
2889
+ const paragraphs = splitParagraphs(normalized).flatMap(splitOversized);
2890
+ if (paragraphs.length === 0) return [{ text: title, words: 0, start: 0, end: 0 }];
2891
+ const chunks = [];
2892
+ let window = [];
2893
+ let windowWords = 0;
2894
+ const flush = () => {
2895
+ const first = window[0];
2896
+ const last = window[window.length - 1];
2897
+ if (!first || !last) return;
2898
+ chunks.push({
2899
+ text: `${title}
2900
+
2901
+ ${window.map((p) => p.text).join("\n\n")}`,
2902
+ words: windowWords,
2903
+ start: first.start,
2904
+ end: last.end
2905
+ });
2906
+ window = [];
2907
+ windowWords = 0;
2908
+ };
2909
+ for (const para of paragraphs) {
2910
+ if (windowWords >= MIN_WINDOW_WORDS || windowWords + para.words > MAX_WINDOW_WORDS) flush();
2911
+ window.push(para);
2912
+ windowWords += para.words;
2913
+ }
2914
+ flush();
2915
+ return chunks;
2916
+ }
2917
+ function splitParagraphs(normalized) {
2918
+ const out = [];
2919
+ const push = (rawStart, raw) => {
2920
+ const text = raw.trim();
2921
+ if (text.length === 0) return;
2922
+ const start = rawStart + (raw.length - raw.trimStart().length);
2923
+ out.push({ text, words: countWords(text), start, end: start + text.length });
2924
+ };
2925
+ const sep4 = /\n[ \t]*\n+/g;
2926
+ let prev = 0;
2927
+ for (const m of normalized.matchAll(sep4)) {
2928
+ push(prev, normalized.slice(prev, m.index));
2929
+ prev = m.index + m[0].length;
2930
+ }
2931
+ push(prev, normalized.slice(prev));
2932
+ return out;
2933
+ }
2934
+ function splitOversized(para) {
2935
+ if (para.words <= MAX_WINDOW_WORDS) return [para];
2936
+ const words = [...para.text.matchAll(/\S+/g)];
2937
+ const pieces = [];
2938
+ for (let i = 0; i < words.length; i += MAX_WINDOW_WORDS) {
2939
+ const slice = words.slice(i, i + MAX_WINDOW_WORDS);
2940
+ const first = slice[0];
2941
+ const last = slice[slice.length - 1];
2942
+ if (!first || !last) continue;
2943
+ pieces.push({
2944
+ text: slice.map((w) => w[0]).join(" "),
2945
+ words: slice.length,
2946
+ start: para.start + first.index,
2947
+ end: para.start + last.index + last[0].length
2948
+ });
2949
+ }
2950
+ return pieces;
2951
+ }
2952
+ function countWords(text) {
2953
+ const m = text.match(/\S+/g);
2954
+ return m ? m.length : 0;
2955
+ }
2956
+
2957
+ // ../core/src/embed/model2vec.ts
2958
+ import { readFile as readFile14 } from "fs/promises";
2959
+ import { basename as basename4, join as join7 } from "path";
2960
+
2961
+ // ../core/src/embed/safetensors.ts
2962
+ function readSafetensors(buf) {
2963
+ if (buf.byteLength < 8) {
2964
+ throw new Error(`safetensors: buffer too small (${buf.byteLength} bytes) to hold a header`);
2965
+ }
2966
+ const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
2967
+ const headerLen = Number(view.getBigUint64(0, true));
2968
+ if (8 + headerLen > buf.byteLength) {
2969
+ throw new Error(
2970
+ `safetensors: header length ${headerLen} exceeds buffer (${buf.byteLength} bytes)`
2971
+ );
2972
+ }
2973
+ const header = JSON.parse(new TextDecoder().decode(buf.subarray(8, 8 + headerLen)));
2974
+ const dataStart = 8 + headerLen;
2975
+ const out = /* @__PURE__ */ new Map();
2976
+ const unsupported = [];
2977
+ for (const [name, meta] of Object.entries(header)) {
2978
+ if (name === "__metadata__") continue;
2979
+ if (meta.dtype !== "F32") {
2980
+ unsupported.push(`${name} (${meta.dtype})`);
2981
+ continue;
2982
+ }
2983
+ const [begin, end] = meta.data_offsets;
2984
+ const byteLen = end - begin;
2985
+ const elems = meta.shape.reduce((a, b) => a * b, 1);
2986
+ if (byteLen !== elems * 4) {
2987
+ throw new Error(
2988
+ `safetensors: tensor "${name}" data span ${byteLen} B does not match shape [${meta.shape.join(", ")}]`
2989
+ );
2990
+ }
2991
+ if (dataStart + end > buf.byteLength) {
2992
+ throw new Error(`safetensors: tensor "${name}" extends past the end of the buffer`);
2993
+ }
2994
+ const absStart = buf.byteOffset + dataStart + begin;
2995
+ let data;
2996
+ if (absStart % 4 === 0) {
2997
+ data = new Float32Array(buf.buffer, absStart, elems);
2998
+ } else {
2999
+ const copy = new Uint8Array(byteLen);
3000
+ copy.set(buf.subarray(dataStart + begin, dataStart + end));
3001
+ data = new Float32Array(copy.buffer);
3002
+ }
3003
+ out.set(name, { dtype: meta.dtype, shape: meta.shape, data });
3004
+ }
3005
+ if (unsupported.length > 0) {
3006
+ throw new Error(`safetensors: only F32 tensors are supported; found ${unsupported.join(", ")}`);
3007
+ }
3008
+ return out;
3009
+ }
3010
+
3011
+ // ../core/src/embed/tokenizer.ts
3012
+ function loadWordPieceTokenizer(tokenizerJson) {
3013
+ const tj = tokenizerJson;
3014
+ const model = tj?.model;
3015
+ if (model?.type !== "WordPiece" || !model.vocab) {
3016
+ throw new Error(
3017
+ `tokenizer: unsupported model type "${model?.type ?? "missing"}" (only WordPiece is implemented)`
3018
+ );
3019
+ }
3020
+ const vocab = new Map(Object.entries(model.vocab));
3021
+ const unkToken = model.unk_token ?? "[UNK]";
3022
+ const unkId = vocab.get(unkToken);
3023
+ if (unkId === void 0) {
3024
+ throw new Error(`tokenizer: unk_token "${unkToken}" is not in the vocab`);
3025
+ }
3026
+ const prefix = model.continuing_subword_prefix ?? "##";
3027
+ const maxChars = model.max_input_chars_per_word ?? 100;
3028
+ const norm = tj.normalizer ?? {};
3029
+ const cleanText = norm.clean_text ?? true;
3030
+ const handleCjk = norm.handle_chinese_chars ?? true;
3031
+ const lowercase = norm.lowercase ?? true;
3032
+ const stripAccents = norm.strip_accents ?? lowercase;
3033
+ const specialIds = new Set(
3034
+ (tj.added_tokens ?? []).filter((t) => t.special).map((t) => t.id)
3035
+ );
3036
+ function normalize(text) {
3037
+ let out = cleanText ? doCleanText(text) : text;
3038
+ if (handleCjk) out = padCjkChars(out);
3039
+ if (stripAccents) out = out.normalize("NFD").replace(new RegExp("\\p{Mn}", "gu"), "");
3040
+ if (lowercase) out = out.toLowerCase();
3041
+ return out;
3042
+ }
3043
+ function wordPiece(word) {
3044
+ const chars = [...word];
3045
+ if (chars.length > maxChars) return [unkId];
3046
+ const ids = [];
3047
+ let start = 0;
3048
+ while (start < chars.length) {
3049
+ let end = chars.length;
3050
+ let id;
3051
+ while (start < end) {
3052
+ const piece = (start > 0 ? prefix : "") + chars.slice(start, end).join("");
3053
+ const found = vocab.get(piece);
3054
+ if (found !== void 0) {
3055
+ id = found;
3056
+ break;
3057
+ }
3058
+ end--;
3059
+ }
3060
+ if (id === void 0) return [unkId];
3061
+ ids.push(id);
3062
+ start = end;
3063
+ }
3064
+ return ids;
3065
+ }
3066
+ return {
3067
+ vocabSize: vocab.size,
3068
+ unkId,
3069
+ specialIds,
3070
+ encode(text) {
3071
+ const ids = [];
3072
+ for (const word of preTokenize(normalize(text))) {
3073
+ for (const id of wordPiece(word)) ids.push(id);
3074
+ }
3075
+ return ids;
3076
+ }
3077
+ };
3078
+ }
3079
+ function preTokenize(text) {
3080
+ const words = [];
3081
+ let cur = "";
3082
+ for (const ch of text) {
3083
+ const cp = ch.codePointAt(0);
3084
+ if (isWhitespace(ch, cp)) {
3085
+ if (cur) {
3086
+ words.push(cur);
3087
+ cur = "";
3088
+ }
3089
+ } else if (isPunctuation(ch, cp)) {
3090
+ if (cur) {
3091
+ words.push(cur);
3092
+ cur = "";
3093
+ }
3094
+ words.push(ch);
3095
+ } else {
3096
+ cur += ch;
3097
+ }
3098
+ }
3099
+ if (cur) words.push(cur);
3100
+ return words;
3101
+ }
3102
+ function doCleanText(text) {
3103
+ let out = "";
3104
+ for (const ch of text) {
3105
+ const cp = ch.codePointAt(0);
3106
+ if (cp === 0 || cp === 65533 || isControl(ch, cp)) continue;
3107
+ out += isWhitespace(ch, cp) ? " " : ch;
3108
+ }
3109
+ return out;
3110
+ }
3111
+ function padCjkChars(text) {
3112
+ let out = "";
3113
+ for (const ch of text) {
3114
+ out += isCjk(ch.codePointAt(0)) ? ` ${ch} ` : ch;
3115
+ }
3116
+ return out;
3117
+ }
3118
+ function isWhitespace(ch, cp) {
3119
+ if (cp === 32 || cp === 9 || cp === 10 || cp === 13) return true;
3120
+ return new RegExp("\\p{Zs}", "u").test(ch);
3121
+ }
3122
+ function isControl(ch, cp) {
3123
+ if (cp === 9 || cp === 10 || cp === 13) return false;
3124
+ return new RegExp("\\p{Cc}|\\p{Cf}", "u").test(ch);
3125
+ }
3126
+ function isPunctuation(ch, cp) {
3127
+ if (cp >= 33 && cp <= 47 || cp >= 58 && cp <= 64 || cp >= 91 && cp <= 96 || cp >= 123 && cp <= 126) {
3128
+ return true;
3129
+ }
3130
+ return new RegExp("\\p{P}", "u").test(ch);
3131
+ }
3132
+ function isCjk(cp) {
3133
+ return cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
3134
+ }
3135
+
3136
+ // ../core/src/embed/model2vec.ts
3137
+ var TENSOR_NAME = "embeddings";
3138
+ async function loadModel2Vec(modelDir) {
3139
+ const [configRaw, tokenizerRaw, weights] = await Promise.all([
3140
+ readFile14(join7(modelDir, "config.json"), "utf8"),
3141
+ readFile14(join7(modelDir, "tokenizer.json"), "utf8"),
3142
+ readFile14(join7(modelDir, "model.safetensors"))
3143
+ ]);
3144
+ const config = JSON.parse(configRaw);
3145
+ const tokenizer = loadWordPieceTokenizer(JSON.parse(tokenizerRaw));
3146
+ const tensors = readSafetensors(weights);
3147
+ const tensor = tensors.get(TENSOR_NAME) ?? (tensors.size === 1 ? [...tensors.values()][0] : void 0);
3148
+ if (!tensor) {
3149
+ throw new Error(
3150
+ `model2vec: no "${TENSOR_NAME}" tensor in ${modelDir}; found: ${[...tensors.keys()].join(", ")}`
3151
+ );
3152
+ }
3153
+ if (tensor.shape.length !== 2) {
3154
+ throw new Error(`model2vec: expected a 2-D embedding matrix, got [${tensor.shape.join(", ")}]`);
3155
+ }
3156
+ const [rows, dim] = tensor.shape;
3157
+ if (rows !== tokenizer.vocabSize) {
3158
+ throw new Error(
3159
+ `model2vec: embedding rows (${rows}) do not match tokenizer vocab size (${tokenizer.vocabSize})`
3160
+ );
3161
+ }
3162
+ if (config.hidden_dim !== void 0 && config.hidden_dim !== dim) {
3163
+ throw new Error(
3164
+ `model2vec: config hidden_dim ${config.hidden_dim} does not match tensor dim ${dim}`
3165
+ );
3166
+ }
3167
+ const table = tensor.data;
3168
+ return {
3169
+ id: basename4(modelDir),
3170
+ dim,
3171
+ embed(text) {
3172
+ const out = new Float32Array(dim);
3173
+ let pooled = 0;
3174
+ for (const id of tokenizer.encode(text)) {
3175
+ if (tokenizer.specialIds.has(id) && id !== tokenizer.unkId) continue;
3176
+ const base = id * dim;
3177
+ for (let j = 0; j < dim; j++) {
3178
+ out[j] = out[j] + table[base + j];
3179
+ }
3180
+ pooled++;
3181
+ }
3182
+ if (pooled === 0) return out;
3183
+ let sumSquares = 0;
3184
+ for (let j = 0; j < dim; j++) {
3185
+ const mean = out[j] / pooled;
3186
+ out[j] = mean;
3187
+ sumSquares += mean * mean;
3188
+ }
3189
+ const norm = Math.sqrt(sumSquares);
3190
+ if (norm > 0) {
3191
+ for (let j = 0; j < dim; j++) out[j] = out[j] / norm;
3192
+ }
3193
+ return out;
3194
+ }
3195
+ };
3196
+ }
3197
+
3198
+ // src/semantic/cache.ts
3199
+ import { createHash as createHash2 } from "crypto";
3200
+ import { mkdir as mkdir6, readFile as readFile15, rename as rename4, writeFile as writeFile3 } from "fs/promises";
3201
+ import { join as join8 } from "path";
3202
+ var CACHE_VERSION = 2;
3203
+ function cachePathsFor(cacheDir, vaultRoot2, modelId) {
3204
+ const vaultKey = createHash2("sha256").update(vaultRoot2).digest("hex").slice(0, 16);
3205
+ const dir = join8(cacheDir, "embeddings", vaultKey);
3206
+ return { dir, bin: join8(dir, `${modelId}.bin`), manifest: join8(dir, `${modelId}.json`) };
3207
+ }
3208
+ async function loadCache(paths, modelId, dim) {
3209
+ try {
3210
+ const manifest = JSON.parse(await readFile15(paths.manifest, "utf8"));
3211
+ if (manifest.version !== CACHE_VERSION || manifest.modelId !== modelId || manifest.dim !== dim) {
3212
+ return void 0;
3213
+ }
3214
+ const buf = await readFile15(paths.bin);
3215
+ const totalChunks = manifest.entries.reduce((a, e) => a + e.chunks, 0);
3216
+ const vectorBytes = totalChunks * dim * 4;
3217
+ const spanBytes = totalChunks * 2 * 4;
3218
+ if (buf.byteLength !== vectorBytes + spanBytes) return void 0;
3219
+ const aligned = new Uint8Array(buf.byteLength);
3220
+ aligned.set(buf);
3221
+ const floats = new Float32Array(aligned.buffer, 0, totalChunks * dim);
3222
+ const spans = new Uint32Array(aligned.buffer, vectorBytes, totalChunks * 2);
3223
+ const vectors = /* @__PURE__ */ new Map();
3224
+ const hashes = /* @__PURE__ */ new Map();
3225
+ let chunkOffset = 0;
3226
+ for (const e of manifest.entries) {
3227
+ if (e.chunks <= 0) return void 0;
3228
+ vectors.set(e.path, {
3229
+ packed: floats.subarray(chunkOffset * dim, (chunkOffset + e.chunks) * dim),
3230
+ spans: spans.subarray(chunkOffset * 2, (chunkOffset + e.chunks) * 2)
3231
+ });
3232
+ hashes.set(e.path, e.contentHash);
3233
+ chunkOffset += e.chunks;
3234
+ }
3235
+ return { vectors, hashes };
3236
+ } catch {
3237
+ return void 0;
3238
+ }
3239
+ }
3240
+ async function saveCache(paths, modelId, dim, notes2, hashes) {
3241
+ const entries = [];
3242
+ const kept = [];
3243
+ let totalChunks = 0;
3244
+ for (const [path2, note] of notes2) {
3245
+ const contentHash2 = hashes.get(path2);
3246
+ if (!contentHash2) continue;
3247
+ const chunks = note.packed.length / dim;
3248
+ entries.push({ path: path2, contentHash: contentHash2, chunks });
3249
+ kept.push(note);
3250
+ totalChunks += chunks;
3251
+ }
3252
+ const bin = new Uint8Array(totalChunks * dim * 4 + totalChunks * 2 * 4);
3253
+ const floats = new Float32Array(bin.buffer, 0, totalChunks * dim);
3254
+ const spans = new Uint32Array(bin.buffer, totalChunks * dim * 4, totalChunks * 2);
3255
+ let floatOffset = 0;
3256
+ let spanOffset = 0;
3257
+ for (const note of kept) {
3258
+ floats.set(note.packed, floatOffset);
3259
+ spans.set(note.spans, spanOffset);
3260
+ floatOffset += note.packed.length;
3261
+ spanOffset += note.spans.length;
3262
+ }
3263
+ const manifest = { version: CACHE_VERSION, modelId, dim, entries };
3264
+ await mkdir6(paths.dir, { recursive: true });
3265
+ await atomicWriteBytes(paths.bin, bin);
3266
+ await atomicWriteBytes(paths.manifest, new TextEncoder().encode(JSON.stringify(manifest)));
3267
+ }
3268
+ async function atomicWriteBytes(absPath, bytes) {
3269
+ const tmpPath = `${absPath}.seekstone-cache-tmp`;
3270
+ await writeFile3(tmpPath, bytes);
3271
+ await rename4(tmpPath, absPath);
3272
+ }
3273
+
3274
+ // src/semantic/store.ts
3275
+ var SemanticStore = class {
3276
+ dim;
3277
+ notes = /* @__PURE__ */ new Map();
3278
+ chunks = 0;
3279
+ constructor(dim) {
3280
+ if (!Number.isInteger(dim) || dim <= 0) {
3281
+ throw new Error(`semantic store: invalid dim ${dim}`);
3282
+ }
3283
+ this.dim = dim;
3284
+ }
3285
+ /** Total chunk vectors across all notes. */
3286
+ get chunkCount() {
3287
+ return this.chunks;
3288
+ }
3289
+ get noteCount() {
3290
+ return this.notes.size;
3291
+ }
3292
+ /** Replace (or insert) a note's packed chunk vectors and spans. */
3293
+ setNote(path2, packed, spans) {
3294
+ if (packed.length === 0 || packed.length % this.dim !== 0) {
3295
+ throw new Error(
3296
+ `semantic store: packed length ${packed.length} is not a positive multiple of dim ${this.dim} (${path2})`
3297
+ );
3298
+ }
3299
+ const n = packed.length / this.dim;
3300
+ if (spans.length !== n * 2) {
3301
+ throw new Error(
3302
+ `semantic store: spans length ${spans.length} does not match ${n} chunks (${path2})`
3303
+ );
3304
+ }
3305
+ const prev = this.notes.get(path2);
3306
+ if (prev) this.chunks -= prev.packed.length / this.dim;
3307
+ this.notes.set(path2, { packed, spans });
3308
+ this.chunks += n;
3309
+ }
3310
+ removeNote(path2) {
3311
+ const prev = this.notes.get(path2);
3312
+ if (!prev) return;
3313
+ this.chunks -= prev.packed.length / this.dim;
3314
+ this.notes.delete(path2);
3315
+ }
3316
+ getNote(path2) {
3317
+ return this.notes.get(path2);
3318
+ }
3319
+ /** Iterate (path, note vectors) — cache serialization uses this. */
3320
+ entries() {
3321
+ return this.notes.entries();
3322
+ }
3323
+ /** Top `k` notes by max-pooled chunk cosine, ties broken path-ascending. */
3324
+ topNotes(query, k) {
3325
+ if (query.length !== this.dim) {
3326
+ throw new Error(`semantic store: query dim ${query.length} does not match ${this.dim}`);
3327
+ }
3328
+ const dim = this.dim;
3329
+ const hits = [];
3330
+ for (const [path2, { packed, spans }] of this.notes) {
3331
+ let best = Number.NEGATIVE_INFINITY;
3332
+ let bestChunk = 0;
3333
+ const n = packed.length / dim;
3334
+ for (let c = 0; c < n; c++) {
3335
+ const base = c * dim;
3336
+ let score = 0;
3337
+ for (let j = 0; j < dim; j++) {
3338
+ score += packed[base + j] * query[j];
3339
+ }
3340
+ if (score > best) {
3341
+ best = score;
3342
+ bestChunk = c;
3343
+ }
3344
+ }
3345
+ hits.push({
3346
+ path: path2,
3347
+ score: best,
3348
+ chunkIndex: bestChunk,
3349
+ start: spans[bestChunk * 2],
3350
+ end: spans[bestChunk * 2 + 1]
3351
+ });
3352
+ }
3353
+ return hits.sort((a, b) => b.score - a.score || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)).slice(0, Math.max(0, k));
3354
+ }
3355
+ };
3356
+
3357
+ // src/semantic/state.ts
3358
+ var Semantic = class _Semantic {
3359
+ embedder;
3360
+ store;
3361
+ progress;
3362
+ ctx;
3363
+ paths;
3364
+ log;
3365
+ debounceMs;
3366
+ saveDebounceMs;
3367
+ yieldEvery;
3368
+ hashes = /* @__PURE__ */ new Map();
3369
+ pending = /* @__PURE__ */ new Map();
3370
+ saveTimer;
3371
+ stopped = false;
3372
+ constructor(embedder, ctx2, cfg, deps) {
3373
+ this.embedder = embedder;
3374
+ this.store = new SemanticStore(embedder.dim);
3375
+ this.ctx = ctx2;
3376
+ this.paths = cachePathsFor(cfg.cacheDir, ctx2.vaultRoot, embedder.id);
3377
+ this.log = deps.log;
3378
+ this.debounceMs = deps.debounceMs ?? 300;
3379
+ this.saveDebounceMs = deps.saveDebounceMs ?? 5e3;
3380
+ this.yieldEvery = deps.yieldEvery ?? 8;
3381
+ this.progress = { state: "building", done: 0, total: ctx2.notes.size };
3382
+ }
3383
+ /**
3384
+ * Load the model and kick off the background index build. Rejects with an
3385
+ * actionable message when the model files are missing — the user enabled
3386
+ * the feature explicitly, so a broken setup should fail loudly at boot.
3387
+ */
3388
+ static async start(ctx2, cfg, deps = {}) {
3389
+ let embedder;
3390
+ try {
3391
+ embedder = await (deps.loadModel ?? loadModel2Vec)(cfg.modelDir);
3392
+ } catch (err) {
3393
+ const reason = err instanceof Error ? err.message : String(err);
3394
+ throw new Error(
3395
+ `semantic search: could not load the embedding model from ${cfg.modelDir} \u2014 run \`npx -y seekstone fetch-model\` to download it, or point SEEKSTONE_MODEL_PATH at a Model2Vec model directory (${reason})`
3396
+ );
3397
+ }
3398
+ const semantic = new _Semantic(embedder, ctx2, cfg, deps);
3399
+ void semantic.build();
3400
+ return semantic;
3401
+ }
3402
+ /** Resolves when the initial build has finished (tests and progress checks). */
3403
+ buildDone = Promise.resolve();
3404
+ ready() {
3405
+ return this.buildDone;
3406
+ }
3407
+ build() {
3408
+ this.buildDone = this.buildInner().catch((err) => {
3409
+ this.log?.error("semantic build failed", {
3410
+ error: err instanceof Error ? err.message : String(err)
3411
+ });
3412
+ });
3413
+ return this.buildDone;
3414
+ }
3415
+ async buildInner() {
3416
+ const t0 = performance.now();
3417
+ const cached = await loadCache(this.paths, this.embedder.id, this.embedder.dim);
3418
+ const ids = [...this.ctx.notes.keys()];
3419
+ this.progress = { state: "building", done: 0, total: ids.length };
3420
+ let reused = 0;
3421
+ let sinceYield = 0;
3422
+ for (const id of ids) {
3423
+ if (this.stopped) return;
3424
+ if (!this.hashes.has(id)) {
3425
+ const note = this.ctx.notes.get(id);
3426
+ if (note) {
3427
+ const current = contentHash(note.raw);
3428
+ const cachedVecs = cached?.hashes.get(id) === current ? cached.vectors.get(id) : void 0;
3429
+ if (cachedVecs !== void 0 && cachedVecs.packed.length > 0) {
3430
+ this.store.setNote(id, cachedVecs.packed, cachedVecs.spans);
3431
+ reused++;
3432
+ } else {
3433
+ const { packed, spans } = this.embedNote(note);
3434
+ this.store.setNote(id, packed, spans);
3435
+ if (++sinceYield >= this.yieldEvery) {
3436
+ sinceYield = 0;
3437
+ await new Promise((r) => setImmediate(r));
3438
+ }
3439
+ }
3440
+ this.hashes.set(id, current);
3441
+ }
3442
+ }
3443
+ if (this.progress.state === "building") this.progress.done++;
3444
+ }
3445
+ this.progress = { state: "ready" };
3446
+ this.log?.info("semantic index ready", {
3447
+ notes: this.store.noteCount,
3448
+ chunks: this.store.chunkCount,
3449
+ reusedFromCache: reused,
3450
+ buildMs: Math.round(performance.now() - t0)
3451
+ });
3452
+ if (reused < this.store.noteCount) await this.save();
3453
+ }
3454
+ embedNote(note) {
3455
+ const chunks = chunkNote(note.title, note.body);
3456
+ const dim = this.embedder.dim;
3457
+ const packed = new Float32Array(chunks.length * dim);
3458
+ const spans = new Uint32Array(chunks.length * 2);
3459
+ for (let i = 0; i < chunks.length; i++) {
3460
+ const chunk = chunks[i];
3461
+ packed.set(this.embedder.embed(chunk.text), i * dim);
3462
+ spans[i * 2] = chunk.start;
3463
+ spans[i * 2 + 1] = chunk.end;
3464
+ }
3465
+ return { packed, spans };
3466
+ }
3467
+ embedQuery(query) {
3468
+ return this.embedder.embed(query);
3469
+ }
3470
+ /** Watcher hook: a note was added or changed (ctx.notes is already fresh). */
3471
+ noteChanged(path2) {
3472
+ if (this.stopped) return;
3473
+ const existing = this.pending.get(path2);
3474
+ if (existing) clearTimeout(existing);
3475
+ const timer = setTimeout(() => {
3476
+ this.pending.delete(path2);
3477
+ this.reembed(path2);
3478
+ }, this.debounceMs);
3479
+ timer.unref?.();
3480
+ this.pending.set(path2, timer);
3481
+ }
3482
+ /** Watcher hook: a note was deleted. */
3483
+ noteRemoved(path2) {
3484
+ const existing = this.pending.get(path2);
3485
+ if (existing) {
3486
+ clearTimeout(existing);
3487
+ this.pending.delete(path2);
3488
+ }
3489
+ if (this.hashes.has(path2) || this.store.getNote(path2)) {
3490
+ this.store.removeNote(path2);
3491
+ this.hashes.delete(path2);
3492
+ this.scheduleSave();
3493
+ }
3494
+ }
3495
+ reembed(path2) {
3496
+ const note = this.ctx.notes.get(path2);
3497
+ if (!note) return;
3498
+ const current = contentHash(note.raw);
3499
+ const prior = this.hashes.get(path2);
3500
+ if (prior === current) return;
3501
+ const { packed, spans } = this.embedNote(note);
3502
+ this.store.setNote(path2, packed, spans);
3503
+ this.hashes.set(path2, current);
3504
+ this.log?.debug("semantic re-embed", { path: path2 });
3505
+ this.scheduleSave();
3506
+ }
3507
+ scheduleSave() {
3508
+ if (this.saveTimer) clearTimeout(this.saveTimer);
3509
+ this.saveTimer = setTimeout(() => {
3510
+ this.saveTimer = void 0;
3511
+ void this.save();
3512
+ }, this.saveDebounceMs);
3513
+ this.saveTimer.unref?.();
3514
+ }
3515
+ async save() {
3516
+ if (this.stopped) return;
3517
+ try {
3518
+ await saveCache(
3519
+ this.paths,
3520
+ this.embedder.id,
3521
+ this.embedder.dim,
3522
+ this.store.entries(),
3523
+ this.hashes
3524
+ );
3525
+ this.log?.debug("semantic cache saved", { notes: this.store.noteCount });
3526
+ } catch (err) {
3527
+ this.log?.warn("semantic cache save failed", {
3528
+ error: err instanceof Error ? err.message : String(err)
3529
+ });
3530
+ }
3531
+ }
3532
+ /** Tear down timers (tests; the process otherwise runs for the session). */
3533
+ stop() {
3534
+ this.stopped = true;
3535
+ for (const timer of this.pending.values()) clearTimeout(timer);
3536
+ this.pending.clear();
3537
+ if (this.saveTimer) clearTimeout(this.saveTimer);
3538
+ this.saveTimer = void 0;
3539
+ }
3540
+ };
3541
+
2775
3542
  // src/tool-list.ts
2776
3543
  var READ_ONLY_ANNOTATIONS = {
2777
3544
  readOnlyHint: true,
@@ -2799,11 +3566,16 @@ var ALL_TOOLS = [
2799
3566
  {
2800
3567
  name: "search",
2801
3568
  annotations: READ_ONLY_ANNOTATIONS,
2802
- description: "Full-text search across the vault. Returns ranked excerpts (~200 chars) \u2014 not full notes \u2014 to minimise context usage. Supports fuzzy matching and prefix search.",
3569
+ description: 'Full-text search across the vault. Returns ranked excerpts (~200 chars) \u2014 not full notes \u2014 to minimise context usage. Supports fuzzy matching and prefix search; with SEEKSTONE_SEMANTIC=1, mode "semantic"/"hybrid" searches by meaning via a local embedding model.',
2803
3570
  inputSchema: {
2804
3571
  type: "object",
2805
3572
  properties: {
2806
3573
  query: { type: "string", description: "Search query." },
3574
+ mode: {
3575
+ type: "string",
3576
+ enum: ["lexical", "semantic", "hybrid"],
3577
+ description: "lexical = keyword search (default). semantic = meaning-based search (requires SEEKSTONE_SEMANTIC=1). hybrid = exact-title lookups go lexical, everything else semantic."
3578
+ },
2807
3579
  limit: { type: "number", description: "Max results (1\u201350, default 10)." },
2808
3580
  folder: { type: "string", description: "Restrict to a vault-relative folder prefix." },
2809
3581
  tag: { type: "string", description: "Restrict to notes with this tag." }
@@ -3288,8 +4060,8 @@ function visibleTools(policy2) {
3288
4060
 
3289
4061
  // src/watcher.ts
3290
4062
  import { realpathSync } from "fs";
3291
- import { readFile as readFile14 } from "fs/promises";
3292
- import { join as join7, relative as relative2, sep as sep3 } from "path";
4063
+ import { readFile as readFile16 } from "fs/promises";
4064
+ import { join as join9, relative as relative2, sep as sep3 } from "path";
3293
4065
  import chokidar from "chokidar";
3294
4066
  function startWatcher(ctx2, log2, opts) {
3295
4067
  const usePolling = opts?.usePolling ?? process.env.SEEKSTONE_WATCH_POLL === "1";
@@ -3301,10 +4073,11 @@ function startWatcher(ctx2, log2, opts) {
3301
4073
  const toRel = (abs) => relative2(vaultRoot2, abs).split(sep3).join("/");
3302
4074
  async function upsert(relPath, op) {
3303
4075
  try {
3304
- const raw = await readFile14(join7(vaultRoot2, relPath), "utf8");
4076
+ const raw = await readFile16(join9(vaultRoot2, relPath), "utf8");
3305
4077
  removeNoteBacklinks(ctx2, relPath);
3306
4078
  upsertDoc(ctx2, buildDoc(relPath, raw));
3307
4079
  addNoteBacklinks(ctx2, relPath, raw);
4080
+ ctx2.semantic?.noteChanged(relPath);
3308
4081
  log2?.debug("index updated", { path: relPath, op });
3309
4082
  } catch {
3310
4083
  }
@@ -3314,6 +4087,7 @@ function startWatcher(ctx2, log2, opts) {
3314
4087
  removeNoteBacklinks(ctx2, relPath);
3315
4088
  ctx2.index.discard(relPath);
3316
4089
  ctx2.notes.delete(relPath);
4090
+ ctx2.semantic?.noteRemoved(relPath);
3317
4091
  log2?.debug("index removed", { path: relPath, op: "delete" });
3318
4092
  }
3319
4093
  }
@@ -3353,7 +4127,7 @@ function startWatcher(ctx2, log2, opts) {
3353
4127
  }
3354
4128
 
3355
4129
  // src/index.ts
3356
- var VERSION = true ? "0.14.0" : "0.0.0-dev";
4130
+ var VERSION = true ? "0.15.0" : "0.0.0-dev";
3357
4131
  var intent = parseCliIntent(process.argv.slice(2));
3358
4132
  if (intent === "version") {
3359
4133
  process.stdout.write(`${VERSION}
@@ -3381,6 +4155,14 @@ if (intent === "init") {
3381
4155
  `);
3382
4156
  process.exit(result.exitCode);
3383
4157
  }
4158
+ if (intent === "fetch-model") {
4159
+ const { runFetchModel } = await import("./fetch-model-7TGOJMZL.js");
4160
+ const { homedir: homedir2 } = await import("os");
4161
+ const result = await runFetchModel({ env: process.env, homedir: homedir2() });
4162
+ process.stdout.write(`${result.output.join("\n")}
4163
+ `);
4164
+ process.exit(result.exitCode);
4165
+ }
3384
4166
  var log = createLogger();
3385
4167
  installProcessGuards(log);
3386
4168
  var rawVaultRoot = process.env.SEEKSTONE_VAULT;
@@ -3396,6 +4178,20 @@ log.info("building index", { vault: vaultRoot });
3396
4178
  var { index, notes, backlinks, buildMs } = await buildIndex(vaultRoot);
3397
4179
  log.info("index ready", { notes: notes.size, buildMs });
3398
4180
  var ctx = { vaultRoot, index, notes, backlinks, policy };
4181
+ var semanticCfg = resolveSemanticConfig(process.env, homedir());
4182
+ if (semanticCfg) {
4183
+ try {
4184
+ ctx.semantic = await Semantic.start(ctx, semanticCfg, { log });
4185
+ log.info("semantic search enabled", {
4186
+ model: ctx.semantic.embedder.id,
4187
+ dim: ctx.semantic.embedder.dim,
4188
+ notes: notes.size
4189
+ });
4190
+ } catch (err) {
4191
+ log.error(err instanceof Error ? err.message : String(err));
4192
+ process.exit(1);
4193
+ }
4194
+ }
3399
4195
  var watcher = startWatcher(ctx, log);
3400
4196
  process.on("exit", () => void watcher.stop());
3401
4197
  var server = new Server({ name: "seekstone", version: VERSION }, { capabilities: { tools: {} } });