seekstone 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/index.js +186 -74
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
Seekstone is an Obsidian MCP server that gives Claude (and any [Model Context Protocol](https://modelcontextprotocol.io) client) direct read and write access to your Obsidian vault. No Obsidian app needs to be running, no plugins are required, and nothing leaves your machine.
|
|
6
6
|
|
|
7
|
-
It reads your vault **directly from disk** instead of routing through the Obsidian Local REST API plugin, and holds a warm full-text index in-process. The practical difference: keyword searches return in **single-digit milliseconds** (semantic search in **~
|
|
7
|
+
It reads your vault **directly from disk** instead of routing through the Obsidian Local REST API plugin, and holds a warm full-text index in-process. The practical difference: keyword searches return in **single-digit milliseconds** (semantic search in **~26 ms**) with **~2 KB payloads** that stay flat as your vault grows — a broad query that costs tens of megabytes of context through a REST-proxy server (up to **95 MB** at 10k notes) costs ~2 KB through Seekstone, up to a **~47,000× reduction**. Benchmarked against 7 other Obsidian MCP servers across committed 1k/5k/10k-note vaults — [fully reproducible](https://github.com/shaqmughal/seekstone/tree/main/packages/harness), full results at [seekstone.dev/benchmarks](https://seekstone.dev/benchmarks).
|
|
8
8
|
|
|
9
9
|
(Previously also published as `obsidian-mcp-seekstone` — that alias is deprecated; existing installs keep working, but install `seekstone` going forward.)
|
|
10
10
|
|
|
@@ -24,7 +24,7 @@ If you use Claude Code, Cursor, or another coding agent, paste this prompt and t
|
|
|
24
24
|
|
|
25
25
|
### Option 1 — One-click (Claude Desktop, no terminal needed)
|
|
26
26
|
|
|
27
|
-
Download `seekstone.mcpb`
|
|
27
|
+
Download [`seekstone.mcpb`](https://github.com/shaqmughal/seekstone/releases/latest/download/seekstone.mcpb), double-click it in Claude Desktop, and pick your Obsidian vault folder when prompted. No JSON editing, no terminal, no Node.js setup required. Want semantic search out of the box? Grab [`seekstone-semantic.mcpb`](https://github.com/shaqmughal/seekstone/releases/latest/download/seekstone-semantic.mcpb) instead — the local embedding model ships inside the bundle, and nothing is downloaded at runtime.
|
|
28
28
|
|
|
29
29
|
### Option 2 — Guided setup (recommended for CLI users)
|
|
30
30
|
|
|
@@ -112,7 +112,7 @@ Other MCP clients (Windsurf, Cline, …) take the Option 3 JSON block in their o
|
|
|
112
112
|
|---|---|
|
|
113
113
|
| `search` | Full-text search. Returns ranked excerpts (default ~120 chars, tunable via `excerptLength`), not full notes. Fuzzy and prefix matching; with `SEEKSTONE_SEMANTIC=1`, `mode: "semantic"`/`"hybrid"` searches by meaning via a local embedding model (nothing leaves your machine). |
|
|
114
114
|
| `query_notes` | Structured metadata query. Filter by frontmatter key/value predicates (`eq`, `ne`, `contains`, `exists`, `missing`, `gt`/`gte`/`lt`/`lte`), tag, folder, modified time, and size; sort and select the fields you need. Returns compact rows, not note content. |
|
|
115
|
-
| `context_pack` | Answer-ready context for a natural-language question in one call, hard-capped at a byte budget (default 2 KB): ranked excerpts, linked neighbor notes with one-line summaries, and follow-up source paths — replaces a search → read → get_backlinks round-trip loop. |
|
|
115
|
+
| `context_pack` | Answer-ready context for a natural-language question in one call, hard-capped at a byte budget (default 2 KB): ranked excerpts, linked neighbor notes with one-line summaries, and follow-up source paths — replaces a search → read → get_backlinks round-trip loop. Scopable by `folder`/`tag` and takes the same `mode` as `search`. |
|
|
116
116
|
| `read_note` | Read the full content of a note by vault-relative path. Supports returning a single section, block, or line range. |
|
|
117
117
|
| `list_notes` | List notes, optionally filtered by folder prefix or tag. |
|
|
118
118
|
| `list_tags` | List all tags in the vault sorted by usage count (or alphabetically). |
|
|
@@ -158,10 +158,11 @@ Every write tool (`append_note`, `patch_note`, `patch_frontmatter`, `replace_in_
|
|
|
158
158
|
| `SEEKSTONE_HISTORY_MAX_ENTRIES` | no | Cap on journal entries (default `1000`); the oldest are dropped past it. |
|
|
159
159
|
| `SEEKSTONE_AUDIT_FILE` | no | Absolute path; off unless set. Appends one JSON-line audit record per write-tool call — ok or refused — with the tool, paths, sha-256 before/after, outcome, and op metadata. Never note content. |
|
|
160
160
|
| `SEEKSTONE_AUDIT_MAX_SIZE` | no | Rotate the audit file to `<file>.1` past this size (e.g. `10mb`; default 10 MB). |
|
|
161
|
-
| `SEEKSTONE_SEMANTIC` | no | Set to `1` to enable semantic search (`search`
|
|
161
|
+
| `SEEKSTONE_SEMANTIC` | no | Set to `1` to enable semantic search (`search` and `context_pack` gain `mode: "semantic"` and `"hybrid"`). Download the local model once with `npx -y seekstone fetch-model`; the running server never touches the network. The `seekstone-semantic.mcpb` bundle sets this automatically and ships the model inside. |
|
|
162
162
|
| `SEEKSTONE_SEMANTIC_MODEL` | no | `potion-base-8M` (default, ~30 MB) or `potion-retrieval-32M` (~129 MB, more accurate, ~2× query latency). Fetch it first: `npx -y seekstone fetch-model --model potion-retrieval-32M`. |
|
|
163
163
|
| `SEEKSTONE_MODEL_PATH` | no | Directory holding the Model2Vec embedding model (default: where `fetch-model` puts the selected model). |
|
|
164
164
|
| `SEEKSTONE_CACHE_DIR` | no | Cache root for the model and per-vault embedding caches (default `~/.cache/seekstone`). |
|
|
165
|
+
| `SEEKSTONE_BUNDLED_MODEL_DIR` | no | Set by the `seekstone-semantic.mcpb` bundle's manifest — sharded model files shipped inside the extension, reassembled into the model directory at boot (disk-only, hash-verified). Not usually set by hand. |
|
|
165
166
|
|
|
166
167
|
---
|
|
167
168
|
|
|
@@ -208,7 +209,7 @@ Any MCP-over-stdio client: Claude Desktop, Claude Code, Cursor, VS Code, Windsur
|
|
|
208
209
|
Yes — tested on macOS, Linux, and Windows in CI on every commit.
|
|
209
210
|
|
|
210
211
|
**Is it safe?**
|
|
211
|
-
No network calls while running, no telemetry (the optional semantic-search model is downloaded once by the explicit `fetch-model` subcommand, SHA-256-verified, before any serving starts). The vault path is sandboxed — no tool reads or writes outside it. Writes are covered by the tested Write-Safety Contract above, and `SEEKSTONE_READ_ONLY=1` removes the write tools entirely.
|
|
212
|
+
No network calls while running, no telemetry (the optional semantic-search model is downloaded once by the explicit `fetch-model` subcommand, SHA-256-verified, before any serving starts — or shipped inside the `seekstone-semantic.mcpb` bundle and reassembled from disk at boot, verified against the same pinned hashes). The vault path is sandboxed — no tool reads or writes outside it. Writes are covered by the tested Write-Safety Contract above, and `SEEKSTONE_READ_ONLY=1` removes the write tools entirely.
|
|
212
213
|
|
|
213
214
|
---
|
|
214
215
|
|
package/dist/index.js
CHANGED
|
@@ -71,9 +71,9 @@ var init_model_manifest = __esm({
|
|
|
71
71
|
});
|
|
72
72
|
|
|
73
73
|
// src/semantic/config.ts
|
|
74
|
-
import { join as
|
|
74
|
+
import { join as join9 } from "path";
|
|
75
75
|
function defaultCacheDir(env, homedir2) {
|
|
76
|
-
return env.SEEKSTONE_CACHE_DIR ||
|
|
76
|
+
return env.SEEKSTONE_CACHE_DIR || join9(homedir2, ".cache", "seekstone");
|
|
77
77
|
}
|
|
78
78
|
function resolveModel(raw) {
|
|
79
79
|
const id = raw?.trim();
|
|
@@ -96,7 +96,7 @@ function resolveSemanticConfig(env, homedir2) {
|
|
|
96
96
|
if (env.SEEKSTONE_SEMANTIC !== "1") return void 0;
|
|
97
97
|
const modelId = resolveModelId(env.SEEKSTONE_SEMANTIC_MODEL);
|
|
98
98
|
const cacheDir = defaultCacheDir(env, homedir2);
|
|
99
|
-
const modelDir = env.SEEKSTONE_MODEL_PATH ||
|
|
99
|
+
const modelDir = env.SEEKSTONE_MODEL_PATH || join9(cacheDir, "models", modelId);
|
|
100
100
|
return { modelId, modelDir, cacheDir };
|
|
101
101
|
}
|
|
102
102
|
var DEFAULT_MODEL_ID;
|
|
@@ -114,10 +114,10 @@ __export(fetch_model_exports, {
|
|
|
114
114
|
runFetchModel: () => runFetchModel,
|
|
115
115
|
selectManifest: () => selectManifest
|
|
116
116
|
});
|
|
117
|
-
import { createHash as
|
|
118
|
-
import { existsSync } from "fs";
|
|
119
|
-
import { mkdir as
|
|
120
|
-
import { join as
|
|
117
|
+
import { createHash as createHash4 } from "crypto";
|
|
118
|
+
import { existsSync as existsSync2 } from "fs";
|
|
119
|
+
import { mkdir as mkdir10, readFile as readFile20, writeFile as writeFile5 } from "fs/promises";
|
|
120
|
+
import { join as join13 } from "path";
|
|
121
121
|
function selectManifest(argv, env) {
|
|
122
122
|
let requested;
|
|
123
123
|
for (let i = 0; i < argv.length; i++) {
|
|
@@ -146,15 +146,15 @@ async function runFetchModel(deps) {
|
|
|
146
146
|
return { output, exitCode: 1 };
|
|
147
147
|
}
|
|
148
148
|
const fetchFn = deps.fetchFn ?? fetch;
|
|
149
|
-
const destDir = deps.env.SEEKSTONE_MODEL_PATH ||
|
|
149
|
+
const destDir = deps.env.SEEKSTONE_MODEL_PATH || join13(defaultCacheDir(deps.env, deps.homedir), "models", manifest.id);
|
|
150
150
|
output.push(`fetch-model: ${manifest.id} \u2192 ${destDir}`);
|
|
151
151
|
try {
|
|
152
|
-
await
|
|
152
|
+
await mkdir10(destDir, { recursive: true });
|
|
153
153
|
let fetched = 0;
|
|
154
154
|
let skipped = 0;
|
|
155
155
|
for (const file of manifest.files) {
|
|
156
|
-
const dest =
|
|
157
|
-
if (
|
|
156
|
+
const dest = join13(destDir, file.name);
|
|
157
|
+
if (existsSync2(dest) && sha2562(await readFile20(dest)) === file.sha256) {
|
|
158
158
|
skipped++;
|
|
159
159
|
continue;
|
|
160
160
|
}
|
|
@@ -162,11 +162,11 @@ async function runFetchModel(deps) {
|
|
|
162
162
|
const res = await fetchFn(file.url);
|
|
163
163
|
if (!res.ok) throw new Error(`fetch ${file.url} failed: ${res.status}`);
|
|
164
164
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
165
|
-
const got =
|
|
165
|
+
const got = sha2562(buf);
|
|
166
166
|
if (got !== file.sha256) {
|
|
167
167
|
throw new Error(`checksum mismatch for ${file.name}: expected ${file.sha256}, got ${got}`);
|
|
168
168
|
}
|
|
169
|
-
await
|
|
169
|
+
await writeFile5(dest, buf);
|
|
170
170
|
fetched++;
|
|
171
171
|
}
|
|
172
172
|
output.push(`\u2713 ${manifest.id}: ${fetched} fetched, ${skipped} already present (verified).`);
|
|
@@ -179,8 +179,8 @@ async function runFetchModel(deps) {
|
|
|
179
179
|
return { output, exitCode: 1 };
|
|
180
180
|
}
|
|
181
181
|
}
|
|
182
|
-
function
|
|
183
|
-
return
|
|
182
|
+
function sha2562(buf) {
|
|
183
|
+
return createHash4("sha256").update(buf).digest("hex");
|
|
184
184
|
}
|
|
185
185
|
var init_fetch_model = __esm({
|
|
186
186
|
"src/semantic/fetch-model.ts"() {
|
|
@@ -271,7 +271,7 @@ function createLogger(opts = {}) {
|
|
|
271
271
|
);
|
|
272
272
|
}
|
|
273
273
|
}
|
|
274
|
-
function
|
|
274
|
+
function writeFile6(record) {
|
|
275
275
|
if (!fileOk || filePath === void 0) return;
|
|
276
276
|
const line = `${JSON.stringify(record)}
|
|
277
277
|
`;
|
|
@@ -293,7 +293,7 @@ function createLogger(opts = {}) {
|
|
|
293
293
|
const f = fields ?? {};
|
|
294
294
|
stderr(`${lvl.toUpperCase().padEnd(5)} ${ts} ${msg}${fmtFields(f)}
|
|
295
295
|
`);
|
|
296
|
-
|
|
296
|
+
writeFile6({ t: ts, level: lvl, msg, ...f });
|
|
297
297
|
}
|
|
298
298
|
return {
|
|
299
299
|
level,
|
|
@@ -1081,22 +1081,25 @@ function basenameNoExt(path2) {
|
|
|
1081
1081
|
const dot = base.lastIndexOf(".");
|
|
1082
1082
|
return dot > 0 ? base.slice(0, dot) : base;
|
|
1083
1083
|
}
|
|
1084
|
-
function
|
|
1084
|
+
function retrieve(ctx2, input) {
|
|
1085
1085
|
const mode = input.mode ?? "lexical";
|
|
1086
1086
|
if (mode === "lexical") return lexicalSearch(ctx2, input);
|
|
1087
1087
|
const semantic = requireSemantic(ctx2);
|
|
1088
1088
|
if (mode === "semantic") return semanticSearch(ctx2, semantic, input);
|
|
1089
1089
|
const words = queryWords(input.query);
|
|
1090
1090
|
if (words.length > 0 && words.length <= MAX_ROUTE_WORDS) {
|
|
1091
|
-
const
|
|
1091
|
+
const lex = lexicalSearch(ctx2, input);
|
|
1092
1092
|
if (routeToLexical(
|
|
1093
1093
|
input.query,
|
|
1094
|
-
|
|
1094
|
+
lex.hits.map((h) => h.path)
|
|
1095
1095
|
))
|
|
1096
|
-
return
|
|
1096
|
+
return lex;
|
|
1097
1097
|
}
|
|
1098
1098
|
return semanticSearch(ctx2, semantic, input);
|
|
1099
1099
|
}
|
|
1100
|
+
function search(ctx2, input) {
|
|
1101
|
+
return retrieve(ctx2, input).hits;
|
|
1102
|
+
}
|
|
1100
1103
|
function requireSemantic(ctx2) {
|
|
1101
1104
|
if (!ctx2.semantic) {
|
|
1102
1105
|
throw new Error(
|
|
@@ -1143,7 +1146,7 @@ function lexicalSearch(ctx2, input) {
|
|
|
1143
1146
|
hits.push(hit);
|
|
1144
1147
|
if (hits.length >= input.limit) break;
|
|
1145
1148
|
}
|
|
1146
|
-
return hits;
|
|
1149
|
+
return { hits, totalCandidates: results.length };
|
|
1147
1150
|
}
|
|
1148
1151
|
function semanticSearch(ctx2, semantic, input) {
|
|
1149
1152
|
const queryVec = semantic.embedQuery(input.query);
|
|
@@ -1166,7 +1169,7 @@ function semanticSearch(ctx2, semantic, input) {
|
|
|
1166
1169
|
hits.push(hit);
|
|
1167
1170
|
if (hits.length >= input.limit) break;
|
|
1168
1171
|
}
|
|
1169
|
-
return hits;
|
|
1172
|
+
return { hits, totalCandidates: candidates.length };
|
|
1170
1173
|
}
|
|
1171
1174
|
function filterNote(ctx2, input, path2) {
|
|
1172
1175
|
if (input.folder && !path2.startsWith(input.folder)) return void 0;
|
|
@@ -1189,7 +1192,12 @@ function decorate(hit, title, tags) {
|
|
|
1189
1192
|
// src/tools/context_pack.ts
|
|
1190
1193
|
var ContextPackInput = z3.object({
|
|
1191
1194
|
query: z3.string().min(1).describe("Natural-language question or topic to assemble context for."),
|
|
1192
|
-
budgetBytes: z3.number().int().min(256).max(16384).default(2048).describe("Hard cap on the response JSON size in bytes. The pack never exceeds it.")
|
|
1195
|
+
budgetBytes: z3.number().int().min(256).max(16384).default(2048).describe("Hard cap on the response JSON size in bytes. The pack never exceeds it."),
|
|
1196
|
+
mode: z3.enum(["lexical", "semantic", "hybrid"]).optional().describe(
|
|
1197
|
+
"lexical = keyword search (default). semantic = meaning-based search over local embeddings (requires SEEKSTONE_SEMANTIC=1 and a fetched model). hybrid = exact-title lookups go lexical, everything else semantic. A question asked in words the note does not use needs semantic or hybrid."
|
|
1198
|
+
),
|
|
1199
|
+
folder: z3.string().optional().describe("Restrict results to notes under this vault-relative folder prefix."),
|
|
1200
|
+
tag: z3.string().optional().describe("Restrict results to notes containing this tag.")
|
|
1193
1201
|
});
|
|
1194
1202
|
var MAX_CANDIDATES = 10;
|
|
1195
1203
|
var NEIGHBOR_HITS = 3;
|
|
@@ -1220,18 +1228,22 @@ function clamp(n, lo, hi) {
|
|
|
1220
1228
|
return Math.min(hi, Math.max(lo, n));
|
|
1221
1229
|
}
|
|
1222
1230
|
function contextPack(ctx2, input) {
|
|
1223
|
-
const
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1231
|
+
const excerptLen = clamp(Math.floor(input.budgetBytes / 10), 80, 400);
|
|
1232
|
+
const { hits: results, totalCandidates } = retrieve(ctx2, {
|
|
1233
|
+
query: input.query,
|
|
1234
|
+
mode: input.mode,
|
|
1235
|
+
folder: input.folder,
|
|
1236
|
+
tag: input.tag,
|
|
1237
|
+
limit: MAX_CANDIDATES,
|
|
1238
|
+
excerptLength: excerptLen
|
|
1227
1239
|
});
|
|
1228
|
-
const totalMatches =
|
|
1229
|
-
if (
|
|
1230
|
-
return { excerpts: [], neighborhood: [], sources: [], totalMatches
|
|
1240
|
+
const totalMatches = totalCandidates;
|
|
1241
|
+
if (results.length === 0) {
|
|
1242
|
+
return { excerpts: [], neighborhood: [], sources: [], totalMatches, confidence: "none" };
|
|
1231
1243
|
}
|
|
1232
1244
|
const terms = input.query.toLowerCase().split(/\s+/).filter((t) => t.length > 1);
|
|
1233
1245
|
const scoreByPath = /* @__PURE__ */ new Map();
|
|
1234
|
-
for (const r of results) scoreByPath.set(r.
|
|
1246
|
+
for (const r of results) scoreByPath.set(r.path, r.score);
|
|
1235
1247
|
const budget = input.budgetBytes;
|
|
1236
1248
|
let used = Buffer.byteLength(
|
|
1237
1249
|
JSON.stringify({
|
|
@@ -1244,24 +1256,20 @@ function contextPack(ctx2, input) {
|
|
|
1244
1256
|
}),
|
|
1245
1257
|
"utf8"
|
|
1246
1258
|
);
|
|
1247
|
-
const excerptLen = clamp(Math.floor(budget / 10), 80, 400);
|
|
1248
1259
|
const reserve = Math.min(Math.floor(budget / 4), 600);
|
|
1249
1260
|
const candidates = results.slice(0, MAX_CANDIDATES);
|
|
1250
1261
|
const excerpts = [];
|
|
1251
1262
|
const overflowHits = [];
|
|
1252
1263
|
let droppedForBudget = false;
|
|
1253
|
-
const buildEntry = (
|
|
1254
|
-
const note = ctx2.notes.get(
|
|
1255
|
-
const body = note?.body ?? "";
|
|
1256
|
-
const noteTags = note?.tags ? note.tags.split(" ").filter(Boolean) : [];
|
|
1264
|
+
const buildEntry = (h, len) => {
|
|
1265
|
+
const note = ctx2.notes.get(h.path);
|
|
1257
1266
|
const entry = {
|
|
1258
|
-
path:
|
|
1259
|
-
score:
|
|
1260
|
-
excerpt: extractExcerpt(
|
|
1267
|
+
path: h.path,
|
|
1268
|
+
score: h.score,
|
|
1269
|
+
excerpt: h.excerpt.length > len ? extractExcerpt(h.excerpt, terms, len) : h.excerpt
|
|
1261
1270
|
};
|
|
1262
|
-
|
|
1263
|
-
if (
|
|
1264
|
-
if (noteTags.length > 0) entry.tags = noteTags;
|
|
1271
|
+
if (h.title) entry.title = h.title;
|
|
1272
|
+
if (h.tags && h.tags.length > 0) entry.tags = h.tags;
|
|
1265
1273
|
const fm = pickFm(note?.fm ?? null);
|
|
1266
1274
|
if (fm) entry.fm = fm;
|
|
1267
1275
|
return entry;
|
|
@@ -1270,7 +1278,7 @@ function contextPack(ctx2, input) {
|
|
|
1270
1278
|
let phaseOpen = true;
|
|
1271
1279
|
for (const r of candidates) {
|
|
1272
1280
|
if (!phaseOpen) {
|
|
1273
|
-
overflowHits.push({ path: r.
|
|
1281
|
+
overflowHits.push({ path: r.path });
|
|
1274
1282
|
continue;
|
|
1275
1283
|
}
|
|
1276
1284
|
let placed = false;
|
|
@@ -1295,7 +1303,7 @@ function contextPack(ctx2, input) {
|
|
|
1295
1303
|
if (!placed) {
|
|
1296
1304
|
phaseOpen = false;
|
|
1297
1305
|
droppedForBudget = true;
|
|
1298
|
-
overflowHits.push({ path: r.
|
|
1306
|
+
overflowHits.push({ path: r.path });
|
|
1299
1307
|
}
|
|
1300
1308
|
}
|
|
1301
1309
|
const includedPaths = new Set(excerpts.map((e) => e.path));
|
|
@@ -3794,6 +3802,82 @@ function installProcessGuards(log2) {
|
|
|
3794
3802
|
};
|
|
3795
3803
|
}
|
|
3796
3804
|
|
|
3805
|
+
// src/semantic/bundled-model.ts
|
|
3806
|
+
init_model_manifest();
|
|
3807
|
+
import { createHash as createHash2 } from "crypto";
|
|
3808
|
+
import { existsSync } from "fs";
|
|
3809
|
+
import { mkdir as mkdir8, readdir as readdir3, readFile as readFile16, rename as rename5, writeFile as writeFile3 } from "fs/promises";
|
|
3810
|
+
import { join as join8 } from "path";
|
|
3811
|
+
async function materializeBundledModel(cfg, bundledDir, deps = {}) {
|
|
3812
|
+
if (!bundledDir) return;
|
|
3813
|
+
if (!existsSync(bundledDir)) {
|
|
3814
|
+
deps.log?.warn("bundled model dir missing \u2014 skipping materialization", { dir: bundledDir });
|
|
3815
|
+
return;
|
|
3816
|
+
}
|
|
3817
|
+
let bundledId;
|
|
3818
|
+
try {
|
|
3819
|
+
const raw = await readFile16(join8(bundledDir, "manifest.json"), "utf8");
|
|
3820
|
+
bundledId = String(JSON.parse(raw).id);
|
|
3821
|
+
} catch (err) {
|
|
3822
|
+
throw new Error(
|
|
3823
|
+
`bundled model: cannot read ${join8(bundledDir, "manifest.json")} \u2014 the extension bundle is corrupt; reinstall it (${err instanceof Error ? err.message : String(err)})`
|
|
3824
|
+
);
|
|
3825
|
+
}
|
|
3826
|
+
if (bundledId !== cfg.modelId) {
|
|
3827
|
+
deps.log?.info("bundled model does not match the requested model \u2014 skipping", {
|
|
3828
|
+
bundled: bundledId,
|
|
3829
|
+
requested: cfg.modelId
|
|
3830
|
+
});
|
|
3831
|
+
return;
|
|
3832
|
+
}
|
|
3833
|
+
const manifest = (deps.manifestFor ?? findModel)(cfg.modelId);
|
|
3834
|
+
if (!manifest) return;
|
|
3835
|
+
await mkdir8(cfg.modelDir, { recursive: true });
|
|
3836
|
+
let written = 0;
|
|
3837
|
+
for (const file of manifest.files) {
|
|
3838
|
+
const dest = join8(cfg.modelDir, file.name);
|
|
3839
|
+
if (existsSync(dest) && sha256(await readFile16(dest)) === file.sha256) continue;
|
|
3840
|
+
const buf = await concatParts(bundledDir, file.name);
|
|
3841
|
+
const got = sha256(buf);
|
|
3842
|
+
if (got !== file.sha256) {
|
|
3843
|
+
throw new Error(
|
|
3844
|
+
`bundled model: checksum mismatch for ${file.name} (expected ${file.sha256}, got ${got}) \u2014 the extension bundle is corrupt; reinstall it`
|
|
3845
|
+
);
|
|
3846
|
+
}
|
|
3847
|
+
const tmp = `${dest}.tmp-${process.pid}`;
|
|
3848
|
+
await writeFile3(tmp, buf);
|
|
3849
|
+
await rename5(tmp, dest);
|
|
3850
|
+
written++;
|
|
3851
|
+
}
|
|
3852
|
+
if (written > 0) {
|
|
3853
|
+
deps.log?.info("bundled model materialized", {
|
|
3854
|
+
model: manifest.id,
|
|
3855
|
+
dir: cfg.modelDir,
|
|
3856
|
+
files: written
|
|
3857
|
+
});
|
|
3858
|
+
}
|
|
3859
|
+
}
|
|
3860
|
+
async function concatParts(bundledDir, name) {
|
|
3861
|
+
const prefix = `${name}.`;
|
|
3862
|
+
const partNum = (f) => Number(f.slice(prefix.length, -".part".length));
|
|
3863
|
+
const parts = (await readdir3(bundledDir)).filter((f) => {
|
|
3864
|
+
if (!f.startsWith(prefix) || !f.endsWith(".part")) return false;
|
|
3865
|
+
const middle = f.slice(prefix.length, -".part".length);
|
|
3866
|
+
return /^\d+$/.test(middle);
|
|
3867
|
+
}).sort((a, b) => partNum(a) - partNum(b));
|
|
3868
|
+
if (parts.length === 0) {
|
|
3869
|
+
throw new Error(
|
|
3870
|
+
`bundled model: no shards for ${name} in ${bundledDir} \u2014 the extension bundle is corrupt; reinstall it`
|
|
3871
|
+
);
|
|
3872
|
+
}
|
|
3873
|
+
const buffers = [];
|
|
3874
|
+
for (const part of parts) buffers.push(await readFile16(join8(bundledDir, part)));
|
|
3875
|
+
return Buffer.concat(buffers);
|
|
3876
|
+
}
|
|
3877
|
+
function sha256(buf) {
|
|
3878
|
+
return createHash2("sha256").update(buf).digest("hex");
|
|
3879
|
+
}
|
|
3880
|
+
|
|
3797
3881
|
// src/index.ts
|
|
3798
3882
|
init_config();
|
|
3799
3883
|
|
|
@@ -3945,8 +4029,8 @@ function candidateSetIdf(queryIds, docs) {
|
|
|
3945
4029
|
}
|
|
3946
4030
|
|
|
3947
4031
|
// ../core/src/embed/model2vec.ts
|
|
3948
|
-
import { readFile as
|
|
3949
|
-
import { basename as basename4, join as
|
|
4032
|
+
import { readFile as readFile17 } from "fs/promises";
|
|
4033
|
+
import { basename as basename4, join as join10 } from "path";
|
|
3950
4034
|
|
|
3951
4035
|
// ../core/src/embed/safetensors.ts
|
|
3952
4036
|
function readSafetensors(buf) {
|
|
@@ -4127,9 +4211,9 @@ function isCjk(cp) {
|
|
|
4127
4211
|
var TENSOR_NAME = "embeddings";
|
|
4128
4212
|
async function loadModel2Vec(modelDir) {
|
|
4129
4213
|
const [configRaw, tokenizerRaw, weights] = await Promise.all([
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4214
|
+
readFile17(join10(modelDir, "config.json"), "utf8"),
|
|
4215
|
+
readFile17(join10(modelDir, "tokenizer.json"), "utf8"),
|
|
4216
|
+
readFile17(join10(modelDir, "model.safetensors"))
|
|
4133
4217
|
]);
|
|
4134
4218
|
const config = JSON.parse(configRaw);
|
|
4135
4219
|
const tokenizer = loadWordPieceTokenizer(JSON.parse(tokenizerRaw));
|
|
@@ -4287,26 +4371,27 @@ function isTokenEmbedder(e) {
|
|
|
4287
4371
|
}
|
|
4288
4372
|
|
|
4289
4373
|
// src/semantic/cache.ts
|
|
4290
|
-
import { createHash as
|
|
4291
|
-
import { mkdir as
|
|
4292
|
-
import { join as
|
|
4293
|
-
var CACHE_VERSION =
|
|
4374
|
+
import { createHash as createHash3 } from "crypto";
|
|
4375
|
+
import { mkdir as mkdir9, readFile as readFile18, rename as rename6, writeFile as writeFile4 } from "fs/promises";
|
|
4376
|
+
import { join as join11 } from "path";
|
|
4377
|
+
var CACHE_VERSION = 3;
|
|
4294
4378
|
function cachePathsFor(cacheDir, vaultRoot2, modelId) {
|
|
4295
|
-
const vaultKey =
|
|
4296
|
-
const dir =
|
|
4297
|
-
return { dir, bin:
|
|
4379
|
+
const vaultKey = createHash3("sha256").update(vaultRoot2).digest("hex").slice(0, 16);
|
|
4380
|
+
const dir = join11(cacheDir, "embeddings", vaultKey);
|
|
4381
|
+
return { dir, bin: join11(dir, `${modelId}.bin`), manifest: join11(dir, `${modelId}.json`) };
|
|
4298
4382
|
}
|
|
4299
4383
|
async function loadCache(paths, modelId, dim) {
|
|
4300
4384
|
try {
|
|
4301
|
-
const manifest = JSON.parse(await
|
|
4385
|
+
const manifest = JSON.parse(await readFile18(paths.manifest, "utf8"));
|
|
4302
4386
|
if (manifest.version !== CACHE_VERSION || manifest.modelId !== modelId || manifest.dim !== dim) {
|
|
4303
4387
|
return void 0;
|
|
4304
4388
|
}
|
|
4305
|
-
const buf = await
|
|
4389
|
+
const buf = await readFile18(paths.bin);
|
|
4306
4390
|
const totalChunks = manifest.entries.reduce((a, e) => a + e.chunks, 0);
|
|
4307
4391
|
const vectorBytes = totalChunks * dim * 4;
|
|
4308
4392
|
const spanBytes = totalChunks * 2 * 4;
|
|
4309
4393
|
if (buf.byteLength !== vectorBytes + spanBytes) return void 0;
|
|
4394
|
+
if (binHashOf(buf) !== manifest.binHash) return void 0;
|
|
4310
4395
|
const aligned = new Uint8Array(buf.byteLength);
|
|
4311
4396
|
aligned.set(buf);
|
|
4312
4397
|
const floats = new Float32Array(aligned.buffer, 0, totalChunks * dim);
|
|
@@ -4329,17 +4414,22 @@ async function loadCache(paths, modelId, dim) {
|
|
|
4329
4414
|
}
|
|
4330
4415
|
}
|
|
4331
4416
|
async function saveCache(paths, modelId, dim, notes2, hashes) {
|
|
4332
|
-
const
|
|
4333
|
-
const kept = [];
|
|
4417
|
+
const rows = [];
|
|
4334
4418
|
let totalChunks = 0;
|
|
4335
4419
|
for (const [path2, note] of notes2) {
|
|
4336
4420
|
const contentHash2 = hashes.get(path2);
|
|
4337
4421
|
if (!contentHash2) continue;
|
|
4338
4422
|
const chunks = note.packed.length / dim;
|
|
4339
|
-
|
|
4340
|
-
kept.push(note);
|
|
4423
|
+
rows.push({ path: path2, contentHash: contentHash2, chunks, note });
|
|
4341
4424
|
totalChunks += chunks;
|
|
4342
4425
|
}
|
|
4426
|
+
rows.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
4427
|
+
const entries = rows.map(({ path: path2, contentHash: contentHash2, chunks }) => ({
|
|
4428
|
+
path: path2,
|
|
4429
|
+
contentHash: contentHash2,
|
|
4430
|
+
chunks
|
|
4431
|
+
}));
|
|
4432
|
+
const kept = rows.map((r) => r.note);
|
|
4343
4433
|
const bin = new Uint8Array(totalChunks * dim * 4 + totalChunks * 2 * 4);
|
|
4344
4434
|
const floats = new Float32Array(bin.buffer, 0, totalChunks * dim);
|
|
4345
4435
|
const spans = new Uint32Array(bin.buffer, totalChunks * dim * 4, totalChunks * 2);
|
|
@@ -4351,15 +4441,29 @@ async function saveCache(paths, modelId, dim, notes2, hashes) {
|
|
|
4351
4441
|
floatOffset += note.packed.length;
|
|
4352
4442
|
spanOffset += note.spans.length;
|
|
4353
4443
|
}
|
|
4354
|
-
const manifest = {
|
|
4355
|
-
|
|
4444
|
+
const manifest = {
|
|
4445
|
+
version: CACHE_VERSION,
|
|
4446
|
+
modelId,
|
|
4447
|
+
dim,
|
|
4448
|
+
binHash: binHashOf(bin),
|
|
4449
|
+
entries
|
|
4450
|
+
};
|
|
4451
|
+
await mkdir9(paths.dir, { recursive: true });
|
|
4356
4452
|
await atomicWriteBytes(paths.bin, bin);
|
|
4357
4453
|
await atomicWriteBytes(paths.manifest, new TextEncoder().encode(JSON.stringify(manifest)));
|
|
4358
4454
|
}
|
|
4455
|
+
function binHashOf(bin) {
|
|
4456
|
+
return createHash3("sha256").update(bin).digest("hex");
|
|
4457
|
+
}
|
|
4458
|
+
var tempSeq = 0;
|
|
4459
|
+
function tempPathFor(absPath) {
|
|
4460
|
+
tempSeq += 1;
|
|
4461
|
+
return `${absPath}.${process.pid}.${tempSeq}.seekstone-cache-tmp`;
|
|
4462
|
+
}
|
|
4359
4463
|
async function atomicWriteBytes(absPath, bytes) {
|
|
4360
|
-
const tmpPath =
|
|
4361
|
-
await
|
|
4362
|
-
await
|
|
4464
|
+
const tmpPath = tempPathFor(absPath);
|
|
4465
|
+
await writeFile4(tmpPath, bytes);
|
|
4466
|
+
await rename6(tmpPath, absPath);
|
|
4363
4467
|
}
|
|
4364
4468
|
|
|
4365
4469
|
// src/semantic/state.ts
|
|
@@ -4831,7 +4935,7 @@ var ALL_TOOLS = [
|
|
|
4831
4935
|
{
|
|
4832
4936
|
name: "context_pack",
|
|
4833
4937
|
annotations: READ_ONLY_ANNOTATIONS,
|
|
4834
|
-
description: 'Assemble everything needed to ANSWER a natural-language question in one call, under a strict byte budget (default 2048): ranked excerpts, linked neighbor notes (backlinks/outlinks) with one-line summaries, and follow-up source paths. Use search to locate notes and query_notes for metadata filters; use context_pack when you want answer-ready context without multiple round-trips. Empty excerpts with confidence "none" or "low" means the vault lacks coverage \u2014 do not infer content.',
|
|
4938
|
+
description: 'Assemble everything needed to ANSWER a natural-language question in one call, under a strict byte budget (default 2048): ranked excerpts, linked neighbor notes (backlinks/outlinks) with one-line summaries, and follow-up source paths. Retrieval is lexical by default: for a question phrased in words the note itself may not use, pass mode "hybrid" or "semantic", and scope with folder/tag exactly as in search. Use search to locate notes and query_notes for metadata filters; use context_pack when you want answer-ready context without multiple round-trips. Empty excerpts with confidence "none" or "low" means the vault lacks coverage \u2014 do not infer content; with folder or tag set, "none" can also mean the filter excluded every match, which totalMatches distinguishes.',
|
|
4835
4939
|
inputSchema: {
|
|
4836
4940
|
type: "object",
|
|
4837
4941
|
properties: {
|
|
@@ -4839,7 +4943,14 @@ var ALL_TOOLS = [
|
|
|
4839
4943
|
budgetBytes: {
|
|
4840
4944
|
type: "number",
|
|
4841
4945
|
description: "Hard cap on response JSON bytes (256\u201316384, default 2048)."
|
|
4842
|
-
}
|
|
4946
|
+
},
|
|
4947
|
+
mode: {
|
|
4948
|
+
type: "string",
|
|
4949
|
+
enum: ["lexical", "semantic", "hybrid"],
|
|
4950
|
+
description: "lexical = keyword search (default). semantic = meaning-based over local embeddings (needs SEEKSTONE_SEMANTIC=1 and a fetched model). hybrid = exact-title lookups go lexical, everything else semantic."
|
|
4951
|
+
},
|
|
4952
|
+
folder: { type: "string", description: "Restrict to a vault-relative folder prefix." },
|
|
4953
|
+
tag: { type: "string", description: "Restrict to notes with this tag." }
|
|
4843
4954
|
},
|
|
4844
4955
|
required: ["query"]
|
|
4845
4956
|
}
|
|
@@ -5283,8 +5394,8 @@ function visibleTools(policy2) {
|
|
|
5283
5394
|
|
|
5284
5395
|
// src/watcher.ts
|
|
5285
5396
|
import { realpathSync } from "fs";
|
|
5286
|
-
import { readFile as
|
|
5287
|
-
import { join as
|
|
5397
|
+
import { readFile as readFile19 } from "fs/promises";
|
|
5398
|
+
import { join as join12, relative as relative2, sep as sep3 } from "path";
|
|
5288
5399
|
import chokidar from "chokidar";
|
|
5289
5400
|
var DEFAULT_POLL_INTERVAL_MS = 1e4;
|
|
5290
5401
|
function resolvePollInterval(optValue, envValue, log2) {
|
|
@@ -5315,7 +5426,7 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
5315
5426
|
const toRel = (abs) => relative2(vaultRoot2, abs).split(sep3).join("/");
|
|
5316
5427
|
async function upsert(relPath, op) {
|
|
5317
5428
|
try {
|
|
5318
|
-
const raw = await
|
|
5429
|
+
const raw = await readFile19(join12(vaultRoot2, relPath), "utf8");
|
|
5319
5430
|
removeNoteBacklinks(ctx2, relPath);
|
|
5320
5431
|
upsertDoc(ctx2, buildDoc(relPath, raw));
|
|
5321
5432
|
addNoteBacklinks(ctx2, relPath, raw);
|
|
@@ -5369,7 +5480,7 @@ function startWatcher(ctx2, log2, opts) {
|
|
|
5369
5480
|
}
|
|
5370
5481
|
|
|
5371
5482
|
// src/index.ts
|
|
5372
|
-
var VERSION = true ? "0.
|
|
5483
|
+
var VERSION = true ? "0.19.0" : "0.0.0-dev";
|
|
5373
5484
|
var intent = parseCliIntent(process.argv.slice(2));
|
|
5374
5485
|
if (intent === "version") {
|
|
5375
5486
|
process.stdout.write(`${VERSION}
|
|
@@ -5454,6 +5565,7 @@ if (auditCfg) {
|
|
|
5454
5565
|
var semanticCfg = resolveSemanticConfig(process.env, homedir());
|
|
5455
5566
|
if (semanticCfg) {
|
|
5456
5567
|
try {
|
|
5568
|
+
await materializeBundledModel(semanticCfg, process.env.SEEKSTONE_BUNDLED_MODEL_DIR, { log });
|
|
5457
5569
|
ctx.semantic = await Semantic.start(ctx, semanticCfg, { log });
|
|
5458
5570
|
log.info("semantic search enabled", {
|
|
5459
5571
|
model: ctx.semantic.embedder.id,
|