trova-cli 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js ADDED
@@ -0,0 +1,4807 @@
1
+ import { createRequire as __trovaCreateRequire } from 'node:module'; const require = __trovaCreateRequire(import.meta.url);
2
+
3
+ // src/cli.ts
4
+ import { spawnSync } from "node:child_process";
5
+ import { existsSync as existsSync4, mkdtempSync, rmSync as rmSync4, writeFileSync as writeFileSync4 } from "node:fs";
6
+ import { tmpdir } from "node:os";
7
+ import { basename as basename2, join as join7, resolve as resolve2 } from "node:path";
8
+
9
+ // ../contract/src/types.ts
10
+ var API_VERSION = "1.0";
11
+ var ERROR_STATUS = {
12
+ bad_request: 400,
13
+ unauthorized: 401,
14
+ not_found: 404,
15
+ unsupported: 422,
16
+ cancelled: 499,
17
+ model_missing: 503,
18
+ model_loading: 503,
19
+ busy: 503,
20
+ internal: 500
21
+ };
22
+ var EngineError = class extends Error {
23
+ constructor(code, message, retryable = false, retryAfterMs) {
24
+ super(message);
25
+ this.code = code;
26
+ this.retryable = retryable;
27
+ this.retryAfterMs = retryAfterMs;
28
+ this.name = "EngineError";
29
+ }
30
+ toBody() {
31
+ return { error: { code: this.code, message: this.message, retryable: this.retryable } };
32
+ }
33
+ };
34
+
35
+ // ../contract/src/query.ts
36
+ var RESERVED_KEYS = ["path", "ext", "tag", "in", "modified", "created", "is", "links"];
37
+ var LANES = ["lex", "vec", "hyde"];
38
+ var DAY_MS = 864e5;
39
+ var KEY_RE = /^[A-Za-z_][A-Za-z0-9_-]*$/;
40
+ function isSpace(ch) {
41
+ return ch === " " || ch === " " || ch === "\n" || ch === "\r";
42
+ }
43
+ function tokenize(input, offset) {
44
+ const tokens = [];
45
+ let i = 0;
46
+ const n = input.length;
47
+ while (i < n) {
48
+ while (i < n && isSpace(input[i])) i++;
49
+ if (i >= n) break;
50
+ const start = i;
51
+ let negated = false;
52
+ let exact = false;
53
+ let escapedLead = false;
54
+ if (input[i] === "\\" && input[i + 1] === "-") {
55
+ escapedLead = true;
56
+ i++;
57
+ } else if (input[i] === "-" && i + 1 < n && !isSpace(input[i + 1])) {
58
+ negated = true;
59
+ i++;
60
+ }
61
+ if (!escapedLead && input[i] === "=" && i + 1 < n && !isSpace(input[i + 1]) && input[i + 1] !== '"') {
62
+ exact = true;
63
+ i++;
64
+ }
65
+ let text = "";
66
+ let key;
67
+ let cmp;
68
+ let quoted = false;
69
+ let sawEscape = escapedLead;
70
+ const readQuoted = () => {
71
+ i++;
72
+ let out2 = "";
73
+ while (i < n && input[i] !== '"') {
74
+ if (input[i] === "\\" && input[i + 1] === '"') i++;
75
+ out2 += input[i];
76
+ i++;
77
+ }
78
+ if (i < n) i++;
79
+ return out2;
80
+ };
81
+ if (input[i] === '"') {
82
+ text = readQuoted();
83
+ quoted = true;
84
+ } else {
85
+ while (i < n && !isSpace(input[i])) {
86
+ const ch = input[i];
87
+ if (ch === "\\" && (input[i + 1] === '"' || input[i + 1] === ":" || input[i + 1] === "-")) {
88
+ text += input[i + 1];
89
+ sawEscape = true;
90
+ i += 2;
91
+ continue;
92
+ }
93
+ if (ch === ":" && key === void 0 && !exact && KEY_RE.test(text) && i + 1 < n && !isSpace(input[i + 1])) {
94
+ if (input[i + 1] === "/" && input[i + 2] === "/") {
95
+ text += ch;
96
+ i++;
97
+ continue;
98
+ }
99
+ key = text;
100
+ text = "";
101
+ i++;
102
+ const two = input.slice(i, i + 2);
103
+ if (two === ">=" || two === "<=") {
104
+ cmp = two;
105
+ i += 2;
106
+ } else if (input[i] === ">" || input[i] === "<" || input[i] === "=") {
107
+ cmp = input[i];
108
+ i++;
109
+ }
110
+ if (input[i] === '"') {
111
+ text = readQuoted();
112
+ quoted = true;
113
+ break;
114
+ }
115
+ if (input.startsWith("[[", i)) {
116
+ const close = input.indexOf("]]", i);
117
+ if (close !== -1) {
118
+ text = input.slice(i, close + 2);
119
+ i = close + 2;
120
+ break;
121
+ }
122
+ }
123
+ continue;
124
+ }
125
+ if (ch === '"' && text === "" && key === void 0) break;
126
+ text += ch;
127
+ i++;
128
+ }
129
+ }
130
+ tokens.push({
131
+ start: start + offset,
132
+ end: i + offset,
133
+ negated,
134
+ key,
135
+ cmp,
136
+ text,
137
+ quoted,
138
+ exact,
139
+ isOr: !negated && !exact && !quoted && !sawEscape && key === void 0 && text === "OR"
140
+ });
141
+ }
142
+ return tokens;
143
+ }
144
+ var CMP_OPS = { ">": "gt", "<": "lt", ">=": "gte", "<=": "lte", "=": "eq" };
145
+ function parseDay(value) {
146
+ const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
147
+ if (!m) return null;
148
+ const d = new Date(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
149
+ if (d.getMonth() !== Number(m[2]) - 1 || d.getDate() !== Number(m[3])) return null;
150
+ return d.getTime();
151
+ }
152
+ function parseRelative(value) {
153
+ const m = /^(\d+)([dwmy])$/i.exec(value);
154
+ if (!m) return null;
155
+ const unit = { d: 1, w: 7, m: 30, y: 365 }[m[2].toLowerCase()];
156
+ return Number(m[1]) * unit * DAY_MS;
157
+ }
158
+ function splitList(text) {
159
+ return text.split(/[\s,]+/).filter(Boolean);
160
+ }
161
+ function buildFilter(tok, now) {
162
+ const key = tok.key.toLowerCase();
163
+ const value = tok.text;
164
+ const cmpOp = tok.cmp ? CMP_OPS[tok.cmp] : void 0;
165
+ const negate = tok.negated ? { negate: true } : {};
166
+ if (value === "") return { bad: `"${key}:" needs a value` };
167
+ switch (key) {
168
+ case "path":
169
+ return { filter: { field: "path", op: "contains", value, ...negate } };
170
+ case "ext": {
171
+ const items = splitList(value).map((e) => e.replace(/^\./, "").toLowerCase());
172
+ if (items.length === 0) return { bad: "ext: needs a file type, such as ext:pdf" };
173
+ return { filter: { field: "ext", op: "in", value: items, ...negate } };
174
+ }
175
+ case "tag": {
176
+ const items = splitList(value).map((t) => t.replace(/^#/, "").toLowerCase()).filter(Boolean);
177
+ if (items.length === 0) return { bad: "tag: needs a tag, such as tag:#zettel" };
178
+ return items.length === 1 ? { filter: { field: "tag", op: "eq", value: items[0], ...negate } } : { filter: { field: "tag", op: "in", value: items, ...negate } };
179
+ }
180
+ case "in": {
181
+ const v = value.toLowerCase();
182
+ if (v === "title" || v === "heading") return { inField: v };
183
+ if (v === "headings") return { inField: "heading" };
184
+ return { bad: `in: accepts "title" or "heading", not "${value}"` };
185
+ }
186
+ case "modified":
187
+ case "created": {
188
+ const rel = parseRelative(value);
189
+ if (rel !== null && !tok.cmp) return { filter: { field: key, op: "gte", value: now - rel, ...negate } };
190
+ const day = parseDay(value);
191
+ if (day === null) return { bad: `"${value}" is not a date. Use YYYY-MM-DD or a span such as 7d` };
192
+ const op = cmpOp ?? "eq";
193
+ if (op === "eq") return { filter: { field: key, op: "eq", value, ...negate } };
194
+ const ms = op === "gt" || op === "lte" ? day + DAY_MS - 1 : day;
195
+ return { filter: { field: key, op, value: ms, ...negate } };
196
+ }
197
+ case "is": {
198
+ const v = value.toLowerCase();
199
+ if (v !== "orphan") return { bad: `is: accepts "orphan", not "${value}"` };
200
+ return { filter: { field: "is", op: "eq", value: v, ...negate } };
201
+ }
202
+ case "links": {
203
+ const target = value.replace(/^\[\[/, "").replace(/\]\]$/, "").split("|")[0].split("#")[0].trim();
204
+ if (!target) return { bad: "links: needs a note, such as links:[[Project plan]]" };
205
+ return { filter: { field: "links", op: "eq", value: target, ...negate } };
206
+ }
207
+ default: {
208
+ const op = cmpOp ?? "eq";
209
+ const num = value.trim() !== "" && !tok.quoted && Number.isFinite(Number(value)) ? Number(value) : null;
210
+ if (op !== "eq" && num === null && parseDay(value) === null) {
211
+ return { bad: `"${tok.cmp}" needs a number or a date, not "${value}"` };
212
+ }
213
+ return { filter: { field: `fm.${key}`, op, value: num ?? value, ...negate } };
214
+ }
215
+ }
216
+ }
217
+ function plainTerms(tok) {
218
+ if (tok.quoted) return tok.text.trim() ? [{ kind: "phrase", text: tok.text.trim() }] : [];
219
+ if (tok.exact) return [{ kind: "exact", text: tok.text }];
220
+ return tok.text ? [{ kind: "word", text: tok.text }] : [];
221
+ }
222
+ function parseQuery(raw, opts = {}) {
223
+ const now = opts.now ?? Date.now();
224
+ const diagnostics = [];
225
+ let rest = raw;
226
+ let offset = 0;
227
+ let mode = "instant";
228
+ let lane;
229
+ const lead = /^\s*\?/.exec(rest);
230
+ if (lead) {
231
+ mode = "ask";
232
+ offset += lead[0].length;
233
+ rest = rest.slice(lead[0].length);
234
+ }
235
+ const laneMatch = /^\s*(lex|vec|hyde):(?=\s|$)/i.exec(rest) ?? /^\s*(lex|vec|hyde):/i.exec(rest);
236
+ if (laneMatch && LANES.includes(laneMatch[1].toLowerCase())) {
237
+ lane = laneMatch[1].toLowerCase();
238
+ offset += laneMatch[0].length;
239
+ rest = rest.slice(laneMatch[0].length);
240
+ }
241
+ if (opts.mode) mode = opts.mode;
242
+ const tokens = tokenize(rest, offset);
243
+ const must = [];
244
+ const mustNot = [];
245
+ const filters = [];
246
+ let inField;
247
+ let pendingOr = false;
248
+ let lastWasTerm = false;
249
+ const addTerms = (terms, negated) => {
250
+ if (terms.length === 0) return;
251
+ if (negated) {
252
+ mustNot.push(...terms);
253
+ return;
254
+ }
255
+ if (pendingOr && lastWasTerm && must.length > 0) must[must.length - 1].push(...terms);
256
+ else for (const t of terms) must.push([t]);
257
+ lastWasTerm = true;
258
+ };
259
+ for (let i = 0; i < tokens.length; i++) {
260
+ const tok = tokens[i];
261
+ if (tok.isOr) {
262
+ const next = tokens[i + 1];
263
+ if (lastWasTerm && next && !next.isOr) {
264
+ pendingOr = true;
265
+ continue;
266
+ }
267
+ addTerms([{ kind: "word", text: "OR" }], false);
268
+ pendingOr = false;
269
+ continue;
270
+ }
271
+ if (tok.key !== void 0) {
272
+ const key = tok.key.toLowerCase();
273
+ const reserved = RESERVED_KEYS.includes(key);
274
+ const known = reserved || !opts.knownKeys || opts.knownKeys.has(key);
275
+ if (!known) {
276
+ diagnostics.push({
277
+ start: tok.start,
278
+ end: tok.end,
279
+ code: "unknown_operator",
280
+ message: `"${tok.key}:" is not an operator or a frontmatter field. It is searched as plain words.`
281
+ });
282
+ addTerms(
283
+ [tok.key, tok.text].filter(Boolean).map((text) => ({ kind: "word", text })),
284
+ tok.negated
285
+ );
286
+ pendingOr = false;
287
+ continue;
288
+ }
289
+ const out2 = buildFilter(tok, now);
290
+ if ("bad" in out2) {
291
+ diagnostics.push({ start: tok.start, end: tok.end, code: "bad_value", message: out2.bad });
292
+ } else if ("inField" in out2) {
293
+ inField = out2.inField;
294
+ } else {
295
+ filters.push(out2.filter);
296
+ }
297
+ pendingOr = false;
298
+ lastWasTerm = false;
299
+ continue;
300
+ }
301
+ const wasOr = pendingOr;
302
+ addTerms(plainTerms(tok), tok.negated);
303
+ if (tok.negated && !wasOr) lastWasTerm = false;
304
+ pendingOr = false;
305
+ }
306
+ if (inField) {
307
+ for (const group of must) for (const t of group) t.field = inField;
308
+ }
309
+ const query = { mode, must, mustNot, filters, raw };
310
+ if (lane) query.lane = lane;
311
+ return { query, diagnostics };
312
+ }
313
+ function queryText(q) {
314
+ return q.must.map((g) => g.map((t) => t.text).join(" ")).join(" ").trim();
315
+ }
316
+ function replaceWord(q, from, to) {
317
+ return {
318
+ ...q,
319
+ must: q.must.map((g) => g.map((t) => t.kind === "word" && t.text.toLowerCase() === from.toLowerCase() ? { ...t, text: to } : t))
320
+ };
321
+ }
322
+
323
+ // ../contract/src/ranking.ts
324
+ var RRF_K = 60;
325
+ var TYPO_RESCUE_MAX_RESULTS = 3;
326
+ var TYPO_MIN_WORD_LENGTH = 5;
327
+ var TYPO_TWO_EDITS_MIN_LENGTH = 9;
328
+ var QMD_BM25_WEIGHTS = { path: 1.5, title: 4, body: 1 };
329
+ var RELATED_NEAREST_CHUNKS = 100;
330
+ var EXTRA_BLOCK_BONUS = 0.05;
331
+ var EXTRA_BLOCK_BONUS_CAP = 0.15;
332
+ var DEFAULT_THRESHOLDS = { strong: 0.6, related: 0.45, loose: 0.3 };
333
+ var MODEL_THRESHOLDS = {
334
+ "embeddinggemma-300M": DEFAULT_THRESHOLDS
335
+ };
336
+ var ASK_LANE_LIMIT = 50;
337
+ var ASK_FAST_LIMIT = 20;
338
+ var ASK_RERANK_TOP = 30;
339
+ var ASK_DEEP_MIN_WORDS = 6;
340
+ var ASK_DEEP_DEADLINE_MS = 5e3;
341
+ var ASK_QUESTION_WORDS = ["who", "what", "when", "where", "why", "how", "which", "does", "do", "is", "are", "was", "were", "can", "could", "should", "would", "will", "has", "have"];
342
+ var ASK_STRONG_RANKS = 3;
343
+ var ASK_RELATED_RANKS = 10;
344
+ var ASK_RELAX_BELOW = 3;
345
+ var ASK_STOPWORDS = new Set(
346
+ "a about an and are as at be been but by can did do does for from had has have how i if in into is it its me my of on or our should so than that the their them then there these they this to was we were what when where which who why will with would you your".split(
347
+ " "
348
+ )
349
+ );
350
+ var BLEND_ALPHA = { top3: 0.75, top10: 0.6, rest: 0.4 };
351
+ var DUPLICATE_BLOCK_COSINE = 0.92;
352
+ var DUPLICATE_BLOCK_MIN_WORDS = 40;
353
+ var DUPLICATE_SOURCE_COSINE = 0.95;
354
+ var MAX_TEXT_BYTES = 2 * 1024 * 1024;
355
+ function rrf(lists, k = RRF_K) {
356
+ const scores = /* @__PURE__ */ new Map();
357
+ for (const list of lists) {
358
+ list.items.forEach((id, i) => {
359
+ scores.set(id, (scores.get(id) ?? 0) + list.weight / (k + i + 1));
360
+ });
361
+ }
362
+ return [...scores.entries()].map(([id, score]) => ({ id, score })).sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : 1));
363
+ }
364
+ function thresholdsFor(modelId) {
365
+ return modelId && MODEL_THRESHOLDS[modelId] || DEFAULT_THRESHOLDS;
366
+ }
367
+ function labelForCosine(cosine2, t = DEFAULT_THRESHOLDS) {
368
+ if (cosine2 >= t.strong) return "strong";
369
+ if (cosine2 >= t.related) return "related";
370
+ if (cosine2 >= t.loose) return "loose";
371
+ return null;
372
+ }
373
+ function labelForRank(rank) {
374
+ if (rank <= ASK_STRONG_RANKS) return "strong";
375
+ if (rank <= ASK_RELATED_RANKS) return "related";
376
+ return "loose";
377
+ }
378
+ function sourceScore(blockScores, t = DEFAULT_THRESHOLDS) {
379
+ if (blockScores.length === 0) return 0;
380
+ const sorted = [...blockScores].sort((a, b) => b - a);
381
+ const extra = sorted.slice(1).filter((s) => s >= t.related).length;
382
+ return sorted[0] + Math.min(EXTRA_BLOCK_BONUS_CAP, extra * EXTRA_BLOCK_BONUS);
383
+ }
384
+ function pickAskMode(q) {
385
+ const text = (typeof q === "string" ? q : queryText(q)).trim();
386
+ const raw = (typeof q === "string" ? q : q.raw).trim();
387
+ const words = text.split(/\s+/).filter(Boolean);
388
+ if (words.length >= ASK_DEEP_MIN_WORDS) return "deep";
389
+ if (raw.endsWith("?")) return "deep";
390
+ const first = (words[0] ?? "").toLowerCase().replace(/[^a-z]/g, "");
391
+ if (ASK_QUESTION_WORDS.includes(first)) return "deep";
392
+ return "fast";
393
+ }
394
+ function blendRerank(fusedRank, rrfNormalised, rerankScore) {
395
+ const alpha = fusedRank <= 3 ? BLEND_ALPHA.top3 : fusedRank <= 10 ? BLEND_ALPHA.top10 : BLEND_ALPHA.rest;
396
+ return alpha * rrfNormalised + (1 - alpha) * rerankScore;
397
+ }
398
+ var CJK_RE = /[぀-ヿ㐀-䶿一-鿿가-힯豈-﫿]/;
399
+ function hasCjk(text) {
400
+ return CJK_RE.test(text);
401
+ }
402
+ function typoMaxEdits(word, userMax = 1) {
403
+ if (userMax === 0 || hasCjk(word)) return 0;
404
+ const len = [...word].length;
405
+ if (len < TYPO_MIN_WORD_LENGTH) return 0;
406
+ if (len >= TYPO_TWO_EDITS_MIN_LENGTH) return 2;
407
+ return userMax === 2 ? 2 : 1;
408
+ }
409
+ function damerauLevenshtein(a, b, max = Infinity) {
410
+ if (a === b) return 0;
411
+ const la = a.length;
412
+ const lb = b.length;
413
+ if (Math.abs(la - lb) > max) return max + 1;
414
+ if (la === 0) return lb;
415
+ if (lb === 0) return la;
416
+ let prev2 = [];
417
+ let prev = Array.from({ length: lb + 1 }, (_, j) => j);
418
+ for (let i = 1; i <= la; i++) {
419
+ const cur = [i];
420
+ let rowMin = i;
421
+ for (let j = 1; j <= lb; j++) {
422
+ const cost = a[i - 1] === b[j - 1] ? 0 : 1;
423
+ let v = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost);
424
+ if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) v = Math.min(v, prev2[j - 2] + 1);
425
+ cur.push(v);
426
+ if (v < rowMin) rowMin = v;
427
+ }
428
+ if (rowMin > max) return max + 1;
429
+ prev2 = prev;
430
+ prev = cur;
431
+ }
432
+ return prev[lb];
433
+ }
434
+ function suggestCorrections(word, vocab, userMax = 1, limit2 = 5, stems = false) {
435
+ const max = typoMaxEdits(word, userMax);
436
+ if (max === 0) return [];
437
+ const w = word.toLowerCase();
438
+ const out2 = [];
439
+ for (const v of vocab) {
440
+ if (v.term.length - w.length > max || !stems && w.length - v.term.length > max) continue;
441
+ let d = damerauLevenshtein(w, v.term, max);
442
+ if (stems && d > 1 && v.term.length >= TYPO_MIN_WORD_LENGTH && v.term.length < w.length) {
443
+ d = Math.min(d, damerauLevenshtein(w.slice(0, v.term.length), v.term, max), damerauLevenshtein(w.slice(0, v.term.length + 1), v.term, max));
444
+ }
445
+ if (d >= 1 && d <= max) out2.push({ term: v.term, docs: v.docs, distance: d });
446
+ }
447
+ out2.sort((a, b) => b.docs - a.docs || a.distance - b.distance || (a.term < b.term ? -1 : 1));
448
+ return out2.slice(0, limit2);
449
+ }
450
+ function cosine(a, b) {
451
+ let dot2 = 0;
452
+ let na = 0;
453
+ let nb = 0;
454
+ const n = Math.min(a.length, b.length);
455
+ for (let i = 0; i < n; i++) {
456
+ dot2 += a[i] * b[i];
457
+ na += a[i] * a[i];
458
+ nb += b[i] * b[i];
459
+ }
460
+ return na === 0 || nb === 0 ? 0 : dot2 / Math.sqrt(na * nb);
461
+ }
462
+ function meanVector(vectors) {
463
+ if (vectors.length === 0) return null;
464
+ const dims = vectors[0].length;
465
+ const out2 = new Float32Array(dims);
466
+ for (const v of vectors) for (let i = 0; i < dims; i++) out2[i] += v[i];
467
+ let norm = 0;
468
+ for (let i = 0; i < dims; i++) norm += out2[i] * out2[i];
469
+ norm = Math.sqrt(norm);
470
+ if (norm === 0) return out2;
471
+ for (let i = 0; i < dims; i++) out2[i] /= norm;
472
+ return out2;
473
+ }
474
+ function ndcg(rankedIds, relevant, k = 10) {
475
+ const dcg = rankedIds.slice(0, k).reduce((sum, id, i) => sum + (2 ** (relevant[id] ?? 0) - 1) / Math.log2(i + 2), 0);
476
+ const ideal = Object.values(relevant).sort((a, b) => b - a).slice(0, k).reduce((sum, g, i) => sum + (2 ** g - 1) / Math.log2(i + 2), 0);
477
+ return ideal === 0 ? 0 : dcg / ideal;
478
+ }
479
+ function recallAt(rankedIds, relevant, k = 20) {
480
+ const ids = Object.keys(relevant).filter((id) => relevant[id] > 0);
481
+ if (ids.length === 0) return 0;
482
+ const top = new Set(rankedIds.slice(0, k));
483
+ return ids.filter((id) => top.has(id)).length / ids.length;
484
+ }
485
+
486
+ // ../contract/src/content.ts
487
+ function sourceId(collection, path) {
488
+ return `${collection}:${path}`;
489
+ }
490
+ function blockId(source, headingPath) {
491
+ return `${source}#${headingPath}`;
492
+ }
493
+ function parseId(id) {
494
+ const colon = id.indexOf(":");
495
+ if (colon <= 0) return null;
496
+ const collection = id.slice(0, colon);
497
+ const rest = id.slice(colon + 1);
498
+ const ext = /\.[A-Za-z0-9]+#/.exec(rest);
499
+ const hash2 = ext ? ext.index + ext[0].length - 1 : /\.[A-Za-z0-9]+$/.test(rest) ? -1 : rest.indexOf("#");
500
+ if (hash2 === -1) return rest ? { collection, path: rest, source: id } : null;
501
+ const path = rest.slice(0, hash2);
502
+ if (!path) return null;
503
+ return { collection, path, source: `${collection}:${path}`, headingPath: rest.slice(hash2 + 1) };
504
+ }
505
+ function slugHeading(text) {
506
+ return text.trim().toLowerCase().replace(/\//g, "-").replace(/\s+/g, "-");
507
+ }
508
+ function frontmatterEnd(text) {
509
+ if (!text.startsWith("---")) return 0;
510
+ const firstNl = text.indexOf("\n");
511
+ if (firstNl === -1 || text.slice(0, firstNl).trim() !== "---") return 0;
512
+ const re = /\n(---|\.\.\.)[ \t]*(\r?\n|$)/g;
513
+ re.lastIndex = firstNl;
514
+ const m = re.exec(text);
515
+ return m ? m.index + m[0].length : 0;
516
+ }
517
+ function frontmatterText(text) {
518
+ const end = frontmatterEnd(text);
519
+ if (end === 0) return null;
520
+ const firstNl = text.indexOf("\n");
521
+ const body = text.slice(firstNl + 1, end);
522
+ return body.replace(/(^|\n)(---|\.\.\.)[ \t]*(\r?\n)?$/, "$1");
523
+ }
524
+ function fencedRanges(text) {
525
+ const out2 = [];
526
+ const re = /^[ \t]{0,3}(```+|~~~+)[^\n]*$/gm;
527
+ let open = null;
528
+ let m;
529
+ while (m = re.exec(text)) {
530
+ const fence = m[1];
531
+ if (!open) open = { at: m.index, fence };
532
+ else if (fence[0] === open.fence[0] && fence.length >= open.fence.length) {
533
+ out2.push([open.at, m.index + m[0].length]);
534
+ open = null;
535
+ }
536
+ }
537
+ if (open) out2.push([open.at, text.length]);
538
+ return out2;
539
+ }
540
+ function inRanges(pos, ranges) {
541
+ for (const [s, e] of ranges) if (pos >= s && pos < e) return true;
542
+ return false;
543
+ }
544
+ function splitBlocks(text) {
545
+ const fmEnd = frontmatterEnd(text);
546
+ const fences = fencedRanges(text);
547
+ const blocks = [];
548
+ const stack = [];
549
+ const seen = /* @__PURE__ */ new Map();
550
+ const re = /^(#{1,6})[ \t]+(.+?)[ \t]*#*[ \t]*$/gm;
551
+ re.lastIndex = fmEnd;
552
+ let m;
553
+ const heads = [];
554
+ while (m = re.exec(text)) {
555
+ if (inRanges(m.index, fences)) continue;
556
+ heads.push({ at: m.index, level: m[1].length, text: m[2].trim() });
557
+ }
558
+ const firstAt = heads.length ? heads[0].at : text.length;
559
+ if (text.slice(fmEnd, firstAt).trim() !== "" || heads.length === 0) {
560
+ blocks.push({ headingPath: "", headings: [], level: 0, start: 0, end: firstAt });
561
+ }
562
+ heads.forEach((h, i) => {
563
+ while (stack.length && stack[stack.length - 1].level >= h.level) stack.pop();
564
+ stack.push({ level: h.level, text: h.text });
565
+ const headings = stack.map((s) => s.text);
566
+ let path = headings.map(slugHeading).join("/");
567
+ const count = (seen.get(path) ?? 0) + 1;
568
+ seen.set(path, count);
569
+ if (count > 1) path = `${path}~${count}`;
570
+ blocks.push({ headingPath: path, headings, level: h.level, start: h.at, end: i + 1 < heads.length ? heads[i + 1].at : text.length });
571
+ });
572
+ return blocks;
573
+ }
574
+ function blockAt(blocks, offset) {
575
+ let found = blocks[0];
576
+ for (const b of blocks) {
577
+ if (b.start <= offset) found = b;
578
+ else break;
579
+ }
580
+ return found;
581
+ }
582
+ function findBlock(blocks, headingPath) {
583
+ return blocks.find((b) => b.headingPath === headingPath);
584
+ }
585
+ function blockText(text, block) {
586
+ const start = block.level === 0 ? Math.max(block.start, frontmatterEnd(text)) : block.start;
587
+ return text.slice(start, block.end);
588
+ }
589
+ function countWords(text) {
590
+ const m = text.match(/[\p{L}\p{N}]+(?:['’-][\p{L}\p{N}]+)*/gu);
591
+ return m ? m.length : 0;
592
+ }
593
+ function extractLinks(text) {
594
+ const fences = fencedRanges(text);
595
+ const out2 = [];
596
+ const wiki = /\[\[([^\]\n]+?)\]\]/g;
597
+ let m;
598
+ while (m = wiki.exec(text)) {
599
+ if (inRanges(m.index, fences)) continue;
600
+ const target = m[1].split("|")[0].split("#")[0].trim();
601
+ if (target) out2.push({ start: m.index, end: m.index + m[0].length, target });
602
+ }
603
+ const md = /\[[^\]\n]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
604
+ while (m = md.exec(text)) {
605
+ if (inRanges(m.index, fences)) continue;
606
+ let target = m[1];
607
+ if (/^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith("#")) continue;
608
+ try {
609
+ target = decodeURIComponent(target);
610
+ } catch {
611
+ }
612
+ target = target.split("#")[0].replace(/^<|>$/g, "");
613
+ if (target) out2.push({ start: m.index, end: m.index + m[0].length, target });
614
+ }
615
+ return out2.sort((a, b) => a.start - b.start);
616
+ }
617
+ function extractInlineTags(text) {
618
+ const fences = fencedRanges(text);
619
+ const fmEnd = frontmatterEnd(text);
620
+ const out2 = /* @__PURE__ */ new Set();
621
+ const re = /(^|[\s(>])#([\p{L}\p{N}_/-]+)/gu;
622
+ let m;
623
+ while (m = re.exec(text)) {
624
+ if (m.index < fmEnd || inRanges(m.index, fences)) continue;
625
+ const tag = m[2].replace(/\/+$/, "");
626
+ if (tag && !/^\d+$/.test(tag)) out2.add(tag.toLowerCase());
627
+ }
628
+ return [...out2];
629
+ }
630
+ function normaliseTags(value) {
631
+ const items = Array.isArray(value) ? value : typeof value === "string" ? value.split(/[\s,]+/) : [];
632
+ return items.map((t) => String(t).trim().replace(/^#/, "").toLowerCase()).filter(Boolean);
633
+ }
634
+ function normaliseAliases(value) {
635
+ const items = Array.isArray(value) ? value : typeof value === "string" ? [value] : [];
636
+ return items.map((a) => String(a).trim()).filter(Boolean);
637
+ }
638
+ function basename(path) {
639
+ const slash = path.lastIndexOf("/");
640
+ return slash === -1 ? path : path.slice(slash + 1);
641
+ }
642
+ function extname(path) {
643
+ const base = basename(path);
644
+ const dot2 = base.lastIndexOf(".");
645
+ return dot2 <= 0 ? "" : base.slice(dot2 + 1).toLowerCase();
646
+ }
647
+ function stripExt(path) {
648
+ const base = basename(path);
649
+ const dot2 = base.lastIndexOf(".");
650
+ return dot2 <= 0 ? path : path.slice(0, path.length - (base.length - dot2));
651
+ }
652
+ function dirname(path) {
653
+ const slash = path.lastIndexOf("/");
654
+ return slash === -1 ? "" : path.slice(0, slash);
655
+ }
656
+ function titleFromPath(path) {
657
+ return stripExt(basename(path));
658
+ }
659
+ function createLinkResolver(paths) {
660
+ const exact = /* @__PURE__ */ new Map();
661
+ const byName = /* @__PURE__ */ new Map();
662
+ const put = (key, path) => {
663
+ const prev = byName.get(key);
664
+ if (!prev || path.length < prev.length || path.length === prev.length && path < prev) byName.set(key, path);
665
+ };
666
+ for (const p of paths) {
667
+ exact.set(p.toLowerCase(), p);
668
+ put(basename(p).toLowerCase(), p);
669
+ put(stripExt(basename(p)).toLowerCase(), p);
670
+ }
671
+ return (target, fromPath) => {
672
+ let t = target.trim().replace(/\\/g, "/");
673
+ if (t.startsWith("./") || t.startsWith("../")) {
674
+ const parts = (fromPath ? dirname(fromPath) : "").split("/").filter(Boolean);
675
+ for (const seg of t.split("/")) {
676
+ if (seg === "..") parts.pop();
677
+ else if (seg !== ".") parts.push(seg);
678
+ }
679
+ t = parts.join("/");
680
+ }
681
+ t = t.replace(/^\/+/, "");
682
+ const key = t.toLowerCase();
683
+ return exact.get(key) ?? exact.get(`${key}.md`) ?? byName.get(basename(key)) ?? null;
684
+ };
685
+ }
686
+
687
+ // ../contract/src/filters.ts
688
+ var DAY_MS2 = 864e5;
689
+ function toTime(value) {
690
+ if (typeof value === "number") return value;
691
+ if (value instanceof Date) return value.getTime();
692
+ if (typeof value === "string") {
693
+ const day = parseDay(value.slice(0, 10));
694
+ if (day !== null && value.length <= 10) return day;
695
+ const t = Date.parse(value);
696
+ return Number.isNaN(t) ? null : t;
697
+ }
698
+ return null;
699
+ }
700
+ function compareScalar(actual, op, expected) {
701
+ if (actual == null) return false;
702
+ if (actual instanceof Date) actual = actual.toISOString();
703
+ const expNum = typeof expected === "number" ? expected : null;
704
+ const actNum = typeof actual === "number" ? actual : typeof actual === "string" && actual.trim() !== "" && Number.isFinite(Number(actual)) ? Number(actual) : null;
705
+ if (op === "eq" || op === "ne") {
706
+ let equal;
707
+ if (expNum !== null && actNum !== null) equal = expNum === actNum;
708
+ else if (typeof expected === "string" && parseDay(expected) !== null && toTime(actual) !== null) {
709
+ const day = parseDay(expected);
710
+ const t = toTime(actual);
711
+ equal = t >= day && t < day + DAY_MS2;
712
+ } else equal = String(actual).trim().toLowerCase() === String(expected).trim().toLowerCase();
713
+ return op === "eq" ? equal : !equal;
714
+ }
715
+ if (op === "contains") return String(actual).toLowerCase().includes(String(expected).toLowerCase());
716
+ let a = actNum;
717
+ let b = expNum;
718
+ if (typeof expected === "string") {
719
+ const day = parseDay(expected);
720
+ if (day === null) return false;
721
+ a = toTime(actual);
722
+ b = op === "gt" || op === "lte" ? day + DAY_MS2 - 1 : day;
723
+ }
724
+ if (a === null || b === null) return false;
725
+ switch (op) {
726
+ case "lt":
727
+ return a < b;
728
+ case "lte":
729
+ return a <= b;
730
+ case "gt":
731
+ return a > b;
732
+ case "gte":
733
+ return a >= b;
734
+ default:
735
+ return false;
736
+ }
737
+ }
738
+ function matchOne(meta, f, ctx) {
739
+ switch (f.field) {
740
+ case "path":
741
+ return meta.path.toLowerCase().includes(String(f.value).toLowerCase());
742
+ case "ext": {
743
+ const list = Array.isArray(f.value) ? f.value : [String(f.value)];
744
+ return list.includes(extname(meta.path));
745
+ }
746
+ case "tag": {
747
+ const wanted = (Array.isArray(f.value) ? f.value : [String(f.value)]).map((t) => t.toLowerCase());
748
+ return meta.tags.some((tag) => wanted.some((w) => tag === w || tag.startsWith(`${w}/`)));
749
+ }
750
+ case "modified":
751
+ case "created": {
752
+ const actual = f.field === "modified" ? meta.modified : meta.created;
753
+ if (Array.isArray(f.value)) return false;
754
+ return compareScalar(actual, f.op, f.value);
755
+ }
756
+ case "is":
757
+ if (f.value === "orphan") return ctx.isOrphan ? ctx.isOrphan(meta.path) : meta.orphan === true;
758
+ return false;
759
+ case "links": {
760
+ const target = String(f.value);
761
+ const resolved = ctx.resolveLink?.(target, meta.path);
762
+ if (resolved) return meta.links.includes(resolved);
763
+ const want = stripExt(basename(target)).toLowerCase();
764
+ return meta.links.some((l) => stripExt(basename(l)).toLowerCase() === want);
765
+ }
766
+ default: {
767
+ if (!f.field.startsWith("fm.")) return false;
768
+ const key = f.field.slice(3).toLowerCase();
769
+ const entry = Object.entries(meta.frontmatter).find(([k]) => k.toLowerCase() === key);
770
+ if (!entry) return f.op === "ne";
771
+ const actual = entry[1];
772
+ const expected = Array.isArray(f.value) ? f.value : [f.value];
773
+ const actuals = Array.isArray(actual) ? actual : [actual];
774
+ if (f.op === "ne") return expected.every((e) => actuals.every((a) => compareScalar(a, "ne", e)));
775
+ const op = f.op === "in" ? "eq" : f.op;
776
+ return expected.some((e) => actuals.some((a) => compareScalar(a, op, e)));
777
+ }
778
+ }
779
+ }
780
+ function matchesFilters(meta, filters, ctx = {}) {
781
+ if (meta.excluded) return false;
782
+ for (const f of filters) {
783
+ const ok = matchOne(meta, f, ctx);
784
+ if (f.negate ? ok : !ok) return false;
785
+ }
786
+ return true;
787
+ }
788
+ function isExcludedByFrontmatter(frontmatter) {
789
+ if (!frontmatter) return false;
790
+ const v = frontmatter["trova"];
791
+ return v === false || v === "false";
792
+ }
793
+ function globToRegExp(glob) {
794
+ let g = glob.trim().replace(/^\.?\//, "");
795
+ if (!/[*?]/.test(g)) g = `${g.replace(/\/+$/, "")}{,/**}`;
796
+ let re = "";
797
+ for (let i = 0; i < g.length; i++) {
798
+ const ch = g[i];
799
+ if (ch === "*") {
800
+ if (g[i + 1] === "*") {
801
+ const slash = g[i + 2] === "/";
802
+ re += slash ? "(?:.*/)?" : ".*";
803
+ i += slash ? 2 : 1;
804
+ } else re += "[^/]*";
805
+ } else if (ch === "?") re += "[^/]";
806
+ else if (ch === "{" && g.startsWith("{,/**}", i)) {
807
+ re += "(?:/.*)?";
808
+ i += 5;
809
+ } else re += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
810
+ }
811
+ return new RegExp(`^${re}$`, "i");
812
+ }
813
+ function inScope(path, include, exclude) {
814
+ if (exclude?.some((g) => g.trim() && globToRegExp(g).test(path))) return false;
815
+ const inc = include?.filter((g) => g.trim()) ?? [];
816
+ return inc.length === 0 || inc.some((g) => globToRegExp(g).test(path));
817
+ }
818
+
819
+ // ../contract/src/match.ts
820
+ function fold(text) {
821
+ const lower = text.toLowerCase();
822
+ if (lower.length === text.length) return lower;
823
+ let out2 = "";
824
+ for (const ch of text) {
825
+ const l = ch.toLowerCase();
826
+ out2 += l.length === ch.length ? l : ch;
827
+ }
828
+ return out2;
829
+ }
830
+ function lightStem(word) {
831
+ let w = word;
832
+ if (w.length <= 3 || /[^a-z]/.test(w)) return w;
833
+ if (w.endsWith("ies") && w.length > 4) w = `${w.slice(0, -3)}y`;
834
+ else if (w.endsWith("sses")) w = w.slice(0, -2);
835
+ else if (w.endsWith("s") && !w.endsWith("ss") && !w.endsWith("us") && !w.endsWith("is")) w = w.slice(0, -1);
836
+ for (const suffix of ["ingly", "edly", "ing", "ed", "ly"]) {
837
+ if (w.endsWith(suffix) && w.length - suffix.length >= 3) {
838
+ w = w.slice(0, -suffix.length);
839
+ if (w.length > 3 && w[w.length - 1] === w[w.length - 2] && !/[lsz]$/.test(w)) w = w.slice(0, -1);
840
+ break;
841
+ }
842
+ }
843
+ if (w.length > 4 && w.endsWith("e")) w = w.slice(0, -1);
844
+ return w;
845
+ }
846
+ var WORD_CHAR = "[\\p{L}\\p{N}_]";
847
+ function escapeRe(text) {
848
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
849
+ }
850
+ function termRegExp(term) {
851
+ const text = fold(term.text.trim());
852
+ if (!text) return null;
853
+ if (term.kind === "phrase") {
854
+ const body = text.split(/\s+/).map(escapeRe).join("\\s+");
855
+ return new RegExp(`(?<!${WORD_CHAR})${body}(?!${WORD_CHAR})`, "gu");
856
+ }
857
+ if (term.kind === "exact") return new RegExp(`(?<!${WORD_CHAR})${escapeRe(text)}(?!${WORD_CHAR})`, "gu");
858
+ const stem = lightStem(text);
859
+ const alt = stem !== text && stem.length >= 3 ? `(?:${escapeRe(text)}|${escapeRe(stem)})` : escapeRe(text);
860
+ return new RegExp(`(?<!${WORD_CHAR})${alt}${WORD_CHAR}*`, "gu");
861
+ }
862
+ function termRanges(text, term, max = 50) {
863
+ const re = termRegExp(term);
864
+ if (!re) return [];
865
+ const hay = fold(text);
866
+ const out2 = [];
867
+ let m;
868
+ while (out2.length < max && (m = re.exec(hay))) {
869
+ if (m[0].length === 0) {
870
+ re.lastIndex++;
871
+ continue;
872
+ }
873
+ out2.push([m.index, m.index + m[0].length]);
874
+ }
875
+ return out2;
876
+ }
877
+ function termMatches(text, term) {
878
+ return termRanges(text, term, 1).length > 0;
879
+ }
880
+ function queryRanges(text, q, max = 200) {
881
+ const all = [];
882
+ for (const group of q.must) for (const t of group) all.push(...termRanges(text, t, max));
883
+ return mergeRanges(all).slice(0, max);
884
+ }
885
+ function mergeRanges(ranges) {
886
+ const sorted = [...ranges].sort((a, b) => a[0] - b[0] || a[1] - b[1]);
887
+ const out2 = [];
888
+ for (const r of sorted) {
889
+ const last = out2[out2.length - 1];
890
+ if (last && r[0] <= last[1]) last[1] = Math.max(last[1], r[1]);
891
+ else out2.push([r[0], r[1]]);
892
+ }
893
+ return out2;
894
+ }
895
+ function fieldText(fields, field) {
896
+ const title = [fields.title, ...fields.aliases ?? []].join("\n");
897
+ if (field === "title") return title;
898
+ const headings = (fields.headings ?? []).join("\n");
899
+ if (field === "heading") return headings;
900
+ return `${title}
901
+ ${headings}
902
+ ${fields.body ?? ""}`;
903
+ }
904
+ function matchesTerms(fields, q) {
905
+ for (const t of q.mustNot) if (termMatches(fieldText(fields, t.field), t)) return false;
906
+ for (const group of q.must) {
907
+ if (!group.some((t) => termMatches(fieldText(fields, t.field), t))) return false;
908
+ }
909
+ return true;
910
+ }
911
+ function makeSnippet(text, ranges, maxLen = 180, from = 0, to = text.length) {
912
+ const inside = ranges.filter((r) => r[0] >= from && r[1] <= to);
913
+ let start = from;
914
+ if (inside.length > 0) {
915
+ let best = 0;
916
+ let bestCount = 0;
917
+ for (let i = 0; i < inside.length; i++) {
918
+ let count = 0;
919
+ for (let j = i; j < inside.length && inside[j][1] - inside[i][0] <= maxLen; j++) count++;
920
+ if (count > bestCount) {
921
+ bestCount = count;
922
+ best = i;
923
+ }
924
+ }
925
+ start = Math.max(from, inside[best][0] - Math.floor(maxLen / 4));
926
+ if (start > from) {
927
+ const space = text.lastIndexOf(" ", start);
928
+ const nl = text.lastIndexOf("\n", start);
929
+ const cut = Math.max(space, nl);
930
+ if (cut >= from && start - cut < 30) start = cut + 1;
931
+ }
932
+ } else {
933
+ const lead = /^[\s#>*-]*/.exec(text.slice(from, to));
934
+ start = from + (lead ? lead[0].length : 0);
935
+ }
936
+ let end = Math.min(to, start + maxLen);
937
+ if (end < to) {
938
+ const space = text.lastIndexOf(" ", end);
939
+ if (space > start + maxLen / 2) end = space;
940
+ }
941
+ const prefix = start > from ? "\u2026" : "";
942
+ const suffix = end < to ? "\u2026" : "";
943
+ const body = text.slice(start, end).replace(/\s/g, " ").trimEnd();
944
+ const highlights = inside.filter((r) => r[0] >= start && r[1] <= end).map((r) => [r[0] - start + prefix.length, r[1] - start + prefix.length]);
945
+ return { snippet: prefix + body + suffix, highlights, offset: start };
946
+ }
947
+
948
+ // ../eval/src/index.ts
949
+ import { readFileSync } from "node:fs";
950
+
951
+ // ../eval/src/fixture.ts
952
+ import { mkdirSync, writeFileSync } from "node:fs";
953
+ import { dirname as dirname2, join } from "node:path";
954
+ var TOPICS = [
955
+ { key: "keyboard", folder: "Tools", title: "Keyboard", terms: ["keyboard", "remap", "layer", "homerow", "modifier", "keymap", "firmware", "chord"], question: "how do I remap modifier keys on a keyboard layer" },
956
+ { key: "search", folder: "Search", title: "Retrieval", terms: ["retrieval", "ranking", "fusion", "bm25", "embedding", "rerank", "recall", "index"], question: "what improves ranking when combining bm25 with embedding retrieval" },
957
+ { key: "garden", folder: "Garden", title: "Garden", terms: ["tomato", "compost", "seedling", "mulch", "prune", "harvest", "trellis", "soil"], question: "when should tomato seedlings move into compost rich soil" },
958
+ { key: "cooking", folder: "Kitchen", title: "Cooking", terms: ["sourdough", "ferment", "starter", "hydration", "crumb", "proof", "oven", "flour"], question: "why does sourdough hydration change the crumb after the proof" },
959
+ { key: "running", folder: "Health", title: "Running", terms: ["marathon", "tempo", "interval", "cadence", "recovery", "mileage", "taper", "stride"], question: "how should weekly mileage change during a marathon taper" },
960
+ { key: "finance", folder: "Money", title: "Finance", terms: ["budget", "invoice", "ledger", "dividend", "expense", "forecast", "savings", "audit"], question: "which expenses belong in the quarterly budget forecast ledger" },
961
+ { key: "astronomy", folder: "Sky", title: "Astronomy", terms: ["telescope", "nebula", "aperture", "eyepiece", "galaxy", "exposure", "tracking", "seeing"], question: "what aperture telescope shows a nebula with long exposure tracking" },
962
+ { key: "travel", folder: "Trips", title: "Travel", terms: ["itinerary", "ferry", "hostel", "passport", "layover", "trailhead", "luggage", "visa"], question: "how to plan a ferry itinerary with a short layover and visa checks" }
963
+ ];
964
+ var FILLER = ["notes", "ideas", "plan", "review", "log", "guide", "checklist", "summary", "draft", "reference", "questions", "decisions"];
965
+ function rng(seed) {
966
+ let s = seed >>> 0;
967
+ return () => {
968
+ s = s + 1831565813 >>> 0;
969
+ let t = Math.imul(s ^ s >>> 15, 1 | s);
970
+ t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
971
+ return ((t ^ t >>> 14) >>> 0) / 4294967296;
972
+ };
973
+ }
974
+ function sentence(topic, random, words = 14) {
975
+ const out2 = [];
976
+ for (let i = 0; i < words; i++) {
977
+ out2.push(random() < 0.55 ? topic.terms[Math.floor(random() * topic.terms.length)] : FILLER[Math.floor(random() * FILLER.length)]);
978
+ }
979
+ const text = out2.join(" ");
980
+ return `${text[0].toUpperCase()}${text.slice(1)}.`;
981
+ }
982
+ function paragraph(topic, random, sentences = 4) {
983
+ return Array.from({ length: sentences }, () => sentence(topic, random)).join(" ");
984
+ }
985
+ function makePdf(pages) {
986
+ const objects = [];
987
+ const kids = pages.map((_, i) => `${3 + i * 2} 0 R`).join(" ");
988
+ objects[0] = "<< /Type /Catalog /Pages 2 0 R >>";
989
+ objects[1] = `<< /Type /Pages /Kids [${kids}] /Count ${pages.length} >>`;
990
+ const fontId = 3 + pages.length * 2;
991
+ pages.forEach((text, i) => {
992
+ const escaped = text.replace(/([\\()])/g, "\\$1");
993
+ const stream = `BT /F1 12 Tf 72 720 Td (${escaped}) Tj ET`;
994
+ objects[2 + i * 2] = `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents ${4 + i * 2} 0 R /Resources << /Font << /F1 ${fontId} 0 R >> >> >>`;
995
+ objects[3 + i * 2] = `<< /Length ${stream.length} >>
996
+ stream
997
+ ${stream}
998
+ endstream`;
999
+ });
1000
+ objects[fontId - 1] = "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>";
1001
+ let pdf = "%PDF-1.4\n";
1002
+ const offsets = [];
1003
+ objects.forEach((body, i) => {
1004
+ offsets.push(pdf.length);
1005
+ pdf += `${i + 1} 0 obj
1006
+ ${body}
1007
+ endobj
1008
+ `;
1009
+ });
1010
+ const xref = pdf.length;
1011
+ pdf += `xref
1012
+ 0 ${objects.length + 1}
1013
+ 0000000000 65535 f
1014
+ `;
1015
+ for (const off of offsets) pdf += `${String(off).padStart(10, "0")} 00000 n
1016
+ `;
1017
+ pdf += `trailer
1018
+ << /Size ${objects.length + 1} /Root 1 0 R >>
1019
+ startxref
1020
+ ${xref}
1021
+ %%EOF
1022
+ `;
1023
+ return Buffer.from(pdf, "latin1");
1024
+ }
1025
+ var KANATA = `---
1026
+ aliases: [KB remap, Kanata tool]
1027
+ tags: [tools, keyboard/software]
1028
+ status: active
1029
+ rating: 5
1030
+ ---
1031
+ Kanata is a cross-platform keyboard remapper. It runs as a daemon and reads one config file.
1032
+
1033
+ # Setup
1034
+ Before anything else, check that the keyboard is detected.
1035
+
1036
+ ## Install
1037
+ Install kanata with Homebrew and grant it input monitoring permission.
1038
+
1039
+ ## Install
1040
+ On Linux, install kanata from the release binary and add a udev rule.
1041
+
1042
+ # Usage
1043
+ Start kanata with the config path. Layers switch with a held key. See [[Trova]] for search notes.
1044
+
1045
+ # Config
1046
+ \`\`\`lisp
1047
+ (defsrc caps a s d f)
1048
+ # this hash line is code, not a heading
1049
+ \`\`\`
1050
+ `;
1051
+ var TROVA = `---
1052
+ aliases: [Trova engine]
1053
+ tags: [project, search]
1054
+ ---
1055
+ # Trova
1056
+ Trova is the search engine behind Trova. It fuses keyword and vector results with reciprocal rank fusion.
1057
+
1058
+ ## Chunk size
1059
+ We decided on a chunk size of 900 tokens with 15 percent overlap after testing recall.
1060
+
1061
+ ## Links
1062
+ Related tools: [[Kanata]] and the [[Projects/Trova/Plan|plan]].
1063
+ `;
1064
+ var PLAN = `---
1065
+ title: Trova plan
1066
+ status: active
1067
+ rating: 4
1068
+ done: false
1069
+ due: 2026-12-01
1070
+ authors: [Ann, Bob]
1071
+ budget: 1250.5
1072
+ meta:
1073
+ owner: Bimal
1074
+ phase: 2
1075
+ tags:
1076
+ - project
1077
+ - planning
1078
+ ---
1079
+ # Plan
1080
+ The plan for the Trova engine: ship the shim first, then the Rust engine. Links: [[Trova]].
1081
+
1082
+ ## Milestones
1083
+ Gate one is the Connections pane. Gate two is the CLI and MCP server.
1084
+ `;
1085
+ var DUP_TEXT = `The quarterly retrospective found that incremental indexing saved most of the time. We measured the scan phase, the embedding phase and the write phase on the reference vault, and the scan phase dominated on spinning disks while the embedding phase dominated on laptops. The team agreed to cache content hashes, to skip unchanged files, and to batch the writes in one transaction per hundred files so that readers never wait for a long write lock.`;
1086
+ function fixtureNotes() {
1087
+ const files = {};
1088
+ const random = rng(20261116);
1089
+ TOPICS.forEach((topic, t) => {
1090
+ for (let i = 0; i < 23; i++) {
1091
+ const name = `${topic.title} ${FILLER[i % FILLER.length]} ${i + 1}`;
1092
+ const path = `${topic.folder}/${name}.md`;
1093
+ const next = `${topic.title} ${FILLER[(i + 1) % FILLER.length]} ${(i + 1) % 23 + 1}`;
1094
+ const tags = `tags: [${topic.key}, area/${topic.key}]`;
1095
+ const fm = i % 3 === 0 ? `---
1096
+ ${tags}
1097
+ status: ${i % 2 ? "active" : "done"}
1098
+ rating: ${i % 5 + 1}
1099
+ ---
1100
+ ` : "";
1101
+ const headings = i % 7 === 6 ? "" : `# ${topic.terms[i % topic.terms.length]} overview
1102
+ `;
1103
+ const second = i % 7 === 6 ? "" : `
1104
+ ## Details
1105
+ ${paragraph(topic, random)}
1106
+
1107
+ ## Details
1108
+ ${paragraph(topic, random, 2)}
1109
+ `;
1110
+ const link = i % 4 === 0 ? "" : `
1111
+ See also [[${next}]].
1112
+ `;
1113
+ const inlineTag = i % 5 === 0 ? ` #${topic.key}` : "";
1114
+ files[path] = `${fm}${headings}${paragraph(topic, random)}${inlineTag}
1115
+ ${second}${link}`;
1116
+ }
1117
+ });
1118
+ files["Tools/Kanata.md"] = KANATA;
1119
+ files["Trova.md"] = TROVA;
1120
+ files["Projects/Trova/Plan.md"] = PLAN;
1121
+ files["NoHeadings.md"] = "This note has no headings at all. It only has a short paragraph about a lighthouse keeper and the fog signal.\n";
1122
+ files["Long.md"] = `# Long note
1123
+ ${Array.from({ length: 60 }, (_, i) => `## Section ${i + 1}
1124
+ ${paragraph(TOPICS[i % TOPICS.length], random, 5)}
1125
+ `).join("\n")}`;
1126
+ files["Secret.md"] = "---\ntrova: false\n---\n# Secret\nThe zebrafish password is hidden here and must never appear in any result.\n";
1127
+ files["Orphan.md"] = "# Orphan\nNothing links here and this note links nowhere. It describes a quiet harbour at dawn.\n";
1128
+ files["Mentions.md"] = "# Mentions\nI tried Kanata yesterday and the KB remap worked. The Trova engine found it. Not linked on purpose.\n";
1129
+ files["Dup A.md"] = `# Retrospective
1130
+ ${DUP_TEXT}
1131
+ `;
1132
+ files["Dup B.md"] = `# Retro copy
1133
+ ${DUP_TEXT} Small addition.
1134
+ `;
1135
+ files["Sprachen/Suchmaschinen.md"] = "# \xDCber Suchmaschinen\nEine Suchmaschine verbindet Schl\xFCsselw\xF6rter und Bedeutung. Die Gr\xF6\xDFe der Abschnitte beeinflusst die Trefferqualit\xE4t.\n";
1136
+ files["Sprachen/\u65E5\u672C\u8A9E.md"] = "# \u65E5\u672C\u8A9E\u306E\u30CE\u30FC\u30C8\n\u691C\u7D22\u30A8\u30F3\u30B8\u30F3\u306F\u30AD\u30FC\u30EF\u30FC\u30C9\u3068\u610F\u5473\u3092\u7D44\u307F\u5408\u308F\u305B\u307E\u3059\u3002\u6771\u4EAC\u306E\u5929\u6C17\u306F\u6674\u308C\u3067\u3059\u3002\n";
1137
+ files["Daily/2026-11-10.md"] = "# 2026-11-10\nWorked on the keyboard remap config and read about fusion ranking.\n";
1138
+ files["Daily/2026-11-11.md"] = "# 2026-11-11\nGarden day: compost and tomato seedlings.\n";
1139
+ files["Templates/Note.md"] = "# {{title}}\nTemplate body.\n";
1140
+ files["Board.canvas"] = JSON.stringify({
1141
+ nodes: [
1142
+ { id: "1", type: "text", text: "Canvas card about periscope design", x: 0, y: 0, width: 200, height: 100 },
1143
+ { id: "2", type: "file", file: "Tools/Kanata.md", x: 300, y: 0, width: 200, height: 100 }
1144
+ ],
1145
+ edges: []
1146
+ });
1147
+ files["Papers/Fusion.pdf"] = makePdf(["Reciprocal rank fusion outperforms single rankers", "Second page discusses the xylophone benchmark"]);
1148
+ return files;
1149
+ }
1150
+ function fixtureQueries() {
1151
+ const queries = [];
1152
+ const notes = fixtureNotes();
1153
+ const paths = Object.keys(notes).filter((p) => p.endsWith(".md"));
1154
+ for (const topic of TOPICS) {
1155
+ const mine = paths.filter((p) => p.startsWith(`${topic.folder}/${topic.title} `));
1156
+ const text = (p) => String(notes[p]).toLowerCase();
1157
+ const pair = topic.terms.slice(0, 2);
1158
+ const relevantShort = {};
1159
+ for (const p of mine) if (pair.every((w) => text(p).includes(w))) relevantShort[p] = 2;
1160
+ queries.push({ id: `${topic.key}-short`, query: pair.join(" "), shape: "short", surface: "instant", relevant: relevantShort });
1161
+ const relevantLong = {};
1162
+ for (const p of mine) relevantLong[p] = 1;
1163
+ queries.push({ id: `${topic.key}-long`, query: topic.question, shape: "long", surface: "ask", relevant: relevantLong });
1164
+ const anchor = mine[0];
1165
+ const relevantRelated = {};
1166
+ for (const p of mine.slice(1)) relevantRelated[p] = 1;
1167
+ queries.push({ id: `${topic.key}-related`, query: "", shape: "long", surface: "related", anchor, relevant: relevantRelated });
1168
+ }
1169
+ queries.push({ id: "kanata-title", query: "kanata", shape: "short", surface: "instant", relevant: { "Tools/Kanata.md": 3, "Mentions.md": 1 } });
1170
+ queries.push({ id: "chunk-size", query: "what did I decide about chunk size", shape: "long", surface: "ask", relevant: { "Trova.md": 3 } });
1171
+ return queries;
1172
+ }
1173
+ function writeFixtureVault(dir) {
1174
+ const files = fixtureNotes();
1175
+ for (const [path, content] of Object.entries(files)) {
1176
+ const target = join(dir, path);
1177
+ mkdirSync(dirname2(target), { recursive: true });
1178
+ writeFileSync(target, content);
1179
+ }
1180
+ return { dir, notes: Object.keys(files), queries: fixtureQueries() };
1181
+ }
1182
+ function writeGeneratedVault(dir, count) {
1183
+ const random = rng(count);
1184
+ for (let i = 0; i < count; i++) {
1185
+ const topic = TOPICS[i % TOPICS.length];
1186
+ const path = join(dir, topic.folder, `${Math.floor(i / 500)}`, `${topic.title} ${i}.md`);
1187
+ mkdirSync(dirname2(path), { recursive: true });
1188
+ const body = `# ${topic.terms[i % topic.terms.length]} ${i}
1189
+ ${paragraph(topic, random, 6)}
1190
+
1191
+ ## Details
1192
+ ${paragraph(topic, random, 6)}
1193
+
1194
+ See also [[${topic.title} ${(i + 8) % count}]].
1195
+ `;
1196
+ writeFileSync(path, body);
1197
+ }
1198
+ }
1199
+
1200
+ // ../eval/src/index.ts
1201
+ function loadQueries(file) {
1202
+ return readFileSync(file, "utf8").split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l));
1203
+ }
1204
+ async function finalResults(stream) {
1205
+ let last = [];
1206
+ for await (const b of stream) if (b.results.length > 0 || b.final) last = b.results.length ? b.results : last;
1207
+ return last;
1208
+ }
1209
+ function rankedKeys(results, relevant) {
1210
+ const out2 = [];
1211
+ for (const r of results) {
1212
+ const key = [r.id, r.source, r.path].find((k) => k in relevant) ?? r.id;
1213
+ if (!out2.includes(key)) out2.push(key);
1214
+ }
1215
+ return out2;
1216
+ }
1217
+ async function runEval(client, queries, collection) {
1218
+ const collections2 = collection ? [collection] : void 0;
1219
+ const perQuery = [];
1220
+ for (const q of queries) {
1221
+ if (q.surface === "related") {
1222
+ if (!q.anchor) continue;
1223
+ const doc = await client.json("POST", "/v1/documents/get", { id: q.anchor, collection });
1224
+ const res = await client.json("POST", "/v1/related", { anchor: { source: doc.id }, limit: 20, collections: collections2, loose: true, granularity: "source" });
1225
+ perQuery.push({ id: q.id, surface: q.surface, shape: q.shape, metric: "recall@20", value: recallAt(rankedKeys(res.results, q.relevant), q.relevant, 20) });
1226
+ continue;
1227
+ }
1228
+ const parsed = parseQuery(q.query, { mode: q.surface === "ask" ? "ask" : "instant" }).query;
1229
+ const results = q.surface === "ask" ? await finalResults(client.stream("/v1/ask", { query: parsed, mode: "auto", collections: collections2, limit: 20 })) : await finalResults(client.stream("/v1/search", { query: parsed, lanes: ["lex"], collections: collections2, limit: 20 }));
1230
+ perQuery.push({ id: q.id, surface: q.surface, shape: q.shape, metric: "ndcg@10", value: ndcg(rankedKeys(results, q.relevant), q.relevant, 10) });
1231
+ }
1232
+ const groups = /* @__PURE__ */ new Map();
1233
+ for (const p of perQuery) {
1234
+ const key = `${p.surface} / ${p.shape}`;
1235
+ const g = groups.get(key) ?? { metric: p.metric, values: [] };
1236
+ g.values.push(p.value);
1237
+ groups.set(key, g);
1238
+ }
1239
+ const rows = [...groups.entries()].sort().map(([group, g]) => ({
1240
+ group,
1241
+ metric: g.metric,
1242
+ queries: g.values.length,
1243
+ value: g.values.reduce((a, b) => a + b, 0) / g.values.length
1244
+ }));
1245
+ return { rows, perQuery };
1246
+ }
1247
+ function percentile(sorted, p) {
1248
+ return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] ?? 0;
1249
+ }
1250
+ async function time(runs, fn) {
1251
+ const times = [];
1252
+ for (let i = 0; i < runs; i++) {
1253
+ const start = performance.now();
1254
+ await fn(i);
1255
+ times.push(performance.now() - start);
1256
+ }
1257
+ times.sort((a, b) => a - b);
1258
+ return { p50: percentile(times, 0.5), p95: percentile(times, 0.95) };
1259
+ }
1260
+ async function drain(stream) {
1261
+ for await (const _ of stream) ;
1262
+ }
1263
+ async function runPerf(client, collection, anchors, terms, runs = 60) {
1264
+ const collections2 = [collection];
1265
+ const word = (i) => terms[i % terms.length];
1266
+ const q = (raw, mode = "instant") => parseQuery(raw, { mode }).query;
1267
+ const rows = [];
1268
+ const add = (interaction, t, targetP50, targetP95) => rows.push({ interaction, ...t, targetP50, targetP95, runs });
1269
+ add("Lex lane (engine part of keystroke to first rows)", await time(runs, (i) => drain(client.stream("/v1/search", { query: q(`${word(i)} ${word(i + 3)}`), lanes: ["lex"], collections: collections2, limit: 20, fuzzy: false }))), 15, 30);
1270
+ add("Lex lane with typo rescue", await time(runs, (i) => drain(client.stream("/v1/search", { query: q(`${word(i).slice(0, -1)}x`), lanes: ["lex"], collections: collections2, limit: 20, fuzzy: true }))), 25, 55);
1271
+ add("Vec lane (semantic rows)", await time(runs, (i) => drain(client.stream("/v1/search", { query: q(`${word(i)} ${word(i + 1)} ${word(i + 2)}`), lanes: ["vec"], collections: collections2, limit: 5 }))), 120, 250);
1272
+ const ids = [];
1273
+ for (const a of anchors) ids.push((await client.json("POST", "/v1/documents/get", { id: a, collection })).id);
1274
+ add("Connections, uncached", await time(runs, (i) => client.json("POST", "/v1/related", { anchor: { source: ids[i % ids.length] }, limit: 20, collections: collections2 })), 150, 400);
1275
+ add("Ask, fast", await time(runs, (i) => drain(client.stream("/v1/ask", { query: q(`${word(i)} ${word(i + 2)}`, "ask"), mode: "fast", collections: collections2 }))), 150, 350);
1276
+ return rows;
1277
+ }
1278
+
1279
+ // src/client.ts
1280
+ import { spawn } from "node:child_process";
1281
+
1282
+ // src/engine.ts
1283
+ import { rmSync as rmSync3, statSync as statSync3, watch } from "node:fs";
1284
+ import { join as join5 } from "node:path";
1285
+ import picomatch from "picomatch";
1286
+
1287
+ // src/sidecar.ts
1288
+ import Database from "better-sqlite3";
1289
+ import { parse as parseYaml } from "yaml";
1290
+ var ATT_SUFFIX = "-att";
1291
+ function publicPath(qmdCollection, qmdPath) {
1292
+ return qmdCollection.endsWith(ATT_SUFFIX) ? qmdPath.replace(/\.md$/, "") : qmdPath;
1293
+ }
1294
+ function parseFrontmatter(text) {
1295
+ const raw = frontmatterText(text);
1296
+ if (!raw) return {};
1297
+ try {
1298
+ const value = parseYaml(raw);
1299
+ return value && typeof value === "object" && !Array.isArray(value) ? value : {};
1300
+ } catch {
1301
+ return {};
1302
+ }
1303
+ }
1304
+ function buildMeta(qmdCollection, qmdPath, hash2, text, times) {
1305
+ const path = publicPath(qmdCollection, qmdPath);
1306
+ const isNote = extname(path) === "md";
1307
+ const fm = isNote ? parseFrontmatter(text) : {};
1308
+ const tags = /* @__PURE__ */ new Set([...normaliseTags(fm["tags"]), ...normaliseTags(fm["tag"]), ...isNote ? extractInlineTags(text) : []]);
1309
+ const fmExcluded = isExcludedByFrontmatter(fm);
1310
+ return {
1311
+ path,
1312
+ title: titleFromPath(path),
1313
+ aliases: [...normaliseAliases(fm["aliases"]), ...normaliseAliases(fm["alias"])],
1314
+ tags: [...tags],
1315
+ links: [],
1316
+ rawLinks: isNote ? [...new Set(extractLinks(text).map((l) => l.target))] : [],
1317
+ frontmatter: fm,
1318
+ created: times.created,
1319
+ modified: times.modified,
1320
+ wordCount: countWords(text.slice(frontmatterEnd(text))),
1321
+ excluded: fmExcluded,
1322
+ fmExcluded,
1323
+ hash: hash2,
1324
+ qmdCollection,
1325
+ qmdPath,
1326
+ oversized: Buffer.byteLength(text) > MAX_TEXT_BYTES
1327
+ };
1328
+ }
1329
+ var SCHEMA = `
1330
+ CREATE TABLE IF NOT EXISTS trova_meta (
1331
+ source_id TEXT PRIMARY KEY, collection TEXT NOT NULL, hash TEXT NOT NULL, title TEXT NOT NULL,
1332
+ aliases TEXT NOT NULL, tags TEXT NOT NULL, links TEXT NOT NULL, frontmatter TEXT NOT NULL,
1333
+ created INTEGER NOT NULL, modified INTEGER NOT NULL, word_count INTEGER NOT NULL, excluded INTEGER NOT NULL,
1334
+ qmd_collection TEXT NOT NULL, qmd_path TEXT NOT NULL, oversized INTEGER NOT NULL DEFAULT 0
1335
+ );
1336
+ CREATE INDEX IF NOT EXISTS idx_trova_meta_collection ON trova_meta(collection);
1337
+ CREATE TABLE IF NOT EXISTS trova_blocks (
1338
+ block_id TEXT PRIMARY KEY, source_id TEXT NOT NULL, heading_path TEXT NOT NULL, headings TEXT NOT NULL,
1339
+ level INTEGER NOT NULL, start INTEGER NOT NULL, end INTEGER NOT NULL
1340
+ );
1341
+ CREATE INDEX IF NOT EXISTS idx_trova_blocks_source ON trova_blocks(source_id, start);
1342
+ CREATE TABLE IF NOT EXISTS trova_paths (old_id TEXT NOT NULL, new_id TEXT NOT NULL, at INTEGER NOT NULL);
1343
+ CREATE INDEX IF NOT EXISTS idx_trova_paths_old ON trova_paths(old_id);
1344
+ CREATE TABLE IF NOT EXISTS trova_jobs (
1345
+ job_id TEXT PRIMARY KEY, kind TEXT NOT NULL, state TEXT NOT NULL, done INTEGER NOT NULL, total INTEGER NOT NULL,
1346
+ error TEXT, started INTEGER, finished INTEGER
1347
+ );
1348
+ CREATE TABLE IF NOT EXISTS trova_collections (name TEXT PRIMARY KEY, path TEXT NOT NULL, options TEXT NOT NULL, last_indexed INTEGER);
1349
+ CREATE TABLE IF NOT EXISTS trova_attachments (
1350
+ collection TEXT NOT NULL, path TEXT NOT NULL, mtime INTEGER NOT NULL, size INTEGER NOT NULL, pushed INTEGER NOT NULL DEFAULT 0,
1351
+ PRIMARY KEY (collection, path)
1352
+ );
1353
+ CREATE TABLE IF NOT EXISTS trova_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL);
1354
+ `;
1355
+ var Sidecar = class {
1356
+ db;
1357
+ metas = /* @__PURE__ */ new Map();
1358
+ graphs = /* @__PURE__ */ new Map();
1359
+ options = /* @__PURE__ */ new Map();
1360
+ constructor(dbPath) {
1361
+ this.db = new Database(dbPath);
1362
+ this.db.pragma("journal_mode = WAL");
1363
+ this.db.pragma("busy_timeout = 5000");
1364
+ this.db.exec(SCHEMA);
1365
+ for (const row of this.db.prepare(`SELECT name, path, options, last_indexed FROM trova_collections`).all()) {
1366
+ this.options.set(row.name, { path: row.path, options: JSON.parse(row.options), lastIndexed: row.last_indexed ?? void 0 });
1367
+ }
1368
+ this.db.prepare(`UPDATE trova_jobs SET state = 'failed', error = 'engine stopped' WHERE state IN ('running', 'queued', 'paused')`).run();
1369
+ }
1370
+ close() {
1371
+ this.db.close();
1372
+ }
1373
+ // ---------- collections ----------
1374
+ collectionNames() {
1375
+ return [...this.options.keys()];
1376
+ }
1377
+ collection(name) {
1378
+ return this.options.get(name);
1379
+ }
1380
+ saveCollection(name, path, options) {
1381
+ const prev = this.options.get(name);
1382
+ this.options.set(name, { path, options, lastIndexed: prev?.lastIndexed });
1383
+ this.db.prepare(`INSERT INTO trova_collections (name, path, options) VALUES (?, ?, ?) ON CONFLICT(name) DO UPDATE SET path = excluded.path, options = excluded.options`).run(name, path, JSON.stringify(options));
1384
+ this.applyScope(name);
1385
+ }
1386
+ markIndexed(name, at) {
1387
+ const c = this.options.get(name);
1388
+ if (c) c.lastIndexed = at;
1389
+ this.db.prepare(`UPDATE trova_collections SET last_indexed = ? WHERE name = ?`).run(at, name);
1390
+ }
1391
+ removeCollection(name) {
1392
+ this.options.delete(name);
1393
+ this.metas.delete(name);
1394
+ this.graphs.delete(name);
1395
+ const prefix = `${name}:`;
1396
+ this.db.prepare(`DELETE FROM trova_collections WHERE name = ?`).run(name);
1397
+ this.db.prepare(`DELETE FROM trova_blocks WHERE substr(source_id, 1, ?) = ?`).run(prefix.length, prefix);
1398
+ this.db.prepare(`DELETE FROM trova_meta WHERE collection = ?`).run(name);
1399
+ this.db.prepare(`DELETE FROM trova_attachments WHERE collection = ?`).run(name);
1400
+ }
1401
+ // ---------- metadata ----------
1402
+ applyScope(collection) {
1403
+ const opts = this.options.get(collection)?.options;
1404
+ const map = this.metas.get(collection);
1405
+ if (!map) return;
1406
+ for (const e of map.values()) e.excluded = e.fmExcluded || !inScope(e.path, opts?.include, opts?.exclude);
1407
+ }
1408
+ entries(collection) {
1409
+ let map = this.metas.get(collection);
1410
+ if (map) return map;
1411
+ map = /* @__PURE__ */ new Map();
1412
+ const rows = this.db.prepare(`SELECT * FROM trova_meta WHERE collection = ?`).all(collection);
1413
+ for (const r of rows) {
1414
+ const path = r.source_id.slice(collection.length + 1);
1415
+ map.set(path, {
1416
+ path,
1417
+ title: r.title,
1418
+ aliases: JSON.parse(r.aliases),
1419
+ tags: JSON.parse(r.tags),
1420
+ links: [],
1421
+ rawLinks: JSON.parse(r.links),
1422
+ frontmatter: JSON.parse(r.frontmatter),
1423
+ created: r.created,
1424
+ modified: r.modified,
1425
+ wordCount: r.word_count,
1426
+ excluded: r.excluded === 1,
1427
+ fmExcluded: r.excluded === 1,
1428
+ hash: r.hash,
1429
+ qmdCollection: r.qmd_collection,
1430
+ qmdPath: r.qmd_path,
1431
+ oversized: r.oversized === 1
1432
+ });
1433
+ }
1434
+ this.metas.set(collection, map);
1435
+ this.applyScope(collection);
1436
+ return map;
1437
+ }
1438
+ entry(collection, path) {
1439
+ return this.entries(collection).get(path);
1440
+ }
1441
+ /** Writes changed sources and removes deleted ones in one transaction. */
1442
+ apply(collection, upserts, removals) {
1443
+ const map = this.entries(collection);
1444
+ const putMeta = this.db.prepare(
1445
+ `INSERT OR REPLACE INTO trova_meta (source_id, collection, hash, title, aliases, tags, links, frontmatter, created, modified, word_count, excluded, qmd_collection, qmd_path, oversized)
1446
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
1447
+ );
1448
+ const delBlocks = this.db.prepare(`DELETE FROM trova_blocks WHERE source_id = ?`);
1449
+ const putBlock = this.db.prepare(
1450
+ `INSERT OR REPLACE INTO trova_blocks (block_id, source_id, heading_path, headings, level, start, end) VALUES (?, ?, ?, ?, ?, ?, ?)`
1451
+ );
1452
+ const delMeta = this.db.prepare(`DELETE FROM trova_meta WHERE source_id = ?`);
1453
+ this.db.transaction(() => {
1454
+ for (const { entry, blocks } of upserts) {
1455
+ const id = `${collection}:${entry.path}`;
1456
+ putMeta.run(
1457
+ id,
1458
+ collection,
1459
+ entry.hash,
1460
+ entry.title,
1461
+ JSON.stringify(entry.aliases),
1462
+ JSON.stringify(entry.tags),
1463
+ JSON.stringify(entry.rawLinks),
1464
+ JSON.stringify(entry.frontmatter),
1465
+ Math.round(entry.created),
1466
+ Math.round(entry.modified),
1467
+ entry.wordCount,
1468
+ entry.fmExcluded ? 1 : 0,
1469
+ entry.qmdCollection,
1470
+ entry.qmdPath,
1471
+ entry.oversized ? 1 : 0
1472
+ );
1473
+ delBlocks.run(id);
1474
+ for (const b of blocks) putBlock.run(`${id}#${b.headingPath}`, id, b.headingPath, JSON.stringify(b.headings), b.level, b.start, b.end);
1475
+ map.set(entry.path, entry);
1476
+ }
1477
+ for (const path of removals) {
1478
+ const id = `${collection}:${path}`;
1479
+ delMeta.run(id);
1480
+ delBlocks.run(id);
1481
+ map.delete(path);
1482
+ }
1483
+ })();
1484
+ this.graphs.delete(collection);
1485
+ this.applyScope(collection);
1486
+ }
1487
+ blocks(source) {
1488
+ const rows = this.db.prepare(`SELECT heading_path, headings, level, start, end FROM trova_blocks WHERE source_id = ? ORDER BY start`).all(source);
1489
+ return rows.map((r) => ({ headingPath: r.heading_path, headings: JSON.parse(r.headings), level: r.level, start: r.start, end: r.end }));
1490
+ }
1491
+ /** The link graph of a collection. Resolved links are written onto each entry. */
1492
+ graph(collection) {
1493
+ let g = this.graphs.get(collection);
1494
+ if (g) return g;
1495
+ const map = this.entries(collection);
1496
+ const resolve3 = createLinkResolver(map.keys());
1497
+ const inbound = /* @__PURE__ */ new Map();
1498
+ for (const e of map.values()) {
1499
+ const links = /* @__PURE__ */ new Set();
1500
+ for (const raw of e.rawLinks) {
1501
+ const target = resolve3(raw, e.path);
1502
+ if (target && target !== e.path) links.add(target);
1503
+ }
1504
+ e.links = [...links];
1505
+ for (const l of links) inbound.set(l, (inbound.get(l) ?? 0) + 1);
1506
+ }
1507
+ g = { resolve: resolve3, inbound };
1508
+ this.graphs.set(collection, g);
1509
+ return g;
1510
+ }
1511
+ isOrphan(collection, path) {
1512
+ const g = this.graph(collection);
1513
+ const e = this.entry(collection, path);
1514
+ return !!e && e.links.length === 0 && !g.inbound.get(path);
1515
+ }
1516
+ // ---------- renames ----------
1517
+ recordRename(oldId, newId) {
1518
+ this.db.prepare(`INSERT INTO trova_paths (old_id, new_id, at) VALUES (?, ?, ?)`).run(oldId, newId, Date.now());
1519
+ }
1520
+ /** Follows the rename history from an old source id to its current id. */
1521
+ currentId(sourceId2) {
1522
+ let id = sourceId2;
1523
+ const stmt = this.db.prepare(`SELECT new_id FROM trova_paths WHERE old_id = ? ORDER BY at DESC, rowid DESC LIMIT 1`);
1524
+ for (let i = 0; i < 20; i++) {
1525
+ const row = stmt.get(id);
1526
+ if (!row || row.new_id === id) break;
1527
+ id = row.new_id;
1528
+ }
1529
+ return id;
1530
+ }
1531
+ renameHistory() {
1532
+ return this.db.prepare(`SELECT old_id, new_id, at FROM trova_paths ORDER BY at`).all();
1533
+ }
1534
+ // ---------- jobs ----------
1535
+ saveJob(job) {
1536
+ this.db.prepare(`INSERT OR REPLACE INTO trova_jobs (job_id, kind, state, done, total, error, started, finished) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(job.jobId, job.kind, job.state, job.done, job.total, job.error ?? null, job.started ?? null, job.finished ?? null);
1537
+ }
1538
+ // ---------- attachments ----------
1539
+ attachmentManifest(collection) {
1540
+ const rows = this.db.prepare(`SELECT path, mtime, size, pushed FROM trova_attachments WHERE collection = ?`).all(collection);
1541
+ return new Map(rows.map((r) => [r.path, { mtime: r.mtime, size: r.size, pushed: r.pushed === 1 }]));
1542
+ }
1543
+ putAttachment(collection, path, mtime, size, pushed) {
1544
+ this.db.prepare(`INSERT OR REPLACE INTO trova_attachments (collection, path, mtime, size, pushed) VALUES (?, ?, ?, ?, ?)`).run(collection, path, Math.round(mtime), size, pushed ? 1 : 0);
1545
+ }
1546
+ removeAttachment(collection, path) {
1547
+ this.db.prepare(`DELETE FROM trova_attachments WHERE collection = ? AND path = ?`).run(collection, path);
1548
+ }
1549
+ // ---------- key-value ----------
1550
+ getValue(key) {
1551
+ const row = this.db.prepare(`SELECT value FROM trova_kv WHERE key = ?`).get(key);
1552
+ return row ? JSON.parse(row.value) : null;
1553
+ }
1554
+ setValue(key, value) {
1555
+ this.db.prepare(`INSERT OR REPLACE INTO trova_kv (key, value) VALUES (?, ?)`).run(key, JSON.stringify(value));
1556
+ }
1557
+ };
1558
+
1559
+ // src/analysis.ts
1560
+ function docIndex(input) {
1561
+ const byHash = /* @__PURE__ */ new Map();
1562
+ const docs = [];
1563
+ for (const c of input.collections) {
1564
+ for (const entry of c.entries.values()) {
1565
+ if (entry.excluded) continue;
1566
+ const doc = { collection: c.name, entry, source: sourceId(c.name, entry.path) };
1567
+ docs.push(doc);
1568
+ const list = byHash.get(entry.hash);
1569
+ if (list) list.push(doc);
1570
+ else byHash.set(entry.hash, [doc]);
1571
+ }
1572
+ }
1573
+ return { byHash, docs };
1574
+ }
1575
+ function chunksOverlapping(chunks, block, length) {
1576
+ const hit = chunks.filter((c, i) => {
1577
+ const end = i + 1 < chunks.length ? chunks[i + 1].pos : length;
1578
+ return c.pos < block.end && Math.max(end, c.pos + 1) > block.start;
1579
+ });
1580
+ return hit.length ? hit : chunks.slice(-1);
1581
+ }
1582
+ async function findDuplicates(input, ctx) {
1583
+ const { reader } = input;
1584
+ const { byHash, docs } = docIndex(input);
1585
+ const blocksOf = (doc) => input.collections.find((c) => c.name === doc.collection).blocks(doc.source);
1586
+ const pairs = /* @__PURE__ */ new Map();
1587
+ const sourcePairs = /* @__PURE__ */ new Set();
1588
+ const pairKey = (a, b) => a < b ? `${a}\0${b}` : `${b}\0${a}`;
1589
+ const noteVecs = /* @__PURE__ */ new Map();
1590
+ const noteVec = (hash2) => {
1591
+ if (!noteVecs.has(hash2)) {
1592
+ const keys = reader.chunks(hash2).map((c) => `${c.hash}_${c.seq}`);
1593
+ noteVecs.set(hash2, meanVector([...reader.vectors(keys).values()]));
1594
+ if (noteVecs.size > 2e3) noteVecs.delete(noteVecs.keys().next().value);
1595
+ }
1596
+ return noteVecs.get(hash2) ?? null;
1597
+ };
1598
+ const addPair = (a, b, kind, similarity, aId = a.source, bId = b.source) => {
1599
+ const key = pairKey(aId, bId);
1600
+ if (pairs.has(key)) return;
1601
+ pairs.set(key, { a: aId, b: bId, aTitle: a.entry.title, bTitle: b.entry.title, aPath: a.entry.path, bPath: b.entry.path, kind, similarity });
1602
+ };
1603
+ for (const list of byHash.values()) {
1604
+ for (let i = 0; i < list.length; i++) {
1605
+ for (let j = i + 1; j < list.length; j++) {
1606
+ addPair(list[i], list[j], "source", 1);
1607
+ sourcePairs.add(pairKey(list[i].source, list[j].source));
1608
+ }
1609
+ }
1610
+ }
1611
+ const seenHash = /* @__PURE__ */ new Set();
1612
+ let done = 0;
1613
+ for (const doc of docs) {
1614
+ ctx.progress(++done, docs.length, { stage: "duplicates" });
1615
+ if (ctx.cancelled()) return [];
1616
+ if (seenHash.has(doc.entry.hash)) continue;
1617
+ seenHash.add(doc.entry.hash);
1618
+ await ctx.pause();
1619
+ const chunks = reader.chunks(doc.entry.hash);
1620
+ if (chunks.length === 0) continue;
1621
+ const text = reader.content(doc.entry.hash) ?? "";
1622
+ const vectors = reader.vectors(chunks.map((c) => `${c.hash}_${c.seq}`));
1623
+ const whole = meanVector([...vectors.values()]);
1624
+ if (!whole) continue;
1625
+ for (const n of reader.knn(whole, 8)) {
1626
+ if (n.hash === doc.entry.hash || n.similarity < DUPLICATE_SOURCE_COSINE - 0.1) continue;
1627
+ const other = noteVec(n.hash);
1628
+ if (!other) continue;
1629
+ const sim = cosine(whole, other);
1630
+ if (sim < DUPLICATE_SOURCE_COSINE) continue;
1631
+ for (const o of byHash.get(n.hash) ?? []) {
1632
+ addPair(doc, o, "source", sim);
1633
+ sourcePairs.add(pairKey(doc.source, o.source));
1634
+ }
1635
+ }
1636
+ const signatures = /* @__PURE__ */ new Set();
1637
+ for (const block of blocksOf(doc)) {
1638
+ if (countWords(blockText(text, block)) < DUPLICATE_BLOCK_MIN_WORDS) continue;
1639
+ const own = chunksOverlapping(chunks, block, text.length);
1640
+ const signature = own.map((c) => c.seq).join(",");
1641
+ if (signatures.has(signature)) continue;
1642
+ signatures.add(signature);
1643
+ const vec = meanVector(own.map((c) => vectors.get(`${c.hash}_${c.seq}`)).filter((v) => !!v));
1644
+ if (!vec) continue;
1645
+ for (const n of reader.knn(vec, 8)) {
1646
+ if (n.hash === doc.entry.hash || n.similarity < DUPLICATE_BLOCK_COSINE - 0.1) continue;
1647
+ for (const o of byHash.get(n.hash) ?? []) {
1648
+ if (sourcePairs.has(pairKey(doc.source, o.source))) continue;
1649
+ const oChunks = reader.chunks(n.hash);
1650
+ const pos = oChunks.find((c) => c.seq === n.seq)?.pos ?? 0;
1651
+ const oText = reader.content(n.hash) ?? "";
1652
+ const oBlock = blockAt(blocksOf(o), pos);
1653
+ if (!oBlock || countWords(blockText(oText, oBlock)) < DUPLICATE_BLOCK_MIN_WORDS) continue;
1654
+ const oOwn = chunksOverlapping(oChunks, oBlock, oText.length);
1655
+ const oVec = meanVector([...reader.vectors(oOwn.map((c) => `${c.hash}_${c.seq}`)).values()]);
1656
+ if (!oVec) continue;
1657
+ const sim = cosine(vec, oVec);
1658
+ if (sim >= DUPLICATE_BLOCK_COSINE) addPair(doc, o, "block", sim, blockId(doc.source, block.headingPath), blockId(o.source, oBlock.headingPath));
1659
+ }
1660
+ }
1661
+ }
1662
+ }
1663
+ return [...pairs.values()].sort((a, b) => b.similarity - a.similarity);
1664
+ }
1665
+ function rng2(seed) {
1666
+ let s = seed >>> 0;
1667
+ return () => {
1668
+ s = s + 1831565813 >>> 0;
1669
+ let t = Math.imul(s ^ s >>> 15, 1 | s);
1670
+ t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
1671
+ return ((t ^ t >>> 14) >>> 0) / 4294967296;
1672
+ };
1673
+ }
1674
+ function dot(a, b) {
1675
+ let s = 0;
1676
+ for (let i = 0; i < a.length; i++) s += a[i] * b[i];
1677
+ return s;
1678
+ }
1679
+ var TOPIC_STOP = new Set("the and for with from that this notes note about into your what when how why are was not but all new one two use using".split(" "));
1680
+ async function findTopics(input, ctx) {
1681
+ const { reader } = input;
1682
+ const { docs } = docIndex(input);
1683
+ const items = [];
1684
+ let done = 0;
1685
+ for (const doc of docs) {
1686
+ if (ctx.cancelled()) return [];
1687
+ if (++done % 200 === 0) {
1688
+ ctx.progress(done, docs.length * 2, { stage: "topics" });
1689
+ await ctx.pause();
1690
+ }
1691
+ const keys = reader.chunks(doc.entry.hash).map((c) => `${c.hash}_${c.seq}`);
1692
+ const vec = meanVector([...reader.vectors(keys).values()]);
1693
+ if (vec) items.push({ doc, vec });
1694
+ }
1695
+ const n = items.length;
1696
+ if (n < 2) return [];
1697
+ const dims = items[0].vec.length;
1698
+ const k = Math.max(2, Math.min(24, Math.round(Math.sqrt(n / 2))));
1699
+ const random = rng2(n * 31 + dims);
1700
+ const centroids = [items[Math.floor(random() * n)].vec.slice()];
1701
+ const dist = new Float64Array(n).fill(Infinity);
1702
+ while (centroids.length < k) {
1703
+ const last = centroids[centroids.length - 1];
1704
+ let total = 0;
1705
+ for (let i = 0; i < n; i++) {
1706
+ dist[i] = Math.min(dist[i], Math.max(0, 1 - dot(items[i].vec, last)));
1707
+ total += dist[i];
1708
+ }
1709
+ let pick = random() * total;
1710
+ let chosen = n - 1;
1711
+ for (let i = 0; i < n; i++) {
1712
+ pick -= dist[i];
1713
+ if (pick <= 0) {
1714
+ chosen = i;
1715
+ break;
1716
+ }
1717
+ }
1718
+ centroids.push(items[chosen].vec.slice());
1719
+ }
1720
+ const assign = new Int32Array(n);
1721
+ for (let iter = 0; iter < 12; iter++) {
1722
+ if (ctx.cancelled()) return [];
1723
+ await ctx.pause();
1724
+ let moved = 0;
1725
+ for (let i = 0; i < n; i++) {
1726
+ let best = 0;
1727
+ let bestSim = -Infinity;
1728
+ for (let c = 0; c < k; c++) {
1729
+ const sim = dot(items[i].vec, centroids[c]);
1730
+ if (sim > bestSim) {
1731
+ bestSim = sim;
1732
+ best = c;
1733
+ }
1734
+ }
1735
+ if (assign[i] !== best) moved++;
1736
+ assign[i] = best;
1737
+ }
1738
+ for (let c = 0; c < k; c++) {
1739
+ const members = items.filter((_, i) => assign[i] === c).map((it) => it.vec);
1740
+ const mean2 = meanVector(members);
1741
+ if (mean2) centroids[c] = mean2;
1742
+ }
1743
+ ctx.progress(docs.length + Math.round((iter + 1) / 12 * docs.length), docs.length * 2, { stage: "topics" });
1744
+ if (moved === 0 && iter > 0) break;
1745
+ }
1746
+ const mean = new Float32Array(dims);
1747
+ for (const it of items) for (let d = 0; d < dims; d++) mean[d] += it.vec[d] / n;
1748
+ const component = (ortho) => {
1749
+ let v = new Float32Array(dims);
1750
+ for (let d = 0; d < dims; d++) v[d] = random() - 0.5;
1751
+ for (let iter = 0; iter < 20; iter++) {
1752
+ const next = new Float32Array(dims);
1753
+ for (const it of items) {
1754
+ let proj = 0;
1755
+ for (let d = 0; d < dims; d++) proj += (it.vec[d] - mean[d]) * v[d];
1756
+ for (let d = 0; d < dims; d++) next[d] += proj * (it.vec[d] - mean[d]);
1757
+ }
1758
+ if (ortho) {
1759
+ const o = dot(next, ortho);
1760
+ for (let d = 0; d < dims; d++) next[d] -= o * ortho[d];
1761
+ }
1762
+ const norm = Math.sqrt(dot(next, next)) || 1;
1763
+ for (let d = 0; d < dims; d++) next[d] /= norm;
1764
+ v = next;
1765
+ }
1766
+ return v;
1767
+ };
1768
+ const pc1 = component(null);
1769
+ await ctx.pause();
1770
+ const pc2 = component(pc1);
1771
+ const project = (vec) => {
1772
+ let x = 0;
1773
+ let y = 0;
1774
+ for (let d = 0; d < dims; d++) {
1775
+ const centred = vec[d] - mean[d];
1776
+ x += centred * pc1[d];
1777
+ y += centred * pc2[d];
1778
+ }
1779
+ return [x, y];
1780
+ };
1781
+ const points = items.map((it) => project(it.vec));
1782
+ const xs = points.map((p) => p[0]);
1783
+ const ys = points.map((p) => p[1]);
1784
+ const [minX, maxX, minY, maxY] = [Math.min(...xs), Math.max(...xs), Math.min(...ys), Math.max(...ys)];
1785
+ const scale = (v, lo, hi) => hi === lo ? 0.5 : (v - lo) / (hi - lo);
1786
+ const termsOf = (it) => {
1787
+ const c = input.collections.find((col) => col.name === it.doc.collection);
1788
+ const headings = c.blocks(it.doc.source).flatMap((b) => b.headings.slice(-1));
1789
+ const text = [it.doc.entry.title, publicPath(it.doc.entry.qmdCollection, it.doc.entry.qmdPath).split("/").slice(0, -1).join(" "), ...headings, ...it.doc.entry.tags].join(" ");
1790
+ return [...new Set((text.toLowerCase().match(/[\p{L}][\p{L}\p{N}]{2,}/gu) ?? []).filter((w) => !TOPIC_STOP.has(w)))];
1791
+ };
1792
+ const tf = Array.from({ length: k }, () => /* @__PURE__ */ new Map());
1793
+ const df = /* @__PURE__ */ new Map();
1794
+ items.forEach((it, i) => {
1795
+ for (const t of termsOf(it)) {
1796
+ const m = tf[assign[i]];
1797
+ if (!m.has(t)) df.set(t, (df.get(t) ?? 0) + 1);
1798
+ m.set(t, (m.get(t) ?? 0) + 1);
1799
+ }
1800
+ });
1801
+ const clusters = [];
1802
+ for (let c = 0; c < k; c++) {
1803
+ const members = items.map((it, i) => ({ it, i })).filter(({ i }) => assign[i] === c);
1804
+ if (members.length === 0) continue;
1805
+ const label = [...tf[c].entries()].map(([term, count]) => ({ term, score: count * Math.log(1 + k / (df.get(term) ?? 1)) })).sort((a, b) => b.score - a.score || (a.term < b.term ? -1 : 1)).slice(0, 3).map((t) => t.term);
1806
+ const sources = members.map(({ it, i }) => ({
1807
+ id: it.doc.source,
1808
+ path: it.doc.entry.path,
1809
+ title: it.doc.entry.title,
1810
+ x: scale(points[i][0], minX, maxX),
1811
+ y: scale(points[i][1], minY, maxY)
1812
+ }));
1813
+ clusters.push({
1814
+ id: clusters.length,
1815
+ label,
1816
+ x: sources.reduce((s, p) => s + p.x, 0) / sources.length,
1817
+ y: sources.reduce((s, p) => s + p.y, 0) / sources.length,
1818
+ sources
1819
+ });
1820
+ }
1821
+ return clusters.sort((a, b) => b.sources.length - a.sources.length).map((c, i) => ({ ...c, id: i }));
1822
+ }
1823
+
1824
+ // src/attachments.ts
1825
+ import { existsSync, mkdirSync as mkdirSync2, readdirSync, readFileSync as readFileSync2, rmSync, statSync, writeFileSync as writeFileSync2 } from "node:fs";
1826
+ import { dirname as dirname3, join as join2, relative, resolve } from "node:path";
1827
+ var DEFAULT_ATTACHMENTS = ["pdf", "canvas", "image"];
1828
+ var TYPE_EXTS = {
1829
+ pdf: ["pdf"],
1830
+ canvas: ["canvas"],
1831
+ office: ["docx", "pptx", "xlsx"],
1832
+ image: ["png", "jpg", "jpeg", "webp", "gif", "bmp"]
1833
+ };
1834
+ var SKIP_DIRS = /* @__PURE__ */ new Set(["node_modules", ".git", ".obsidian", ".trash"]);
1835
+ function attachmentType(path) {
1836
+ const ext = extname(path);
1837
+ for (const [type, exts] of Object.entries(TYPE_EXTS)) if (exts.includes(ext)) return type;
1838
+ return null;
1839
+ }
1840
+ function truncate(text) {
1841
+ if (Buffer.byteLength(text) <= MAX_TEXT_BYTES) return text;
1842
+ return Buffer.from(text).subarray(0, MAX_TEXT_BYTES).toString("utf8");
1843
+ }
1844
+ function decodeXml(text) {
1845
+ return text.replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))).replace(/&amp;/g, "&");
1846
+ }
1847
+ function xmlTexts(xml, tag) {
1848
+ const re = new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)</${tag}>`, "g");
1849
+ const out2 = [];
1850
+ let m;
1851
+ while (m = re.exec(xml)) out2.push(decodeXml(m[1]));
1852
+ return out2;
1853
+ }
1854
+ async function extractPdf(data) {
1855
+ const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
1856
+ const task = pdfjs.getDocument({ data, useSystemFonts: true, disableFontFace: true, isEvalSupported: false, verbosity: 0 });
1857
+ const doc = await task.promise;
1858
+ const pages = [];
1859
+ let bytes = 0;
1860
+ try {
1861
+ for (let n = 1; n <= doc.numPages && bytes < MAX_TEXT_BYTES; n++) {
1862
+ const content = await (await doc.getPage(n)).getTextContent();
1863
+ const text = content.items.map((it) => (it.str ?? "") + (it.hasEOL ? "\n" : " ")).join("").replace(/[ \t]+/g, " ").trim();
1864
+ if (!text) continue;
1865
+ pages.push(`## Page ${n}
1866
+
1867
+ ${text}`);
1868
+ bytes += text.length;
1869
+ }
1870
+ } finally {
1871
+ await task.destroy().catch(() => void 0);
1872
+ }
1873
+ return pages.join("\n\n");
1874
+ }
1875
+ function extractCanvas(json) {
1876
+ let data;
1877
+ try {
1878
+ data = JSON.parse(json);
1879
+ } catch {
1880
+ return "";
1881
+ }
1882
+ const cards = [];
1883
+ const files = [];
1884
+ for (const node of data.nodes ?? []) {
1885
+ if (node.type === "text" && node.text) cards.push(node.text.trim());
1886
+ else if (node.type === "file" && node.file) files.push(node.file.split("/").pop());
1887
+ else if (node.type === "link" && node.url) cards.push(node.url);
1888
+ else if (node.type === "group" && node.label) cards.push(node.label);
1889
+ }
1890
+ const parts = [...cards];
1891
+ if (files.length) parts.push(`## Files
1892
+
1893
+ ${files.map((f) => `- ${f}`).join("\n")}`);
1894
+ return parts.join("\n\n");
1895
+ }
1896
+ async function extractOffice(data, ext) {
1897
+ const { unzipSync, strFromU8 } = await import("fflate");
1898
+ const files = unzipSync(data, {
1899
+ filter: (f) => /^(word\/document\.xml|ppt\/slides\/slide\d+\.xml|xl\/sharedStrings\.xml)$/.test(f.name)
1900
+ });
1901
+ if (ext === "docx") {
1902
+ const xml = files["word/document.xml"];
1903
+ if (!xml) return "";
1904
+ return xmlTexts(strFromU8(xml), "w:p").map((p) => xmlTexts(p, "w:t").join("")).filter((p) => p.trim()).join("\n\n");
1905
+ }
1906
+ if (ext === "pptx") {
1907
+ return Object.keys(files).filter((n) => n.startsWith("ppt/slides/")).sort((a, b) => Number(/\d+/.exec(a)[0]) - Number(/\d+/.exec(b)[0])).map((name, i) => `## Slide ${i + 1}
1908
+
1909
+ ${xmlTexts(strFromU8(files[name]), "a:t").join("\n")}`).join("\n\n");
1910
+ }
1911
+ const shared = files["xl/sharedStrings.xml"];
1912
+ return shared ? xmlTexts(strFromU8(shared), "t").join("\n") : "";
1913
+ }
1914
+ async function extract(file, ext) {
1915
+ if (ext === "canvas") return extractCanvas(readFileSync2(file, "utf8"));
1916
+ const data = new Uint8Array(readFileSync2(file));
1917
+ if (ext === "pdf") return extractPdf(data);
1918
+ return extractOffice(data, ext);
1919
+ }
1920
+ function walk(root, rel, out2) {
1921
+ let names;
1922
+ try {
1923
+ names = readdirSync(join2(root, rel), { withFileTypes: true });
1924
+ } catch {
1925
+ return;
1926
+ }
1927
+ for (const d of names) {
1928
+ if (d.name.startsWith(".") || SKIP_DIRS.has(d.name)) continue;
1929
+ const child = rel ? `${rel}/${d.name}` : d.name;
1930
+ if (d.isDirectory()) walk(root, child, out2);
1931
+ else if (d.isFile()) out2.push(child);
1932
+ }
1933
+ }
1934
+ function attachmentDir(attachmentsRoot, collection) {
1935
+ return join2(attachmentsRoot, collection);
1936
+ }
1937
+ function textFile(dir, path) {
1938
+ const target = resolve(dir, `${path}.md`);
1939
+ const rel = relative(resolve(dir), target);
1940
+ if (!path || rel.startsWith("..") || resolve(rel) === rel) throw new EngineError("bad_request", `"${path}" is not a path inside the collection`);
1941
+ return target;
1942
+ }
1943
+ function writeText(dir, path, text) {
1944
+ const target = textFile(dir, path);
1945
+ mkdirSync2(dirname3(target), { recursive: true });
1946
+ writeFileSync2(target, truncate(text));
1947
+ }
1948
+ function removeText(dir, path) {
1949
+ rmSync(textFile(dir, path), { force: true });
1950
+ }
1951
+ function storePushedText(sidecar, attachmentsRoot, collection, path, text) {
1952
+ const dir = attachmentDir(attachmentsRoot, collection);
1953
+ if (!text.trim()) {
1954
+ removeText(dir, path);
1955
+ sidecar.removeAttachment(collection, path);
1956
+ return;
1957
+ }
1958
+ writeText(dir, path, text);
1959
+ sidecar.putAttachment(collection, path, Date.now(), text.length, true);
1960
+ }
1961
+ async function syncAttachments(sidecar, attachmentsRoot, collection, root, options, pause) {
1962
+ const types = options.attachments ?? DEFAULT_ATTACHMENTS;
1963
+ const exts = new Set(types.filter((t) => t !== "image").flatMap((t) => TYPE_EXTS[t] ?? []));
1964
+ const dir = attachmentDir(attachmentsRoot, collection);
1965
+ mkdirSync2(dir, { recursive: true });
1966
+ const manifest = sidecar.attachmentManifest(collection);
1967
+ const seen = /* @__PURE__ */ new Set();
1968
+ const files = [];
1969
+ if (exts.size > 0) walk(root, "", files);
1970
+ for (const rel of files) {
1971
+ const ext = extname(rel);
1972
+ if (!exts.has(ext) || !inScope(rel, options.include, options.exclude)) continue;
1973
+ seen.add(rel);
1974
+ const file = join2(root, rel);
1975
+ let stat;
1976
+ try {
1977
+ stat = statSync(file);
1978
+ } catch {
1979
+ continue;
1980
+ }
1981
+ const known = manifest.get(rel);
1982
+ if (known && !known.pushed && known.mtime === Math.round(stat.mtimeMs) && known.size === stat.size) continue;
1983
+ await pause();
1984
+ let text = "";
1985
+ try {
1986
+ text = await extract(file, ext);
1987
+ } catch {
1988
+ text = "";
1989
+ }
1990
+ if (text.trim()) writeText(dir, rel, text);
1991
+ else removeText(dir, rel);
1992
+ sidecar.putAttachment(collection, rel, stat.mtimeMs, stat.size, false);
1993
+ }
1994
+ for (const [path, info] of manifest) {
1995
+ const gone = info.pushed ? !existsSync(join2(root, path)) || !types.includes("image") : !seen.has(path);
1996
+ if (gone) {
1997
+ removeText(dir, path);
1998
+ sidecar.removeAttachment(collection, path);
1999
+ }
2000
+ }
2001
+ return sidecar.attachmentManifest(collection).size > 0;
2002
+ }
2003
+
2004
+ // src/backend.ts
2005
+ import { existsSync as existsSync2, readdirSync as readdirSync2, statSync as statSync2 } from "node:fs";
2006
+ import { createRequire } from "node:module";
2007
+ import { dirname as dirname4, join as join3 } from "node:path";
2008
+ import { fileURLToPath, pathToFileURL } from "node:url";
2009
+ import { createStore } from "@tobilu/qmd";
2010
+ var require2 = createRequire(import.meta.url);
2011
+ function qmdDist() {
2012
+ try {
2013
+ const url = import.meta.resolve("@tobilu/qmd");
2014
+ if (url.startsWith("file:")) return dirname4(fileURLToPath(url));
2015
+ } catch {
2016
+ }
2017
+ for (const start of [import.meta.dirname, process.cwd()]) {
2018
+ let dir = start;
2019
+ for (; ; ) {
2020
+ const candidate = join3(dir, "node_modules", "@tobilu", "qmd", "dist");
2021
+ if (existsSync2(join3(candidate, "index.js"))) return candidate;
2022
+ const parent = dirname4(dir);
2023
+ if (parent === dir) break;
2024
+ dir = parent;
2025
+ }
2026
+ }
2027
+ throw new Error("Cannot find the @tobilu/qmd package");
2028
+ }
2029
+ var STOP_POLL_MS = 200;
2030
+ var EMBED_MODEL_BYTES = 33e7;
2031
+ var DEEP_MODEL_BYTES = 17e8;
2032
+ function modelIdFromUri(uri) {
2033
+ const file = uri.split("/").pop() ?? uri;
2034
+ return file.replace(/\.gguf$/i, "").replace(/[-_.](q\d[\w]*|f16|f32)$/i, "");
2035
+ }
2036
+ var CHUNK_CHARS = 3600;
2037
+ var CHUNK_OVERLAP = Math.floor(CHUNK_CHARS * 0.15);
2038
+ function chunkText(text) {
2039
+ const out2 = [];
2040
+ let pos = 0;
2041
+ while (pos < text.length) {
2042
+ let end = Math.min(text.length, pos + CHUNK_CHARS);
2043
+ if (end < text.length) {
2044
+ const para = text.lastIndexOf("\n\n", end);
2045
+ const line = text.lastIndexOf("\n", end);
2046
+ if (para > pos + CHUNK_CHARS / 2) end = para;
2047
+ else if (line > pos + CHUNK_CHARS / 2) end = line;
2048
+ }
2049
+ out2.push({ pos, text: text.slice(pos, end) });
2050
+ if (end >= text.length) break;
2051
+ pos = Math.max(pos + 1, end - CHUNK_OVERLAP);
2052
+ }
2053
+ return out2;
2054
+ }
2055
+ var QmdBackend = class _QmdBackend {
2056
+ constructor(opts) {
2057
+ this.opts = opts;
2058
+ try {
2059
+ const pkgPath = join3(qmdDist(), "..", "package.json");
2060
+ this.version = require2(pkgPath).version;
2061
+ } catch {
2062
+ this.version = "unknown";
2063
+ }
2064
+ }
2065
+ store;
2066
+ llm = null;
2067
+ downloading = /* @__PURE__ */ new Set();
2068
+ version;
2069
+ static async open(opts) {
2070
+ const backend = new _QmdBackend(opts);
2071
+ if (opts.embedModel) process.env.QMD_EMBED_MODEL = opts.embedModel;
2072
+ backend.store = await createStore({ dbPath: opts.dbPath });
2073
+ return backend;
2074
+ }
2075
+ get name() {
2076
+ return `qmd ${this.version}`;
2077
+ }
2078
+ async close() {
2079
+ await this.store.close();
2080
+ }
2081
+ async llmModule() {
2082
+ if (!this.llm) {
2083
+ const file = join3(qmdDist(), "llm.js");
2084
+ this.llm = await import(pathToFileURL(file).href);
2085
+ }
2086
+ return this.llm;
2087
+ }
2088
+ // ---------- collections ----------
2089
+ async listCollections() {
2090
+ return (await this.store.listCollections()).map((c) => ({ name: c.name, path: c.pwd }));
2091
+ }
2092
+ async addCollection(name, path, ignore = []) {
2093
+ await this.store.addCollection(name, { path, pattern: "**/*.md", ignore });
2094
+ }
2095
+ async removeCollection(name) {
2096
+ return this.store.removeCollection(name);
2097
+ }
2098
+ async renameCollection(from, to) {
2099
+ return this.store.renameCollection(from, to);
2100
+ }
2101
+ async setContexts(collection, context) {
2102
+ const existing = (await this.store.listContexts()).filter((c) => c.collection === collection);
2103
+ for (const c of existing) await this.store.removeContext(collection, c.path);
2104
+ for (const [path, text] of Object.entries(context)) {
2105
+ if (text.trim()) await this.store.addContext(collection, path || "/", text.trim());
2106
+ }
2107
+ }
2108
+ async addContext(collection, path, text) {
2109
+ return this.store.addContext(collection, path, text);
2110
+ }
2111
+ async removeContext(collection, path) {
2112
+ return this.store.removeContext(collection, path);
2113
+ }
2114
+ async listContexts() {
2115
+ return this.store.listContexts();
2116
+ }
2117
+ // ---------- indexing ----------
2118
+ async update(collections2, onProgress) {
2119
+ if (collections2.length === 0) return;
2120
+ await this.store.update({ collections: collections2, onProgress: (p) => onProgress?.(p.current, p.total) });
2121
+ }
2122
+ /**
2123
+ * Embeds every document that has no vectors yet. Returns the number of chunks embedded. When
2124
+ * `shouldStop` turns true (the job was paused or cancelled) the run stops: the vectors written so
2125
+ * far are kept, and the remaining documents stay pending.
2126
+ */
2127
+ async embedPending(onProgress, shouldStop) {
2128
+ const embedder = this.opts.embedder;
2129
+ if (!embedder) {
2130
+ const controller = new AbortController();
2131
+ const poll = shouldStop ? setInterval(() => shouldStop() && controller.abort(), STOP_POLL_MS) : null;
2132
+ let embedded = 0;
2133
+ const run = (force) => this.store.embed({
2134
+ force,
2135
+ signal: controller.signal,
2136
+ onProgress: (p) => {
2137
+ embedded = p.chunksEmbedded;
2138
+ onProgress?.(p.chunksEmbedded, p.totalChunks);
2139
+ }
2140
+ });
2141
+ this.downloading.add("embed");
2142
+ try {
2143
+ const result = await run(false).catch((err) => {
2144
+ if (!controller.signal.aborted && /dimension mismatch/i.test(err.message)) return run(true);
2145
+ throw err;
2146
+ });
2147
+ return result.chunksEmbedded;
2148
+ } catch (err) {
2149
+ if (controller.signal.aborted) return embedded;
2150
+ throw err;
2151
+ } finally {
2152
+ if (poll) clearInterval(poll);
2153
+ this.downloading.delete("embed");
2154
+ }
2155
+ }
2156
+ const internal = this.store.internal;
2157
+ const rows = internal.db.prepare(
2158
+ `SELECT d.hash AS hash, c.doc AS body FROM documents d JOIN content c ON c.hash = d.hash
2159
+ WHERE d.active = 1 AND NOT EXISTS (SELECT 1 FROM content_vectors cv WHERE cv.hash = d.hash) GROUP BY d.hash`
2160
+ ).all();
2161
+ if (rows.length === 0) return 0;
2162
+ internal.ensureVecTable(embedder.dims);
2163
+ let done = 0;
2164
+ let chunks = 0;
2165
+ for (const row of rows) {
2166
+ if (shouldStop?.()) break;
2167
+ const parts = chunkText(row.body);
2168
+ for (let seq = 0; seq < parts.length; seq++) {
2169
+ const vec = await embedder.embed(parts[seq].text, false);
2170
+ internal.insertEmbedding(row.hash, seq, parts[seq].pos, vec, embedder.id, (/* @__PURE__ */ new Date()).toISOString(), parts.length, "");
2171
+ chunks++;
2172
+ }
2173
+ onProgress?.(++done, rows.length);
2174
+ }
2175
+ return chunks;
2176
+ }
2177
+ /** Documents waiting to be embedded. qmd's own count is per its model, so an injected embedder counts its own. */
2178
+ async pendingEmbeddings() {
2179
+ if (this.opts.embedder) {
2180
+ const row = this.store.internal.db.prepare(
2181
+ `SELECT COUNT(DISTINCT d.hash) AS n FROM documents d WHERE d.active = 1 AND NOT EXISTS (SELECT 1 FROM content_vectors cv WHERE cv.hash = d.hash)`
2182
+ ).get();
2183
+ return row?.n ?? 0;
2184
+ }
2185
+ return (await this.store.getStatus()).needsEmbedding;
2186
+ }
2187
+ // ---------- models ----------
2188
+ embedModelUri() {
2189
+ return this.opts.embedder?.id ?? this.store.internal.llm?.embedModelName ?? "";
2190
+ }
2191
+ embedModelId() {
2192
+ return this.opts.embedder ? this.opts.embedder.id : modelIdFromUri(this.embedModelUri());
2193
+ }
2194
+ cachedBytes(uri) {
2195
+ if (!uri) return 0;
2196
+ if (!uri.startsWith("hf:")) return existsSync2(uri) ? statSync2(uri).size : 0;
2197
+ const file = uri.split("/").pop();
2198
+ const dir = this.llm?.DEFAULT_MODEL_CACHE_DIR ?? join3(process.env.XDG_CACHE_HOME || join3(process.env.HOME ?? "", ".cache"), "qmd", "models");
2199
+ try {
2200
+ return readdirSync2(dir).filter((name) => name.includes(file) && !name.endsWith(".etag")).reduce((sum, name) => sum + statSync2(join3(dir, name)).size, 0);
2201
+ } catch {
2202
+ return 0;
2203
+ }
2204
+ }
2205
+ fileState(uri, kind, expected) {
2206
+ const bytes = this.cachedBytes(uri);
2207
+ if (this.downloading.has(kind) && bytes < expected * 0.9) {
2208
+ return { state: "downloading", progress: Math.min(0.99, bytes / expected) };
2209
+ }
2210
+ return { state: bytes > 0 ? "ready" : "missing" };
2211
+ }
2212
+ modelStates() {
2213
+ if (this.opts.embedder) {
2214
+ const deep = this.opts.deep ? "ready" : "missing";
2215
+ return { embed: { id: this.opts.embedder.id, state: "ready" }, rerank: { state: deep }, expand: { state: deep } };
2216
+ }
2217
+ const llm = this.store.internal.llm;
2218
+ const rerankUri = llm?.rerankModelName ?? "";
2219
+ const expandUri = llm?.generateModelName ?? "";
2220
+ return {
2221
+ embed: { id: this.embedModelId(), ...this.fileState(this.embedModelUri(), "embed", EMBED_MODEL_BYTES) },
2222
+ rerank: { id: modelIdFromUri(rerankUri), ...this.fileState(rerankUri, "deep", DEEP_MODEL_BYTES / 3) },
2223
+ expand: { id: modelIdFromUri(expandUri), ...this.fileState(expandUri, "deep", DEEP_MODEL_BYTES * 2 / 3) }
2224
+ };
2225
+ }
2226
+ deepReady() {
2227
+ const m = this.modelStates();
2228
+ return m.rerank.state === "ready" && m.expand.state === "ready";
2229
+ }
2230
+ /** Downloads the rerank and expansion models (about 1.7 GB). Called only after the user confirms. */
2231
+ async downloadDeepModels() {
2232
+ if (this.opts.embedder) return;
2233
+ const llm = this.store.internal.llm;
2234
+ if (!llm) return;
2235
+ const mod = await this.llmModule();
2236
+ this.downloading.add("deep");
2237
+ try {
2238
+ await mod.pullModels([llm.rerankModelName, llm.generateModelName]);
2239
+ } finally {
2240
+ this.downloading.delete("deep");
2241
+ }
2242
+ }
2243
+ // ---------- model calls ----------
2244
+ async embedText(text, isQuery) {
2245
+ if (this.opts.embedder) return this.opts.embedder.embed(text, isQuery);
2246
+ const llm = this.store.internal.llm;
2247
+ if (!llm) return null;
2248
+ const mod = await this.llmModule();
2249
+ const uri = llm.embedModelName;
2250
+ const formatted = isQuery ? mod.formatQueryForEmbedding(text, uri) : mod.formatDocForEmbedding(text, void 0, uri);
2251
+ const result = await llm.embed(formatted, { model: uri, isQuery });
2252
+ return result ? new Float32Array(result.embedding) : null;
2253
+ }
2254
+ async expand(query) {
2255
+ if (this.opts.embedder) return this.opts.deep ? this.opts.deep.expand(query) : [];
2256
+ return (await this.store.expandQuery(query)).map((q) => ({ type: q.type, query: q.query }));
2257
+ }
2258
+ async rerank(query, docs) {
2259
+ if (this.opts.embedder) return this.opts.deep ? this.opts.deep.rerank(query, docs) : /* @__PURE__ */ new Map();
2260
+ const scored = await this.store.internal.rerank(
2261
+ query,
2262
+ docs.map((d) => ({ file: d.id, text: d.text }))
2263
+ );
2264
+ return new Map(scored.map((s) => [s.file, s.score]));
2265
+ }
2266
+ /** Touches the models so qmd does not unload them while Obsidian is focused. */
2267
+ async keepWarm() {
2268
+ if (this.opts.embedder) return;
2269
+ if (this.modelStates().embed.state === "ready") await this.embedText("warm", true).catch(() => null);
2270
+ }
2271
+ };
2272
+
2273
+ // src/jobs.ts
2274
+ var Jobs = class {
2275
+ constructor(sidecar, emit) {
2276
+ this.sidecar = sidecar;
2277
+ this.emit = emit;
2278
+ }
2279
+ seq = 0;
2280
+ all = /* @__PURE__ */ new Map();
2281
+ queue = [];
2282
+ current = null;
2283
+ interactive = 0;
2284
+ lastEmit = 0;
2285
+ paused = false;
2286
+ enqueue(kind, key, run) {
2287
+ const queued = this.queue.find((j) => j.key === key && !j.cancel);
2288
+ if (queued) return queued.info.jobId;
2289
+ const jobId = `j_${String(++this.seq).padStart(2, "0")}`;
2290
+ const job = { info: { jobId, kind, state: "queued", done: 0, total: 0 }, key, cancel: false, run };
2291
+ this.all.set(jobId, job);
2292
+ this.queue.push(job);
2293
+ this.sidecar.saveJob(job.info);
2294
+ queueMicrotask(() => void this.pump());
2295
+ return jobId;
2296
+ }
2297
+ async pump() {
2298
+ if (this.current || this.paused) return;
2299
+ const job = this.queue.shift();
2300
+ if (!job) return;
2301
+ if (job.cancel) return void this.pump();
2302
+ this.current = job;
2303
+ job.info.state = "running";
2304
+ job.info.started = Date.now();
2305
+ this.sidecar.saveJob(job.info);
2306
+ const ctx = {
2307
+ info: job.info,
2308
+ cancelled: () => job.cancel,
2309
+ progress: (done, total, extra) => {
2310
+ job.info.done = done;
2311
+ job.info.total = total;
2312
+ const now = Date.now();
2313
+ if (now - this.lastEmit < 100 && done < total) return;
2314
+ this.lastEmit = now;
2315
+ this.emit({ event: "index.progress", data: { jobId: job.info.jobId, kind: job.info.kind, done, total, ...extra } });
2316
+ },
2317
+ pause: () => this.idle(job)
2318
+ };
2319
+ try {
2320
+ await job.run(ctx);
2321
+ job.info.state = job.cancel ? "cancelled" : "done";
2322
+ } catch (err) {
2323
+ job.info.state = "failed";
2324
+ job.info.error = err.message;
2325
+ this.emit({ event: "engine.state", data: { state: "error", message: job.info.error, jobId: job.info.jobId } });
2326
+ }
2327
+ job.info.finished = Date.now();
2328
+ this.sidecar.saveJob(job.info);
2329
+ this.current = null;
2330
+ const finished = [...this.all.values()].filter((j) => j.info.finished);
2331
+ for (const old of finished.slice(0, Math.max(0, finished.length - 20))) this.all.delete(old.info.jobId);
2332
+ void this.pump();
2333
+ }
2334
+ async idle(job) {
2335
+ await new Promise((r) => setImmediate(r));
2336
+ while ((this.interactive > 0 || this.paused) && !job.cancel) {
2337
+ if (this.paused && job.info.state === "running") job.info.state = "paused";
2338
+ await new Promise((r) => setTimeout(r, 25));
2339
+ }
2340
+ if (job.info.state === "paused") job.info.state = "running";
2341
+ }
2342
+ /** Marks an interactive request as in flight, so background work yields to it. */
2343
+ async interactiveCall(fn) {
2344
+ this.interactive++;
2345
+ try {
2346
+ return await fn();
2347
+ } finally {
2348
+ this.interactive--;
2349
+ }
2350
+ }
2351
+ beginInteractive() {
2352
+ this.interactive++;
2353
+ let ended = false;
2354
+ return () => {
2355
+ if (!ended) {
2356
+ ended = true;
2357
+ this.interactive--;
2358
+ }
2359
+ };
2360
+ }
2361
+ cancel(jobId) {
2362
+ const job = this.all.get(jobId);
2363
+ if (!job) return false;
2364
+ job.cancel = true;
2365
+ if (job.info.state === "queued") {
2366
+ job.info.state = "cancelled";
2367
+ job.info.finished = Date.now();
2368
+ this.sidecar.saveJob(job.info);
2369
+ }
2370
+ return true;
2371
+ }
2372
+ pause() {
2373
+ this.paused = true;
2374
+ this.emit({ event: "engine.state", data: { state: "paused" } });
2375
+ }
2376
+ resume() {
2377
+ this.paused = false;
2378
+ this.emit({ event: "engine.state", data: { state: "running" } });
2379
+ void this.pump();
2380
+ }
2381
+ get(jobId) {
2382
+ return this.all.get(jobId)?.info;
2383
+ }
2384
+ /** Queued, running and paused jobs. */
2385
+ active() {
2386
+ return [...this.all.values()].map((j) => j.info).filter((j) => j.state === "queued" || j.state === "running" || j.state === "paused");
2387
+ }
2388
+ list() {
2389
+ return [...this.all.values()].map((j) => ({ ...j.info }));
2390
+ }
2391
+ /** Resolves when no job is queued or running. Used by the CLI and by tests. */
2392
+ async drain() {
2393
+ while (this.current || this.queue.some((j) => !j.cancel)) await new Promise((r) => setTimeout(r, 10));
2394
+ }
2395
+ };
2396
+
2397
+ // src/paths.ts
2398
+ import { chmodSync, existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync3, renameSync, rmSync as rmSync2, writeFileSync as writeFileSync3 } from "node:fs";
2399
+ import { homedir } from "node:os";
2400
+ import { join as join4 } from "node:path";
2401
+ var ENGINE_NAME = "trova";
2402
+ var ENGINE_VERSION = "0.1.0";
2403
+ var DEFAULT_HOME = join4(homedir(), ".cache", "trova");
2404
+ function trovaHome() {
2405
+ return process.env.TROVA_HOME || DEFAULT_HOME;
2406
+ }
2407
+ function dataPaths(home = trovaHome()) {
2408
+ mkdirSync3(home, { recursive: true, mode: 448 });
2409
+ if (home === DEFAULT_HOME) {
2410
+ try {
2411
+ chmodSync(home, 448);
2412
+ } catch {
2413
+ }
2414
+ }
2415
+ mkdirSync3(join4(home, "logs"), { recursive: true, mode: 448 });
2416
+ mkdirSync3(join4(home, "attachments"), { recursive: true, mode: 448 });
2417
+ return {
2418
+ home,
2419
+ qmdDb: join4(home, "qmd.sqlite"),
2420
+ sidecarDb: join4(home, "trova.sqlite"),
2421
+ daemonJson: join4(home, "daemon.json"),
2422
+ logs: join4(home, "logs"),
2423
+ attachments: join4(home, "attachments")
2424
+ };
2425
+ }
2426
+ function readDaemonInfo(home = trovaHome()) {
2427
+ try {
2428
+ const info = JSON.parse(readFileSync3(join4(home, "daemon.json"), "utf8"));
2429
+ return typeof info.port === "number" && typeof info.token === "string" ? info : null;
2430
+ } catch {
2431
+ return null;
2432
+ }
2433
+ }
2434
+ function writeDaemonInfo(info, home = trovaHome()) {
2435
+ const file = join4(home, "daemon.json");
2436
+ const temp = `${file}.${process.pid}.tmp`;
2437
+ rmSync2(temp, { force: true });
2438
+ writeFileSync3(temp, JSON.stringify(info, null, 2), { mode: 384, flag: "wx" });
2439
+ renameSync(temp, file);
2440
+ }
2441
+ function removeDaemonInfo(home = trovaHome()) {
2442
+ const file = join4(home, "daemon.json");
2443
+ if (existsSync3(file)) rmSync2(file, { force: true });
2444
+ }
2445
+ function isProcessAlive(pid) {
2446
+ try {
2447
+ process.kill(pid, 0);
2448
+ return true;
2449
+ } catch (err) {
2450
+ return err.code === "EPERM";
2451
+ }
2452
+ }
2453
+
2454
+ // src/reader.ts
2455
+ import Database2 from "better-sqlite3";
2456
+ import * as sqliteVec from "sqlite-vec";
2457
+ var CJK_RUN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
2458
+ var CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
2459
+ function ftsTokens(text) {
2460
+ return text.replace(CJK_RUN, (run) => ` ${Array.from(run).join(" ")} `).split(/[^\p{L}\p{N}]+/u).filter(Boolean);
2461
+ }
2462
+ function ftsTerm(term) {
2463
+ const tokens = ftsTokens(term.text);
2464
+ if (tokens.length === 0) return null;
2465
+ const phrase = `"${tokens.join(" ")}"`;
2466
+ const prefix = term.kind === "word" && !CJK_CHAR.test(tokens[tokens.length - 1]) ? "*" : "";
2467
+ const scoped = term.field === "title" ? `{filepath title} : ${phrase}${prefix}` : `${phrase}${prefix}`;
2468
+ return scoped;
2469
+ }
2470
+ function buildFtsQuery(q) {
2471
+ const groups = [];
2472
+ for (const group of q.must) {
2473
+ const parts = group.map(ftsTerm).filter((p) => p !== null);
2474
+ if (parts.length === 0) continue;
2475
+ groups.push(parts.length === 1 ? parts[0] : `(${parts.join(" OR ")})`);
2476
+ }
2477
+ if (groups.length === 0) return null;
2478
+ let expr = groups.join(" AND ");
2479
+ const negatives = q.mustNot.map(ftsTerm).filter((p) => p !== null);
2480
+ for (const n of negatives) expr = `(${expr}) NOT ${n}`;
2481
+ return expr;
2482
+ }
2483
+ var VEC_TABLE = "vectors_by_collection";
2484
+ var SCHEMA2 = {
2485
+ documents: ["id", "collection", "path", "title", "hash", "created_at", "modified_at", "active"],
2486
+ content: ["hash", "doc"],
2487
+ content_vectors: ["hash", "seq", "pos", "model"],
2488
+ documents_fts: ["filepath", "title", "body"],
2489
+ vector_rows: ["id", "hash", "seq", "collection_id"],
2490
+ vector_collection_ids: ["id", "name"]
2491
+ };
2492
+ var MAX_K = 4096;
2493
+ var QmdReader = class {
2494
+ constructor(dbPath) {
2495
+ this.dbPath = dbPath;
2496
+ this.db = new Database2(dbPath, { readonly: true, fileMustExist: true });
2497
+ this.db.pragma("busy_timeout = 5000");
2498
+ try {
2499
+ sqliteVec.load(this.db);
2500
+ this.vecLoaded = true;
2501
+ } catch {
2502
+ this.vecLoaded = false;
2503
+ }
2504
+ }
2505
+ db;
2506
+ vecLoaded = false;
2507
+ vocabReady = false;
2508
+ stmts = /* @__PURE__ */ new Map();
2509
+ close() {
2510
+ this.stmts.clear();
2511
+ this.db.close();
2512
+ }
2513
+ stmt(sql) {
2514
+ let s = this.stmts.get(sql);
2515
+ if (!s) {
2516
+ s = this.db.prepare(sql);
2517
+ this.stmts.set(sql, s);
2518
+ }
2519
+ return s;
2520
+ }
2521
+ placeholders(n) {
2522
+ return Array.from({ length: n }, () => "?").join(",");
2523
+ }
2524
+ /**
2525
+ * What is missing from the index for this reader, one line each; empty when the layout matches.
2526
+ * A mismatch means the installed qmd and this engine build disagree, so the engine refuses to
2527
+ * open rather than show no results.
2528
+ */
2529
+ schemaProblems() {
2530
+ const problems = [];
2531
+ const columnsOf = this.db.prepare(`SELECT name FROM pragma_table_info(?)`);
2532
+ for (const [table, wanted] of Object.entries(SCHEMA2)) {
2533
+ const have = columnsOf.all(table).map((r) => r.name);
2534
+ if (have.length === 0) problems.push(`table ${table} is missing`);
2535
+ else if (table === "documents_fts" && have.join(",") !== wanted.join(",")) problems.push(`documents_fts has columns ${have.join(", ")}, not ${wanted.join(", ")}`);
2536
+ else for (const column of wanted) if (!have.includes(column)) problems.push(`column ${table}.${column} is missing`);
2537
+ }
2538
+ const vec = this.stmt(`SELECT sql FROM sqlite_master WHERE type='table' AND name='${VEC_TABLE}'`).get();
2539
+ if (vec && !(/collection_id\s+INTEGER\s+PARTITION KEY/i.test(vec.sql) && /embedding\s+float\[\d+\]/i.test(vec.sql))) {
2540
+ problems.push(`${VEC_TABLE} is not partitioned by collection_id with an embedding column`);
2541
+ }
2542
+ return problems;
2543
+ }
2544
+ hasVectors() {
2545
+ if (!this.vecLoaded) return false;
2546
+ return !!this.stmt(`SELECT name FROM sqlite_master WHERE type='table' AND name='${VEC_TABLE}'`).get();
2547
+ }
2548
+ /** Embedding dimensions, read from the vec0 table definition. */
2549
+ dims() {
2550
+ const row = this.stmt(`SELECT sql FROM sqlite_master WHERE type='table' AND name='${VEC_TABLE}'`).get();
2551
+ const m = row ? /float\[(\d+)\]/.exec(row.sql) : null;
2552
+ return m ? Number(m[1]) : null;
2553
+ }
2554
+ // ---------- documents ----------
2555
+ activeDocs(collection) {
2556
+ return this.stmt(
2557
+ `SELECT id, collection, path, title, hash, created_at, modified_at FROM documents WHERE collection = ? AND active = 1`
2558
+ ).all(collection);
2559
+ }
2560
+ doc(collection, path) {
2561
+ return this.stmt(
2562
+ `SELECT id, collection, path, title, hash, created_at, modified_at FROM documents WHERE collection = ? AND path = ? AND active = 1`
2563
+ ).get(collection, path);
2564
+ }
2565
+ content(hash2) {
2566
+ const row = this.stmt(`SELECT doc FROM content WHERE hash = ?`).get(hash2);
2567
+ return row ? row.doc : null;
2568
+ }
2569
+ docsByHashes(hashes) {
2570
+ const out2 = /* @__PURE__ */ new Map();
2571
+ for (let i = 0; i < hashes.length; i += 400) {
2572
+ const part = hashes.slice(i, i + 400);
2573
+ const rows = this.db.prepare(
2574
+ `SELECT id, collection, path, title, hash, created_at, modified_at FROM documents WHERE active = 1 AND hash IN (${this.placeholders(part.length)})`
2575
+ ).all(...part);
2576
+ for (const r of rows) {
2577
+ const list = out2.get(r.hash);
2578
+ if (list) list.push(r);
2579
+ else out2.set(r.hash, [r]);
2580
+ }
2581
+ }
2582
+ return out2;
2583
+ }
2584
+ counts(collection) {
2585
+ const sources = this.stmt(`SELECT COUNT(*) AS n FROM documents WHERE collection = ? AND active = 1`).get(collection).n;
2586
+ const embedded = this.stmt(
2587
+ `SELECT COUNT(*) AS n FROM documents d WHERE d.collection = ? AND d.active = 1 AND EXISTS (SELECT 1 FROM content_vectors cv WHERE cv.hash = d.hash)`
2588
+ ).get(collection).n;
2589
+ return { sources, embedded };
2590
+ }
2591
+ /** Model ids that stored vectors were made with, most common first. */
2592
+ vectorModels() {
2593
+ return this.stmt(`SELECT model, COUNT(*) AS chunks FROM content_vectors GROUP BY model ORDER BY chunks DESC`).all();
2594
+ }
2595
+ // ---------- lex ----------
2596
+ lex(match, collections2, limit2) {
2597
+ if (collections2.length === 0) return [];
2598
+ const { path, title, body } = QMD_BM25_WEIGHTS;
2599
+ const sql = `
2600
+ SELECT d.id, d.collection, d.path, d.title, d.hash, d.created_at, d.modified_at,
2601
+ bm25(documents_fts, ${path}, ${title}, ${body}) AS score
2602
+ FROM documents_fts
2603
+ JOIN documents d ON d.id = documents_fts.rowid
2604
+ WHERE documents_fts MATCH ? AND d.active = 1 AND d.collection IN (${this.placeholders(collections2.length)})
2605
+ ORDER BY score LIMIT ?`;
2606
+ try {
2607
+ return this.db.prepare(sql).all(match, ...collections2, limit2).map((r) => ({ ...r, score: -r.score }));
2608
+ } catch (err) {
2609
+ if (String(err.message).includes("fts5")) return [];
2610
+ throw err;
2611
+ }
2612
+ }
2613
+ /** True when some indexed term starts with this word (after FTS tokenising and stemming). */
2614
+ hasPrefix(word) {
2615
+ const tokens = ftsTokens(word);
2616
+ if (tokens.length === 0) return true;
2617
+ try {
2618
+ return !!this.stmt(`SELECT rowid FROM documents_fts WHERE documents_fts MATCH ? LIMIT 1`).get(`"${tokens.join(" ")}"*`);
2619
+ } catch {
2620
+ return true;
2621
+ }
2622
+ }
2623
+ /** The FTS vocabulary with document counts, read through an fts5vocab table. */
2624
+ vocabulary() {
2625
+ if (!this.vocabReady) {
2626
+ this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.trova_vocab USING fts5vocab('main', 'documents_fts', 'row')`);
2627
+ this.vocabReady = true;
2628
+ }
2629
+ return this.stmt(`SELECT term, doc AS docs FROM temp.trova_vocab`).all();
2630
+ }
2631
+ // ---------- vectors ----------
2632
+ chunks(hash2) {
2633
+ return this.stmt(`SELECT hash, seq, pos FROM content_vectors WHERE hash = ? ORDER BY seq`).all(hash2);
2634
+ }
2635
+ chunksFor(hashes) {
2636
+ const out2 = /* @__PURE__ */ new Map();
2637
+ for (let i = 0; i < hashes.length; i += 400) {
2638
+ const part = hashes.slice(i, i + 400);
2639
+ const rows = this.db.prepare(`SELECT hash, seq, pos FROM content_vectors WHERE hash IN (${this.placeholders(part.length)}) ORDER BY hash, seq`).all(...part);
2640
+ for (const r of rows) {
2641
+ const list = out2.get(r.hash);
2642
+ if (list) list.push(r);
2643
+ else out2.set(r.hash, [r]);
2644
+ }
2645
+ }
2646
+ return out2;
2647
+ }
2648
+ toVector(blob) {
2649
+ const copy = new Uint8Array(blob.byteLength);
2650
+ copy.set(blob);
2651
+ return new Float32Array(copy.buffer);
2652
+ }
2653
+ /**
2654
+ * Stored vectors by `hash_seq` key. A chunk held by several collections is stored once in each,
2655
+ * with the same vector, so any one of its rows serves.
2656
+ */
2657
+ vectors(keys) {
2658
+ const out2 = /* @__PURE__ */ new Map();
2659
+ if (!this.hasVectors() || keys.length === 0) return out2;
2660
+ const wanted = new Set(keys);
2661
+ const hashes = [...new Set(keys.map((k) => k.slice(0, k.lastIndexOf("_"))))];
2662
+ const vectorOf = this.stmt(`SELECT embedding FROM ${VEC_TABLE} WHERE rowid = ?`);
2663
+ for (let i = 0; i < hashes.length; i += 400) {
2664
+ const part = hashes.slice(i, i + 400);
2665
+ const rows = this.stmt(`SELECT MIN(id) AS id, hash, seq FROM vector_rows WHERE hash IN (${this.placeholders(part.length)}) GROUP BY hash, seq`).all(...part);
2666
+ for (const r of rows) {
2667
+ const key = `${r.hash}_${r.seq}`;
2668
+ if (!wanted.has(key)) continue;
2669
+ const row = vectorOf.get(BigInt(r.id));
2670
+ if (row) out2.set(key, this.toVector(row.embedding));
2671
+ }
2672
+ }
2673
+ return out2;
2674
+ }
2675
+ /**
2676
+ * The k nearest stored chunks to a vector, with cosine similarity, in the given qmd collections
2677
+ * or in all of them. Each collection is scanned on its own, as qmd does, so a small collection is
2678
+ * never crowded out by a large one; a chunk held by several collections counts once.
2679
+ */
2680
+ knn(vector, k, collections2) {
2681
+ if (!this.hasVectors()) return [];
2682
+ const ids = collections2 ? collections2.length === 0 ? [] : this.stmt(`SELECT id FROM vector_collection_ids WHERE name IN (${this.placeholders(collections2.length)})`).all(...collections2) : this.stmt(`SELECT id FROM vector_collection_ids`).all();
2683
+ const embedding = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
2684
+ const kk = Math.max(1, Math.min(MAX_K, Math.floor(k)));
2685
+ const scan = this.stmt(`SELECT rowid, distance FROM ${VEC_TABLE} WHERE embedding MATCH ? AND k = ? AND collection_id = ?`);
2686
+ const chunkOf = this.stmt(`SELECT hash, seq FROM vector_rows WHERE id = ?`);
2687
+ const best = /* @__PURE__ */ new Map();
2688
+ for (const { id } of ids) {
2689
+ for (const r of scan.all(embedding, kk, BigInt(id))) {
2690
+ const chunk = chunkOf.get(r.rowid);
2691
+ if (!chunk) continue;
2692
+ const key = `${chunk.hash}_${chunk.seq}`;
2693
+ const similarity = 1 - r.distance;
2694
+ if ((best.get(key)?.similarity ?? -Infinity) < similarity) best.set(key, { hash: chunk.hash, seq: chunk.seq, similarity });
2695
+ }
2696
+ }
2697
+ return [...best.values()].sort((a, b) => b.similarity - a.similarity).slice(0, kk);
2698
+ }
2699
+ /** Iterates every stored chunk vector of the given collections' active documents. */
2700
+ allChunkKeys(collections2) {
2701
+ if (collections2.length === 0) return [];
2702
+ return this.db.prepare(
2703
+ `SELECT DISTINCT cv.hash, cv.seq, cv.pos FROM content_vectors cv JOIN documents d ON d.hash = cv.hash AND d.active = 1
2704
+ WHERE d.collection IN (${this.placeholders(collections2.length)}) ORDER BY cv.hash, cv.seq`
2705
+ ).all(...collections2);
2706
+ }
2707
+ };
2708
+
2709
+ // src/engine.ts
2710
+ var GRAPH_EDGES_PER_NODE = 3;
2711
+ var STOPWORDS = new Set(
2712
+ "about after again also because been before being between both could does doing down during each from further have having here into itself just more most other over same should some such than that their them then there these they this those through under until very were what when where which while will with would your".split(
2713
+ " "
2714
+ )
2715
+ );
2716
+ function keywords(text, max = 60) {
2717
+ const counts = /* @__PURE__ */ new Map();
2718
+ for (const w of text.toLowerCase().match(/[\p{L}\p{N}]{4,}/gu) ?? []) {
2719
+ if (!STOPWORDS.has(w)) counts.set(w, (counts.get(w) ?? 0) + 1);
2720
+ }
2721
+ return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, max).map(([w]) => w);
2722
+ }
2723
+ function withTimeout(promise, ms) {
2724
+ if (!Number.isFinite(ms)) return promise;
2725
+ if (ms <= 0) return Promise.resolve("timeout");
2726
+ let timer;
2727
+ return Promise.race([
2728
+ promise.finally(() => clearTimeout(timer)),
2729
+ new Promise((resolve3) => {
2730
+ timer = setTimeout(() => resolve3("timeout"), ms);
2731
+ })
2732
+ ]);
2733
+ }
2734
+ function checkAborted(signal) {
2735
+ if (signal?.aborted) throw new EngineError("cancelled", "The client aborted the request");
2736
+ }
2737
+ var Engine = class _Engine {
2738
+ constructor(opts, backend, reader, sidecar, paths) {
2739
+ this.opts = opts;
2740
+ this.backend = backend;
2741
+ this.reader = reader;
2742
+ this.sidecar = sidecar;
2743
+ this.paths = paths;
2744
+ this.jobs = new Jobs(sidecar, (e) => this.emit(e));
2745
+ this.duplicates = sidecar.getValue("duplicates") ?? this.duplicates;
2746
+ this.topics = sidecar.getValue("topics") ?? this.topics;
2747
+ if (this.duplicates.state === "running") this.duplicates.state = "none";
2748
+ if (this.topics.state === "running") this.topics.state = "none";
2749
+ }
2750
+ paths;
2751
+ jobs;
2752
+ listeners = /* @__PURE__ */ new Set();
2753
+ attRegistered = /* @__PURE__ */ new Set();
2754
+ vocabCache = null;
2755
+ contextCache = null;
2756
+ watchers = /* @__PURE__ */ new Map();
2757
+ watchTimers = /* @__PURE__ */ new Map();
2758
+ closed = false;
2759
+ duplicates = { state: "none", pairs: [] };
2760
+ topics = { state: "none", clusters: [] };
2761
+ /** Last time a client was seen. The daemon uses it for its idle exit. */
2762
+ lastActivity = Date.now();
2763
+ static async open(opts = {}) {
2764
+ const paths = dataPaths(opts.home);
2765
+ const backend = await QmdBackend.open({ dbPath: paths.qmdDb, embedder: opts.embedder, deep: opts.deep, embedModel: opts.embedModel });
2766
+ const reader = new QmdReader(paths.qmdDb);
2767
+ const problems = reader.schemaProblems();
2768
+ if (problems.length > 0) {
2769
+ reader.close();
2770
+ await backend.close();
2771
+ throw new Error(
2772
+ `The index at ${paths.qmdDb} does not have the layout this trova build reads with ${backend.name}: ${problems.join("; ")}. Reinstall trova's dependencies (npm install) so its qmd version matches, then start it again.`
2773
+ );
2774
+ }
2775
+ const sidecar = new Sidecar(paths.sidecarDb);
2776
+ const engine = new _Engine(opts, backend, reader, sidecar, paths);
2777
+ const known = new Set((await backend.listCollections()).map((c) => c.name));
2778
+ for (const name of sidecar.collectionNames()) {
2779
+ if (known.has(name + ATT_SUFFIX)) engine.attRegistered.add(name);
2780
+ if (opts.watch) engine.startWatcher(name);
2781
+ }
2782
+ return engine;
2783
+ }
2784
+ async close() {
2785
+ if (this.closed) return;
2786
+ this.closed = true;
2787
+ for (const w of this.watchers.values()) w.close();
2788
+ for (const t of this.watchTimers.values()) clearTimeout(t);
2789
+ for (const j of this.jobs.active()) this.jobs.cancel(j.jobId);
2790
+ this.reader.close();
2791
+ this.sidecar.close();
2792
+ await this.backend.close();
2793
+ }
2794
+ // ---------- events ----------
2795
+ on(listener) {
2796
+ this.listeners.add(listener);
2797
+ return () => this.listeners.delete(listener);
2798
+ }
2799
+ get listenerCount() {
2800
+ return this.listeners.size;
2801
+ }
2802
+ emit(e) {
2803
+ for (const l of this.listeners) {
2804
+ try {
2805
+ l(e);
2806
+ } catch {
2807
+ }
2808
+ }
2809
+ }
2810
+ // ---------- collections ----------
2811
+ collectionsFor(requested) {
2812
+ const all = this.sidecar.collectionNames();
2813
+ if (!requested || requested.length === 0) return all;
2814
+ return requested.filter((n) => all.includes(n));
2815
+ }
2816
+ qmdNames(collections2) {
2817
+ return collections2.flatMap((n) => this.attRegistered.has(n) ? [n, n + ATT_SUFFIX] : [n]);
2818
+ }
2819
+ publicCollection(qmdCollection) {
2820
+ return qmdCollection.endsWith(ATT_SUFFIX) && this.attRegistered.has(qmdCollection.slice(0, -ATT_SUFFIX.length)) ? qmdCollection.slice(0, -ATT_SUFFIX.length) : qmdCollection;
2821
+ }
2822
+ async listCollections() {
2823
+ const contexts = await this.backend.listContexts();
2824
+ return this.sidecar.collectionNames().map((name) => {
2825
+ const c = this.sidecar.collection(name);
2826
+ const context = {};
2827
+ for (const ctx of contexts) if (ctx.collection === name) context[ctx.path] = ctx.context;
2828
+ return { name, path: c.path, options: c.options, context };
2829
+ });
2830
+ }
2831
+ async addCollection(req) {
2832
+ if (!req.name || !/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(req.name) || req.name.endsWith(ATT_SUFFIX)) {
2833
+ throw new EngineError("bad_request", `"${req.name}" is not a valid collection name`);
2834
+ }
2835
+ const existing = this.sidecar.collection(req.name);
2836
+ const path = req.path ?? existing?.path;
2837
+ if (!path) throw new EngineError("bad_request", "A collection needs a path");
2838
+ try {
2839
+ if (!statSync3(path).isDirectory()) throw new Error("not a directory");
2840
+ } catch {
2841
+ throw new EngineError("bad_request", `"${path}" is not a folder`);
2842
+ }
2843
+ if (!existing || existing.path !== path) await this.backend.addCollection(req.name, path);
2844
+ this.sidecar.saveCollection(req.name, path, req.options ?? existing?.options ?? {});
2845
+ if (req.context) {
2846
+ await this.backend.setContexts(req.name, req.context);
2847
+ this.contextCache = null;
2848
+ }
2849
+ if (this.opts.watch) this.startWatcher(req.name);
2850
+ return (await this.listCollections()).find((c) => c.name === req.name);
2851
+ }
2852
+ async removeCollection(name) {
2853
+ if (!this.sidecar.collection(name)) throw new EngineError("not_found", `Unknown collection "${name}"`);
2854
+ this.watchers.get(name)?.close();
2855
+ this.watchers.delete(name);
2856
+ await this.backend.removeCollection(name);
2857
+ this.attRegistered.delete(name);
2858
+ await this.backend.removeCollection(name + ATT_SUFFIX);
2859
+ rmSync3(attachmentDir(this.paths.attachments, name), { recursive: true, force: true });
2860
+ this.sidecar.removeCollection(name);
2861
+ this.vocabCache = null;
2862
+ this.contextCache = null;
2863
+ }
2864
+ async renameCollection(from, to) {
2865
+ const c = this.sidecar.collection(from);
2866
+ if (!c) throw new EngineError("not_found", `Unknown collection "${from}"`);
2867
+ await this.removeCollection(from);
2868
+ await this.addCollection({ name: to, path: c.path, options: c.options });
2869
+ this.sync({ collection: to, full: true });
2870
+ }
2871
+ async setContext(collection, path, text) {
2872
+ if (!this.sidecar.collection(collection)) throw new EngineError("not_found", `Unknown collection "${collection}"`);
2873
+ if (text) await this.backend.addContext(collection, path, text);
2874
+ else await this.backend.removeContext(collection, path);
2875
+ this.contextCache = null;
2876
+ }
2877
+ async contextFor(collection, path) {
2878
+ if (!this.contextCache) {
2879
+ this.contextCache = /* @__PURE__ */ new Map();
2880
+ for (const c of await this.backend.listContexts()) {
2881
+ const list2 = this.contextCache.get(c.collection) ?? [];
2882
+ list2.push({ path: c.path.replace(/^\/+|\/+$/g, ""), context: c.context });
2883
+ this.contextCache.set(c.collection, list2);
2884
+ }
2885
+ for (const list2 of this.contextCache.values()) list2.sort((a, b) => b.path.length - a.path.length);
2886
+ }
2887
+ const list = this.contextCache.get(collection);
2888
+ return list?.find((c) => c.path === "" || path === c.path || path.startsWith(`${c.path}/`))?.context;
2889
+ }
2890
+ // ---------- watcher ----------
2891
+ startWatcher(name) {
2892
+ if (this.watchers.has(name)) return;
2893
+ const c = this.sidecar.collection(name);
2894
+ if (!c) return;
2895
+ try {
2896
+ const w = watch(c.path, { recursive: true }, (_event, file) => {
2897
+ if (file && /(^|[\\/])\./.test(String(file))) return;
2898
+ clearTimeout(this.watchTimers.get(name));
2899
+ this.watchTimers.set(
2900
+ name,
2901
+ setTimeout(() => {
2902
+ if (!this.closed) this.sync({ collection: name, full: true });
2903
+ }, this.opts.watchDebounceMs ?? 2e3)
2904
+ );
2905
+ });
2906
+ w.on("error", () => void 0);
2907
+ this.watchers.set(name, w);
2908
+ } catch {
2909
+ }
2910
+ }
2911
+ // ---------- indexing ----------
2912
+ sync(req) {
2913
+ const col = this.sidecar.collection(req.collection);
2914
+ if (!col) throw new EngineError("not_found", `Unknown collection "${req.collection}"`);
2915
+ for (const r of req.renamed ?? []) {
2916
+ this.sidecar.recordRename(sourceId(req.collection, r.from), sourceId(req.collection, r.to));
2917
+ }
2918
+ const jobId = this.jobs.enqueue("sync", `sync:${req.collection}`, (ctx) => this.runSync(ctx, req.collection));
2919
+ return { jobId };
2920
+ }
2921
+ async runSync(ctx, name) {
2922
+ const col = this.sidecar.collection(name);
2923
+ if (!col) return;
2924
+ const hasAttachments = await syncAttachments(this.sidecar, this.paths.attachments, name, col.path, { ...col.options }, () => ctx.pause());
2925
+ if (hasAttachments && !this.attRegistered.has(name)) {
2926
+ await this.backend.addCollection(name + ATT_SUFFIX, attachmentDir(this.paths.attachments, name));
2927
+ this.attRegistered.add(name);
2928
+ }
2929
+ if (ctx.cancelled()) return;
2930
+ await ctx.pause();
2931
+ await this.backend.update(this.qmdNames([name]), (done, total) => ctx.progress(done, total, { stage: "scan", collection: name }));
2932
+ if (ctx.cancelled()) return;
2933
+ const changed = this.refreshMeta(name);
2934
+ this.vocabCache = null;
2935
+ this.sidecar.markIndexed(name, Date.now());
2936
+ if (changed.length > 0) this.emit({ event: "results.stale", data: { collection: name, sources: changed } });
2937
+ if (await this.backend.pendingEmbeddings() > 0) {
2938
+ this.jobs.enqueue("embed", "embed", (c) => this.runEmbed(c));
2939
+ } else {
2940
+ this.emitReady();
2941
+ }
2942
+ }
2943
+ async runEmbed(ctx) {
2944
+ const stored = this.reader.vectorModels();
2945
+ const current = this.backend.embedModelUri();
2946
+ if (stored.length > 0 && current && stored.some((m) => m.model !== current)) {
2947
+ this.emit({ event: "model.changed", data: { model: this.backend.embedModelId(), previous: stored.map((m) => m.model) } });
2948
+ }
2949
+ await ctx.pause();
2950
+ await this.backend.embedPending(
2951
+ (done, total) => ctx.progress(done, total, { stage: "embed" }),
2952
+ () => ctx.cancelled() || this.jobs.paused
2953
+ );
2954
+ if (!ctx.cancelled() && !this.jobs.paused) this.emitReady();
2955
+ }
2956
+ emitReady() {
2957
+ const collections2 = this.sidecar.collectionNames().map((name) => ({ name, ...this.countsFor(name) }));
2958
+ this.emit({ event: "index.ready", data: { collections: collections2 } });
2959
+ }
2960
+ countsFor(name) {
2961
+ let sources = 0;
2962
+ let embedded = 0;
2963
+ for (const q of this.qmdNames([name])) {
2964
+ const c = this.reader.counts(q);
2965
+ sources += c.sources;
2966
+ embedded += c.embedded;
2967
+ }
2968
+ return { sources, embedded };
2969
+ }
2970
+ /** Brings trova_meta and trova_blocks in line with qmd's documents. Returns the changed source ids. */
2971
+ refreshMeta(name) {
2972
+ const col = this.sidecar.collection(name);
2973
+ const map = this.sidecar.entries(name);
2974
+ const seen = /* @__PURE__ */ new Set();
2975
+ const upserts = [];
2976
+ const changed = [];
2977
+ for (const qmdCollection of this.qmdNames([name])) {
2978
+ for (const row of this.reader.activeDocs(qmdCollection)) {
2979
+ const path = publicPath(qmdCollection, row.path);
2980
+ seen.add(path);
2981
+ const existing = map.get(path);
2982
+ if (existing && existing.hash === row.hash) continue;
2983
+ const text = this.reader.content(row.hash) ?? "";
2984
+ let created = Date.parse(row.created_at) || Date.now();
2985
+ let modified = Date.parse(row.modified_at) || Date.now();
2986
+ try {
2987
+ const stat = statSync3(join5(col.path, path));
2988
+ created = stat.birthtimeMs || created;
2989
+ modified = stat.mtimeMs;
2990
+ } catch {
2991
+ }
2992
+ upserts.push({ entry: buildMeta(qmdCollection, row.path, row.hash, text, { created, modified }), blocks: splitBlocks(text) });
2993
+ changed.push(sourceId(name, path));
2994
+ }
2995
+ }
2996
+ const removals = [...map.keys()].filter((p) => !seen.has(p));
2997
+ for (const p of removals) changed.push(sourceId(name, p));
2998
+ if (upserts.length > 0 || removals.length > 0) this.sidecar.apply(name, upserts, removals);
2999
+ return changed;
3000
+ }
3001
+ attachmentText(req) {
3002
+ if (!this.sidecar.collection(req.collection)) throw new EngineError("not_found", `Unknown collection "${req.collection}"`);
3003
+ if (attachmentType(req.path) !== "image") throw new EngineError("bad_request", "Only image text can be pushed");
3004
+ storePushedText(this.sidecar, this.paths.attachments, req.collection, req.path, req.text);
3005
+ return this.sync({ collection: req.collection, changed: [req.path] });
3006
+ }
3007
+ cancelJob(jobId) {
3008
+ if (!this.jobs.cancel(jobId)) throw new EngineError("not_found", `Unknown job "${jobId}"`);
3009
+ }
3010
+ // ---------- status ----------
3011
+ status() {
3012
+ const models = this.backend.modelStates();
3013
+ const dims = this.reader.dims() ?? void 0;
3014
+ const hasVec = this.reader.hasVectors();
3015
+ const capabilities = ["lex", "fuzzy", "blocks", "attachments"];
3016
+ if (hasVec || models.embed.state !== "missing") capabilities.push("vec", "relatedByVector", "duplicates", "topics", "graph");
3017
+ if (models.rerank.state === "ready") capabilities.push("rerank");
3018
+ if (models.expand.state === "ready") capabilities.push("expansion");
3019
+ const collections2 = this.sidecar.collectionNames().map((name) => {
3020
+ const c = this.sidecar.collection(name);
3021
+ const counts = this.countsFor(name);
3022
+ const oversized = [...this.sidecar.entries(name).values()].filter((e) => e.oversized).map((e) => e.path);
3023
+ return {
3024
+ name,
3025
+ path: c.path,
3026
+ sources: counts.sources,
3027
+ embedded: counts.embedded,
3028
+ pending: Math.max(0, counts.sources - counts.embedded),
3029
+ ...oversized.length ? { oversized } : {},
3030
+ ...c.lastIndexed ? { lastIndexed: c.lastIndexed } : {}
3031
+ };
3032
+ });
3033
+ const status = {
3034
+ api: API_VERSION,
3035
+ engine: { name: ENGINE_NAME, version: ENGINE_VERSION, backend: this.backend.name },
3036
+ capabilities,
3037
+ models: { embed: { ...models.embed, ...dims ? { dims } : {} }, rerank: models.rerank, expand: models.expand },
3038
+ collections: collections2,
3039
+ jobs: this.jobs.active(),
3040
+ paused: this.jobs.paused
3041
+ };
3042
+ const stored = this.reader.vectorModels();
3043
+ const current = this.backend.embedModelUri();
3044
+ const stale = stored.filter((m) => m.model !== current).reduce((sum, m) => sum + m.chunks, 0);
3045
+ if (stale > 0 && current) {
3046
+ const total = stored.reduce((sum, m) => sum + m.chunks, 0);
3047
+ status.reembedding = { done: total - stale, total };
3048
+ }
3049
+ return status;
3050
+ }
3051
+ pause() {
3052
+ this.jobs.pause();
3053
+ }
3054
+ async resume() {
3055
+ this.jobs.resume();
3056
+ if (await this.backend.pendingEmbeddings() > 0 && !this.closed) this.jobs.enqueue("embed", "embed", (c) => this.runEmbed(c));
3057
+ }
3058
+ async drain() {
3059
+ await this.jobs.drain();
3060
+ }
3061
+ keepWarm() {
3062
+ void this.backend.keepWarm();
3063
+ }
3064
+ /** Starts the download of the Deep models. The caller must have the user's confirmation. */
3065
+ downloadModels() {
3066
+ const jobId = this.jobs.enqueue("models", "models", async () => {
3067
+ this.emit({ event: "engine.state", data: { state: "downloading", models: ["rerank", "expand"] } });
3068
+ await this.backend.downloadDeepModels();
3069
+ this.emit({ event: "engine.state", data: { state: "running", models: "ready" } });
3070
+ });
3071
+ return { jobId };
3072
+ }
3073
+ // ---------- result building ----------
3074
+ filterCtx(collection) {
3075
+ const graph = this.sidecar.graph(collection);
3076
+ return { isOrphan: (path) => this.sidecar.isOrphan(collection, path), resolveLink: graph.resolve };
3077
+ }
3078
+ passes(collection, entry, filters) {
3079
+ if (entry.excluded) return false;
3080
+ if (filters.length === 0) return true;
3081
+ return matchesFilters(entry, filters, this.filterCtx(collection));
3082
+ }
3083
+ async toResult(hit, q, label) {
3084
+ const { collection, entry } = hit;
3085
+ const source = sourceId(collection, entry.path);
3086
+ const text = this.reader.content(entry.hash) ?? "";
3087
+ const blocks = this.sidecar.blocks(source);
3088
+ const fmEnd = frontmatterEnd(text);
3089
+ const ranges = q ? queryRanges(text, q).filter((r) => r[0] >= fmEnd) : [];
3090
+ let block;
3091
+ let from = fmEnd;
3092
+ if (hit.pos !== void 0) {
3093
+ block = blockAt(blocks, hit.pos);
3094
+ from = Math.max(fmEnd, block?.start ?? 0, hit.pos);
3095
+ } else if (ranges.length > 0) {
3096
+ let best = 0;
3097
+ for (const b of blocks) {
3098
+ const count = ranges.filter((r) => r[0] >= b.start && r[0] < b.end).length;
3099
+ if (count > best) {
3100
+ best = count;
3101
+ block = b;
3102
+ }
3103
+ }
3104
+ from = Math.max(fmEnd, block?.start ?? 0);
3105
+ }
3106
+ const to = block ? block.end : text.length;
3107
+ let snip = makeSnippet(text, ranges, 180, Math.min(from, to), to);
3108
+ if (snip.snippet.trim() === "" && block) snip = makeSnippet(text, ranges, 180, Math.max(fmEnd, block.start), to);
3109
+ const result = {
3110
+ id: block ? blockId(source, block.headingPath) : source,
3111
+ source,
3112
+ path: entry.path,
3113
+ title: entry.title,
3114
+ collection,
3115
+ snippet: snip.snippet,
3116
+ highlights: snip.highlights,
3117
+ scores: hit.scores,
3118
+ matchedBy: hit.matchedBy,
3119
+ label
3120
+ };
3121
+ if (block) result.block = { id: blockId(source, block.headingPath), headings: block.headings, start: block.start, end: block.end };
3122
+ const type = extname(entry.path) === "md" ? null : attachmentType(entry.path) ?? extname(entry.path);
3123
+ if (type) result.type = type;
3124
+ const context = await this.contextFor(collection, entry.path);
3125
+ if (context) result.context = context;
3126
+ return result;
3127
+ }
3128
+ // ---------- lanes ----------
3129
+ lexHits(q, collections2, limit2) {
3130
+ const match = buildFtsQuery(q);
3131
+ const hits = [];
3132
+ if (!match) {
3133
+ if (q.filters.length === 0 || q.must.length > 0) return [];
3134
+ for (const collection of collections2) {
3135
+ for (const entry of this.sidecar.entries(collection).values()) {
3136
+ if (this.passes(collection, entry, q.filters)) hits.push({ collection, entry, scores: { lex: 0 }, matchedBy: ["lex"] });
3137
+ }
3138
+ }
3139
+ return hits.sort((a, b) => b.entry.modified - a.entry.modified || (a.entry.path < b.entry.path ? -1 : 1)).slice(0, limit2);
3140
+ }
3141
+ const strictGroups = q.must.filter((g) => g.some((t) => t.kind !== "word" || t.field === "heading"));
3142
+ const strictNot = q.mustNot.filter((t) => t.kind !== "word");
3143
+ const strict = strictGroups.length || strictNot.length ? { ...q, must: strictGroups, mustNot: strictNot } : null;
3144
+ const selective = strict !== null || q.filters.length > 0;
3145
+ const rows = this.reader.lex(match, this.qmdNames(collections2), selective ? Math.max(200, limit2 * 10) : limit2 * 2);
3146
+ const seen = /* @__PURE__ */ new Set();
3147
+ for (const row of rows) {
3148
+ const collection = this.publicCollection(row.collection);
3149
+ const entry = this.sidecar.entry(collection, publicPath(row.collection, row.path));
3150
+ if (!entry || entry.hash !== row.hash || !this.passes(collection, entry, q.filters)) continue;
3151
+ const key = `${collection}:${entry.path}`;
3152
+ if (seen.has(key)) continue;
3153
+ if (strict) {
3154
+ const text = this.reader.content(entry.hash) ?? "";
3155
+ const headings = this.sidecar.blocks(key).flatMap((b) => b.headings.slice(-1));
3156
+ if (!matchesTerms({ title: entry.title, aliases: entry.aliases, headings, body: text }, strict)) continue;
3157
+ }
3158
+ seen.add(key);
3159
+ hits.push({ collection, entry, scores: { lex: row.score }, matchedBy: ["lex"] });
3160
+ if (hits.length >= limit2) break;
3161
+ }
3162
+ return hits;
3163
+ }
3164
+ vocabulary() {
3165
+ if (!this.vocabCache) this.vocabCache = this.reader.vocabulary();
3166
+ return this.vocabCache;
3167
+ }
3168
+ /** The lex lane with typo rescue: when few results arrive, misspelt words are replaced from the vocabulary. */
3169
+ lexLane(q, collections2, limit2, fuzzy) {
3170
+ const hits = this.lexHits(q, collections2, limit2);
3171
+ if (!fuzzy || hits.length >= TYPO_RESCUE_MAX_RESULTS) return { hits, query: q };
3172
+ let rescued = q;
3173
+ const from = [];
3174
+ const to = [];
3175
+ for (const group of q.must) {
3176
+ for (const term of group) {
3177
+ if (term.kind !== "word" || typoMaxEdits(term.text) === 0 || this.reader.hasPrefix(term.text)) continue;
3178
+ const best = suggestCorrections(term.text, this.vocabulary(), 1, 5, true)[0];
3179
+ if (!best) continue;
3180
+ rescued = replaceWord(rescued, term.text, best.term);
3181
+ from.push(term.text);
3182
+ to.push(best.term);
3183
+ }
3184
+ }
3185
+ if (from.length === 0) return { hits, query: q };
3186
+ const fixed = this.lexHits(rescued, collections2, limit2).map((h) => ({ ...h, matchedBy: ["fuzzy"] }));
3187
+ if (fixed.length <= hits.length) return { hits, query: q };
3188
+ return { hits: fixed, correction: { from: from.join(" "), to: to.join(" ") }, query: rescued };
3189
+ }
3190
+ async vecHits(text, collections2, limit2, filters, asDocument = false) {
3191
+ if (!text.trim() || !this.reader.hasVectors()) return [];
3192
+ const vector = await this.backend.embedText(text, !asDocument);
3193
+ if (!vector) return [];
3194
+ return this.nearest(vector, collections2, limit2, filters, /* @__PURE__ */ new Set());
3195
+ }
3196
+ /** Nearest stored chunks mapped to sources: one hit per source, at its best chunk. */
3197
+ nearest(vector, collections2, limit2, filters, skipHashes, k = limit2 * 6) {
3198
+ const allowed = new Set(collections2);
3199
+ const near = this.reader.knn(vector, k + skipHashes.size * 4, this.qmdNames(collections2)).filter((n) => !skipHashes.has(n.hash));
3200
+ const docs = this.reader.docsByHashes([...new Set(near.map((n) => n.hash))]);
3201
+ const chunkPos = this.reader.chunksFor([...docs.keys()]);
3202
+ const best = /* @__PURE__ */ new Map();
3203
+ for (const n of near) {
3204
+ for (const doc of docs.get(n.hash) ?? []) {
3205
+ const collection = this.publicCollection(doc.collection);
3206
+ if (!allowed.has(collection)) continue;
3207
+ const entry = this.sidecar.entry(collection, publicPath(doc.collection, doc.path));
3208
+ if (!entry || entry.hash !== n.hash || !this.passes(collection, entry, filters)) continue;
3209
+ const key = `${collection}:${entry.path}`;
3210
+ const prev = best.get(key);
3211
+ if (prev && (prev.scores.vec ?? 0) >= n.similarity) continue;
3212
+ const pos = chunkPos.get(n.hash)?.find((c) => c.seq === n.seq)?.pos ?? 0;
3213
+ best.set(key, { collection, entry, pos, scores: { vec: n.similarity }, matchedBy: ["vec"] });
3214
+ }
3215
+ }
3216
+ return [...best.values()].sort((a, b) => (b.scores.vec ?? 0) - (a.scores.vec ?? 0)).slice(0, limit2);
3217
+ }
3218
+ hitKey(h) {
3219
+ return `${h.collection}:${h.entry.path}`;
3220
+ }
3221
+ fuse(lists) {
3222
+ const byKey = /* @__PURE__ */ new Map();
3223
+ for (const list of lists) {
3224
+ for (const h of list) {
3225
+ const key = this.hitKey(h);
3226
+ const prev = byKey.get(key);
3227
+ if (!prev) byKey.set(key, { ...h, scores: { ...h.scores }, matchedBy: [...h.matchedBy] });
3228
+ else {
3229
+ prev.scores = { ...h.scores, ...prev.scores, lex: prev.scores.lex ?? h.scores.lex, vec: prev.scores.vec ?? h.scores.vec };
3230
+ for (const m of h.matchedBy) if (!prev.matchedBy.includes(m)) prev.matchedBy.push(m);
3231
+ if (prev.pos === void 0 && !prev.matchedBy.includes("lex") && !prev.matchedBy.includes("fuzzy")) prev.pos = h.pos;
3232
+ }
3233
+ }
3234
+ }
3235
+ const fused = rrf(lists.map((items) => ({ weight: 1, items: items.map((h) => this.hitKey(h)) })));
3236
+ return fused.map((f, i) => {
3237
+ const h = byKey.get(f.id);
3238
+ h.scores.fused = f.score;
3239
+ h.scores.fusedRank = i + 1;
3240
+ if (h.matchedBy.includes("lex") || h.matchedBy.includes("fuzzy")) delete h.pos;
3241
+ return h;
3242
+ });
3243
+ }
3244
+ async toResults(hits, q, labelOf) {
3245
+ const out2 = [];
3246
+ for (let i = 0; i < hits.length; i++) {
3247
+ const label = labelOf(hits[i], i + 1);
3248
+ if (label) out2.push(await this.toResult(hits[i], q, label));
3249
+ }
3250
+ return out2;
3251
+ }
3252
+ cosineLabel = (hit) => labelForCosine(hit.scores.vec ?? 0, thresholdsFor(this.backend.embedModelId()));
3253
+ // ---------- search ----------
3254
+ async *search(req, signal) {
3255
+ const end = this.jobs.beginInteractive();
3256
+ try {
3257
+ this.validateQuery(req.query);
3258
+ const q = req.query;
3259
+ const collections2 = this.collectionsFor(req.collections);
3260
+ const limit2 = Math.max(1, Math.min(200, req.limit ?? 20));
3261
+ const fuzzy = req.fuzzy !== false;
3262
+ const deadline = req.deadlineMs ? Date.now() + req.deadlineMs : Infinity;
3263
+ const text = queryText(q);
3264
+ const lanes = q.lane ? [q.lane === "lex" ? "lex" : "vec"] : req.lanes?.length ? req.lanes : ["lex"];
3265
+ let lex = null;
3266
+ const getLex = () => lex ??= this.lexLane(q, collections2, limit2, fuzzy);
3267
+ for (let i = 0; i < lanes.length; i++) {
3268
+ checkAborted(signal);
3269
+ const lane = lanes[i];
3270
+ const last = i === lanes.length - 1;
3271
+ if (lane === "lex") {
3272
+ const l = getLex();
3273
+ const results = await this.toResults(l.hits, l.query, (_h, rank) => labelForRank(rank));
3274
+ yield { stage: "lex", results, final: last, ...l.correction ? { correction: l.correction } : {} };
3275
+ } else if (lane === "vec") {
3276
+ const hits = await withTimeout(this.vecHits(text, collections2, limit2, q.filters, q.lane === "hyde"), deadline - Date.now());
3277
+ checkAborted(signal);
3278
+ const results = hits === "timeout" ? [] : await this.toResults(hits, q, this.cosineLabel);
3279
+ yield { stage: "vec", results, final: last };
3280
+ } else {
3281
+ const l = getLex();
3282
+ if (!lanes.includes("lex")) {
3283
+ const results = await this.toResults(l.hits, l.query, (_h, rank) => labelForRank(rank));
3284
+ yield { stage: "lex", results, final: false, ...l.correction ? { correction: l.correction } : {} };
3285
+ }
3286
+ const vec = await withTimeout(this.vecHits(text, collections2, limit2, q.filters), deadline - Date.now());
3287
+ checkAborted(signal);
3288
+ const fused = this.fuse([l.hits, vec === "timeout" ? [] : vec]).slice(0, limit2);
3289
+ yield { stage: "fused", results: await this.toResults(fused, l.query, (_h, rank) => labelForRank(rank)), final: last };
3290
+ }
3291
+ if (Date.now() > deadline && !last) {
3292
+ yield { stage: lane === "hybrid" ? "fused" : lane, results: [], final: true };
3293
+ return;
3294
+ }
3295
+ }
3296
+ } finally {
3297
+ end();
3298
+ }
3299
+ }
3300
+ validateQuery(q) {
3301
+ const p = q;
3302
+ if (!p || !Array.isArray(p.must) || !Array.isArray(p.mustNot) || !Array.isArray(p.filters)) {
3303
+ throw new EngineError("bad_request", "The request needs a parsed query with must, mustNot and filters");
3304
+ }
3305
+ }
3306
+ // ---------- ask ----------
3307
+ async *ask(req, signal) {
3308
+ const end = this.jobs.beginInteractive();
3309
+ try {
3310
+ this.validateQuery(req.query);
3311
+ const q = req.query;
3312
+ const collections2 = this.collectionsFor(req.collections);
3313
+ const limit2 = Math.max(1, Math.min(100, req.limit ?? ASK_FAST_LIMIT));
3314
+ const fuzzy = req.fuzzy !== false;
3315
+ const text = queryText(q);
3316
+ const rankLabel = (_h, rank) => labelForRank(rank);
3317
+ if (q.lane) {
3318
+ if (q.lane === "lex") {
3319
+ const l = this.lexLane(q, collections2, limit2, fuzzy);
3320
+ yield { stage: "lex", results: await this.toResults(l.hits, l.query, rankLabel), final: true, ...l.correction ? { correction: l.correction } : {} };
3321
+ } else {
3322
+ const hits = await this.vecHits(text, collections2, limit2, q.filters, q.lane === "hyde");
3323
+ yield { stage: "vec", results: await this.toResults(hits, q, rankLabel), final: true };
3324
+ }
3325
+ return;
3326
+ }
3327
+ let mode = req.mode === "auto" || !req.mode ? pickAskMode(q) : req.mode;
3328
+ const started = Date.now();
3329
+ const deadline = started + (req.deadlineMs ?? ASK_DEEP_DEADLINE_MS);
3330
+ const lex = this.lexLane(q, collections2, ASK_LANE_LIMIT, fuzzy);
3331
+ this.relaxLex(lex, collections2);
3332
+ yield {
3333
+ stage: "lex",
3334
+ results: await this.toResults(lex.hits.slice(0, limit2), lex.query, rankLabel),
3335
+ final: false,
3336
+ mode,
3337
+ ...lex.correction ? { correction: lex.correction } : {}
3338
+ };
3339
+ checkAborted(signal);
3340
+ const vec = await this.vecHits(text, collections2, ASK_LANE_LIMIT, q.filters);
3341
+ checkAborted(signal);
3342
+ const fused = this.fuse([lex.hits, vec]);
3343
+ const fusedResults = await this.toResults(fused.slice(0, limit2), lex.query, rankLabel);
3344
+ let note;
3345
+ if (mode === "deep" && !this.backend.deepReady()) {
3346
+ if (req.allowModelDownload) this.downloadModels();
3347
+ mode = "fast";
3348
+ note = "model_missing";
3349
+ }
3350
+ if (mode === "fast") {
3351
+ yield { stage: "fused", results: fusedResults, final: true, mode, ...note ? { note } : {} };
3352
+ return;
3353
+ }
3354
+ yield { stage: "fused", results: fusedResults, final: false, mode };
3355
+ const deep = await withTimeout(this.deepPass(q, lex, vec, collections2, limit2, fuzzy, signal), deadline - Date.now());
3356
+ checkAborted(signal);
3357
+ if (deep === "timeout") {
3358
+ yield { stage: "fused", results: fusedResults, final: true, mode, note: "rerank_skipped" };
3359
+ return;
3360
+ }
3361
+ yield { stage: "rerank", results: deep.results, final: true, mode, expansions: deep.expansions };
3362
+ } finally {
3363
+ end();
3364
+ }
3365
+ }
3366
+ /**
3367
+ * A question rarely has every word in one note. When the all-words query finds few hits, Ask
3368
+ * adds the notes that match any of its content words, ranked by BM25 behind the strict hits.
3369
+ */
3370
+ relaxLex(lex, collections2) {
3371
+ const q = lex.query;
3372
+ if (lex.hits.length >= ASK_RELAX_BELOW || q.must.length < 2) return;
3373
+ const words = q.must.flat().filter((t) => t.kind !== "word" || !ASK_STOPWORDS.has(t.text.toLowerCase().replace(/[^\p{L}\p{N}]/gu, "")));
3374
+ if (words.length < 2) return;
3375
+ const relaxed = { ...q, must: [words] };
3376
+ const seen = new Set(lex.hits.map((h) => this.hitKey(h)));
3377
+ for (const h of this.lexHits(relaxed, collections2, ASK_LANE_LIMIT)) {
3378
+ if (!seen.has(this.hitKey(h)) && lex.hits.length < ASK_LANE_LIMIT) lex.hits.push(h);
3379
+ }
3380
+ lex.query = relaxed;
3381
+ }
3382
+ async deepPass(q, lex, vec, collections2, limit2, fuzzy, signal) {
3383
+ const text = queryText(q);
3384
+ const expansions = (await this.backend.expand(text)).filter((e) => e.query.trim() && e.query.trim().toLowerCase() !== text.toLowerCase());
3385
+ const lists = [lex.hits, vec];
3386
+ for (const sub of expansions) {
3387
+ if (signal?.aborted) break;
3388
+ if (sub.type === "lex") {
3389
+ const parsed = parseQuery(sub.query, { mode: "ask" }).query;
3390
+ lists.push(this.lexLane({ ...parsed, filters: q.filters }, collections2, ASK_LANE_LIMIT, fuzzy).hits);
3391
+ } else {
3392
+ lists.push(await this.vecHits(sub.query, collections2, ASK_LANE_LIMIT, q.filters, sub.type === "hyde"));
3393
+ }
3394
+ }
3395
+ const fused = this.fuse(lists);
3396
+ const top = fused.slice(0, ASK_RERANK_TOP);
3397
+ const prelim = await this.toResults(top, lex.query, (_h, rank) => labelForRank(rank));
3398
+ const docs = prelim.map((r) => {
3399
+ const entry = this.sidecar.entry(r.collection, r.path);
3400
+ const full = this.reader.content(entry.hash) ?? "";
3401
+ const body = r.block ? full.slice(r.block.start, r.block.end) : full.slice(frontmatterEnd(full));
3402
+ return { id: r.id, text: `${r.title}
3403
+ ${body}`.slice(0, 2400) };
3404
+ });
3405
+ const scores = await this.backend.rerank(text, docs);
3406
+ const maxFused = Math.max(...prelim.map((r) => r.scores.fused ?? 0), 1e-9);
3407
+ const blended = prelim.map((r, i) => {
3408
+ const rerank = scores.get(r.id);
3409
+ const final = rerank === void 0 ? (r.scores.fused ?? 0) / maxFused : blendRerank(i + 1, (r.scores.fused ?? 0) / maxFused, rerank);
3410
+ const matchedBy = rerank === void 0 ? r.matchedBy : [...r.matchedBy, "rerank"];
3411
+ return { r: { ...r, matchedBy, scores: { ...r.scores, ...rerank === void 0 ? {} : { rerank } } }, final };
3412
+ }).sort((a, b) => b.final - a.final).slice(0, limit2).map(({ r }, i) => ({ ...r, label: labelForRank(i + 1) }));
3413
+ return { results: blended, expansions };
3414
+ }
3415
+ // ---------- related ----------
3416
+ locate(id) {
3417
+ let parsed = parseId(id);
3418
+ if (!parsed) return null;
3419
+ if (!this.sidecar.collection(parsed.collection)) return null;
3420
+ let entry = this.sidecar.entry(parsed.collection, parsed.path);
3421
+ if (!entry) {
3422
+ const current = parseId(this.sidecar.currentId(parsed.source));
3423
+ if (current) {
3424
+ entry = this.sidecar.entry(current.collection, current.path);
3425
+ if (entry) parsed = { ...current, headingPath: parsed.headingPath };
3426
+ }
3427
+ }
3428
+ if (!entry || entry.excluded) return null;
3429
+ return { collection: parsed.collection, entry, headingPath: parsed.headingPath };
3430
+ }
3431
+ /** The vector of an anchor: stored vectors for a source or block, a fresh embedding for text. */
3432
+ async anchorVector(anchor) {
3433
+ const set = [anchor.source, anchor.block, anchor.text].filter((v) => v != null && v !== "");
3434
+ if (set.length !== 1) throw new EngineError("bad_request", "Exactly one of source, block or text must be set on the anchor");
3435
+ if (anchor.text) {
3436
+ if (!this.reader.hasVectors()) return null;
3437
+ const vector2 = await this.backend.embedText(anchor.text, false);
3438
+ return vector2 ? { vector: vector2, text: anchor.text } : null;
3439
+ }
3440
+ const id = anchor.block ?? anchor.source;
3441
+ const found = this.locate(id);
3442
+ if (!found) throw new EngineError("not_found", `Unknown source or block "${id}"`);
3443
+ const { collection, entry } = found;
3444
+ const chunks = this.reader.chunks(entry.hash);
3445
+ if (chunks.length === 0) return null;
3446
+ const full = this.reader.content(entry.hash) ?? "";
3447
+ let wanted = chunks;
3448
+ let text = full;
3449
+ if (anchor.block) {
3450
+ const block = findBlock(this.sidecar.blocks(sourceId(collection, entry.path)), found.headingPath ?? "");
3451
+ if (!block) throw new EngineError("not_found", `Unknown block "${id}"`);
3452
+ wanted = chunks.filter((c, i) => {
3453
+ const chunkEnd = i + 1 < chunks.length ? chunks[i + 1].pos : full.length;
3454
+ return c.pos < block.end && Math.max(chunkEnd, c.pos + 1) > block.start;
3455
+ });
3456
+ if (wanted.length === 0) wanted = [chunks[chunks.length - 1]];
3457
+ text = blockText(full, block);
3458
+ }
3459
+ const vectors = this.reader.vectors(wanted.map((c) => `${c.hash}_${c.seq}`));
3460
+ const vector = meanVector([...vectors.values()]);
3461
+ return vector ? { vector, collection, entry, text } : null;
3462
+ }
3463
+ async related(req, signal) {
3464
+ return this.jobs.interactiveCall(async () => {
3465
+ if (!req.anchor) throw new EngineError("bad_request", "The request needs an anchor");
3466
+ const anchor = await this.anchorVector(req.anchor);
3467
+ checkAborted(signal);
3468
+ if (!anchor) return { anchorIndexed: false, results: [] };
3469
+ const collections2 = this.collectionsFor(req.collections);
3470
+ const limit2 = Math.max(1, Math.min(100, req.limit ?? 20));
3471
+ const perSource = Math.max(1, req.blocksPerSource ?? 3);
3472
+ const thresholds = thresholdsFor(this.backend.embedModelId());
3473
+ const granularity = req.granularity ?? "both";
3474
+ const filters = req.filters ?? [];
3475
+ const excluded = new Set((req.exclude?.sources ?? []).map((s) => parseId(s)?.source ?? s));
3476
+ const anchorSource = anchor.entry ? sourceId(anchor.collection, anchor.entry.path) : null;
3477
+ if (anchor.entry && req.exclude?.linkedFromAnchor) {
3478
+ this.sidecar.graph(anchor.collection);
3479
+ for (const l of anchor.entry.links) excluded.add(sourceId(anchor.collection, l));
3480
+ }
3481
+ const own = anchor.entry ? /* @__PURE__ */ new Set([anchor.entry.hash]) : /* @__PURE__ */ new Set();
3482
+ const ownChunks = anchor.entry ? this.reader.chunks(anchor.entry.hash).length : 0;
3483
+ const near = this.reader.knn(anchor.vector, RELATED_NEAREST_CHUNKS + ownChunks, this.qmdNames(collections2)).filter((n) => !own.has(n.hash)).slice(0, RELATED_NEAREST_CHUNKS);
3484
+ const docs = this.reader.docsByHashes([...new Set(near.map((n) => n.hash))]);
3485
+ const chunkPos = this.reader.chunksFor([...docs.keys()]);
3486
+ const sources = /* @__PURE__ */ new Map();
3487
+ const allowed = new Set(collections2);
3488
+ for (const n of near) {
3489
+ for (const doc of docs.get(n.hash) ?? []) {
3490
+ const collection = this.publicCollection(doc.collection);
3491
+ if (!allowed.has(collection)) continue;
3492
+ const entry = this.sidecar.entry(collection, publicPath(doc.collection, doc.path));
3493
+ if (!entry || entry.hash !== n.hash) continue;
3494
+ const source = sourceId(collection, entry.path);
3495
+ if (source === anchorSource || excluded.has(source) || !this.passes(collection, entry, filters)) continue;
3496
+ const pos = chunkPos.get(n.hash)?.find((c) => c.seq === n.seq)?.pos ?? 0;
3497
+ const block = blockAt(this.sidecar.blocks(source), pos);
3498
+ if (!block) continue;
3499
+ let s = sources.get(source);
3500
+ if (!s) sources.set(source, s = { collection, entry, blocks: /* @__PURE__ */ new Map() });
3501
+ const prev = s.blocks.get(block.headingPath);
3502
+ if (!prev || prev.score < n.similarity) s.blocks.set(block.headingPath, { block, score: n.similarity, pos });
3503
+ }
3504
+ }
3505
+ const anchorKeywords = new Set(anchor.text ? keywords(anchor.text) : []);
3506
+ const anchorLinks = new Set(anchor.entry?.links ?? []);
3507
+ const anchorTags = new Set(anchor.entry?.tags ?? []);
3508
+ const scored = [...sources.values()].map((s) => {
3509
+ const blocks = [...s.blocks.values()].sort((a, b) => b.score - a.score);
3510
+ return { ...s, ranked: blocks, best: blocks[0].score, score: sourceScore(blocks.map((b) => b.score), thresholds) };
3511
+ }).filter((s) => {
3512
+ const label = labelForCosine(s.best, thresholds);
3513
+ return label !== null && (label !== "loose" || req.loose === true);
3514
+ }).sort((a, b) => b.score - a.score || (a.entry.path < b.entry.path ? -1 : 1));
3515
+ const results = [];
3516
+ for (const s of scored) {
3517
+ checkAborted(signal);
3518
+ const blockResults = [];
3519
+ for (const b of s.ranked.slice(0, perSource)) {
3520
+ const label = labelForCosine(b.score, thresholds);
3521
+ if (!label || label === "loose" && !req.loose) continue;
3522
+ blockResults.push(await this.toResult({ collection: s.collection, entry: s.entry, pos: b.block.start, scores: { vec: b.score }, matchedBy: ["vec"] }, null, label));
3523
+ }
3524
+ if (granularity === "block") {
3525
+ results.push(...blockResults);
3526
+ continue;
3527
+ }
3528
+ const top = await this.toResult(
3529
+ { collection: s.collection, entry: s.entry, pos: s.ranked[0].block.start, scores: { vec: s.best, fused: s.score }, matchedBy: ["vec"] },
3530
+ null,
3531
+ labelForCosine(s.best, thresholds)
3532
+ );
3533
+ const sourceResult = { ...top, id: top.source };
3534
+ delete sourceResult.block;
3535
+ if (granularity === "both") sourceResult.blocks = blockResults;
3536
+ const full = this.reader.content(s.entry.hash) ?? "";
3537
+ this.sidecar.graph(s.collection);
3538
+ const sameCollection = s.collection === anchor.collection;
3539
+ const links = sameCollection ? s.entry.links.filter((l) => anchorLinks.has(l)) : [];
3540
+ if (sameCollection && anchor.entry) {
3541
+ if (s.entry.links.includes(anchor.entry.path)) links.push(anchor.entry.path);
3542
+ if (anchorLinks.has(s.entry.path)) links.push(s.entry.path);
3543
+ }
3544
+ sourceResult.why = {
3545
+ keywords: keywords(full.slice(s.ranked[0].block.start, s.ranked[0].block.end), 200).filter((w) => anchorKeywords.has(w)).slice(0, 6),
3546
+ links: [...new Set(links)],
3547
+ tags: s.entry.tags.filter((t) => anchorTags.has(t))
3548
+ };
3549
+ results.push(sourceResult);
3550
+ }
3551
+ if (granularity === "block") results.sort((a, b) => (b.scores.vec ?? 0) - (a.scores.vec ?? 0));
3552
+ return { anchorIndexed: true, results: results.slice(0, limit2) };
3553
+ });
3554
+ }
3555
+ async score(req) {
3556
+ if (!Array.isArray(req.pairs)) throw new EngineError("bad_request", "The request needs pairs");
3557
+ const scores = [];
3558
+ for (const [a, b] of req.pairs) {
3559
+ const va = await this.anchorVector(a);
3560
+ const vb = await this.anchorVector(b);
3561
+ scores.push(va && vb ? Math.max(0, Math.min(1, cosine(va.vector, vb.vector))) : null);
3562
+ }
3563
+ return { scores };
3564
+ }
3565
+ // ---------- similarity graph ----------
3566
+ noteVectors = /* @__PURE__ */ new Map();
3567
+ /** The vector of a whole source: the normalised mean of its chunk vectors. Cached by content hash. */
3568
+ noteVector(hash2) {
3569
+ const hit = this.noteVectors.get(hash2);
3570
+ if (hit !== void 0) return hit;
3571
+ const keys = this.reader.chunks(hash2).map((c) => `${c.hash}_${c.seq}`);
3572
+ const vector = keys.length ? meanVector([...this.reader.vectors(keys).values()]) : null;
3573
+ if (this.noteVectors.size >= 4e3) this.noteVectors.delete(this.noteVectors.keys().next().value);
3574
+ this.noteVectors.set(hash2, vector ?? null);
3575
+ return vector ?? null;
3576
+ }
3577
+ /**
3578
+ * The similarity graph around one anchor: the related sources, an edge from the anchor to each,
3579
+ * and edges between related sources whose note vectors are similar. Each source keeps at most
3580
+ * `GRAPH_EDGES_PER_NODE` such edges, so the picture stays readable.
3581
+ */
3582
+ async graph(req, signal) {
3583
+ const limit2 = Math.max(1, Math.min(60, req.limit ?? 24));
3584
+ const related = await this.related({ anchor: req.anchor, granularity: "source", limit: limit2, collections: req.collections, filters: req.filters, loose: req.loose }, signal);
3585
+ if (!related.anchorIndexed) return { anchorIndexed: false, anchor: null, nodes: [], edges: [] };
3586
+ return this.jobs.interactiveCall(async () => {
3587
+ const thresholds = thresholdsFor(this.backend.embedModelId());
3588
+ const anchorId = req.anchor.source ?? (req.anchor.block ? parseId(req.anchor.block)?.source : null) ?? null;
3589
+ const anchorFound = anchorId ? this.locate(anchorId) : null;
3590
+ const anchor = anchorFound ? { id: sourceId(anchorFound.collection, anchorFound.entry.path), path: anchorFound.entry.path, title: anchorFound.entry.title, collection: anchorFound.collection, score: 1, label: "strong", snippet: "", tags: anchorFound.entry.tags } : null;
3591
+ const located = /* @__PURE__ */ new Map();
3592
+ const nodes = [];
3593
+ for (const r of related.results) {
3594
+ const found = this.locate(r.source);
3595
+ if (!found) continue;
3596
+ this.sidecar.graph(found.collection);
3597
+ located.set(r.source, found);
3598
+ nodes.push({ id: r.source, path: r.path, title: r.title, collection: found.collection, score: r.scores.vec ?? 0, label: r.label, snippet: r.snippet, tags: found.entry.tags });
3599
+ }
3600
+ const linked = (a, b) => a.collection === b.collection && (a.entry.links.includes(b.entry.path) || b.entry.links.includes(a.entry.path));
3601
+ const edges = [];
3602
+ if (anchor && anchorFound) {
3603
+ this.sidecar.graph(anchorFound.collection);
3604
+ for (const n of nodes) edges.push({ a: anchor.id, b: n.id, similarity: n.score, linked: linked(anchorFound, located.get(n.id)) });
3605
+ }
3606
+ checkAborted(signal);
3607
+ const vectors = nodes.map((n) => this.noteVector(located.get(n.id).entry.hash));
3608
+ const candidates = [];
3609
+ for (let i = 0; i < nodes.length; i++) {
3610
+ const vi = vectors[i];
3611
+ if (!vi) continue;
3612
+ for (let j = i + 1; j < nodes.length; j++) {
3613
+ const vj = vectors[j];
3614
+ if (!vj) continue;
3615
+ const similarity = cosine(vi, vj);
3616
+ if (similarity >= thresholds.related) candidates.push({ i, j, similarity });
3617
+ }
3618
+ }
3619
+ candidates.sort((x, y) => y.similarity - x.similarity);
3620
+ const degree = new Array(nodes.length).fill(0);
3621
+ for (const c of candidates) {
3622
+ if (degree[c.i] >= GRAPH_EDGES_PER_NODE || degree[c.j] >= GRAPH_EDGES_PER_NODE) continue;
3623
+ degree[c.i]++;
3624
+ degree[c.j]++;
3625
+ const a = nodes[c.i];
3626
+ const b = nodes[c.j];
3627
+ edges.push({ a: a.id, b: b.id, similarity: Math.min(1, c.similarity), linked: linked(located.get(a.id), located.get(b.id)) });
3628
+ }
3629
+ return { anchorIndexed: true, anchor, nodes, edges };
3630
+ });
3631
+ }
3632
+ // ---------- vocabulary and metadata ----------
3633
+ vocabSuggest(req) {
3634
+ if (!Array.isArray(req.terms)) throw new EngineError("bad_request", "The request needs terms");
3635
+ const suggestions = {};
3636
+ for (const term of req.terms) {
3637
+ suggestions[term] = this.reader.hasPrefix(term) && typoMaxEdits(term) === 0 ? [] : suggestCorrections(term, this.vocabulary(), 1, 5, true);
3638
+ }
3639
+ return { suggestions };
3640
+ }
3641
+ metadataKeys(collections2) {
3642
+ const keys = /* @__PURE__ */ new Map();
3643
+ for (const collection of this.collectionsFor(collections2)) {
3644
+ for (const entry of this.sidecar.entries(collection).values()) {
3645
+ if (entry.excluded) continue;
3646
+ for (const [key, value] of Object.entries(entry.frontmatter)) {
3647
+ const k = key.toLowerCase();
3648
+ let set = keys.get(k);
3649
+ if (!set) keys.set(k, set = /* @__PURE__ */ new Set());
3650
+ for (const v of Array.isArray(value) ? value : [value]) {
3651
+ if (set.size < 50 && v != null && typeof v !== "object") set.add(String(v));
3652
+ }
3653
+ }
3654
+ }
3655
+ }
3656
+ return { keys: Object.fromEntries([...keys.entries()].sort().map(([k, v]) => [k, [...v].sort()])) };
3657
+ }
3658
+ // ---------- documents ----------
3659
+ body(collection, entry, headingPath, neighbours = 0) {
3660
+ const source = sourceId(collection, entry.path);
3661
+ const text = this.reader.content(entry.hash) ?? "";
3662
+ if (headingPath === void 0) {
3663
+ return { id: source, source, path: entry.path, title: entry.title, content: text };
3664
+ }
3665
+ const blocks = this.sidecar.blocks(source);
3666
+ const index = blocks.findIndex((b) => b.headingPath === headingPath);
3667
+ if (index === -1) throw new EngineError("not_found", `Unknown block "${source}#${headingPath}"`);
3668
+ const make = (b) => ({
3669
+ id: blockId(source, b.headingPath),
3670
+ source,
3671
+ path: entry.path,
3672
+ title: entry.title,
3673
+ block: { id: blockId(source, b.headingPath), headings: b.headings, start: b.start, end: b.end },
3674
+ content: blockText(text, b)
3675
+ });
3676
+ const doc = make(blocks[index]);
3677
+ if (neighbours > 0) {
3678
+ doc.neighbours = {
3679
+ before: blocks.slice(Math.max(0, index - neighbours), index).map(make),
3680
+ after: blocks.slice(index + 1, index + 1 + neighbours).map(make)
3681
+ };
3682
+ }
3683
+ return doc;
3684
+ }
3685
+ get(req) {
3686
+ if (!req.id) throw new EngineError("bad_request", "The request needs an id");
3687
+ let found = this.locate(req.id);
3688
+ if (!found) {
3689
+ const [path, heading] = req.id.split(/#(.*)/s);
3690
+ for (const collection of this.collectionsFor(req.collection ? [req.collection] : void 0)) {
3691
+ const entry = this.sidecar.entry(collection, path) ?? this.sidecar.entry(collection, `${path}.md`);
3692
+ if (entry && !entry.excluded) {
3693
+ found = { collection, entry, headingPath: heading };
3694
+ break;
3695
+ }
3696
+ }
3697
+ }
3698
+ if (!found) throw new EngineError("not_found", `Unknown source or block "${req.id}"`);
3699
+ return this.body(found.collection, found.entry, found.headingPath, req.neighbours ?? 0);
3700
+ }
3701
+ multiGet(req) {
3702
+ const documents = [];
3703
+ const errors = [];
3704
+ const maxBytes = req.maxBytes ?? 512 * 1024;
3705
+ let bytes = 0;
3706
+ const push = (doc) => {
3707
+ bytes += Buffer.byteLength(doc.content);
3708
+ if (bytes > maxBytes && documents.length > 0) return false;
3709
+ documents.push(doc);
3710
+ return true;
3711
+ };
3712
+ for (const id of req.ids ?? []) {
3713
+ try {
3714
+ if (!push(this.get({ id, collection: req.collection }))) break;
3715
+ } catch (err) {
3716
+ errors.push({ id, code: err instanceof EngineError ? err.code : "internal" });
3717
+ }
3718
+ }
3719
+ if (req.glob) {
3720
+ const isMatch = picomatch(req.glob, { nocase: true });
3721
+ outer: for (const collection of this.collectionsFor(req.collection ? [req.collection] : void 0)) {
3722
+ for (const entry of [...this.sidecar.entries(collection).values()].sort((a, b) => a.path < b.path ? -1 : 1)) {
3723
+ if (entry.excluded || !isMatch(entry.path)) continue;
3724
+ if (!push(this.body(collection, entry, void 0))) break outer;
3725
+ }
3726
+ }
3727
+ }
3728
+ return { documents, errors };
3729
+ }
3730
+ // ---------- background analysis ----------
3731
+ analysisInput() {
3732
+ return {
3733
+ reader: this.reader,
3734
+ collections: this.sidecar.collectionNames().map((name) => ({
3735
+ name,
3736
+ qmdNames: this.qmdNames([name]),
3737
+ entries: this.sidecar.entries(name),
3738
+ blocks: (source) => this.sidecar.blocks(source)
3739
+ }))
3740
+ };
3741
+ }
3742
+ getDuplicates(start) {
3743
+ if (start && this.duplicates.state !== "running") {
3744
+ const jobId = this.jobs.enqueue("duplicates", "duplicates", async (ctx) => {
3745
+ const pairs = await findDuplicates(this.analysisInput(), ctx);
3746
+ if (ctx.cancelled()) {
3747
+ this.duplicates = { ...this.duplicates, state: this.duplicates.computedAt ? "done" : "none" };
3748
+ return;
3749
+ }
3750
+ this.duplicates = { state: "done", pairs, computedAt: Date.now() };
3751
+ this.sidecar.setValue("duplicates", this.duplicates);
3752
+ });
3753
+ this.duplicates = { ...this.duplicates, state: "running", jobId };
3754
+ }
3755
+ return this.duplicates;
3756
+ }
3757
+ getTopics(start) {
3758
+ if (start && this.topics.state !== "running") {
3759
+ const jobId = this.jobs.enqueue("topics", "topics", async (ctx) => {
3760
+ const clusters = await findTopics(this.analysisInput(), ctx);
3761
+ if (ctx.cancelled()) {
3762
+ this.topics = { ...this.topics, state: this.topics.computedAt ? "done" : "none" };
3763
+ return;
3764
+ }
3765
+ this.topics = { state: "done", clusters, computedAt: Date.now() };
3766
+ this.sidecar.setValue("topics", this.topics);
3767
+ });
3768
+ this.topics = { ...this.topics, state: "running", jobId };
3769
+ }
3770
+ return this.topics;
3771
+ }
3772
+ renameHistory() {
3773
+ return this.sidecar.renameHistory();
3774
+ }
3775
+ };
3776
+
3777
+ // src/routes.ts
3778
+ async function dispatch(engine, method, path, body, params, signal) {
3779
+ const b = body;
3780
+ const json = (value, status = 200) => ({ kind: "json", status, body: value });
3781
+ switch (`${method} ${path}`) {
3782
+ case "POST /v1/search":
3783
+ return { kind: "stream", batches: engine.search(b, signal) };
3784
+ case "POST /v1/ask":
3785
+ return { kind: "stream", batches: engine.ask(b, signal) };
3786
+ case "POST /v1/related":
3787
+ return json(await engine.related(b, signal));
3788
+ case "POST /v1/graph":
3789
+ return json(await engine.graph(b, signal));
3790
+ case "POST /v1/vocab/suggest":
3791
+ return json(engine.vocabSuggest(b));
3792
+ case "POST /v1/score":
3793
+ return json(await engine.score(b));
3794
+ case "POST /v1/documents/get":
3795
+ return json(engine.get(b));
3796
+ case "POST /v1/documents/multi-get":
3797
+ return json(engine.multiGet(b));
3798
+ case "POST /v1/index/sync":
3799
+ if (typeof body["collection"] !== "string") throw new EngineError("bad_request", "The request needs a collection");
3800
+ return json(engine.sync(b), 202);
3801
+ case "GET /v1/index/status":
3802
+ return json(engine.status());
3803
+ case "POST /v1/index/pause":
3804
+ engine.pause();
3805
+ return json({ paused: true });
3806
+ case "POST /v1/index/resume":
3807
+ await engine.resume();
3808
+ return json({ paused: false });
3809
+ case "GET /v1/collections":
3810
+ return json({ collections: await engine.listCollections() });
3811
+ case "POST /v1/collections":
3812
+ if (typeof body["rename"] === "string" && typeof body["name"] === "string") {
3813
+ await engine.renameCollection(body["name"], body["rename"]);
3814
+ return json({ collections: await engine.listCollections() });
3815
+ }
3816
+ if (typeof body["contextPath"] === "string" && typeof body["name"] === "string") {
3817
+ await engine.setContext(body["name"], body["contextPath"], typeof body["contextText"] === "string" ? body["contextText"] : null);
3818
+ return json({ collections: await engine.listCollections() });
3819
+ }
3820
+ return json(await engine.addCollection(b));
3821
+ case "DELETE /v1/collections": {
3822
+ const name = params.get("name") ?? body["name"];
3823
+ if (!name) throw new EngineError("bad_request", "The request needs a collection name");
3824
+ await engine.removeCollection(name);
3825
+ return { kind: "empty", status: 204 };
3826
+ }
3827
+ case "GET /v1/metadata/keys":
3828
+ return json(engine.metadataKeys(params.getAll("collection")));
3829
+ case "POST /v1/attachments/text":
3830
+ return json(engine.attachmentText(b), 202);
3831
+ case "GET /v1/duplicates":
3832
+ return json(engine.getDuplicates(false));
3833
+ case "POST /v1/duplicates":
3834
+ return json(engine.getDuplicates(true), 202);
3835
+ case "GET /v1/topics":
3836
+ return json(engine.getTopics(false));
3837
+ case "POST /v1/topics":
3838
+ return json(engine.getTopics(true), 202);
3839
+ case "POST /v1/models/download":
3840
+ return json(engine.downloadModels(), 202);
3841
+ case "POST /v1/keepalive":
3842
+ engine.keepWarm();
3843
+ return json({ ok: true });
3844
+ }
3845
+ const cancel = /^\/v1\/jobs\/([^/]+)\/cancel$/.exec(path);
3846
+ if (cancel && method === "POST") {
3847
+ engine.cancelJob(decodeURIComponent(cancel[1]));
3848
+ return { kind: "empty", status: 204 };
3849
+ }
3850
+ throw new EngineError("not_found", `No route for ${method} ${path}`);
3851
+ }
3852
+
3853
+ // src/client.ts
3854
+ function splitPath(path) {
3855
+ const url = new URL(path, "http://127.0.0.1");
3856
+ return { pathname: url.pathname, params: url.searchParams };
3857
+ }
3858
+ function localTrova(engine, ownsEngine = true) {
3859
+ return {
3860
+ local: true,
3861
+ async json(method, path, body) {
3862
+ const { pathname, params } = splitPath(path);
3863
+ const out2 = await dispatch(engine, method, pathname, body ?? {}, params);
3864
+ if (out2.kind === "json") return out2.body;
3865
+ return void 0;
3866
+ },
3867
+ async *stream(path, body, signal) {
3868
+ const out2 = await dispatch(engine, "POST", path, body, new URLSearchParams(), signal);
3869
+ if (out2.kind === "stream") yield* out2.batches;
3870
+ },
3871
+ async wait(onProgress) {
3872
+ const off = engine.on((e) => {
3873
+ if (e.event === "index.progress") onProgress?.(String(e.data["stage"] ?? e.data["kind"]), Number(e.data["done"]), Number(e.data["total"]));
3874
+ });
3875
+ await engine.drain();
3876
+ off();
3877
+ },
3878
+ async close() {
3879
+ if (ownsEngine) await engine.close();
3880
+ }
3881
+ };
3882
+ }
3883
+ function httpTrova(port, token) {
3884
+ const base = `http://127.0.0.1:${port}`;
3885
+ const headers = () => ({ Authorization: `Bearer ${token}`, "Content-Type": "application/json", "X-Request-Id": crypto.randomUUID() });
3886
+ const fail = async (res) => {
3887
+ const body = await res.json().catch(() => null);
3888
+ throw new EngineError(body?.error.code ?? "internal", body?.error.message ?? `HTTP ${res.status}`, body?.error.retryable ?? false);
3889
+ };
3890
+ const trova = {
3891
+ local: false,
3892
+ async json(method, path, body) {
3893
+ const res = await fetch(base + path, { method, headers: headers(), body: body === void 0 ? void 0 : JSON.stringify(body) });
3894
+ if (!res.ok) await fail(res);
3895
+ if (res.status === 204) return void 0;
3896
+ return await res.json();
3897
+ },
3898
+ async *stream(path, body, signal) {
3899
+ const res = await fetch(base + path, { method: "POST", headers: headers(), body: JSON.stringify(body), signal });
3900
+ if (!res.ok || !res.body) await fail(res);
3901
+ const decoder = new TextDecoder();
3902
+ let buffer = "";
3903
+ for await (const chunk of res.body) {
3904
+ buffer += decoder.decode(chunk, { stream: true });
3905
+ let nl;
3906
+ while ((nl = buffer.indexOf("\n")) !== -1) {
3907
+ const line = buffer.slice(0, nl).trim();
3908
+ buffer = buffer.slice(nl + 1);
3909
+ if (!line) continue;
3910
+ const value = JSON.parse(line);
3911
+ if (value.error && !value.stage) throw new EngineError(value.error.code, value.error.message, value.error.retryable);
3912
+ yield value;
3913
+ }
3914
+ }
3915
+ },
3916
+ async wait(onProgress) {
3917
+ for (; ; ) {
3918
+ const status = await trova.json("GET", "/v1/index/status");
3919
+ if (status.jobs.length === 0) return;
3920
+ const job = status.jobs[0];
3921
+ onProgress?.(job.kind, job.done, job.total);
3922
+ await new Promise((r) => setTimeout(r, 300));
3923
+ }
3924
+ },
3925
+ async close() {
3926
+ }
3927
+ };
3928
+ return trova;
3929
+ }
3930
+ async function daemonTrova(home = trovaHome()) {
3931
+ const info = readDaemonInfo(home);
3932
+ if (!info) return null;
3933
+ try {
3934
+ const res = await fetch(`http://127.0.0.1:${info.port}/health`, { signal: AbortSignal.timeout(1e3) });
3935
+ if (!res.ok) return null;
3936
+ } catch {
3937
+ return null;
3938
+ }
3939
+ return httpTrova(info.port, info.token);
3940
+ }
3941
+ async function connect(home = trovaHome()) {
3942
+ return await daemonTrova(home) ?? localTrova(await Engine.open({ home }));
3943
+ }
3944
+ async function connectOrStartDaemon(home = trovaHome(), cliPath = process.argv[1]) {
3945
+ const running = await daemonTrova(home);
3946
+ if (running) return running;
3947
+ const child = spawn(process.execPath, [cliPath, "watch"], { detached: true, stdio: "ignore", env: { ...process.env, TROVA_HOME: home } });
3948
+ child.unref();
3949
+ for (let i = 0; i < 60; i++) {
3950
+ await new Promise((r) => setTimeout(r, 250));
3951
+ const trova = await daemonTrova(home);
3952
+ if (trova) return trova;
3953
+ }
3954
+ return localTrova(await Engine.open({ home }));
3955
+ }
3956
+
3957
+ // src/daemon.ts
3958
+ import { randomBytes } from "node:crypto";
3959
+
3960
+ // src/server.ts
3961
+ import { appendFileSync } from "node:fs";
3962
+ import { createServer } from "node:http";
3963
+ import { timingSafeEqual } from "node:crypto";
3964
+ import { join as join6 } from "node:path";
3965
+ var ALLOWED_ORIGIN = "app://obsidian.md";
3966
+ var MAX_BODY_BYTES = 12 * 1024 * 1024;
3967
+ var MAX_INTERACTIVE = 32;
3968
+ var HEARTBEAT_MS = 15e3;
3969
+ function toError(err) {
3970
+ if (err instanceof EngineError) return err;
3971
+ if (err instanceof SyntaxError) return new EngineError("bad_request", "The request body is not valid JSON");
3972
+ const message = err instanceof Error ? err.message : String(err);
3973
+ if (/model/i.test(message) && /(download|not found|missing)/i.test(message)) return new EngineError("model_missing", message, true);
3974
+ return new EngineError("internal", message);
3975
+ }
3976
+ function sendJson(res, status, body, headers = {}) {
3977
+ const text = body === void 0 ? "" : JSON.stringify(body);
3978
+ res.writeHead(status, { "Content-Type": "application/json", ...headers });
3979
+ res.end(text);
3980
+ }
3981
+ function sendError(res, err) {
3982
+ const headers = {};
3983
+ if (err.code === "model_loading" || err.code === "busy") headers["Retry-After"] = String(Math.ceil((err.retryAfterMs ?? 1e3) / 1e3));
3984
+ sendJson(res, ERROR_STATUS[err.code], err.toBody(), headers);
3985
+ }
3986
+ async function readBody(req) {
3987
+ const chunks = [];
3988
+ let size = 0;
3989
+ for await (const chunk of req) {
3990
+ size += chunk.length;
3991
+ if (size > MAX_BODY_BYTES) throw new EngineError("bad_request", "The request body is too large");
3992
+ chunks.push(chunk);
3993
+ }
3994
+ if (size === 0) return {};
3995
+ const value = JSON.parse(Buffer.concat(chunks).toString("utf8"));
3996
+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new EngineError("bad_request", "The request body must be a JSON object");
3997
+ return value;
3998
+ }
3999
+ async function startServer(engine, opts) {
4000
+ const expected = Buffer.from(`Bearer ${opts.token}`);
4001
+ const sse = /* @__PURE__ */ new Set();
4002
+ let interactive = 0;
4003
+ let debug = opts.debug === true;
4004
+ const log = (line) => {
4005
+ if (!debug) return;
4006
+ try {
4007
+ appendFileSync(join6(engine.paths.logs, "daemon.log"), `${(/* @__PURE__ */ new Date()).toISOString()} ${line}
4008
+ `);
4009
+ } catch {
4010
+ }
4011
+ };
4012
+ const authorised = (req) => {
4013
+ const got = Buffer.from(req.headers.authorization ?? "");
4014
+ return got.length === expected.length && timingSafeEqual(got, expected);
4015
+ };
4016
+ const stream = async (res, controller, iterator) => {
4017
+ if (interactive >= MAX_INTERACTIVE) throw new EngineError("busy", "The interactive queue is full", true, 200);
4018
+ interactive++;
4019
+ try {
4020
+ const first = await iterator.next();
4021
+ res.writeHead(200, { "Content-Type": "application/x-ndjson", "Cache-Control": "no-store", "X-Accel-Buffering": "no" });
4022
+ try {
4023
+ let step = first;
4024
+ while (!step.done) {
4025
+ if (controller.signal.aborted) break;
4026
+ res.write(`${JSON.stringify(step.value)}
4027
+ `);
4028
+ step = await iterator.next();
4029
+ }
4030
+ } catch (err) {
4031
+ const e = toError(err);
4032
+ if (e.code !== "cancelled") res.write(`${JSON.stringify(e.toBody())}
4033
+ `);
4034
+ }
4035
+ res.end();
4036
+ } finally {
4037
+ interactive--;
4038
+ void iterator.return(void 0);
4039
+ }
4040
+ };
4041
+ const route = async (req, res) => {
4042
+ const url = new URL(req.url ?? "/", "http://127.0.0.1");
4043
+ const path = url.pathname.replace(/\/+$/, "") || "/";
4044
+ const method = req.method ?? "GET";
4045
+ if (path === "/health" && method === "GET") {
4046
+ return sendJson(res, 200, { ok: true, name: ENGINE_NAME, version: ENGINE_VERSION, api: API_VERSION });
4047
+ }
4048
+ if (!authorised(req)) throw new EngineError("unauthorized", "Missing or wrong token");
4049
+ engine.lastActivity = Date.now();
4050
+ const body = method === "POST" || method === "DELETE" ? await readBody(req) : {};
4051
+ if (path === "/v1/events" && method === "GET") {
4052
+ res.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-store", Connection: "keep-alive" });
4053
+ res.write(": connected\n\n");
4054
+ sse.add(res);
4055
+ const off = engine.on((e) => res.write(`event: ${e.event}
4056
+ data: ${JSON.stringify(e.data)}
4057
+
4058
+ `));
4059
+ const beat = setInterval(() => res.write(": heartbeat\n\n"), opts.heartbeatMs ?? HEARTBEAT_MS);
4060
+ res.on("close", () => {
4061
+ clearInterval(beat);
4062
+ off();
4063
+ sse.delete(res);
4064
+ engine.lastActivity = Date.now();
4065
+ });
4066
+ return;
4067
+ }
4068
+ if (path === "/v1/debug" && method === "POST") {
4069
+ debug = body.on === true;
4070
+ return sendJson(res, 200, { debug });
4071
+ }
4072
+ if (path === "/v1/search" || path === "/v1/ask") log(`${path} ${JSON.stringify(body.query?.raw ?? "")}`);
4073
+ const controller = new AbortController();
4074
+ const onClose = () => {
4075
+ if (!res.writableEnded) controller.abort();
4076
+ };
4077
+ res.on("close", onClose);
4078
+ const out2 = await dispatch(engine, method, path, body, url.searchParams, controller.signal);
4079
+ if (out2.kind === "stream") return stream(res, controller, out2.batches);
4080
+ res.off("close", onClose);
4081
+ if (out2.kind === "empty") {
4082
+ res.writeHead(out2.status);
4083
+ return void res.end();
4084
+ }
4085
+ return sendJson(res, out2.status, out2.body);
4086
+ };
4087
+ const server = createServer((req, res) => {
4088
+ const origin = req.headers.origin;
4089
+ if (origin !== void 0) {
4090
+ if (origin !== ALLOWED_ORIGIN) {
4091
+ return sendJson(res, 403, { error: { code: "unauthorized", message: "Origin not allowed", retryable: false } });
4092
+ }
4093
+ res.setHeader("Access-Control-Allow-Origin", ALLOWED_ORIGIN);
4094
+ res.setHeader("Vary", "Origin");
4095
+ }
4096
+ if (req.method === "OPTIONS") {
4097
+ res.writeHead(204, {
4098
+ "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
4099
+ "Access-Control-Allow-Headers": "Authorization, Content-Type, X-Request-Id",
4100
+ "Access-Control-Max-Age": "600"
4101
+ });
4102
+ return void res.end();
4103
+ }
4104
+ const started = Date.now();
4105
+ const requestId = req.headers["x-request-id"];
4106
+ if (typeof requestId === "string") res.setHeader("X-Request-Id", requestId);
4107
+ route(req, res).catch((err) => {
4108
+ const e = toError(err);
4109
+ if (res.headersSent) {
4110
+ if (!res.writableEnded) res.end();
4111
+ return;
4112
+ }
4113
+ if (e.code === "cancelled") {
4114
+ res.writeHead(ERROR_STATUS.cancelled);
4115
+ return void res.end();
4116
+ }
4117
+ sendError(res, e);
4118
+ }).finally(() => log(`${req.method} ${req.url} ${res.statusCode} ${Date.now() - started}ms`));
4119
+ });
4120
+ await new Promise((resolve3, reject) => {
4121
+ server.once("error", reject);
4122
+ server.listen(opts.port ?? 0, "127.0.0.1", () => resolve3());
4123
+ });
4124
+ const address = server.address();
4125
+ const port = typeof address === "object" && address ? address.port : 0;
4126
+ return {
4127
+ server,
4128
+ port,
4129
+ clients: () => sse.size,
4130
+ setDebug: (on) => {
4131
+ debug = on;
4132
+ },
4133
+ close: () => new Promise((resolve3) => {
4134
+ for (const res of sse) res.end();
4135
+ server.close(() => resolve3());
4136
+ server.closeAllConnections();
4137
+ })
4138
+ };
4139
+ }
4140
+
4141
+ // src/daemon.ts
4142
+ var IDLE_EXIT_MS = 30 * 60 * 1e3;
4143
+ var DaemonRunningError = class extends Error {
4144
+ constructor(pid, port) {
4145
+ super(`A trova daemon is already running (pid ${pid}, port ${port})`);
4146
+ this.pid = pid;
4147
+ this.port = port;
4148
+ }
4149
+ };
4150
+ async function healthy(port) {
4151
+ try {
4152
+ const res = await fetch(`http://127.0.0.1:${port}/health`, { signal: AbortSignal.timeout(1e3) });
4153
+ return res.ok;
4154
+ } catch {
4155
+ return false;
4156
+ }
4157
+ }
4158
+ async function startDaemon(opts = {}) {
4159
+ const home = opts.home ?? trovaHome();
4160
+ const existing = readDaemonInfo(home);
4161
+ if (existing && existing.pid !== process.pid && isProcessAlive(existing.pid) && await healthy(existing.port)) {
4162
+ throw new DaemonRunningError(existing.pid, existing.port);
4163
+ }
4164
+ const engine = await Engine.open({ ...opts, home });
4165
+ const token = randomBytes(24).toString("hex");
4166
+ const server = await startServer(engine, { token, port: opts.port, debug: opts.debug ?? process.env.TROVA_DEBUG === "1" });
4167
+ writeDaemonInfo({ pid: process.pid, port: server.port, token, version: ENGINE_VERSION, api: API_VERSION }, home);
4168
+ let stopped = false;
4169
+ const stop = async () => {
4170
+ if (stopped) return;
4171
+ stopped = true;
4172
+ clearInterval(idleTimer);
4173
+ const info = readDaemonInfo(home);
4174
+ if (info?.pid === process.pid) removeDaemonInfo(home);
4175
+ await server.close();
4176
+ await engine.close();
4177
+ };
4178
+ const idleMs = opts.idleExitMs ?? IDLE_EXIT_MS;
4179
+ const idleTimer = setInterval(
4180
+ () => {
4181
+ if (server.clients() > 0 || engine.jobs.active().length > 0) {
4182
+ engine.lastActivity = Date.now();
4183
+ return;
4184
+ }
4185
+ if (Date.now() - engine.lastActivity >= idleMs) void stop().then(() => opts.onExit?.());
4186
+ },
4187
+ Math.min(6e4, Math.max(50, idleMs / 4))
4188
+ );
4189
+ idleTimer.unref();
4190
+ return { engine, server, token, stop };
4191
+ }
4192
+
4193
+ // src/mcp.ts
4194
+ import { createServer as createServer2 } from "node:http";
4195
+ import { Server } from "@modelcontextprotocol/sdk/server/index.js";
4196
+ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
4197
+ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
4198
+ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
4199
+ var str = { type: "string" };
4200
+ var collections = { type: "array", items: str, description: "Collection names to search. Default: all." };
4201
+ var limit = { type: "number", description: "Maximum number of results. Default 10." };
4202
+ var MCP_TOOLS = [
4203
+ {
4204
+ name: "search",
4205
+ description: "Search notes with typed sub-queries. Each sub-query has a type (lex = keywords, vec = meaning, hyde = a hypothetical answer passage) and a query in the Trova query language (filters such as tag:, path:, modified: are allowed).",
4206
+ inputSchema: {
4207
+ type: "object",
4208
+ properties: {
4209
+ searches: {
4210
+ type: "array",
4211
+ items: { type: "object", properties: { type: { type: "string", enum: ["lex", "vec", "hyde"] }, query: str }, required: ["type", "query"] }
4212
+ },
4213
+ query: { ...str, description: "Shortcut for one hybrid query, used when `searches` is absent." },
4214
+ collections,
4215
+ limit
4216
+ }
4217
+ }
4218
+ },
4219
+ {
4220
+ name: "ask",
4221
+ description: "Hybrid question search: keyword and vector results fused, reranked in deep mode.",
4222
+ inputSchema: {
4223
+ type: "object",
4224
+ properties: { question: str, mode: { type: "string", enum: ["auto", "fast", "deep"] }, collections, limit },
4225
+ required: ["question"]
4226
+ }
4227
+ },
4228
+ {
4229
+ name: "related",
4230
+ description: "Related notes and blocks for a note path (optionally with #heading-path) or for a text.",
4231
+ inputSchema: { type: "object", properties: { path: str, text: str, collections, limit } }
4232
+ },
4233
+ {
4234
+ name: "get",
4235
+ description: "Read one source or block by id or path.",
4236
+ inputSchema: { type: "object", properties: { id: str, collection: str }, required: ["id"] }
4237
+ },
4238
+ {
4239
+ name: "multi_get",
4240
+ description: "Read several sources by ids or by a glob over paths.",
4241
+ inputSchema: { type: "object", properties: { ids: { type: "array", items: str }, glob: str, collection: str } }
4242
+ },
4243
+ { name: "status", description: "Index health: counts, models, jobs and capabilities.", inputSchema: { type: "object", properties: {} } },
4244
+ { name: "metadata", description: "Frontmatter keys and their values, for building filters.", inputSchema: { type: "object", properties: { collections } } }
4245
+ ];
4246
+ function formatResults(results) {
4247
+ if (results.length === 0) return "No results.";
4248
+ return results.map((r, i) => {
4249
+ const heading = r.block?.headings.length ? ` > ${r.block.headings.join(" > ")}` : "";
4250
+ const score = r.scores.rerank ?? r.scores.fused ?? r.scores.vec ?? r.scores.lex ?? 0;
4251
+ return `${i + 1}. ${r.path}${heading}
4252
+ id: ${r.id}
4253
+ score: ${score.toFixed(3)} (${r.matchedBy.join(", ")}, ${r.label})
4254
+ ${r.snippet}`;
4255
+ }).join("\n");
4256
+ }
4257
+ async function lastBatch(stream) {
4258
+ let last = null;
4259
+ for await (const b of stream) if (b.results.length > 0 || !last) last = b;
4260
+ return last;
4261
+ }
4262
+ async function callTool(trova, name, args) {
4263
+ const cols = Array.isArray(args["collections"]) ? args["collections"] : void 0;
4264
+ const max = typeof args["limit"] === "number" ? args["limit"] : 10;
4265
+ switch (name) {
4266
+ case "search": {
4267
+ const searches = Array.isArray(args["searches"]) ? args["searches"] : [];
4268
+ if (searches.length === 0) {
4269
+ if (typeof args["query"] !== "string") throw new Error("search needs `searches` or `query`");
4270
+ const batch = await lastBatch(trova.stream("/v1/search", { query: parseQuery(args["query"]).query, lanes: ["hybrid"], collections: cols, limit: max }));
4271
+ return formatResults(batch?.results ?? []);
4272
+ }
4273
+ const best = /* @__PURE__ */ new Map();
4274
+ for (const s of searches) {
4275
+ const query = parseQuery(`${s.type}: ${s.query}`).query;
4276
+ const batch = await lastBatch(trova.stream("/v1/search", { query, lanes: [s.type === "lex" ? "lex" : "vec"], collections: cols, limit: max }));
4277
+ (batch?.results ?? []).forEach((result, rank) => {
4278
+ const prev = best.get(result.id);
4279
+ if (!prev || rank < prev.rank) best.set(result.id, { rank, result });
4280
+ });
4281
+ }
4282
+ return formatResults([...best.values()].sort((a, b) => a.rank - b.rank).slice(0, max).map((b) => b.result));
4283
+ }
4284
+ case "ask": {
4285
+ const query = parseQuery(String(args["question"] ?? ""), { mode: "ask" }).query;
4286
+ const batch = await lastBatch(trova.stream("/v1/ask", { query, mode: args["mode"] ?? "auto", collections: cols, limit: max }));
4287
+ return formatResults(batch?.results ?? []);
4288
+ }
4289
+ case "related": {
4290
+ let anchor;
4291
+ if (typeof args["text"] === "string" && args["text"]) anchor = { text: args["text"] };
4292
+ else {
4293
+ const doc = await trova.json("POST", "/v1/documents/get", { id: String(args["path"] ?? "") });
4294
+ anchor = doc.block ? { block: doc.id } : { source: doc.id };
4295
+ }
4296
+ const res = await trova.json("POST", "/v1/related", { anchor, limit: max, collections: cols, granularity: "both" });
4297
+ if (!res.anchorIndexed) return "The anchor is not embedded yet.";
4298
+ return formatResults(res.results);
4299
+ }
4300
+ case "get": {
4301
+ const doc = await trova.json("POST", "/v1/documents/get", { id: String(args["id"]), collection: args["collection"] });
4302
+ return `# ${doc.path}${doc.block?.headings.length ? ` > ${doc.block.headings.join(" > ")}` : ""}
4303
+
4304
+ ${doc.content}`;
4305
+ }
4306
+ case "multi_get": {
4307
+ const res = await trova.json("POST", "/v1/documents/multi-get", args);
4308
+ const docs = res.documents.map((d) => `# ${d.path}
4309
+
4310
+ ${d.content}`).join("\n\n---\n\n");
4311
+ const errors = res.errors.map((e) => `${e.id}: ${e.code}`).join("\n");
4312
+ return [docs, errors && `Errors:
4313
+ ${errors}`].filter(Boolean).join("\n\n") || "No documents.";
4314
+ }
4315
+ case "status":
4316
+ return JSON.stringify(await trova.json("GET", "/v1/index/status"), null, 2);
4317
+ case "metadata": {
4318
+ const qs = (cols ?? []).map((c) => `collection=${encodeURIComponent(c)}`).join("&");
4319
+ return JSON.stringify(await trova.json("GET", `/v1/metadata/keys${qs ? `?${qs}` : ""}`), null, 2);
4320
+ }
4321
+ default:
4322
+ throw new Error(`Unknown tool "${name}"`);
4323
+ }
4324
+ }
4325
+ function createMcpServer(trova) {
4326
+ const server = new Server({ name: "trova", version: ENGINE_VERSION }, { capabilities: { tools: {} } });
4327
+ server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: MCP_TOOLS }));
4328
+ server.setRequestHandler(CallToolRequestSchema, async (req) => {
4329
+ try {
4330
+ const text = await callTool(trova, req.params.name, req.params.arguments ?? {});
4331
+ return { content: [{ type: "text", text }] };
4332
+ } catch (err) {
4333
+ return { content: [{ type: "text", text: `Error: ${err.message}` }], isError: true };
4334
+ }
4335
+ });
4336
+ return server;
4337
+ }
4338
+ async function runMcpStdio(trova) {
4339
+ await createMcpServer(trova).connect(new StdioServerTransport());
4340
+ }
4341
+ var LOOPBACK_HOST = /^(127\.0\.0\.1|localhost|\[::1\])(:\d+)?$/i;
4342
+ function isLoopbackRequest(host, origin) {
4343
+ if (!host || !LOOPBACK_HOST.test(host)) return false;
4344
+ if (origin === void 0) return true;
4345
+ try {
4346
+ return LOOPBACK_HOST.test(new URL(origin).host);
4347
+ } catch {
4348
+ return false;
4349
+ }
4350
+ }
4351
+ async function runMcpHttp(trova, port) {
4352
+ const http = createServer2(async (req, res) => {
4353
+ if (!isLoopbackRequest(req.headers.host, req.headers.origin)) {
4354
+ res.writeHead(403).end();
4355
+ return;
4356
+ }
4357
+ if (!req.url?.startsWith("/mcp")) {
4358
+ res.writeHead(404).end();
4359
+ return;
4360
+ }
4361
+ const chunks = [];
4362
+ for await (const c of req) chunks.push(c);
4363
+ let body;
4364
+ try {
4365
+ body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : void 0;
4366
+ } catch {
4367
+ res.writeHead(400).end();
4368
+ return;
4369
+ }
4370
+ const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: void 0 });
4371
+ const server = createMcpServer(trova);
4372
+ res.on("close", () => {
4373
+ void transport.close();
4374
+ void server.close();
4375
+ });
4376
+ await server.connect(transport);
4377
+ await transport.handleRequest(req, res, body);
4378
+ });
4379
+ await new Promise((resolve3) => http.listen(port, "127.0.0.1", () => resolve3()));
4380
+ const address = http.address();
4381
+ return typeof address === "object" && address ? address.port : port;
4382
+ }
4383
+
4384
+ // src/testing.ts
4385
+ var STOP = new Set("the a an and or of to in is it for on with as at by be this that are was from".split(" "));
4386
+ function hash(text) {
4387
+ let h = 2166136261;
4388
+ for (let i = 0; i < text.length; i++) {
4389
+ h ^= text.charCodeAt(i);
4390
+ h = Math.imul(h, 16777619);
4391
+ }
4392
+ return h >>> 0;
4393
+ }
4394
+ function hashEmbedder(dims = 96, id = "hash-embedder") {
4395
+ return {
4396
+ id,
4397
+ dims,
4398
+ async embed(text) {
4399
+ const vec = new Float32Array(dims);
4400
+ for (const raw of text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []) {
4401
+ if (STOP.has(raw) || raw.length < 2) continue;
4402
+ const word = lightStem(raw);
4403
+ const h = hash(word);
4404
+ vec[h % dims] += 1;
4405
+ vec[(h >>> 8) % dims] += 0.5;
4406
+ }
4407
+ let norm = 0;
4408
+ for (const v of vec) norm += v * v;
4409
+ norm = Math.sqrt(norm);
4410
+ if (norm === 0) {
4411
+ vec[0] = 1;
4412
+ return vec;
4413
+ }
4414
+ for (let i = 0; i < dims; i++) vec[i] /= norm;
4415
+ return vec;
4416
+ }
4417
+ };
4418
+ }
4419
+ function fakeDeep(delayMs = 0) {
4420
+ const words = (t) => new Set((t.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []).filter((w) => !STOP.has(w)).map(lightStem));
4421
+ return {
4422
+ async expand(query) {
4423
+ if (delayMs) await new Promise((r) => setTimeout(r, delayMs));
4424
+ return [
4425
+ { type: "lex", query: query.split(/\s+/).slice(0, 3).join(" ") },
4426
+ { type: "vec", query: `notes about ${query}` }
4427
+ ];
4428
+ },
4429
+ async rerank(query, docs) {
4430
+ if (delayMs) await new Promise((r) => setTimeout(r, delayMs));
4431
+ const q = words(query);
4432
+ const out2 = /* @__PURE__ */ new Map();
4433
+ for (const d of docs) {
4434
+ const w = words(d.text);
4435
+ let hit = 0;
4436
+ for (const t of q) if (w.has(t)) hit++;
4437
+ out2.set(d.id, q.size ? hit / q.size : 0);
4438
+ }
4439
+ return out2;
4440
+ }
4441
+ };
4442
+ }
4443
+
4444
+ // src/cli.ts
4445
+ var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["json", "perf", "http", "debug", "help", "version", "real", "full"]);
4446
+ function parseArgs(argv) {
4447
+ const positional = [];
4448
+ const flags = {};
4449
+ for (let i = 0; i < argv.length; i++) {
4450
+ const a = argv[i];
4451
+ if (a === "--") {
4452
+ positional.push(...argv.slice(i + 1));
4453
+ break;
4454
+ }
4455
+ if (a.startsWith("--")) {
4456
+ const [key, inline] = a.slice(2).split(/=(.*)/s);
4457
+ if (inline !== void 0) flags[key] = inline;
4458
+ else if (BOOLEAN_FLAGS.has(key) || i + 1 >= argv.length || argv[i + 1].startsWith("--")) flags[key] = true;
4459
+ else flags[key] = argv[++i];
4460
+ } else if (a === "-n") flags["limit"] = argv[++i] ?? "10";
4461
+ else if (a === "-c") flags["collection"] = argv[++i] ?? "";
4462
+ else positional.push(a);
4463
+ }
4464
+ return { positional, flags };
4465
+ }
4466
+ var HELP = `trova ${ENGINE_VERSION} - local keyword and semantic search for Markdown notes
4467
+
4468
+ Usage: trova <command> [options]
4469
+
4470
+ collection add <path> [--name <name>] Add a folder as a collection
4471
+ collection ls | rm <name> | rename <old> <new>
4472
+ context add <collection> <path> <text> Describe a folder to improve ranking
4473
+ context ls | rm <collection> <path>
4474
+ update [-c <collection>] Rescan collections for changes
4475
+ embed Embed new and changed blocks
4476
+ watch [--port <n>] [--debug] Run the daemon: watcher, indexer and HTTP API
4477
+ serve [--port <n>] [--debug] HTTP API only, no watcher
4478
+ search <query> Keyword lane, typo tolerant
4479
+ vsearch <query> Vector lane
4480
+ query <query> [--mode fast|deep] Hybrid search (alias: ask)
4481
+ related <path>[#heading-path] Related sources and blocks
4482
+ get <id|path> Print one source or block
4483
+ multi-get <glob | id,id,...> Print several sources
4484
+ status | doctor Health, counts and model identity
4485
+ mcp [--http] [--port <n>] MCP server over stdio, or HTTP on 127.0.0.1 (port 8181)
4486
+ eval [--queries <file>] [--real] nDCG@10 and Recall@20 on the fixture vault or your queries
4487
+ eval --perf [--notes <n>] Latency budgets on a generated vault (10,000 notes)
4488
+
4489
+ Common options: --json, -n/--limit <n>, -c/--collection <name>`;
4490
+ function limitOf(flags, fallback = 10) {
4491
+ const raw = flags["limit"];
4492
+ if (raw === void 0) return fallback;
4493
+ const n = Number(raw);
4494
+ if (!Number.isInteger(n) || n < 1) throw new Error(`--limit needs a whole number of at least 1, not "${raw}"`);
4495
+ return n;
4496
+ }
4497
+ function out(flags, value, text) {
4498
+ console.log(flags["json"] ? JSON.stringify(value, null, 2) : text());
4499
+ }
4500
+ function formatResults2(results) {
4501
+ if (results.length === 0) return "No matches.";
4502
+ return results.map((r) => {
4503
+ const score = r.scores.rerank ?? r.scores.fused ?? r.scores.vec ?? r.scores.lex ?? 0;
4504
+ const heading = r.block?.headings.length ? `#${r.block.headings.join(" > ")}` : "";
4505
+ return `${score.toFixed(3).padStart(7)} ${r.path}${heading} [${r.matchedBy.join(",")} ${r.label}]
4506
+ ${r.snippet}`;
4507
+ }).join("\n");
4508
+ }
4509
+ function progressLine(kind, done, total) {
4510
+ if (!process.stderr.isTTY) return;
4511
+ process.stderr.write(`\r${kind} ${done.toLocaleString()} / ${total.toLocaleString()} `);
4512
+ }
4513
+ function sqliteWorks() {
4514
+ const probe = spawnSync(
4515
+ process.execPath,
4516
+ ["-e", "const D=require('better-sqlite3');const d=new D(':memory:');d.prepare('select 1').get();require('sqlite-vec').load(d)"],
4517
+ { cwd: import.meta.dirname, stdio: "ignore" }
4518
+ );
4519
+ return probe.status === 0;
4520
+ }
4521
+ function requireSqlite() {
4522
+ const marker = join7(trovaHome(), `.sqlite-ok-${process.version}`);
4523
+ if (existsSync4(marker)) return;
4524
+ if (!sqliteWorks()) {
4525
+ throw new Error(
4526
+ `SQLite cannot load on Node ${process.version}. Run trova with a newer Node (24 is verified) or rebuild: npm rebuild better-sqlite3 --build-from-source`
4527
+ );
4528
+ }
4529
+ try {
4530
+ writeFileSync4(marker, "");
4531
+ } catch {
4532
+ }
4533
+ }
4534
+ async function collectionsOf(trova) {
4535
+ return (await trova.json("GET", "/v1/collections")).collections;
4536
+ }
4537
+ async function syncAll(trova, only) {
4538
+ const cols = (await collectionsOf(trova)).filter((c) => !only || c.name === only);
4539
+ if (cols.length === 0) throw new Error(only ? `Unknown collection "${only}"` : "No collections. Add one with: trova collection add <path>");
4540
+ for (const c of cols) await trova.json("POST", "/v1/index/sync", { collection: c.name, full: true });
4541
+ await trova.wait(progressLine);
4542
+ if (process.stderr.isTTY) process.stderr.write("\r\x1B[K");
4543
+ }
4544
+ async function lastBatch2(stream) {
4545
+ let last = { stage: "lex", results: [], final: true };
4546
+ for await (const b of stream) if (b.results.length > 0 || b.final) last = b.results.length || !last.results.length ? b : { ...last, note: b.note };
4547
+ return last;
4548
+ }
4549
+ async function runSearch(trova, args, kind) {
4550
+ const raw = args.positional.join(" ");
4551
+ if (!raw.trim()) throw new Error(`Usage: trova ${kind} <query>`);
4552
+ const limit2 = limitOf(args.flags);
4553
+ const collections2 = typeof args.flags["collection"] === "string" ? [args.flags["collection"]] : void 0;
4554
+ const query = parseQuery(raw, { mode: kind === "query" ? "ask" : "instant" }).query;
4555
+ const batch = kind === "query" ? await lastBatch2(trova.stream("/v1/ask", { query, mode: args.flags["mode"] ?? "auto", collections: collections2, limit: limit2 })) : await lastBatch2(trova.stream("/v1/search", { query, lanes: [kind === "search" ? "lex" : "vec"], collections: collections2, limit: limit2 }));
4556
+ out(args.flags, batch, () => {
4557
+ const head = batch.correction ? `Showing results for "${batch.correction.to}" instead of "${batch.correction.from}"
4558
+ ` : "";
4559
+ const note = batch.note === "rerank_skipped" ? "\n(rerank skipped: deadline passed)" : batch.note === "model_missing" ? "\n(Deep models are not downloaded; ran in Fast mode)" : "";
4560
+ return head + formatResults2(batch.results) + note;
4561
+ });
4562
+ }
4563
+ async function runEvalCommand(args) {
4564
+ const queriesFile = typeof args.flags["queries"] === "string" ? args.flags["queries"] : null;
4565
+ if (queriesFile && !args.flags["perf"]) {
4566
+ const trova2 = await connect();
4567
+ try {
4568
+ const report = await runEval(trova2, loadQueries(queriesFile), typeof args.flags["collection"] === "string" ? args.flags["collection"] : void 0);
4569
+ out(args.flags, report, () => report.rows.map((r) => `${r.group.padEnd(20)} ${r.metric.padEnd(10)} ${r.value.toFixed(3)} (${r.queries} queries)`).join("\n"));
4570
+ } finally {
4571
+ await trova2.close();
4572
+ }
4573
+ return;
4574
+ }
4575
+ const root = mkdtempSync(join7(tmpdir(), "trova-eval-"));
4576
+ const real = args.flags["real"] === true;
4577
+ const engine = await Engine.open(real ? { home: join7(root, "home") } : { home: join7(root, "home"), embedder: hashEmbedder(), deep: fakeDeep() });
4578
+ const trova = localTrova(engine);
4579
+ try {
4580
+ const vault = join7(root, "vault");
4581
+ if (args.flags["perf"]) {
4582
+ const count = Number(args.flags["notes"] ?? 1e4);
4583
+ console.error(`Generating and indexing ${count.toLocaleString()} notes with a stand-in embedder...`);
4584
+ writeGeneratedVault(vault, count);
4585
+ await trova.json("POST", "/v1/collections", { name: "perf", path: vault });
4586
+ await syncAll(trova, "perf");
4587
+ const anchors = Array.from({ length: 20 }, (_, i) => `${TOPICS[i % TOPICS.length].folder}/0/${TOPICS[i % TOPICS.length].title} ${i}.md`);
4588
+ const rows = await runPerf(trova, "perf", anchors, TOPICS.flatMap((t) => t.terms));
4589
+ out(
4590
+ args.flags,
4591
+ rows,
4592
+ () => rows.map((r) => {
4593
+ const ok = r.p95 <= r.targetP95 ? "ok " : "OVER";
4594
+ return `${ok} ${r.interaction.padEnd(52)} p50 ${r.p50.toFixed(1).padStart(7)} ms (target ${r.targetP50}) p95 ${r.p95.toFixed(1).padStart(7)} ms (target ${r.targetP95})`;
4595
+ }).join("\n") + "\nNot measured here: paint-time budgets, which need Obsidian. Vector timings use a stand-in embedder, so they exclude model time."
4596
+ );
4597
+ } else {
4598
+ const fixture = writeFixtureVault(vault);
4599
+ await trova.json("POST", "/v1/collections", { name: "fixture", path: vault });
4600
+ await syncAll(trova, "fixture");
4601
+ const report = await runEval(trova, fixture.queries, "fixture");
4602
+ out(args.flags, report, () => report.rows.map((r) => `${r.group.padEnd(20)} ${r.metric.padEnd(10)} ${r.value.toFixed(3)} (${r.queries} queries)`).join("\n") + (real ? "\nFixture vault with the real models." : "\nFixture vault with a stand-in embedder. Pass --real for the real models, or --queries <file> to evaluate the real index."));
4603
+ }
4604
+ } finally {
4605
+ await trova.close();
4606
+ rmSync4(root, { recursive: true, force: true });
4607
+ }
4608
+ }
4609
+ async function runDaemon(args, watch2) {
4610
+ try {
4611
+ const daemon = await startDaemon({
4612
+ watch: watch2,
4613
+ port: args.flags["port"] ? Number(args.flags["port"]) : void 0,
4614
+ debug: args.flags["debug"] === true,
4615
+ onExit: () => process.exit(0)
4616
+ });
4617
+ process.stderr.on("error", () => void 0);
4618
+ console.error(`trova ${watch2 ? "watch" : "serve"}: listening on 127.0.0.1:${daemon.server.port} (pid ${process.pid})`);
4619
+ const shutdown = () => void daemon.stop().then(() => process.exit(0));
4620
+ process.on("SIGINT", shutdown);
4621
+ process.on("SIGTERM", shutdown);
4622
+ await new Promise(() => void 0);
4623
+ } catch (err) {
4624
+ if (err instanceof DaemonRunningError) {
4625
+ console.error(err.message);
4626
+ process.exit(0);
4627
+ }
4628
+ throw err;
4629
+ }
4630
+ }
4631
+ async function doctor(args) {
4632
+ const checks = [];
4633
+ const major = Number(process.versions.node.split(".")[0]);
4634
+ checks.push({ name: "Node version", ok: major >= 22, detail: process.version });
4635
+ const sqlite = sqliteWorks();
4636
+ checks.push({ name: "SQLite and sqlite-vec load", ok: sqlite, detail: sqlite ? "ok" : `crashes on Node ${process.version}; use Node 24 or rebuild better-sqlite3 from source` });
4637
+ const info = readDaemonInfo();
4638
+ checks.push({ name: "Daemon", ok: true, detail: info && isProcessAlive(info.pid) ? `running, pid ${info.pid}, port ${info.port}` : "not running (commands run in-process)" });
4639
+ checks.push({ name: "Data directory", ok: true, detail: trovaHome() });
4640
+ let trova = null;
4641
+ if (sqlite) {
4642
+ try {
4643
+ trova = await connect();
4644
+ } catch (err) {
4645
+ checks.push({ name: "Index", ok: false, detail: err.message });
4646
+ }
4647
+ }
4648
+ if (trova) {
4649
+ try {
4650
+ const status = await trova.json("GET", "/v1/index/status");
4651
+ checks.push({ name: "Backend", ok: true, detail: status.engine.backend });
4652
+ checks.push({ name: "Embedding model", ok: status.models.embed.state !== "missing", detail: `${status.models.embed.id ?? "?"}: ${status.models.embed.state}` });
4653
+ checks.push({ name: "Deep models", ok: true, detail: `rerank ${status.models.rerank.state}, expansion ${status.models.expand.state}` });
4654
+ for (const c of status.collections) checks.push({ name: `Collection ${c.name}`, ok: existsSync4(c.path), detail: `${c.sources} sources, ${c.embedded} embedded, ${c.path}` });
4655
+ } finally {
4656
+ await trova.close();
4657
+ }
4658
+ }
4659
+ out(args.flags, checks, () => checks.map((c) => `${c.ok ? "ok " : "FAIL"} ${c.name}: ${c.detail}`).join("\n"));
4660
+ if (checks.some((c) => !c.ok)) process.exitCode = 1;
4661
+ }
4662
+ async function main(argv) {
4663
+ const [command, ...rest] = argv;
4664
+ const args = parseArgs(rest);
4665
+ if (!command || command === "help" || command === "--help" || args.flags["help"]) return void console.log(HELP);
4666
+ if (command === "--version" || command === "version") return void console.log(ENGINE_VERSION);
4667
+ if (command === "doctor") return doctor(args);
4668
+ requireSqlite();
4669
+ if (command === "watch") return runDaemon(args, true);
4670
+ if (command === "serve") return runDaemon(args, false);
4671
+ if (command === "eval") return runEvalCommand(args);
4672
+ if (command === "mcp") {
4673
+ const trova2 = await connectOrStartDaemon();
4674
+ if (args.flags["http"]) {
4675
+ const port = await runMcpHttp(trova2, Number(args.flags["port"] ?? 8181));
4676
+ console.error(`trova mcp: http://127.0.0.1:${port}/mcp`);
4677
+ } else await runMcpStdio(trova2);
4678
+ return;
4679
+ }
4680
+ const trova = await connect();
4681
+ const controller = new AbortController();
4682
+ process.on("SIGINT", () => {
4683
+ controller.abort();
4684
+ void trova.close().then(() => process.exit(130));
4685
+ });
4686
+ try {
4687
+ const [sub, ...subArgs] = args.positional;
4688
+ switch (command) {
4689
+ case "collection": {
4690
+ if (sub === "add") {
4691
+ const path = resolve2(subArgs[0] ?? ".");
4692
+ const name = typeof args.flags["name"] === "string" ? args.flags["name"] : basename2(path).toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "collection";
4693
+ const info = await trova.json("POST", "/v1/collections", { name, path, options: { kind: "folder" } });
4694
+ await syncAll(trova, name);
4695
+ out(args.flags, info, () => `Added collection "${name}" at ${path}`);
4696
+ } else if (sub === "rm") {
4697
+ if (!subArgs[0]) throw new Error("Usage: trova collection rm <name>");
4698
+ await trova.json("DELETE", `/v1/collections?name=${encodeURIComponent(subArgs[0])}`);
4699
+ out(args.flags, { removed: subArgs[0] }, () => `Removed collection "${subArgs[0]}"`);
4700
+ } else if (sub === "rename") {
4701
+ if (!subArgs[0] || !subArgs[1]) throw new Error("Usage: trova collection rename <old> <new>");
4702
+ await trova.json("POST", "/v1/collections", { name: subArgs[0], rename: subArgs[1] });
4703
+ await trova.wait(progressLine);
4704
+ out(args.flags, { renamed: subArgs }, () => `Renamed "${subArgs[0]}" to "${subArgs[1]}"`);
4705
+ } else {
4706
+ const cols = await collectionsOf(trova);
4707
+ out(args.flags, cols, () => cols.length ? cols.map((c) => `${c.name} ${c.path}`).join("\n") : "No collections.");
4708
+ }
4709
+ break;
4710
+ }
4711
+ case "context": {
4712
+ if (sub === "add") {
4713
+ const [collection, path, ...text] = subArgs;
4714
+ if (!collection || path === void 0 || text.length === 0) throw new Error("Usage: trova context add <collection> <path> <text>");
4715
+ await trova.json("POST", "/v1/collections", { name: collection, contextPath: path, contextText: text.join(" ") });
4716
+ out(args.flags, { collection, path }, () => `Context set for ${collection}:${path}`);
4717
+ } else if (sub === "rm") {
4718
+ const [collection, path] = subArgs;
4719
+ if (!collection || path === void 0) throw new Error("Usage: trova context rm <collection> <path>");
4720
+ await trova.json("POST", "/v1/collections", { name: collection, contextPath: path });
4721
+ out(args.flags, { collection, path }, () => `Context removed for ${collection}:${path}`);
4722
+ } else {
4723
+ const cols = await collectionsOf(trova);
4724
+ const rows = cols.flatMap((c) => Object.entries(c.context).map(([path, text]) => ({ collection: c.name, path, text })));
4725
+ out(args.flags, rows, () => rows.length ? rows.map((r) => `${r.collection}:${r.path} ${r.text}`).join("\n") : "No contexts.");
4726
+ }
4727
+ break;
4728
+ }
4729
+ case "update":
4730
+ case "embed": {
4731
+ await syncAll(trova, typeof args.flags["collection"] === "string" ? args.flags["collection"] : void 0);
4732
+ const status = await trova.json("GET", "/v1/index/status");
4733
+ out(args.flags, status.collections, () => status.collections.map((c) => `${c.name}: ${c.sources} sources, ${c.embedded} embedded`).join("\n"));
4734
+ break;
4735
+ }
4736
+ case "search":
4737
+ await runSearch(trova, args, "search");
4738
+ break;
4739
+ case "vsearch":
4740
+ await runSearch(trova, args, "vsearch");
4741
+ break;
4742
+ case "query":
4743
+ case "ask":
4744
+ await runSearch(trova, args, "query");
4745
+ break;
4746
+ case "related": {
4747
+ if (!sub) throw new Error("Usage: trova related <path>[#heading-path]");
4748
+ const collection = typeof args.flags["collection"] === "string" ? args.flags["collection"] : void 0;
4749
+ const doc = await trova.json("POST", "/v1/documents/get", { id: sub, collection });
4750
+ const res = await trova.json("POST", "/v1/related", {
4751
+ anchor: doc.block ? { block: doc.id } : { source: doc.id },
4752
+ limit: limitOf(args.flags),
4753
+ granularity: "both"
4754
+ });
4755
+ out(
4756
+ args.flags,
4757
+ res,
4758
+ () => !res.anchorIndexed ? "Not indexed yet. Run: trova embed" : res.results.length === 0 ? "No related sources." : res.results.map((r) => `${(r.scores.fused ?? r.scores.vec ?? 0).toFixed(3).padStart(7)} ${r.path} [${r.label}]${(r.blocks ?? []).map((b) => `
4759
+ ${b.block?.headings.join(" > ") || "(top)"}: ${b.snippet.slice(0, 90)}`).join("")}`).join("\n")
4760
+ );
4761
+ break;
4762
+ }
4763
+ case "get": {
4764
+ if (!sub) throw new Error("Usage: trova get <id|path>");
4765
+ const doc = await trova.json("POST", "/v1/documents/get", { id: sub, collection: args.flags["collection"] });
4766
+ out(args.flags, doc, () => doc.content);
4767
+ break;
4768
+ }
4769
+ case "multi-get": {
4770
+ if (!sub) throw new Error("Usage: trova multi-get <glob | id,id,...>");
4771
+ const isGlob = /[*?{]/.test(sub);
4772
+ const res = await trova.json("POST", "/v1/documents/multi-get", isGlob ? { glob: sub, collection: args.flags["collection"] } : { ids: sub.split(","), collection: args.flags["collection"] });
4773
+ out(args.flags, res, () => res.documents.map((d) => `=== ${d.path} ===
4774
+ ${d.content}`).join("\n\n") + res.errors.map((e) => `
4775
+ ${e.id}: ${e.code}`).join(""));
4776
+ break;
4777
+ }
4778
+ case "status": {
4779
+ const status = await trova.json("GET", "/v1/index/status");
4780
+ out(
4781
+ args.flags,
4782
+ status,
4783
+ () => [
4784
+ `${status.engine.name} ${status.engine.version} (${status.engine.backend}), API ${status.api}${trova.local ? ", in-process" : ", daemon"}`,
4785
+ `Embedding model: ${status.models.embed.id ?? "?"} (${status.models.embed.state}${status.models.embed.dims ? `, ${status.models.embed.dims} dims` : ""})`,
4786
+ `Rerank: ${status.models.rerank.state}, expansion: ${status.models.expand.state}`,
4787
+ `Capabilities: ${status.capabilities.join(", ")}`,
4788
+ ...status.collections.map((c) => `${c.name}: ${c.sources} sources, ${c.embedded} embedded, ${c.pending} pending ${c.path}`),
4789
+ ...status.jobs.map((j) => `job ${j.jobId} ${j.kind} ${j.state} ${j.done}/${j.total}`)
4790
+ ].join("\n")
4791
+ );
4792
+ break;
4793
+ }
4794
+ default:
4795
+ throw new Error(`Unknown command "${command}". Run: trova help`);
4796
+ }
4797
+ } catch (err) {
4798
+ if (err instanceof EngineError && args.flags["json"]) console.log(JSON.stringify(err.toBody()));
4799
+ else console.error(`trova: ${err.message}`);
4800
+ process.exitCode = 1;
4801
+ } finally {
4802
+ await trova.close();
4803
+ }
4804
+ }
4805
+ export {
4806
+ main
4807
+ };