sensemaking 0.9.5 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/README.md +27 -22
  2. package/dist/cjs/cli/download.d.cts +3 -0
  3. package/dist/cjs/cli/download.d.ts +3 -0
  4. package/dist/cjs/cli/download.js +214 -0
  5. package/dist/cjs/cli/download.js.map +1 -0
  6. package/dist/cjs/cli/index.d.cts +6 -5
  7. package/dist/cjs/cli/index.d.ts +6 -5
  8. package/dist/cjs/cli/index.js +20 -14
  9. package/dist/cjs/cli/index.js.map +1 -1
  10. package/dist/cjs/cli/init.js +4 -1
  11. package/dist/cjs/cli/init.js.map +1 -1
  12. package/dist/cjs/cli/map.js +2 -2
  13. package/dist/cjs/cli/map.js.map +1 -1
  14. package/dist/cjs/cli/named.js +13 -19
  15. package/dist/cjs/cli/named.js.map +1 -1
  16. package/dist/cjs/cli/path.d.cts +3 -0
  17. package/dist/cjs/cli/path.d.ts +3 -0
  18. package/dist/cjs/cli/path.js +161 -0
  19. package/dist/cjs/cli/path.js.map +1 -0
  20. package/dist/cjs/cli/peek.js +3 -2
  21. package/dist/cjs/cli/peek.js.map +1 -1
  22. package/dist/cjs/cli/related.d.cts +3 -0
  23. package/dist/cjs/cli/related.d.ts +3 -0
  24. package/dist/cjs/cli/related.js +279 -0
  25. package/dist/cjs/cli/related.js.map +1 -0
  26. package/dist/cjs/cli/search.js +3 -7
  27. package/dist/cjs/cli/search.js.map +1 -1
  28. package/dist/cjs/cli/shared.d.cts +4 -2
  29. package/dist/cjs/cli/shared.d.ts +4 -2
  30. package/dist/cjs/cli/shared.js +49 -6
  31. package/dist/cjs/cli/shared.js.map +1 -1
  32. package/dist/cjs/cli/sql.d.cts +3 -0
  33. package/dist/cjs/cli/sql.d.ts +3 -0
  34. package/dist/cjs/cli/{query.js → sql.js} +33 -5
  35. package/dist/cjs/cli/sql.js.map +1 -0
  36. package/dist/cjs/cli/status.d.cts +1 -0
  37. package/dist/cjs/cli/status.d.ts +1 -0
  38. package/dist/cjs/cli/status.js +80 -10
  39. package/dist/cjs/cli/status.js.map +1 -1
  40. package/dist/cjs/cli.js +14 -15
  41. package/dist/cjs/cli.js.map +1 -1
  42. package/dist/cjs/commands.d.cts +13 -4
  43. package/dist/cjs/commands.d.ts +13 -4
  44. package/dist/cjs/commands.js +161 -61
  45. package/dist/cjs/commands.js.map +1 -1
  46. package/dist/cjs/config.d.cts +6 -4
  47. package/dist/cjs/config.d.ts +6 -4
  48. package/dist/cjs/config.js +137 -58
  49. package/dist/cjs/config.js.map +1 -1
  50. package/dist/cjs/db.d.cts +2 -2
  51. package/dist/cjs/db.d.ts +2 -2
  52. package/dist/cjs/db.js +25 -46
  53. package/dist/cjs/db.js.map +1 -1
  54. package/dist/cjs/errors.d.cts +1 -1
  55. package/dist/cjs/errors.d.ts +1 -1
  56. package/dist/cjs/errors.js.map +1 -1
  57. package/dist/cjs/features/embed.d.cts +15 -0
  58. package/dist/cjs/features/embed.d.ts +15 -0
  59. package/dist/cjs/features/embed.js +277 -61
  60. package/dist/cjs/features/embed.js.map +1 -1
  61. package/dist/cjs/features/links.js +7 -16
  62. package/dist/cjs/features/links.js.map +1 -1
  63. package/dist/cjs/features/rank.js +2 -6
  64. package/dist/cjs/features/rank.js.map +1 -1
  65. package/dist/cjs/index.d.cts +2 -2
  66. package/dist/cjs/index.d.ts +2 -2
  67. package/dist/cjs/index.js +3 -3
  68. package/dist/cjs/index.js.map +1 -1
  69. package/dist/cjs/output.d.cts +3 -0
  70. package/dist/cjs/output.d.ts +3 -0
  71. package/dist/cjs/output.js +17 -15
  72. package/dist/cjs/output.js.map +1 -1
  73. package/dist/cjs/progress.js +2 -4
  74. package/dist/cjs/progress.js.map +1 -1
  75. package/dist/cjs/scan.d.cts +2 -0
  76. package/dist/cjs/scan.d.ts +2 -0
  77. package/dist/cjs/scan.js +150 -44
  78. package/dist/cjs/scan.js.map +1 -1
  79. package/dist/cjs/search-error.js +4 -6
  80. package/dist/cjs/search-error.js.map +1 -1
  81. package/dist/cjs/traverse.d.cts +8 -0
  82. package/dist/cjs/traverse.d.ts +8 -0
  83. package/dist/cjs/traverse.js +97 -0
  84. package/dist/cjs/traverse.js.map +1 -0
  85. package/dist/esm/cli/download.d.ts +3 -0
  86. package/dist/esm/cli/download.js +27 -0
  87. package/dist/esm/cli/download.js.map +1 -0
  88. package/dist/esm/cli/index.d.ts +6 -5
  89. package/dist/esm/cli/index.js +13 -11
  90. package/dist/esm/cli/index.js.map +1 -1
  91. package/dist/esm/cli/init.js +4 -1
  92. package/dist/esm/cli/init.js.map +1 -1
  93. package/dist/esm/cli/map.js +3 -2
  94. package/dist/esm/cli/map.js.map +1 -1
  95. package/dist/esm/cli/named.js +13 -17
  96. package/dist/esm/cli/named.js.map +1 -1
  97. package/dist/esm/cli/path.d.ts +3 -0
  98. package/dist/esm/cli/path.js +51 -0
  99. package/dist/esm/cli/path.js.map +1 -0
  100. package/dist/esm/cli/peek.js +4 -2
  101. package/dist/esm/cli/peek.js.map +1 -1
  102. package/dist/esm/cli/related.d.ts +3 -0
  103. package/dist/esm/cli/related.js +26 -0
  104. package/dist/esm/cli/related.js.map +1 -0
  105. package/dist/esm/cli/search.js +2 -5
  106. package/dist/esm/cli/search.js.map +1 -1
  107. package/dist/esm/cli/shared.d.ts +4 -2
  108. package/dist/esm/cli/shared.js +49 -6
  109. package/dist/esm/cli/shared.js.map +1 -1
  110. package/dist/esm/cli/sql.d.ts +3 -0
  111. package/dist/esm/cli/{query.js → sql.js} +8 -5
  112. package/dist/esm/cli/sql.js.map +1 -0
  113. package/dist/esm/cli/status.d.ts +1 -0
  114. package/dist/esm/cli/status.js +72 -8
  115. package/dist/esm/cli/status.js.map +1 -1
  116. package/dist/esm/cli.js +13 -10
  117. package/dist/esm/cli.js.map +1 -1
  118. package/dist/esm/commands.d.ts +13 -4
  119. package/dist/esm/commands.js +135 -72
  120. package/dist/esm/commands.js.map +1 -1
  121. package/dist/esm/config.d.ts +6 -4
  122. package/dist/esm/config.js +126 -75
  123. package/dist/esm/config.js.map +1 -1
  124. package/dist/esm/db.d.ts +2 -2
  125. package/dist/esm/db.js +28 -48
  126. package/dist/esm/db.js.map +1 -1
  127. package/dist/esm/errors.d.ts +1 -1
  128. package/dist/esm/errors.js.map +1 -1
  129. package/dist/esm/features/embed.d.ts +15 -0
  130. package/dist/esm/features/embed.js +137 -42
  131. package/dist/esm/features/embed.js.map +1 -1
  132. package/dist/esm/features/links.js +8 -17
  133. package/dist/esm/features/links.js.map +1 -1
  134. package/dist/esm/features/rank.js +2 -6
  135. package/dist/esm/features/rank.js.map +1 -1
  136. package/dist/esm/features/types.js.map +1 -1
  137. package/dist/esm/index.d.ts +2 -2
  138. package/dist/esm/index.js +1 -1
  139. package/dist/esm/index.js.map +1 -1
  140. package/dist/esm/output.d.ts +3 -0
  141. package/dist/esm/output.js +19 -17
  142. package/dist/esm/output.js.map +1 -1
  143. package/dist/esm/progress.js +2 -4
  144. package/dist/esm/progress.js.map +1 -1
  145. package/dist/esm/scan.d.ts +2 -0
  146. package/dist/esm/scan.js +123 -47
  147. package/dist/esm/scan.js.map +1 -1
  148. package/dist/esm/search-error.js +4 -6
  149. package/dist/esm/search-error.js.map +1 -1
  150. package/dist/esm/traverse.d.ts +8 -0
  151. package/dist/esm/traverse.js +71 -0
  152. package/dist/esm/traverse.js.map +1 -0
  153. package/package.json +7 -3
  154. package/schema.json +12 -16
  155. package/skills/sense/EXAMPLES.md +8 -8
  156. package/skills/sense/SKILL.md +51 -23
  157. package/skills/sense-setup/EXAMPLES.md +24 -21
  158. package/skills/sense-setup/SKILL.md +9 -9
  159. package/dist/cjs/cli/check.d.cts +0 -3
  160. package/dist/cjs/cli/check.d.ts +0 -3
  161. package/dist/cjs/cli/check.js +0 -398
  162. package/dist/cjs/cli/check.js.map +0 -1
  163. package/dist/cjs/cli/query.d.cts +0 -3
  164. package/dist/cjs/cli/query.d.ts +0 -3
  165. package/dist/cjs/cli/query.js.map +0 -1
  166. package/dist/cjs/cli/rebuild.d.cts +0 -3
  167. package/dist/cjs/cli/rebuild.d.ts +0 -3
  168. package/dist/cjs/cli/rebuild.js +0 -50
  169. package/dist/cjs/cli/rebuild.js.map +0 -1
  170. package/dist/esm/cli/check.d.ts +0 -3
  171. package/dist/esm/cli/check.js +0 -94
  172. package/dist/esm/cli/check.js.map +0 -1
  173. package/dist/esm/cli/query.d.ts +0 -3
  174. package/dist/esm/cli/query.js.map +0 -1
  175. package/dist/esm/cli/rebuild.d.ts +0 -3
  176. package/dist/esm/cli/rebuild.js +0 -13
  177. package/dist/esm/cli/rebuild.js.map +0 -1
@@ -1,18 +1,21 @@
1
1
  import type { DatabaseSync } from 'node:sqlite';
2
- import type { FeatureName, ResolvedConfig } from './config.js';
2
+ import type { FeatureName, ResolvedConfig, SearchOverrides } from './config.js';
3
3
  import type { Row } from './output.js';
4
4
  export interface SearchOptions {
5
5
  k?: number;
6
6
  where?: string;
7
7
  preset?: string;
8
8
  include?: string[];
9
- semantic?: boolean;
9
+ exclude?: string[];
10
+ noExclude?: boolean;
10
11
  }
11
12
  export declare function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<Row[]>;
13
+ export declare function scopedPaths(db: DatabaseSync, cfg: ResolvedConfig, overrides: SearchOverrides): Set<string>;
12
14
  export interface PresetCoverage {
13
15
  name: string;
14
16
  files: number;
15
17
  embedded: number;
18
+ semantic: boolean;
16
19
  }
17
20
  export declare function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[];
18
21
  export interface TreeMap {
@@ -30,11 +33,13 @@ export interface TreeMap {
30
33
  hubs: Row[];
31
34
  recent: Row[];
32
35
  }
33
- export declare function mapTree(db: DatabaseSync, cfg: ResolvedConfig): TreeMap;
36
+ export declare function mapTree(db: DatabaseSync, cfg: ResolvedConfig, overrides?: SearchOverrides): TreeMap;
37
+ export declare function resolveNote(paths: string[], arg: string): string;
34
38
  export interface Peek {
35
39
  path: string;
36
40
  tokens: number;
37
41
  frontmatter: Row;
42
+ parseError: string | null;
38
43
  sections: Row[];
39
44
  outbound: string[];
40
45
  backlinks: string[];
@@ -45,4 +50,8 @@ export interface Peek {
45
50
  unresolvedTotal: number;
46
51
  off: FeatureName[];
47
52
  }
48
- export declare function peek(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string): Peek;
53
+ export declare function peek(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides?: SearchOverrides): Peek;
54
+ export declare function relatedNotes(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{
55
+ path: string;
56
+ similarity: number;
57
+ }>>;
@@ -18,6 +18,15 @@ _export(exports, {
18
18
  get presetCoverage () {
19
19
  return presetCoverage;
20
20
  },
21
+ get relatedNotes () {
22
+ return relatedNotes;
23
+ },
24
+ get resolveNote () {
25
+ return resolveNote;
26
+ },
27
+ get scopedPaths () {
28
+ return scopedPaths;
29
+ },
21
30
  get search () {
22
31
  return search;
23
32
  }
@@ -226,11 +235,8 @@ function _ts_generator(thisArg, body) {
226
235
  // Each returns data; cli.ts renders. All of them degrade when a feature is off.
227
236
  var WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0)';
228
237
  var RRF_K = 60;
229
- // FTS5 snippet() re-tokenizes the whole doc per candidate row, superlinearly: measured
230
- // 2ms at 4KB docs but 189ms at 64KB for one query's pool, and ~10s for a single 1MB doc
231
- // (plans/performance-findings.md finding A). 16KB keeps the whole pool's snippet cost in
232
- // the low-ms range; bigger docs get the JS-computed excerpt, linear and only for the rows
233
- // actually returned.
238
+ // snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
239
+ // (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.
234
240
  var SNIPPET_BOUND = 16384;
235
241
  var EXCERPT_WINDOW = 160;
236
242
  // Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt
@@ -367,8 +373,8 @@ function lineRangeFor(db, path, line) {
367
373
  return row ? "L".concat(row.start_line, "-").concat(row.end_line) : null;
368
374
  }
369
375
  // node:path's matchesGlob is experimental (stable behind an unstable-API flag) as of the
370
- // engines floor (Node >=22.16); scope filtering only ever needs single-pattern matching, so
371
- // it's used here in JS rather than pulling fast-glob's directory walk into the query path.
376
+ // engines floor (Node >=22.20); scope filtering only ever needs single-pattern matching, so
377
+ // it's used here in JS rather than running a directory walk in the query path.
372
378
  function inScope(path, include, exclude) {
373
379
  if (!include.some(function(g) {
374
380
  return (0, _nodepath.matchesGlob)(path, g);
@@ -387,7 +393,7 @@ function scopeHasEmbeddings(db, cfg, scopedPaths) {
387
393
  }
388
394
  function search(_0, _1, _2) {
389
395
  return _async_to_generator(function(db, cfg, terms) {
390
- var opts, _hits_get, _chunkLines_get, _chunkSimilarity_get, effective, k, include, exclude, allPaths, scopedPaths, scopeActive, fetch, semanticEnabled, scope, whereJoin, whereCond, matchSql, matchRows, hits, oversized, candidates, edges, linked, seeds, ranked, chunkLines, chunkSimilarity, vec, insert, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, path, c, where, similarityCol, rows, bareTerms, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, row, text, _computeExcerpt, excerpt, offset;
396
+ var opts, _hits_get, _chunkLines_get, _chunkSimilarity_get, effective, k, include, exclude, allPaths, adHocScope, scopedPaths, scopeActive, fetch, wantsVectors, semanticEnabled, scope, whereJoin, whereCond, matchSql, matchRows, hits, oversized, candidates, edges, linked, seeds, ranked, chunkLines, chunkSimilarity, vec, insert, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, path, c, where, similarityCol, rows, bareTerms, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, row, text, _computeExcerpt, excerpt, offset;
391
397
  var _arguments = arguments;
392
398
  return _ts_generator(this, function(_state) {
393
399
  switch(_state.label){
@@ -398,11 +404,10 @@ function search(_0, _1, _2) {
398
404
  allPaths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
399
405
  return r.path;
400
406
  });
401
- // A preset scope reads preset_files -- the coverage reconcile actually indexed with
402
- // (fast-glob semantics); re-matching globs here with node:path's matchesGlob could
403
- // silently disagree with it. Ad-hoc --include has no persisted coverage, so only that
404
- // path pays the JS glob match.
405
- scopedPaths = opts.include ? new Set(allPaths.filter(function(p) {
407
+ // A preset scope reads the coverage reconcile indexed; re-matching globs here could disagree
408
+ // with it. Only ad-hoc scope flags, which have no persisted coverage, pay the JS match.
409
+ adHocScope = opts.include !== undefined || opts.exclude !== undefined || opts.noExclude === true;
410
+ scopedPaths = adHocScope ? new Set(allPaths.filter(function(p) {
406
411
  return inScope(p, include, exclude);
407
412
  })) : new Set(db.prepare('SELECT "path" FROM preset_files WHERE preset = ?').all(effective.presetName).map(function(r) {
408
413
  return r.path;
@@ -412,16 +417,16 @@ function search(_0, _1, _2) {
412
417
  // list that a plain over-fetch wouldn't have reached.
413
418
  scopeActive = scopedPaths.size < allPaths.length;
414
419
  fetch = scopeActive ? Math.max(k * 5, 50) : Math.max(k * 3, 30);
415
- // Semantic participation: vectors run only when the effective setting allows it AND the
416
- // scope actually has embedded files -- a scope covered only by semantic-false presets (or
417
- // by no preset embedding at all) searches lexically, automatically, no error.
418
- semanticEnabled = effective.semantic && scopeHasEmbeddings(db, cfg, scopedPaths);
419
- // Terms pass verbatim to FTS5 MATCH: bare words AND-join, operators are the caller's.
420
- // Invalid syntax propagates as an error, zero matches return zero -- no silent rewrites.
421
- // --where applies inside the candidate query (a post-filter over the top-N would drop
422
- // matches ranked past the pool) and again on the final select for link-derived rows.
423
- // An explicit --where replaces the preset's declared where rather than ANDing with it,
424
- // so a caller can always widen back to the whole scope.
420
+ // Asking for vectors without a model is a misconfiguration, not a mode: degrading would make
421
+ // the same search answer differently before and after a download. semantic:false never asks.
422
+ wantsVectors = effective.semantic && (0, _configts.anyPresetEmbeds)(cfg);
423
+ if (wantsVectors && !(0, _embedts.modelPresent)(cfg)) {
424
+ throw new _errorsts.SenseError('EMBED_MODEL_MISSING', 'preset "'.concat(effective.presetName, '" searches with vectors, but the embedding model is not available; run `sense download`, or set "semantic": false on that preset to search on words and links'));
425
+ }
426
+ semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, scopedPaths);
427
+ // Terms pass verbatim to MATCH; invalid syntax errors rather than being rewritten. --where
428
+ // applies inside the candidate query (a post-filter would drop matches ranked past the pool)
429
+ // and again on the final select, for link-derived rows.
425
430
  scope = effective.where;
426
431
  whereJoin = scope ? 'JOIN frontmatter f ON f."path" = content.path' : '';
427
432
  whereCond = scope ? "AND (".concat(scope, ")") : '';
@@ -461,10 +466,8 @@ function search(_0, _1, _2) {
461
466
  });
462
467
  edges = (0, _configts.featureEnabled)(cfg, 'links') && matchRows.length > 0 ? (0, _indexts.linkEdges)(db) : [];
463
468
  if (edges.length > 0) {
464
- // `linked` gates the label only, not the score: PPR restart mass gives every seed a
465
- // nonzero rank even without an incident edge, which is not link evidence — but dropping
466
- // that mass from the score list reweights fusion toward connectivity and measurably
467
- // wrecks ranking on link-dense corpora (FEVER hit@10 0.997 -> 0.907; fusion-tuning.md).
469
+ // Gates the label only, not the score: restart mass ranks every seed without an incident
470
+ // edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).
468
471
  linked = new Set(edges.flat());
469
472
  seeds = new Map(matchRows.map(function(r, i) {
470
473
  return [
@@ -525,9 +528,9 @@ function search(_0, _1, _2) {
525
528
  });
526
529
  _state.label = 2;
527
530
  case 2:
528
- db.exec('DROP TABLE IF EXISTS _find');
529
- db.exec('CREATE TEMP TABLE _find ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');
530
- insert = db.prepare('INSERT INTO _find ("path", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');
531
+ db.exec('DROP TABLE IF EXISTS _search');
532
+ db.exec('CREATE TEMP TABLE _search ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');
533
+ insert = db.prepare('INSERT INTO _search ("path", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');
531
534
  _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
532
535
  try {
533
536
  for(_iterator = candidates[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
@@ -548,15 +551,13 @@ function search(_0, _1, _2) {
548
551
  }
549
552
  }
550
553
  }
551
- // Every candidate list was already filtered to scopedPaths above, so _find only ever holds
552
- // in-scope rows -- the final select just needs --where again, same as before, for rows
553
- // whichever candidate query didn't itself apply it to (link-derived rows never touched
554
- // whereCond above).
554
+ // Already scope-filtered above; the final select reapplies --where only for link-derived
555
+ // rows, which never passed through whereCond.
555
556
  where = scope ? "WHERE (".concat(scope, ")") : '';
556
557
  // lines is always present now: semantic rows carry their chunk's range, oversized-doc
557
558
  // lexical rows gain one below, everything else stays null. similarity stays semantic-only.
558
- similarityCol = semanticEnabled ? ', _find.similarity' : '';
559
- rows = db.prepare('SELECT f."path" AS path, content.title, content.summary, _find.hit, _find.via, round(_find.score, 4) AS score, _find.lines'.concat(similarityCol, '\n FROM _find JOIN frontmatter f ON f."path" = _find."path" JOIN content ON content.path = _find."path"\n ').concat(where, " ORDER BY _find.score DESC LIMIT ?")).all(k);
559
+ similarityCol = semanticEnabled ? ', _search.similarity' : '';
560
+ rows = db.prepare('SELECT f."path" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines'.concat(similarityCol, '\n FROM _search JOIN frontmatter f ON f."path" = _search."path" JOIN content ON content.path = _search."path"\n ').concat(where, " ORDER BY _search.score DESC LIMIT ?")).all(k);
560
561
  if (oversized.size > 0) {
561
562
  bareTerms = extractBareTerms(terms);
562
563
  _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
@@ -597,6 +598,27 @@ function search(_0, _1, _2) {
597
598
  });
598
599
  }).apply(this, arguments);
599
600
  }
601
+ function scopedPaths(db, cfg, overrides) {
602
+ var effective = (0, _configts.resolveSearch)(cfg, overrides);
603
+ var include = effective.include, exclude = effective.exclude, where = effective.where;
604
+ var allPaths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
605
+ return r.path;
606
+ });
607
+ var adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;
608
+ var paths = adHocScope ? new Set(allPaths.filter(function(p) {
609
+ return inScope(p, include, exclude);
610
+ })) : new Set(db.prepare('SELECT "path" FROM preset_files WHERE preset = ?').all(effective.presetName).map(function(r) {
611
+ return r.path;
612
+ }));
613
+ if (!where) return paths;
614
+ var whereRows = db.prepare('SELECT "path" FROM frontmatter f WHERE ('.concat(where, ")")).all();
615
+ var wherePaths = new Set(whereRows.map(function(r) {
616
+ return r.path;
617
+ }));
618
+ return new Set(_to_consumable_array(paths).filter(function(p) {
619
+ return wherePaths.has(p);
620
+ }));
621
+ }
600
622
  function presetCoverage(db, cfg) {
601
623
  var embedActive = (0, _configts.anyPresetEmbeds)(cfg);
602
624
  return (0, _configts.presetNames)(cfg).map(function(name) {
@@ -605,7 +627,8 @@ function presetCoverage(db, cfg) {
605
627
  return {
606
628
  name: name,
607
629
  files: files,
608
- embedded: embedded
630
+ embedded: embedded,
631
+ semantic: (0, _configts.presetSemanticEnabled)(cfg, name)
609
632
  };
610
633
  });
611
634
  }
@@ -613,7 +636,8 @@ var INTERNAL_COLUMNS = new Set([
613
636
  'path',
614
637
  '_mtime',
615
638
  '_size',
616
- '_rank'
639
+ '_rank',
640
+ '_parse_error'
617
641
  ]);
618
642
  // A result row is capped at SQLITE_MAX_COLUMN (2000, default); two aggregate expressions
619
643
  // per field keeps a chunk's row safely under that regardless of how many fields the tree has.
@@ -623,19 +647,26 @@ function chunk(items, size) {
623
647
  for(var i = 0; i < items.length; i += size)out.push(items.slice(i, i + size));
624
648
  return out;
625
649
  }
650
+ // Materializes the resolved scope into a temp table (same shape as traverse.ts's
651
+ // allowed_nodes) so every mapTree query can join/filter against it cheaply.
652
+ function setupMapScope(db, paths) {
653
+ db.exec('DROP TABLE IF EXISTS _map_scope');
654
+ db.exec('CREATE TEMP TABLE _map_scope ("path" TEXT PRIMARY KEY)');
655
+ db.prepare('INSERT INTO _map_scope SELECT DISTINCT value FROM json_each(?1)').run(JSON.stringify(_to_consumable_array(paths)));
656
+ }
626
657
  function mapTree(db, cfg) {
627
- var docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM("_size"), 0) AS bytes FROM frontmatter').get();
658
+ var overrides = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : {};
659
+ setupMapScope(db, scopedPaths(db, cfg, overrides));
660
+ var scopeWhere = 'WHERE "path" IN (SELECT "path" FROM _map_scope)';
661
+ var scopeAnd = 'AND f."path" IN (SELECT "path" FROM _map_scope)';
662
+ var docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM("_size"), 0) AS bytes FROM frontmatter '.concat(scopeWhere)).get();
628
663
  var columns = db.prepare('PRAGMA table_info(frontmatter)').all().map(function(r) {
629
664
  return r.name;
630
665
  }).filter(function(name) {
631
666
  return !INTERNAL_COLUMNS.has(name);
632
667
  });
633
- // Observed storage class, not a declared one: these columns are added dynamically and
634
- // SQLite types per value, so a field can be text in most notes and numeric in a few.
635
- // Listing every distinct type makes both the type and any drift visible.
636
- // One aggregate scan per chunk of columns rather than one scan per column: COUNT already
637
- // ignores NULL, and FILTER keeps typeof() looking only at the non-null rows COUNT counted,
638
- // matching the old per-column `WHERE ... IS NOT NULL` exactly.
668
+ // Observed types, not declared: SQLite types per value, so a field can be text in most notes
669
+ // and numeric in a few. One aggregate scan per chunk of columns; FILTER matches COUNT's nulls.
639
670
  var allFields = [];
640
671
  var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
641
672
  try {
@@ -645,7 +676,7 @@ function mapTree(db, cfg) {
645
676
  var quoted = '"'.concat(name.split('"').join('""'), '"');
646
677
  return "COUNT(".concat(quoted, ") AS n").concat(i, ", GROUP_CONCAT(DISTINCT typeof(").concat(quoted, ")) FILTER (WHERE ").concat(quoted, " IS NOT NULL) AS t").concat(i);
647
678
  });
648
- var result = db.prepare("SELECT ".concat(exprs.join(', '), " FROM frontmatter")).get();
679
+ var result = db.prepare("SELECT ".concat(exprs.join(', '), " FROM frontmatter ").concat(scopeWhere)).get();
649
680
  group.forEach(function(name, i) {
650
681
  var _result_;
651
682
  return allFields.push({
@@ -674,8 +705,8 @@ function mapTree(db, cfg) {
674
705
  return b.coverage - a.coverage;
675
706
  });
676
707
  var fields = allFields.slice(0, 20);
677
- var hubs = (0, _configts.featureEnabled)(cfg, 'rank') ? db.prepare('SELECT f."path" AS path, round(f."_rank" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f."path" WHERE f."_rank" IS NOT NULL ORDER BY f."_rank" DESC LIMIT 8').all() : [];
678
- var recent = db.prepare('SELECT "path", datetime("_mtime" / 1000, \'unixepoch\') AS modified FROM frontmatter ORDER BY "_mtime" DESC LIMIT 5').all();
708
+ var hubs = (0, _configts.featureEnabled)(cfg, 'rank') ? db.prepare('SELECT f."path" AS path, round(f."_rank" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f."path" WHERE f."_rank" IS NOT NULL '.concat(scopeAnd, ' ORDER BY f."_rank" DESC LIMIT 8')).all() : [];
709
+ var recent = db.prepare('SELECT "path", datetime("_mtime" / 1000, \'unixepoch\') AS modified FROM frontmatter '.concat(scopeWhere, ' ORDER BY "_mtime" DESC LIMIT 5')).all();
679
710
  return {
680
711
  docs: docs,
681
712
  fields: fields,
@@ -686,25 +717,29 @@ function mapTree(db, cfg) {
686
717
  recent: recent
687
718
  };
688
719
  }
720
+ function resolveNote(paths, arg) {
721
+ var exact = paths.find(function(p) {
722
+ return p === arg;
723
+ });
724
+ if (exact) return exact;
725
+ var base = _posix.default.basename(arg).replace(/\.md$/i, '').toLowerCase();
726
+ var matches = paths.filter(function(p) {
727
+ return _posix.default.basename(p).replace(/\.md$/i, '').toLowerCase() === base;
728
+ });
729
+ if (matches.length === 1) return matches[0];
730
+ if (matches.length > 1) throw new _errorsts.SenseError('NOTE_AMBIGUOUS', '"'.concat(arg, '" is ambiguous: ').concat(matches.join(', ')));
731
+ throw new _errorsts.SenseError('NOTE_NOT_FOUND', 'no note matches "'.concat(arg, '"'));
732
+ }
689
733
  var PEEK_LIST_LIMIT = 20;
690
734
  function peek(db, cfg, pathArg) {
691
- var _row__size;
735
+ var overrides = arguments.length > 3 && arguments[3] !== void 0 ? arguments[3] : {};
736
+ var _row__parse_error, _row__size;
692
737
  var paths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
693
738
  return r.path;
694
739
  });
695
- var path = paths.find(function(p) {
696
- return p === pathArg;
697
- });
698
- if (!path) {
699
- var base = _posix.default.basename(pathArg).replace(/\.md$/i, '').toLowerCase();
700
- var matches = paths.filter(function(p) {
701
- return _posix.default.basename(p).replace(/\.md$/i, '').toLowerCase() === base;
702
- });
703
- if (matches.length === 1) path = matches[0];
704
- else if (matches.length > 1) throw new _errorsts.SenseError('NOTE_AMBIGUOUS', '"'.concat(pathArg, '" is ambiguous: ').concat(matches.join(', ')));
705
- else throw new _errorsts.SenseError('NOTE_NOT_FOUND', 'no note matches "'.concat(pathArg, '"'));
706
- }
740
+ var path = resolveNote(paths, pathArg);
707
741
  var row = db.prepare('SELECT * FROM frontmatter WHERE "path" = ?').get(path);
742
+ var parseError = (_row__parse_error = row._parse_error) !== null && _row__parse_error !== void 0 ? _row__parse_error : null;
708
743
  var frontmatter = {};
709
744
  var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
710
745
  try {
@@ -732,6 +767,7 @@ function peek(db, cfg, pathArg) {
732
767
  var backlinks = [];
733
768
  var unresolved = [];
734
769
  var backlinksTotal = 0;
770
+ var _allowed = scopedPaths(db, cfg, overrides);
735
771
  if ((0, _configts.featureEnabled)(cfg, 'links')) {
736
772
  var out = db.prepare('SELECT target, dst FROM links WHERE src = ? ORDER BY target').all(path);
737
773
  outbound = _to_consumable_array(new Set(out.filter(function(l) {
@@ -753,6 +789,7 @@ function peek(db, cfg, pathArg) {
753
789
  path: path,
754
790
  tokens: Math.ceil(((_row__size = row._size) !== null && _row__size !== void 0 ? _row__size : 0) / 4),
755
791
  frontmatter: frontmatter,
792
+ parseError: parseError,
756
793
  sections: sections,
757
794
  outbound: outbound.slice(0, PEEK_LIST_LIMIT),
758
795
  backlinks: backlinks,
@@ -769,4 +806,67 @@ function peek(db, cfg, pathArg) {
769
806
  })
770
807
  };
771
808
  }
809
+ function relatedNotes(db, cfg, pathArg, overrides, k) {
810
+ return _async_to_generator(function() {
811
+ var paths, path, outbound, backlinks, exclude, effective, allowed;
812
+ return _ts_generator(this, function(_state) {
813
+ switch(_state.label){
814
+ case 0:
815
+ paths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
816
+ return r.path;
817
+ });
818
+ path = resolveNote(paths, pathArg);
819
+ outbound = db.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL').all(path).map(function(r) {
820
+ return r.dst;
821
+ });
822
+ backlinks = db.prepare('SELECT DISTINCT src FROM links WHERE dst = ?').all(path).map(function(r) {
823
+ return r.src;
824
+ });
825
+ exclude = new Set([
826
+ path
827
+ ].concat(_to_consumable_array(outbound), _to_consumable_array(backlinks)));
828
+ // Vectors are the only signal `related` has, so every way of not having them is an error
829
+ // naming the cause. An empty table then means one thing: nothing near in meaning that this
830
+ // note does not already link to, which is a real answer.
831
+ effective = (0, _configts.resolveSearch)(cfg, overrides);
832
+ if (!(0, _configts.embedEnabled)(cfg)) {
833
+ throw new _errorsts.SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an "embed" block naming one to sense.config.json, then run `sense download` (search works without it, on words and links)');
834
+ }
835
+ // search gates on the same flag (see wantsVectors above); reading it here too keeps
836
+ // `semantic: false` meaning one thing. Without this, an overlapping semantic-on preset's
837
+ // vectors would answer for a scope that declined them.
838
+ if (!effective.semantic) {
839
+ throw new _errorsts.SenseError('PRESET_NOT_SEMANTIC', 'preset "'.concat(effective.presetName, '" sets "semantic": false, so this scope has no vectors and related has no other signal; search it instead (words and links), or set semantic back on for that preset'));
840
+ }
841
+ if (!(0, _embedts.modelPresent)(cfg)) {
842
+ throw new _errorsts.SenseError('EMBED_MODEL_MISSING', 'related ranks notes by meaning, so it needs the embedding model, which is not downloaded; run `sense download` (search still works without it, on words and links)');
843
+ }
844
+ allowed = scopedPaths(db, cfg, overrides);
845
+ // Top up pending rows before the seed check, or a fresh index reports every note as
846
+ // having no indexed text until some search has run.
847
+ return [
848
+ 4,
849
+ (0, _embedts.embedPending)(db, cfg, cfg.baseDir)
850
+ ];
851
+ case 1:
852
+ _state.sent();
853
+ if (!(0, _embedts.hasEmbedding)(db, path)) {
854
+ throw new _errorsts.SenseError('NOTE_NOT_EMBEDDED', "".concat(path, " has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning"));
855
+ }
856
+ if (!scopeHasEmbeddings(db, cfg, allowed)) return [
857
+ 2,
858
+ []
859
+ ];
860
+ return [
861
+ 2,
862
+ (0, _embedts.similarNotes)(db, cfg, path, {
863
+ exclude: exclude,
864
+ allowed: allowed,
865
+ k: k
866
+ })
867
+ ];
868
+ }
869
+ });
870
+ })();
871
+ }
772
872
  /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }