sensemaking 0.9.5 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -22
- package/dist/cjs/cli/download.d.cts +3 -0
- package/dist/cjs/cli/download.d.ts +3 -0
- package/dist/cjs/cli/download.js +214 -0
- package/dist/cjs/cli/download.js.map +1 -0
- package/dist/cjs/cli/index.d.cts +6 -5
- package/dist/cjs/cli/index.d.ts +6 -5
- package/dist/cjs/cli/index.js +20 -14
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/init.js +4 -1
- package/dist/cjs/cli/init.js.map +1 -1
- package/dist/cjs/cli/map.js +2 -2
- package/dist/cjs/cli/map.js.map +1 -1
- package/dist/cjs/cli/named.js +13 -19
- package/dist/cjs/cli/named.js.map +1 -1
- package/dist/cjs/cli/path.d.cts +3 -0
- package/dist/cjs/cli/path.d.ts +3 -0
- package/dist/cjs/cli/path.js +161 -0
- package/dist/cjs/cli/path.js.map +1 -0
- package/dist/cjs/cli/peek.js +3 -2
- package/dist/cjs/cli/peek.js.map +1 -1
- package/dist/cjs/cli/related.d.cts +3 -0
- package/dist/cjs/cli/related.d.ts +3 -0
- package/dist/cjs/cli/related.js +279 -0
- package/dist/cjs/cli/related.js.map +1 -0
- package/dist/cjs/cli/search.js +3 -7
- package/dist/cjs/cli/search.js.map +1 -1
- package/dist/cjs/cli/shared.d.cts +4 -2
- package/dist/cjs/cli/shared.d.ts +4 -2
- package/dist/cjs/cli/shared.js +49 -6
- package/dist/cjs/cli/shared.js.map +1 -1
- package/dist/cjs/cli/sql.d.cts +3 -0
- package/dist/cjs/cli/sql.d.ts +3 -0
- package/dist/cjs/cli/{query.js → sql.js} +33 -5
- package/dist/cjs/cli/sql.js.map +1 -0
- package/dist/cjs/cli/status.d.cts +1 -0
- package/dist/cjs/cli/status.d.ts +1 -0
- package/dist/cjs/cli/status.js +80 -10
- package/dist/cjs/cli/status.js.map +1 -1
- package/dist/cjs/cli.js +14 -15
- package/dist/cjs/cli.js.map +1 -1
- package/dist/cjs/commands.d.cts +13 -4
- package/dist/cjs/commands.d.ts +13 -4
- package/dist/cjs/commands.js +161 -61
- package/dist/cjs/commands.js.map +1 -1
- package/dist/cjs/config.d.cts +6 -4
- package/dist/cjs/config.d.ts +6 -4
- package/dist/cjs/config.js +137 -58
- package/dist/cjs/config.js.map +1 -1
- package/dist/cjs/db.d.cts +2 -2
- package/dist/cjs/db.d.ts +2 -2
- package/dist/cjs/db.js +25 -46
- package/dist/cjs/db.js.map +1 -1
- package/dist/cjs/errors.d.cts +1 -1
- package/dist/cjs/errors.d.ts +1 -1
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/features/embed.d.cts +15 -0
- package/dist/cjs/features/embed.d.ts +15 -0
- package/dist/cjs/features/embed.js +277 -61
- package/dist/cjs/features/embed.js.map +1 -1
- package/dist/cjs/features/links.js +7 -16
- package/dist/cjs/features/links.js.map +1 -1
- package/dist/cjs/features/rank.js +2 -6
- package/dist/cjs/features/rank.js.map +1 -1
- package/dist/cjs/index.d.cts +2 -2
- package/dist/cjs/index.d.ts +2 -2
- package/dist/cjs/index.js +3 -3
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/output.d.cts +3 -0
- package/dist/cjs/output.d.ts +3 -0
- package/dist/cjs/output.js +17 -15
- package/dist/cjs/output.js.map +1 -1
- package/dist/cjs/progress.js +2 -4
- package/dist/cjs/progress.js.map +1 -1
- package/dist/cjs/scan.d.cts +2 -0
- package/dist/cjs/scan.d.ts +2 -0
- package/dist/cjs/scan.js +150 -44
- package/dist/cjs/scan.js.map +1 -1
- package/dist/cjs/search-error.js +4 -6
- package/dist/cjs/search-error.js.map +1 -1
- package/dist/cjs/traverse.d.cts +8 -0
- package/dist/cjs/traverse.d.ts +8 -0
- package/dist/cjs/traverse.js +97 -0
- package/dist/cjs/traverse.js.map +1 -0
- package/dist/esm/cli/download.d.ts +3 -0
- package/dist/esm/cli/download.js +27 -0
- package/dist/esm/cli/download.js.map +1 -0
- package/dist/esm/cli/index.d.ts +6 -5
- package/dist/esm/cli/index.js +13 -11
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/init.js +4 -1
- package/dist/esm/cli/init.js.map +1 -1
- package/dist/esm/cli/map.js +3 -2
- package/dist/esm/cli/map.js.map +1 -1
- package/dist/esm/cli/named.js +13 -17
- package/dist/esm/cli/named.js.map +1 -1
- package/dist/esm/cli/path.d.ts +3 -0
- package/dist/esm/cli/path.js +51 -0
- package/dist/esm/cli/path.js.map +1 -0
- package/dist/esm/cli/peek.js +4 -2
- package/dist/esm/cli/peek.js.map +1 -1
- package/dist/esm/cli/related.d.ts +3 -0
- package/dist/esm/cli/related.js +26 -0
- package/dist/esm/cli/related.js.map +1 -0
- package/dist/esm/cli/search.js +2 -5
- package/dist/esm/cli/search.js.map +1 -1
- package/dist/esm/cli/shared.d.ts +4 -2
- package/dist/esm/cli/shared.js +49 -6
- package/dist/esm/cli/shared.js.map +1 -1
- package/dist/esm/cli/sql.d.ts +3 -0
- package/dist/esm/cli/{query.js → sql.js} +8 -5
- package/dist/esm/cli/sql.js.map +1 -0
- package/dist/esm/cli/status.d.ts +1 -0
- package/dist/esm/cli/status.js +72 -8
- package/dist/esm/cli/status.js.map +1 -1
- package/dist/esm/cli.js +13 -10
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/commands.d.ts +13 -4
- package/dist/esm/commands.js +135 -72
- package/dist/esm/commands.js.map +1 -1
- package/dist/esm/config.d.ts +6 -4
- package/dist/esm/config.js +126 -75
- package/dist/esm/config.js.map +1 -1
- package/dist/esm/db.d.ts +2 -2
- package/dist/esm/db.js +28 -48
- package/dist/esm/db.js.map +1 -1
- package/dist/esm/errors.d.ts +1 -1
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/features/embed.d.ts +15 -0
- package/dist/esm/features/embed.js +137 -42
- package/dist/esm/features/embed.js.map +1 -1
- package/dist/esm/features/links.js +8 -17
- package/dist/esm/features/links.js.map +1 -1
- package/dist/esm/features/rank.js +2 -6
- package/dist/esm/features/rank.js.map +1 -1
- package/dist/esm/features/types.js.map +1 -1
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/output.d.ts +3 -0
- package/dist/esm/output.js +19 -17
- package/dist/esm/output.js.map +1 -1
- package/dist/esm/progress.js +2 -4
- package/dist/esm/progress.js.map +1 -1
- package/dist/esm/scan.d.ts +2 -0
- package/dist/esm/scan.js +123 -47
- package/dist/esm/scan.js.map +1 -1
- package/dist/esm/search-error.js +4 -6
- package/dist/esm/search-error.js.map +1 -1
- package/dist/esm/traverse.d.ts +8 -0
- package/dist/esm/traverse.js +71 -0
- package/dist/esm/traverse.js.map +1 -0
- package/package.json +7 -3
- package/schema.json +12 -16
- package/skills/sense/EXAMPLES.md +8 -8
- package/skills/sense/SKILL.md +51 -23
- package/skills/sense-setup/EXAMPLES.md +24 -21
- package/skills/sense-setup/SKILL.md +9 -9
- package/dist/cjs/cli/check.d.cts +0 -3
- package/dist/cjs/cli/check.d.ts +0 -3
- package/dist/cjs/cli/check.js +0 -398
- package/dist/cjs/cli/check.js.map +0 -1
- package/dist/cjs/cli/query.d.cts +0 -3
- package/dist/cjs/cli/query.d.ts +0 -3
- package/dist/cjs/cli/query.js.map +0 -1
- package/dist/cjs/cli/rebuild.d.cts +0 -3
- package/dist/cjs/cli/rebuild.d.ts +0 -3
- package/dist/cjs/cli/rebuild.js +0 -50
- package/dist/cjs/cli/rebuild.js.map +0 -1
- package/dist/esm/cli/check.d.ts +0 -3
- package/dist/esm/cli/check.js +0 -94
- package/dist/esm/cli/check.js.map +0 -1
- package/dist/esm/cli/query.d.ts +0 -3
- package/dist/esm/cli/query.js.map +0 -1
- package/dist/esm/cli/rebuild.d.ts +0 -3
- package/dist/esm/cli/rebuild.js +0 -13
- package/dist/esm/cli/rebuild.js.map +0 -1
package/dist/cjs/commands.d.ts
CHANGED
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
-
import type { FeatureName, ResolvedConfig } from './config.js';
|
|
2
|
+
import type { FeatureName, ResolvedConfig, SearchOverrides } from './config.js';
|
|
3
3
|
import type { Row } from './output.js';
|
|
4
4
|
export interface SearchOptions {
|
|
5
5
|
k?: number;
|
|
6
6
|
where?: string;
|
|
7
7
|
preset?: string;
|
|
8
8
|
include?: string[];
|
|
9
|
-
|
|
9
|
+
exclude?: string[];
|
|
10
|
+
noExclude?: boolean;
|
|
10
11
|
}
|
|
11
12
|
export declare function search(db: DatabaseSync, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<Row[]>;
|
|
13
|
+
export declare function scopedPaths(db: DatabaseSync, cfg: ResolvedConfig, overrides: SearchOverrides): Set<string>;
|
|
12
14
|
export interface PresetCoverage {
|
|
13
15
|
name: string;
|
|
14
16
|
files: number;
|
|
15
17
|
embedded: number;
|
|
18
|
+
semantic: boolean;
|
|
16
19
|
}
|
|
17
20
|
export declare function presetCoverage(db: DatabaseSync, cfg: ResolvedConfig): PresetCoverage[];
|
|
18
21
|
export interface TreeMap {
|
|
@@ -30,11 +33,13 @@ export interface TreeMap {
|
|
|
30
33
|
hubs: Row[];
|
|
31
34
|
recent: Row[];
|
|
32
35
|
}
|
|
33
|
-
export declare function mapTree(db: DatabaseSync, cfg: ResolvedConfig): TreeMap;
|
|
36
|
+
export declare function mapTree(db: DatabaseSync, cfg: ResolvedConfig, overrides?: SearchOverrides): TreeMap;
|
|
37
|
+
export declare function resolveNote(paths: string[], arg: string): string;
|
|
34
38
|
export interface Peek {
|
|
35
39
|
path: string;
|
|
36
40
|
tokens: number;
|
|
37
41
|
frontmatter: Row;
|
|
42
|
+
parseError: string | null;
|
|
38
43
|
sections: Row[];
|
|
39
44
|
outbound: string[];
|
|
40
45
|
backlinks: string[];
|
|
@@ -45,4 +50,8 @@ export interface Peek {
|
|
|
45
50
|
unresolvedTotal: number;
|
|
46
51
|
off: FeatureName[];
|
|
47
52
|
}
|
|
48
|
-
export declare function peek(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string): Peek;
|
|
53
|
+
export declare function peek(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides?: SearchOverrides): Peek;
|
|
54
|
+
export declare function relatedNotes(db: DatabaseSync, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{
|
|
55
|
+
path: string;
|
|
56
|
+
similarity: number;
|
|
57
|
+
}>>;
|
package/dist/cjs/commands.js
CHANGED
|
@@ -18,6 +18,15 @@ _export(exports, {
|
|
|
18
18
|
get presetCoverage () {
|
|
19
19
|
return presetCoverage;
|
|
20
20
|
},
|
|
21
|
+
get relatedNotes () {
|
|
22
|
+
return relatedNotes;
|
|
23
|
+
},
|
|
24
|
+
get resolveNote () {
|
|
25
|
+
return resolveNote;
|
|
26
|
+
},
|
|
27
|
+
get scopedPaths () {
|
|
28
|
+
return scopedPaths;
|
|
29
|
+
},
|
|
21
30
|
get search () {
|
|
22
31
|
return search;
|
|
23
32
|
}
|
|
@@ -226,11 +235,8 @@ function _ts_generator(thisArg, body) {
|
|
|
226
235
|
// Each returns data; cli.ts renders. All of them degrade when a feature is off.
|
|
227
236
|
var WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0)';
|
|
228
237
|
var RRF_K = 60;
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
// (plans/performance-findings.md finding A). 16KB keeps the whole pool's snippet cost in
|
|
232
|
-
// the low-ms range; bigger docs get the JS-computed excerpt, linear and only for the rows
|
|
233
|
-
// actually returned.
|
|
238
|
+
// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
|
|
239
|
+
// (BENCHMARKING.md). Past this bound, rows get the linear JS excerpt instead.
|
|
234
240
|
var SNIPPET_BOUND = 16384;
|
|
235
241
|
var EXCERPT_WINDOW = 160;
|
|
236
242
|
// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt
|
|
@@ -367,8 +373,8 @@ function lineRangeFor(db, path, line) {
|
|
|
367
373
|
return row ? "L".concat(row.start_line, "-").concat(row.end_line) : null;
|
|
368
374
|
}
|
|
369
375
|
// node:path's matchesGlob is experimental (stable behind an unstable-API flag) as of the
|
|
370
|
-
// engines floor (Node >=22.
|
|
371
|
-
// it's used here in JS rather than
|
|
376
|
+
// engines floor (Node >=22.20); scope filtering only ever needs single-pattern matching, so
|
|
377
|
+
// it's used here in JS rather than running a directory walk in the query path.
|
|
372
378
|
function inScope(path, include, exclude) {
|
|
373
379
|
if (!include.some(function(g) {
|
|
374
380
|
return (0, _nodepath.matchesGlob)(path, g);
|
|
@@ -387,7 +393,7 @@ function scopeHasEmbeddings(db, cfg, scopedPaths) {
|
|
|
387
393
|
}
|
|
388
394
|
function search(_0, _1, _2) {
|
|
389
395
|
return _async_to_generator(function(db, cfg, terms) {
|
|
390
|
-
var opts, _hits_get, _chunkLines_get, _chunkSimilarity_get, effective, k, include, exclude, allPaths, scopedPaths, scopeActive, fetch, semanticEnabled, scope, whereJoin, whereCond, matchSql, matchRows, hits, oversized, candidates, edges, linked, seeds, ranked, chunkLines, chunkSimilarity, vec, insert, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, path, c, where, similarityCol, rows, bareTerms, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, row, text, _computeExcerpt, excerpt, offset;
|
|
396
|
+
var opts, _hits_get, _chunkLines_get, _chunkSimilarity_get, effective, k, include, exclude, allPaths, adHocScope, scopedPaths, scopeActive, fetch, wantsVectors, semanticEnabled, scope, whereJoin, whereCond, matchSql, matchRows, hits, oversized, candidates, edges, linked, seeds, ranked, chunkLines, chunkSimilarity, vec, insert, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, path, c, where, similarityCol, rows, bareTerms, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, row, text, _computeExcerpt, excerpt, offset;
|
|
391
397
|
var _arguments = arguments;
|
|
392
398
|
return _ts_generator(this, function(_state) {
|
|
393
399
|
switch(_state.label){
|
|
@@ -398,11 +404,10 @@ function search(_0, _1, _2) {
|
|
|
398
404
|
allPaths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
|
|
399
405
|
return r.path;
|
|
400
406
|
});
|
|
401
|
-
// A preset scope reads
|
|
402
|
-
//
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
scopedPaths = opts.include ? new Set(allPaths.filter(function(p) {
|
|
407
|
+
// A preset scope reads the coverage reconcile indexed; re-matching globs here could disagree
|
|
408
|
+
// with it. Only ad-hoc scope flags, which have no persisted coverage, pay the JS match.
|
|
409
|
+
adHocScope = opts.include !== undefined || opts.exclude !== undefined || opts.noExclude === true;
|
|
410
|
+
scopedPaths = adHocScope ? new Set(allPaths.filter(function(p) {
|
|
406
411
|
return inScope(p, include, exclude);
|
|
407
412
|
})) : new Set(db.prepare('SELECT "path" FROM preset_files WHERE preset = ?').all(effective.presetName).map(function(r) {
|
|
408
413
|
return r.path;
|
|
@@ -412,16 +417,16 @@ function search(_0, _1, _2) {
|
|
|
412
417
|
// list that a plain over-fetch wouldn't have reached.
|
|
413
418
|
scopeActive = scopedPaths.size < allPaths.length;
|
|
414
419
|
fetch = scopeActive ? Math.max(k * 5, 50) : Math.max(k * 3, 30);
|
|
415
|
-
//
|
|
416
|
-
//
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
//
|
|
423
|
-
//
|
|
424
|
-
//
|
|
420
|
+
// Asking for vectors without a model is a misconfiguration, not a mode: degrading would make
|
|
421
|
+
// the same search answer differently before and after a download. semantic:false never asks.
|
|
422
|
+
wantsVectors = effective.semantic && (0, _configts.anyPresetEmbeds)(cfg);
|
|
423
|
+
if (wantsVectors && !(0, _embedts.modelPresent)(cfg)) {
|
|
424
|
+
throw new _errorsts.SenseError('EMBED_MODEL_MISSING', 'preset "'.concat(effective.presetName, '" searches with vectors, but the embedding model is not available; run `sense download`, or set "semantic": false on that preset to search on words and links'));
|
|
425
|
+
}
|
|
426
|
+
semanticEnabled = wantsVectors && scopeHasEmbeddings(db, cfg, scopedPaths);
|
|
427
|
+
// Terms pass verbatim to MATCH; invalid syntax errors rather than being rewritten. --where
|
|
428
|
+
// applies inside the candidate query (a post-filter would drop matches ranked past the pool)
|
|
429
|
+
// and again on the final select, for link-derived rows.
|
|
425
430
|
scope = effective.where;
|
|
426
431
|
whereJoin = scope ? 'JOIN frontmatter f ON f."path" = content.path' : '';
|
|
427
432
|
whereCond = scope ? "AND (".concat(scope, ")") : '';
|
|
@@ -461,10 +466,8 @@ function search(_0, _1, _2) {
|
|
|
461
466
|
});
|
|
462
467
|
edges = (0, _configts.featureEnabled)(cfg, 'links') && matchRows.length > 0 ? (0, _indexts.linkEdges)(db) : [];
|
|
463
468
|
if (edges.length > 0) {
|
|
464
|
-
//
|
|
465
|
-
//
|
|
466
|
-
// that mass from the score list reweights fusion toward connectivity and measurably
|
|
467
|
-
// wrecks ranking on link-dense corpora (FEVER hit@10 0.997 -> 0.907; fusion-tuning.md).
|
|
469
|
+
// Gates the label only, not the score: restart mass ranks every seed without an incident
|
|
470
|
+
// edge, but dropping it from the score reweights fusion (FEVER hit@10 0.997 -> 0.907).
|
|
468
471
|
linked = new Set(edges.flat());
|
|
469
472
|
seeds = new Map(matchRows.map(function(r, i) {
|
|
470
473
|
return [
|
|
@@ -525,9 +528,9 @@ function search(_0, _1, _2) {
|
|
|
525
528
|
});
|
|
526
529
|
_state.label = 2;
|
|
527
530
|
case 2:
|
|
528
|
-
db.exec('DROP TABLE IF EXISTS
|
|
529
|
-
db.exec('CREATE TEMP TABLE
|
|
530
|
-
insert = db.prepare('INSERT INTO
|
|
531
|
+
db.exec('DROP TABLE IF EXISTS _search');
|
|
532
|
+
db.exec('CREATE TEMP TABLE _search ("path" TEXT PRIMARY KEY, score REAL, via TEXT, hit TEXT, lines TEXT, similarity REAL)');
|
|
533
|
+
insert = db.prepare('INSERT INTO _search ("path", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)');
|
|
531
534
|
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
532
535
|
try {
|
|
533
536
|
for(_iterator = candidates[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
@@ -548,15 +551,13 @@ function search(_0, _1, _2) {
|
|
|
548
551
|
}
|
|
549
552
|
}
|
|
550
553
|
}
|
|
551
|
-
//
|
|
552
|
-
//
|
|
553
|
-
// whichever candidate query didn't itself apply it to (link-derived rows never touched
|
|
554
|
-
// whereCond above).
|
|
554
|
+
// Already scope-filtered above; the final select reapplies --where only for link-derived
|
|
555
|
+
// rows, which never passed through whereCond.
|
|
555
556
|
where = scope ? "WHERE (".concat(scope, ")") : '';
|
|
556
557
|
// lines is always present now: semantic rows carry their chunk's range, oversized-doc
|
|
557
558
|
// lexical rows gain one below, everything else stays null. similarity stays semantic-only.
|
|
558
|
-
similarityCol = semanticEnabled ? ',
|
|
559
|
-
rows = db.prepare('SELECT f."path" AS path, content.title, content.summary,
|
|
559
|
+
similarityCol = semanticEnabled ? ', _search.similarity' : '';
|
|
560
|
+
rows = db.prepare('SELECT f."path" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines'.concat(similarityCol, '\n FROM _search JOIN frontmatter f ON f."path" = _search."path" JOIN content ON content.path = _search."path"\n ').concat(where, " ORDER BY _search.score DESC LIMIT ?")).all(k);
|
|
560
561
|
if (oversized.size > 0) {
|
|
561
562
|
bareTerms = extractBareTerms(terms);
|
|
562
563
|
_iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
|
|
@@ -597,6 +598,27 @@ function search(_0, _1, _2) {
|
|
|
597
598
|
});
|
|
598
599
|
}).apply(this, arguments);
|
|
599
600
|
}
|
|
601
|
+
function scopedPaths(db, cfg, overrides) {
|
|
602
|
+
var effective = (0, _configts.resolveSearch)(cfg, overrides);
|
|
603
|
+
var include = effective.include, exclude = effective.exclude, where = effective.where;
|
|
604
|
+
var allPaths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
|
|
605
|
+
return r.path;
|
|
606
|
+
});
|
|
607
|
+
var adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;
|
|
608
|
+
var paths = adHocScope ? new Set(allPaths.filter(function(p) {
|
|
609
|
+
return inScope(p, include, exclude);
|
|
610
|
+
})) : new Set(db.prepare('SELECT "path" FROM preset_files WHERE preset = ?').all(effective.presetName).map(function(r) {
|
|
611
|
+
return r.path;
|
|
612
|
+
}));
|
|
613
|
+
if (!where) return paths;
|
|
614
|
+
var whereRows = db.prepare('SELECT "path" FROM frontmatter f WHERE ('.concat(where, ")")).all();
|
|
615
|
+
var wherePaths = new Set(whereRows.map(function(r) {
|
|
616
|
+
return r.path;
|
|
617
|
+
}));
|
|
618
|
+
return new Set(_to_consumable_array(paths).filter(function(p) {
|
|
619
|
+
return wherePaths.has(p);
|
|
620
|
+
}));
|
|
621
|
+
}
|
|
600
622
|
function presetCoverage(db, cfg) {
|
|
601
623
|
var embedActive = (0, _configts.anyPresetEmbeds)(cfg);
|
|
602
624
|
return (0, _configts.presetNames)(cfg).map(function(name) {
|
|
@@ -605,7 +627,8 @@ function presetCoverage(db, cfg) {
|
|
|
605
627
|
return {
|
|
606
628
|
name: name,
|
|
607
629
|
files: files,
|
|
608
|
-
embedded: embedded
|
|
630
|
+
embedded: embedded,
|
|
631
|
+
semantic: (0, _configts.presetSemanticEnabled)(cfg, name)
|
|
609
632
|
};
|
|
610
633
|
});
|
|
611
634
|
}
|
|
@@ -613,7 +636,8 @@ var INTERNAL_COLUMNS = new Set([
|
|
|
613
636
|
'path',
|
|
614
637
|
'_mtime',
|
|
615
638
|
'_size',
|
|
616
|
-
'_rank'
|
|
639
|
+
'_rank',
|
|
640
|
+
'_parse_error'
|
|
617
641
|
]);
|
|
618
642
|
// A result row is capped at SQLITE_MAX_COLUMN (2000, default); two aggregate expressions
|
|
619
643
|
// per field keeps a chunk's row safely under that regardless of how many fields the tree has.
|
|
@@ -623,19 +647,26 @@ function chunk(items, size) {
|
|
|
623
647
|
for(var i = 0; i < items.length; i += size)out.push(items.slice(i, i + size));
|
|
624
648
|
return out;
|
|
625
649
|
}
|
|
650
|
+
// Materializes the resolved scope into a temp table (same shape as traverse.ts's
|
|
651
|
+
// allowed_nodes) so every mapTree query can join/filter against it cheaply.
|
|
652
|
+
function setupMapScope(db, paths) {
|
|
653
|
+
db.exec('DROP TABLE IF EXISTS _map_scope');
|
|
654
|
+
db.exec('CREATE TEMP TABLE _map_scope ("path" TEXT PRIMARY KEY)');
|
|
655
|
+
db.prepare('INSERT INTO _map_scope SELECT DISTINCT value FROM json_each(?1)').run(JSON.stringify(_to_consumable_array(paths)));
|
|
656
|
+
}
|
|
626
657
|
function mapTree(db, cfg) {
|
|
627
|
-
var
|
|
658
|
+
var overrides = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : {};
|
|
659
|
+
setupMapScope(db, scopedPaths(db, cfg, overrides));
|
|
660
|
+
var scopeWhere = 'WHERE "path" IN (SELECT "path" FROM _map_scope)';
|
|
661
|
+
var scopeAnd = 'AND f."path" IN (SELECT "path" FROM _map_scope)';
|
|
662
|
+
var docs = db.prepare('SELECT COUNT(*) AS count, COALESCE(SUM("_size"), 0) AS bytes FROM frontmatter '.concat(scopeWhere)).get();
|
|
628
663
|
var columns = db.prepare('PRAGMA table_info(frontmatter)').all().map(function(r) {
|
|
629
664
|
return r.name;
|
|
630
665
|
}).filter(function(name) {
|
|
631
666
|
return !INTERNAL_COLUMNS.has(name);
|
|
632
667
|
});
|
|
633
|
-
// Observed
|
|
634
|
-
//
|
|
635
|
-
// Listing every distinct type makes both the type and any drift visible.
|
|
636
|
-
// One aggregate scan per chunk of columns rather than one scan per column: COUNT already
|
|
637
|
-
// ignores NULL, and FILTER keeps typeof() looking only at the non-null rows COUNT counted,
|
|
638
|
-
// matching the old per-column `WHERE ... IS NOT NULL` exactly.
|
|
668
|
+
// Observed types, not declared: SQLite types per value, so a field can be text in most notes
|
|
669
|
+
// and numeric in a few. One aggregate scan per chunk of columns; FILTER matches COUNT's nulls.
|
|
639
670
|
var allFields = [];
|
|
640
671
|
var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
641
672
|
try {
|
|
@@ -645,7 +676,7 @@ function mapTree(db, cfg) {
|
|
|
645
676
|
var quoted = '"'.concat(name.split('"').join('""'), '"');
|
|
646
677
|
return "COUNT(".concat(quoted, ") AS n").concat(i, ", GROUP_CONCAT(DISTINCT typeof(").concat(quoted, ")) FILTER (WHERE ").concat(quoted, " IS NOT NULL) AS t").concat(i);
|
|
647
678
|
});
|
|
648
|
-
var result = db.prepare("SELECT ".concat(exprs.join(', '), " FROM frontmatter")).get();
|
|
679
|
+
var result = db.prepare("SELECT ".concat(exprs.join(', '), " FROM frontmatter ").concat(scopeWhere)).get();
|
|
649
680
|
group.forEach(function(name, i) {
|
|
650
681
|
var _result_;
|
|
651
682
|
return allFields.push({
|
|
@@ -674,8 +705,8 @@ function mapTree(db, cfg) {
|
|
|
674
705
|
return b.coverage - a.coverage;
|
|
675
706
|
});
|
|
676
707
|
var fields = allFields.slice(0, 20);
|
|
677
|
-
var hubs = (0, _configts.featureEnabled)(cfg, 'rank') ? db.prepare('SELECT f."path" AS path, round(f."_rank" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f."path" WHERE f."_rank" IS NOT NULL ORDER BY f."_rank" DESC LIMIT 8').all() : [];
|
|
678
|
-
var recent = db.prepare('SELECT "path", datetime("_mtime" / 1000, \'unixepoch\') AS modified FROM frontmatter ORDER BY "_mtime" DESC LIMIT 5').all();
|
|
708
|
+
var hubs = (0, _configts.featureEnabled)(cfg, 'rank') ? db.prepare('SELECT f."path" AS path, round(f."_rank" * 100, 2) AS rank, content.title FROM frontmatter f JOIN content ON content.path = f."path" WHERE f."_rank" IS NOT NULL '.concat(scopeAnd, ' ORDER BY f."_rank" DESC LIMIT 8')).all() : [];
|
|
709
|
+
var recent = db.prepare('SELECT "path", datetime("_mtime" / 1000, \'unixepoch\') AS modified FROM frontmatter '.concat(scopeWhere, ' ORDER BY "_mtime" DESC LIMIT 5')).all();
|
|
679
710
|
return {
|
|
680
711
|
docs: docs,
|
|
681
712
|
fields: fields,
|
|
@@ -686,25 +717,29 @@ function mapTree(db, cfg) {
|
|
|
686
717
|
recent: recent
|
|
687
718
|
};
|
|
688
719
|
}
|
|
720
|
+
function resolveNote(paths, arg) {
|
|
721
|
+
var exact = paths.find(function(p) {
|
|
722
|
+
return p === arg;
|
|
723
|
+
});
|
|
724
|
+
if (exact) return exact;
|
|
725
|
+
var base = _posix.default.basename(arg).replace(/\.md$/i, '').toLowerCase();
|
|
726
|
+
var matches = paths.filter(function(p) {
|
|
727
|
+
return _posix.default.basename(p).replace(/\.md$/i, '').toLowerCase() === base;
|
|
728
|
+
});
|
|
729
|
+
if (matches.length === 1) return matches[0];
|
|
730
|
+
if (matches.length > 1) throw new _errorsts.SenseError('NOTE_AMBIGUOUS', '"'.concat(arg, '" is ambiguous: ').concat(matches.join(', ')));
|
|
731
|
+
throw new _errorsts.SenseError('NOTE_NOT_FOUND', 'no note matches "'.concat(arg, '"'));
|
|
732
|
+
}
|
|
689
733
|
var PEEK_LIST_LIMIT = 20;
|
|
690
734
|
function peek(db, cfg, pathArg) {
|
|
691
|
-
var
|
|
735
|
+
var overrides = arguments.length > 3 && arguments[3] !== void 0 ? arguments[3] : {};
|
|
736
|
+
var _row__parse_error, _row__size;
|
|
692
737
|
var paths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
|
|
693
738
|
return r.path;
|
|
694
739
|
});
|
|
695
|
-
var path = paths
|
|
696
|
-
return p === pathArg;
|
|
697
|
-
});
|
|
698
|
-
if (!path) {
|
|
699
|
-
var base = _posix.default.basename(pathArg).replace(/\.md$/i, '').toLowerCase();
|
|
700
|
-
var matches = paths.filter(function(p) {
|
|
701
|
-
return _posix.default.basename(p).replace(/\.md$/i, '').toLowerCase() === base;
|
|
702
|
-
});
|
|
703
|
-
if (matches.length === 1) path = matches[0];
|
|
704
|
-
else if (matches.length > 1) throw new _errorsts.SenseError('NOTE_AMBIGUOUS', '"'.concat(pathArg, '" is ambiguous: ').concat(matches.join(', ')));
|
|
705
|
-
else throw new _errorsts.SenseError('NOTE_NOT_FOUND', 'no note matches "'.concat(pathArg, '"'));
|
|
706
|
-
}
|
|
740
|
+
var path = resolveNote(paths, pathArg);
|
|
707
741
|
var row = db.prepare('SELECT * FROM frontmatter WHERE "path" = ?').get(path);
|
|
742
|
+
var parseError = (_row__parse_error = row._parse_error) !== null && _row__parse_error !== void 0 ? _row__parse_error : null;
|
|
708
743
|
var frontmatter = {};
|
|
709
744
|
var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
710
745
|
try {
|
|
@@ -732,6 +767,7 @@ function peek(db, cfg, pathArg) {
|
|
|
732
767
|
var backlinks = [];
|
|
733
768
|
var unresolved = [];
|
|
734
769
|
var backlinksTotal = 0;
|
|
770
|
+
var _allowed = scopedPaths(db, cfg, overrides);
|
|
735
771
|
if ((0, _configts.featureEnabled)(cfg, 'links')) {
|
|
736
772
|
var out = db.prepare('SELECT target, dst FROM links WHERE src = ? ORDER BY target').all(path);
|
|
737
773
|
outbound = _to_consumable_array(new Set(out.filter(function(l) {
|
|
@@ -753,6 +789,7 @@ function peek(db, cfg, pathArg) {
|
|
|
753
789
|
path: path,
|
|
754
790
|
tokens: Math.ceil(((_row__size = row._size) !== null && _row__size !== void 0 ? _row__size : 0) / 4),
|
|
755
791
|
frontmatter: frontmatter,
|
|
792
|
+
parseError: parseError,
|
|
756
793
|
sections: sections,
|
|
757
794
|
outbound: outbound.slice(0, PEEK_LIST_LIMIT),
|
|
758
795
|
backlinks: backlinks,
|
|
@@ -769,4 +806,67 @@ function peek(db, cfg, pathArg) {
|
|
|
769
806
|
})
|
|
770
807
|
};
|
|
771
808
|
}
|
|
809
|
+
function relatedNotes(db, cfg, pathArg, overrides, k) {
|
|
810
|
+
return _async_to_generator(function() {
|
|
811
|
+
var paths, path, outbound, backlinks, exclude, effective, allowed;
|
|
812
|
+
return _ts_generator(this, function(_state) {
|
|
813
|
+
switch(_state.label){
|
|
814
|
+
case 0:
|
|
815
|
+
paths = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
|
|
816
|
+
return r.path;
|
|
817
|
+
});
|
|
818
|
+
path = resolveNote(paths, pathArg);
|
|
819
|
+
outbound = db.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL').all(path).map(function(r) {
|
|
820
|
+
return r.dst;
|
|
821
|
+
});
|
|
822
|
+
backlinks = db.prepare('SELECT DISTINCT src FROM links WHERE dst = ?').all(path).map(function(r) {
|
|
823
|
+
return r.src;
|
|
824
|
+
});
|
|
825
|
+
exclude = new Set([
|
|
826
|
+
path
|
|
827
|
+
].concat(_to_consumable_array(outbound), _to_consumable_array(backlinks)));
|
|
828
|
+
// Vectors are the only signal `related` has, so every way of not having them is an error
|
|
829
|
+
// naming the cause. An empty table then means one thing: nothing near in meaning that this
|
|
830
|
+
// note does not already link to, which is a real answer.
|
|
831
|
+
effective = (0, _configts.resolveSearch)(cfg, overrides);
|
|
832
|
+
if (!(0, _configts.embedEnabled)(cfg)) {
|
|
833
|
+
throw new _errorsts.SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an "embed" block naming one to sense.config.json, then run `sense download` (search works without it, on words and links)');
|
|
834
|
+
}
|
|
835
|
+
// search gates on the same flag (see wantsVectors above); reading it here too keeps
|
|
836
|
+
// `semantic: false` meaning one thing. Without this, an overlapping semantic-on preset's
|
|
837
|
+
// vectors would answer for a scope that declined them.
|
|
838
|
+
if (!effective.semantic) {
|
|
839
|
+
throw new _errorsts.SenseError('PRESET_NOT_SEMANTIC', 'preset "'.concat(effective.presetName, '" sets "semantic": false, so this scope has no vectors and related has no other signal; search it instead (words and links), or set semantic back on for that preset'));
|
|
840
|
+
}
|
|
841
|
+
if (!(0, _embedts.modelPresent)(cfg)) {
|
|
842
|
+
throw new _errorsts.SenseError('EMBED_MODEL_MISSING', 'related ranks notes by meaning, so it needs the embedding model, which is not downloaded; run `sense download` (search still works without it, on words and links)');
|
|
843
|
+
}
|
|
844
|
+
allowed = scopedPaths(db, cfg, overrides);
|
|
845
|
+
// Top up pending rows before the seed check, or a fresh index reports every note as
|
|
846
|
+
// having no indexed text until some search has run.
|
|
847
|
+
return [
|
|
848
|
+
4,
|
|
849
|
+
(0, _embedts.embedPending)(db, cfg, cfg.baseDir)
|
|
850
|
+
];
|
|
851
|
+
case 1:
|
|
852
|
+
_state.sent();
|
|
853
|
+
if (!(0, _embedts.hasEmbedding)(db, path)) {
|
|
854
|
+
throw new _errorsts.SenseError('NOTE_NOT_EMBEDDED', "".concat(path, " has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning"));
|
|
855
|
+
}
|
|
856
|
+
if (!scopeHasEmbeddings(db, cfg, allowed)) return [
|
|
857
|
+
2,
|
|
858
|
+
[]
|
|
859
|
+
];
|
|
860
|
+
return [
|
|
861
|
+
2,
|
|
862
|
+
(0, _embedts.similarNotes)(db, cfg, path, {
|
|
863
|
+
exclude: exclude,
|
|
864
|
+
allowed: allowed,
|
|
865
|
+
k: k
|
|
866
|
+
})
|
|
867
|
+
];
|
|
868
|
+
}
|
|
869
|
+
});
|
|
870
|
+
})();
|
|
871
|
+
}
|
|
772
872
|
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|