sensemaking 0.23.1 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/dist/cjs/cli/index.d.cts +1 -1
- package/dist/cjs/cli/index.d.ts +1 -1
- package/dist/cjs/cli/index.js +1 -1
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/named.js +6 -2
- package/dist/cjs/cli/named.js.map +1 -1
- package/dist/cjs/cli/search.js +6 -2
- package/dist/cjs/cli/search.js.map +1 -1
- package/dist/cjs/cli/shared.d.cts +2 -0
- package/dist/cjs/cli/shared.d.ts +2 -0
- package/dist/cjs/cli/shared.js +24 -0
- package/dist/cjs/cli/shared.js.map +1 -1
- package/dist/cjs/commands/map.js +2 -2
- package/dist/cjs/commands/map.js.map +1 -1
- package/dist/cjs/commands/search.d.cts +10 -0
- package/dist/cjs/commands/search.d.ts +10 -0
- package/dist/cjs/commands/search.js +488 -180
- package/dist/cjs/commands/search.js.map +1 -1
- package/dist/cjs/output/output.js +45 -8
- package/dist/cjs/output/output.js.map +1 -1
- package/dist/cjs/output/search-error.d.cts +1 -0
- package/dist/cjs/output/search-error.d.ts +1 -0
- package/dist/cjs/output/search-error.js +28 -4
- package/dist/cjs/output/search-error.js.map +1 -1
- package/dist/cjs/scan/frontmatter.js +1 -1
- package/dist/cjs/scan/frontmatter.js.map +1 -1
- package/dist/cjs/scan/index.js +1 -0
- package/dist/cjs/scan/index.js.map +1 -1
- package/dist/cjs/store/duckdb/fieldStats.js +4 -7
- package/dist/cjs/store/duckdb/fieldStats.js.map +1 -1
- package/dist/cjs/store/duckdb/lexical.d.cts +2 -2
- package/dist/cjs/store/duckdb/lexical.d.ts +2 -2
- package/dist/cjs/store/duckdb/lexical.js +215 -64
- package/dist/cjs/store/duckdb/lexical.js.map +1 -1
- package/dist/cjs/store/duckdb/reconcile.js +7 -2
- package/dist/cjs/store/duckdb/reconcile.js.map +1 -1
- package/dist/cjs/store/duckdb/store.js +1 -3
- package/dist/cjs/store/duckdb/store.js.map +1 -1
- package/dist/cjs/store/duckdb/vectors.js +13 -8
- package/dist/cjs/store/duckdb/vectors.js.map +1 -1
- package/dist/cjs/store/native.js +5 -4
- package/dist/cjs/store/native.js.map +1 -1
- package/dist/cjs/store/open.js +7 -0
- package/dist/cjs/store/open.js.map +1 -1
- package/dist/cjs/store/shared.js.map +1 -1
- package/dist/cjs/store/sqlite/lexical.js +1 -4
- package/dist/cjs/store/sqlite/lexical.js.map +1 -1
- package/dist/cjs/store/sqlite/store.js +1 -4
- package/dist/cjs/store/sqlite/store.js.map +1 -1
- package/dist/cjs/store/sqlite/vectors.js +78 -24
- package/dist/cjs/store/sqlite/vectors.js.map +1 -1
- package/dist/cjs/store/turso/connection.js +2 -1
- package/dist/cjs/store/turso/connection.js.map +1 -1
- package/dist/cjs/store/turso/lexical-text.d.cts +1 -0
- package/dist/cjs/store/turso/lexical-text.d.ts +1 -0
- package/dist/cjs/store/turso/lexical-text.js +40 -0
- package/dist/cjs/store/turso/lexical-text.js.map +1 -0
- package/dist/cjs/store/turso/lexical.js +89 -16
- package/dist/cjs/store/turso/lexical.js.map +1 -1
- package/dist/cjs/store/turso/open.d.cts +1 -1
- package/dist/cjs/store/turso/open.d.ts +1 -1
- package/dist/cjs/store/turso/open.js +2 -2
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/reconcile.d.cts +1 -1
- package/dist/cjs/store/turso/reconcile.d.ts +1 -1
- package/dist/cjs/store/turso/reconcile.js +6 -2
- package/dist/cjs/store/turso/reconcile.js.map +1 -1
- package/dist/cjs/store/turso/sql-functions.d.cts +1 -0
- package/dist/cjs/store/turso/sql-functions.d.ts +1 -0
- package/dist/cjs/store/turso/sql-functions.js +174 -0
- package/dist/cjs/store/turso/sql-functions.js.map +1 -0
- package/dist/cjs/store/turso/store.js +2 -3
- package/dist/cjs/store/turso/store.js.map +1 -1
- package/dist/cjs/store/turso/vectors.js +50 -14
- package/dist/cjs/store/turso/vectors.js.map +1 -1
- package/dist/cjs/store/types.d.cts +1 -2
- package/dist/cjs/store/types.d.ts +1 -2
- package/dist/cjs/store/types.js +1 -4
- package/dist/cjs/store/types.js.map +1 -1
- package/dist/cjs/store/vectors.d.cts +7 -0
- package/dist/cjs/store/vectors.d.ts +7 -0
- package/dist/cjs/store/vectors.js +8 -0
- package/dist/cjs/store/vectors.js.map +1 -1
- package/dist/cjs/text/segment.d.cts +13 -0
- package/dist/cjs/text/segment.d.ts +13 -0
- package/dist/cjs/text/segment.js +109 -4
- package/dist/cjs/text/segment.js.map +1 -1
- package/dist/esm/cli/index.d.ts +1 -1
- package/dist/esm/cli/index.js +1 -1
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/named.js +6 -2
- package/dist/esm/cli/named.js.map +1 -1
- package/dist/esm/cli/search.js +5 -1
- package/dist/esm/cli/search.js.map +1 -1
- package/dist/esm/cli/shared.d.ts +2 -0
- package/dist/esm/cli/shared.js +19 -0
- package/dist/esm/cli/shared.js.map +1 -1
- package/dist/esm/commands/map.js +2 -2
- package/dist/esm/commands/map.js.map +1 -1
- package/dist/esm/commands/search.d.ts +10 -0
- package/dist/esm/commands/search.js +271 -91
- package/dist/esm/commands/search.js.map +1 -1
- package/dist/esm/output/output.js +27 -4
- package/dist/esm/output/output.js.map +1 -1
- package/dist/esm/output/search-error.d.ts +1 -0
- package/dist/esm/output/search-error.js +17 -1
- package/dist/esm/output/search-error.js.map +1 -1
- package/dist/esm/scan/frontmatter.js +1 -1
- package/dist/esm/scan/frontmatter.js.map +1 -1
- package/dist/esm/scan/index.js +1 -0
- package/dist/esm/scan/index.js.map +1 -1
- package/dist/esm/store/duckdb/fieldStats.js +4 -7
- package/dist/esm/store/duckdb/fieldStats.js.map +1 -1
- package/dist/esm/store/duckdb/lexical.d.ts +2 -2
- package/dist/esm/store/duckdb/lexical.js +94 -40
- package/dist/esm/store/duckdb/lexical.js.map +1 -1
- package/dist/esm/store/duckdb/reconcile.js +2 -2
- package/dist/esm/store/duckdb/reconcile.js.map +1 -1
- package/dist/esm/store/duckdb/store.js +2 -4
- package/dist/esm/store/duckdb/store.js.map +1 -1
- package/dist/esm/store/duckdb/vectors.js +26 -13
- package/dist/esm/store/duckdb/vectors.js.map +1 -1
- package/dist/esm/store/native.js +5 -4
- package/dist/esm/store/native.js.map +1 -1
- package/dist/esm/store/open.js +7 -0
- package/dist/esm/store/open.js.map +1 -1
- package/dist/esm/store/shared.js +1 -2
- package/dist/esm/store/shared.js.map +1 -1
- package/dist/esm/store/sqlite/lexical.js +2 -5
- package/dist/esm/store/sqlite/lexical.js.map +1 -1
- package/dist/esm/store/sqlite/store.js +1 -4
- package/dist/esm/store/sqlite/store.js.map +1 -1
- package/dist/esm/store/sqlite/vectors.js +48 -19
- package/dist/esm/store/sqlite/vectors.js.map +1 -1
- package/dist/esm/store/turso/connection.js +3 -2
- package/dist/esm/store/turso/connection.js.map +1 -1
- package/dist/esm/store/turso/lexical-text.d.ts +1 -0
- package/dist/esm/store/turso/lexical-text.js +12 -0
- package/dist/esm/store/turso/lexical-text.js.map +1 -0
- package/dist/esm/store/turso/lexical.js +56 -21
- package/dist/esm/store/turso/lexical.js.map +1 -1
- package/dist/esm/store/turso/open.d.ts +1 -1
- package/dist/esm/store/turso/open.js +2 -2
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/reconcile.d.ts +1 -1
- package/dist/esm/store/turso/reconcile.js +6 -2
- package/dist/esm/store/turso/reconcile.js.map +1 -1
- package/dist/esm/store/turso/sql-functions.d.ts +1 -0
- package/dist/esm/store/turso/sql-functions.js +179 -0
- package/dist/esm/store/turso/sql-functions.js.map +1 -0
- package/dist/esm/store/turso/store.js +3 -5
- package/dist/esm/store/turso/store.js.map +1 -1
- package/dist/esm/store/turso/vectors.js +19 -10
- package/dist/esm/store/turso/vectors.js.map +1 -1
- package/dist/esm/store/types.d.ts +1 -2
- package/dist/esm/store/types.js +6 -8
- package/dist/esm/store/types.js.map +1 -1
- package/dist/esm/store/vectors.d.ts +7 -0
- package/dist/esm/store/vectors.js +8 -2
- package/dist/esm/store/vectors.js.map +1 -1
- package/dist/esm/text/segment.d.ts +13 -0
- package/dist/esm/text/segment.js +70 -0
- package/dist/esm/text/segment.js.map +1 -1
- package/package.json +4 -3
- package/schema.json +1 -1
- package/skills/sense/EXAMPLES.md +3 -3
- package/skills/sense/SKILL.md +6 -6
- package/skills/sense-setup/SKILL.md +1 -1
|
@@ -1,96 +1,266 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { stemmer } from 'stemmer';
|
|
3
4
|
import { embedConfig, featureEnabled, resolveSearch } from '../config/index.js';
|
|
4
5
|
import { localModelMissing, MODEL_FILENAMES } from '../embed/store.js';
|
|
5
6
|
import { SenseError } from '../errors.js';
|
|
6
|
-
import { searchError } from '../output/search-error.js';
|
|
7
|
+
import { bareTermSyntaxError, searchError } from '../output/search-error.js';
|
|
8
|
+
import { foldForSearch, hasUnspacedRun, searchTokens, unspacedRuns } from '../text/segment.js';
|
|
7
9
|
import { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.js';
|
|
8
10
|
import { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.js';
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
//
|
|
11
|
+
// Default budget behind --snippet-char-limit/--snippet-count-limit: the limit is whatever the
|
|
12
|
+
// caller passes, these only say what happens when they pass nothing (DESIGN.md "Search snippets").
|
|
13
|
+
export const SNIPPET_CHAR_LIMIT_DEFAULT = 80;
|
|
14
|
+
export const SNIPPET_COUNT_LIMIT_DEFAULT = 1;
|
|
15
|
+
// Bare terms from an FTS5 query string: strips operators/quoting, not real terms, so the
|
|
16
|
+
// snippet scan matches what the query matched -- FTS5 operators are uppercase-only, so this is too.
|
|
14
17
|
function extractBareTerms(query) {
|
|
15
|
-
const cleaned = query.replace(/"/g, ' ').replace(/[()*]/g, ' ').replace(/\b(AND|OR|NOT|NEAR)\b(\/\d+)?/
|
|
18
|
+
const cleaned = query.replace(/"/g, ' ').replace(/[()*]/g, ' ').replace(/\b(AND|OR|NOT|NEAR)\b(\/\d+)?/g, ' ');
|
|
16
19
|
return cleaned.split(/\s+/).map((tok)=>tok.replace(/^[A-Za-z_]\w*:/, '')) // column filter, e.g. title:term
|
|
17
|
-
.map((tok)=>tok.toLowerCase().trim()).filter((tok)=>tok.length > 0);
|
|
20
|
+
.flatMap((tok)=>searchTokens(tok).map(({ text })=>text)).map((tok)=>tok.toLowerCase().trim()).filter((tok)=>tok.length > 0);
|
|
18
21
|
}
|
|
19
|
-
|
|
22
|
+
// House convention (src/chunk/group.ts): Intl.Segmenter word boundaries, not a bare \b, which
|
|
23
|
+
// is ASCII-only. Built on first use: construction is a few ms, paid only when a term matches.
|
|
24
|
+
let wordSegmenter;
|
|
25
|
+
function findOccurrences(text, terms) {
|
|
20
26
|
const occ = [];
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
const textLower = text.toLowerCase();
|
|
28
|
+
// Unspaced scripts (UNSPACED_SCRIPTS) mark no word boundaries: the sense-setup skill promises
|
|
29
|
+
// substring semantics there, so a matching term is found wherever it occurs in the run.
|
|
30
|
+
const unspacedTerms = terms.filter(hasUnspacedRun);
|
|
31
|
+
if (unspacedTerms.length > 0) {
|
|
32
|
+
for (const run of unspacedRuns(text)){
|
|
33
|
+
const runLower = textLower.slice(run.start, run.end);
|
|
34
|
+
for (const term of unspacedTerms){
|
|
35
|
+
let idx = 0;
|
|
36
|
+
for(;;){
|
|
37
|
+
const found = runLower.indexOf(term, idx);
|
|
38
|
+
if (found === -1) break;
|
|
39
|
+
occ.push({
|
|
40
|
+
start: run.start + found,
|
|
41
|
+
end: run.start + found + term.length,
|
|
42
|
+
term
|
|
43
|
+
});
|
|
44
|
+
idx = found + term.length;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
// Spaced scripts: a word is marked when its porter stem equals a query term's stem, the same
|
|
50
|
+
// algorithm both indexes tokenize with (sqlite's TOKENIZE = 'porter unicode61'; duckdb's fts default).
|
|
51
|
+
const spacedTerms = terms.filter((t)=>!hasUnspacedRun(t));
|
|
52
|
+
if (spacedTerms.length > 0) {
|
|
53
|
+
const termByStem = new Map(spacedTerms.map((t)=>[
|
|
54
|
+
stemmer(foldForSearch(t)),
|
|
55
|
+
t
|
|
56
|
+
]));
|
|
57
|
+
const stemInitials = new Set(Array.from(termByStem.keys(), (term)=>term[0]));
|
|
58
|
+
for (const match of searchTokens(text)){
|
|
59
|
+
const segment = match.text;
|
|
60
|
+
if (match.unspaced) continue;
|
|
61
|
+
const folded = foldForSearch(segment);
|
|
62
|
+
// Porter stemming changes suffixes only, so a different initial cannot match a query stem.
|
|
63
|
+
if (!stemInitials.has(folded[0])) continue;
|
|
64
|
+
const term = termByStem.get(stemmer(folded));
|
|
65
|
+
if (term !== undefined) occ.push({
|
|
66
|
+
start: match.start,
|
|
67
|
+
end: match.end,
|
|
29
68
|
term
|
|
30
69
|
});
|
|
31
|
-
idx = found + term.length;
|
|
32
70
|
}
|
|
33
71
|
}
|
|
34
|
-
// Longest span first at equal start, so "tests" beats its substring "test" and the
|
|
35
|
-
// whole word gets highlighted; the emit loop then absorbs the shorter overlap.
|
|
36
72
|
return occ.sort((a, b)=>a.start - b.start || b.end - a.end);
|
|
37
73
|
}
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
74
|
+
// Segment boundaries of the whole text, word granularity: the only cut points a snippet edge
|
|
75
|
+
// may land on. One rule for every script (DESIGN.md "Cutting") -- a regex \b is ASCII-only and
|
|
76
|
+
// would mis-cut Japanese, Chinese and Thai, which Intl.Segmenter divides by dictionary instead.
|
|
77
|
+
function wordBoundaries(text) {
|
|
78
|
+
wordSegmenter !== null && wordSegmenter !== void 0 ? wordSegmenter : wordSegmenter = new Intl.Segmenter('en', {
|
|
79
|
+
granularity: 'word'
|
|
80
|
+
});
|
|
81
|
+
const bounds = [
|
|
82
|
+
0
|
|
83
|
+
];
|
|
84
|
+
for (const { index, segment } of wordSegmenter.segment(text))bounds.push(index + segment.length);
|
|
85
|
+
return bounds;
|
|
86
|
+
}
|
|
87
|
+
// bounds is sorted: snaps are binary searches, since each runs once per candidate window.
|
|
88
|
+
function snapForward(bounds, pos) {
|
|
89
|
+
let lo = 0;
|
|
90
|
+
let hi = bounds.length - 1;
|
|
91
|
+
while(lo < hi){
|
|
92
|
+
const mid = lo + hi >> 1;
|
|
93
|
+
if (bounds[mid] < pos) lo = mid + 1;
|
|
94
|
+
else hi = mid;
|
|
95
|
+
}
|
|
96
|
+
return bounds[lo];
|
|
97
|
+
}
|
|
98
|
+
function snapBackward(bounds, pos) {
|
|
99
|
+
let lo = 0;
|
|
100
|
+
let hi = bounds.length - 1;
|
|
101
|
+
let result = bounds[0];
|
|
102
|
+
while(lo < hi){
|
|
103
|
+
const mid = lo + hi + 1 >> 1;
|
|
104
|
+
if (bounds[mid] <= pos) {
|
|
105
|
+
lo = mid;
|
|
106
|
+
result = bounds[mid];
|
|
107
|
+
} else hi = mid - 1;
|
|
108
|
+
}
|
|
109
|
+
return result;
|
|
110
|
+
}
|
|
111
|
+
function firstBoundAfter(bounds, pos) {
|
|
112
|
+
let lo = 0;
|
|
113
|
+
let hi = bounds.length;
|
|
114
|
+
while(lo < hi){
|
|
115
|
+
const mid = lo + hi >> 1;
|
|
116
|
+
if (bounds[mid] <= pos) lo = mid + 1;
|
|
117
|
+
else hi = mid;
|
|
118
|
+
}
|
|
119
|
+
return bounds[lo];
|
|
120
|
+
}
|
|
121
|
+
// A raw, unmarked upper bound: [start, end) inside [anchor, anchor+charLimit), edges on
|
|
122
|
+
// boundaries. Markup (guillemets, ellipses) added on top of this can push the rendered result
|
|
123
|
+
// past charLimit, which is what fitEnd() below corrects for; this is only the starting guess.
|
|
124
|
+
// A single segment spanning the whole target window (a word longer than the char limit) is the
|
|
125
|
+
// one case where holding the boundary rule means the budget is not a hard ceiling.
|
|
126
|
+
function cutWindow(text, bounds, anchor, charLimit) {
|
|
127
|
+
var _firstBoundAfter;
|
|
128
|
+
const rawEnd = Math.min(text.length, anchor + charLimit);
|
|
129
|
+
const start = snapForward(bounds, anchor);
|
|
130
|
+
let end = snapBackward(bounds, rawEnd);
|
|
131
|
+
if (end <= start) end = (_firstBoundAfter = firstBoundAfter(bounds, start)) !== null && _firstBoundAfter !== void 0 ? _firstBoundAfter : text.length;
|
|
132
|
+
return {
|
|
133
|
+
start,
|
|
134
|
+
end: Math.min(end, text.length)
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
// The largest boundary <= upperBound, > floor, whose rendered [*, end) fits charLimit. Markup
|
|
138
|
+
// only ever adds characters as end grows, so rendered length is non-decreasing in end and
|
|
139
|
+
// shrinking from upperBound is guaranteed to find the true fit rather than guess at it.
|
|
140
|
+
// `floor` is the one end this never shrinks past (the anchor's own marked word, or the first
|
|
141
|
+
// word in the unmarked fallback): DESIGN.md's budget is a ceiling except here, where returning
|
|
142
|
+
// nothing is worse than a single word that overruns it.
|
|
143
|
+
function fitEnd(render, bounds, floor, upperBound, charLimit) {
|
|
144
|
+
let lo = 0;
|
|
145
|
+
let hi = bounds.length - 1;
|
|
146
|
+
while(lo < hi){
|
|
147
|
+
const mid = lo + hi + 1 >> 1;
|
|
148
|
+
if (bounds[mid] <= upperBound) lo = mid;
|
|
149
|
+
else hi = mid - 1;
|
|
150
|
+
}
|
|
151
|
+
for(; lo >= 0 && bounds[lo] > floor; lo--){
|
|
152
|
+
if (render(bounds[lo]).length <= charLimit) return bounds[lo];
|
|
153
|
+
}
|
|
154
|
+
return floor;
|
|
155
|
+
}
|
|
156
|
+
// occ is start-sorted: a window's first candidate occurrence is a binary search, not a walk.
|
|
157
|
+
function firstOccAtOrAfter(occ, pos) {
|
|
158
|
+
let lo = 0;
|
|
159
|
+
let hi = occ.length;
|
|
160
|
+
while(lo < hi){
|
|
161
|
+
const mid = lo + hi >> 1;
|
|
162
|
+
if (occ[mid].start < pos) lo = mid + 1;
|
|
163
|
+
else hi = mid;
|
|
164
|
+
}
|
|
165
|
+
return lo;
|
|
166
|
+
}
|
|
167
|
+
function renderWindow(text, occ, win) {
|
|
168
|
+
let out = '';
|
|
169
|
+
let cursor = win.start;
|
|
170
|
+
for(let i = firstOccAtOrAfter(occ, win.start); i < occ.length && occ[i].start < win.end; i++){
|
|
171
|
+
const o = occ[i];
|
|
172
|
+
if (o.end > win.end) continue;
|
|
173
|
+
// Occurrences of duplicate or substring-overlapping terms ("test tests") can overlap;
|
|
174
|
+
// emitting each would duplicate document text. Keep the first, absorb the rest.
|
|
175
|
+
if (o.start < cursor) continue;
|
|
176
|
+
out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;
|
|
177
|
+
cursor = o.end;
|
|
178
|
+
}
|
|
179
|
+
out += text.slice(cursor, win.end);
|
|
180
|
+
const prefix = win.start > 0 ? '…' : '';
|
|
181
|
+
const suffix = win.end < text.length ? '…' : '';
|
|
182
|
+
return `${prefix}${out.replace(/\s+/g, ' ')}${suffix}`;
|
|
183
|
+
}
|
|
184
|
+
// Scores every occurrence-anchored candidate window: most distinct terms wins, ties broken by
|
|
185
|
+
// most total hits -- coverage now that occurrences are real words, not the false-fragment counts
|
|
186
|
+
// a substring scan produced. Selection itself is unevaluated (DESIGN.md "Selection, unsettled").
|
|
187
|
+
// The window's `end` is fitted to the *rendered* text, markup included, not the raw span: a
|
|
188
|
+
// window dense with marks (every guillemet pair adds two characters) shrinks to fit the same
|
|
189
|
+
// budget a sparse one fills almost exactly.
|
|
190
|
+
function scoreWindows(occ, bounds, text, charLimit) {
|
|
191
|
+
const candidates = [];
|
|
44
192
|
for(let i = 0; i < occ.length; i++){
|
|
45
|
-
const winEnd = occ[i].start +
|
|
193
|
+
const winEnd = occ[i].start + charLimit;
|
|
46
194
|
const seen = new Set();
|
|
47
195
|
let count = 0;
|
|
48
196
|
for(let j = i; j < occ.length && occ[j].start < winEnd; j++){
|
|
49
197
|
seen.add(occ[j].term);
|
|
50
198
|
count++;
|
|
51
199
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
200
|
+
const start = snapForward(bounds, occ[i].start);
|
|
201
|
+
const floor = Math.max(start, occ[i].end);
|
|
202
|
+
const upperBound = Math.max(floor, cutWindow(text, bounds, occ[i].start, charLimit).end);
|
|
203
|
+
const end = fitEnd((e)=>renderWindow(text, occ, {
|
|
204
|
+
start,
|
|
205
|
+
end: e
|
|
206
|
+
}), bounds, floor, upperBound, charLimit);
|
|
207
|
+
candidates.push({
|
|
208
|
+
start,
|
|
209
|
+
end,
|
|
210
|
+
distinct: seen.size,
|
|
211
|
+
count
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
return candidates;
|
|
215
|
+
}
|
|
216
|
+
function overlaps(a, b) {
|
|
217
|
+
return a.start < b.end && b.start < a.end;
|
|
218
|
+
}
|
|
219
|
+
// Same scoring function as a single snippet, kept rather than discarded (DESIGN.md "Several
|
|
220
|
+
// snippets per note"): rank by score, take the top non-overlapping windows, then reorder to the
|
|
221
|
+
// note's own order, since several passages from one note read as an abridged note.
|
|
222
|
+
function selectWindows(candidates, countLimit) {
|
|
223
|
+
const ranked = candidates.slice().sort((a, b)=>b.distinct - a.distinct || b.count - a.count);
|
|
224
|
+
const chosen = [];
|
|
225
|
+
for (const c of ranked){
|
|
226
|
+
if (chosen.length >= countLimit) break;
|
|
227
|
+
if (chosen.some((s)=>overlaps(s, c))) continue;
|
|
228
|
+
chosen.push(c);
|
|
57
229
|
}
|
|
58
|
-
|
|
230
|
+
chosen.sort((a, b)=>a.start - b.start);
|
|
231
|
+
return {
|
|
232
|
+
chosen,
|
|
233
|
+
best: ranked[0]
|
|
234
|
+
};
|
|
59
235
|
}
|
|
60
|
-
//
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
const
|
|
236
|
+
// Builds bounded marked passages and returns the best window's offset for section lookup.
|
|
237
|
+
export function computeSnippets(text, terms, charLimit, countLimit) {
|
|
238
|
+
const occ = findOccurrences(text, terms);
|
|
239
|
+
const bounds = wordBoundaries(text);
|
|
64
240
|
if (occ.length === 0) {
|
|
65
|
-
|
|
66
|
-
//
|
|
67
|
-
|
|
241
|
+
var _bounds_find;
|
|
242
|
+
// A term matched only via title/summary, not this body text, finds no occurrence here;
|
|
243
|
+
// fall back to the doc's start, unmarked.
|
|
244
|
+
const render = (end)=>`${text.slice(0, end).replace(/\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`;
|
|
245
|
+
const floor = (_bounds_find = bounds.find((b)=>b > 0)) !== null && _bounds_find !== void 0 ? _bounds_find : text.length;
|
|
246
|
+
const upperBound = Math.max(floor, cutWindow(text, bounds, 0, charLimit).end);
|
|
247
|
+
const end = fitEnd(render, bounds, floor, upperBound, charLimit);
|
|
68
248
|
return {
|
|
69
|
-
|
|
249
|
+
snippets: [
|
|
250
|
+
render(end)
|
|
251
|
+
],
|
|
70
252
|
offset: 0
|
|
71
253
|
};
|
|
72
254
|
}
|
|
73
|
-
const
|
|
74
|
-
const
|
|
75
|
-
let out = '';
|
|
76
|
-
let cursor = start;
|
|
77
|
-
for (const o of occ){
|
|
78
|
-
if (o.start < start || o.end > end) continue;
|
|
79
|
-
// Occurrences of duplicate or substring-overlapping terms ("test tests") can overlap;
|
|
80
|
-
// emitting each would duplicate document text. Keep the first, absorb the rest.
|
|
81
|
-
if (o.start < cursor) continue;
|
|
82
|
-
out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;
|
|
83
|
-
cursor = o.end;
|
|
84
|
-
}
|
|
85
|
-
out += text.slice(cursor, end);
|
|
86
|
-
const prefix = start > 0 ? '…' : '';
|
|
87
|
-
const suffix = end < text.length ? '…' : '';
|
|
255
|
+
const candidates = scoreWindows(occ, bounds, text, charLimit);
|
|
256
|
+
const { chosen, best } = selectWindows(candidates, countLimit);
|
|
88
257
|
return {
|
|
89
|
-
|
|
90
|
-
offset: start
|
|
258
|
+
snippets: chosen.map((win)=>renderWindow(text, occ, win)),
|
|
259
|
+
offset: best.start
|
|
91
260
|
};
|
|
92
261
|
}
|
|
93
|
-
|
|
262
|
+
// Converts an offset in the original text to its one-based line number.
|
|
263
|
+
export function lineNumberAt(text, offset) {
|
|
94
264
|
let line = 1;
|
|
95
265
|
for(let i = 0; i < offset; i++)if (text.charCodeAt(i) === 10) line++;
|
|
96
266
|
return line;
|
|
@@ -100,6 +270,26 @@ async function lineRangeFor(store, path, line) {
|
|
|
100
270
|
const row = await stmt.get(path, line, line);
|
|
101
271
|
return row ? `L${row.start_line}-${row.end_line}` : null;
|
|
102
272
|
}
|
|
273
|
+
// Hydrates already-ranked rows with snippets and containing-section ranges without reranking.
|
|
274
|
+
export async function hydrateSearchRows(store, cfg, rows, matchedPaths, terms, opts = {}) {
|
|
275
|
+
var _opts_snippetCharLimit, _opts_snippetCountLimit;
|
|
276
|
+
if (matchedPaths.size === 0) return;
|
|
277
|
+
const bareTerms = extractBareTerms(terms);
|
|
278
|
+
const charLimit = (_opts_snippetCharLimit = opts.snippetCharLimit) !== null && _opts_snippetCharLimit !== void 0 ? _opts_snippetCharLimit : SNIPPET_CHAR_LIMIT_DEFAULT;
|
|
279
|
+
const countLimit = (_opts_snippetCountLimit = opts.snippetCountLimit) !== null && _opts_snippetCountLimit !== void 0 ? _opts_snippetCountLimit : SNIPPET_COUNT_LIMIT_DEFAULT;
|
|
280
|
+
for (const row of rows){
|
|
281
|
+
if (!matchedPaths.has(row.path)) continue;
|
|
282
|
+
let text;
|
|
283
|
+
try {
|
|
284
|
+
text = readFileSync(join(cfg.baseDir, row.path), 'utf8');
|
|
285
|
+
} catch {
|
|
286
|
+
continue; // vanished since the match; leave snippets/lines empty rather than throw
|
|
287
|
+
}
|
|
288
|
+
const { snippets, offset } = computeSnippets(text, bareTerms, charLimit, countLimit);
|
|
289
|
+
row.snippets = snippets;
|
|
290
|
+
if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path, lineNumberAt(text, offset)) : null;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
103
293
|
// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.
|
|
104
294
|
// `opts` arrives already resolved (config.ts:resolveSearch).
|
|
105
295
|
export async function search(store, cfg, terms, opts = {}) {
|
|
@@ -138,22 +328,24 @@ export async function search(store, cfg, terms, opts = {}) {
|
|
|
138
328
|
const scopeCond = scopeActive ? `AND content.path IN (SELECT "path" FROM _search_scope)` : '';
|
|
139
329
|
const candidates = new Map();
|
|
140
330
|
let matchRows = [];
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
331
|
+
const hasEmptyPhrase = [
|
|
332
|
+
...terms.matchAll(/"([^"]*)"/g)
|
|
333
|
+
].some((match)=>!/[\p{L}\p{N}_]/u.test(match[1]));
|
|
334
|
+
const hasBareSyntaxOnly = /[()]/u.test(terms.replace(/"[^"]*"/g, ' '));
|
|
335
|
+
// A query that is only whitespace or punctuation has no lexical rows; other declared signals
|
|
336
|
+
// compose normally. In particular, unspaced-script punctuation can rewrite to an empty MATCH.
|
|
337
|
+
if (signals.words !== undefined && terms.trim() !== '' && (/[\p{L}\p{N}_]/u.test(terms) || hasBareSyntaxOnly) && !hasEmptyPhrase) {
|
|
338
|
+
const syntaxError = bareTermSyntaxError(terms);
|
|
339
|
+
if (syntaxError) throw syntaxError;
|
|
144
340
|
try {
|
|
145
341
|
matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);
|
|
146
342
|
} catch (err) {
|
|
147
343
|
throw searchError(err, terms, scope);
|
|
148
344
|
}
|
|
149
345
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
]));
|
|
154
|
-
// hit === null here means the bound suppressed snippet(), not "no match" -- distinguish
|
|
155
|
-
// from via='link' rows (never in matchRows, so absent from this set) below.
|
|
156
|
-
const oversized = new Set(matchRows.filter((r)=>r.hit === null).map((r)=>r.path));
|
|
346
|
+
// Every store's lexical `snippets` is NULL; the snippet text is always computed below.
|
|
347
|
+
// Distinguish from via='link' rows, which never appear in matchRows.
|
|
348
|
+
const matchedPaths = new Set(matchRows.map((r)=>r.path));
|
|
157
349
|
if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);
|
|
158
350
|
let chunkLines = new Map();
|
|
159
351
|
let chunkSimilarity = new Map();
|
|
@@ -163,17 +355,16 @@ export async function search(store, cfg, terms, opts = {}) {
|
|
|
163
355
|
await store.exec('DROP TABLE IF EXISTS _search');
|
|
164
356
|
// DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the
|
|
165
357
|
// rounded score/similarity are printed at full precision in rows.
|
|
166
|
-
await store.exec('CREATE TEMP TABLE _search ("path" TEXT PRIMARY KEY, score DOUBLE, via TEXT,
|
|
358
|
+
await store.exec('CREATE TEMP TABLE _search ("path" TEXT PRIMARY KEY, score DOUBLE, via TEXT, lines TEXT, similarity DOUBLE)');
|
|
167
359
|
if (candidates.size > 0) {
|
|
168
|
-
await store.runBatch('INSERT INTO _search ("path", score, via,
|
|
360
|
+
await store.runBatch('INSERT INTO _search ("path", score, via, lines, similarity) VALUES (?, ?, ?, ?, ?)', [
|
|
169
361
|
...candidates
|
|
170
362
|
].map(([path, c])=>{
|
|
171
|
-
var
|
|
363
|
+
var _chunkLines_get, _chunkSimilarity_get;
|
|
172
364
|
return [
|
|
173
365
|
path,
|
|
174
366
|
c.score,
|
|
175
367
|
c.via,
|
|
176
|
-
(_hits_get = hits.get(path)) !== null && _hits_get !== void 0 ? _hits_get : null,
|
|
177
368
|
(_chunkLines_get = chunkLines.get(path)) !== null && _chunkLines_get !== void 0 ? _chunkLines_get : null,
|
|
178
369
|
(_chunkSimilarity_get = chunkSimilarity.get(path)) !== null && _chunkSimilarity_get !== void 0 ? _chunkSimilarity_get : null
|
|
179
370
|
];
|
|
@@ -182,27 +373,16 @@ export async function search(store, cfg, terms, opts = {}) {
|
|
|
182
373
|
// Reapplies --where even though _search is already scope+where filtered, since the join to
|
|
183
374
|
// frontmatter is already needed for the path column.
|
|
184
375
|
const where = scope ? `WHERE (${scope})` : '';
|
|
185
|
-
// lines: semantic rows carry their chunk's range,
|
|
186
|
-
//
|
|
376
|
+
// lines: semantic rows carry their chunk's range, lexical rows gain one below,
|
|
377
|
+
// everything else stays null; similarity stays semantic-only.
|
|
187
378
|
const similarityCol = semanticEnabled ? ', _search.similarity' : '';
|
|
188
|
-
|
|
379
|
+
// NULL AS snippets holds this column's position (between summary and via) for every row;
|
|
380
|
+
// overwritten below, always to an array, never left null (DESIGN.md).
|
|
381
|
+
const selectStmt = await store.prepare(`SELECT f."path" AS path, content.title, content.summary, NULL AS snippets, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}
|
|
189
382
|
FROM _search JOIN frontmatter f ON f."path" = _search."path" JOIN content ON content.path = _search."path"
|
|
190
383
|
${where} ORDER BY _search.score DESC LIMIT ?`);
|
|
191
384
|
const rows = await selectStmt.all(k);
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
for (const row of rows){
|
|
195
|
-
if (row.hit !== null || !oversized.has(row.path)) continue;
|
|
196
|
-
let text;
|
|
197
|
-
try {
|
|
198
|
-
text = readFileSync(join(cfg.baseDir, row.path), 'utf8');
|
|
199
|
-
} catch {
|
|
200
|
-
continue; // vanished since the match; leave hit/lines null rather than throw
|
|
201
|
-
}
|
|
202
|
-
const { excerpt, offset } = computeExcerpt(text, bareTerms);
|
|
203
|
-
row.hit = excerpt;
|
|
204
|
-
if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path, lineNumberAt(text, offset)) : null;
|
|
205
|
-
}
|
|
206
|
-
}
|
|
385
|
+
for (const row of rows)row.snippets = [];
|
|
386
|
+
await hydrateSearchRows(store, cfg, rows, matchedPaths, terms, opts);
|
|
207
387
|
return rows;
|
|
208
388
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { searchError } from '../output/search-error.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-0.13.2-hub-release-battery.md). Past this bound, rows get the JS excerpt.\nconst EXCERPT_WINDOW = 160;\n\n// Bare terms from an FTS5 query string: strips operators/quoting so the oversized-doc excerpt\n// scan matches the same words the query matched on, not FTS5 syntax.\nfunction extractBareTerms(query: string): string[] {\n const cleaned = query\n .replace(/\"/g, ' ')\n .replace(/[()*]/g, ' ')\n .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/gi, ' ');\n return cleaned\n .split(/\\s+/)\n .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n .map((tok) => tok.toLowerCase().trim())\n .filter((tok) => tok.length > 0);\n}\n\nfunction findOccurrences(haystackLower: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n const occ: Array<{ start: number; end: number; term: string }> = [];\n for (const term of terms) {\n let idx = 0;\n for (;;) {\n const found = haystackLower.indexOf(term, idx);\n if (found === -1) break;\n occ.push({ start: found, end: found + term.length, term });\n idx = found + term.length;\n }\n }\n // Longest span first at equal start, so \"tests\" beats its substring \"test\" and the\n // whole word gets highlighted; the emit loop then absorbs the shorter overlap.\n return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Slides a window over the sorted occurrences, same semantics as FTS5's own best-window\n// pick: most distinct terms, ties broken by most total hits.\nfunction bestWindowStart(occ: Array<{ start: number; end: number; term: string }>, windowSize: number): number {\n let best = occ[0].start;\n let bestDistinct = 0;\n let bestCount = 0;\n for (let i = 0; i < occ.length; i++) {\n const winEnd = occ[i].start + windowSize;\n const seen = new Set<string>();\n let count = 0;\n for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n seen.add(occ[j].term);\n count++;\n }\n if (seen.size > bestDistinct || (seen.size === bestDistinct && count > bestCount)) {\n best = occ[i].start;\n bestDistinct = seen.size;\n bestCount = count;\n }\n }\n return best;\n}\n\n// snippet()-shaped excerpt: matched terms wrapped in «», … at cut edges. Linear in doc\n// length, only run for rows actually returned (<= k), unlike snippet()'s per-row cost.\nfunction computeExcerpt(text: string, terms: string[]): { excerpt: string; offset: number } {\n const occ = findOccurrences(text.toLowerCase(), terms);\n if (occ.length === 0) {\n // Raw substring scan, not porter-stemmed: a doc matched only through a stemmed variant\n // finds no occurrence here, so fall back to the doc's start, unmarked.\n const end = Math.min(text.length, EXCERPT_WINDOW);\n return { excerpt: `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`, offset: 0 };\n }\n const start = bestWindowStart(occ, EXCERPT_WINDOW);\n const end = Math.min(text.length, start + EXCERPT_WINDOW);\n let out = '';\n let cursor = start;\n for (const o of occ) {\n if (o.start < start || o.end > end) continue;\n // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n // emitting each would duplicate document text. Keep the first, absorb the rest.\n if (o.start < cursor) continue;\n out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n cursor = o.end;\n }\n out += text.slice(cursor, end);\n const prefix = start > 0 ? '…' : '';\n const suffix = end < text.length ? '…' : '';\n return { excerpt: `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`, offset: start };\n}\n\nfunction lineNumberAt(text: string, offset: number): number {\n let line = 1;\n for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n return line;\n}\n\nasync function lineRangeFor(store: Store, path: string, line: number): Promise<string | null> {\n const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');\n const row = (await stmt.get(path, line, line)) as { start_line: number; end_line: number } | undefined;\n return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n k?: number;\n where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(store: Store, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n const effective = resolveSearch(cfg, opts);\n const { k, signals } = effective;\n\n const allPaths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n const scopePaths = await rawScope(store, cfg, opts, allPaths);\n const scopeActive = scopePaths.size < allPaths.length;\n // The set every candidate pool must be filtered to before truncation: scope narrowed by\n // --where, the same composition scopedPaths() gives the other commands.\n let allowedPaths: Set<string>;\n try {\n allowedPaths = await narrowByWhere(store, scopePaths, effective.where);\n } catch (err) {\n throw searchError(err as Error, terms, effective.where);\n }\n const fetch = Math.max(k * 3, 30);\n\n // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n // local path with missing files errors here, since nothing will ever fetch it for itself.\n const wantsVectors = signals.vectors !== undefined;\n if (wantsVectors) {\n const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n if (localModelMissing(e)) {\n throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n }\n }\n const semanticEnabled = wantsVectors && (await scopeHasEmbeddings(store, cfg, allowedPaths));\n\n // --where applies inside the candidate query (a post-filter would drop matches ranked past\n // the pool) and again on the final select, for link-derived rows.\n const scope = effective.where;\n const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n const whereCond = scope ? `AND (${scope})` : '';\n // Filtered before LIMIT, not after, or scoped notes ranking below the global top-`fetch`\n // never reach the filter; joined against a temp table since real scopes exceed SQLITE_MAX_VARIABLE_NUMBER.\n if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);\n const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n\n const candidates = new Map<string, { score: number; via: string }>();\n let matchRows: LexicalHit[] = [];\n // A query that is only whitespace has no searchable words: zero lexical rows, and the other\n // signals compose normally.\n if (signals.words !== undefined && terms.trim() !== '') {\n try {\n matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);\n } catch (err) {\n throw searchError(err as Error, terms, scope);\n }\n }\n const hits = new Map(matchRows.map((r) => [r.path, r.hit]));\n // hit === null here means the bound suppressed snippet(), not \"no match\" -- distinguish\n // from via='link' rows (never in matchRows, so absent from this set) below.\n const oversized = new Set(matchRows.filter((r) => r.hit === null).map((r) => r.path));\n\n if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n let chunkLines = new Map<string, string>();\n let chunkSimilarity = new Map<string, number>();\n if (semanticEnabled) {\n ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n }\n\n await store.exec('DROP TABLE IF EXISTS _search');\n // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the\n // rounded score/similarity are printed at full precision in rows.\n await store.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score DOUBLE, via TEXT, hit TEXT, lines TEXT, similarity DOUBLE)');\n if (candidates.size > 0) {\n await store.runBatch(\n 'INSERT INTO _search (\"path\", score, via, hit, lines, similarity) VALUES (?, ?, ?, ?, ?, ?)',\n [...candidates].map(([path, c]) => [path, c.score, c.via, hits.get(path) ?? null, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null])\n );\n }\n\n // Reapplies --where even though _search is already scope+where filtered, since the join to\n // frontmatter is already needed for the path column.\n const where = scope ? `WHERE (${scope})` : '';\n // lines: semantic rows carry their chunk's range, oversized-doc lexical rows gain one\n // below, everything else stays null; similarity stays semantic-only.\n const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n const selectStmt = await store.prepare(\n `SELECT f.\"path\" AS path, content.title, content.summary, _search.hit, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n ${where} ORDER BY _search.score DESC LIMIT ?`\n );\n const rows = (await selectStmt.all(k)) as Row[];\n\n if (oversized.size > 0) {\n const bareTerms = extractBareTerms(terms);\n for (const row of rows) {\n if (row.hit !== null || !oversized.has(row.path as string)) continue;\n let text: string;\n try {\n text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n } catch {\n continue; // vanished since the match; leave hit/lines null rather than throw\n }\n const { excerpt, offset } = computeExcerpt(text, bareTerms);\n row.hit = excerpt;\n if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path as string, lineNumberAt(text, offset)) : null;\n }\n }\n\n return rows;\n}\n"],"names":["readFileSync","join","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","searchError","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","EXCERPT_WINDOW","extractBareTerms","query","cleaned","replace","split","map","tok","toLowerCase","trim","filter","length","findOccurrences","haystackLower","terms","occ","term","idx","found","indexOf","push","start","end","sort","a","b","bestWindowStart","windowSize","best","bestDistinct","bestCount","i","winEnd","seen","Set","count","j","add","size","computeExcerpt","text","Math","min","excerpt","slice","offset","out","cursor","o","prefix","suffix","lineNumberAt","line","charCodeAt","lineRangeFor","store","path","stmt","prepare","row","get","start_line","end_line","search","cfg","opts","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","max","wantsVectors","vectors","undefined","e","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","candidates","Map","matchRows","words","hits","hit","oversized","links","chunkLines","chunkSimilarity","exec","runBatch","c","score","via","similarityCol","selectStmt","rows","bareTerms","has","baseDir","lines"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AAChF,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,WAAW,QAAQ,4BAA4B;AAExD,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,iFAAiF;AACjF,0GAA0G;AAC1G,MAAMC,iBAAiB;AAEvB,8FAA8F;AAC9F,qEAAqE;AACrE,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,mCAAmC;IAC9C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFE,GAAG,CAAC,CAACC,MAAQA,IAAIC,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACH,MAAQA,IAAII,MAAM,GAAG;AAClC;AAEA,SAASC,gBAAgBC,aAAqB,EAAEC,KAAe;IAC7D,MAAMC,MAA2D,EAAE;IACnE,KAAK,MAAMC,QAAQF,MAAO;QACxB,IAAIG,MAAM;QACV,OAAS;YACP,MAAMC,QAAQL,cAAcM,OAAO,CAACH,MAAMC;YAC1C,IAAIC,UAAU,CAAC,GAAG;YAClBH,IAAIK,IAAI,CAAC;gBAAEC,OAAOH;gBAAOI,KAAKJ,QAAQF,KAAKL,MAAM;gBAAEK;YAAK;YACxDC,MAAMC,QAAQF,KAAKL,MAAM;QAC3B;IACF;IACA,mFAAmF;IACnF,+EAA+E;IAC/E,OAAOI,IAAIQ,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEH,KAAK,GAAGI,EAAEJ,KAAK,IAAII,EAAEH,GAAG,GAAGE,EAAEF,GAAG;AAC9D;AAEA,wFAAwF;AACxF,6DAA6D;AAC7D,SAASI,gBAAgBX,GAAwD,EAAEY,UAAkB;IACnG,IAAIC,OAAOb,GAAG,CAAC,EAAE,CAACM,KAAK;IACvB,IAAIQ,eAAe;IACnB,IAAIC,YAAY;IAChB,IAAK,IAAIC,IAAI,GAAGA,IAAIhB,IAAIJ,MAAM,EAAEoB,IAAK;QACnC,MAAMC,SAASjB,GAAG,CAACgB,EAAE,CAACV,KAAK,GAAGM;QAC9B,MAAMM,OAAO,IAAIC;QACjB,IAAIC,QAAQ;QACZ,IAAK,IAAIC,IAAIL,GAAGK,IAAIrB,IAAIJ,MAAM,IAAII,GAAG,CAACqB,EAAE,CAACf,KAAK,GAAGW,QAAQI,IAAK;YAC5DH,KAAKI,GAAG,CAACtB,GAAG,CAACqB,EAAE,CAACpB,IAAI;YACpBmB;QACF;QACA,IAAIF,KAAKK,IAAI,GAAGT,gBAAiBI,KAAKK,IAAI,KAAKT,gBAAgBM,QAAQL,WAAY;YACjFF,OAAOb,GAAG,CAACgB,EAAE,CAACV,KAAK;YACnBQ,eAAeI,KAAKK,IAAI;YACxBR,YAAYK;QACd;IACF;IACA,OAAOP;AACT;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,SAASW,eAAeC,IAAY,EAAE1B,KAAe;IACnD,MAAMC,MAAMH,gBAAgB4B,KAAKhC,WAAW,IAAIM;IAChD,IAAIC,IAAIJ,MAAM,KAAK,GAAG;QACpB,uFAAuF;QACvF,uEAAuE;QACvE,MAAMW,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEX;QAClC,OAAO;YAAE2C,SAAS,GAAGH,KAAKI,KAAK,CAAC,GAAGtB,KAAKlB,OAAO,CAAC,QAAQ,KAAKK,IAAI,KAAKa,MAAMkB,KAAK7B,MAAM,GAAG,MAAM,IAAI;YAAEkC,QAAQ;QAAE;IAClH;IACA,MAAMxB,QAAQK,gBAAgBX,KAAKf;IACnC,MAAMsB,MAAMmB,KAAKC,GAAG,CAACF,KAAK7B,MAAM,EAAEU,QAAQrB;IAC1C,IAAI8C,MAAM;IACV,IAAIC,SAAS1B;IACb,KAAK,MAAM2B,KAAKjC,IAAK;QACnB,IAAIiC,EAAE3B,KAAK,GAAGA,SAAS2B,EAAE1B,GAAG,GAAGA,KAAK;QACpC,sFAAsF;QACtF,gFAAgF;QAChF,IAAI0B,EAAE3B,KAAK,GAAG0B,QAAQ;QACtBD,OAAO,GAAGN,KAAKI,KAAK,CAACG,QAAQC,EAAE3B,KAAK,EAAE,CAAC,EAAEmB,KAAKI,KAAK,CAACI,EAAE3B,KAAK,EAAE2B,EAAE1B,GAAG,EAAE,CAAC,CAAC;QACtEyB,SAASC,EAAE1B,GAAG;IAChB;IACAwB,OAAON,KAAKI,KAAK,CAACG,QAAQzB;IAC1B,MAAM2B,SAAS5B,QAAQ,IAAI,MAAM;IACjC,MAAM6B,SAAS5B,MAAMkB,KAAK7B,MAAM,GAAG,MAAM;IACzC,OAAO;QAAEgC,SAAS,GAAGM,SAASH,IAAI1C,OAAO,CAAC,QAAQ,OAAO8C,QAAQ;QAAEL,QAAQxB;IAAM;AACnF;AAEA,SAAS8B,aAAaX,IAAY,EAAEK,MAAc;IAChD,IAAIO,OAAO;IACX,IAAK,IAAIrB,IAAI,GAAGA,IAAIc,QAAQd,IAAK,IAAIS,KAAKa,UAAU,CAACtB,OAAO,IAAIqB;IAChE,OAAOA;AACT;AAEA,eAAeE,aAAaC,KAAY,EAAEC,IAAY,EAAEJ,IAAY;IAClE,MAAMK,OAAO,MAAMF,MAAMG,OAAO,CAAC;IACjC,MAAMC,MAAO,MAAMF,KAAKG,GAAG,CAACJ,MAAMJ,MAAMA;IACxC,OAAOO,MAAM,CAAC,CAAC,EAAEA,IAAIE,UAAU,CAAC,CAAC,EAAEF,IAAIG,QAAQ,EAAE,GAAG;AACtD;AAWA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAeC,OAAOR,KAAY,EAAES,GAAmB,EAAElD,KAAa,EAAEmD,OAAsB,CAAC,CAAC;IACrG,MAAMC,YAAY9E,cAAc4E,KAAKC;IACrC,MAAM,EAAEE,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMd,MAAMG,OAAO,CAAC,iCAAgC,EAAGY,GAAG,EAAC,EAA+BhE,GAAG,CAAC,CAACiE,IAAMA,EAAEf,IAAI;IACrI,MAAMgB,aAAa,MAAM7E,SAAS4D,OAAOS,KAAKC,MAAMI;IACpD,MAAMI,cAAcD,WAAWlC,IAAI,GAAG+B,SAAS1D,MAAM;IACrD,wFAAwF;IACxF,wEAAwE;IACxE,IAAI+D;IACJ,IAAI;QACFA,eAAe,MAAMhF,cAAc6D,OAAOiB,YAAYN,UAAUS,KAAK;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAMpF,YAAYoF,KAAc9D,OAAOoD,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQpC,KAAKqC,GAAG,CAACX,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMY,eAAeX,QAAQY,OAAO,KAAKC;IACzC,IAAIF,cAAc;QAChB,MAAMG,IAAIhG,YAAY8E,MAAM,oEAAoE;QAChG,IAAI3E,kBAAkB6F,IAAI;YACxB,MAAM,IAAI3F,WAAW,uBAAuB,CAAC,QAAQ,EAAE2E,UAAUiB,UAAU,CAAC,mDAAmD,EAAED,EAAEE,KAAK,CAAC,aAAa,EAAE9F,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAM+F,kBAAkBN,gBAAiB,MAAMnF,mBAAmB2D,OAAOS,KAAKU;IAE9E,2FAA2F;IAC3F,kEAAkE;IAClE,MAAMY,QAAQpB,UAAUS,KAAK;IAC7B,MAAMY,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,2GAA2G;IAC3G,IAAIb,aAAa,MAAMhF,iBAAiB8D,OAAO,iBAAiBiB;IAChE,MAAMiB,YAAYhB,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAE3F,MAAMiB,aAAa,IAAIC;IACvB,IAAIC,YAA0B,EAAE;IAChC,4FAA4F;IAC5F,4BAA4B;IAC5B,IAAIxB,QAAQyB,KAAK,KAAKZ,aAAanE,MAAML,IAAI,OAAO,IAAI;QACtD,IAAI;YACFmF,YAAY,MAAM7F,gBAAgBwD,OAAOmC,YAAY5E,OAAOyE,WAAWC,WAAWC,WAAWZ,OAAOT,QAAQyB,KAAK;QACnH,EAAE,OAAOjB,KAAK;YACZ,MAAMpF,YAAYoF,KAAc9D,OAAOwE;QACzC;IACF;IACA,MAAMQ,OAAO,IAAIH,IAAIC,UAAUtF,GAAG,CAAC,CAACiE,IAAM;YAACA,EAAEf,IAAI;YAAEe,EAAEwB,GAAG;SAAC;IACzD,wFAAwF;IACxF,4EAA4E;IAC5E,MAAMC,YAAY,IAAI9D,IAAI0D,UAAUlF,MAAM,CAAC,CAAC6D,IAAMA,EAAEwB,GAAG,KAAK,MAAMzF,GAAG,CAAC,CAACiE,IAAMA,EAAEf,IAAI;IAEnF,IAAIY,QAAQ6B,KAAK,KAAKhB,WAAW,MAAMpF,gBAAgB0D,OAAOmC,YAAYE,WAAWvB,UAAUK,cAAcG,OAAOT,QAAQ6B,KAAK;IAEjI,IAAIC,aAAa,IAAIP;IACrB,IAAIQ,kBAAkB,IAAIR;IAC1B,IAAIN,iBAAiB;QAClB,CAAA,EAAEa,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMrG,kBAAkByD,OAAOS,KAAK0B,YAAY5E,OAAO+D,OAAOH,cAAcN,QAAQY,OAAO,CAAU;IAC1I;IAEA,MAAMzB,MAAM6C,IAAI,CAAC;IACjB,8FAA8F;IAC9F,kEAAkE;IAClE,MAAM7C,MAAM6C,IAAI,CAAC;IACjB,IAAIV,WAAWpD,IAAI,GAAG,GAAG;QACvB,MAAMiB,MAAM8C,QAAQ,CAClB,8FACA;eAAIX;SAAW,CAACpF,GAAG,CAAC,CAAC,CAACkD,MAAM8C,EAAE;gBAA4BR,WAAwBI,iBAA8BC;mBAA7E;gBAAC3C;gBAAM8C,EAAEC,KAAK;gBAAED,EAAEE,GAAG;iBAAEV,YAAAA,KAAKlC,GAAG,CAACJ,mBAATsC,uBAAAA,YAAkB;iBAAMI,kBAAAA,WAAWtC,GAAG,CAACJ,mBAAf0C,6BAAAA,kBAAwB;iBAAMC,uBAAAA,gBAAgBvC,GAAG,CAACJ,mBAApB2C,kCAAAA,uBAA6B;aAAK;;IAEtJ;IAEA,2FAA2F;IAC3F,qDAAqD;IACrD,MAAMxB,QAAQW,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,sFAAsF;IACtF,qEAAqE;IACrE,MAAMmB,gBAAgBpB,kBAAkB,yBAAyB;IACjE,MAAMqB,aAAa,MAAMnD,MAAMG,OAAO,CACpC,CAAC,kIAAkI,EAAE+C,cAAc;;OAEhJ,EAAE9B,MAAM,oCAAoC,CAAC;IAElD,MAAMgC,OAAQ,MAAMD,WAAWpC,GAAG,CAACH;IAEnC,IAAI6B,UAAU1D,IAAI,GAAG,GAAG;QACtB,MAAMsE,YAAY3G,iBAAiBa;QACnC,KAAK,MAAM6C,OAAOgD,KAAM;YACtB,IAAIhD,IAAIoC,GAAG,KAAK,QAAQ,CAACC,UAAUa,GAAG,CAAClD,IAAIH,IAAI,GAAa;YAC5D,IAAIhB;YACJ,IAAI;gBACFA,OAAOxD,aAAaC,KAAK+E,IAAI8C,OAAO,EAAEnD,IAAIH,IAAI,GAAa;YAC7D,EAAE,OAAM;gBACN,UAAU,mEAAmE;YAC/E;YACA,MAAM,EAAEb,OAAO,EAAEE,MAAM,EAAE,GAAGN,eAAeC,MAAMoE;YACjDjD,IAAIoC,GAAG,GAAGpD;YACV,IAAIgB,IAAIoD,KAAK,IAAI,MAAMpD,IAAIoD,KAAK,GAAG5H,eAAe6E,KAAK,cAAc,MAAMV,aAAaC,OAAOI,IAAIH,IAAI,EAAYL,aAAaX,MAAMK,WAAW;QACnJ;IACF;IAEA,OAAO8D;AACT"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/search.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { stemmer } from 'stemmer';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { embedConfig, featureEnabled, resolveSearch } from '../config/index.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Row } from '../output/output.ts';\nimport { bareTermSyntaxError, searchError } from '../output/search-error.ts';\nimport type { LexicalHit, Store } from '../store/types.ts';\nimport { foldForSearch, hasUnspacedRun, searchTokens, unspacedRuns } from '../text/segment.ts';\nimport { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.ts';\nimport { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.ts';\n\n// Default budget behind --snippet-char-limit/--snippet-count-limit: the limit is whatever the\n// caller passes, these only say what happens when they pass nothing (DESIGN.md \"Search snippets\").\nexport const SNIPPET_CHAR_LIMIT_DEFAULT = 80;\nexport const SNIPPET_COUNT_LIMIT_DEFAULT = 1;\n\n// Bare terms from an FTS5 query string: strips operators/quoting, not real terms, so the\n// snippet scan matches what the query matched -- FTS5 operators are uppercase-only, so this is too.\nfunction extractBareTerms(query: string): string[] {\n const cleaned = query\n .replace(/\"/g, ' ')\n .replace(/[()*]/g, ' ')\n .replace(/\\b(AND|OR|NOT|NEAR)\\b(\\/\\d+)?/g, ' ');\n return cleaned\n .split(/\\s+/)\n .map((tok) => tok.replace(/^[A-Za-z_]\\w*:/, '')) // column filter, e.g. title:term\n .flatMap((tok) => searchTokens(tok).map(({ text }) => text))\n .map((tok) => tok.toLowerCase().trim())\n .filter((tok) => tok.length > 0);\n}\n\n// House convention (src/chunk/group.ts): Intl.Segmenter word boundaries, not a bare \\b, which\n// is ASCII-only. Built on first use: construction is a few ms, paid only when a term matches.\nlet wordSegmenter: Intl.Segmenter | undefined;\n\nfunction findOccurrences(text: string, terms: string[]): Array<{ start: number; end: number; term: string }> {\n const occ: Array<{ start: number; end: number; term: string }> = [];\n const textLower = text.toLowerCase();\n\n // Unspaced scripts (UNSPACED_SCRIPTS) mark no word boundaries: the sense-setup skill promises\n // substring semantics there, so a matching term is found wherever it occurs in the run.\n const unspacedTerms = terms.filter(hasUnspacedRun);\n if (unspacedTerms.length > 0) {\n for (const run of unspacedRuns(text)) {\n const runLower = textLower.slice(run.start, run.end);\n for (const term of unspacedTerms) {\n let idx = 0;\n for (;;) {\n const found = runLower.indexOf(term, idx);\n if (found === -1) break;\n occ.push({ start: run.start + found, end: run.start + found + term.length, term });\n idx = found + term.length;\n }\n }\n }\n }\n\n // Spaced scripts: a word is marked when its porter stem equals a query term's stem, the same\n // algorithm both indexes tokenize with (sqlite's TOKENIZE = 'porter unicode61'; duckdb's fts default).\n const spacedTerms = terms.filter((t) => !hasUnspacedRun(t));\n if (spacedTerms.length > 0) {\n const termByStem = new Map(spacedTerms.map((t) => [stemmer(foldForSearch(t)), t]));\n const stemInitials = new Set(Array.from(termByStem.keys(), (term) => term[0]));\n for (const match of searchTokens(text)) {\n const segment = match.text;\n if (match.unspaced) continue;\n const folded = foldForSearch(segment);\n // Porter stemming changes suffixes only, so a different initial cannot match a query stem.\n if (!stemInitials.has(folded[0])) continue;\n const term = termByStem.get(stemmer(folded));\n if (term !== undefined) occ.push({ start: match.start, end: match.end, term });\n }\n }\n\n return occ.sort((a, b) => a.start - b.start || b.end - a.end);\n}\n\n// Segment boundaries of the whole text, word granularity: the only cut points a snippet edge\n// may land on. One rule for every script (DESIGN.md \"Cutting\") -- a regex \\b is ASCII-only and\n// would mis-cut Japanese, Chinese and Thai, which Intl.Segmenter divides by dictionary instead.\nfunction wordBoundaries(text: string): number[] {\n wordSegmenter ??= new Intl.Segmenter('en', { granularity: 'word' });\n const bounds = [0];\n for (const { index, segment } of wordSegmenter.segment(text)) bounds.push(index + segment.length);\n return bounds;\n}\n\n// bounds is sorted: snaps are binary searches, since each runs once per candidate window.\nfunction snapForward(bounds: number[], pos: number): number {\n let lo = 0;\n let hi = bounds.length - 1;\n while (lo < hi) {\n const mid = (lo + hi) >> 1;\n if (bounds[mid] < pos) lo = mid + 1;\n else hi = mid;\n }\n return bounds[lo];\n}\n\nfunction snapBackward(bounds: number[], pos: number): number {\n let lo = 0;\n let hi = bounds.length - 1;\n let result = bounds[0];\n while (lo < hi) {\n const mid = (lo + hi + 1) >> 1;\n if (bounds[mid] <= pos) {\n lo = mid;\n result = bounds[mid];\n } else hi = mid - 1;\n }\n return result;\n}\n\nfunction firstBoundAfter(bounds: number[], pos: number): number | undefined {\n let lo = 0;\n let hi = bounds.length;\n while (lo < hi) {\n const mid = (lo + hi) >> 1;\n if (bounds[mid] <= pos) lo = mid + 1;\n else hi = mid;\n }\n return bounds[lo];\n}\n\n// A raw, unmarked upper bound: [start, end) inside [anchor, anchor+charLimit), edges on\n// boundaries. Markup (guillemets, ellipses) added on top of this can push the rendered result\n// past charLimit, which is what fitEnd() below corrects for; this is only the starting guess.\n// A single segment spanning the whole target window (a word longer than the char limit) is the\n// one case where holding the boundary rule means the budget is not a hard ceiling.\nfunction cutWindow(text: string, bounds: number[], anchor: number, charLimit: number): { start: number; end: number } {\n const rawEnd = Math.min(text.length, anchor + charLimit);\n const start = snapForward(bounds, anchor);\n let end = snapBackward(bounds, rawEnd);\n if (end <= start) end = firstBoundAfter(bounds, start) ?? text.length;\n return { start, end: Math.min(end, text.length) };\n}\n\n// The largest boundary <= upperBound, > floor, whose rendered [*, end) fits charLimit. Markup\n// only ever adds characters as end grows, so rendered length is non-decreasing in end and\n// shrinking from upperBound is guaranteed to find the true fit rather than guess at it.\n// `floor` is the one end this never shrinks past (the anchor's own marked word, or the first\n// word in the unmarked fallback): DESIGN.md's budget is a ceiling except here, where returning\n// nothing is worse than a single word that overruns it.\nfunction fitEnd(render: (end: number) => string, bounds: number[], floor: number, upperBound: number, charLimit: number): number {\n let lo = 0;\n let hi = bounds.length - 1;\n while (lo < hi) {\n const mid = (lo + hi + 1) >> 1;\n if (bounds[mid] <= upperBound) lo = mid;\n else hi = mid - 1;\n }\n for (; lo >= 0 && bounds[lo] > floor; lo--) {\n if (render(bounds[lo]).length <= charLimit) return bounds[lo];\n }\n return floor;\n}\n\ninterface Span {\n start: number;\n end: number;\n}\n\ninterface SnippetWindow extends Span {\n distinct: number;\n count: number;\n}\n\n// occ is start-sorted: a window's first candidate occurrence is a binary search, not a walk.\nfunction firstOccAtOrAfter(occ: Array<{ start: number; end: number; term: string }>, pos: number): number {\n let lo = 0;\n let hi = occ.length;\n while (lo < hi) {\n const mid = (lo + hi) >> 1;\n if (occ[mid].start < pos) lo = mid + 1;\n else hi = mid;\n }\n return lo;\n}\n\nfunction renderWindow(text: string, occ: Array<{ start: number; end: number; term: string }>, win: Span): string {\n let out = '';\n let cursor = win.start;\n for (let i = firstOccAtOrAfter(occ, win.start); i < occ.length && occ[i].start < win.end; i++) {\n const o = occ[i];\n if (o.end > win.end) continue;\n // Occurrences of duplicate or substring-overlapping terms (\"test tests\") can overlap;\n // emitting each would duplicate document text. Keep the first, absorb the rest.\n if (o.start < cursor) continue;\n out += `${text.slice(cursor, o.start)}«${text.slice(o.start, o.end)}»`;\n cursor = o.end;\n }\n out += text.slice(cursor, win.end);\n const prefix = win.start > 0 ? '…' : '';\n const suffix = win.end < text.length ? '…' : '';\n return `${prefix}${out.replace(/\\s+/g, ' ')}${suffix}`;\n}\n\n// Scores every occurrence-anchored candidate window: most distinct terms wins, ties broken by\n// most total hits -- coverage now that occurrences are real words, not the false-fragment counts\n// a substring scan produced. Selection itself is unevaluated (DESIGN.md \"Selection, unsettled\").\n// The window's `end` is fitted to the *rendered* text, markup included, not the raw span: a\n// window dense with marks (every guillemet pair adds two characters) shrinks to fit the same\n// budget a sparse one fills almost exactly.\nfunction scoreWindows(occ: Array<{ start: number; end: number; term: string }>, bounds: number[], text: string, charLimit: number): SnippetWindow[] {\n const candidates: SnippetWindow[] = [];\n for (let i = 0; i < occ.length; i++) {\n const winEnd = occ[i].start + charLimit;\n const seen = new Set<string>();\n let count = 0;\n for (let j = i; j < occ.length && occ[j].start < winEnd; j++) {\n seen.add(occ[j].term);\n count++;\n }\n const start = snapForward(bounds, occ[i].start);\n const floor = Math.max(start, occ[i].end);\n const upperBound = Math.max(floor, cutWindow(text, bounds, occ[i].start, charLimit).end);\n const end = fitEnd((e) => renderWindow(text, occ, { start, end: e }), bounds, floor, upperBound, charLimit);\n candidates.push({ start, end, distinct: seen.size, count });\n }\n return candidates;\n}\n\nfunction overlaps(a: Span, b: Span): boolean {\n return a.start < b.end && b.start < a.end;\n}\n\n// Same scoring function as a single snippet, kept rather than discarded (DESIGN.md \"Several\n// snippets per note\"): rank by score, take the top non-overlapping windows, then reorder to the\n// note's own order, since several passages from one note read as an abridged note.\nfunction selectWindows(candidates: SnippetWindow[], countLimit: number): { chosen: SnippetWindow[]; best: SnippetWindow } {\n const ranked = candidates.slice().sort((a, b) => b.distinct - a.distinct || b.count - a.count);\n const chosen: SnippetWindow[] = [];\n for (const c of ranked) {\n if (chosen.length >= countLimit) break;\n if (chosen.some((s) => overlaps(s, c))) continue;\n chosen.push(c);\n }\n chosen.sort((a, b) => a.start - b.start);\n return { chosen, best: ranked[0] };\n}\n\n// Builds bounded marked passages and returns the best window's offset for section lookup.\nexport function computeSnippets(text: string, terms: string[], charLimit: number, countLimit: number): { snippets: string[]; offset: number } {\n const occ = findOccurrences(text, terms);\n const bounds = wordBoundaries(text);\n if (occ.length === 0) {\n // A term matched only via title/summary, not this body text, finds no occurrence here;\n // fall back to the doc's start, unmarked.\n const render = (end: number): string => `${text.slice(0, end).replace(/\\s+/g, ' ').trim()}${end < text.length ? '…' : ''}`;\n const floor = bounds.find((b) => b > 0) ?? text.length;\n const upperBound = Math.max(floor, cutWindow(text, bounds, 0, charLimit).end);\n const end = fitEnd(render, bounds, floor, upperBound, charLimit);\n return { snippets: [render(end)], offset: 0 };\n }\n const candidates = scoreWindows(occ, bounds, text, charLimit);\n const { chosen, best } = selectWindows(candidates, countLimit);\n return { snippets: chosen.map((win) => renderWindow(text, occ, win)), offset: best.start };\n}\n\n// Converts an offset in the original text to its one-based line number.\nexport function lineNumberAt(text: string, offset: number): number {\n let line = 1;\n for (let i = 0; i < offset; i++) if (text.charCodeAt(i) === 10) line++;\n return line;\n}\n\nasync function lineRangeFor(store: Store, path: string, line: number): Promise<string | null> {\n const stmt = await store.prepare('SELECT start_line, end_line FROM sections WHERE \"path\" = ? AND start_line <= ? AND end_line >= ? ORDER BY start_line DESC LIMIT 1');\n const row = (await stmt.get(path, line, line)) as { start_line: number; end_line: number } | undefined;\n return row ? `L${row.start_line}-${row.end_line}` : null;\n}\n\nexport interface SearchOptions {\n k?: number;\n where?: string; // SQL fragment against frontmatter alias `f`, e.g. \"f.status = 'active'\"\n preset?: string; // named preset; unknown name throws listing declared presets, undefined -> \"default\"\n include?: string[]; // ad hoc scope override (repeatable --include); independent of exclude\n exclude?: string[]; // ad hoc scope override (repeatable --exclude); independent of include\n noExclude?: boolean; // --no-exclude: drop the preset's exclude for this command\n snippetCharLimit?: number; // --snippet-char-limit; defaults to SNIPPET_CHAR_LIMIT_DEFAULT\n snippetCountLimit?: number; // --snippet-count-limit; defaults to SNIPPET_COUNT_LIMIT_DEFAULT\n}\n\n// Hydrates already-ranked rows with snippets and containing-section ranges without reranking.\nexport async function hydrateSearchRows(store: Store, cfg: ResolvedConfig, rows: Row[], matchedPaths: Set<string>, terms: string, opts: Pick<SearchOptions, 'snippetCharLimit' | 'snippetCountLimit'> = {}): Promise<void> {\n if (matchedPaths.size === 0) return;\n const bareTerms = extractBareTerms(terms);\n const charLimit = opts.snippetCharLimit ?? SNIPPET_CHAR_LIMIT_DEFAULT;\n const countLimit = opts.snippetCountLimit ?? SNIPPET_COUNT_LIMIT_DEFAULT;\n for (const row of rows) {\n if (!matchedPaths.has(row.path as string)) continue;\n let text: string;\n try {\n text = readFileSync(join(cfg.baseDir, row.path as string), 'utf8');\n } catch {\n continue; // vanished since the match; leave snippets/lines empty rather than throw\n }\n const { snippets, offset } = computeSnippets(text, bareTerms, charLimit, countLimit);\n row.snippets = snippets;\n if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path as string, lineNumberAt(text, offset)) : null;\n }\n}\n\n// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.\n// `opts` arrives already resolved (config.ts:resolveSearch).\nexport async function search(store: Store, cfg: ResolvedConfig, terms: string, opts: SearchOptions = {}): Promise<Row[]> {\n const effective = resolveSearch(cfg, opts);\n const { k, signals } = effective;\n\n const allPaths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((r) => r.path);\n const scopePaths = await rawScope(store, cfg, opts, allPaths);\n const scopeActive = scopePaths.size < allPaths.length;\n // The set every candidate pool must be filtered to before truncation: scope narrowed by\n // --where, the same composition scopedPaths() gives the other commands.\n let allowedPaths: Set<string>;\n try {\n allowedPaths = await narrowByWhere(store, scopePaths, effective.where);\n } catch (err) {\n throw searchError(err as Error, terms, effective.where);\n }\n const fetch = Math.max(k * 3, 30);\n\n // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a\n // local path with missing files errors here, since nothing will ever fetch it for itself.\n const wantsVectors = signals.vectors !== undefined;\n if (wantsVectors) {\n const e = embedConfig(cfg); // validate.ts guarantees this is set whenever \"vectors\" is declared\n if (localModelMissing(e)) {\n throw new SenseError('EMBED_MODEL_MISSING', `preset \"${effective.presetName}\" searches with vectors, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop \"vectors\" from that preset's signals to search without them`);\n }\n }\n const semanticEnabled = wantsVectors && (await scopeHasEmbeddings(store, cfg, allowedPaths));\n\n // --where applies inside the candidate query (a post-filter would drop matches ranked past\n // the pool) and again on the final select, for link-derived rows.\n const scope = effective.where;\n const whereJoin = scope ? `JOIN frontmatter f ON f.\"path\" = content.path` : '';\n const whereCond = scope ? `AND (${scope})` : '';\n // Filtered before LIMIT, not after, or scoped notes ranking below the global top-`fetch`\n // never reach the filter; joined against a temp table since real scopes exceed SQLITE_MAX_VARIABLE_NUMBER.\n if (scopeActive) await materializeScope(store, '_search_scope', scopePaths);\n const scopeCond = scopeActive ? `AND content.path IN (SELECT \"path\" FROM _search_scope)` : '';\n\n const candidates = new Map<string, { score: number; via: string }>();\n let matchRows: LexicalHit[] = [];\n const hasEmptyPhrase = [...terms.matchAll(/\"([^\"]*)\"/g)].some((match) => !/[\\p{L}\\p{N}_]/u.test(match[1]));\n const hasBareSyntaxOnly = /[()]/u.test(terms.replace(/\"[^\"]*\"/g, ' '));\n // A query that is only whitespace or punctuation has no lexical rows; other declared signals\n // compose normally. In particular, unspaced-script punctuation can rewrite to an empty MATCH.\n if (signals.words !== undefined && terms.trim() !== '' && (/[\\p{L}\\p{N}_]/u.test(terms) || hasBareSyntaxOnly) && !hasEmptyPhrase) {\n const syntaxError = bareTermSyntaxError(terms);\n if (syntaxError) throw syntaxError;\n try {\n matchRows = await wordsCandidates(store, candidates, terms, whereJoin, whereCond, scopeCond, fetch, signals.words);\n } catch (err) {\n throw searchError(err as Error, terms, scope);\n }\n }\n // Every store's lexical `snippets` is NULL; the snippet text is always computed below.\n // Distinguish from via='link' rows, which never appear in matchRows.\n const matchedPaths = new Set(matchRows.map((r) => r.path));\n\n if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);\n\n let chunkLines = new Map<string, string>();\n let chunkSimilarity = new Map<string, number>();\n if (semanticEnabled) {\n ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors as number));\n }\n\n await store.exec('DROP TABLE IF EXISTS _search');\n // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the\n // rounded score/similarity are printed at full precision in rows.\n await store.exec('CREATE TEMP TABLE _search (\"path\" TEXT PRIMARY KEY, score DOUBLE, via TEXT, lines TEXT, similarity DOUBLE)');\n if (candidates.size > 0) {\n await store.runBatch(\n 'INSERT INTO _search (\"path\", score, via, lines, similarity) VALUES (?, ?, ?, ?, ?)',\n [...candidates].map(([path, c]) => [path, c.score, c.via, chunkLines.get(path) ?? null, chunkSimilarity.get(path) ?? null])\n );\n }\n\n // Reapplies --where even though _search is already scope+where filtered, since the join to\n // frontmatter is already needed for the path column.\n const where = scope ? `WHERE (${scope})` : '';\n // lines: semantic rows carry their chunk's range, lexical rows gain one below,\n // everything else stays null; similarity stays semantic-only.\n const similarityCol = semanticEnabled ? ', _search.similarity' : '';\n // NULL AS snippets holds this column's position (between summary and via) for every row;\n // overwritten below, always to an array, never left null (DESIGN.md).\n const selectStmt = await store.prepare(\n `SELECT f.\"path\" AS path, content.title, content.summary, NULL AS snippets, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}\n FROM _search JOIN frontmatter f ON f.\"path\" = _search.\"path\" JOIN content ON content.path = _search.\"path\"\n ${where} ORDER BY _search.score DESC LIMIT ?`\n );\n const rows = (await selectStmt.all(k)) as Row[];\n for (const row of rows) row.snippets = [];\n\n await hydrateSearchRows(store, cfg, rows, matchedPaths, terms, opts);\n\n return rows;\n}\n"],"names":["readFileSync","join","stemmer","embedConfig","featureEnabled","resolveSearch","localModelMissing","MODEL_FILENAMES","SenseError","bareTermSyntaxError","searchError","foldForSearch","hasUnspacedRun","searchTokens","unspacedRuns","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","linksCandidates","vectorsCandidates","wordsCandidates","SNIPPET_CHAR_LIMIT_DEFAULT","SNIPPET_COUNT_LIMIT_DEFAULT","extractBareTerms","query","cleaned","replace","split","map","tok","flatMap","text","toLowerCase","trim","filter","length","wordSegmenter","findOccurrences","terms","occ","textLower","unspacedTerms","run","runLower","slice","start","end","term","idx","found","indexOf","push","spacedTerms","t","termByStem","Map","stemInitials","Set","Array","from","keys","match","segment","unspaced","folded","has","get","undefined","sort","a","b","wordBoundaries","Intl","Segmenter","granularity","bounds","index","snapForward","pos","lo","hi","mid","snapBackward","result","firstBoundAfter","cutWindow","anchor","charLimit","rawEnd","Math","min","fitEnd","render","floor","upperBound","firstOccAtOrAfter","renderWindow","win","out","cursor","i","o","prefix","suffix","scoreWindows","candidates","winEnd","seen","count","j","add","max","e","distinct","size","overlaps","selectWindows","countLimit","ranked","chosen","c","some","s","best","computeSnippets","find","snippets","offset","lineNumberAt","line","charCodeAt","lineRangeFor","store","path","stmt","prepare","row","start_line","end_line","hydrateSearchRows","cfg","rows","matchedPaths","opts","bareTerms","snippetCharLimit","snippetCountLimit","baseDir","lines","search","effective","k","signals","allPaths","all","r","scopePaths","scopeActive","allowedPaths","where","err","fetch","wantsVectors","vectors","presetName","model","semanticEnabled","scope","whereJoin","whereCond","scopeCond","matchRows","hasEmptyPhrase","matchAll","test","hasBareSyntaxOnly","words","syntaxError","links","chunkLines","chunkSimilarity","exec","runBatch","score","via","similarityCol","selectStmt"],"mappings":"AAAA,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,OAAO,QAAQ,UAAU;AAElC,SAASC,WAAW,EAAEC,cAAc,EAAEC,aAAa,QAAQ,qBAAqB;AAChF,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,mBAAmB,EAAEC,WAAW,QAAQ,4BAA4B;AAE7E,SAASC,aAAa,EAAEC,cAAc,EAAEC,YAAY,EAAEC,YAAY,QAAQ,qBAAqB;AAC/F,SAASC,gBAAgB,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,kBAAkB,QAAQ,aAAa;AAC3F,SAASC,eAAe,EAAEC,iBAAiB,EAAEC,eAAe,QAAQ,eAAe;AAEnF,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAMC,6BAA6B,GAAG;AAC7C,OAAO,MAAMC,8BAA8B,EAAE;AAE7C,yFAAyF;AACzF,oGAAoG;AACpG,SAASC,iBAAiBC,KAAa;IACrC,MAAMC,UAAUD,MACbE,OAAO,CAAC,MAAM,KACdA,OAAO,CAAC,UAAU,KAClBA,OAAO,CAAC,kCAAkC;IAC7C,OAAOD,QACJE,KAAK,CAAC,OACNC,GAAG,CAAC,CAACC,MAAQA,IAAIH,OAAO,CAAC,kBAAkB,KAAK,iCAAiC;KACjFI,OAAO,CAAC,CAACD,MAAQjB,aAAaiB,KAAKD,GAAG,CAAC,CAAC,EAAEG,IAAI,EAAE,GAAKA,OACrDH,GAAG,CAAC,CAACC,MAAQA,IAAIG,WAAW,GAAGC,IAAI,IACnCC,MAAM,CAAC,CAACL,MAAQA,IAAIM,MAAM,GAAG;AAClC;AAEA,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAIC;AAEJ,SAASC,gBAAgBN,IAAY,EAAEO,KAAe;IACpD,MAAMC,MAA2D,EAAE;IACnE,MAAMC,YAAYT,KAAKC,WAAW;IAElC,8FAA8F;IAC9F,wFAAwF;IACxF,MAAMS,gBAAgBH,MAAMJ,MAAM,CAACvB;IACnC,IAAI8B,cAAcN,MAAM,GAAG,GAAG;QAC5B,KAAK,MAAMO,OAAO7B,aAAakB,MAAO;YACpC,MAAMY,WAAWH,UAAUI,KAAK,CAACF,IAAIG,KAAK,EAAEH,IAAII,GAAG;YACnD,KAAK,MAAMC,QAAQN,cAAe;gBAChC,IAAIO,MAAM;gBACV,OAAS;oBACP,MAAMC,QAAQN,SAASO,OAAO,CAACH,MAAMC;oBACrC,IAAIC,UAAU,CAAC,GAAG;oBAClBV,IAAIY,IAAI,CAAC;wBAAEN,OAAOH,IAAIG,KAAK,GAAGI;wBAAOH,KAAKJ,IAAIG,KAAK,GAAGI,QAAQF,KAAKZ,MAAM;wBAAEY;oBAAK;oBAChFC,MAAMC,QAAQF,KAAKZ,MAAM;gBAC3B;YACF;QACF;IACF;IAEA,6FAA6F;IAC7F,uGAAuG;IACvG,MAAMiB,cAAcd,MAAMJ,MAAM,CAAC,CAACmB,IAAM,CAAC1C,eAAe0C;IACxD,IAAID,YAAYjB,MAAM,GAAG,GAAG;QAC1B,MAAMmB,aAAa,IAAIC,IAAIH,YAAYxB,GAAG,CAAC,CAACyB,IAAM;gBAACpD,QAAQS,cAAc2C;gBAAKA;aAAE;QAChF,MAAMG,eAAe,IAAIC,IAAIC,MAAMC,IAAI,CAACL,WAAWM,IAAI,IAAI,CAACb,OAASA,IAAI,CAAC,EAAE;QAC5E,KAAK,MAAMc,SAASjD,aAAamB,MAAO;YACtC,MAAM+B,UAAUD,MAAM9B,IAAI;YAC1B,IAAI8B,MAAME,QAAQ,EAAE;YACpB,MAAMC,SAAStD,cAAcoD;YAC7B,2FAA2F;YAC3F,IAAI,CAACN,aAAaS,GAAG,CAACD,MAAM,CAAC,EAAE,GAAG;YAClC,MAAMjB,OAAOO,WAAWY,GAAG,CAACjE,QAAQ+D;YACpC,IAAIjB,SAASoB,WAAW5B,IAAIY,IAAI,CAAC;gBAAEN,OAAOgB,MAAMhB,KAAK;gBAAEC,KAAKe,MAAMf,GAAG;gBAAEC;YAAK;QAC9E;IACF;IAEA,OAAOR,IAAI6B,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAExB,KAAK,GAAGyB,EAAEzB,KAAK,IAAIyB,EAAExB,GAAG,GAAGuB,EAAEvB,GAAG;AAC9D;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,gGAAgG;AAChG,SAASyB,eAAexC,IAAY;IAClCK,0BAAAA,2BAAAA,gBAAAA,gBAAkB,IAAIoC,KAAKC,SAAS,CAAC,MAAM;QAAEC,aAAa;IAAO;IACjE,MAAMC,SAAS;QAAC;KAAE;IAClB,KAAK,MAAM,EAAEC,KAAK,EAAEd,OAAO,EAAE,IAAI1B,cAAc0B,OAAO,CAAC/B,MAAO4C,OAAOxB,IAAI,CAACyB,QAAQd,QAAQ3B,MAAM;IAChG,OAAOwC;AACT;AAEA,0FAA0F;AAC1F,SAASE,YAAYF,MAAgB,EAAEG,GAAW;IAChD,IAAIC,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM,GAAG;IACzB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,MAAO;QACzB,IAAIL,MAAM,CAACM,IAAI,GAAGH,KAAKC,KAAKE,MAAM;aAC7BD,KAAKC;IACZ;IACA,OAAON,MAAM,CAACI,GAAG;AACnB;AAEA,SAASG,aAAaP,MAAgB,EAAEG,GAAW;IACjD,IAAIC,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM,GAAG;IACzB,IAAIgD,SAASR,MAAM,CAAC,EAAE;IACtB,MAAOI,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,KAAK,KAAM;QAC7B,IAAIL,MAAM,CAACM,IAAI,IAAIH,KAAK;YACtBC,KAAKE;YACLE,SAASR,MAAM,CAACM,IAAI;QACtB,OAAOD,KAAKC,MAAM;IACpB;IACA,OAAOE;AACT;AAEA,SAASC,gBAAgBT,MAAgB,EAAEG,GAAW;IACpD,IAAIC,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM;IACtB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,MAAO;QACzB,IAAIL,MAAM,CAACM,IAAI,IAAIH,KAAKC,KAAKE,MAAM;aAC9BD,KAAKC;IACZ;IACA,OAAON,MAAM,CAACI,GAAG;AACnB;AAEA,wFAAwF;AACxF,8FAA8F;AAC9F,8FAA8F;AAC9F,+FAA+F;AAC/F,mFAAmF;AACnF,SAASM,UAAUtD,IAAY,EAAE4C,MAAgB,EAAEW,MAAc,EAAEC,SAAiB;QAI1DH;IAHxB,MAAMI,SAASC,KAAKC,GAAG,CAAC3D,KAAKI,MAAM,EAAEmD,SAASC;IAC9C,MAAM1C,QAAQgC,YAAYF,QAAQW;IAClC,IAAIxC,MAAMoC,aAAaP,QAAQa;IAC/B,IAAI1C,OAAOD,OAAOC,OAAMsC,mBAAAA,gBAAgBT,QAAQ9B,oBAAxBuC,8BAAAA,mBAAkCrD,KAAKI,MAAM;IACrE,OAAO;QAAEU;QAAOC,KAAK2C,KAAKC,GAAG,CAAC5C,KAAKf,KAAKI,MAAM;IAAE;AAClD;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,wFAAwF;AACxF,6FAA6F;AAC7F,+FAA+F;AAC/F,wDAAwD;AACxD,SAASwD,OAAOC,MAA+B,EAAEjB,MAAgB,EAAEkB,KAAa,EAAEC,UAAkB,EAAEP,SAAiB;IACrH,IAAIR,KAAK;IACT,IAAIC,KAAKL,OAAOxC,MAAM,GAAG;IACzB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,KAAK,KAAM;QAC7B,IAAIL,MAAM,CAACM,IAAI,IAAIa,YAAYf,KAAKE;aAC/BD,KAAKC,MAAM;IAClB;IACA,MAAOF,MAAM,KAAKJ,MAAM,CAACI,GAAG,GAAGc,OAAOd,KAAM;QAC1C,IAAIa,OAAOjB,MAAM,CAACI,GAAG,EAAE5C,MAAM,IAAIoD,WAAW,OAAOZ,MAAM,CAACI,GAAG;IAC/D;IACA,OAAOc;AACT;AAYA,6FAA6F;AAC7F,SAASE,kBAAkBxD,GAAwD,EAAEuC,GAAW;IAC9F,IAAIC,KAAK;IACT,IAAIC,KAAKzC,IAAIJ,MAAM;IACnB,MAAO4C,KAAKC,GAAI;QACd,MAAMC,MAAM,AAACF,KAAKC,MAAO;QACzB,IAAIzC,GAAG,CAAC0C,IAAI,CAACpC,KAAK,GAAGiC,KAAKC,KAAKE,MAAM;aAChCD,KAAKC;IACZ;IACA,OAAOF;AACT;AAEA,SAASiB,aAAajE,IAAY,EAAEQ,GAAwD,EAAE0D,GAAS;IACrG,IAAIC,MAAM;IACV,IAAIC,SAASF,IAAIpD,KAAK;IACtB,IAAK,IAAIuD,IAAIL,kBAAkBxD,KAAK0D,IAAIpD,KAAK,GAAGuD,IAAI7D,IAAIJ,MAAM,IAAII,GAAG,CAAC6D,EAAE,CAACvD,KAAK,GAAGoD,IAAInD,GAAG,EAAEsD,IAAK;QAC7F,MAAMC,IAAI9D,GAAG,CAAC6D,EAAE;QAChB,IAAIC,EAAEvD,GAAG,GAAGmD,IAAInD,GAAG,EAAE;QACrB,sFAAsF;QACtF,gFAAgF;QAChF,IAAIuD,EAAExD,KAAK,GAAGsD,QAAQ;QACtBD,OAAO,GAAGnE,KAAKa,KAAK,CAACuD,QAAQE,EAAExD,KAAK,EAAE,CAAC,EAAEd,KAAKa,KAAK,CAACyD,EAAExD,KAAK,EAAEwD,EAAEvD,GAAG,EAAE,CAAC,CAAC;QACtEqD,SAASE,EAAEvD,GAAG;IAChB;IACAoD,OAAOnE,KAAKa,KAAK,CAACuD,QAAQF,IAAInD,GAAG;IACjC,MAAMwD,SAASL,IAAIpD,KAAK,GAAG,IAAI,MAAM;IACrC,MAAM0D,SAASN,IAAInD,GAAG,GAAGf,KAAKI,MAAM,GAAG,MAAM;IAC7C,OAAO,GAAGmE,SAASJ,IAAIxE,OAAO,CAAC,QAAQ,OAAO6E,QAAQ;AACxD;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,iGAAiG;AACjG,4FAA4F;AAC5F,6FAA6F;AAC7F,4CAA4C;AAC5C,SAASC,aAAajE,GAAwD,EAAEoC,MAAgB,EAAE5C,IAAY,EAAEwD,SAAiB;IAC/H,MAAMkB,aAA8B,EAAE;IACtC,IAAK,IAAIL,IAAI,GAAGA,IAAI7D,IAAIJ,MAAM,EAAEiE,IAAK;QACnC,MAAMM,SAASnE,GAAG,CAAC6D,EAAE,CAACvD,KAAK,GAAG0C;QAC9B,MAAMoB,OAAO,IAAIlD;QACjB,IAAImD,QAAQ;QACZ,IAAK,IAAIC,IAAIT,GAAGS,IAAItE,IAAIJ,MAAM,IAAII,GAAG,CAACsE,EAAE,CAAChE,KAAK,GAAG6D,QAAQG,IAAK;YAC5DF,KAAKG,GAAG,CAACvE,GAAG,CAACsE,EAAE,CAAC9D,IAAI;YACpB6D;QACF;QACA,MAAM/D,QAAQgC,YAAYF,QAAQpC,GAAG,CAAC6D,EAAE,CAACvD,KAAK;QAC9C,MAAMgD,QAAQJ,KAAKsB,GAAG,CAAClE,OAAON,GAAG,CAAC6D,EAAE,CAACtD,GAAG;QACxC,MAAMgD,aAAaL,KAAKsB,GAAG,CAAClB,OAAOR,UAAUtD,MAAM4C,QAAQpC,GAAG,CAAC6D,EAAE,CAACvD,KAAK,EAAE0C,WAAWzC,GAAG;QACvF,MAAMA,MAAM6C,OAAO,CAACqB,IAAMhB,aAAajE,MAAMQ,KAAK;gBAAEM;gBAAOC,KAAKkE;YAAE,IAAIrC,QAAQkB,OAAOC,YAAYP;QACjGkB,WAAWtD,IAAI,CAAC;YAAEN;YAAOC;YAAKmE,UAAUN,KAAKO,IAAI;YAAEN;QAAM;IAC3D;IACA,OAAOH;AACT;AAEA,SAASU,SAAS9C,CAAO,EAAEC,CAAO;IAChC,OAAOD,EAAExB,KAAK,GAAGyB,EAAExB,GAAG,IAAIwB,EAAEzB,KAAK,GAAGwB,EAAEvB,GAAG;AAC3C;AAEA,4FAA4F;AAC5F,gGAAgG;AAChG,mFAAmF;AACnF,SAASsE,cAAcX,UAA2B,EAAEY,UAAkB;IACpE,MAAMC,SAASb,WAAW7D,KAAK,GAAGwB,IAAI,CAAC,CAACC,GAAGC,IAAMA,EAAE2C,QAAQ,GAAG5C,EAAE4C,QAAQ,IAAI3C,EAAEsC,KAAK,GAAGvC,EAAEuC,KAAK;IAC7F,MAAMW,SAA0B,EAAE;IAClC,KAAK,MAAMC,KAAKF,OAAQ;QACtB,IAAIC,OAAOpF,MAAM,IAAIkF,YAAY;QACjC,IAAIE,OAAOE,IAAI,CAAC,CAACC,IAAMP,SAASO,GAAGF,KAAK;QACxCD,OAAOpE,IAAI,CAACqE;IACd;IACAD,OAAOnD,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAExB,KAAK,GAAGyB,EAAEzB,KAAK;IACvC,OAAO;QAAE0E;QAAQI,MAAML,MAAM,CAAC,EAAE;IAAC;AACnC;AAEA,0FAA0F;AAC1F,OAAO,SAASM,gBAAgB7F,IAAY,EAAEO,KAAe,EAAEiD,SAAiB,EAAE8B,UAAkB;IAClG,MAAM9E,MAAMF,gBAAgBN,MAAMO;IAClC,MAAMqC,SAASJ,eAAexC;IAC9B,IAAIQ,IAAIJ,MAAM,KAAK,GAAG;YAINwC;QAHd,uFAAuF;QACvF,0CAA0C;QAC1C,MAAMiB,SAAS,CAAC9C,MAAwB,GAAGf,KAAKa,KAAK,CAAC,GAAGE,KAAKpB,OAAO,CAAC,QAAQ,KAAKO,IAAI,KAAKa,MAAMf,KAAKI,MAAM,GAAG,MAAM,IAAI;QAC1H,MAAM0D,SAAQlB,eAAAA,OAAOkD,IAAI,CAAC,CAACvD,IAAMA,IAAI,gBAAvBK,0BAAAA,eAA6B5C,KAAKI,MAAM;QACtD,MAAM2D,aAAaL,KAAKsB,GAAG,CAAClB,OAAOR,UAAUtD,MAAM4C,QAAQ,GAAGY,WAAWzC,GAAG;QAC5E,MAAMA,MAAM6C,OAAOC,QAAQjB,QAAQkB,OAAOC,YAAYP;QACtD,OAAO;YAAEuC,UAAU;gBAAClC,OAAO9C;aAAK;YAAEiF,QAAQ;QAAE;IAC9C;IACA,MAAMtB,aAAaD,aAAajE,KAAKoC,QAAQ5C,MAAMwD;IACnD,MAAM,EAAEgC,MAAM,EAAEI,IAAI,EAAE,GAAGP,cAAcX,YAAYY;IACnD,OAAO;QAAES,UAAUP,OAAO3F,GAAG,CAAC,CAACqE,MAAQD,aAAajE,MAAMQ,KAAK0D;QAAO8B,QAAQJ,KAAK9E,KAAK;IAAC;AAC3F;AAEA,wEAAwE;AACxE,OAAO,SAASmF,aAAajG,IAAY,EAAEgG,MAAc;IACvD,IAAIE,OAAO;IACX,IAAK,IAAI7B,IAAI,GAAGA,IAAI2B,QAAQ3B,IAAK,IAAIrE,KAAKmG,UAAU,CAAC9B,OAAO,IAAI6B;IAChE,OAAOA;AACT;AAEA,eAAeE,aAAaC,KAAY,EAAEC,IAAY,EAAEJ,IAAY;IAClE,MAAMK,OAAO,MAAMF,MAAMG,OAAO,CAAC;IACjC,MAAMC,MAAO,MAAMF,KAAKpE,GAAG,CAACmE,MAAMJ,MAAMA;IACxC,OAAOO,MAAM,CAAC,CAAC,EAAEA,IAAIC,UAAU,CAAC,CAAC,EAAED,IAAIE,QAAQ,EAAE,GAAG;AACtD;AAaA,8FAA8F;AAC9F,OAAO,eAAeC,kBAAkBP,KAAY,EAAEQ,GAAmB,EAAEC,IAAW,EAAEC,YAAyB,EAAExG,KAAa,EAAEyG,OAAsE,CAAC,CAAC;QAGtLA,wBACCA;IAHnB,IAAID,aAAa5B,IAAI,KAAK,GAAG;IAC7B,MAAM8B,YAAYzH,iBAAiBe;IACnC,MAAMiD,aAAYwD,yBAAAA,KAAKE,gBAAgB,cAArBF,oCAAAA,yBAAyB1H;IAC3C,MAAMgG,cAAa0B,0BAAAA,KAAKG,iBAAiB,cAAtBH,qCAAAA,0BAA0BzH;IAC7C,KAAK,MAAMkH,OAAOK,KAAM;QACtB,IAAI,CAACC,aAAa7E,GAAG,CAACuE,IAAIH,IAAI,GAAa;QAC3C,IAAItG;QACJ,IAAI;YACFA,OAAOhC,aAAaC,KAAK4I,IAAIO,OAAO,EAAEX,IAAIH,IAAI,GAAa;QAC7D,EAAE,OAAM;YACN,UAAU,yEAAyE;QACrF;QACA,MAAM,EAAEP,QAAQ,EAAEC,MAAM,EAAE,GAAGH,gBAAgB7F,MAAMiH,WAAWzD,WAAW8B;QACzEmB,IAAIV,QAAQ,GAAGA;QACf,IAAIU,IAAIY,KAAK,IAAI,MAAMZ,IAAIY,KAAK,GAAGjJ,eAAeyI,KAAK,cAAc,MAAMT,aAAaC,OAAOI,IAAIH,IAAI,EAAYL,aAAajG,MAAMgG,WAAW;IACnJ;AACF;AAEA,+FAA+F;AAC/F,6DAA6D;AAC7D,OAAO,eAAesB,OAAOjB,KAAY,EAAEQ,GAAmB,EAAEtG,KAAa,EAAEyG,OAAsB,CAAC,CAAC;IACrG,MAAMO,YAAYlJ,cAAcwI,KAAKG;IACrC,MAAM,EAAEQ,CAAC,EAAEC,OAAO,EAAE,GAAGF;IAEvB,MAAMG,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMrB,MAAMG,OAAO,CAAC,iCAAgC,EAAGmB,GAAG,EAAC,EAA+B9H,GAAG,CAAC,CAAC+H,IAAMA,EAAEtB,IAAI;IACrI,MAAMuB,aAAa,MAAM5I,SAASoH,OAAOQ,KAAKG,MAAMU;IACpD,MAAMI,cAAcD,WAAW1C,IAAI,GAAGuC,SAAStH,MAAM;IACrD,wFAAwF;IACxF,wEAAwE;IACxE,IAAI2H;IACJ,IAAI;QACFA,eAAe,MAAM/I,cAAcqH,OAAOwB,YAAYN,UAAUS,KAAK;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAMvJ,YAAYuJ,KAAc1H,OAAOgH,UAAUS,KAAK;IACxD;IACA,MAAME,QAAQxE,KAAKsB,GAAG,CAACwC,IAAI,GAAG;IAE9B,oFAAoF;IACpF,0FAA0F;IAC1F,MAAMW,eAAeV,QAAQW,OAAO,KAAKhG;IACzC,IAAI+F,cAAc;QAChB,MAAMlD,IAAI9G,YAAY0I,MAAM,oEAAoE;QAChG,IAAIvI,kBAAkB2G,IAAI;YACxB,MAAM,IAAIzG,WAAW,uBAAuB,CAAC,QAAQ,EAAE+I,UAAUc,UAAU,CAAC,mDAAmD,EAAEpD,EAAEqD,KAAK,CAAC,aAAa,EAAE/J,gBAAgB,uHAAuH,CAAC;QAClS;IACF;IACA,MAAMgK,kBAAkBJ,gBAAiB,MAAMjJ,mBAAmBmH,OAAOQ,KAAKkB;IAE9E,2FAA2F;IAC3F,kEAAkE;IAClE,MAAMS,QAAQjB,UAAUS,KAAK;IAC7B,MAAMS,YAAYD,QAAQ,CAAC,6CAA6C,CAAC,GAAG;IAC5E,MAAME,YAAYF,QAAQ,CAAC,KAAK,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC7C,yFAAyF;IACzF,2GAA2G;IAC3G,IAAIV,aAAa,MAAM/I,iBAAiBsH,OAAO,iBAAiBwB;IAChE,MAAMc,YAAYb,cAAc,CAAC,sDAAsD,CAAC,GAAG;IAE3F,MAAMpD,aAAa,IAAIlD;IACvB,IAAIoH,YAA0B,EAAE;IAChC,MAAMC,iBAAiB;WAAItI,MAAMuI,QAAQ,CAAC;KAAc,CAACpD,IAAI,CAAC,CAAC5D,QAAU,CAAC,iBAAiBiH,IAAI,CAACjH,KAAK,CAAC,EAAE;IACxG,MAAMkH,oBAAoB,QAAQD,IAAI,CAACxI,MAAMZ,OAAO,CAAC,YAAY;IACjE,6FAA6F;IAC7F,8FAA8F;IAC9F,IAAI8H,QAAQwB,KAAK,KAAK7G,aAAa7B,MAAML,IAAI,OAAO,MAAO,CAAA,iBAAiB6I,IAAI,CAACxI,UAAUyI,iBAAgB,KAAM,CAACH,gBAAgB;QAChI,MAAMK,cAAczK,oBAAoB8B;QACxC,IAAI2I,aAAa,MAAMA;QACvB,IAAI;YACFN,YAAY,MAAMvJ,gBAAgBgH,OAAO3B,YAAYnE,OAAOkI,WAAWC,WAAWC,WAAWT,OAAOT,QAAQwB,KAAK;QACnH,EAAE,OAAOhB,KAAK;YACZ,MAAMvJ,YAAYuJ,KAAc1H,OAAOiI;QACzC;IACF;IACA,uFAAuF;IACvF,qEAAqE;IACrE,MAAMzB,eAAe,IAAIrF,IAAIkH,UAAU/I,GAAG,CAAC,CAAC+H,IAAMA,EAAEtB,IAAI;IAExD,IAAImB,QAAQ0B,KAAK,KAAK/G,WAAW,MAAMjD,gBAAgBkH,OAAO3B,YAAYkE,WAAWlB,UAAUK,cAAcG,OAAOT,QAAQ0B,KAAK;IAEjI,IAAIC,aAAa,IAAI5H;IACrB,IAAI6H,kBAAkB,IAAI7H;IAC1B,IAAI+G,iBAAiB;QAClB,CAAA,EAAEa,UAAU,EAAEC,eAAe,EAAE,GAAG,MAAMjK,kBAAkBiH,OAAOQ,KAAKnC,YAAYnE,OAAO2H,OAAOH,cAAcN,QAAQW,OAAO,CAAU;IAC1I;IAEA,MAAM/B,MAAMiD,IAAI,CAAC;IACjB,8FAA8F;IAC9F,kEAAkE;IAClE,MAAMjD,MAAMiD,IAAI,CAAC;IACjB,IAAI5E,WAAWS,IAAI,GAAG,GAAG;QACvB,MAAMkB,MAAMkD,QAAQ,CAClB,sFACA;eAAI7E;SAAW,CAAC7E,GAAG,CAAC,CAAC,CAACyG,MAAMb,EAAE;gBAA4B2D,iBAA8BC;mBAArD;gBAAC/C;gBAAMb,EAAE+D,KAAK;gBAAE/D,EAAEgE,GAAG;iBAAEL,kBAAAA,WAAWjH,GAAG,CAACmE,mBAAf8C,6BAAAA,kBAAwB;iBAAMC,uBAAAA,gBAAgBlH,GAAG,CAACmE,mBAApB+C,kCAAAA,uBAA6B;aAAK;;IAE9H;IAEA,2FAA2F;IAC3F,qDAAqD;IACrD,MAAMrB,QAAQQ,QAAQ,CAAC,OAAO,EAAEA,MAAM,CAAC,CAAC,GAAG;IAC3C,+EAA+E;IAC/E,8DAA8D;IAC9D,MAAMkB,gBAAgBnB,kBAAkB,yBAAyB;IACjE,yFAAyF;IACzF,sEAAsE;IACtE,MAAMoB,aAAa,MAAMtD,MAAMG,OAAO,CACpC,CAAC,uIAAuI,EAAEkD,cAAc;;OAErJ,EAAE1B,MAAM,oCAAoC,CAAC;IAElD,MAAMlB,OAAQ,MAAM6C,WAAWhC,GAAG,CAACH;IACnC,KAAK,MAAMf,OAAOK,KAAML,IAAIV,QAAQ,GAAG,EAAE;IAEzC,MAAMa,kBAAkBP,OAAOQ,KAAKC,MAAMC,cAAcxG,OAAOyG;IAE/D,OAAOF;AACT"}
|