sensemaking 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -10
- package/dist/cjs/cli/index.d.cts +1 -1
- package/dist/cjs/cli/index.d.ts +1 -1
- package/dist/cjs/cli/index.js +1 -1
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/path.js +5 -29
- package/dist/cjs/cli/path.js.map +1 -1
- package/dist/cjs/cli/related.js +4 -3
- package/dist/cjs/cli/related.js.map +1 -1
- package/dist/cjs/commands/index.d.cts +6 -3
- package/dist/cjs/commands/index.d.ts +6 -3
- package/dist/cjs/commands/index.js +4 -0
- package/dist/cjs/commands/index.js.map +1 -1
- package/dist/cjs/commands/map.d.cts +14 -5
- package/dist/cjs/commands/map.d.ts +14 -5
- package/dist/cjs/commands/map.js +42 -35
- package/dist/cjs/commands/map.js.map +1 -1
- package/dist/cjs/commands/path.d.cts +14 -0
- package/dist/cjs/commands/path.d.ts +14 -0
- package/dist/cjs/commands/path.js +209 -0
- package/dist/cjs/commands/path.js.map +1 -0
- package/dist/cjs/commands/peek.d.cts +16 -1
- package/dist/cjs/commands/peek.d.ts +16 -1
- package/dist/cjs/commands/peek.js +71 -33
- package/dist/cjs/commands/peek.js.map +1 -1
- package/dist/cjs/commands/related.d.cts +16 -3
- package/dist/cjs/commands/related.d.ts +16 -3
- package/dist/cjs/commands/related.js +131 -98
- package/dist/cjs/commands/related.js.map +1 -1
- package/dist/cjs/commands/scope.d.cts +1 -0
- package/dist/cjs/commands/scope.d.ts +1 -0
- package/dist/cjs/commands/scope.js +149 -0
- package/dist/cjs/commands/scope.js.map +1 -1
- package/dist/cjs/commands/search.d.cts +26 -3
- package/dist/cjs/commands/search.d.ts +26 -3
- package/dist/cjs/commands/search.js +211 -71
- package/dist/cjs/commands/search.js.map +1 -1
- package/dist/cjs/commands/signals.d.cts +4 -3
- package/dist/cjs/commands/signals.d.ts +4 -3
- package/dist/cjs/commands/signals.js +13 -4
- package/dist/cjs/commands/signals.js.map +1 -1
- package/dist/cjs/commands/snapshot.d.cts +3 -0
- package/dist/cjs/commands/snapshot.d.ts +3 -0
- package/dist/cjs/commands/snapshot.js +180 -0
- package/dist/cjs/commands/snapshot.js.map +1 -0
- package/dist/cjs/embed/cohere.js +41 -16
- package/dist/cjs/embed/cohere.js.map +1 -1
- package/dist/cjs/embed/endpoint.d.cts +2 -0
- package/dist/cjs/embed/endpoint.d.ts +2 -0
- package/dist/cjs/embed/endpoint.js +30 -0
- package/dist/cjs/embed/endpoint.js.map +1 -0
- package/dist/cjs/embed/handoff.d.cts +7 -2
- package/dist/cjs/embed/handoff.d.ts +7 -2
- package/dist/cjs/embed/handoff.js +10 -0
- package/dist/cjs/embed/handoff.js.map +1 -1
- package/dist/cjs/embed/http.d.cts +18 -1
- package/dist/cjs/embed/http.d.ts +18 -1
- package/dist/cjs/embed/http.js +239 -40
- package/dist/cjs/embed/http.js.map +1 -1
- package/dist/cjs/embed/openai.js +39 -17
- package/dist/cjs/embed/openai.js.map +1 -1
- package/dist/cjs/embed/query.d.cts +9 -3
- package/dist/cjs/embed/query.d.ts +9 -3
- package/dist/cjs/embed/query.js +257 -90
- package/dist/cjs/embed/query.js.map +1 -1
- package/dist/cjs/embed/registry.d.cts +2 -2
- package/dist/cjs/embed/registry.d.ts +2 -2
- package/dist/cjs/embed/registry.js +36 -4
- package/dist/cjs/embed/registry.js.map +1 -1
- package/dist/cjs/embed/static.js +6 -2
- package/dist/cjs/embed/static.js.map +1 -1
- package/dist/cjs/embed/store.js +42 -77
- package/dist/cjs/embed/store.js.map +1 -1
- package/dist/cjs/embed/types.d.cts +5 -2
- package/dist/cjs/embed/types.d.ts +5 -2
- package/dist/cjs/embed/types.js.map +1 -1
- package/dist/cjs/features/embed.js +290 -43
- package/dist/cjs/features/embed.js.map +1 -1
- package/dist/cjs/graph/traverse.d.cts +2 -2
- package/dist/cjs/graph/traverse.d.ts +2 -2
- package/dist/cjs/graph/traverse.js +15 -5
- package/dist/cjs/graph/traverse.js.map +1 -1
- package/dist/cjs/index.d.cts +2 -2
- package/dist/cjs/index.d.ts +2 -2
- package/dist/cjs/index.js +6 -0
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/output/output.d.cts +4 -36
- package/dist/cjs/output/output.d.ts +4 -36
- package/dist/cjs/output/output.js +1 -2
- package/dist/cjs/output/output.js.map +1 -1
- package/dist/cjs/scan/pool.d.cts +2 -1
- package/dist/cjs/scan/pool.d.ts +2 -1
- package/dist/cjs/scan/pool.js +116 -27
- package/dist/cjs/scan/pool.js.map +1 -1
- package/dist/cjs/scan/reparse.d.cts +1 -0
- package/dist/cjs/scan/reparse.d.ts +1 -0
- package/dist/cjs/scan/reparse.js +75 -21
- package/dist/cjs/scan/reparse.js.map +1 -1
- package/dist/cjs/store/builder.d.cts +11 -21
- package/dist/cjs/store/builder.d.ts +11 -21
- package/dist/cjs/store/builder.js +1077 -438
- package/dist/cjs/store/builder.js.map +1 -1
- package/dist/cjs/store/duckdb/connection.js +16 -2
- package/dist/cjs/store/duckdb/connection.js.map +1 -1
- package/dist/cjs/store/duckdb/open.d.cts +1 -1
- package/dist/cjs/store/duckdb/open.d.ts +1 -1
- package/dist/cjs/store/duckdb/open.js +73 -65
- package/dist/cjs/store/duckdb/open.js.map +1 -1
- package/dist/cjs/store/duckdb/reconcile.js +533 -15
- package/dist/cjs/store/duckdb/reconcile.js.map +1 -1
- package/dist/cjs/store/embed-scope.js +19 -9
- package/dist/cjs/store/embed-scope.js.map +1 -1
- package/dist/cjs/store/index.d.cts +6 -0
- package/dist/cjs/store/index.d.ts +6 -0
- package/dist/cjs/store/index.js +8 -4
- package/dist/cjs/store/index.js.map +1 -1
- package/dist/cjs/store/open.d.cts +6 -1
- package/dist/cjs/store/open.d.ts +6 -1
- package/dist/cjs/store/open.js +230 -307
- package/dist/cjs/store/open.js.map +1 -1
- package/dist/cjs/store/readiness.d.cts +5 -0
- package/dist/cjs/store/readiness.d.ts +5 -0
- package/dist/cjs/store/readiness.js +39 -0
- package/dist/cjs/store/readiness.js.map +1 -0
- package/dist/cjs/store/reconcile.d.cts +26 -1
- package/dist/cjs/store/reconcile.d.ts +26 -1
- package/dist/cjs/store/reconcile.js +670 -576
- package/dist/cjs/store/reconcile.js.map +1 -1
- package/dist/cjs/store/signature.d.cts +1 -0
- package/dist/cjs/store/signature.d.ts +1 -0
- package/dist/cjs/store/signature.js +9 -0
- package/dist/cjs/store/signature.js.map +1 -1
- package/dist/cjs/store/sqlite/open.d.cts +1 -1
- package/dist/cjs/store/sqlite/open.d.ts +1 -1
- package/dist/cjs/store/sqlite/open.js +84 -90
- package/dist/cjs/store/sqlite/open.js.map +1 -1
- package/dist/cjs/store/sqlite/reconcile.js.map +1 -1
- package/dist/cjs/store/sqlite/store.js +63 -60
- package/dist/cjs/store/sqlite/store.js.map +1 -1
- package/dist/cjs/store/sqlite/vectors.d.cts +2 -2
- package/dist/cjs/store/sqlite/vectors.d.ts +2 -2
- package/dist/cjs/store/sqlite/vectors.js +8 -4
- package/dist/cjs/store/sqlite/vectors.js.map +1 -1
- package/dist/cjs/store/stages.js.map +1 -1
- package/dist/cjs/store/turso/connection.d.cts +1 -0
- package/dist/cjs/store/turso/connection.d.ts +1 -0
- package/dist/cjs/store/turso/connection.js +319 -94
- package/dist/cjs/store/turso/connection.js.map +1 -1
- package/dist/cjs/store/turso/open.d.cts +1 -1
- package/dist/cjs/store/turso/open.d.ts +1 -1
- package/dist/cjs/store/turso/open.js +70 -74
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/store.js +25 -23
- package/dist/cjs/store/turso/store.js.map +1 -1
- package/dist/cjs/store/turso/vectors.d.cts +2 -2
- package/dist/cjs/store/turso/vectors.d.ts +2 -2
- package/dist/cjs/store/turso/vectors.js +26 -16
- package/dist/cjs/store/turso/vectors.js.map +1 -1
- package/dist/cjs/store/types.d.cts +4 -1
- package/dist/cjs/store/types.d.ts +4 -1
- package/dist/cjs/store/types.js.map +1 -1
- package/dist/cjs/store/vectors.d.cts +14 -1
- package/dist/cjs/store/vectors.d.ts +14 -1
- package/dist/cjs/store/vectors.js +61 -0
- package/dist/cjs/store/vectors.js.map +1 -1
- package/dist/cjs/watch.d.cts +6 -0
- package/dist/cjs/watch.d.ts +6 -0
- package/dist/cjs/watch.js +42 -10
- package/dist/cjs/watch.js.map +1 -1
- package/dist/cjs/workers/parse.d.cts +4 -3
- package/dist/cjs/workers/parse.d.ts +4 -3
- package/dist/cjs/workers/parse.js +19 -17
- package/dist/cjs/workers/parse.js.map +1 -1
- package/dist/esm/cli/index.d.ts +1 -1
- package/dist/esm/cli/index.js +1 -1
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/path.js +4 -10
- package/dist/esm/cli/path.js.map +1 -1
- package/dist/esm/cli/related.js +5 -3
- package/dist/esm/cli/related.js.map +1 -1
- package/dist/esm/commands/index.d.ts +6 -3
- package/dist/esm/commands/index.js +1 -0
- package/dist/esm/commands/index.js.map +1 -1
- package/dist/esm/commands/map.d.ts +14 -5
- package/dist/esm/commands/map.js +2 -0
- package/dist/esm/commands/map.js.map +1 -1
- package/dist/esm/commands/path.d.ts +14 -0
- package/dist/esm/commands/path.js +33 -0
- package/dist/esm/commands/path.js.map +1 -0
- package/dist/esm/commands/peek.d.ts +16 -1
- package/dist/esm/commands/peek.js +35 -9
- package/dist/esm/commands/peek.js.map +1 -1
- package/dist/esm/commands/related.d.ts +16 -3
- package/dist/esm/commands/related.js +45 -28
- package/dist/esm/commands/related.js.map +1 -1
- package/dist/esm/commands/scope.d.ts +1 -0
- package/dist/esm/commands/scope.js +58 -0
- package/dist/esm/commands/scope.js.map +1 -1
- package/dist/esm/commands/search.d.ts +26 -3
- package/dist/esm/commands/search.js +85 -18
- package/dist/esm/commands/search.js.map +1 -1
- package/dist/esm/commands/signals.d.ts +4 -3
- package/dist/esm/commands/signals.js +11 -4
- package/dist/esm/commands/signals.js.map +1 -1
- package/dist/esm/commands/snapshot.d.ts +3 -0
- package/dist/esm/commands/snapshot.js +25 -0
- package/dist/esm/commands/snapshot.js.map +1 -0
- package/dist/esm/embed/cohere.js +35 -10
- package/dist/esm/embed/cohere.js.map +1 -1
- package/dist/esm/embed/endpoint.d.ts +2 -0
- package/dist/esm/embed/endpoint.js +11 -0
- package/dist/esm/embed/endpoint.js.map +1 -0
- package/dist/esm/embed/handoff.d.ts +7 -2
- package/dist/esm/embed/handoff.js +7 -0
- package/dist/esm/embed/handoff.js.map +1 -1
- package/dist/esm/embed/http.d.ts +18 -1
- package/dist/esm/embed/http.js +137 -17
- package/dist/esm/embed/http.js.map +1 -1
- package/dist/esm/embed/openai.js +34 -12
- package/dist/esm/embed/openai.js.map +1 -1
- package/dist/esm/embed/query.d.ts +9 -3
- package/dist/esm/embed/query.js +98 -26
- package/dist/esm/embed/query.js.map +1 -1
- package/dist/esm/embed/registry.d.ts +2 -2
- package/dist/esm/embed/registry.js +34 -5
- package/dist/esm/embed/registry.js.map +1 -1
- package/dist/esm/embed/static.js +10 -2
- package/dist/esm/embed/static.js.map +1 -1
- package/dist/esm/embed/store.js +30 -21
- package/dist/esm/embed/store.js.map +1 -1
- package/dist/esm/embed/types.d.ts +5 -2
- package/dist/esm/embed/types.js.map +1 -1
- package/dist/esm/features/embed.js +82 -21
- package/dist/esm/features/embed.js.map +1 -1
- package/dist/esm/graph/traverse.d.ts +2 -2
- package/dist/esm/graph/traverse.js +4 -2
- package/dist/esm/graph/traverse.js.map +1 -1
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/output/output.d.ts +4 -36
- package/dist/esm/output/output.js +1 -2
- package/dist/esm/output/output.js.map +1 -1
- package/dist/esm/scan/pool.d.ts +2 -1
- package/dist/esm/scan/pool.js +80 -23
- package/dist/esm/scan/pool.js.map +1 -1
- package/dist/esm/scan/reparse.d.ts +1 -0
- package/dist/esm/scan/reparse.js +40 -12
- package/dist/esm/scan/reparse.js.map +1 -1
- package/dist/esm/store/builder.d.ts +11 -21
- package/dist/esm/store/builder.js +337 -133
- package/dist/esm/store/builder.js.map +1 -1
- package/dist/esm/store/duckdb/connection.js +9 -1
- package/dist/esm/store/duckdb/connection.js.map +1 -1
- package/dist/esm/store/duckdb/open.d.ts +1 -1
- package/dist/esm/store/duckdb/open.js +13 -9
- package/dist/esm/store/duckdb/open.js.map +1 -1
- package/dist/esm/store/duckdb/reconcile.js +145 -10
- package/dist/esm/store/duckdb/reconcile.js.map +1 -1
- package/dist/esm/store/embed-scope.js +16 -6
- package/dist/esm/store/embed-scope.js.map +1 -1
- package/dist/esm/store/index.d.ts +6 -0
- package/dist/esm/store/index.js +10 -4
- package/dist/esm/store/index.js.map +1 -1
- package/dist/esm/store/open.d.ts +6 -1
- package/dist/esm/store/open.js +77 -87
- package/dist/esm/store/open.js.map +1 -1
- package/dist/esm/store/readiness.d.ts +5 -0
- package/dist/esm/store/readiness.js +11 -0
- package/dist/esm/store/readiness.js.map +1 -0
- package/dist/esm/store/reconcile.d.ts +26 -1
- package/dist/esm/store/reconcile.js +142 -120
- package/dist/esm/store/reconcile.js.map +1 -1
- package/dist/esm/store/signature.d.ts +1 -0
- package/dist/esm/store/signature.js +8 -0
- package/dist/esm/store/signature.js.map +1 -1
- package/dist/esm/store/sqlite/open.d.ts +1 -1
- package/dist/esm/store/sqlite/open.js +12 -12
- package/dist/esm/store/sqlite/open.js.map +1 -1
- package/dist/esm/store/sqlite/reconcile.js +2 -2
- package/dist/esm/store/sqlite/reconcile.js.map +1 -1
- package/dist/esm/store/sqlite/store.js +21 -18
- package/dist/esm/store/sqlite/store.js.map +1 -1
- package/dist/esm/store/sqlite/vectors.d.ts +2 -2
- package/dist/esm/store/sqlite/vectors.js +7 -5
- package/dist/esm/store/sqlite/vectors.js.map +1 -1
- package/dist/esm/store/stages.js.map +1 -1
- package/dist/esm/store/turso/connection.d.ts +1 -0
- package/dist/esm/store/turso/connection.js +97 -28
- package/dist/esm/store/turso/connection.js.map +1 -1
- package/dist/esm/store/turso/open.d.ts +1 -1
- package/dist/esm/store/turso/open.js +9 -8
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/store.js +16 -14
- package/dist/esm/store/turso/store.js.map +1 -1
- package/dist/esm/store/turso/vectors.d.ts +2 -2
- package/dist/esm/store/turso/vectors.js +27 -17
- package/dist/esm/store/turso/vectors.js.map +1 -1
- package/dist/esm/store/types.d.ts +4 -1
- package/dist/esm/store/types.js.map +1 -1
- package/dist/esm/store/vectors.d.ts +14 -1
- package/dist/esm/store/vectors.js +55 -0
- package/dist/esm/store/vectors.js.map +1 -1
- package/dist/esm/watch.d.ts +6 -0
- package/dist/esm/watch.js +40 -14
- package/dist/esm/watch.js.map +1 -1
- package/dist/esm/workers/parse.d.ts +4 -3
- package/dist/esm/workers/parse.js +19 -17
- package/dist/esm/workers/parse.js.map +1 -1
- package/package.json +1 -1
- package/skills/sense/SKILL.md +4 -2
- package/skills/sense/references/stores/duckdb.md +1 -1
- package/skills/sense-setup/SKILL.md +2 -2
- package/skills/sense-setup/references/embeddings.md +1 -1
- package/skills/sense-setup/references/store-benchmarks.md +5 -5
- package/skills/sense-setup/references/stores/duckdb.md +1 -1
|
@@ -10,6 +10,7 @@ Object.defineProperty(exports, "embed", {
|
|
|
10
10
|
});
|
|
11
11
|
var _indexts = require("../chunk/index.js");
|
|
12
12
|
var _indexts1 = require("../config/index.js");
|
|
13
|
+
var _endpointts = require("../embed/endpoint.js");
|
|
13
14
|
var _handoffts = require("../embed/handoff.js");
|
|
14
15
|
var _identityts = require("../embed/identity.js");
|
|
15
16
|
var _frontmatterts = require("../scan/frontmatter.js");
|
|
@@ -139,6 +140,27 @@ function _ts_generator(thisArg, body) {
|
|
|
139
140
|
};
|
|
140
141
|
}
|
|
141
142
|
}
|
|
143
|
+
function _ts_values(o) {
|
|
144
|
+
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
|
|
145
|
+
if (m) {
|
|
146
|
+
return m.call(o);
|
|
147
|
+
}
|
|
148
|
+
if (o && typeof o.length === "number") {
|
|
149
|
+
return {
|
|
150
|
+
next: function() {
|
|
151
|
+
if (o && i >= o.length) {
|
|
152
|
+
o = void 0;
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
value: o && o[i++],
|
|
156
|
+
done: !o
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
|
|
162
|
+
}
|
|
163
|
+
var CHUNK_LOOKUP_PAGE = 512;
|
|
142
164
|
// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so
|
|
143
165
|
// `offset` shifts line numbers back onto raw and a range stays a direct Read range.
|
|
144
166
|
function chunksOf(blocks, body, search) {
|
|
@@ -168,7 +190,7 @@ var embed = {
|
|
|
168
190
|
case 0:
|
|
169
191
|
return [
|
|
170
192
|
4,
|
|
171
|
-
db.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))')
|
|
193
|
+
db.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))')
|
|
172
194
|
];
|
|
173
195
|
case 1:
|
|
174
196
|
_state.sent();
|
|
@@ -185,19 +207,24 @@ var embed = {
|
|
|
185
207
|
// blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.
|
|
186
208
|
return chunksOf(blocks !== null && blocks !== void 0 ? blocks : (0, _indexts.parse)(body), body, search, (0, _frontmatterts.countLines)(raw) - (0, _frontmatterts.countLines)(body), cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
|
|
187
209
|
},
|
|
188
|
-
remove: function remove(db, paths) {
|
|
210
|
+
remove: function remove(db, paths, delta) {
|
|
189
211
|
return _async_to_generator(function() {
|
|
212
|
+
var vanishedSet, vanished;
|
|
190
213
|
return _ts_generator(this, function(_state) {
|
|
191
214
|
switch(_state.label){
|
|
192
215
|
case 0:
|
|
193
|
-
|
|
216
|
+
vanishedSet = new Set(delta.vanished);
|
|
217
|
+
vanished = paths.filter(function(path) {
|
|
218
|
+
return vanishedSet.has(path);
|
|
219
|
+
});
|
|
220
|
+
if (vanished.length === 0) return [
|
|
194
221
|
2
|
|
195
222
|
];
|
|
196
223
|
return [
|
|
197
224
|
4,
|
|
198
|
-
db.runBatch('DELETE FROM embeddings WHERE "path" = ?',
|
|
225
|
+
db.runBatch('DELETE FROM embeddings WHERE "path" = ?', vanished.map(function(path) {
|
|
199
226
|
return [
|
|
200
|
-
|
|
227
|
+
path
|
|
201
228
|
];
|
|
202
229
|
}))
|
|
203
230
|
];
|
|
@@ -212,50 +239,268 @@ var embed = {
|
|
|
212
239
|
},
|
|
213
240
|
store: // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file
|
|
214
241
|
// filter skips it), so extracted is undefined here -- those docs contribute no rows.
|
|
215
|
-
function store(db, docs) {
|
|
242
|
+
function store(db, docs, delta) {
|
|
216
243
|
return _async_to_generator(function() {
|
|
217
|
-
var rows, texts, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step;
|
|
244
|
+
var rows, texts, added, firstPage, nextPage, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step, err;
|
|
218
245
|
return _ts_generator(this, function(_state) {
|
|
219
246
|
switch(_state.label){
|
|
220
247
|
case 0:
|
|
221
248
|
rows = [];
|
|
222
249
|
texts = new Map();
|
|
250
|
+
added = new Set(delta.added);
|
|
223
251
|
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
252
|
+
_state.label = 1;
|
|
253
|
+
case 1:
|
|
254
|
+
_state.trys.push([
|
|
255
|
+
1,
|
|
256
|
+
6,
|
|
257
|
+
7,
|
|
258
|
+
8
|
|
259
|
+
]);
|
|
260
|
+
_loop = function() {
|
|
261
|
+
var _step_value, path, extracted, chunks, identified, kept, cursor, page, _tmp, _tmp1, removed, moved, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step1, old, index, chunk, before;
|
|
262
|
+
return _ts_generator(this, function(_state) {
|
|
263
|
+
switch(_state.label){
|
|
264
|
+
case 0:
|
|
265
|
+
_step_value = _step.value, path = _step_value.path, extracted = _step_value.extracted;
|
|
266
|
+
chunks = extracted !== null && extracted !== void 0 ? extracted : [];
|
|
267
|
+
if (!(chunks.length === 0)) return [
|
|
268
|
+
3,
|
|
269
|
+
3
|
|
270
|
+
];
|
|
271
|
+
if (!!added.has(path)) return [
|
|
272
|
+
3,
|
|
273
|
+
2
|
|
274
|
+
];
|
|
275
|
+
return [
|
|
276
|
+
4,
|
|
277
|
+
db.runBatch('DELETE FROM embeddings WHERE "path" = ?', [
|
|
278
|
+
[
|
|
279
|
+
path
|
|
280
|
+
]
|
|
281
|
+
])
|
|
282
|
+
];
|
|
283
|
+
case 1:
|
|
284
|
+
_state.sent();
|
|
285
|
+
_state.label = 2;
|
|
286
|
+
case 2:
|
|
287
|
+
return [
|
|
288
|
+
2,
|
|
289
|
+
"continue"
|
|
290
|
+
];
|
|
291
|
+
case 3:
|
|
292
|
+
identified = chunks.map(function(chunk) {
|
|
293
|
+
return (0, _handoffts.identifyChunkText)(chunk.text);
|
|
294
|
+
});
|
|
295
|
+
kept = new Set();
|
|
296
|
+
if (!!added.has(path)) return [
|
|
297
|
+
3,
|
|
298
|
+
20
|
|
299
|
+
];
|
|
300
|
+
cursor = void 0;
|
|
301
|
+
_state.label = 4;
|
|
302
|
+
case 4:
|
|
303
|
+
page = void 0;
|
|
304
|
+
if (!(cursor === undefined)) return [
|
|
305
|
+
3,
|
|
306
|
+
9
|
|
307
|
+
];
|
|
308
|
+
if (!(firstPage !== null && firstPage !== void 0)) return [
|
|
309
|
+
3,
|
|
310
|
+
5
|
|
311
|
+
];
|
|
312
|
+
_tmp = firstPage;
|
|
313
|
+
return [
|
|
314
|
+
3,
|
|
315
|
+
7
|
|
316
|
+
];
|
|
317
|
+
case 5:
|
|
318
|
+
return [
|
|
319
|
+
4,
|
|
320
|
+
db.prepare('SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE "path" = ? ORDER BY chunk LIMIT '.concat(CHUNK_LOOKUP_PAGE))
|
|
321
|
+
];
|
|
322
|
+
case 6:
|
|
323
|
+
_tmp = firstPage = _state.sent();
|
|
324
|
+
_state.label = 7;
|
|
325
|
+
case 7:
|
|
326
|
+
_tmp;
|
|
327
|
+
return [
|
|
328
|
+
4,
|
|
329
|
+
firstPage.all(path)
|
|
330
|
+
];
|
|
331
|
+
case 8:
|
|
332
|
+
page = _state.sent();
|
|
333
|
+
return [
|
|
334
|
+
3,
|
|
335
|
+
14
|
|
336
|
+
];
|
|
337
|
+
case 9:
|
|
338
|
+
if (!(nextPage !== null && nextPage !== void 0)) return [
|
|
339
|
+
3,
|
|
340
|
+
10
|
|
341
|
+
];
|
|
342
|
+
_tmp1 = nextPage;
|
|
343
|
+
return [
|
|
344
|
+
3,
|
|
345
|
+
12
|
|
346
|
+
];
|
|
347
|
+
case 10:
|
|
348
|
+
return [
|
|
349
|
+
4,
|
|
350
|
+
db.prepare('SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE "path" = ? AND chunk > ? ORDER BY chunk LIMIT '.concat(CHUNK_LOOKUP_PAGE))
|
|
351
|
+
];
|
|
352
|
+
case 11:
|
|
353
|
+
_tmp1 = nextPage = _state.sent();
|
|
354
|
+
_state.label = 12;
|
|
355
|
+
case 12:
|
|
356
|
+
_tmp1;
|
|
357
|
+
return [
|
|
358
|
+
4,
|
|
359
|
+
nextPage.all(path, cursor)
|
|
360
|
+
];
|
|
361
|
+
case 13:
|
|
362
|
+
page = _state.sent();
|
|
363
|
+
_state.label = 14;
|
|
364
|
+
case 14:
|
|
365
|
+
removed = [];
|
|
366
|
+
moved = [];
|
|
367
|
+
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
368
|
+
try {
|
|
369
|
+
for(_iterator = page[Symbol.iterator](); !(_iteratorNormalCompletion = (_step1 = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
370
|
+
old = _step1.value;
|
|
371
|
+
index = Number(old.chunk);
|
|
372
|
+
chunk = chunks[index];
|
|
373
|
+
if (!chunk || old.content_identity !== identified[index].identity) {
|
|
374
|
+
removed.push([
|
|
375
|
+
path,
|
|
376
|
+
index
|
|
377
|
+
]);
|
|
378
|
+
} else {
|
|
379
|
+
kept.add(index);
|
|
380
|
+
if (Number(old.start_line) !== chunk.startLine || Number(old.end_line) !== chunk.endLine) moved.push([
|
|
381
|
+
chunk.startLine,
|
|
382
|
+
chunk.endLine,
|
|
383
|
+
path,
|
|
384
|
+
index
|
|
385
|
+
]);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
} catch (err) {
|
|
389
|
+
_didIteratorError = true;
|
|
390
|
+
_iteratorError = err;
|
|
391
|
+
} finally{
|
|
392
|
+
try {
|
|
393
|
+
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
394
|
+
_iterator.return();
|
|
395
|
+
}
|
|
396
|
+
} finally{
|
|
397
|
+
if (_didIteratorError) {
|
|
398
|
+
throw _iteratorError;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
if (!(removed.length > 0)) return [
|
|
403
|
+
3,
|
|
404
|
+
16
|
|
405
|
+
];
|
|
406
|
+
return [
|
|
407
|
+
4,
|
|
408
|
+
db.runBatch('DELETE FROM embeddings WHERE "path" = ? AND chunk = ?', removed)
|
|
409
|
+
];
|
|
410
|
+
case 15:
|
|
411
|
+
_state.sent();
|
|
412
|
+
_state.label = 16;
|
|
413
|
+
case 16:
|
|
414
|
+
if (!(moved.length > 0)) return [
|
|
415
|
+
3,
|
|
416
|
+
18
|
|
417
|
+
];
|
|
418
|
+
return [
|
|
419
|
+
4,
|
|
420
|
+
db.runBatch('UPDATE embeddings SET start_line = ?, end_line = ? WHERE "path" = ? AND chunk = ?', moved)
|
|
421
|
+
];
|
|
422
|
+
case 17:
|
|
423
|
+
_state.sent();
|
|
424
|
+
_state.label = 18;
|
|
425
|
+
case 18:
|
|
426
|
+
if (page.length < CHUNK_LOOKUP_PAGE) return [
|
|
427
|
+
3,
|
|
428
|
+
20
|
|
429
|
+
];
|
|
430
|
+
cursor = Number(page[page.length - 1].chunk);
|
|
431
|
+
_state.label = 19;
|
|
432
|
+
case 19:
|
|
433
|
+
return [
|
|
434
|
+
3,
|
|
435
|
+
4
|
|
436
|
+
];
|
|
437
|
+
case 20:
|
|
438
|
+
before = rows.length;
|
|
439
|
+
chunks.forEach(function(chunk, index) {
|
|
440
|
+
if (!kept.has(index)) rows.push([
|
|
441
|
+
path,
|
|
442
|
+
index,
|
|
443
|
+
chunk.startLine,
|
|
444
|
+
chunk.endLine,
|
|
445
|
+
identified[index].identity
|
|
446
|
+
]);
|
|
447
|
+
});
|
|
448
|
+
// No new pending rows means no retained handoff. Matching pending rows can use
|
|
449
|
+
// embedPending's exact indexed-source fallback, including after repeated store calls.
|
|
450
|
+
if (rows.length > before) texts.set(path, identified);
|
|
451
|
+
return [
|
|
452
|
+
2
|
|
453
|
+
];
|
|
454
|
+
}
|
|
455
|
+
});
|
|
456
|
+
};
|
|
457
|
+
_iterator = docs[Symbol.iterator]();
|
|
458
|
+
_state.label = 2;
|
|
459
|
+
case 2:
|
|
460
|
+
if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
|
|
461
|
+
3,
|
|
462
|
+
5
|
|
463
|
+
];
|
|
464
|
+
return [
|
|
465
|
+
5,
|
|
466
|
+
_ts_values(_loop())
|
|
467
|
+
];
|
|
468
|
+
case 3:
|
|
469
|
+
_state.sent();
|
|
470
|
+
_state.label = 4;
|
|
471
|
+
case 4:
|
|
472
|
+
_iteratorNormalCompletion = true;
|
|
473
|
+
return [
|
|
474
|
+
3,
|
|
475
|
+
2
|
|
476
|
+
];
|
|
477
|
+
case 5:
|
|
478
|
+
return [
|
|
479
|
+
3,
|
|
480
|
+
8
|
|
481
|
+
];
|
|
482
|
+
case 6:
|
|
483
|
+
err = _state.sent();
|
|
484
|
+
_didIteratorError = true;
|
|
485
|
+
_iteratorError = err;
|
|
486
|
+
return [
|
|
487
|
+
3,
|
|
488
|
+
8
|
|
489
|
+
];
|
|
490
|
+
case 7:
|
|
224
491
|
try {
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
var chunks = extracted;
|
|
229
|
-
chunks.forEach(function(c, idx) {
|
|
230
|
-
return rows.push([
|
|
231
|
-
path,
|
|
232
|
-
idx,
|
|
233
|
-
c.startLine,
|
|
234
|
-
c.endLine
|
|
235
|
-
]);
|
|
236
|
-
});
|
|
237
|
-
texts.set(path, chunks.map(function(c) {
|
|
238
|
-
return c.text;
|
|
239
|
-
}));
|
|
240
|
-
};
|
|
241
|
-
for(_iterator = docs[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true)_loop();
|
|
242
|
-
} catch (err) {
|
|
243
|
-
_didIteratorError = true;
|
|
244
|
-
_iteratorError = err;
|
|
492
|
+
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
493
|
+
_iterator.return();
|
|
494
|
+
}
|
|
245
495
|
} finally{
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
_iterator.return();
|
|
249
|
-
}
|
|
250
|
-
} finally{
|
|
251
|
-
if (_didIteratorError) {
|
|
252
|
-
throw _iteratorError;
|
|
253
|
-
}
|
|
496
|
+
if (_didIteratorError) {
|
|
497
|
+
throw _iteratorError;
|
|
254
498
|
}
|
|
255
499
|
}
|
|
256
|
-
|
|
257
|
-
|
|
500
|
+
return [
|
|
501
|
+
7
|
|
258
502
|
];
|
|
503
|
+
case 8:
|
|
259
504
|
// scale and vector are written by the embed pass, not here, so they are not in the column list
|
|
260
505
|
// and take the table's own default on the append path.
|
|
261
506
|
return [
|
|
@@ -264,13 +509,13 @@ var embed = {
|
|
|
264
509
|
'path',
|
|
265
510
|
'chunk',
|
|
266
511
|
'start_line',
|
|
267
|
-
'end_line'
|
|
268
|
-
|
|
512
|
+
'end_line',
|
|
513
|
+
'content_identity'
|
|
514
|
+
], 'INSERT INTO embeddings ("path", chunk, start_line, end_line, content_identity, scale, vector) VALUES (?, ?, ?, ?, ?, NULL, NULL)', rows)
|
|
269
515
|
];
|
|
270
|
-
case
|
|
516
|
+
case 9:
|
|
271
517
|
_state.sent();
|
|
272
|
-
//
|
|
273
|
-
// process rather than re-derived from the file there (src/embed/handoff.ts).
|
|
518
|
+
// Hand off only paths with new pending rows; missing entries use the indexed source.
|
|
274
519
|
(0, _handoffts.stashChunkText)(db, texts);
|
|
275
520
|
return [
|
|
276
521
|
2
|
|
@@ -293,7 +538,9 @@ var embed = {
|
|
|
293
538
|
// W3b: chunkTokens rides the version token, since it's the one owner lever the chunker
|
|
294
539
|
// itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.
|
|
295
540
|
var chunkVersion = e.chunkTokens !== undefined ? "".concat(_indexts.CHUNK_VERSION, ":").concat(e.chunkTokens) : _indexts.CHUNK_VERSION;
|
|
296
|
-
|
|
541
|
+
var endpoint = (0, _endpointts.embedEndpointIdentity)(e.provider, e.url);
|
|
542
|
+
var endpointPart = endpoint === undefined ? '' : ":endpoint=".concat(endpoint);
|
|
543
|
+
return "embed:".concat(e.provider, ":").concat(e.model).concat(endpointPart, ":").concat(chunkVersion).concat(identity !== undefined ? "@".concat(identity) : '');
|
|
297
544
|
}
|
|
298
545
|
};
|
|
299
546
|
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and build (or an authorised scoped CLI query) prepares them before semantic retrieval.\n\nexport type { Chunk };\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n // for every chunker input not already carried in config.\n return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n startLine: c.startLine + offset,\n endLine: c.endLine + offset,\n text: prefix ? `${prefix}\\n${c.text}` : c.text,\n }));\n}\n\nexport const embed: Feature = {\n name: 'embed',\n async schema(db) {\n await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n },\n extract(raw, body, search, _data, cfg, blocks) {\n // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n },\n async remove(db, paths) {\n if (paths.length === 0) return;\n await db.runBatch(\n 'DELETE FROM embeddings WHERE \"path\" = ?',\n paths.map((p) => [p])\n );\n },\n // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n async store(db, docs) {\n const rows: unknown[][] = [];\n const texts = new Map<string, string[]>();\n for (const { path, extracted } of docs) {\n if (!extracted) continue;\n const chunks = extracted as Chunk[];\n chunks.forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n texts.set(\n path,\n chunks.map((c) => c.text)\n );\n }\n if (rows.length === 0) return;\n // scale and vector are written by the embed pass, not here, so they are not in the column list\n // and take the table's own default on the append path.\n await appendRows(db, 'embeddings', ['path', 'chunk', 'start_line', 'end_line'], 'INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n // The text extract() just computed, handed to the embed pass that follows in this same\n // process rather than re-derived from the file there (src/embed/handoff.ts).\n stashChunkText(db, texts);\n },\n enabledForFile(_cfg, file) {\n return file.embed;\n },\n // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n // a model can be configured with no preset yet using vectors.\n signature(cfg) {\n const e = embedConfig(cfg);\n if (!e) return 'embed:off';\n // Weight identity from identity.ts, no network: a static model's resolved sha or local\n // size+mtime, appended once known so changed weights re-embed.\n const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n },\n};\n"],"names":["embed","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","chunkFromBlocks","undefined","targetTokens","map","c","startLine","endLine","text","name","schema","db","exec","extract","raw","_data","cfg","parse","countLines","remove","paths","length","runBatch","p","store","docs","rows","texts","Map","path","extracted","chunks","forEach","idx","push","set","appendRows","stashChunkText","enabledForFile","_cfg","file","signature","e","embedConfig","identity","provider","modelIdentity","model","chunkVersion","CHUNK_VERSION"],"mappings":";;;;+BA2BaA;;;eAAAA;;;uBA1ByC;wBAC1B;yBACG;0BACD;6BACH;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQ3B,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C;QAAEC,SAAAA,iEAAS,GAAGC;IACxG,IAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOC,IAAAA,wBAAe,EAACX,QAAQC,MAAMG,gBAAgBQ,YAAY;QAAEC,cAAcT;IAAY,IAAIQ,WAAWE,GAAG,CAAC,SAACC;eAAO;YACtHC,WAAWD,EAAEC,SAAS,GAAGb;YACzBc,SAASF,EAAEE,OAAO,GAAGd;YACrBe,MAAMb,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAW,OAAPU,EAAEG,IAAI,IAAKH,EAAEG,IAAI;QAChD;;AACF;AAEO,IAAMpB,QAAiB;IAC5BqB,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAU2B,IAAAA,cAAK,EAAC1B,OAAOA,MAAMC,QAAQ0B,IAAAA,yBAAU,EAACJ,OAAOI,IAAAA,yBAAU,EAAC3B,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAK5B,KAAK,cAAV4B,iCAAAA,WAAYtB,WAAW;IAClH;IACMyB,QAAN,SAAMA,OAAOR,EAAE,EAAES,KAAK;;;;;wBACpB,IAAIA,MAAMC,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMV,GAAGW,QAAQ,CACf,2CACAF,MAAMhB,GAAG,CAAC,SAACmB;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IAGMC,OAFN,8FAA8F;IAC9F,qFAAqF;IACrF,SAAMA,MAAMb,EAAE,EAAEc,IAAI;;gBACZC,MACAC,OACD,2BAAA,mBAAA,uBAAA,WAAA;;;;wBAFCD;wBACAC,QAAQ,IAAIC;wBACb,kCAAA,2BAAA;;;gCAAA,kBAAA,aAAQC,mBAAAA,MAAMC,wBAAAA;gCACjB,IAAI,CAACA,WAAW,OAAA;gCAChB,IAAMC,SAASD;gCACfC,OAAOC,OAAO,CAAC,SAAC3B,GAAG4B;2CAAQP,KAAKQ,IAAI,CAAC;wCAACL;wCAAMI;wCAAK5B,EAAEC,SAAS;wCAAED,EAAEE,OAAO;qCAAC;;gCACxEoB,MAAMQ,GAAG,CACPN,MACAE,OAAO3B,GAAG,CAAC,SAACC;2CAAMA,EAAEG,IAAI;;4BAE5B;4BARA,IAAK,YAA6BiB,2BAA7B,6BAAA,QAAA,yBAAA;;4BAAA;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBASL,IAAIC,KAAKL,MAAM,KAAK,GAAG;;;wBACvB,+FAA+F;wBAC/F,uDAAuD;wBACvD;;4BAAMe,IAAAA,oBAAU,EAACzB,IAAI;gCAAe;gCAAQ;gCAAS;gCAAc;+BAAa,+GAA+Ge;;;wBAA/L;wBACA,uFAAuF;wBACvF,6EAA6E;wBAC7EW,IAAAA,yBAAc,EAAC1B,IAAIgB;;;;;;QACrB;;IACAW,gBAAAA,SAAAA,eAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKpD,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9DqD,WAAAA,SAAAA,UAAUzB,GAAG;QACX,IAAM0B,IAAIC,IAAAA,qBAAW,EAAC3B;QACtB,IAAI,CAAC0B,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,IAAME,WAAWF,EAAEG,QAAQ,KAAK,WAAWC,IAAAA,yBAAa,EAACJ,EAAEK,KAAK,IAAI7C;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,IAAM8C,eAAeN,EAAEhD,WAAW,KAAKQ,YAAY,AAAC,GAAmBwC,OAAjBO,sBAAa,EAAC,KAAiB,OAAdP,EAAEhD,WAAW,IAAKuD,sBAAa;QACtG,OAAO,AAAC,SAAsBP,OAAdA,EAAEG,QAAQ,EAAC,KAAcG,OAAXN,EAAEK,KAAK,EAAC,KAAkBH,OAAfI,cAA4D,OAA7CJ,aAAa1C,YAAY,AAAC,IAAY,OAAT0C,YAAa;IACpG;AACF"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { embedEndpointIdentity } from '../embed/endpoint.ts';\nimport { identifyChunkText, stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and build (or an authorised scoped CLI query) prepares them before semantic retrieval.\n\nexport type { Chunk };\n\nconst CHUNK_LOOKUP_PAGE = 512;\ninterface ChunkRow {\n chunk: number;\n start_line: number;\n end_line: number;\n content_identity: string;\n}\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n // for every chunker input not already carried in config.\n return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n startLine: c.startLine + offset,\n endLine: c.endLine + offset,\n text: prefix ? `${prefix}\\n${c.text}` : c.text,\n }));\n}\n\nexport const embed: Feature = {\n name: 'embed',\n async schema(db) {\n await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n },\n extract(raw, body, search, _data, cfg, blocks) {\n // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n },\n async remove(db, paths, delta) {\n const vanishedSet = new Set(delta.vanished);\n const vanished = paths.filter((path) => vanishedSet.has(path));\n if (vanished.length === 0) return;\n await db.runBatch(\n 'DELETE FROM embeddings WHERE \"path\" = ?',\n vanished.map((path) => [path])\n );\n },\n // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n async store(db, docs, delta) {\n const rows: unknown[][] = [];\n const texts = new Map<string, ReturnType<typeof identifyChunkText>[]>();\n const added = new Set(delta.added);\n // Keyset pages bound comparison metadata without reading native vector payloads.\n let firstPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n let nextPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n for (const { path, extracted } of docs) {\n const chunks = (extracted as Chunk[] | undefined) ?? [];\n if (chunks.length === 0) {\n if (!added.has(path)) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ?', [[path]]);\n continue;\n }\n const identified = chunks.map((chunk) => identifyChunkText(chunk.text));\n const kept = new Set<number>();\n if (!added.has(path)) {\n let cursor: number | undefined;\n for (;;) {\n let page: ChunkRow[];\n if (cursor === undefined) {\n firstPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n page = (await firstPage.all(path)) as ChunkRow[];\n } else {\n nextPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? AND chunk > ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n page = (await nextPage.all(path, cursor)) as ChunkRow[];\n }\n const removed: unknown[][] = [];\n const moved: unknown[][] = [];\n for (const old of page) {\n const index = Number(old.chunk);\n const chunk = chunks[index];\n if (!chunk || old.content_identity !== identified[index].identity) {\n removed.push([path, index]);\n } else {\n kept.add(index);\n if (Number(old.start_line) !== chunk.startLine || Number(old.end_line) !== chunk.endLine) moved.push([chunk.startLine, chunk.endLine, path, index]);\n }\n }\n if (removed.length > 0) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ? AND chunk = ?', removed);\n if (moved.length > 0) await db.runBatch('UPDATE embeddings SET start_line = ?, end_line = ? WHERE \"path\" = ? AND chunk = ?', moved);\n if (page.length < CHUNK_LOOKUP_PAGE) break;\n cursor = Number(page[page.length - 1].chunk);\n }\n }\n const before = rows.length;\n chunks.forEach((chunk, index) => {\n if (!kept.has(index)) rows.push([path, index, chunk.startLine, chunk.endLine, identified[index].identity]);\n });\n // No new pending rows means no retained handoff. Matching pending rows can use\n // embedPending's exact indexed-source fallback, including after repeated store calls.\n if (rows.length > before) texts.set(path, identified);\n }\n // scale and vector are written by the embed pass, not here, so they are not in the column list\n // and take the table's own default on the append path.\n await appendRows(db, 'embeddings', ['path', 'chunk', 'start_line', 'end_line', 'content_identity'], 'INSERT INTO embeddings (\"path\", chunk, start_line, end_line, content_identity, scale, vector) VALUES (?, ?, ?, ?, ?, NULL, NULL)', rows);\n // Hand off only paths with new pending rows; missing entries use the indexed source.\n stashChunkText(db, texts);\n },\n enabledForFile(_cfg, file) {\n return file.embed;\n },\n // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n // a model can be configured with no preset yet using vectors.\n signature(cfg) {\n const e = embedConfig(cfg);\n if (!e) return 'embed:off';\n // Weight identity from identity.ts, no network: a static model's resolved sha or local\n // size+mtime, appended once known so changed weights re-embed.\n const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n const endpoint = embedEndpointIdentity(e.provider, e.url);\n const endpointPart = endpoint === undefined ? '' : `:endpoint=${endpoint}`;\n return `embed:${e.provider}:${e.model}${endpointPart}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n },\n};\n"],"names":["embed","CHUNK_LOOKUP_PAGE","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","chunkFromBlocks","undefined","targetTokens","map","c","startLine","endLine","text","name","schema","db","exec","extract","raw","_data","cfg","parse","countLines","remove","paths","delta","vanishedSet","vanished","Set","path","has","length","runBatch","store","docs","rows","texts","added","firstPage","nextPage","Map","extracted","chunks","identified","kept","cursor","page","removed","moved","old","index","chunk","before","identifyChunkText","prepare","all","Number","content_identity","identity","push","add","start_line","end_line","forEach","set","appendRows","stashChunkText","enabledForFile","_cfg","file","signature","e","embedConfig","provider","modelIdentity","model","chunkVersion","CHUNK_VERSION","endpoint","embedEndpointIdentity","url","endpointPart"],"mappings":";;;;+BAoCaA;;;eAAAA;;;uBAnCyC;wBAC1B;0BACU;yBACY;0BACpB;6BACH;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQ3B,IAAMC,oBAAoB;AAQ1B,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C;QAAEC,SAAAA,iEAAS,GAAGC;IACxG,IAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOC,IAAAA,wBAAe,EAACX,QAAQC,MAAMG,gBAAgBQ,YAAY;QAAEC,cAAcT;IAAY,IAAIQ,WAAWE,GAAG,CAAC,SAACC;eAAO;YACtHC,WAAWD,EAAEC,SAAS,GAAGb;YACzBc,SAASF,EAAEE,OAAO,GAAGd;YACrBe,MAAMb,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAW,OAAPU,EAAEG,IAAI,IAAKH,EAAEG,IAAI;QAChD;;AACF;AAEO,IAAMrB,QAAiB;IAC5BsB,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAU2B,IAAAA,cAAK,EAAC1B,OAAOA,MAAMC,QAAQ0B,IAAAA,yBAAU,EAACJ,OAAOI,IAAAA,yBAAU,EAAC3B,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAK7B,KAAK,cAAV6B,iCAAAA,WAAYtB,WAAW;IAClH;IACMyB,QAAN,SAAMA,OAAOR,EAAE,EAAES,KAAK,EAAEC,KAAK;;gBACrBC,aACAC;;;;wBADAD,cAAc,IAAIE,IAAIH,MAAME,QAAQ;wBACpCA,WAAWH,MAAMtB,MAAM,CAAC,SAAC2B;mCAASH,YAAYI,GAAG,CAACD;;wBACxD,IAAIF,SAASI,MAAM,KAAK,GAAG;;;wBAC3B;;4BAAMhB,GAAGiB,QAAQ,CACf,2CACAL,SAASnB,GAAG,CAAC,SAACqB;uCAAS;oCAACA;iCAAK;;;;wBAF/B;;;;;;QAIF;;IAGMI,OAFN,8FAA8F;IAC9F,qFAAqF;IACrF,SAAMA,MAAMlB,EAAE,EAAEmB,IAAI,EAAET,KAAK;;gBACnBU,MACAC,OACAC,OAEFC,WACAC,UACC,2BAAA,mBAAA,uBAAA,WAAA;;;;wBANCJ;wBACAC,QAAQ,IAAII;wBACZH,QAAQ,IAAIT,IAAIH,MAAMY,KAAK;wBAI5B,kCAAA,2BAAA;;;;;;;;;;6CAAQR,MAAMY,WACXC,QAKAC,YACAC,MAEAC,QAEEC,mBAQEC,SACAC,OACD,2BAAA,mBAAA,gBAAA,WAAA,QAAMC,KACHC,OACAC,OAcNC;;;;sDArCH,aAAQvB,mBAAAA,MAAMY,wBAAAA;wCACXC,SAAUD,sBAAAA,uBAAAA;6CACZC,CAAAA,OAAOX,MAAM,KAAK,CAAA,GAAlBW;;;;6CACE,CAACL,MAAMP,GAAG,CAACD,OAAX;;;;wCAAkB;;4CAAMd,GAAGiB,QAAQ,CAAC;;oDAA6CH;;;;;wCAA/D;;;wCACtB;;4CAAA;;;wCAEIc,aAAaD,OAAOlC,GAAG,CAAC,SAAC2C;mDAAUE,IAAAA,4BAAiB,EAACF,MAAMvC,IAAI;;wCAC/DgC,OAAO,IAAIhB;6CACb,CAACS,MAAMP,GAAG,CAACD,OAAX;;;;wCACEgB,SAAAA,KAAAA;;;wCAEEC,OAAAA,KAAAA;6CACAD,CAAAA,WAAWvC,SAAQ,GAAnBuC;;;;8CACFP,sBAAAA;;;;+CAAAA;;;;;;wCAAc;;4CAAMvB,GAAGuC,OAAO,CAAC,AAAC,8GAA+H,OAAlB9D;;;+CAA7I8C,YAAc;;;;wCACN;;4CAAMA,UAAUiB,GAAG,CAAC1B;;;wCAA5BiB,OAAQ;;;;;;8CAERP,qBAAAA;;;;gDAAAA;;;;;;wCAAa;;4CAAMxB,GAAGuC,OAAO,CAAC,AAAC,4HAA6I,OAAlB9D;;;gDAA1J+C,WAAa;;;;wCACL;;4CAAMA,SAASgB,GAAG,CAAC1B,MAAMgB;;;wCAAjCC,OAAQ;;;wCAEJC;wCACAC;wCACD,kCAAA,2BAAA;;4CAAL,IAAK,YAAaF,2BAAb,6BAAA,SAAA,yBAAA,iCAAmB;gDAAbG,MAAN;gDACGC,QAAQM,OAAOP,IAAIE,KAAK;gDACxBA,QAAQT,MAAM,CAACQ,MAAM;gDAC3B,IAAI,CAACC,SAASF,IAAIQ,gBAAgB,KAAKd,UAAU,CAACO,MAAM,CAACQ,QAAQ,EAAE;oDACjEX,QAAQY,IAAI;wDAAE9B;wDAAMqB;;gDACtB,OAAO;oDACLN,KAAKgB,GAAG,CAACV;oDACT,IAAIM,OAAOP,IAAIY,UAAU,MAAMV,MAAMzC,SAAS,IAAI8C,OAAOP,IAAIa,QAAQ,MAAMX,MAAMxC,OAAO,EAAEqC,MAAMW,IAAI;wDAAER,MAAMzC,SAAS;wDAAEyC,MAAMxC,OAAO;wDAAEkB;wDAAMqB;;gDAC9I;4CACF;;4CATK;4CAAA;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;6CAUDH,CAAAA,QAAQhB,MAAM,GAAG,CAAA,GAAjBgB;;;;wCAAoB;;4CAAMhC,GAAGiB,QAAQ,CAAC,yDAAyDe;;;wCAA3E;;;6CACpBC,CAAAA,MAAMjB,MAAM,GAAG,CAAA,GAAfiB;;;;wCAAkB;;4CAAMjC,GAAGiB,QAAQ,CAAC,qFAAqFgB;;;wCAAvG;;;wCACtB,IAAIF,KAAKf,MAAM,GAAGvC,mBAAmB;;;;wCACrCqD,SAASW,OAAOV,IAAI,CAACA,KAAKf,MAAM,GAAG,EAAE,CAACoB,KAAK;;;;;;;;wCAGzCC,SAASjB,KAAKJ,MAAM;wCAC1BW,OAAOqB,OAAO,CAAC,SAACZ,OAAOD;4CACrB,IAAI,CAACN,KAAKd,GAAG,CAACoB,QAAQf,KAAKwB,IAAI,CAAC;gDAAC9B;gDAAMqB;gDAAOC,MAAMzC,SAAS;gDAAEyC,MAAMxC,OAAO;gDAAEgC,UAAU,CAACO,MAAM,CAACQ,QAAQ;6CAAC;wCAC3G;wCACA,+EAA+E;wCAC/E,sFAAsF;wCACtF,IAAIvB,KAAKJ,MAAM,GAAGqB,QAAQhB,MAAM4B,GAAG,CAACnC,MAAMc;;;;;;wBAC5C;wBA5CK,YAA6BT;;;+BAA7B,6BAAA,QAAA;;;;;;;;;;;;wBAAA;;;;;;;;;;;;wBAAA;wBAAA;;;;;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;;wBA6CL,+FAA+F;wBAC/F,uDAAuD;wBACvD;;4BAAM+B,IAAAA,oBAAU,EAAClD,IAAI;gCAAe;gCAAQ;gCAAS;gCAAc;gCAAY;+BAAqB,oIAAoIoB;;;wBAAxO;wBACA,qFAAqF;wBACrF+B,IAAAA,yBAAc,EAACnD,IAAIqB;;;;;;QACrB;;IACA+B,gBAAAA,SAAAA,eAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAK9E,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9D+E,WAAAA,SAAAA,UAAUlD,GAAG;QACX,IAAMmD,IAAIC,IAAAA,qBAAW,EAACpD;QACtB,IAAI,CAACmD,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,IAAMb,WAAWa,EAAEE,QAAQ,KAAK,WAAWC,IAAAA,yBAAa,EAACH,EAAEI,KAAK,IAAIrE;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,IAAMsE,eAAeL,EAAEzE,WAAW,KAAKQ,YAAY,AAAC,GAAmBiE,OAAjBM,sBAAa,EAAC,KAAiB,OAAdN,EAAEzE,WAAW,IAAK+E,sBAAa;QACtG,IAAMC,WAAWC,IAAAA,iCAAqB,EAACR,EAAEE,QAAQ,EAAEF,EAAES,GAAG;QACxD,IAAMC,eAAeH,aAAaxE,YAAY,KAAK,AAAC,aAAqB,OAATwE;QAChE,OAAO,AAAC,SAAsBP,OAAdA,EAAEE,QAAQ,EAAC,KAAaQ,OAAVV,EAAEI,KAAK,EAAmBC,OAAhBK,cAAa,KAAkBvB,OAAfkB,cAA4D,OAA7ClB,aAAapD,YAAY,AAAC,IAAY,OAAToD,YAAa;IACnH;AACF"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { Store } from '../store/types.js';
|
|
2
|
-
interface FindPathOptions {
|
|
2
|
+
export interface FindPathOptions {
|
|
3
3
|
directed?: boolean;
|
|
4
4
|
maxDepth?: number;
|
|
5
5
|
allowed?: Set<string>;
|
|
6
6
|
}
|
|
7
7
|
export declare function findPath(store: Store, from: string, to: string, opts?: FindPathOptions): Promise<string[] | null>;
|
|
8
|
-
export
|
|
8
|
+
export declare function findPathInSnapshot(store: Store, from: string, to: string, opts: FindPathOptions): Promise<string[] | null>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { Store } from '../store/types.js';
|
|
2
|
-
interface FindPathOptions {
|
|
2
|
+
export interface FindPathOptions {
|
|
3
3
|
directed?: boolean;
|
|
4
4
|
maxDepth?: number;
|
|
5
5
|
allowed?: Set<string>;
|
|
6
6
|
}
|
|
7
7
|
export declare function findPath(store: Store, from: string, to: string, opts?: FindPathOptions): Promise<string[] | null>;
|
|
8
|
-
export
|
|
8
|
+
export declare function findPathInSnapshot(store: Store, from: string, to: string, opts: FindPathOptions): Promise<string[] | null>;
|
|
@@ -4,10 +4,18 @@
|
|
|
4
4
|
Object.defineProperty(exports, "__esModule", {
|
|
5
5
|
value: true
|
|
6
6
|
});
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
function _export(target, all) {
|
|
8
|
+
for(var name in all)Object.defineProperty(target, name, {
|
|
9
|
+
enumerable: true,
|
|
10
|
+
get: Object.getOwnPropertyDescriptor(all, name).get
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
_export(exports, {
|
|
14
|
+
get findPath () {
|
|
10
15
|
return findPath;
|
|
16
|
+
},
|
|
17
|
+
get findPathInSnapshot () {
|
|
18
|
+
return findPathInSnapshot;
|
|
11
19
|
}
|
|
12
20
|
});
|
|
13
21
|
var _serialqueryts = require("../lib/serial-query.js");
|
|
@@ -283,10 +291,12 @@ function reconstructPath(store, to) {
|
|
|
283
291
|
function findPath(store, from, to) {
|
|
284
292
|
var opts = arguments.length > 3 && arguments[3] !== void 0 ? arguments[3] : {};
|
|
285
293
|
return (0, _serialqueryts.serialQuery)(store, function() {
|
|
286
|
-
return
|
|
294
|
+
return store.transaction(function() {
|
|
295
|
+
return findPathInSnapshot(store, from, to, opts);
|
|
296
|
+
});
|
|
287
297
|
});
|
|
288
298
|
}
|
|
289
|
-
function
|
|
299
|
+
function findPathInSnapshot(store, from, to, opts) {
|
|
290
300
|
return _async_to_generator(function() {
|
|
291
301
|
var _opts_maxDepth, direction, maxDepth, insertStart, stmt, ring, rows;
|
|
292
302
|
return _ts_generator(this, function(_state) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/graph/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\n\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/graph/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\n\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\nexport interface FindPathOptions {\n directed?: boolean;\n maxDepth?: number;\n allowed?: Set<string>;\n}\n\nasync function resetVisited(store: Store): Promise<void> {\n await store.exec('DROP TABLE IF EXISTS temp.visited');\n await store.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');\n}\n\n// Materializes the caller-resolved scope Set into a temp table so it joins like any other\n// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.\nasync function setupAllowed(store: Store, allowed: Set<string>, extra?: string): Promise<void> {\n const all = new Set(allowed);\n if (extra !== undefined) all.add(extra);\n await store.exec('DROP TABLE IF EXISTS temp.allowed_nodes');\n await store.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');\n await store.runBatch(\n 'INSERT INTO allowed_nodes (path) VALUES (?)',\n [...all].map((p) => [p])\n );\n}\n\n// Only resolved links (dst IS NOT NULL) are followed, in either direction.\nfunction candidateSelect(direction: Direction): string {\n const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n if (direction === 'forward') return forward;\n if (direction === 'reverse') return reverse;\n return `${forward} UNION ALL ${reverse}`;\n}\n\n// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against\n// everything already visited, optionally filtered to the allowed set, inserted at the new depth (?2); RETURNING gives what this ring newly reached.\nfunction ringSql(direction: Direction, hasAllowed: boolean): string {\n return `\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (${candidateSelect(direction)}) cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}\nGROUP BY cand.path\nORDER BY cand.path\nRETURNING path`;\n}\n\nasync function reconstructPath(store: Store, to: string): Promise<string[]> {\n const lookup = await store.prepare('SELECT pred FROM visited WHERE path = ?');\n const path: string[] = [];\n let cur: string | null = to;\n while (cur !== null) {\n path.push(cur);\n const row = (await lookup.get(cur)) as { pred: string | null } | undefined;\n cur = row ? row.pred : null;\n }\n return path.reverse();\n}\n\n// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching\n// personalizedRank; null when `to` is unreached within `maxDepth`.\nexport function findPath(store: Store, from: string, to: string, opts: FindPathOptions = {}): Promise<string[] | null> {\n return serialQuery(store, () => store.transaction(() => findPathInSnapshot(store, from, to, opts)));\n}\n\n// Internal entry for the scoped command wrapper, whose transaction also owns endpoint resolution\n// and scope reads. Direct graph callers use findPath() above and receive the same snapshot rule.\nexport async function findPathInSnapshot(store: Store, from: string, to: string, opts: FindPathOptions): Promise<string[] | null> {\n if (from === to) return [from];\n\n const direction: Direction = opts.directed ? 'forward' : 'both';\n const maxDepth = opts.maxDepth ?? Infinity;\n if (maxDepth <= 0) return null;\n\n await resetVisited(store);\n const insertStart = await store.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)');\n await insertStart.run(from);\n if (opts.allowed) await setupAllowed(store, opts.allowed, to);\n\n const stmt = await store.prepare(ringSql(direction, !!opts.allowed));\n for (let ring = 1; ring <= maxDepth; ring++) {\n const rows = (await stmt.all(ring - 1, ring)) as Array<{ path: string }>;\n if (rows.length === 0) break;\n if (rows.some((row) => row.path === to)) return reconstructPath(store, to);\n }\n return null;\n}\n"],"names":["findPath","findPathInSnapshot","resetVisited","store","exec","setupAllowed","allowed","extra","all","Set","undefined","add","runBatch","map","p","candidateSelect","direction","forward","reverse","ringSql","hasAllowed","reconstructPath","to","lookup","path","cur","row","prepare","push","get","pred","from","opts","serialQuery","transaction","maxDepth","insertStart","stmt","ring","rows","directed","Infinity","run","length","some"],"mappings":"AAAA,2FAA2F;AAC3F,oEAAoE;;;;;;;;;;;;QAoEpDA;eAAAA;;QAMMC;eAAAA;;;6BAxEM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAW5B,SAAeC,aAAaC,KAAY;;;;;oBACtC;;wBAAMA,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMC,IAAI,CAAC;;;oBAAjB;;;;;;IACF;;AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,SAAeC,aAAaF,KAAY,EAAEG,OAAoB,EAAEC,KAAc;;YACtEC;;;;oBAAAA,MAAM,IAAIC,IAAIH;oBACpB,IAAIC,UAAUG,WAAWF,IAAIG,GAAG,CAACJ;oBACjC;;wBAAMJ,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMC,IAAI,CAAC;;;oBAAjB;oBACA;;wBAAMD,MAAMS,QAAQ,CAClB,+CACA,AAAC,qBAAGJ,KAAKK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFzB;;;;;;IAIF;;AAEA,2EAA2E;AAC3E,SAASC,gBAAgBC,SAAoB;IAC3C,IAAMC,UAAU;IAChB,IAAMC,UAAU;IAChB,IAAIF,cAAc,WAAW,OAAOC;IACpC,IAAID,cAAc,WAAW,OAAOE;IACpC,OAAO,AAAC,GAAuBA,OAArBD,SAAQ,eAAqB,OAARC;AACjC;AAEA,0FAA0F;AAC1F,oJAAoJ;AACpJ,SAASC,QAAQH,SAAoB,EAAEI,UAAmB;IACxD,OAAO,AAAC,0FAKRA,OAFML,gBAAgBC,YAAW,mFAEuD,OAAxFI,aAAa,wEAAwE,IAAG;AAI1F;AAEA,SAAeC,gBAAgBlB,KAAY,EAAEmB,EAAU;;YAC/CC,QACAC,MACFC,KAGIC;;;;oBALO;;wBAAMvB,MAAMwB,OAAO,CAAC;;;oBAA7BJ,SAAS;oBACTC;oBACFC,MAAqBH;;;yBAClBG,CAAAA,QAAQ,IAAG;;;;oBAChBD,KAAKI,IAAI,CAACH;oBACG;;wBAAMF,OAAOM,GAAG,CAACJ;;;oBAAxBC,MAAO;oBACbD,MAAMC,MAAMA,IAAII,IAAI,GAAG;;;;;;oBAEzB;;wBAAON,KAAKN,OAAO;;;;IACrB;;AAIO,SAASlB,SAASG,KAAY,EAAE4B,IAAY,EAAET,EAAU;QAAEU,OAAAA,iEAAwB,CAAC;IACxF,OAAOC,IAAAA,0BAAW,EAAC9B,OAAO;eAAMA,MAAM+B,WAAW,CAAC;mBAAMjC,mBAAmBE,OAAO4B,MAAMT,IAAIU;;;AAC9F;AAIO,SAAe/B,mBAAmBE,KAAY,EAAE4B,IAAY,EAAET,EAAU,EAAEU,IAAqB;;YAInFA,gBADXhB,WACAmB,UAIAC,aAIAC,MACGC,MACDC;;;;oBAbR,IAAIR,SAAST,IAAI;;;4BAAQS;;;oBAEnBf,YAAuBgB,KAAKQ,QAAQ,GAAG,YAAY;oBACnDL,YAAWH,iBAAAA,KAAKG,QAAQ,cAAbH,4BAAAA,iBAAiBS;oBAClC,IAAIN,YAAY,GAAG;;wBAAO;;oBAE1B;;wBAAMjC,aAAaC;;;oBAAnB;oBACoB;;wBAAMA,MAAMwB,OAAO,CAAC;;;oBAAlCS,cAAc;oBACpB;;wBAAMA,YAAYM,GAAG,CAACX;;;oBAAtB;yBACIC,KAAK1B,OAAO,EAAZ0B;;;;oBAAc;;wBAAM3B,aAAaF,OAAO6B,KAAK1B,OAAO,EAAEgB;;;oBAAxC;;;oBAEL;;wBAAMnB,MAAMwB,OAAO,CAACR,QAAQH,WAAW,CAAC,CAACgB,KAAK1B,OAAO;;;oBAA5D+B,OAAO;oBACJC,OAAO;;;yBAAGA,CAAAA,QAAQH,QAAO;;;;oBAClB;;wBAAME,KAAK7B,GAAG,CAAC8B,OAAO,GAAGA;;;oBAAjCC,OAAQ;oBACd,IAAIA,KAAKI,MAAM,KAAK,GAAG;;;;oBACvB,IAAIJ,KAAKK,IAAI,CAAC,SAAClB;+BAAQA,IAAIF,IAAI,KAAKF;wBAAK;;wBAAOD,gBAAgBlB,OAAOmB;;;;oBAHpCgB;;;;;;oBAKrC;;wBAAO;;;;IACT"}
|
package/dist/cjs/index.d.cts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.js';
|
|
2
|
-
export { mapTree, peek, presetCoverage, search } from './commands/index.js';
|
|
1
|
+
export type { PathOptions, Peek, PeekSection, PresetCoverage, RelatedOptions, RelatedResult, SearchOptions, SearchResult, SearchResultVia, TreeMap, TreeMapField, TreeMapHub, TreeMapRecent } from './commands/index.js';
|
|
2
|
+
export { findPath, mapTree, peek, presetCoverage, relatedNotes, search } from './commands/index.js';
|
|
3
3
|
export type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.js';
|
|
4
4
|
export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
|
|
5
5
|
export type { SenseErrorCode } from './errors.js';
|
package/dist/cjs/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.js';
|
|
2
|
-
export { mapTree, peek, presetCoverage, search } from './commands/index.js';
|
|
1
|
+
export type { PathOptions, Peek, PeekSection, PresetCoverage, RelatedOptions, RelatedResult, SearchOptions, SearchResult, SearchResultVia, TreeMap, TreeMapField, TreeMapHub, TreeMapRecent } from './commands/index.js';
|
|
2
|
+
export { findPath, mapTree, peek, presetCoverage, relatedNotes, search } from './commands/index.js';
|
|
3
3
|
export type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.js';
|
|
4
4
|
export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
|
|
5
5
|
export type { SenseErrorCode } from './errors.js';
|
package/dist/cjs/index.js
CHANGED
|
@@ -38,6 +38,9 @@ _export(exports, {
|
|
|
38
38
|
get clearCache () {
|
|
39
39
|
return _indexts2.clearCache;
|
|
40
40
|
},
|
|
41
|
+
get findPath () {
|
|
42
|
+
return _indexts.findPath;
|
|
43
|
+
},
|
|
41
44
|
get initConfig () {
|
|
42
45
|
return _indexts1.initConfig;
|
|
43
46
|
},
|
|
@@ -62,6 +65,9 @@ _export(exports, {
|
|
|
62
65
|
get printRows () {
|
|
63
66
|
return _outputts.printRows;
|
|
64
67
|
},
|
|
68
|
+
get relatedNotes () {
|
|
69
|
+
return _indexts.relatedNotes;
|
|
70
|
+
},
|
|
65
71
|
get runWatch () {
|
|
66
72
|
return _watchts.runWatch;
|
|
67
73
|
},
|
package/dist/cjs/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { BuildOptions, BuildResult, OpenOptions, OpenResult, Stages } from './store/index.ts';\nexport { buildIndex as build, CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","build","buildIndex","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAWzCA;eAAAA,0BAAgB;;QANrCC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIIC;eAAdC,oBAAU;;QAA6BC;eAAAA,oBAAU;;
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { PathOptions, Peek, PeekSection, PresetCoverage, RelatedOptions, RelatedResult, SearchOptions, SearchResult, SearchResultVia, TreeMap, TreeMapField, TreeMapHub, TreeMapRecent } from './commands/index.ts';\nexport { findPath, mapTree, peek, presetCoverage, relatedNotes, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { BuildOptions, BuildResult, OpenOptions, OpenResult, Stages } from './store/index.ts';\nexport { buildIndex as build, CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","build","buildIndex","clearCache","findPath","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","relatedNotes","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAWzCA;eAAAA,0BAAgB;;QANrCC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIIC;eAAdC,oBAAU;;QAA6BC;eAAAA,oBAAU;;QARjDC;eAAAA,iBAAQ;;QAESC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAF7BC;eAAAA,gBAAO;;QAEwBC;eAAAA,uBAAa;;QAMUC;eAAbC,mBAAS;;QARzCC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAMvCC;eAAAA,mBAAS;;QANgCC;eAAAA,qBAAY;;QAYrDC;eAAAA,iBAAQ;;QAZ+CC;eAAAA,eAAM;;;uBAAQ;wBAEyD;wBAE5G;wBAED;wBAE2D;uBAI5D"}
|