sensemaking 0.25.1 → 0.27.0

package/dist/cjs/commands/related.js

@@ -9,6 +9,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get RELATED_DEFAULT_K () {
+        return RELATED_DEFAULT_K;
+    },
     get relatedNotes () {
         return relatedNotes;
     },
@@ -23,6 +26,7 @@ var _errorsts = require("../errors.js");
 var _serialqueryts = require("../lib/serial-query.js");
 var _peekts = require("./peek.js");
 var _scopets = require("./scope.js");
+var _snapshotts = require("./snapshot.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -175,6 +179,7 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
+var RELATED_DEFAULT_K = 5;
 function resolveRelatedSeed(store, pathArg) {
     return _async_to_generator(function() {
         var paths;
@@ -202,111 +207,139 @@ function resolveRelatedSeed(store, pathArg) {
         });
     })();
 }
-function relatedNotes(store, cfg, pathArg, overrides, k) {
+function relatedNotes(store, cfg, pathArg) {
+    var options = arguments.length > 3 && arguments[3] !== void 0 ? arguments[3] : {};
     return (0, _serialqueryts.serialQuery)(store, function() {
-        return relatedIndexed(store, cfg, pathArg, overrides, k);
+        var _options_k;
+        return relatedIndexed(store, cfg, pathArg, options, (_options_k = options.k) !== null && _options_k !== void 0 ? _options_k : RELATED_DEFAULT_K);
     });
 }
 function relatedIndexed(store, cfg, pathArg, overrides, k) {
     return _async_to_generator(function() {
-        var path, outbound, backlinks, exclude, effective, e, allowed;
+        var effective, e;
         return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    return [
-                        4,
-                        resolveRelatedSeed(store, pathArg)
-                    ];
-                case 1:
-                    path = _state.sent();
-                    return [
-                        4,
-                        store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')
-                    ];
-                case 2:
-                    return [
-                        4,
-                        _state.sent().all(path)
-                    ];
-                case 3:
-                    outbound = _state.sent().map(function(r) {
-                        return r.dst;
-                    });
-                    return [
-                        4,
-                        store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')
-                    ];
-                case 4:
-                    return [
-                        4,
-                        _state.sent().all(path)
-                    ];
-                case 5:
-                    backlinks = _state.sent().map(function(r) {
-                        return r.src;
-                    });
-                    exclude = new Set([
-                        path
-                    ].concat(_to_consumable_array(outbound), _to_consumable_array(backlinks)));
-                    // Vectors are the only signal `related` has, so every way of not having them is an error
-                    // naming the cause; an empty result means one thing, nothing near in meaning.
-                    effective = (0, _indexts.resolveSearch)(cfg, overrides);
-                    if (!(0, _indexts.embedEnabled)(cfg)) {
-                        throw new _errorsts.SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an "embed" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');
-                    }
-                    // search gates on the same signal, so a preset's declared signals mean one thing: without
-                    // this check an overlapping vectors-on preset would answer for a scope that declined them.
-                    if (effective.signals.vectors === undefined) {
-                        throw new _errorsts.SenseError('PRESET_NOT_SEMANTIC', 'preset "'.concat(effective.presetName, '" has no "vectors" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add "vectors" to that preset\'s signals'));
-                    }
-                    // A downloadable HF id can be fetched by preparation. Only a local path with missing files
-                    // can never fetch itself.
-                    e = (0, _indexts.embedConfig)(cfg); // embedEnabled(cfg) above guarantees this is set
-                    if ((0, _storets.localModelMissing)(e)) {
-                        throw new _errorsts.SenseError('EMBED_MODEL_MISSING', 'related ranks notes by meaning, so it needs the embedding model, but the local model path "'.concat(e.model, '" is missing ').concat(_storets.MODEL_FILENAMES, "; point embed.model at a directory containing them (search still works without it, on words and links)"));
-                    }
-                    return [
-                        4,
-                        (0, _scopets.scopedPaths)(store, cfg, overrides)
-                    ];
-                case 6:
-                    allowed = _state.sent();
-                    // The seed may sit outside a narrowed candidate scope; both it and every eligible result must
-                    // be ready, while pending vectors in unrelated notes do not block this operation.
-                    return [
-                        4,
-                        (0, _queryts.ensureDocumentEmbeddings)(store, cfg, new Set(_to_consumable_array(allowed).concat([
-                            path
-                        ])))
-                    ];
-                case 7:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _queryts.hasEmbedding)(store, path)
-                    ];
-                case 8:
-                    if (!_state.sent()) {
-                        throw new _errorsts.SenseError('NOTE_NOT_EMBEDDED', "".concat(path, " has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning"));
-                    }
-                    return [
-                        4,
-                        (0, _scopets.scopeHasEmbeddings)(store, cfg, allowed)
-                    ];
-                case 9:
-                    if (!_state.sent()) return [
-                        2,
-                        []
-                    ];
-                    return [
-                        2,
-                        (0, _queryts.similarNotes)(store, cfg, path, {
-                            exclude: exclude,
-                            allowed: allowed,
-                            k: k
-                        })
-                    ];
+            if (!Number.isFinite(k) || !Number.isInteger(k) || k <= 0) {
+                throw new _errorsts.SenseError('SEARCH_OPTION_INVALID', 'related option "k" must be a positive finite integer, got '.concat(String(k)));
+            }
+            // Vectors are the only signal `related` has, so every way of not having them is an error
+            // naming the cause; an empty result means one thing, nothing near in meaning.
+            effective = (0, _indexts.resolveSearch)(cfg, overrides);
+            if (!(0, _indexts.embedEnabled)(cfg)) {
+                throw new _errorsts.SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an "embed" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');
             }
+            // search gates on the same signal, so a preset's declared signals mean one thing: without
+            // this check an overlapping vectors-on preset would answer for a scope that declined them.
+            if (effective.signals.vectors === undefined) {
+                throw new _errorsts.SenseError('PRESET_NOT_SEMANTIC', 'preset "'.concat(effective.presetName, '" has no "vectors" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add "vectors" to that preset\'s signals'));
+            }
+            // A downloadable HF id can be fetched by preparation. Only a local path with missing files
+            // can never fetch itself.
+            e = (0, _indexts.embedConfig)(cfg); // embedEnabled(cfg) above guarantees this is set
+            if ((0, _storets.localModelMissing)(e)) {
+                throw new _errorsts.SenseError('EMBED_MODEL_MISSING', 'related ranks notes by meaning, so it needs the embedding model, but the local model path "'.concat(e.model, '" is missing ').concat(_storets.MODEL_FILENAMES, "; point embed.model at a directory containing them (search still works without it, on words and links)"));
+            }
+            return [
+                2,
+                store.transaction(function() {
+                    return _async_to_generator(function() {
+                        var current, path, outbound, backlinks, exclude, allowed;
+                        return _ts_generator(this, function(_state) {
+                            switch(_state.label){
+                                case 0:
+                                    return [
+                                        4,
+                                        (0, _snapshotts.assertQuerySnapshot)(store, cfg)
+                                    ];
+                                case 1:
+                                    _state.sent();
+                                    // Repeat effective preset validation inside the snapshot that supplies its durable scope.
+                                    current = (0, _indexts.resolveSearch)(cfg, overrides);
+                                    if (current.signals.vectors === undefined) {
+                                        throw new _errorsts.SenseError('PRESET_NOT_SEMANTIC', 'preset "'.concat(current.presetName, '" has no "vectors" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add "vectors" to that preset\'s signals'));
+                                    }
+                                    return [
+                                        4,
+                                        resolveRelatedSeed(store, pathArg)
+                                    ];
+                                case 2:
+                                    path = _state.sent();
+                                    return [
+                                        4,
+                                        store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')
+                                    ];
+                                case 3:
+                                    return [
+                                        4,
+                                        _state.sent().all(path)
+                                    ];
+                                case 4:
+                                    outbound = _state.sent().map(function(r) {
+                                        return r.dst;
+                                    });
+                                    return [
+                                        4,
+                                        store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')
+                                    ];
+                                case 5:
+                                    return [
+                                        4,
+                                        _state.sent().all(path)
+                                    ];
+                                case 6:
+                                    backlinks = _state.sent().map(function(r) {
+                                        return r.src;
+                                    });
+                                    exclude = new Set([
+                                        path
+                                    ].concat(_to_consumable_array(outbound), _to_consumable_array(backlinks)));
+                                    return [
+                                        4,
+                                        (0, _scopets.scopedPaths)(store, cfg, overrides)
+                                    ];
+                                case 7:
+                                    allowed = _state.sent();
+                                    // The seed may sit outside a narrowed candidate scope; both it and every eligible result must
+                                    // be ready, while pending vectors in unrelated notes do not block this operation.
+                                    return [
+                                        4,
+                                        (0, _queryts.ensureDocumentEmbeddings)(store, cfg, new Set(_to_consumable_array(allowed).concat([
+                                            path
+                                        ])))
+                                    ];
+                                case 8:
+                                    _state.sent();
+                                    return [
+                                        4,
+                                        (0, _queryts.hasEmbedding)(store, path)
+                                    ];
+                                case 9:
+                                    if (!_state.sent()) {
+                                        throw new _errorsts.SenseError('NOTE_NOT_EMBEDDED', "".concat(path, " has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning"));
+                                    }
+                                    return [
+                                        4,
+                                        (0, _scopets.scopeHasEmbeddings)(store, cfg, allowed)
+                                    ];
+                                case 10:
+                                    if (!_state.sent()) return [
+                                        2,
+                                        []
+                                    ];
+                                    return [
+                                        2,
+                                        (0, _scopets.withMaterializedVectorScope)(store, allowed, function() {
+                                            return (0, _queryts.similarNotes)(store, cfg, path, {
+                                                exclude: exclude,
+                                                allowed: allowed,
+                                                k: k
+                                            });
+                                        })
+                                    ];
+                            }
+                        });
+                    })();
+                })
+            ];
         });
     })();
 }

package/dist/cjs/commands/related.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedConfig, embedEnabled, resolveSearch } from '../config/index.ts';\nimport { ensureDocumentEmbeddings, hasEmbedding, similarNotes } from '../embed/query.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings } from './scope.ts';\n\nexport async function resolveRelatedSeed(store: Store, pathArg: string): Promise<string> {\n  const paths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((row) => row.path);\n  return resolveNote(paths, pathArg);\n}\n\n// Notes most similar by cosine, excluding self and everything already linked either way.\n// Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.\nexport function relatedNotes(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  return serialQuery(store, () => relatedIndexed(store, cfg, pathArg, overrides, k));\n}\n\nasync function relatedIndexed(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  const path = await resolveRelatedSeed(store, pathArg);\n\n  const outbound = ((await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)) as Array<{ dst: string }>).map((r) => r.dst);\n  const backlinks = ((await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)) as Array<{ src: string }>).map((r) => r.src);\n  const exclude = new Set([path, ...outbound, ...backlinks]);\n\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause; an empty result means one thing, nothing near in meaning.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');\n  }\n  // search gates on the same signal, so a preset's declared signals mean one thing: without\n  // this check an overlapping vectors-on preset would answer for a scope that declined them.\n  if (effective.signals.vectors === undefined) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n  }\n  // A downloadable HF id can be fetched by preparation. Only a local path with missing files\n  // can never fetch itself.\n  const e = embedConfig(cfg); // embedEnabled(cfg) above guarantees this is set\n  if (localModelMissing(e)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);\n  }\n  const allowed = await scopedPaths(store, cfg, overrides);\n  // The seed may sit outside a narrowed candidate scope; both it and every eligible result must\n  // be ready, while pending vectors in unrelated notes do not block this operation.\n  await ensureDocumentEmbeddings(store, cfg, new Set([...allowed, path]));\n  if (!(await hasEmbedding(store, path))) {\n    throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n  }\n  if (!(await scopeHasEmbeddings(store, cfg, allowed))) return [];\n  return similarNotes(store, cfg, path, { exclude, allowed, k });\n}\n"],"names":["relatedNotes","resolveRelatedSeed","store","pathArg","paths","prepare","all","map","row","path","resolveNote","cfg","overrides","k","serialQuery","relatedIndexed","outbound","backlinks","exclude","effective","e","allowed","r","dst","src","Set","resolveSearch","embedEnabled","SenseError","signals","vectors","undefined","presetName","embedConfig","localModelMissing","MODEL_FILENAMES","model","scopedPaths","ensureDocumentEmbeddings","hasEmbedding","scopeHasEmbeddings","similarNotes"],"mappings":";;;;;;;;;;;QAiBgBA;eAAAA;;QAPMC;eAAAA;;;uBATmC;uBACY;uBAClB;wBACxB;6BACC;sBAEA;uBACoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEzC,SAAeA,mBAAmBC,KAAY,EAAEC,OAAe;;YAC9DC;;;;oBAAiB;;wBAAMF,MAAMG,OAAO,CAAC;;;oBAA3B;;wBAAO,cAAuDC,GAAG;;;oBAA3EF,QAAQ,AAAE,cAAiGG,GAAG,CAAC,SAACC;+BAAQA,IAAIC,IAAI;;oBACtI;;wBAAOC,IAAAA,mBAAW,EAACN,OAAOD;;;;IAC5B;;AAIO,SAASH,aAAaE,KAAY,EAAES,GAAmB,EAAER,OAAe,EAAES,SAA0B,EAAEC,CAAS;IACpH,OAAOC,IAAAA,0BAAW,EAACZ,OAAO;eAAMa,eAAeb,OAAOS,KAAKR,SAASS,WAAWC;;AACjF;AAEA,SAAeE,eAAeb,KAAY,EAAES,GAAmB,EAAER,OAAe,EAAES,SAA0B,EAAEC,CAAS;;YAC/GJ,MAEAO,UACAC,WACAC,SAIAC,WAWAC,GAIAC;;;;oBAvBO;;wBAAMpB,mBAAmBC,OAAOC;;;oBAAvCM,OAAO;oBAEa;;wBAAMP,MAAMG,OAAO,CAAC;;;oBAA3B;;wBAAO,cAAwGC,GAAG,CAACG;;;oBAAhIO,WAAW,AAAE,cAAqJT,GAAG,CAAC,SAACe;+BAAMA,EAAEC,GAAG;;oBAC7J;;wBAAMrB,MAAMG,OAAO,CAAC;;;oBAA3B;;wBAAO,cAAoFC,GAAG,CAACG;;;oBAA7GQ,YAAY,AAAE,cAAiIV,GAAG,CAAC,SAACe;+BAAMA,EAAEE,GAAG;;oBAC/JN,UAAU,IAAIO,IAAI;wBAAChB;sBAAD,OAAO,qBAAGO,WAAU,qBAAGC;oBAE/C,yFAAyF;oBACzF,8EAA8E;oBACxEE,YAAYO,IAAAA,sBAAa,EAACf,KAAKC;oBACrC,IAAI,CAACe,IAAAA,qBAAY,EAAChB,MAAM;wBACtB,MAAM,IAAIiB,oBAAU,CAAC,kBAAkB;oBACzC;oBACA,0FAA0F;oBAC1F,2FAA2F;oBAC3F,IAAIT,UAAUU,OAAO,CAACC,OAAO,KAAKC,WAAW;wBAC3C,MAAM,IAAIH,oBAAU,CAAC,uBAAuB,AAAC,WAA+B,OAArBT,UAAUa,UAAU,EAAC;oBAC9E;oBACA,2FAA2F;oBAC3F,0BAA0B;oBACpBZ,IAAIa,IAAAA,oBAAW,EAACtB,MAAM,iDAAiD;oBAC7E,IAAIuB,IAAAA,0BAAiB,EAACd,IAAI;wBACxB,MAAM,IAAIQ,oBAAU,CAAC,uBAAuB,AAAC,8FAAoHO,OAAvBf,EAAEgB,KAAK,EAAC,iBAA+B,OAAhBD,wBAAe,EAAC;oBACnL;oBACgB;;wBAAME,IAAAA,oBAAW,EAACnC,OAAOS,KAAKC;;;oBAAxCS,UAAU;oBAChB,8FAA8F;oBAC9F,kFAAkF;oBAClF;;wBAAMiB,IAAAA,iCAAwB,EAACpC,OAAOS,KAAK,IAAIc,IAAI,AAAC,qBAAGJ;4BAASZ;;;;oBAAhE;oBACM;;wBAAM8B,IAAAA,qBAAY,EAACrC,OAAOO;;;oBAAhC,IAAI,CAAE,eAAkC;wBACtC,MAAM,IAAImB,oBAAU,CAAC,qBAAqB,AAAC,GAAO,OAALnB,MAAK;oBACpD;oBACM;;wBAAM+B,IAAAA,2BAAkB,EAACtC,OAAOS,KAAKU;;;oBAA3C,IAAI,CAAE,eAAgD;;;;oBACtD;;wBAAOoB,IAAAA,qBAAY,EAACvC,OAAOS,KAAKF,MAAM;4BAAES,SAAAA;4BAASG,SAAAA;4BAASR,GAAAA;wBAAE;;;;IAC9D"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedConfig, embedEnabled, resolveSearch } from '../config/index.ts';\nimport { ensureDocumentEmbeddings, hasEmbedding, similarNotes } from '../embed/query.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings, withMaterializedVectorScope } from './scope.ts';\nimport { assertQuerySnapshot } from './snapshot.ts';\n\nexport const RELATED_DEFAULT_K = 5;\n\n/**\n * Options for {@link relatedNotes}. Scope fields filter result candidates, not the seed note;\n * `k` defaults to 5, matching the CLI.\n */\nexport interface RelatedOptions extends SearchOverrides {\n  k?: number;\n}\n\nexport type RelatedResult = {\n  path: string;\n  similarity: number;\n};\n\nexport async function resolveRelatedSeed(store: Store, pathArg: string): Promise<string> {\n  const paths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((row) => row.path);\n  return resolveNote(paths, pathArg);\n}\n\n/**\n * Returns the notes most similar to an indexed seed, excluding the seed and notes linked either\n * way. The seed resolves across the indexed tree even when it is outside the candidate scope.\n */\nexport function relatedNotes(store: Store, cfg: ResolvedConfig, pathArg: string, options: RelatedOptions = {}): Promise<RelatedResult[]> {\n  return serialQuery(store, () => relatedIndexed(store, cfg, pathArg, options, options.k ?? RELATED_DEFAULT_K));\n}\n\nasync function relatedIndexed(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: RelatedOptions, k: number): Promise<RelatedResult[]> {\n  if (!Number.isFinite(k) || !Number.isInteger(k) || k <= 0) {\n    throw new SenseError('SEARCH_OPTION_INVALID', `related option \"k\" must be a positive finite integer, got ${String(k)}`);\n  }\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause; an empty result means one thing, nothing near in meaning.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');\n  }\n  // search gates on the same signal, so a preset's declared signals mean one thing: without\n  // this check an overlapping vectors-on preset would answer for a scope that declined them.\n  if (effective.signals.vectors === undefined) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n  }\n  // A downloadable HF id can be fetched by preparation. Only a local path with missing files\n  // can never fetch itself.\n  const e = embedConfig(cfg); // embedEnabled(cfg) above guarantees this is set\n  if (localModelMissing(e)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);\n  }\n\n  return store.transaction(async () => {\n    await assertQuerySnapshot(store, cfg);\n    // Repeat effective preset validation inside the snapshot that supplies its durable scope.\n    const current = resolveSearch(cfg, overrides);\n    if (current.signals.vectors === undefined) {\n      throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${current.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n    }\n\n    const path = await resolveRelatedSeed(store, pathArg);\n    const outbound = ((await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)) as Array<{ dst: string }>).map((r) => r.dst);\n    const backlinks = ((await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)) as Array<{ src: string }>).map((r) => r.src);\n    const exclude = new Set([path, ...outbound, ...backlinks]);\n    const allowed = await scopedPaths(store, cfg, overrides);\n    // The seed may sit outside a narrowed candidate scope; both it and every eligible result must\n    // be ready, while pending vectors in unrelated notes do not block this operation.\n    await ensureDocumentEmbeddings(store, cfg, new Set([...allowed, path]));\n    if (!(await hasEmbedding(store, path))) {\n      throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n    }\n    if (!(await scopeHasEmbeddings(store, cfg, allowed))) return [];\n    return withMaterializedVectorScope(store, allowed, () => similarNotes(store, cfg, path, { exclude, allowed, k }));\n  });\n}\n"],"names":["RELATED_DEFAULT_K","relatedNotes","resolveRelatedSeed","store","pathArg","paths","prepare","all","map","row","path","resolveNote","cfg","options","serialQuery","relatedIndexed","k","overrides","effective","e","Number","isFinite","isInteger","SenseError","String","resolveSearch","embedEnabled","signals","vectors","undefined","presetName","embedConfig","localModelMissing","MODEL_FILENAMES","model","transaction","current","outbound","backlinks","exclude","allowed","assertQuerySnapshot","r","dst","src","Set","scopedPaths","ensureDocumentEmbeddings","hasEmbedding","scopeHasEmbeddings","withMaterializedVectorScope","similarNotes"],"mappings":";;;;;;;;;;;QAWaA;eAAAA;;QAwBGC;eAAAA;;QATMC;eAAAA;;;uBAzBmC;uBACY;uBAClB;wBACxB;6BACC;sBAEA;uBACiD;0BACzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7B,IAAMF,oBAAoB;AAe1B,SAAeE,mBAAmBC,KAAY,EAAEC,OAAe;;YAC9DC;;;;oBAAiB;;wBAAMF,MAAMG,OAAO,CAAC;;;oBAA3B;;wBAAO,cAAuDC,GAAG;;;oBAA3EF,QAAQ,AAAE,cAAiGG,GAAG,CAAC,SAACC;+BAAQA,IAAIC,IAAI;;oBACtI;;wBAAOC,IAAAA,mBAAW,EAACN,OAAOD;;;;IAC5B;;AAMO,SAASH,aAAaE,KAAY,EAAES,GAAmB,EAAER,OAAe;QAAES,UAAAA,iEAA0B,CAAC;IAC1G,OAAOC,IAAAA,0BAAW,EAACX,OAAO;YAAmDU;eAA7CE,eAAeZ,OAAOS,KAAKR,SAASS,UAASA,aAAAA,QAAQG,CAAC,cAATH,wBAAAA,aAAab;;AAC5F;AAEA,SAAee,eAAeZ,KAAY,EAAES,GAAmB,EAAER,OAAe,EAAEa,SAAyB,EAAED,CAAS;;YAM9GE,WAWAC;;YAhBN,IAAI,CAACC,OAAOC,QAAQ,CAACL,MAAM,CAACI,OAAOE,SAAS,CAACN,MAAMA,KAAK,GAAG;gBACzD,MAAM,IAAIO,oBAAU,CAAC,yBAAyB,AAAC,6DAAsE,OAAVC,OAAOR;YACpH;YACA,yFAAyF;YACzF,8EAA8E;YACxEE,YAAYO,IAAAA,sBAAa,EAACb,KAAKK;YACrC,IAAI,CAACS,IAAAA,qBAAY,EAACd,MAAM;gBACtB,MAAM,IAAIW,oBAAU,CAAC,kBAAkB;YACzC;YACA,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAIL,UAAUS,OAAO,CAACC,OAAO,KAAKC,WAAW;gBAC3C,MAAM,IAAIN,oBAAU,CAAC,uBAAuB,AAAC,WAA+B,OAArBL,UAAUY,UAAU,EAAC;YAC9E;YACA,2FAA2F;YAC3F,0BAA0B;YACpBX,IAAIY,IAAAA,oBAAW,EAACnB,MAAM,iDAAiD;YAC7E,IAAIoB,IAAAA,0BAAiB,EAACb,IAAI;gBACxB,MAAM,IAAII,oBAAU,CAAC,uBAAuB,AAAC,8FAAoHU,OAAvBd,EAAEe,KAAK,EAAC,iBAA+B,OAAhBD,wBAAe,EAAC;YACnL;YAEA;;gBAAO9B,MAAMgC,WAAW,CAAC;;4BAGjBC,SAKA1B,MACA2B,UACAC,WACAC,SACAC;;;;oCAXN;;wCAAMC,IAAAA,+BAAmB,EAACtC,OAAOS;;;oCAAjC;oCACA,0FAA0F;oCACpFwB,UAAUX,IAAAA,sBAAa,EAACb,KAAKK;oCACnC,IAAImB,QAAQT,OAAO,CAACC,OAAO,KAAKC,WAAW;wCACzC,MAAM,IAAIN,oBAAU,CAAC,uBAAuB,AAAC,WAA6B,OAAnBa,QAAQN,UAAU,EAAC;oCAC5E;oCAEa;;wCAAM5B,mBAAmBC,OAAOC;;;oCAAvCM,OAAO;oCACa;;wCAAMP,MAAMG,OAAO,CAAC;;;oCAA3B;;wCAAO,cAAwGC,GAAG,CAACG;;;oCAAhI2B,WAAW,AAAE,cAAqJ7B,GAAG,CAAC,SAACkC;+CAAMA,EAAEC,GAAG;;oCAC7J;;wCAAMxC,MAAMG,OAAO,CAAC;;;oCAA3B;;wCAAO,cAAoFC,GAAG,CAACG;;;oCAA7G4B,YAAY,AAAE,cAAiI9B,GAAG,CAAC,SAACkC;+CAAMA,EAAEE,GAAG;;oCAC/JL,UAAU,IAAIM,IAAI;wCAACnC;sCAAD,OAAO,qBAAG2B,WAAU,qBAAGC;oCAC/B;;wCAAMQ,IAAAA,oBAAW,EAAC3C,OAAOS,KAAKK;;;oCAAxCuB,UAAU;oCAChB,8FAA8F;oCAC9F,kFAAkF;oCAClF;;wCAAMO,IAAAA,iCAAwB,EAAC5C,OAAOS,KAAK,IAAIiC,IAAI,AAAC,qBAAGL;4CAAS9B;;;;oCAAhE;oCACM;;wCAAMsC,IAAAA,qBAAY,EAAC7C,OAAOO;;;oCAAhC,IAAI,CAAE,eAAkC;wCACtC,MAAM,IAAIa,oBAAU,CAAC,qBAAqB,AAAC,GAAO,OAALb,MAAK;oCACpD;oCACM;;wCAAMuC,IAAAA,2BAAkB,EAAC9C,OAAOS,KAAK4B;;;oCAA3C,IAAI,CAAE,eAAgD;;;;oCACtD;;wCAAOU,IAAAA,oCAA2B,EAAC/C,OAAOqC,SAAS;mDAAMW,IAAAA,qBAAY,EAAChD,OAAOS,KAAKF,MAAM;gDAAE6B,SAAAA;gDAASC,SAAAA;gDAASxB,GAAAA;4CAAE;;;;;oBAChH;;;;IACF"}

package/dist/cjs/commands/scope.d.cts

@@ -7,4 +7,5 @@ export declare function rawScope(store: Store, cfg: ResolvedConfig, overrides: S
 export declare function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>>;
 export declare function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>>;
 export declare function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void>;
+export declare function withMaterializedVectorScope<T>(store: Store, allowed: Set<string>, callback: () => Promise<T>, knownTotal?: number): Promise<T>;
 export declare function setupMapScope(store: Store, paths: Set<string>): Promise<void>;

package/dist/cjs/commands/scope.d.ts

@@ -7,4 +7,5 @@ export declare function rawScope(store: Store, cfg: ResolvedConfig, overrides: S
 export declare function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>>;
 export declare function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>>;
 export declare function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void>;
+export declare function withMaterializedVectorScope<T>(store: Store, allowed: Set<string>, callback: () => Promise<T>, knownTotal?: number): Promise<T>;
 export declare function setupMapScope(store: Store, paths: Set<string>): Promise<void>;

package/dist/cjs/commands/scope.js

@@ -32,11 +32,15 @@ _export(exports, {
     },
     get setupMapScope () {
         return setupMapScope;
+    },
+    get withMaterializedVectorScope () {
+        return withMaterializedVectorScope;
     }
 });
 var _nodepath = require("node:path");
 var _indexts = require("../config/index.js");
 var _columnhintts = require("../output/column-hint.js");
+var _vectorsts = require("../store/vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -448,6 +452,151 @@ function materializeScope(store, table, paths) {
         });
     })();
 }
+function withMaterializedVectorScope(store, allowed, callback, knownTotal) {
+    return _async_to_generator(function() {
+        var reservation, total, stmt, row, err, operationFailed, operationError, result, err1, cleanupFailed, cleanupError, err2;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    reservation = (0, _vectorsts.reserveNativeVectorScope)(store.vectors, allowed);
+                    if (!reservation) return [
+                        2,
+                        callback()
+                    ];
+                    if (allowed.size === 0) {
+                        reservation.invalidate();
+                        return [
+                            2,
+                            callback()
+                        ];
+                    }
+                    total = knownTotal;
+                    if (!(total === undefined)) return [
+                        3,
+                        5
+                    ];
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        4,
+                        ,
+                        5
+                    ]);
+                    return [
+                        4,
+                        store.prepare('SELECT COUNT(*) AS count FROM frontmatter')
+                    ];
+                case 2:
+                    stmt = _state.sent();
+                    return [
+                        4,
+                        stmt.get()
+                    ];
+                case 3:
+                    row = _state.sent();
+                    total = Number(row.count);
+                    return [
+                        3,
+                        5
+                    ];
+                case 4:
+                    err = _state.sent();
+                    reservation.invalidate();
+                    throw err;
+                case 5:
+                    // Materialization only buys work when this is a nonempty true narrowing and the allowed side
+                    // is no larger than its complement. Direct adapter calls and broader scopes retain JS filtering.
+                    if (allowed.size >= total || allowed.size > total - allowed.size) {
+                        reservation.invalidate();
+                        return [
+                            2,
+                            callback()
+                        ];
+                    }
+                    operationFailed = false;
+                    _state.label = 6;
+                case 6:
+                    _state.trys.push([
+                        6,
+                        9,
+                        ,
+                        10
+                    ]);
+                    return [
+                        4,
+                        materializeScope(store, _vectorsts.NATIVE_VECTOR_SCOPE_TABLE, allowed)
+                    ];
+                case 7:
+                    _state.sent();
+                    reservation.activate();
+                    return [
+                        4,
+                        callback()
+                    ];
+                case 8:
+                    result = _state.sent();
+                    return [
+                        3,
+                        10
+                    ];
+                case 9:
+                    err1 = _state.sent();
+                    operationFailed = true;
+                    operationError = err1;
+                    return [
+                        3,
+                        10
+                    ];
+                case 10:
+                    cleanupFailed = false;
+                    _state.label = 11;
+                case 11:
+                    _state.trys.push([
+                        11,
+                        13,
+                        14,
+                        15
+                    ]);
+                    reservation.deactivate();
+                    return [
+                        4,
+                        store.exec("DROP TABLE IF EXISTS ".concat(_vectorsts.NATIVE_VECTOR_SCOPE_TABLE))
+                    ];
+                case 12:
+                    _state.sent();
+                    return [
+                        3,
+                        15
+                    ];
+                case 13:
+                    err2 = _state.sent();
+                    cleanupFailed = true;
+                    cleanupError = err2;
+                    return [
+                        3,
+                        15
+                    ];
+                case 14:
+                    reservation.invalidate();
+                    return [
+                        7
+                    ];
+                case 15:
+                    if (operationFailed && cleanupFailed) throw new AggregateError([
+                        operationError,
+                        cleanupError
+                    ], 'native vector scope operation and cleanup failed');
+                    if (operationFailed) throw operationError;
+                    if (cleanupFailed) throw cleanupError;
+                    return [
+                        2,
+                        result
+                    ];
+            }
+        });
+    })();
+}
 function setupMapScope(store, paths) {
     return _async_to_generator(function() {
         return _ts_generator(this, function(_state) {

package/dist/cjs/commands/scope.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/scope.ts"],"sourcesContent":["import { matchesGlob } from 'node:path';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { anyPresetEmbeds, resolveSearch } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\nimport type { Store } from '../store/types.ts';\n\nexport const INTERNAL_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error']);\n\n// node:path's matchesGlob is experimental (stable behind an unstable-API flag) at the\n// Node >=22.20 engines floor; used here in JS rather than a directory walk in the query path.\nexport function inScope(path: string, include: string[], exclude?: string[]): boolean {\n  if (!include.some((g) => matchesGlob(path, g))) return false;\n  if (exclude?.some((g) => matchesGlob(path, g))) return false;\n  return true;\n}\n\nexport async function scopeHasEmbeddings(store: Store, cfg: ResolvedConfig, scopedPaths: Set<string>): Promise<boolean> {\n  if (!anyPresetEmbeds(cfg)) return false; // the embeddings table doesn't exist at all in this case\n  const stmt = await store.prepare('SELECT DISTINCT \"path\" FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string }>;\n  return rows.some((r) => scopedPaths.has(r.path));\n}\n\n// Scope only (no --where): preset_files for a named preset, JS glob matching for an ad hoc\n// include/exclude override. Shared by scopedPaths() and search()'s candidate-pool sizing.\nexport async function rawScope(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  const { include, exclude } = effective;\n  const adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;\n  if (!adHocScope) {\n    const stmt = await store.prepare('SELECT \"path\" FROM preset_files WHERE preset = ?');\n    return new Set(((await stmt.all(effective.presetName)) as Array<{ path: string }>).map((r) => r.path));\n  }\n  const paths = allPaths ?? (await allFrontmatterPaths(store));\n  return new Set(paths.filter((p) => inScope(p, include, exclude)));\n}\n\nasync function allFrontmatterPaths(store: Store): Promise<string[]> {\n  const stmt = await store.prepare('SELECT \"path\" FROM frontmatter');\n  return ((await stmt.all()) as Array<{ path: string }>).map((r) => r.path);\n}\n\nexport async function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>> {\n  if (!where) return paths;\n  let whereRows: Array<{ path: string }>;\n  try {\n    const stmt = await store.prepare(`SELECT \"path\" FROM frontmatter f WHERE (${where})`);\n    whereRows = (await stmt.all()) as Array<{ path: string }>;\n  } catch (err) {\n    throw columnHint(await store.docs.columns(), err as Error);\n  }\n  const wherePaths = new Set(whereRows.map((r) => r.path));\n  return new Set([...paths].filter((p) => wherePaths.has(p)));\n}\n\n// The scope resolver for non-search commands (path, peek, map): same coverage rule search()\n// applies, then narrowed by the resolved `where`.\nexport async function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  return narrowByWhere(store, await rawScope(store, cfg, overrides), effective.where);\n}\n\n// Materializes a path set into a named temp table (same shape as traverse.ts's allowed_nodes)\n// so a query can join/filter against it cheaply instead of binding a parameter per path.\nexport async function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void> {\n  await store.exec(`DROP TABLE IF EXISTS ${table}`);\n  await store.exec(`CREATE TEMP TABLE ${table} (\"path\" TEXT PRIMARY KEY)`);\n  await store.runBatch(\n    `INSERT INTO ${table} (\"path\") VALUES (?)`,\n    [...paths].map((p) => [p])\n  );\n}\n\nexport async function setupMapScope(store: Store, paths: Set<string>): Promise<void> {\n  await materializeScope(store, '_map_scope', paths);\n}\n"],"names":["INTERNAL_COLUMNS","inScope","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","scopedPaths","setupMapScope","Set","path","include","exclude","some","g","matchesGlob","store","cfg","stmt","rows","anyPresetEmbeds","prepare","all","r","has","overrides","allPaths","effective","adHocScope","paths","resolveSearch","undefined","noExclude","presetName","map","allFrontmatterPaths","filter","p","where","whereRows","err","wherePaths","docs","columns","columnHint","table","exec","runBatch"],"mappings":";;;;;;;;;;;QAMaA;eAAAA;;QAIGC;eAAAA;;QAsDMC;eAAAA;;QAtBAC;eAAAA;;QAjBAC;eAAAA;;QATAC;eAAAA;;QAyCAC;eAAAA;;QAgBAC;eAAAA;;;wBAzEM;uBAEmB;4BACpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGpB,IAAMP,mBAAmB,IAAIQ,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;CAAe;AAI/F,SAASP,QAAQQ,IAAY,EAAEC,OAAiB,EAAEC,OAAkB;IACzE,IAAI,CAACD,QAAQE,IAAI,CAAC,SAACC;eAAMC,IAAAA,qBAAW,EAACL,MAAMI;QAAK,OAAO;IACvD,IAAIF,oBAAAA,8BAAAA,QAASC,IAAI,CAAC,SAACC;eAAMC,IAAAA,qBAAW,EAACL,MAAMI;QAAK,OAAO;IACvD,OAAO;AACT;AAEO,SAAeR,mBAAmBU,KAAY,EAAEC,GAAmB,EAAEV,WAAwB;;YAE5FW,MACAC;;;;oBAFN,IAAI,CAACC,IAAAA,wBAAe,EAACH,MAAM;;wBAAO;uBAAO,yDAAyD;oBACrF;;wBAAMD,MAAMK,OAAO,CAAC;;;oBAA3BH,OAAO;oBACC;;wBAAMA,KAAKI,GAAG;;;oBAAtBH,OAAQ;oBACd;;wBAAOA,KAAKN,IAAI,CAAC,SAACU;mCAAMhB,YAAYiB,GAAG,CAACD,EAAEb,IAAI;;;;;IAChD;;AAIO,SAAeL,SAASW,KAAY,EAAEC,GAAmB,EAAEQ,SAA0B,EAAEC,QAAmB;;YACzGC,WACEhB,SAASC,SACXgB,YAEEV,SAGFW;;;;oBAPAF,YAAYG,IAAAA,sBAAa,EAACb,KAAKQ;oBAC7Bd,UAAqBgB,UAArBhB,SAASC,UAAYe,UAAZf;oBACXgB,aAAaH,UAAUd,OAAO,KAAKoB,aAAaN,UAAUb,OAAO,KAAKmB,aAAaN,UAAUO,SAAS,KAAK;yBAC7G,CAACJ,YAAD;;;;oBACW;;wBAAMZ,MAAMK,OAAO,CAAC;;;oBAA3BH,OAAO;wBACFT;oBAAM;;wBAAMS,KAAKI,GAAG,CAACK,UAAUM,UAAU;;;oBAApD;;wBAAO,IAAA,CAAA,EAAA,MAAIxB;;4BAAM,cAAkEyB,GAAG,CAAC,SAACX;uCAAMA,EAAEb,IAAI;;0BAAC;;;0BAEzFgB,qBAAAA;;;;2BAAAA;;;;;;oBAAa;;wBAAMS,oBAAoBnB;;;2BAA1B;;;oBAArBa;oBACN;;wBAAO,IAAIpB,IAAIoB,MAAMO,MAAM,CAAC,SAACC;mCAAMnC,QAAQmC,GAAG1B,SAASC;;;;;IACzD;;AAEA,SAAeuB,oBAAoBnB,KAAY;;YACvCE;;;;oBAAO;;wBAAMF,MAAMK,OAAO,CAAC;;;oBAA3BH,OAAO;oBACJ;;wBAAMA,KAAKI,GAAG;;;oBAAvB;;wBAAS,cAA8CY,GAAG,CAAC,SAACX;mCAAMA,EAAEb,IAAI;;;;;IAC1E;;AAEO,SAAeN,cAAcY,KAAY,EAAEa,KAAkB,EAAES,KAAyB;;YAEzFC,WAEIrB,MAECsB,KAGHC;;;;oBARN,IAAI,CAACH,OAAO;;wBAAOT;;;;;;;;;;oBAGJ;;wBAAMb,MAAMK,OAAO,CAAC,AAAC,2CAAgD,OAANiB,OAAM;;;oBAA5EpB,OAAO;oBACA;;wBAAMA,KAAKI,GAAG;;;oBAA3BiB,YAAa;;;;;;oBACNC;oBACU;;wBAAMxB,MAAM0B,IAAI,CAACC,OAAO;;;oBAAzC,MAAMC,wBAAU;wBAAC;wBAA4BJ;;;oBAEzCC,aAAa,IAAIhC,IAAI8B,UAAUL,GAAG,CAAC,SAACX;+BAAMA,EAAEb,IAAI;;oBACtD;;wBAAO,IAAID,IAAI,AAAC,qBAAGoB,OAAOO,MAAM,CAAC,SAACC;mCAAMI,WAAWjB,GAAG,CAACa;;;;;IACzD;;AAIO,SAAe9B,YAAYS,KAAY,EAAEC,GAAmB,EAAEQ,SAA0B;;YACvFE;;;;oBAAAA,YAAYG,IAAAA,sBAAa,EAACb,KAAKQ;;wBAChBT;;oBAAO;;wBAAMX,SAASW,OAAOC,KAAKQ;;;oBAAvD;;wBAAOrB;4BAAqB;4BAAuCuB,UAAUW,KAAK;;;;;IACpF;;AAIO,SAAenC,iBAAiBa,KAAY,EAAE6B,KAAa,EAAEhB,KAAkB;;;;;oBACpF;;wBAAMb,MAAM8B,IAAI,CAAC,AAAC,wBAA6B,OAAND;;;oBAAzC;oBACA;;wBAAM7B,MAAM8B,IAAI,CAAC,AAAC,qBAA0B,OAAND,OAAM;;;oBAA5C;oBACA;;wBAAM7B,MAAM+B,QAAQ,CAClB,AAAC,eAAoB,OAANF,OAAM,yBACrB,AAAC,qBAAGhB,OAAOK,GAAG,CAAC,SAACG;mCAAM;gCAACA;6BAAE;;;;oBAF3B;;;;;;IAIF;;AAEO,SAAe7B,cAAcQ,KAAY,EAAEa,KAAkB;;;;;oBAClE;;wBAAM1B,iBAAiBa,OAAO,cAAca;;;oBAA5C;;;;;;IACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/scope.ts"],"sourcesContent":["import { matchesGlob } from 'node:path';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { anyPresetEmbeds, resolveSearch } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\nimport type { Store } from '../store/types.ts';\nimport { NATIVE_VECTOR_SCOPE_TABLE, reserveNativeVectorScope } from '../store/vectors.ts';\n\nexport const INTERNAL_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error']);\n\n// node:path's matchesGlob is experimental (stable behind an unstable-API flag) at the\n// Node >=22.20 engines floor; used here in JS rather than a directory walk in the query path.\nexport function inScope(path: string, include: string[], exclude?: string[]): boolean {\n  if (!include.some((g) => matchesGlob(path, g))) return false;\n  if (exclude?.some((g) => matchesGlob(path, g))) return false;\n  return true;\n}\n\nexport async function scopeHasEmbeddings(store: Store, cfg: ResolvedConfig, scopedPaths: Set<string>): Promise<boolean> {\n  if (!anyPresetEmbeds(cfg)) return false; // the embeddings table doesn't exist at all in this case\n  const stmt = await store.prepare('SELECT DISTINCT \"path\" FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string }>;\n  return rows.some((r) => scopedPaths.has(r.path));\n}\n\n// Scope only (no --where): preset_files for a named preset, JS glob matching for an ad hoc\n// include/exclude override. Shared by scopedPaths() and search()'s candidate-pool sizing.\nexport async function rawScope(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  const { include, exclude } = effective;\n  const adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;\n  if (!adHocScope) {\n    const stmt = await store.prepare('SELECT \"path\" FROM preset_files WHERE preset = ?');\n    return new Set(((await stmt.all(effective.presetName)) as Array<{ path: string }>).map((r) => r.path));\n  }\n  const paths = allPaths ?? (await allFrontmatterPaths(store));\n  return new Set(paths.filter((p) => inScope(p, include, exclude)));\n}\n\nasync function allFrontmatterPaths(store: Store): Promise<string[]> {\n  const stmt = await store.prepare('SELECT \"path\" FROM frontmatter');\n  return ((await stmt.all()) as Array<{ path: string }>).map((r) => r.path);\n}\n\nexport async function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>> {\n  if (!where) return paths;\n  let whereRows: Array<{ path: string }>;\n  try {\n    const stmt = await store.prepare(`SELECT \"path\" FROM frontmatter f WHERE (${where})`);\n    whereRows = (await stmt.all()) as Array<{ path: string }>;\n  } catch (err) {\n    throw columnHint(await store.docs.columns(), err as Error);\n  }\n  const wherePaths = new Set(whereRows.map((r) => r.path));\n  return new Set([...paths].filter((p) => wherePaths.has(p)));\n}\n\n// The scope resolver for non-search commands (path, peek, map): same coverage rule search()\n// applies, then narrowed by the resolved `where`.\nexport async function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  return narrowByWhere(store, await rawScope(store, cfg, overrides), effective.where);\n}\n\n// Materializes a path set into a named temp table (same shape as traverse.ts's allowed_nodes)\n// so a query can join/filter against it cheaply instead of binding a parameter per path.\nexport async function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void> {\n  await store.exec(`DROP TABLE IF EXISTS ${table}`);\n  await store.exec(`CREATE TEMP TABLE ${table} (\"path\" TEXT PRIMARY KEY)`);\n  await store.runBatch(\n    `INSERT INTO ${table} (\"path\") VALUES (?)`,\n    [...paths].map((p) => [p])\n  );\n}\n\n// Native vector adapters can consume a small exact scope through their connection-local temp\n// table. The reservation is tied to this Store's VectorStore wrapper, so another Store cannot\n// activate it by receiving the same Set object.\nexport async function withMaterializedVectorScope<T>(store: Store, allowed: Set<string>, callback: () => Promise<T>, knownTotal?: number): Promise<T> {\n  const reservation = reserveNativeVectorScope(store.vectors, allowed);\n  if (!reservation) return callback();\n\n  if (allowed.size === 0) {\n    reservation.invalidate();\n    return callback();\n  }\n\n  let total = knownTotal;\n  if (total === undefined) {\n    try {\n      const stmt = await store.prepare('SELECT COUNT(*) AS count FROM frontmatter');\n      const row = (await stmt.get()) as { count: number | bigint };\n      total = Number(row.count);\n    } catch (err) {\n      reservation.invalidate();\n      throw err;\n    }\n  }\n  // Materialization only buys work when this is a nonempty true narrowing and the allowed side\n  // is no larger than its complement. Direct adapter calls and broader scopes retain JS filtering.\n  if (allowed.size >= total || allowed.size > total - allowed.size) {\n    reservation.invalidate();\n    return callback();\n  }\n\n  let operationFailed = false;\n  let operationError: unknown;\n  let result!: T;\n  try {\n    await materializeScope(store, NATIVE_VECTOR_SCOPE_TABLE, allowed);\n    reservation.activate();\n    result = await callback();\n  } catch (err) {\n    operationFailed = true;\n    operationError = err;\n  }\n\n  let cleanupFailed = false;\n  let cleanupError: unknown;\n  try {\n    reservation.deactivate();\n    await store.exec(`DROP TABLE IF EXISTS ${NATIVE_VECTOR_SCOPE_TABLE}`);\n  } catch (err) {\n    cleanupFailed = true;\n    cleanupError = err;\n  } finally {\n    reservation.invalidate();\n  }\n\n  if (operationFailed && cleanupFailed) throw new AggregateError([operationError, cleanupError], 'native vector scope operation and cleanup failed');\n  if (operationFailed) throw operationError;\n  if (cleanupFailed) throw cleanupError;\n  return result;\n}\n\nexport async function setupMapScope(store: Store, paths: Set<string>): Promise<void> {\n  await materializeScope(store, '_map_scope', paths);\n}\n"],"names":["INTERNAL_COLUMNS","inScope","materializeScope","narrowByWhere","rawScope","scopeHasEmbeddings","scopedPaths","setupMapScope","withMaterializedVectorScope","Set","path","include","exclude","some","g","matchesGlob","store","cfg","stmt","rows","anyPresetEmbeds","prepare","all","r","has","overrides","allPaths","effective","adHocScope","paths","resolveSearch","undefined","noExclude","presetName","map","allFrontmatterPaths","filter","p","where","whereRows","err","wherePaths","docs","columns","columnHint","table","exec","runBatch","allowed","callback","knownTotal","reservation","total","row","operationFailed","operationError","result","cleanupFailed","cleanupError","reserveNativeVectorScope","vectors","size","invalidate","get","Number","count","NATIVE_VECTOR_SCOPE_TABLE","activate","deactivate","AggregateError"],"mappings":";;;;;;;;;;;QAOaA;eAAAA;;QAIGC;eAAAA;;QAsDMC;eAAAA;;QAtBAC;eAAAA;;QAjBAC;eAAAA;;QATAC;eAAAA;;QAyCAC;eAAAA;;QA4EAC;eAAAA;;QAzDAC;eAAAA;;;wBA7EM;uBAEmB;4BACpB;yBAEyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7D,IAAMR,mBAAmB,IAAIS,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;CAAe;AAI/F,SAASR,QAAQS,IAAY,EAAEC,OAAiB,EAAEC,OAAkB;IACzE,IAAI,CAACD,QAAQE,IAAI,CAAC,SAACC;eAAMC,IAAAA,qBAAW,EAACL,MAAMI;QAAK,OAAO;IACvD,IAAIF,oBAAAA,8BAAAA,QAASC,IAAI,CAAC,SAACC;eAAMC,IAAAA,qBAAW,EAACL,MAAMI;QAAK,OAAO;IACvD,OAAO;AACT;AAEO,SAAeT,mBAAmBW,KAAY,EAAEC,GAAmB,EAAEX,WAAwB;;YAE5FY,MACAC;;;;oBAFN,IAAI,CAACC,IAAAA,wBAAe,EAACH,MAAM;;wBAAO;uBAAO,yDAAyD;oBACrF;;wBAAMD,MAAMK,OAAO,CAAC;;;oBAA3BH,OAAO;oBACC;;wBAAMA,KAAKI,GAAG;;;oBAAtBH,OAAQ;oBACd;;wBAAOA,KAAKN,IAAI,CAAC,SAACU;mCAAMjB,YAAYkB,GAAG,CAACD,EAAEb,IAAI;;;;;IAChD;;AAIO,SAAeN,SAASY,KAAY,EAAEC,GAAmB,EAAEQ,SAA0B,EAAEC,QAAmB;;YACzGC,WACEhB,SAASC,SACXgB,YAEEV,SAGFW;;;;oBAPAF,YAAYG,IAAAA,sBAAa,EAACb,KAAKQ;oBAC7Bd,UAAqBgB,UAArBhB,SAASC,UAAYe,UAAZf;oBACXgB,aAAaH,UAAUd,OAAO,KAAKoB,aAAaN,UAAUb,OAAO,KAAKmB,aAAaN,UAAUO,SAAS,KAAK;yBAC7G,CAACJ,YAAD;;;;oBACW;;wBAAMZ,MAAMK,OAAO,CAAC;;;oBAA3BH,OAAO;wBACFT;oBAAM;;wBAAMS,KAAKI,GAAG,CAACK,UAAUM,UAAU;;;oBAApD;;wBAAO,IAAA,CAAA,EAAA,MAAIxB;;4BAAM,cAAkEyB,GAAG,CAAC,SAACX;uCAAMA,EAAEb,IAAI;;0BAAC;;;0BAEzFgB,qBAAAA;;;;2BAAAA;;;;;;oBAAa;;wBAAMS,oBAAoBnB;;;2BAA1B;;;oBAArBa;oBACN;;wBAAO,IAAIpB,IAAIoB,MAAMO,MAAM,CAAC,SAACC;mCAAMpC,QAAQoC,GAAG1B,SAASC;;;;;IACzD;;AAEA,SAAeuB,oBAAoBnB,KAAY;;YACvCE;;;;oBAAO;;wBAAMF,MAAMK,OAAO,CAAC;;;oBAA3BH,OAAO;oBACJ;;wBAAMA,KAAKI,GAAG;;;oBAAvB;;wBAAS,cAA8CY,GAAG,CAAC,SAACX;mCAAMA,EAAEb,IAAI;;;;;IAC1E;;AAEO,SAAeP,cAAca,KAAY,EAAEa,KAAkB,EAAES,KAAyB;;YAEzFC,WAEIrB,MAECsB,KAGHC;;;;oBARN,IAAI,CAACH,OAAO;;wBAAOT;;;;;;;;;;oBAGJ;;wBAAMb,MAAMK,OAAO,CAAC,AAAC,2CAAgD,OAANiB,OAAM;;;oBAA5EpB,OAAO;oBACA;;wBAAMA,KAAKI,GAAG;;;oBAA3BiB,YAAa;;;;;;oBACNC;oBACU;;wBAAMxB,MAAM0B,IAAI,CAACC,OAAO;;;oBAAzC,MAAMC,wBAAU;wBAAC;wBAA4BJ;;;oBAEzCC,aAAa,IAAIhC,IAAI8B,UAAUL,GAAG,CAAC,SAACX;+BAAMA,EAAEb,IAAI;;oBACtD;;wBAAO,IAAID,IAAI,AAAC,qBAAGoB,OAAOO,MAAM,CAAC,SAACC;mCAAMI,WAAWjB,GAAG,CAACa;;;;;IACzD;;AAIO,SAAe/B,YAAYU,KAAY,EAAEC,GAAmB,EAAEQ,SAA0B;;YACvFE;;;;oBAAAA,YAAYG,IAAAA,sBAAa,EAACb,KAAKQ;;wBAChBT;;oBAAO;;wBAAMZ,SAASY,OAAOC,KAAKQ;;;oBAAvD;;wBAAOtB;4BAAqB;4BAAuCwB,UAAUW,KAAK;;;;;IACpF;;AAIO,SAAepC,iBAAiBc,KAAY,EAAE6B,KAAa,EAAEhB,KAAkB;;;;;oBACpF;;wBAAMb,MAAM8B,IAAI,CAAC,AAAC,wBAA6B,OAAND;;;oBAAzC;oBACA;;wBAAM7B,MAAM8B,IAAI,CAAC,AAAC,qBAA0B,OAAND,OAAM;;;oBAA5C;oBACA;;wBAAM7B,MAAM+B,QAAQ,CAClB,AAAC,eAAoB,OAANF,OAAM,yBACrB,AAAC,qBAAGhB,OAAOK,GAAG,CAAC,SAACG;mCAAM;gCAACA;6BAAE;;;;oBAF3B;;;;;;IAIF;;AAKO,SAAe7B,4BAA+BQ,KAAY,EAAEgC,OAAoB,EAAEC,QAA0B,EAAEC,UAAmB;;YAChIC,aAQFC,OAGMlC,MACAmC,KAECb,KAYPc,iBACAC,gBACAC,QAKKhB,MAKLiB,eACAC,cAIKlB;;;;oBA3CHW,cAAcQ,IAAAA,mCAAwB,EAAC3C,MAAM4C,OAAO,EAAEZ;oBAC5D,IAAI,CAACG,aAAa;;wBAAOF;;oBAEzB,IAAID,QAAQa,IAAI,KAAK,GAAG;wBACtBV,YAAYW,UAAU;wBACtB;;4BAAOb;;oBACT;oBAEIG,QAAQF;yBACRE,CAAAA,UAAUrB,SAAQ,GAAlBqB;;;;;;;;;;;;oBAEa;;wBAAMpC,MAAMK,OAAO,CAAC;;;oBAA3BH,OAAO;oBACA;;wBAAMA,KAAK6C,GAAG;;;oBAArBV,MAAO;oBACbD,QAAQY,OAAOX,IAAIY,KAAK;;;;;;oBACjBzB;oBACPW,YAAYW,UAAU;oBACtB,MAAMtB;;oBAGV,6FAA6F;oBAC7F,iGAAiG;oBACjG,IAAIQ,QAAQa,IAAI,IAAIT,SAASJ,QAAQa,IAAI,GAAGT,QAAQJ,QAAQa,IAAI,EAAE;wBAChEV,YAAYW,UAAU;wBACtB;;4BAAOb;;oBACT;oBAEIK,kBAAkB;;;;;;;;;oBAIpB;;wBAAMpD,iBAAiBc,OAAOkD,oCAAyB,EAAElB;;;oBAAzD;oBACAG,YAAYgB,QAAQ;oBACX;;wBAAMlB;;;oBAAfO,SAAS;;;;;;oBACFhB;oBACPc,kBAAkB;oBAClBC,iBAAiBf;;;;;;oBAGfiB,gBAAgB;;;;;;;;;oBAGlBN,YAAYiB,UAAU;oBACtB;;wBAAMpD,MAAM8B,IAAI,CAAC,AAAC,wBAAiD,OAA1BoB,oCAAyB;;;oBAAlE;;;;;;oBACO1B;oBACPiB,gBAAgB;oBAChBC,eAAelB;;;;;;oBAEfW,YAAYW,UAAU;;;;;oBAGxB,IAAIR,mBAAmBG,eAAe,MAAM,IAAIY;wBAAgBd;wBAAgBG;uBAAe;oBAC/F,IAAIJ,iBAAiB,MAAMC;oBAC3B,IAAIE,eAAe,MAAMC;oBACzB;;wBAAOF;;;;IACT;;AAEO,SAAejD,cAAcS,KAAY,EAAEa,KAAkB;;;;;oBAClE;;wBAAM3B,iBAAiBc,OAAO,cAAca;;;oBAA5C;;;;;;IACF"}

package/dist/cjs/commands/search.d.cts

@@ -1,6 +1,7 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { Row } from '../output/output.js';
 import type { Store } from '../store/types.js';
+import type { SearchResultVia, SearchSignalContribution } from './signals.js';
+export type { SearchResultVia, SearchSignalContribution } from './signals.js';
 export declare const SNIPPET_CHAR_LIMIT_DEFAULT = 80;
 export declare const SNIPPET_COUNT_LIMIT_DEFAULT = 1;
 export declare function computeSnippets(text: string, terms: string[], charLimit: number, countLimit: number): {
@@ -8,8 +9,11 @@ export declare function computeSnippets(text: string, terms: string[], charLimit
     offset: number;
 };
 export declare function lineNumberAt(text: string, offset: number): number;
+/** Options for the root {@link search} API. */
 export interface SearchOptions {
     k?: number;
+    /** Collects ranking contributions for returned candidates; omitted or false preserves ordinary output. */
+    explain?: boolean;
     where?: string;
     preset?: string;
     include?: string[];
@@ -17,6 +21,29 @@ export interface SearchOptions {
     noExclude?: boolean;
     snippetCharLimit?: number;
     snippetCountLimit?: number;
+    /** Cancels supported provider I/O and stops between phases after active native work finishes. */
+    signal?: AbortSignal;
 }
-export declare function hydrateSearchRows(store: Store, cfg: ResolvedConfig, rows: Row[], matchedPaths: Set<string>, terms: string, opts?: Pick<SearchOptions, 'snippetCharLimit' | 'snippetCountLimit'>): Promise<void>;
-export declare function search(store: Store, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<Row[]>;
+export type SearchResult = {
+    path: string;
+    title: string;
+    summary: string;
+    snippets: string[];
+    via: SearchResultVia;
+    score: number;
+    lines: string | null;
+    similarity?: number | null;
+    /** Present only with explain: true, in words/links/vectors order; contributions can differ from via labels. */
+    explanation?: SearchSignalContribution[];
+};
+export interface SearchHydrationRow {
+    path: string;
+    snippets?: string[];
+    lines?: string | null;
+}
+export declare function hydrateSearchRows(store: Store, cfg: ResolvedConfig, rows: SearchHydrationRow[], matchedPaths: Set<string>, terms: string, opts?: Pick<SearchOptions, 'snippetCharLimit' | 'snippetCountLimit'>): Promise<void>;
+/**
+ * Searches one committed published generation. Provider work precedes the final read transaction;
+ * cancellation waits for active native work and transaction cleanup before rejection.
+ */
+export declare function search(store: Store, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<SearchResult[]>;