sensemaking 0.25.0 → 0.26.0

package/dist/esm/commands/related.js

@@ -4,25 +4,26 @@ import { localModelMissing, MODEL_FILENAMES } from '../embed/store.js';
 import { SenseError } from '../errors.js';
 import { serialQuery } from '../lib/serial-query.js';
 import { resolveNote } from './peek.js';
-import { scopedPaths, scopeHasEmbeddings } from './scope.js';
+import { scopedPaths, scopeHasEmbeddings, withMaterializedVectorScope } from './scope.js';
+import { assertQuerySnapshot } from './snapshot.js';
+export const RELATED_DEFAULT_K = 5;
 export async function resolveRelatedSeed(store, pathArg) {
     const paths = (await (await store.prepare('SELECT "path" FROM frontmatter')).all()).map((row)=>row.path);
     return resolveNote(paths, pathArg);
 }
-// Notes most similar by cosine, excluding self and everything already linked either way.
-// Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.
-export function relatedNotes(store, cfg, pathArg, overrides, k) {
-    return serialQuery(store, ()=>relatedIndexed(store, cfg, pathArg, overrides, k));
+/**
+ * Returns the notes most similar to an indexed seed, excluding the seed and notes linked either
+ * way. The seed resolves across the indexed tree even when it is outside the candidate scope.
+ */ export function relatedNotes(store, cfg, pathArg, options = {}) {
+    return serialQuery(store, ()=>{
+        var _options_k;
+        return relatedIndexed(store, cfg, pathArg, options, (_options_k = options.k) !== null && _options_k !== void 0 ? _options_k : RELATED_DEFAULT_K);
+    });
 }
 async function relatedIndexed(store, cfg, pathArg, overrides, k) {
-    const path = await resolveRelatedSeed(store, pathArg);
-    const outbound = (await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)).map((r)=>r.dst);
-    const backlinks = (await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)).map((r)=>r.src);
-    const exclude = new Set([
-        path,
-        ...outbound,
-        ...backlinks
-    ]);
+    if (!Number.isFinite(k) || !Number.isInteger(k) || k <= 0) {
+        throw new SenseError('SEARCH_OPTION_INVALID', `related option "k" must be a positive finite integer, got ${String(k)}`);
+    }
     // Vectors are the only signal `related` has, so every way of not having them is an error
     // naming the cause; an empty result means one thing, nothing near in meaning.
     const effective = resolveSearch(cfg, overrides);
@@ -40,20 +41,36 @@ async function relatedIndexed(store, cfg, pathArg, overrides, k) {
     if (localModelMissing(e)) {
         throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path "${e.model}" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);
     }
-    const allowed = await scopedPaths(store, cfg, overrides);
-    // The seed may sit outside a narrowed candidate scope; both it and every eligible result must
-    // be ready, while pending vectors in unrelated notes do not block this operation.
-    await ensureDocumentEmbeddings(store, cfg, new Set([
-        ...allowed,
-        path
-    ]));
-    if (!await hasEmbedding(store, path)) {
-        throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);
-    }
-    if (!await scopeHasEmbeddings(store, cfg, allowed)) return [];
-    return similarNotes(store, cfg, path, {
-        exclude,
-        allowed,
-        k
+    return store.transaction(async ()=>{
+        await assertQuerySnapshot(store, cfg);
+        // Repeat effective preset validation inside the snapshot that supplies its durable scope.
+        const current = resolveSearch(cfg, overrides);
+        if (current.signals.vectors === undefined) {
+            throw new SenseError('PRESET_NOT_SEMANTIC', `preset "${current.presetName}" has no "vectors" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add "vectors" to that preset's signals`);
+        }
+        const path = await resolveRelatedSeed(store, pathArg);
+        const outbound = (await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)).map((r)=>r.dst);
+        const backlinks = (await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)).map((r)=>r.src);
+        const exclude = new Set([
+            path,
+            ...outbound,
+            ...backlinks
+        ]);
+        const allowed = await scopedPaths(store, cfg, overrides);
+        // The seed may sit outside a narrowed candidate scope; both it and every eligible result must
+        // be ready, while pending vectors in unrelated notes do not block this operation.
+        await ensureDocumentEmbeddings(store, cfg, new Set([
+            ...allowed,
+            path
+        ]));
+        if (!await hasEmbedding(store, path)) {
+            throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);
+        }
+        if (!await scopeHasEmbeddings(store, cfg, allowed)) return [];
+        return withMaterializedVectorScope(store, allowed, ()=>similarNotes(store, cfg, path, {
+                exclude,
+                allowed,
+                k
+            }));
     });
 }

package/dist/esm/commands/related.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedConfig, embedEnabled, resolveSearch } from '../config/index.ts';\nimport { ensureDocumentEmbeddings, hasEmbedding, similarNotes } from '../embed/query.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings } from './scope.ts';\n\nexport async function resolveRelatedSeed(store: Store, pathArg: string): Promise<string> {\n  const paths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((row) => row.path);\n  return resolveNote(paths, pathArg);\n}\n\n// Notes most similar by cosine, excluding self and everything already linked either way.\n// Its own command, not a peek section: a full embeddings scan is ~480ms at 26k notes.\nexport function relatedNotes(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  return serialQuery(store, () => relatedIndexed(store, cfg, pathArg, overrides, k));\n}\n\nasync function relatedIndexed(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: SearchOverrides, k: number): Promise<Array<{ path: string; similarity: number }>> {\n  const path = await resolveRelatedSeed(store, pathArg);\n\n  const outbound = ((await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)) as Array<{ dst: string }>).map((r) => r.dst);\n  const backlinks = ((await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)) as Array<{ src: string }>).map((r) => r.src);\n  const exclude = new Set([path, ...outbound, ...backlinks]);\n\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause; an empty result means one thing, nothing near in meaning.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');\n  }\n  // search gates on the same signal, so a preset's declared signals mean one thing: without\n  // this check an overlapping vectors-on preset would answer for a scope that declined them.\n  if (effective.signals.vectors === undefined) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n  }\n  // A downloadable HF id can be fetched by preparation. Only a local path with missing files\n  // can never fetch itself.\n  const e = embedConfig(cfg); // embedEnabled(cfg) above guarantees this is set\n  if (localModelMissing(e)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);\n  }\n  const allowed = await scopedPaths(store, cfg, overrides);\n  // The seed may sit outside a narrowed candidate scope; both it and every eligible result must\n  // be ready, while pending vectors in unrelated notes do not block this operation.\n  await ensureDocumentEmbeddings(store, cfg, new Set([...allowed, path]));\n  if (!(await hasEmbedding(store, path))) {\n    throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n  }\n  if (!(await scopeHasEmbeddings(store, cfg, allowed))) return [];\n  return similarNotes(store, cfg, path, { exclude, allowed, k });\n}\n"],"names":["embedConfig","embedEnabled","resolveSearch","ensureDocumentEmbeddings","hasEmbedding","similarNotes","localModelMissing","MODEL_FILENAMES","SenseError","serialQuery","resolveNote","scopedPaths","scopeHasEmbeddings","resolveRelatedSeed","store","pathArg","paths","prepare","all","map","row","path","relatedNotes","cfg","overrides","k","relatedIndexed","outbound","r","dst","backlinks","src","exclude","Set","effective","signals","vectors","undefined","presetName","e","model","allowed"],"mappings":"AACA,SAASA,WAAW,EAAEC,YAAY,EAAEC,aAAa,QAAQ,qBAAqB;AAC9E,SAASC,wBAAwB,EAAEC,YAAY,EAAEC,YAAY,QAAQ,oBAAoB;AACzF,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,WAAW,QAAQ,yBAAyB;AAErD,SAASC,WAAW,QAAQ,YAAY;AACxC,SAASC,WAAW,EAAEC,kBAAkB,QAAQ,aAAa;AAE7D,OAAO,eAAeC,mBAAmBC,KAAY,EAAEC,OAAe;IACpE,MAAMC,QAAQ,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMF,MAAMG,OAAO,CAAC,iCAAgC,EAAGC,GAAG,EAAC,EAA+BC,GAAG,CAAC,CAACC,MAAQA,IAAIC,IAAI;IACtI,OAAOX,YAAYM,OAAOD;AAC5B;AAEA,yFAAyF;AACzF,sFAAsF;AACtF,OAAO,SAASO,aAAaR,KAAY,EAAES,GAAmB,EAAER,OAAe,EAAES,SAA0B,EAAEC,CAAS;IACpH,OAAOhB,YAAYK,OAAO,IAAMY,eAAeZ,OAAOS,KAAKR,SAASS,WAAWC;AACjF;AAEA,eAAeC,eAAeZ,KAAY,EAAES,GAAmB,EAAER,OAAe,EAAES,SAA0B,EAAEC,CAAS;IACrH,MAAMJ,OAAO,MAAMR,mBAAmBC,OAAOC;IAE7C,MAAMY,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMb,MAAMG,OAAO,CAAC,kFAAiF,EAAGC,GAAG,CAACG,KAAI,EAA8BF,GAAG,CAAC,CAACS,IAAMA,EAAEC,GAAG;IACxL,MAAMC,YAAY,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMhB,MAAMG,OAAO,CAAC,8DAA6D,EAAGC,GAAG,CAACG,KAAI,EAA8BF,GAAG,CAAC,CAACS,IAAMA,EAAEG,GAAG;IACrK,MAAMC,UAAU,IAAIC,IAAI;QAACZ;WAASM;WAAaG;KAAU;IAEzD,yFAAyF;IACzF,8EAA8E;IAC9E,MAAMI,YAAYhC,cAAcqB,KAAKC;IACrC,IAAI,CAACvB,aAAasB,MAAM;QACtB,MAAM,IAAIf,WAAW,kBAAkB;IACzC;IACA,0FAA0F;IAC1F,2FAA2F;IAC3F,IAAI0B,UAAUC,OAAO,CAACC,OAAO,KAAKC,WAAW;QAC3C,MAAM,IAAI7B,WAAW,uBAAuB,CAAC,QAAQ,EAAE0B,UAAUI,UAAU,CAAC,uKAAuK,CAAC;IACtP;IACA,2FAA2F;IAC3F,0BAA0B;IAC1B,MAAMC,IAAIvC,YAAYuB,MAAM,iDAAiD;IAC7E,IAAIjB,kBAAkBiC,IAAI;QACxB,MAAM,IAAI/B,WAAW,uBAAuB,CAAC,2FAA2F,EAAE+B,EAAEC,KAAK,CAAC,aAAa,EAAEjC,gBAAgB,sGAAsG,CAAC;IAC1R;IACA,MAAMkC,UAAU,MAAM9B,YAAYG,OAAOS,KAAKC;IAC9C,8FAA8F;IAC9F,kFAAkF;IAClF,MAAMrB,yBAAyBW,OAAOS,KAAK,IAAIU,IAAI;WAAIQ;QAASpB;KAAK;IACrE,IAAI,CAAE,MAAMjB,aAAaU,OAAOO,OAAQ;QACtC,MAAM,IAAIb,WAAW,qBAAqB,GAAGa,KAAK,4GAA4G,CAAC;IACjK;IACA,IAAI,CAAE,MAAMT,mBAAmBE,OAAOS,KAAKkB,UAAW,OAAO,EAAE;IAC/D,OAAOpC,aAAaS,OAAOS,KAAKF,MAAM;QAAEW;QAASS;QAAShB;IAAE;AAC9D"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/related.ts"],"sourcesContent":["import type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { embedConfig, embedEnabled, resolveSearch } from '../config/index.ts';\nimport { ensureDocumentEmbeddings, hasEmbedding, similarNotes } from '../embed/query.ts';\nimport { localModelMissing, MODEL_FILENAMES } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\nimport { resolveNote } from './peek.ts';\nimport { scopedPaths, scopeHasEmbeddings, withMaterializedVectorScope } from './scope.ts';\nimport { assertQuerySnapshot } from './snapshot.ts';\n\nexport const RELATED_DEFAULT_K = 5;\n\n/**\n * Options for {@link relatedNotes}. Scope fields filter result candidates, not the seed note;\n * `k` defaults to 5, matching the CLI.\n */\nexport interface RelatedOptions extends SearchOverrides {\n  k?: number;\n}\n\nexport type RelatedResult = {\n  path: string;\n  similarity: number;\n};\n\nexport async function resolveRelatedSeed(store: Store, pathArg: string): Promise<string> {\n  const paths = ((await (await store.prepare('SELECT \"path\" FROM frontmatter')).all()) as Array<{ path: string }>).map((row) => row.path);\n  return resolveNote(paths, pathArg);\n}\n\n/**\n * Returns the notes most similar to an indexed seed, excluding the seed and notes linked either\n * way. The seed resolves across the indexed tree even when it is outside the candidate scope.\n */\nexport function relatedNotes(store: Store, cfg: ResolvedConfig, pathArg: string, options: RelatedOptions = {}): Promise<RelatedResult[]> {\n  return serialQuery(store, () => relatedIndexed(store, cfg, pathArg, options, options.k ?? RELATED_DEFAULT_K));\n}\n\nasync function relatedIndexed(store: Store, cfg: ResolvedConfig, pathArg: string, overrides: RelatedOptions, k: number): Promise<RelatedResult[]> {\n  if (!Number.isFinite(k) || !Number.isInteger(k) || k <= 0) {\n    throw new SenseError('SEARCH_OPTION_INVALID', `related option \"k\" must be a positive finite integer, got ${String(k)}`);\n  }\n  // Vectors are the only signal `related` has, so every way of not having them is an error\n  // naming the cause; an empty result means one thing, nothing near in meaning.\n  const effective = resolveSearch(cfg, overrides);\n  if (!embedEnabled(cfg)) {\n    throw new SenseError('EMBED_DISABLED', 'related ranks notes by meaning, and this tree has no embedding model; add an \"embed\" block naming one to sense.config.json (a Hugging Face id fetches automatically at first use; `sense download` prefetches it) -- search works without it, on words and links');\n  }\n  // search gates on the same signal, so a preset's declared signals mean one thing: without\n  // this check an overlapping vectors-on preset would answer for a scope that declined them.\n  if (effective.signals.vectors === undefined) {\n    throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${effective.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n  }\n  // A downloadable HF id can be fetched by preparation. Only a local path with missing files\n  // can never fetch itself.\n  const e = embedConfig(cfg); // embedEnabled(cfg) above guarantees this is set\n  if (localModelMissing(e)) {\n    throw new SenseError('EMBED_MODEL_MISSING', `related ranks notes by meaning, so it needs the embedding model, but the local model path \"${e.model}\" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them (search still works without it, on words and links)`);\n  }\n\n  return store.transaction(async () => {\n    await assertQuerySnapshot(store, cfg);\n    // Repeat effective preset validation inside the snapshot that supplies its durable scope.\n    const current = resolveSearch(cfg, overrides);\n    if (current.signals.vectors === undefined) {\n      throw new SenseError('PRESET_NOT_SEMANTIC', `preset \"${current.presetName}\" has no \"vectors\" signal, so this scope has no vectors and related has no other signal; search it instead (words and links), or add \"vectors\" to that preset's signals`);\n    }\n\n    const path = await resolveRelatedSeed(store, pathArg);\n    const outbound = ((await (await store.prepare('SELECT DISTINCT dst FROM links WHERE src = ? AND dst IS NOT NULL AND dst != src')).all(path)) as Array<{ dst: string }>).map((r) => r.dst);\n    const backlinks = ((await (await store.prepare('SELECT DISTINCT src FROM links WHERE dst = ? AND src != dst')).all(path)) as Array<{ src: string }>).map((r) => r.src);\n    const exclude = new Set([path, ...outbound, ...backlinks]);\n    const allowed = await scopedPaths(store, cfg, overrides);\n    // The seed may sit outside a narrowed candidate scope; both it and every eligible result must\n    // be ready, while pending vectors in unrelated notes do not block this operation.\n    await ensureDocumentEmbeddings(store, cfg, new Set([...allowed, path]));\n    if (!(await hasEmbedding(store, path))) {\n      throw new SenseError('NOTE_NOT_EMBEDDED', `${path} has no indexed text to compare -- a note that is frontmatter only, or empty, has nothing to rank by meaning`);\n    }\n    if (!(await scopeHasEmbeddings(store, cfg, allowed))) return [];\n    return withMaterializedVectorScope(store, allowed, () => similarNotes(store, cfg, path, { exclude, allowed, k }));\n  });\n}\n"],"names":["embedConfig","embedEnabled","resolveSearch","ensureDocumentEmbeddings","hasEmbedding","similarNotes","localModelMissing","MODEL_FILENAMES","SenseError","serialQuery","resolveNote","scopedPaths","scopeHasEmbeddings","withMaterializedVectorScope","assertQuerySnapshot","RELATED_DEFAULT_K","resolveRelatedSeed","store","pathArg","paths","prepare","all","map","row","path","relatedNotes","cfg","options","relatedIndexed","k","overrides","Number","isFinite","isInteger","String","effective","signals","vectors","undefined","presetName","e","model","transaction","current","outbound","r","dst","backlinks","src","exclude","Set","allowed"],"mappings":"AACA,SAASA,WAAW,EAAEC,YAAY,EAAEC,aAAa,QAAQ,qBAAqB;AAC9E,SAASC,wBAAwB,EAAEC,YAAY,EAAEC,YAAY,QAAQ,oBAAoB;AACzF,SAASC,iBAAiB,EAAEC,eAAe,QAAQ,oBAAoB;AACvE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,WAAW,QAAQ,yBAAyB;AAErD,SAASC,WAAW,QAAQ,YAAY;AACxC,SAASC,WAAW,EAAEC,kBAAkB,EAAEC,2BAA2B,QAAQ,aAAa;AAC1F,SAASC,mBAAmB,QAAQ,gBAAgB;AAEpD,OAAO,MAAMC,oBAAoB,EAAE;AAenC,OAAO,eAAeC,mBAAmBC,KAAY,EAAEC,OAAe;IACpE,MAAMC,QAAQ,AAAE,CAAA,MAAM,AAAC,CAAA,MAAMF,MAAMG,OAAO,CAAC,iCAAgC,EAAGC,GAAG,EAAC,EAA+BC,GAAG,CAAC,CAACC,MAAQA,IAAIC,IAAI;IACtI,OAAOd,YAAYS,OAAOD;AAC5B;AAEA;;;CAGC,GACD,OAAO,SAASO,aAAaR,KAAY,EAAES,GAAmB,EAAER,OAAe,EAAES,UAA0B,CAAC,CAAC;IAC3G,OAAOlB,YAAYQ,OAAO;YAAmDU;eAA7CC,eAAeX,OAAOS,KAAKR,SAASS,UAASA,aAAAA,QAAQE,CAAC,cAATF,wBAAAA,aAAaZ;;AAC5F;AAEA,eAAea,eAAeX,KAAY,EAAES,GAAmB,EAAER,OAAe,EAAEY,SAAyB,EAAED,CAAS;IACpH,IAAI,CAACE,OAAOC,QAAQ,CAACH,MAAM,CAACE,OAAOE,SAAS,CAACJ,MAAMA,KAAK,GAAG;QACzD,MAAM,IAAIrB,WAAW,yBAAyB,CAAC,0DAA0D,EAAE0B,OAAOL,IAAI;IACxH;IACA,yFAAyF;IACzF,8EAA8E;IAC9E,MAAMM,YAAYjC,cAAcwB,KAAKI;IACrC,IAAI,CAAC7B,aAAayB,MAAM;QACtB,MAAM,IAAIlB,WAAW,kBAAkB;IACzC;IACA,0FAA0F;IAC1F,2FAA2F;IAC3F,IAAI2B,UAAUC,OAAO,CAACC,OAAO,KAAKC,WAAW;QAC3C,MAAM,IAAI9B,WAAW,uBAAuB,CAAC,QAAQ,EAAE2B,UAAUI,UAAU,CAAC,uKAAuK,CAAC;IACtP;IACA,2FAA2F;IAC3F,0BAA0B;IAC1B,MAAMC,IAAIxC,YAAY0B,MAAM,iDAAiD;IAC7E,IAAIpB,kBAAkBkC,IAAI;QACxB,MAAM,IAAIhC,WAAW,uBAAuB,CAAC,2FAA2F,EAAEgC,EAAEC,KAAK,CAAC,aAAa,EAAElC,gBAAgB,sGAAsG,CAAC;IAC1R;IAEA,OAAOU,MAAMyB,WAAW,CAAC;QACvB,MAAM5B,oBAAoBG,OAAOS;QACjC,0FAA0F;QAC1F,MAAMiB,UAAUzC,cAAcwB,KAAKI;QACnC,IAAIa,QAAQP,OAAO,CAACC,OAAO,KAAKC,WAAW;YACzC,MAAM,IAAI9B,WAAW,uBAAuB,CAAC,QAAQ,EAAEmC,QAAQJ,UAAU,CAAC,uKAAuK,CAAC;QACpP;QAEA,MAAMf,OAAO,MAAMR,mBAAmBC,OAAOC;QAC7C,MAAM0B,WAAW,AAAE,CAAA,MAAM,AAAC,CAAA,MAAM3B,MAAMG,OAAO,CAAC,kFAAiF,EAAGC,GAAG,CAACG,KAAI,EAA8BF,GAAG,CAAC,CAACuB,IAAMA,EAAEC,GAAG;QACxL,MAAMC,YAAY,AAAE,CAAA,MAAM,AAAC,CAAA,MAAM9B,MAAMG,OAAO,CAAC,8DAA6D,EAAGC,GAAG,CAACG,KAAI,EAA8BF,GAAG,CAAC,CAACuB,IAAMA,EAAEG,GAAG;QACrK,MAAMC,UAAU,IAAIC,IAAI;YAAC1B;eAASoB;eAAaG;SAAU;QACzD,MAAMI,UAAU,MAAMxC,YAAYM,OAAOS,KAAKI;QAC9C,8FAA8F;QAC9F,kFAAkF;QAClF,MAAM3B,yBAAyBc,OAAOS,KAAK,IAAIwB,IAAI;eAAIC;YAAS3B;SAAK;QACrE,IAAI,CAAE,MAAMpB,aAAaa,OAAOO,OAAQ;YACtC,MAAM,IAAIhB,WAAW,qBAAqB,GAAGgB,KAAK,4GAA4G,CAAC;QACjK;QACA,IAAI,CAAE,MAAMZ,mBAAmBK,OAAOS,KAAKyB,UAAW,OAAO,EAAE;QAC/D,OAAOtC,4BAA4BI,OAAOkC,SAAS,IAAM9C,aAAaY,OAAOS,KAAKF,MAAM;gBAAEyB;gBAASE;gBAAStB;YAAE;IAChH;AACF"}

package/dist/esm/commands/scope.d.ts

@@ -7,4 +7,5 @@ export declare function rawScope(store: Store, cfg: ResolvedConfig, overrides: S
 export declare function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>>;
 export declare function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>>;
 export declare function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void>;
+export declare function withMaterializedVectorScope<T>(store: Store, allowed: Set<string>, callback: () => Promise<T>, knownTotal?: number): Promise<T>;
 export declare function setupMapScope(store: Store, paths: Set<string>): Promise<void>;

package/dist/esm/commands/scope.js

@@ -1,6 +1,7 @@
 import { matchesGlob } from 'node:path';
 import { anyPresetEmbeds, resolveSearch } from '../config/index.js';
 import { columnHint } from '../output/column-hint.js';
+import { NATIVE_VECTOR_SCOPE_TABLE, reserveNativeVectorScope } from '../store/vectors.js';
 export const INTERNAL_COLUMNS = new Set([
     'path',
     '_mtime',
@@ -70,6 +71,63 @@ export async function materializeScope(store, table, paths) {
             p
         ]));
 }
+// Native vector adapters can consume a small exact scope through their connection-local temp
+// table. The reservation is tied to this Store's VectorStore wrapper, so another Store cannot
+// activate it by receiving the same Set object.
+export async function withMaterializedVectorScope(store, allowed, callback, knownTotal) {
+    const reservation = reserveNativeVectorScope(store.vectors, allowed);
+    if (!reservation) return callback();
+    if (allowed.size === 0) {
+        reservation.invalidate();
+        return callback();
+    }
+    let total = knownTotal;
+    if (total === undefined) {
+        try {
+            const stmt = await store.prepare('SELECT COUNT(*) AS count FROM frontmatter');
+            const row = await stmt.get();
+            total = Number(row.count);
+        } catch (err) {
+            reservation.invalidate();
+            throw err;
+        }
+    }
+    // Materialization only buys work when this is a nonempty true narrowing and the allowed side
+    // is no larger than its complement. Direct adapter calls and broader scopes retain JS filtering.
+    if (allowed.size >= total || allowed.size > total - allowed.size) {
+        reservation.invalidate();
+        return callback();
+    }
+    let operationFailed = false;
+    let operationError;
+    let result;
+    try {
+        await materializeScope(store, NATIVE_VECTOR_SCOPE_TABLE, allowed);
+        reservation.activate();
+        result = await callback();
+    } catch (err) {
+        operationFailed = true;
+        operationError = err;
+    }
+    let cleanupFailed = false;
+    let cleanupError;
+    try {
+        reservation.deactivate();
+        await store.exec(`DROP TABLE IF EXISTS ${NATIVE_VECTOR_SCOPE_TABLE}`);
+    } catch (err) {
+        cleanupFailed = true;
+        cleanupError = err;
+    } finally{
+        reservation.invalidate();
+    }
+    if (operationFailed && cleanupFailed) throw new AggregateError([
+        operationError,
+        cleanupError
+    ], 'native vector scope operation and cleanup failed');
+    if (operationFailed) throw operationError;
+    if (cleanupFailed) throw cleanupError;
+    return result;
+}
 export async function setupMapScope(store, paths) {
     await materializeScope(store, '_map_scope', paths);
 }

package/dist/esm/commands/scope.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/scope.ts"],"sourcesContent":["import { matchesGlob } from 'node:path';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { anyPresetEmbeds, resolveSearch } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\nimport type { Store } from '../store/types.ts';\n\nexport const INTERNAL_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error']);\n\n// node:path's matchesGlob is experimental (stable behind an unstable-API flag) at the\n// Node >=22.20 engines floor; used here in JS rather than a directory walk in the query path.\nexport function inScope(path: string, include: string[], exclude?: string[]): boolean {\n  if (!include.some((g) => matchesGlob(path, g))) return false;\n  if (exclude?.some((g) => matchesGlob(path, g))) return false;\n  return true;\n}\n\nexport async function scopeHasEmbeddings(store: Store, cfg: ResolvedConfig, scopedPaths: Set<string>): Promise<boolean> {\n  if (!anyPresetEmbeds(cfg)) return false; // the embeddings table doesn't exist at all in this case\n  const stmt = await store.prepare('SELECT DISTINCT \"path\" FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string }>;\n  return rows.some((r) => scopedPaths.has(r.path));\n}\n\n// Scope only (no --where): preset_files for a named preset, JS glob matching for an ad hoc\n// include/exclude override. Shared by scopedPaths() and search()'s candidate-pool sizing.\nexport async function rawScope(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  const { include, exclude } = effective;\n  const adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;\n  if (!adHocScope) {\n    const stmt = await store.prepare('SELECT \"path\" FROM preset_files WHERE preset = ?');\n    return new Set(((await stmt.all(effective.presetName)) as Array<{ path: string }>).map((r) => r.path));\n  }\n  const paths = allPaths ?? (await allFrontmatterPaths(store));\n  return new Set(paths.filter((p) => inScope(p, include, exclude)));\n}\n\nasync function allFrontmatterPaths(store: Store): Promise<string[]> {\n  const stmt = await store.prepare('SELECT \"path\" FROM frontmatter');\n  return ((await stmt.all()) as Array<{ path: string }>).map((r) => r.path);\n}\n\nexport async function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>> {\n  if (!where) return paths;\n  let whereRows: Array<{ path: string }>;\n  try {\n    const stmt = await store.prepare(`SELECT \"path\" FROM frontmatter f WHERE (${where})`);\n    whereRows = (await stmt.all()) as Array<{ path: string }>;\n  } catch (err) {\n    throw columnHint(await store.docs.columns(), err as Error);\n  }\n  const wherePaths = new Set(whereRows.map((r) => r.path));\n  return new Set([...paths].filter((p) => wherePaths.has(p)));\n}\n\n// The scope resolver for non-search commands (path, peek, map): same coverage rule search()\n// applies, then narrowed by the resolved `where`.\nexport async function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  return narrowByWhere(store, await rawScope(store, cfg, overrides), effective.where);\n}\n\n// Materializes a path set into a named temp table (same shape as traverse.ts's allowed_nodes)\n// so a query can join/filter against it cheaply instead of binding a parameter per path.\nexport async function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void> {\n  await store.exec(`DROP TABLE IF EXISTS ${table}`);\n  await store.exec(`CREATE TEMP TABLE ${table} (\"path\" TEXT PRIMARY KEY)`);\n  await store.runBatch(\n    `INSERT INTO ${table} (\"path\") VALUES (?)`,\n    [...paths].map((p) => [p])\n  );\n}\n\nexport async function setupMapScope(store: Store, paths: Set<string>): Promise<void> {\n  await materializeScope(store, '_map_scope', paths);\n}\n"],"names":["matchesGlob","anyPresetEmbeds","resolveSearch","columnHint","INTERNAL_COLUMNS","Set","inScope","path","include","exclude","some","g","scopeHasEmbeddings","store","cfg","scopedPaths","stmt","prepare","rows","all","r","has","rawScope","overrides","allPaths","effective","adHocScope","undefined","noExclude","presetName","map","paths","allFrontmatterPaths","filter","p","narrowByWhere","where","whereRows","err","docs","columns","wherePaths","materializeScope","table","exec","runBatch","setupMapScope"],"mappings":"AAAA,SAASA,WAAW,QAAQ,YAAY;AAExC,SAASC,eAAe,EAAEC,aAAa,QAAQ,qBAAqB;AACpE,SAASC,UAAU,QAAQ,2BAA2B;AAGtD,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;CAAe,EAAE;AAExG,sFAAsF;AACtF,8FAA8F;AAC9F,OAAO,SAASC,QAAQC,IAAY,EAAEC,OAAiB,EAAEC,OAAkB;IACzE,IAAI,CAACD,QAAQE,IAAI,CAAC,CAACC,IAAMX,YAAYO,MAAMI,KAAK,OAAO;IACvD,IAAIF,oBAAAA,8BAAAA,QAASC,IAAI,CAAC,CAACC,IAAMX,YAAYO,MAAMI,KAAK,OAAO;IACvD,OAAO;AACT;AAEA,OAAO,eAAeC,mBAAmBC,KAAY,EAAEC,GAAmB,EAAEC,WAAwB;IAClG,IAAI,CAACd,gBAAgBa,MAAM,OAAO,OAAO,yDAAyD;IAClG,MAAME,OAAO,MAAMH,MAAMI,OAAO,CAAC;IACjC,MAAMC,OAAQ,MAAMF,KAAKG,GAAG;IAC5B,OAAOD,KAAKR,IAAI,CAAC,CAACU,IAAML,YAAYM,GAAG,CAACD,EAAEb,IAAI;AAChD;AAEA,2FAA2F;AAC3F,0FAA0F;AAC1F,OAAO,eAAee,SAAST,KAAY,EAAEC,GAAmB,EAAES,SAA0B,EAAEC,QAAmB;IAC/G,MAAMC,YAAYvB,cAAcY,KAAKS;IACrC,MAAM,EAAEf,OAAO,EAAEC,OAAO,EAAE,GAAGgB;IAC7B,MAAMC,aAAaH,UAAUf,OAAO,KAAKmB,aAAaJ,UAAUd,OAAO,KAAKkB,aAAaJ,UAAUK,SAAS,KAAK;IACjH,IAAI,CAACF,YAAY;QACf,MAAMV,OAAO,MAAMH,MAAMI,OAAO,CAAC;QACjC,OAAO,IAAIZ,IAAI,AAAE,CAAA,MAAMW,KAAKG,GAAG,CAACM,UAAUI,UAAU,CAAA,EAA+BC,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;IACtG;IACA,MAAMwB,QAAQP,qBAAAA,sBAAAA,WAAa,MAAMQ,oBAAoBnB;IACrD,OAAO,IAAIR,IAAI0B,MAAME,MAAM,CAAC,CAACC,IAAM5B,QAAQ4B,GAAG1B,SAASC;AACzD;AAEA,eAAeuB,oBAAoBnB,KAAY;IAC7C,MAAMG,OAAO,MAAMH,MAAMI,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKG,GAAG,EAAC,EAA+BW,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;AAC1E;AAEA,OAAO,eAAe4B,cAActB,KAAY,EAAEkB,KAAkB,EAAEK,KAAyB;IAC7F,IAAI,CAACA,OAAO,OAAOL;IACnB,IAAIM;IACJ,IAAI;QACF,MAAMrB,OAAO,MAAMH,MAAMI,OAAO,CAAC,CAAC,wCAAwC,EAAEmB,MAAM,CAAC,CAAC;QACpFC,YAAa,MAAMrB,KAAKG,GAAG;IAC7B,EAAE,OAAOmB,KAAK;QACZ,MAAMnC,WAAW,MAAMU,MAAM0B,IAAI,CAACC,OAAO,IAAIF;IAC/C;IACA,MAAMG,aAAa,IAAIpC,IAAIgC,UAAUP,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;IACtD,OAAO,IAAIF,IAAI;WAAI0B;KAAM,CAACE,MAAM,CAAC,CAACC,IAAMO,WAAWpB,GAAG,CAACa;AACzD;AAEA,4FAA4F;AAC5F,kDAAkD;AAClD,OAAO,eAAenB,YAAYF,KAAY,EAAEC,GAAmB,EAAES,SAA0B;IAC7F,MAAME,YAAYvB,cAAcY,KAAKS;IACrC,OAAOY,cAActB,OAAO,MAAMS,SAAST,OAAOC,KAAKS,YAAYE,UAAUW,KAAK;AACpF;AAEA,8FAA8F;AAC9F,yFAAyF;AACzF,OAAO,eAAeM,iBAAiB7B,KAAY,EAAE8B,KAAa,EAAEZ,KAAkB;IACpF,MAAMlB,MAAM+B,IAAI,CAAC,CAAC,qBAAqB,EAAED,OAAO;IAChD,MAAM9B,MAAM+B,IAAI,CAAC,CAAC,kBAAkB,EAAED,MAAM,0BAA0B,CAAC;IACvE,MAAM9B,MAAMgC,QAAQ,CAClB,CAAC,YAAY,EAAEF,MAAM,oBAAoB,CAAC,EAC1C;WAAIZ;KAAM,CAACD,GAAG,CAAC,CAACI,IAAM;YAACA;SAAE;AAE7B;AAEA,OAAO,eAAeY,cAAcjC,KAAY,EAAEkB,KAAkB;IAClE,MAAMW,iBAAiB7B,OAAO,cAAckB;AAC9C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/commands/scope.ts"],"sourcesContent":["import { matchesGlob } from 'node:path';\nimport type { ResolvedConfig, SearchOverrides } from '../config/index.ts';\nimport { anyPresetEmbeds, resolveSearch } from '../config/index.ts';\nimport { columnHint } from '../output/column-hint.ts';\nimport type { Store } from '../store/types.ts';\nimport { NATIVE_VECTOR_SCOPE_TABLE, reserveNativeVectorScope } from '../store/vectors.ts';\n\nexport const INTERNAL_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_rank', '_parse_error']);\n\n// node:path's matchesGlob is experimental (stable behind an unstable-API flag) at the\n// Node >=22.20 engines floor; used here in JS rather than a directory walk in the query path.\nexport function inScope(path: string, include: string[], exclude?: string[]): boolean {\n  if (!include.some((g) => matchesGlob(path, g))) return false;\n  if (exclude?.some((g) => matchesGlob(path, g))) return false;\n  return true;\n}\n\nexport async function scopeHasEmbeddings(store: Store, cfg: ResolvedConfig, scopedPaths: Set<string>): Promise<boolean> {\n  if (!anyPresetEmbeds(cfg)) return false; // the embeddings table doesn't exist at all in this case\n  const stmt = await store.prepare('SELECT DISTINCT \"path\" FROM embeddings WHERE vector IS NOT NULL');\n  const rows = (await stmt.all()) as Array<{ path: string }>;\n  return rows.some((r) => scopedPaths.has(r.path));\n}\n\n// Scope only (no --where): preset_files for a named preset, JS glob matching for an ad hoc\n// include/exclude override. Shared by scopedPaths() and search()'s candidate-pool sizing.\nexport async function rawScope(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides, allPaths?: string[]): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  const { include, exclude } = effective;\n  const adHocScope = overrides.include !== undefined || overrides.exclude !== undefined || overrides.noExclude === true;\n  if (!adHocScope) {\n    const stmt = await store.prepare('SELECT \"path\" FROM preset_files WHERE preset = ?');\n    return new Set(((await stmt.all(effective.presetName)) as Array<{ path: string }>).map((r) => r.path));\n  }\n  const paths = allPaths ?? (await allFrontmatterPaths(store));\n  return new Set(paths.filter((p) => inScope(p, include, exclude)));\n}\n\nasync function allFrontmatterPaths(store: Store): Promise<string[]> {\n  const stmt = await store.prepare('SELECT \"path\" FROM frontmatter');\n  return ((await stmt.all()) as Array<{ path: string }>).map((r) => r.path);\n}\n\nexport async function narrowByWhere(store: Store, paths: Set<string>, where: string | undefined): Promise<Set<string>> {\n  if (!where) return paths;\n  let whereRows: Array<{ path: string }>;\n  try {\n    const stmt = await store.prepare(`SELECT \"path\" FROM frontmatter f WHERE (${where})`);\n    whereRows = (await stmt.all()) as Array<{ path: string }>;\n  } catch (err) {\n    throw columnHint(await store.docs.columns(), err as Error);\n  }\n  const wherePaths = new Set(whereRows.map((r) => r.path));\n  return new Set([...paths].filter((p) => wherePaths.has(p)));\n}\n\n// The scope resolver for non-search commands (path, peek, map): same coverage rule search()\n// applies, then narrowed by the resolved `where`.\nexport async function scopedPaths(store: Store, cfg: ResolvedConfig, overrides: SearchOverrides): Promise<Set<string>> {\n  const effective = resolveSearch(cfg, overrides);\n  return narrowByWhere(store, await rawScope(store, cfg, overrides), effective.where);\n}\n\n// Materializes a path set into a named temp table (same shape as traverse.ts's allowed_nodes)\n// so a query can join/filter against it cheaply instead of binding a parameter per path.\nexport async function materializeScope(store: Store, table: string, paths: Set<string>): Promise<void> {\n  await store.exec(`DROP TABLE IF EXISTS ${table}`);\n  await store.exec(`CREATE TEMP TABLE ${table} (\"path\" TEXT PRIMARY KEY)`);\n  await store.runBatch(\n    `INSERT INTO ${table} (\"path\") VALUES (?)`,\n    [...paths].map((p) => [p])\n  );\n}\n\n// Native vector adapters can consume a small exact scope through their connection-local temp\n// table. The reservation is tied to this Store's VectorStore wrapper, so another Store cannot\n// activate it by receiving the same Set object.\nexport async function withMaterializedVectorScope<T>(store: Store, allowed: Set<string>, callback: () => Promise<T>, knownTotal?: number): Promise<T> {\n  const reservation = reserveNativeVectorScope(store.vectors, allowed);\n  if (!reservation) return callback();\n\n  if (allowed.size === 0) {\n    reservation.invalidate();\n    return callback();\n  }\n\n  let total = knownTotal;\n  if (total === undefined) {\n    try {\n      const stmt = await store.prepare('SELECT COUNT(*) AS count FROM frontmatter');\n      const row = (await stmt.get()) as { count: number | bigint };\n      total = Number(row.count);\n    } catch (err) {\n      reservation.invalidate();\n      throw err;\n    }\n  }\n  // Materialization only buys work when this is a nonempty true narrowing and the allowed side\n  // is no larger than its complement. Direct adapter calls and broader scopes retain JS filtering.\n  if (allowed.size >= total || allowed.size > total - allowed.size) {\n    reservation.invalidate();\n    return callback();\n  }\n\n  let operationFailed = false;\n  let operationError: unknown;\n  let result!: T;\n  try {\n    await materializeScope(store, NATIVE_VECTOR_SCOPE_TABLE, allowed);\n    reservation.activate();\n    result = await callback();\n  } catch (err) {\n    operationFailed = true;\n    operationError = err;\n  }\n\n  let cleanupFailed = false;\n  let cleanupError: unknown;\n  try {\n    reservation.deactivate();\n    await store.exec(`DROP TABLE IF EXISTS ${NATIVE_VECTOR_SCOPE_TABLE}`);\n  } catch (err) {\n    cleanupFailed = true;\n    cleanupError = err;\n  } finally {\n    reservation.invalidate();\n  }\n\n  if (operationFailed && cleanupFailed) throw new AggregateError([operationError, cleanupError], 'native vector scope operation and cleanup failed');\n  if (operationFailed) throw operationError;\n  if (cleanupFailed) throw cleanupError;\n  return result;\n}\n\nexport async function setupMapScope(store: Store, paths: Set<string>): Promise<void> {\n  await materializeScope(store, '_map_scope', paths);\n}\n"],"names":["matchesGlob","anyPresetEmbeds","resolveSearch","columnHint","NATIVE_VECTOR_SCOPE_TABLE","reserveNativeVectorScope","INTERNAL_COLUMNS","Set","inScope","path","include","exclude","some","g","scopeHasEmbeddings","store","cfg","scopedPaths","stmt","prepare","rows","all","r","has","rawScope","overrides","allPaths","effective","adHocScope","undefined","noExclude","presetName","map","paths","allFrontmatterPaths","filter","p","narrowByWhere","where","whereRows","err","docs","columns","wherePaths","materializeScope","table","exec","runBatch","withMaterializedVectorScope","allowed","callback","knownTotal","reservation","vectors","size","invalidate","total","row","get","Number","count","operationFailed","operationError","result","activate","cleanupFailed","cleanupError","deactivate","AggregateError","setupMapScope"],"mappings":"AAAA,SAASA,WAAW,QAAQ,YAAY;AAExC,SAASC,eAAe,EAAEC,aAAa,QAAQ,qBAAqB;AACpE,SAASC,UAAU,QAAQ,2BAA2B;AAEtD,SAASC,yBAAyB,EAAEC,wBAAwB,QAAQ,sBAAsB;AAE1F,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;IAAS;CAAe,EAAE;AAExG,sFAAsF;AACtF,8FAA8F;AAC9F,OAAO,SAASC,QAAQC,IAAY,EAAEC,OAAiB,EAAEC,OAAkB;IACzE,IAAI,CAACD,QAAQE,IAAI,CAAC,CAACC,IAAMb,YAAYS,MAAMI,KAAK,OAAO;IACvD,IAAIF,oBAAAA,8BAAAA,QAASC,IAAI,CAAC,CAACC,IAAMb,YAAYS,MAAMI,KAAK,OAAO;IACvD,OAAO;AACT;AAEA,OAAO,eAAeC,mBAAmBC,KAAY,EAAEC,GAAmB,EAAEC,WAAwB;IAClG,IAAI,CAAChB,gBAAgBe,MAAM,OAAO,OAAO,yDAAyD;IAClG,MAAME,OAAO,MAAMH,MAAMI,OAAO,CAAC;IACjC,MAAMC,OAAQ,MAAMF,KAAKG,GAAG;IAC5B,OAAOD,KAAKR,IAAI,CAAC,CAACU,IAAML,YAAYM,GAAG,CAACD,EAAEb,IAAI;AAChD;AAEA,2FAA2F;AAC3F,0FAA0F;AAC1F,OAAO,eAAee,SAAST,KAAY,EAAEC,GAAmB,EAAES,SAA0B,EAAEC,QAAmB;IAC/G,MAAMC,YAAYzB,cAAcc,KAAKS;IACrC,MAAM,EAAEf,OAAO,EAAEC,OAAO,EAAE,GAAGgB;IAC7B,MAAMC,aAAaH,UAAUf,OAAO,KAAKmB,aAAaJ,UAAUd,OAAO,KAAKkB,aAAaJ,UAAUK,SAAS,KAAK;IACjH,IAAI,CAACF,YAAY;QACf,MAAMV,OAAO,MAAMH,MAAMI,OAAO,CAAC;QACjC,OAAO,IAAIZ,IAAI,AAAE,CAAA,MAAMW,KAAKG,GAAG,CAACM,UAAUI,UAAU,CAAA,EAA+BC,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;IACtG;IACA,MAAMwB,QAAQP,qBAAAA,sBAAAA,WAAa,MAAMQ,oBAAoBnB;IACrD,OAAO,IAAIR,IAAI0B,MAAME,MAAM,CAAC,CAACC,IAAM5B,QAAQ4B,GAAG1B,SAASC;AACzD;AAEA,eAAeuB,oBAAoBnB,KAAY;IAC7C,MAAMG,OAAO,MAAMH,MAAMI,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMD,KAAKG,GAAG,EAAC,EAA+BW,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;AAC1E;AAEA,OAAO,eAAe4B,cAActB,KAAY,EAAEkB,KAAkB,EAAEK,KAAyB;IAC7F,IAAI,CAACA,OAAO,OAAOL;IACnB,IAAIM;IACJ,IAAI;QACF,MAAMrB,OAAO,MAAMH,MAAMI,OAAO,CAAC,CAAC,wCAAwC,EAAEmB,MAAM,CAAC,CAAC;QACpFC,YAAa,MAAMrB,KAAKG,GAAG;IAC7B,EAAE,OAAOmB,KAAK;QACZ,MAAMrC,WAAW,MAAMY,MAAM0B,IAAI,CAACC,OAAO,IAAIF;IAC/C;IACA,MAAMG,aAAa,IAAIpC,IAAIgC,UAAUP,GAAG,CAAC,CAACV,IAAMA,EAAEb,IAAI;IACtD,OAAO,IAAIF,IAAI;WAAI0B;KAAM,CAACE,MAAM,CAAC,CAACC,IAAMO,WAAWpB,GAAG,CAACa;AACzD;AAEA,4FAA4F;AAC5F,kDAAkD;AAClD,OAAO,eAAenB,YAAYF,KAAY,EAAEC,GAAmB,EAAES,SAA0B;IAC7F,MAAME,YAAYzB,cAAcc,KAAKS;IACrC,OAAOY,cAActB,OAAO,MAAMS,SAAST,OAAOC,KAAKS,YAAYE,UAAUW,KAAK;AACpF;AAEA,8FAA8F;AAC9F,yFAAyF;AACzF,OAAO,eAAeM,iBAAiB7B,KAAY,EAAE8B,KAAa,EAAEZ,KAAkB;IACpF,MAAMlB,MAAM+B,IAAI,CAAC,CAAC,qBAAqB,EAAED,OAAO;IAChD,MAAM9B,MAAM+B,IAAI,CAAC,CAAC,kBAAkB,EAAED,MAAM,0BAA0B,CAAC;IACvE,MAAM9B,MAAMgC,QAAQ,CAClB,CAAC,YAAY,EAAEF,MAAM,oBAAoB,CAAC,EAC1C;WAAIZ;KAAM,CAACD,GAAG,CAAC,CAACI,IAAM;YAACA;SAAE;AAE7B;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,gDAAgD;AAChD,OAAO,eAAeY,4BAA+BjC,KAAY,EAAEkC,OAAoB,EAAEC,QAA0B,EAAEC,UAAmB;IACtI,MAAMC,cAAc/C,yBAAyBU,MAAMsC,OAAO,EAAEJ;IAC5D,IAAI,CAACG,aAAa,OAAOF;IAEzB,IAAID,QAAQK,IAAI,KAAK,GAAG;QACtBF,YAAYG,UAAU;QACtB,OAAOL;IACT;IAEA,IAAIM,QAAQL;IACZ,IAAIK,UAAU3B,WAAW;QACvB,IAAI;YACF,MAAMX,OAAO,MAAMH,MAAMI,OAAO,CAAC;YACjC,MAAMsC,MAAO,MAAMvC,KAAKwC,GAAG;YAC3BF,QAAQG,OAAOF,IAAIG,KAAK;QAC1B,EAAE,OAAOpB,KAAK;YACZY,YAAYG,UAAU;YACtB,MAAMf;QACR;IACF;IACA,6FAA6F;IAC7F,iGAAiG;IACjG,IAAIS,QAAQK,IAAI,IAAIE,SAASP,QAAQK,IAAI,GAAGE,QAAQP,QAAQK,IAAI,EAAE;QAChEF,YAAYG,UAAU;QACtB,OAAOL;IACT;IAEA,IAAIW,kBAAkB;IACtB,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,MAAMnB,iBAAiB7B,OAAOX,2BAA2B6C;QACzDG,YAAYY,QAAQ;QACpBD,SAAS,MAAMb;IACjB,EAAE,OAAOV,KAAK;QACZqB,kBAAkB;QAClBC,iBAAiBtB;IACnB;IAEA,IAAIyB,gBAAgB;IACpB,IAAIC;IACJ,IAAI;QACFd,YAAYe,UAAU;QACtB,MAAMpD,MAAM+B,IAAI,CAAC,CAAC,qBAAqB,EAAE1C,2BAA2B;IACtE,EAAE,OAAOoC,KAAK;QACZyB,gBAAgB;QAChBC,eAAe1B;IACjB,SAAU;QACRY,YAAYG,UAAU;IACxB;IAEA,IAAIM,mBAAmBI,eAAe,MAAM,IAAIG,eAAe;QAACN;QAAgBI;KAAa,EAAE;IAC/F,IAAIL,iBAAiB,MAAMC;IAC3B,IAAIG,eAAe,MAAMC;IACzB,OAAOH;AACT;AAEA,OAAO,eAAeM,cAActD,KAAY,EAAEkB,KAAkB;IAClE,MAAMW,iBAAiB7B,OAAO,cAAckB;AAC9C"}

package/dist/esm/commands/search.d.ts

@@ -1,6 +1,7 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { Row } from '../output/output.js';
 import type { Store } from '../store/types.js';
+import type { SearchResultVia } from './signals.js';
+export type { SearchResultVia } from './signals.js';
 export declare const SNIPPET_CHAR_LIMIT_DEFAULT = 80;
 export declare const SNIPPET_COUNT_LIMIT_DEFAULT = 1;
 export declare function computeSnippets(text: string, terms: string[], charLimit: number, countLimit: number): {
@@ -8,6 +9,7 @@ export declare function computeSnippets(text: string, terms: string[], charLimit
     offset: number;
 };
 export declare function lineNumberAt(text: string, offset: number): number;
+/** Options for the root {@link search} API. */
 export interface SearchOptions {
     k?: number;
     where?: string;
@@ -17,6 +19,27 @@ export interface SearchOptions {
     noExclude?: boolean;
     snippetCharLimit?: number;
     snippetCountLimit?: number;
+    /** Cancels supported provider I/O and stops between phases after active native work finishes. */
+    signal?: AbortSignal;
 }
-export declare function hydrateSearchRows(store: Store, cfg: ResolvedConfig, rows: Row[], matchedPaths: Set<string>, terms: string, opts?: Pick<SearchOptions, 'snippetCharLimit' | 'snippetCountLimit'>): Promise<void>;
-export declare function search(store: Store, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<Row[]>;
+export type SearchResult = {
+    path: string;
+    title: string;
+    summary: string;
+    snippets: string[];
+    via: SearchResultVia;
+    score: number;
+    lines: string | null;
+    similarity?: number | null;
+};
+export interface SearchHydrationRow {
+    path: string;
+    snippets?: string[];
+    lines?: string | null;
+}
+export declare function hydrateSearchRows(store: Store, cfg: ResolvedConfig, rows: SearchHydrationRow[], matchedPaths: Set<string>, terms: string, opts?: Pick<SearchOptions, 'snippetCharLimit' | 'snippetCountLimit'>): Promise<void>;
+/**
+ * Searches one committed published generation. Provider work precedes the final read transaction;
+ * cancellation waits for active native work and transaction cleanup before rejection.
+ */
+export declare function search(store: Store, cfg: ResolvedConfig, terms: string, opts?: SearchOptions): Promise<SearchResult[]>;

package/dist/esm/commands/search.js

@@ -1,6 +1,7 @@
+import { channel } from 'node:diagnostics_channel';
 import { stemmer } from 'stemmer';
 import { embedConfig, featureEnabled, resolveSearch } from '../config/index.js';
-import { ensureDocumentEmbeddings } from '../embed/query.js';
+import { ensureDocumentEmbeddings, prepareSemanticQuery } from '../embed/query.js';
 import { localModelMissing, MODEL_FILENAMES } from '../embed/store.js';
 import { SenseError } from '../errors.js';
 import { serialQuery } from '../lib/serial-query.js';
@@ -8,10 +9,12 @@ import { bareTermSyntaxError, searchError } from '../output/search-error.js';
 import { foldForSearch, hasUnspacedRun, searchTokens, unspacedRuns } from '../text/segment.js';
 import { materializeScope, narrowByWhere, rawScope, scopeHasEmbeddings } from './scope.js';
 import { linksCandidates, vectorsCandidates, wordsCandidates } from './signals.js';
+import { assertQuerySnapshot } from './snapshot.js';
 // Default budget behind --snippet-char-limit/--snippet-count-limit: the limit is whatever the
 // caller passes, these only say what happens when they pass nothing (DESIGN.md "Search snippets").
 export const SNIPPET_CHAR_LIMIT_DEFAULT = 80;
 export const SNIPPET_COUNT_LIMIT_DEFAULT = 1;
+const searchCandidatesRead = channel('sensemaking.search.candidates-read');
 // Bare terms from an FTS5 query string: strips operators/quoting, not real terms, so the
 // snippet scan matches what the query matched -- FTS5 operators are uppercase-only, so this is too.
 function extractBareTerms(query) {
@@ -308,38 +311,99 @@ export async function hydrateSearchRows(store, cfg, rows, matchedPaths, terms, o
         if (row.lines == null) row.lines = featureEnabled(cfg, 'sections') ? await lineRangeFor(store, row.path, lineNumberAt(text, offset)) : null;
     }
 }
-// The declared or defaulted signals compose via RRF; `via` names which ones produced each row.
-// `opts` arrives already resolved (config.ts:resolveSearch).
-export function search(store, cfg, terms, opts = {}) {
+/**
+ * Searches one committed published generation. Provider work precedes the final read transaction;
+ * cancellation waits for active native work and transaction cleanup before rejection.
+ */ export function search(store, cfg, terms, opts = {}) {
+    var _opts_signal;
+    if ((_opts_signal = opts.signal) === null || _opts_signal === void 0 ? void 0 : _opts_signal.aborted) return Promise.reject(opts.signal.reason);
     return serialQuery(store, ()=>searchIndexed(store, cfg, terms, opts));
 }
-async function searchIndexed(store, cfg, terms, opts) {
-    validateSearchOptions(opts);
+async function readSearchScope(store, cfg, terms, opts) {
+    const signature = await assertQuerySnapshot(store, cfg);
     const effective = resolveSearch(cfg, opts);
-    const { k, signals } = effective;
     const allPaths = (await (await store.prepare('SELECT "path" FROM frontmatter')).all()).map((r)=>r.path);
     const scopePaths = await rawScope(store, cfg, opts, allPaths);
-    const scopeActive = scopePaths.size < allPaths.length;
-    // The set every candidate pool must be filtered to before truncation: scope narrowed by
-    // --where, the same composition scopedPaths() gives the other commands.
     let allowedPaths;
     try {
         allowedPaths = await narrowByWhere(store, scopePaths, effective.where);
     } catch (err) {
         throw searchError(err, terms, effective.where);
     }
-    const fetch = Math.max(k * 3, 30);
+    const wantsVectors = effective.signals.vectors !== undefined;
+    if (wantsVectors) await ensureDocumentEmbeddings(store, cfg, allowedPaths);
+    const semanticEligible = wantsVectors && await scopeHasEmbeddings(store, cfg, allowedPaths);
+    return {
+        signature,
+        effective,
+        allPaths,
+        scopePaths,
+        allowedPaths,
+        semanticEligible
+    };
+}
+function needsSemanticPreparation(result) {
+    return !Array.isArray(result);
+}
+async function searchIndexed(store, cfg, terms, opts) {
+    var _opts_signal, _opts_signal1, _opts_signal2, _opts_signal3, _opts_signal4;
+    validateSearchOptions(opts);
+    const initial = resolveSearch(cfg, opts);
+    (_opts_signal = opts.signal) === null || _opts_signal === void 0 ? void 0 : _opts_signal.throwIfAborted();
     // A downloadable HF id proceeds -- getProvider fetches it lazily on consent. Only a
     // local path with missing files errors here, since nothing will ever fetch it for itself.
-    const wantsVectors = signals.vectors !== undefined;
+    const wantsVectors = initial.signals.vectors !== undefined;
     if (wantsVectors) {
         const e = embedConfig(cfg); // validate.ts guarantees this is set whenever "vectors" is declared
         if (localModelMissing(e)) {
-            throw new SenseError('EMBED_MODEL_MISSING', `preset "${effective.presetName}" searches with vectors, but the local model path "${e.model}" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop "vectors" from that preset's signals to search without them`);
+            throw new SenseError('EMBED_MODEL_MISSING', `preset "${initial.presetName}" searches with vectors, but the local model path "${e.model}" is missing ${MODEL_FILENAMES}; point embed.model at a directory containing them, or drop "vectors" from that preset's signals to search without them`);
+        }
+    }
+    let prepared;
+    let preparedSignature;
+    if (wantsVectors) {
+        var _opts_signal5;
+        // This first snapshot only decides whether a semantic query can contribute. It closes before
+        // provider/model work, and the complete scope/readiness decision is repeated below.
+        const preflight = await store.transaction(()=>readSearchScope(store, cfg, terms, opts));
+        (_opts_signal5 = opts.signal) === null || _opts_signal5 === void 0 ? void 0 : _opts_signal5.throwIfAborted();
+        if (preflight.semanticEligible) {
+            var _opts_signal6;
+            preparedSignature = preflight.signature;
+            prepared = await prepareSemanticQuery(cfg, terms, {
+                signal: opts.signal
+            });
+            (_opts_signal6 = opts.signal) === null || _opts_signal6 === void 0 ? void 0 : _opts_signal6.throwIfAborted();
         }
-        await ensureDocumentEmbeddings(store, cfg, allowedPaths);
     }
-    const semanticEnabled = wantsVectors && await scopeHasEmbeddings(store, cfg, allowedPaths);
+    (_opts_signal1 = opts.signal) === null || _opts_signal1 === void 0 ? void 0 : _opts_signal1.throwIfAborted();
+    let result = await store.transaction(()=>searchSnapshot(store, cfg, terms, opts, prepared, preparedSignature));
+    (_opts_signal2 = opts.signal) === null || _opts_signal2 === void 0 ? void 0 : _opts_signal2.throwIfAborted();
+    if (!needsSemanticPreparation(result)) return result;
+    // The scope became vector-eligible after an empty/ineligible preflight. Leave that snapshot,
+    // prepare once, then take a fresh final snapshot rather than waiting on the provider inside it.
+    preparedSignature = result.signature;
+    prepared = await prepareSemanticQuery(cfg, terms, {
+        signal: opts.signal
+    });
+    (_opts_signal3 = opts.signal) === null || _opts_signal3 === void 0 ? void 0 : _opts_signal3.throwIfAborted();
+    result = await store.transaction(()=>searchSnapshot(store, cfg, terms, opts, prepared, preparedSignature));
+    (_opts_signal4 = opts.signal) === null || _opts_signal4 === void 0 ? void 0 : _opts_signal4.throwIfAborted();
+    if (needsSemanticPreparation(result)) throw new SenseError('INDEX_NOT_READY', 'the semantic search scope changed while its query was being prepared; retry the search against a stable published index');
+    return result;
+}
+async function searchSnapshot(store, cfg, terms, opts, prepared, preparedSignature) {
+    const { signature, effective, allPaths, scopePaths, allowedPaths, semanticEligible } = await readSearchScope(store, cfg, terms, opts);
+    if (prepared !== undefined && signature !== preparedSignature) {
+        throw new SenseError('INDEX_NOT_READY', 'the index configuration changed while the semantic query was being prepared; retry after the current build finishes, or rebuild the index for this configuration');
+    }
+    if (semanticEligible && prepared === undefined) return {
+        signature
+    };
+    const { k, signals } = effective;
+    const fetch = Math.max(k * 3, 30);
+    const scopeActive = scopePaths.size < allPaths.length;
+    const semanticEnabled = semanticEligible && prepared !== undefined;
     // --where applies inside the candidate query (a post-filter would drop matches ranked past
     // the pool) and again on the final select, for link-derived rows.
     const scope = effective.where;
@@ -372,8 +436,8 @@ async function searchIndexed(store, cfg, terms, opts) {
     if (signals.links !== undefined) await linksCandidates(store, candidates, matchRows, allPaths, allowedPaths, fetch, signals.links);
     let chunkLines = new Map();
     let chunkSimilarity = new Map();
-    if (semanticEnabled) {
-        ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, cfg, candidates, terms, fetch, allowedPaths, signals.vectors));
+    if (semanticEnabled && signals.vectors !== undefined && prepared !== undefined) {
+        ({ chunkLines, chunkSimilarity } = await vectorsCandidates(store, prepared, candidates, fetch, allowedPaths, allPaths.length, signals.vectors));
     }
     await store.exec('DROP TABLE IF EXISTS _search');
     // DOUBLE, not REAL: sqlite's REAL is an 8-byte double but duckdb's is a 4-byte float, and the
@@ -403,8 +467,11 @@ async function searchIndexed(store, cfg, terms, opts) {
     // overwritten below, always to an array, never left null (DESIGN.md).
     const selectStmt = await store.prepare(`SELECT f."path" AS path, content.title, content.summary, NULL AS snippets, _search.via, round(_search.score, 4) AS score, _search.lines${similarityCol}
        FROM _search JOIN frontmatter f ON f."path" = _search."path" JOIN content ON content.path = _search."path"
-       ${where} ORDER BY _search.score DESC LIMIT ?`);
+       ${where} ORDER BY _search.score DESC, _search."path" ASC LIMIT ?`);
     const rows = await selectStmt.all(k);
+    if (searchCandidatesRead.hasSubscribers) searchCandidatesRead.publish({
+        paths: rows.map((row)=>row.path)
+    });
     for (const row of rows)row.snippets = [];
     await hydrateSearchRows(store, cfg, rows, matchedPaths, terms, opts);
     return rows;