sensemaking 0.18.3 → 0.19.0

package/dist/cjs/embed/openai.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // Symmetric for now: per-model prefixes (nomic's search_document/search_query) are a\n  // later workstream.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["openaiProvider","BATCH_CAP","model","url","keyEnv","base","headers","key","dims","post","texts","res","body","data","fetchWithRetry","method","JSON","stringify","input","ok","SenseError","status","json","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","replace","process","env","undefined","authorization","id","batchCap","embedDocuments","embedQuery","text"],"mappings":";;;;+BAQsBA;;;eAAAA;;;wBARK;sBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;;YAE/FC,MACAC,SACAC,KAgBAC;QAbN,SAAeC,KAAKC,KAAe;;oBAC3BC,KAEAC,MAGAC;;;;4BALM;;gCAAMC,IAAAA,sBAAc,EAAC,AAAC,GAAO,OAALT,MAAK,gBAAc;oCAAEU,QAAQ;oCAAQT,SAAAA;oCAASM,MAAMI,KAAKC,SAAS,CAAC;wCAAEf,OAAAA;wCAAOgB,OAAOR;oCAAM;gCAAG;;;4BAA1HC,MAAM;4BACZ,IAAI,CAACA,IAAIQ,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BT,OAA3BN,MAAK,wBAAiC,OAAXM,IAAIU,MAAM;4BAC3E;;gCAAMV,IAAIW,IAAI;;;4BAAtBV,OAAQ;4BACd,yFAAyF;4BACzF,kFAAkF;4BAC5EC,OAAOD,KAAKC,IAAI,CAACU,KAAK,CAAC,SAACC;uCAAM,OAAOA,EAAEC,KAAK,KAAK;iCAAY,AAAC,qBAAGb,KAAKC,IAAI,EAAEa,IAAI,CAAC,SAACC,GAAGC;uCAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK;iCAAeb,KAAKC,IAAI;4BACvJ,wFAAwF;4BACxF,qEAAqE;4BACrE,IAAIA,KAAKgB,MAAM,KAAKnB,MAAMmB,MAAM,EAAE,MAAM,IAAIT,oBAAU,CAAC,eAAe,AAAC,GAA8BP,OAA5BR,MAAK,yBAAqDK,OAA9BG,KAAKgB,MAAM,EAAC,oBAA+B,OAAbnB,MAAMmB,MAAM,EAAC;4BAChJ;;gCAAOhB,KAAKiB,GAAG,CAAC,SAACN;2CAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;;;;;YACtD;;;;;oBAjBA,IAAI,CAAC9B,KAAK,MAAM,IAAIiB,oBAAU,CAAC,eAAe;oBACxCf,OAAOF,IAAI+B,OAAO,CAAC,QAAQ;oBAC3B5B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAAS+B,QAAQC,GAAG,CAAChC,OAAO,GAAGiC;oBAC3C,IAAI9B,KAAKD,QAAQgC,aAAa,GAAG,AAAC,UAAa,OAAJ/B;oBAe7B;;wBAAME;4BAAM;;;;oBAApBD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACqB,MAAM;oBACxD,qFAAqF;oBACrF,oBAAoB;oBACpB;;wBAAO;4BAAEU,IAAI,AAAC,UAAiBrC,OAARG,MAAK,KAAS,OAANH;4BAASM,MAAAA;4BAAMgC,UAAUvC;4BAAWwC,gBAAgBhC;4BAAMiC,YAAY,SAAZA,WAAmBC;;;;;gDAAU;;oDAAMlC;wDAAMkC;;;;;;oDAAZ,aAAmB,CAAC,EAAE;;;;;;wBAAC;;;;IAC/I"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["openaiProvider","BATCH_CAP","model","url","keyEnv","base","headers","key","dims","post","texts","res","body","data","fetchWithRetry","method","JSON","stringify","input","ok","SenseError","status","json","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","replace","process","env","undefined","authorization","id","batchCap","embedDocuments","embedQuery","text"],"mappings":";;;;+BAQsBA;;;eAAAA;;;wBARK;sBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;;YAE/FC,MACAC,SACAC,KAgBAC;QAbN,SAAeC,KAAKC,KAAe;;oBAC3BC,KAEAC,MAGAC;;;;4BALM;;gCAAMC,IAAAA,sBAAc,EAAC,AAAC,GAAO,OAALT,MAAK,gBAAc;oCAAEU,QAAQ;oCAAQT,SAAAA;oCAASM,MAAMI,KAAKC,SAAS,CAAC;wCAAEf,OAAAA;wCAAOgB,OAAOR;oCAAM;gCAAG;;;4BAA1HC,MAAM;4BACZ,IAAI,CAACA,IAAIQ,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BT,OAA3BN,MAAK,wBAAiC,OAAXM,IAAIU,MAAM;4BAC3E;;gCAAMV,IAAIW,IAAI;;;4BAAtBV,OAAQ;4BACd,yFAAyF;4BACzF,kFAAkF;4BAC5EC,OAAOD,KAAKC,IAAI,CAACU,KAAK,CAAC,SAACC;uCAAM,OAAOA,EAAEC,KAAK,KAAK;iCAAY,AAAC,qBAAGb,KAAKC,IAAI,EAAEa,IAAI,CAAC,SAACC,GAAGC;uCAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK;iCAAeb,KAAKC,IAAI;4BACvJ,wFAAwF;4BACxF,qEAAqE;4BACrE,IAAIA,KAAKgB,MAAM,KAAKnB,MAAMmB,MAAM,EAAE,MAAM,IAAIT,oBAAU,CAAC,eAAe,AAAC,GAA8BP,OAA5BR,MAAK,yBAAqDK,OAA9BG,KAAKgB,MAAM,EAAC,oBAA+B,OAAbnB,MAAMmB,MAAM,EAAC;4BAChJ;;gCAAOhB,KAAKiB,GAAG,CAAC,SAACN;2CAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;;;;;YACtD;;;;;oBAjBA,IAAI,CAAC9B,KAAK,MAAM,IAAIiB,oBAAU,CAAC,eAAe;oBACxCf,OAAOF,IAAI+B,OAAO,CAAC,QAAQ;oBAC3B5B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAAS+B,QAAQC,GAAG,CAAChC,OAAO,GAAGiC;oBAC3C,IAAI9B,KAAKD,QAAQgC,aAAa,GAAG,AAAC,UAAa,OAAJ/B;oBAe7B;;wBAAME;4BAAM;;;;oBAApBD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACqB,MAAM;oBACxD,oFAAoF;oBACpF,gDAAgD;oBAChD;;wBAAO;4BAAEU,IAAI,AAAC,UAAiBrC,OAARG,MAAK,KAAS,OAANH;4BAASM,MAAAA;4BAAMgC,UAAUvC;4BAAWwC,gBAAgBhC;4BAAMiC,YAAY,SAAZA,WAAmBC;;;;;gDAAU;;oDAAMlC;wDAAMkC;;;;;;oDAAZ,aAAmB,CAAC,EAAE;;;;;;wBAAC;;;;IAC/I"}

package/dist/cjs/embed/query.js

@@ -25,11 +25,13 @@ _export(exports, {
         return toStore;
     }
 });
+var _nodefs = require("node:fs");
 var _nodepath = require("node:path");
 var _errorsts = require("../errors.js");
 var _embedts = require("../features/embed.js");
 var _progressts = require("../output/progress.js");
 var _indexts = require("../scan/index.js");
+var _handoffts = require("./handoff.js");
 var _langfitts = require("./langfit.js");
 var _registryts = require("./registry.js");
 var _typests = require("./types.js");
@@ -243,7 +245,7 @@ function toStore(full, dims, int8) {
 }
 function embedPending(store, cfg, baseDir) {
     return _async_to_generator(function() {
-        var _loop, provider, dirty, storeDims, byPath, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, _byPath_get, list, jobs, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, _step_value, path, chunkIdxs, chunks, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, idx, report, i;
+        var _loop, _takeChunkText, provider, dirty, storeDims, byPath, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, _byPath_get, list, textByPath, jobs, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _loop1, _iterator1, _step1, report, i;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -324,15 +326,49 @@ function embedPending(store, cfg, baseDir) {
                             }
                         }
                     }
+                    textByPath = (_takeChunkText = (0, _handoffts.takeChunkText)(store)) !== null && _takeChunkText !== void 0 ? _takeChunkText : new Map();
                     jobs = [];
                     _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
                     try {
-                        for(_iterator1 = byPath[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
-                            _step_value = _sliced_to_array(_step1.value, 2), path = _step_value[0], chunkIdxs = _step_value[1];
-                            chunks = void 0;
+                        _loop1 = function() {
+                            var _step_value = _sliced_to_array(_step1.value, 2), path = _step_value[0], chunkIdxs = _step_value[1];
+                            var texts = textByPath.get(path);
+                            if (texts && chunkIdxs.every(function(idx) {
+                                return idx < texts.length;
+                            })) {
+                                // A stat, not a parse: a file that vanished since reconcile is skipped here even though its
+                                // text comes from memory, and the next reconcile removes its rows.
+                                if (!(0, _nodefs.existsSync)((0, _nodepath.join)(baseDir, path))) return "continue";
+                                var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                try {
+                                    for(var _iterator = chunkIdxs[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                        var idx = _step.value;
+                                        jobs.push({
+                                            path: path,
+                                            chunk: idx,
+                                            text: texts[idx]
+                                        });
+                                    }
+                                } catch (err) {
+                                    _didIteratorError = true;
+                                    _iteratorError = err;
+                                } finally{
+                                    try {
+                                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                            _iterator.return();
+                                        }
+                                    } finally{
+                                        if (_didIteratorError) {
+                                            throw _iteratorError;
+                                        }
+                                    }
+                                }
+                                return "continue";
+                            }
+                            // Reconcile ran in another process (`sense map` earlier, or a background `sense watch`), so
+                            // its chunk text is gone: re-derive this one file.
+                            var chunks = void 0;
                             try {
-                                // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already
-                                // has embeddings rows means the tree had an embedding model at reconcile time.
                                 chunks = (0, _indexts.parseFile)({
                                     relPath: path,
                                     absPath: (0, _nodepath.join)(baseDir, path),
@@ -345,33 +381,35 @@ function embedPending(store, cfg, baseDir) {
                                     _embedts.embed
                                 ], cfg).doc.extracted.embed;
                             } catch (unused) {
-                                continue; // vanished since reconcile; the next reconcile removes its rows
+                                return "continue" // vanished since reconcile; the next reconcile removes its rows
+                                ;
                             }
-                            _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+                            var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
                             try {
-                                for(_iterator2 = chunkIdxs[Symbol.iterator](); !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true){
-                                    idx = _step2.value;
-                                    if (chunks[idx]) jobs.push({
+                                for(var _iterator1 = chunkIdxs[Symbol.iterator](), _step2; !(_iteratorNormalCompletion1 = (_step2 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                                    var idx1 = _step2.value;
+                                    if (chunks[idx1]) jobs.push({
                                         path: path,
-                                        chunk: idx,
-                                        text: chunks[idx].text
+                                        chunk: idx1,
+                                        text: chunks[idx1].text
                                     });
                                 }
                             } catch (err) {
-                                _didIteratorError2 = true;
-                                _iteratorError2 = err;
+                                _didIteratorError1 = true;
+                                _iteratorError1 = err;
                             } finally{
                                 try {
-                                    if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
-                                        _iterator2.return();
+                                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                                        _iterator1.return();
                                     }
                                 } finally{
-                                    if (_didIteratorError2) {
-                                        throw _iteratorError2;
+                                    if (_didIteratorError1) {
+                                        throw _iteratorError1;
                                     }
                                 }
                             }
-                        }
+                        };
+                        for(_iterator1 = byPath[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true)_loop1();
                     } catch (err) {
                         _didIteratorError1 = true;
                         _iteratorError1 = err;

package/dist/cjs/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = await store.vectors.pending();\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(store, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["embedPending","hasEmbedding","semanticCandidates","similarNotes","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","store","cfg","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","path","chunkIdxs","chunks","idx","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","writeVectors","job","q","Int8Array","round","chunk","vector","Buffer","from","buffer","tick","min","length","getProvider","pending","STORE_DIMS","Map","get","push","set","parseFile","relPath","absPath","join","mtimeMs","ctimeMs","size","presets","embed","doc","extracted","checkLanguageFit","progress","finish","terms","fetch","allowed","qv","SenseError","match","filter","t","includes","embedQuery","candidates","hasVector","_cfg","opts","similar"],"mappings":";;;;;;;;;;;QA4BsBA;eAAAA;;QAuEAC;eAAAA;;QAfAC;eAAAA;;QAqBAC;eAAAA;;QA3FNC;eAAAA;;;wBAdK;wBAEM;uBAEL;0BACG;uBACC;yBAEO;0BACL;uBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIpB,SAASA,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAef,aAAaiB,KAAY,EAAEC,GAAW,EAAEC,OAAe;;mBACrEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOC,MAAMC,WACZC,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,KAaPC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQP,KAAKS,KAAK,CAACH,GAAGA,IAAIZ,SAASgB,QAAQ;oCACjC;;wCAAMhB,SAASiB,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChB;;wCAAMjB,MAAMiB,OAAO,CAACO,YAAY,CAC9BR,MAAMK,GAAG,CAAC,SAACI,KAAKH;4CACd,IAAqBnC,WAAAA,QAAQ8B,OAAO,CAACK,EAAE,EAAEjB,WAAW,OAA5Cd,IAAaJ,SAAbI,GAAGM,QAAUV,SAAVU;4CACX,IAAM6B,IAAI,IAAIC,UAAUtB;4CACxB,IAAK,IAAIX,IAAI,GAAGA,IAAIW,WAAWX,IAAKgC,CAAC,CAAChC,EAAE,GAAGC,KAAKiC,KAAK,CAACrC,CAAC,CAACG,EAAE,GAAGG;4CAC7D,OAAO;gDAAEa,MAAMe,IAAIf,IAAI;gDAAEmB,OAAOJ,IAAII,KAAK;gDAAEhC,OAAAA;gDAAOiC,QAAQC,OAAOC,IAAI,CAACN,EAAEO,MAAM;4CAAE;wCAClF;;;oCANF;oCAQAnB,OAAOoB,IAAI,CAACvC,KAAKwC,GAAG,CAACpB,IAAIZ,SAASgB,QAAQ,EAAEV,KAAK2B,MAAM;;;;;;oBACzD;oBAhDiB;;wBAAMC,IAAAA,uBAAW,EAACpC;;;oBAA7BE,WAAW;oBACH;;wBAAMH,MAAMiB,OAAO,CAACqB,OAAO;;;oBAAnClC,QAAQ;oBACd,IAAIA,MAAMgC,MAAM,KAAK,GAAG;;;oBAClB/B,YAAYV,KAAKwC,GAAG,CAACI,mBAAU,EAAEpC,SAASd,IAAI;oBAE9CiB,SAAS,IAAIkC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAapC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOmC,GAAG,CAAClC,IAAIG,IAAI,eAAnBJ,yBAAAA;4BACbE,KAAKkC,IAAI,CAACnC,IAAIsB,KAAK;4BACnBvB,OAAOqC,GAAG,CAACpC,IAAIG,IAAI,EAAEF;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOI,uBAAMC;4BACZC,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,+EAA+E;gCAC/EA,SAASgC,IAAAA,kBAAS,EAAC;oCAAEC,SAASnC;oCAAMoC,SAASC,IAAAA,cAAI,EAAC7C,SAASQ;oCAAOsC,SAAS;oCAAGC,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMC,OAAO;gCAAK;oCAAIA,cAAK;mCAAGnD,KAAKoD,GAAG,CAACC,SAAS,CAACF,KAAK;4BAClK,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAazC,gCAAb,8BAAA,SAAA,0BAAA;oCAAME,MAAN;oCAAwB,IAAID,MAAM,CAACC,IAAI,EAAEJ,KAAKiC,IAAI,CAAC;wCAAEhC,MAAAA;wCAAMmB,OAAOhB;wCAAKU,MAAMX,MAAM,CAACC,IAAI,CAACU,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAMgC,IAAAA,2BAAgB,EACpBvD,OACAG,UACAM,KAAKY,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMA,8EAA8E;oBAC9E,6DAA6D;oBACvDT,SAAS0C,IAAAA,oBAAQ,EAAC,oBAAoB/C,KAAK2B,MAAM;oBAC9CrB,IAAI;;;yBAAGA,CAAAA,IAAIN,KAAK2B,MAAM,AAAD;;;;;;;;;;;;oBAAGrB,KAAKZ,SAASgB,QAAQ;;;;;;oBAavDL,OAAO2C,MAAM;;;;;;IACf;;AAKO,SAAexE,mBAAmBe,KAAY,EAAEC,GAAW,EAAEyD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;;YAOvGF,cANRxD,SAIAC,UACAE,WACAkB,MACYpC,UAAP0E;;;;oBAPL3D,UAAU,AAACD,IAAgCC,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAI4D,oBAAU,CAAC,eAAe;oBAClD;;wBAAM/E,aAAaiB,OAAOC,KAAKC;;;oBAA/B;oBAEiB;;wBAAMmC,IAAAA,uBAAW,EAACpC;;;oBAA7BE,WAAW;oBACXE,YAAYV,KAAKwC,GAAG,CAACI,mBAAU,EAAEpC,SAASd,IAAI;oBAC9CkC,OAAO,EAACmC,eAAAA,MAAMK,KAAK,CAAC,2naAAZL,0BAAAA,mBAAsCM,MAAM,CAAC,SAACC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIlB,IAAI,CAAC;oBACxF;;wBAAM5C,SAASgE,UAAU,CAAC5C;;;oBAAlCpC,WAAAA;wBAAQ;wBAAiCkB;wBAAW;wBAA3DwD,KAAO1E,SAAVI;oBAER;;wBAAOS,MAAMiB,OAAO,CAACmD,UAAU,CAACP,IAAIxD,WAAWsD,OAAOC;;;;IACxD;;AAIO,SAAe5E,aAAagB,KAAY,EAAEU,IAAY;;;YAC3D;;gBAAOV,MAAMiB,OAAO,CAACoD,SAAS,CAAC3D;;;IACjC;;AAIO,SAAexB,aAAac,KAAY,EAAEsE,IAAY,EAAE5D,IAAY,EAAE6D,IAAgE;;;YAC3I;;gBAAOvE,MAAMiB,OAAO,CAACuD,OAAO,CAAC9D,MAAM6D;;;IACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the file.\nexport async function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = await store.vectors.pending();\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map<string, string[]>();\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && chunkIdxs.every((idx) => idx < texts.length)) {\n      // A stat, not a parse: a file that vanished since reconcile is skipped here even though its\n      // text comes from memory, and the next reconcile removes its rows.\n      if (!existsSync(join(baseDir, path))) continue;\n      for (const idx of chunkIdxs) jobs.push({ path, chunk: idx, text: texts[idx] });\n      continue;\n    }\n    // Reconcile ran in another process (`sense map` earlier, or a background `sense watch`), so\n    // its chunk text is gone: re-derive this one file.\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(store, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["embedPending","hasEmbedding","semanticCandidates","similarNotes","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","store","cfg","baseDir","takeChunkText","provider","dirty","storeDims","byPath","row","list","textByPath","jobs","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","writeVectors","job","q","Int8Array","round","path","chunk","vector","Buffer","from","buffer","tick","min","length","getProvider","pending","STORE_DIMS","Map","get","push","set","chunkIdxs","texts","every","idx","existsSync","join","chunks","parseFile","relPath","absPath","mtimeMs","ctimeMs","size","presets","embed","doc","extracted","checkLanguageFit","progress","finish","terms","fetch","allowed","qv","SenseError","match","filter","t","includes","embedQuery","candidates","hasVector","_cfg","opts","similar"],"mappings":";;;;;;;;;;;QA8BsBA;eAAAA;;QAgFAC;eAAAA;;QAfAC;eAAAA;;QAqBAC;eAAAA;;QApGNC;eAAAA;;;sBAhBW;wBACN;wBAEM;uBAEL;0BACG;uBACC;yBAEI;yBACG;0BACL;uBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIpB,SAASA,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAef,aAAaiB,KAAY,EAAEC,GAAW,EAAEC,OAAe;;mBAaxDC,gBAZbC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,YAEAC,MACD,4BAAA,oBAAA,yBAAA,YAAA,QA8BCC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQH,KAAKK,KAAK,CAACH,GAAGA,IAAIT,SAASa,QAAQ;oCACjC;;wCAAMb,SAASc,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChB;;wCAAMf,MAAMe,OAAO,CAACO,YAAY,CAC9BR,MAAMK,GAAG,CAAC,SAACI,KAAKH;4CACd,IAAqBjC,WAAAA,QAAQ4B,OAAO,CAACK,EAAE,EAAEd,WAAW,OAA5Cf,IAAaJ,SAAbI,GAAGM,QAAUV,SAAVU;4CACX,IAAM2B,IAAI,IAAIC,UAAUnB;4CACxB,IAAK,IAAIZ,IAAI,GAAGA,IAAIY,WAAWZ,IAAK8B,CAAC,CAAC9B,EAAE,GAAGC,KAAK+B,KAAK,CAACnC,CAAC,CAACG,EAAE,GAAGG;4CAC7D,OAAO;gDAAE8B,MAAMJ,IAAII,IAAI;gDAAEC,OAAOL,IAAIK,KAAK;gDAAE/B,OAAAA;gDAAOgC,QAAQC,OAAOC,IAAI,CAACP,EAAEQ,MAAM;4CAAE;wCAClF;;;oCANF;oCAQApB,OAAOqB,IAAI,CAACtC,KAAKuC,GAAG,CAACrB,IAAIT,SAASa,QAAQ,EAAEN,KAAKwB,MAAM;;;;;;oBACzD;oBA1DiB;;wBAAMC,IAAAA,uBAAW,EAACnC;;;oBAA7BG,WAAW;oBACH;;wBAAMJ,MAAMe,OAAO,CAACsB,OAAO;;;oBAAnChC,QAAQ;oBACd,IAAIA,MAAM8B,MAAM,KAAK,GAAG;;;oBAClB7B,YAAYX,KAAKuC,GAAG,CAACI,mBAAU,EAAElC,SAASf,IAAI;oBAE9CkB,SAAS,IAAIgC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAalC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOiC,GAAG,CAAChC,IAAImB,IAAI,eAAnBpB,yBAAAA;4BACbE,KAAKgC,IAAI,CAACjC,IAAIoB,KAAK;4BACnBrB,OAAOmC,GAAG,CAAClC,IAAImB,IAAI,EAAElB;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC,cAAaP,iBAAAA,IAAAA,wBAAa,EAACH,oBAAdG,4BAAAA,iBAAwB,IAAIoC;oBAEzC5B;oBACD,mCAAA,4BAAA;;;4BAAA,mCAAA,kBAAOgB,uBAAMgB;4BAChB,IAAMC,QAAQlC,WAAW8B,GAAG,CAACb;4BAC7B,IAAIiB,SAASD,UAAUE,KAAK,CAAC,SAACC;uCAAQA,MAAMF,MAAMT,MAAM;gCAAG;gCACzD,4FAA4F;gCAC5F,mEAAmE;gCACnE,IAAI,CAACY,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAAC9C,SAASyB,QAAQ,OAAA;oCACjC,kCAAA,2BAAA;;oCAAL,QAAK,YAAagB,8BAAb,SAAA,6BAAA,QAAA,yBAAA;wCAAA,IAAMG,MAAN;wCAAwBnC,KAAK8B,IAAI,CAAC;4CAAEd,MAAAA;4CAAMC,OAAOkB;4CAAKzB,MAAMuB,KAAK,CAACE,IAAI;wCAAC;;;oCAAvE;oCAAA;;;6CAAA,6BAAA;4CAAA;;;4CAAA;kDAAA;;;;gCACL,OAAA;4BACF;4BACA,4FAA4F;4BAC5F,mDAAmD;4BACnD,IAAIG,SAAAA,KAAAA;4BACJ,IAAI;gCACFA,SAASC,IAAAA,kBAAS,EAAC;oCAAEC,SAASxB;oCAAMyB,SAASJ,IAAAA,cAAI,EAAC9C,SAASyB;oCAAO0B,SAAS;oCAAGC,SAAS;oCAAGC,MAAM;oCAAGC,SAAS,EAAE;oCAAEC,OAAO;gCAAK,GAAG;oCAACA,cAAK;iCAAC,EAAExD,KAAKyD,GAAG,CAACC,SAAS,CAACF,KAAK;4BAClK,EAAE,eAAM;gCACN,OAAA,WAAU,gEAAgE;;4BAC5E;gCACK,mCAAA,4BAAA;;gCAAL,QAAK,aAAad,8BAAb,UAAA,8BAAA,SAAA,0BAAA;oCAAA,IAAMG,OAAN;oCAAwB,IAAIG,MAAM,CAACH,KAAI,EAAEnC,KAAK8B,IAAI,CAAC;wCAAEd,MAAAA;wCAAMC,OAAOkB;wCAAKzB,MAAM4B,MAAM,CAACH,KAAI,CAACzB,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;wBAlBA,IAAK,aAA2Bd,6BAA3B,8BAAA,SAAA,0BAAA;;wBAAA;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAoBL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAMqD,IAAAA,2BAAgB,EACpB5D,OACAI,UACAO,KAAKQ,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMA,8EAA8E;oBAC9E,6DAA6D;oBACvDT,SAASiD,IAAAA,oBAAQ,EAAC,oBAAoBlD,KAAKwB,MAAM;oBAC9CtB,IAAI;;;yBAAGA,CAAAA,IAAIF,KAAKwB,MAAM,AAAD;;;;;;;;;;;;oBAAGtB,KAAKT,SAASa,QAAQ;;;;;;oBAavDL,OAAOkD,MAAM;;;;;;IACf;;AAIO,SAAe7E,mBAAmBe,KAAY,EAAEC,GAAW,EAAE8D,KAAa,EAAEC,KAAa,EAAEC,OAAqB;;YAOvGF,cANR7D,SAIAE,UACAE,WACAe,MACYlC,UAAP+E;;;;oBAPLhE,UAAU,AAACD,IAAgCC,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIiE,oBAAU,CAAC,eAAe;oBAClD;;wBAAMpF,aAAaiB,OAAOC,KAAKC;;;oBAA/B;oBAEiB;;wBAAMkC,IAAAA,uBAAW,EAACnC;;;oBAA7BG,WAAW;oBACXE,YAAYX,KAAKuC,GAAG,CAACI,mBAAU,EAAElC,SAASf,IAAI;oBAC9CgC,OAAO,EAAC0C,eAAAA,MAAMK,KAAK,CAAC,2naAAZL,0BAAAA,mBAAsCM,MAAM,CAAC,SAACC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAItB,IAAI,CAAC;oBACxF;;wBAAM5C,SAASoE,UAAU,CAACnD;;;oBAAlClC,WAAAA;wBAAQ;wBAAiCmB;wBAAW;wBAA3D4D,KAAO/E,SAAVI;oBAER;;wBAAOS,MAAMe,OAAO,CAAC0D,UAAU,CAACP,IAAI5D,WAAW0D,OAAOC;;;;IACxD;;AAIO,SAAejF,aAAagB,KAAY,EAAE2B,IAAY;;;YAC3D;;gBAAO3B,MAAMe,OAAO,CAAC2D,SAAS,CAAC/C;;;IACjC;;AAIO,SAAezC,aAAac,KAAY,EAAE2E,IAAY,EAAEhD,IAAY,EAAEiD,IAAgE;;;YAC3I;;gBAAO5E,MAAMe,OAAO,CAAC8D,OAAO,CAAClD,MAAMiD;;;IACrC"}

package/dist/cjs/embed/registry.js

@@ -83,8 +83,7 @@ function getProvider(cfg) {
     var factory = REGISTRY[e.provider];
     if (!factory) throw new _errorsts.SenseError('EMBED_MODEL', 'embed.provider "'.concat(e.provider, '" has no provider implementation in this build'));
     // Every input that shapes the provider is in the key: two trees sharing an endpoint and
-    // model but declaring different languages or reading different key env vars are two
-    // providers, not one, however long a process lives.
+    // model but declaring different languages or key env vars are two providers, not one.
     var sig = JSON.stringify([
         e.provider,
         e.model,

package/dist/cjs/embed/registry.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or reading different key env vars are two\n  // providers, not one, however long a process lives.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["getProvider","REGISTRY","static","model","staticProvider","openai","url","key","openaiProvider","cohere","cohereProvider","providers","Map","cfg","e","embedConfig","SenseError","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":";;;;+BAoBgBA;;;eAAAA;;;uBAnBY;wBACD;wBACI;wBACA;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK/B,yFAAyF;AACzF,iCAAiC;AACjC,IAAMC,WAA4C;IAChDC,QAAQ,SAARA,QAASC;eAAUC,IAAAA,wBAAc,EAACD;;IAClCE,QAAQ,SAARA,OAASF,OAAOG,KAAKC;eAAQC,IAAAA,wBAAc,EAACL,OAAOG,KAAKC;;IACxDE,QAAQ,SAARA,OAASN,OAAOG,KAAKC;eAAQG,IAAAA,wBAAc,EAACP,OAAOG,KAAKC;;AAC1D;AAEA,IAAMI,YAAY,IAAIC;AAEf,SAASZ,YAAYa,GAAW;QAQYC,QAAaA,QAAaA;IAP3E,IAAMA,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIE,oBAAU,CAAC,kBAAkB;IAC/C,IAAMC,UAAUhB,QAAQ,CAACa,EAAEI,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAID,oBAAU,CAAC,eAAe,AAAC,mBAA6B,OAAXF,EAAEI,QAAQ,EAAC;IAChF,wFAAwF;IACxF,oFAAoF;IACpF,oDAAoD;IACpD,IAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACP,EAAEI,QAAQ;QAAEJ,EAAEX,KAAK;SAAEW,SAAAA,EAAER,GAAG,cAALQ,oBAAAA,SAAS;SAAIA,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,eAAAA,EAAEQ,SAAS,cAAXR,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIS,IAAIZ,UAAUa,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQH,EAAEX,KAAK,EAAEW,EAAER,GAAG,EAAEQ,EAAEP,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIO,EAAEQ,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,SAACC;mBAAU,wCAAKA;gBAAMJ,WAAWR,EAAEQ,SAAS;;;QACxEX,UAAUgB,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC;mBAAMjB,UAAUkB,MAAM,CAACV;;IACjC;IACA,OAAOI;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["getProvider","REGISTRY","static","model","staticProvider","openai","url","key","openaiProvider","cohere","cohereProvider","providers","Map","cfg","e","embedConfig","SenseError","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":";;;;+BAoBgBA;;;eAAAA;;;uBAnBY;wBACD;wBACI;wBACA;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK/B,yFAAyF;AACzF,iCAAiC;AACjC,IAAMC,WAA4C;IAChDC,QAAQ,SAARA,QAASC;eAAUC,IAAAA,wBAAc,EAACD;;IAClCE,QAAQ,SAARA,OAASF,OAAOG,KAAKC;eAAQC,IAAAA,wBAAc,EAACL,OAAOG,KAAKC;;IACxDE,QAAQ,SAARA,OAASN,OAAOG,KAAKC;eAAQG,IAAAA,wBAAc,EAACP,OAAOG,KAAKC;;AAC1D;AAEA,IAAMI,YAAY,IAAIC;AAEf,SAASZ,YAAYa,GAAW;QAOYC,QAAaA,QAAaA;IAN3E,IAAMA,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIE,oBAAU,CAAC,kBAAkB;IAC/C,IAAMC,UAAUhB,QAAQ,CAACa,EAAEI,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAID,oBAAU,CAAC,eAAe,AAAC,mBAA6B,OAAXF,EAAEI,QAAQ,EAAC;IAChF,wFAAwF;IACxF,sFAAsF;IACtF,IAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACP,EAAEI,QAAQ;QAAEJ,EAAEX,KAAK;SAAEW,SAAAA,EAAER,GAAG,cAALQ,oBAAAA,SAAS;SAAIA,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,eAAAA,EAAEQ,SAAS,cAAXR,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIS,IAAIZ,UAAUa,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQH,EAAEX,KAAK,EAAEW,EAAER,GAAG,EAAEQ,EAAEP,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIO,EAAEQ,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,SAACC;mBAAU,wCAAKA;gBAAMJ,WAAWR,EAAEQ,SAAS;;;QACxEX,UAAUgB,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC;mBAAMjB,UAAUkB,MAAM,CAACV;;IACjC;IACA,OAAOI;AACT"}

package/dist/cjs/embed/static.d.cts

@@ -1,2 +1,2 @@
 import type { EmbedProvider } from './types.js';
-export declare function staticProvider(model: string): Promise<EmbedProvider>;
+export declare function staticProvider(model: string, root?: string): Promise<EmbedProvider>;

package/dist/cjs/embed/static.d.ts

@@ -1,2 +1,2 @@
 import type { EmbedProvider } from './types.js';
-export declare function staticProvider(model: string): Promise<EmbedProvider>;
+export declare function staticProvider(model: string, root?: string): Promise<EmbedProvider>;

package/dist/cjs/embed/static.js

@@ -190,17 +190,15 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
 var BATCH_CAP = 64;
-// @huggingface/tokenizers is a 540K static import; a top-level import put it on every command's
-// module graph regardless of whether the semantic path ran, shipped as the 0.17.0 regression.
-// Deferred to a synchronous require inside staticProvider so only the embed path pays for it.
+// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every
+// command's module graph, so it's deferred to a synchronous require inside staticProvider.
 var _require = typeof require === 'undefined' ? _nodemodule.default.createRequire(require("url").pathToFileURL(__filename).toString()) : require;
-function staticProvider(model) {
+function staticProvider(model, root) {
     return _async_to_generator(function() {
         var _ref, _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1, dir, languages, cached, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, raw, headerLen, header, entry, spec, dims, dataStart, matrix, tokenizerJson, Tokenizer, tok, unkId;
         function one(text) {
-            // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an
-            // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the
-            // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.
+            // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it
+            // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.
             var ids = tok.encode(text, {
                 add_special_tokens: false
             }).ids.filter(function(id) {
@@ -249,20 +247,20 @@ function staticProvider(model) {
                         4,
                         (0, _storets.downloadModel)(model, function(file, into) {
                             return console.error("sense: fetching ".concat(model, "/").concat(file, " into ").concat(into));
-                        })
+                        }, root)
                     ];
                 case 1:
                     // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.
                     dir = _state.sent();
                     // Cached beside the repo; a local model directory has no card to read, so it has none.
-                    cached = (0, _storets.readLanguages)(model);
+                    cached = (0, _storets.readLanguages)(model, root);
                     languages = cached && cached.length > 0 ? cached : undefined;
                     return [
                         3,
                         3
                     ];
                 case 2:
-                    dir = (0, _storets.modelDir)(model);
+                    dir = (0, _storets.modelDir)(model, root);
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
                         for(_iterator = _storets.MODEL_FILES[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){

package/dist/cjs/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import put it on every command's\n// module graph regardless of whether the semantic path ran, shipped as the 0.17.0 regression.\n// Deferred to a synchronous require inside staticProvider so only the embed path pays for it.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["staticProvider","BATCH_CAP","_require","require","Module","createRequire","model","tokenizerJson","dir","languages","cached","file","raw","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","text","ids","encode","add_special_tokens","filter","id","Number","isInteger","v","Float32Array","length","off","d","norm","Math","sqrt","isDownloadable","downloadModel","into","console","error","readLanguages","undefined","modelDir","MODEL_FILES","existsSync","join","SenseError","MODEL_FILENAMES","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","slice","vocab","unk_token","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":";;;;+BAgBsBA;;;eAAAA;;;sBAhBmB;iEACtB;wBACE;wBACM;uBAC0E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGrG,IAAMC,YAAY;AAElB,gGAAgG;AAChG,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAInF,SAAeH,eAAeM,KAAa;;kBA+BLC,sBAA7BA,4BAAAA,uBA9BVC,KACAC,WAKIC,QAID,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAOPC,KACAC,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAZ,eACEa,WACFC,KACAC;QAEN,SAASC,IAAIC,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMC,MAAM,AAACJ,IAAIK,MAAM,CAACF,MAAM;gBAAEG,oBAAoB;YAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,SAACC;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOP;;YAC5H,IAAMU,IAAI,IAAIC,aAAahB;YAC3B,IAAIQ,IAAIS,MAAM,KAAK,GAAG,OAAOF;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYP,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMI,KAAN;oBACH,IAAMM,MAAMN,KAAKZ;oBACjB,IAAK,IAAImB,IAAI,GAAGA,IAAInB,MAAMmB,IAAKJ,CAAC,CAACI,EAAE,IAAIjB,MAAM,CAACgB,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAK;gBAC7BJ,CAAC,CAACI,GAAE,IAAIX,IAAIS,MAAM;gBAClBG,QAAQL,CAAC,CAACI,GAAE,GAAGJ,CAAC,CAACI,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAKJ,CAAC,CAACI,GAAE,IAAIC;YACvC,OAAOL;QACT;;;;yBAjDIQ,IAAAA,uBAAc,EAAClC,QAAfkC;;;;oBAEI;;wBAAMC,IAAAA,sBAAa,EAACnC,OAAO,SAACK,MAAM+B;mCAASC,QAAQC,KAAK,CAAC,AAAC,mBAA2BjC,OAATL,OAAM,KAAgBoC,OAAb/B,MAAK,UAAa,OAAL+B;;;;oBADxG,qFAAqF;oBACrFlC,MAAM;oBACN,uFAAuF;oBACjFE,SAASmC,IAAAA,sBAAa,EAACvC;oBAC7BG,YAAYC,UAAUA,OAAOwB,MAAM,GAAG,IAAIxB,SAASoC;;;;;;oBAEnDtC,MAAMuC,IAAAA,iBAAQ,EAACzC;oBACV,kCAAA,2BAAA;;wBAAL,IAAK,YAAc0C,oBAAW,uBAAzB,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBrC,OAAN;4BACH,IAAI,CAACsC,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAAC1C,KAAKG,QAAQ;gCAChC,MAAM,IAAIwC,oBAAU,CAAC,uBAAuB,AAAC,eAAmD3C,OAArCF,OAAM,iCAAiD8C,OAAlB5C,KAAI,gBAA8B,OAAhB4C,wBAAe,EAAC;4BACpI;wBACF;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAODxC,MAAMyC,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK;oBAC7BK,YAAYiB,OAAOlB,IAAI0C,eAAe,CAAC;oBACvCxC,SAASyC,KAAKC,KAAK,CAAC5C,IAAI6C,QAAQ,CAAC,GAAG,IAAI5C,WAAW6C,QAAQ,CAAC;oBAC5D3C,QAAQ4C,OAAOC,OAAO,CAAC9C,QAAQ+C,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC/C,SAASA,KAAK,CAAC,EAAE,CAACgD,KAAK,KAAK,OAAO,MAAM,IAAIZ,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAAN7C,OAAM;oBAC/EU,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKgD,KAAK,CAAC,EAAE;oBACpB9C,YAAYN,IAAIqD,UAAU,GAAG,IAAIpD,YAAYG,KAAKkD,YAAY,CAAC,EAAE;oBACjE/C,SAASD,YAAY,MAAM,IAAI,IAAIe,aAAarB,IAAIuD,MAAM,EAAEjD,WAAWF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,QAAQ,IAAIgB,aAAarB,IAAIuD,MAAM,CAACC,KAAK,CAAClD,WAAWA,YAAYF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,OAAO;oBAE/KV,gBAAgBgD,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK,mBAAmB;oBACnEY,YAAclB,SAAS,2BAAvBkB;oBACFC,MAAM,IAAID,UAAUb,eAAe,CAAC;oBACpCe,iBAAQf,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqB8D,KAAK,cAA1B9D,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqB+D,SAAS,CAAC,uCAAI,CAAC;oBAuB/E,mEAAmE;oBACnE;;wBAAO;4BAAEzC,IAAI,AAAC,UAAe,OAANvB;4BAASW,MAAAA;4BAAMsD,UAAUtE;4BAAWQ,WAAAA;4BAAW+D,gBAAgB,SAAhBA,eAAuBC;;;;;4CAAUA,MAAMC,GAAG,CAACnD;;;;;4BAAMoD,YAAY,SAAZA,WAAmBnD;;;;;4CAASD,IAAIC;;;;;wBAAM;;;;IAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["staticProvider","BATCH_CAP","_require","require","Module","createRequire","model","root","tokenizerJson","dir","languages","cached","file","raw","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","text","ids","encode","add_special_tokens","filter","id","Number","isInteger","v","Float32Array","length","off","d","norm","Math","sqrt","isDownloadable","downloadModel","into","console","error","readLanguages","undefined","modelDir","MODEL_FILES","existsSync","join","SenseError","MODEL_FILENAMES","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","slice","vocab","unk_token","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":";;;;+BAesBA;;;eAAAA;;;sBAfmB;iEACtB;wBACE;wBACM;uBAC0E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGrG,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAInF,SAAeH,eAAeM,KAAa,EAAEC,IAAa;;kBA+BpBC,sBAA7BA,4BAAAA,uBA9BVC,KACAC,WAKIC,QAID,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAOPC,KACAC,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAZ,eACEa,WACFC,KACAC;QAEN,SAASC,IAAIC,IAAY;YACvB,0FAA0F;YAC1F,0FAA0F;YAC1F,IAAMC,MAAM,AAACJ,IAAIK,MAAM,CAACF,MAAM;gBAAEG,oBAAoB;YAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,SAACC;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOP;;YAC5H,IAAMU,IAAI,IAAIC,aAAahB;YAC3B,IAAIQ,IAAIS,MAAM,KAAK,GAAG,OAAOF;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYP,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMI,KAAN;oBACH,IAAMM,MAAMN,KAAKZ;oBACjB,IAAK,IAAImB,IAAI,GAAGA,IAAInB,MAAMmB,IAAKJ,CAAC,CAACI,EAAE,IAAIjB,MAAM,CAACgB,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAK;gBAC7BJ,CAAC,CAACI,GAAE,IAAIX,IAAIS,MAAM;gBAClBG,QAAQL,CAAC,CAACI,GAAE,GAAGJ,CAAC,CAACI,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAKJ,CAAC,CAACI,GAAE,IAAIC;YACvC,OAAOL;QACT;;;;yBAhDIQ,IAAAA,uBAAc,EAACnC,QAAfmC;;;;oBAEI;;wBAAMC,IAAAA,sBAAa,EAACpC,OAAO,SAACM,MAAM+B;mCAASC,QAAQC,KAAK,CAAC,AAAC,mBAA2BjC,OAATN,OAAM,KAAgBqC,OAAb/B,MAAK,UAAa,OAAL+B;2BAASpC;;;oBADjH,qFAAqF;oBACrFE,MAAM;oBACN,uFAAuF;oBACjFE,SAASmC,IAAAA,sBAAa,EAACxC,OAAOC;oBACpCG,YAAYC,UAAUA,OAAOwB,MAAM,GAAG,IAAIxB,SAASoC;;;;;;oBAEnDtC,MAAMuC,IAAAA,iBAAQ,EAAC1C,OAAOC;oBACjB,kCAAA,2BAAA;;wBAAL,IAAK,YAAc0C,oBAAW,uBAAzB,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBrC,OAAN;4BACH,IAAI,CAACsC,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAAC1C,KAAKG,QAAQ;gCAChC,MAAM,IAAIwC,oBAAU,CAAC,uBAAuB,AAAC,eAAmD3C,OAArCH,OAAM,iCAAiD+C,OAAlB5C,KAAI,gBAA8B,OAAhB4C,wBAAe,EAAC;4BACpI;wBACF;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAODxC,MAAMyC,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK;oBAC7BK,YAAYiB,OAAOlB,IAAI0C,eAAe,CAAC;oBACvCxC,SAASyC,KAAKC,KAAK,CAAC5C,IAAI6C,QAAQ,CAAC,GAAG,IAAI5C,WAAW6C,QAAQ,CAAC;oBAC5D3C,QAAQ4C,OAAOC,OAAO,CAAC9C,QAAQ+C,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC/C,SAASA,KAAK,CAAC,EAAE,CAACgD,KAAK,KAAK,OAAO,MAAM,IAAIZ,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAAN9C,OAAM;oBAC/EW,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKgD,KAAK,CAAC,EAAE;oBACpB9C,YAAYN,IAAIqD,UAAU,GAAG,IAAIpD,YAAYG,KAAKkD,YAAY,CAAC,EAAE;oBACjE/C,SAASD,YAAY,MAAM,IAAI,IAAIe,aAAarB,IAAIuD,MAAM,EAAEjD,WAAWF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,QAAQ,IAAIgB,aAAarB,IAAIuD,MAAM,CAACC,KAAK,CAAClD,WAAWA,YAAYF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,OAAO;oBAE/KV,gBAAgBgD,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK,mBAAmB;oBACnEY,YAAcnB,SAAS,2BAAvBmB;oBACFC,MAAM,IAAID,UAAUb,eAAe,CAAC;oBACpCe,iBAAQf,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqB8D,KAAK,cAA1B9D,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqB+D,SAAS,CAAC,uCAAI,CAAC;oBAsB/E,mEAAmE;oBACnE;;wBAAO;4BAAEzC,IAAI,AAAC,UAAe,OAANxB;4BAASY,MAAAA;4BAAMsD,UAAUvE;4BAAWS,WAAAA;4BAAW+D,gBAAgB,SAAhBA,eAAuBC;;;;;4CAAUA,MAAMC,GAAG,CAACnD;;;;;4BAAMoD,YAAY,SAAZA,WAAmBnD;;;;;4CAASD,IAAIC;;;;;wBAAM;;;;IAC/J"}

package/dist/cjs/embed/store.d.cts

@@ -4,5 +4,5 @@ export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir,
 export declare function modelPresent(cfg: Config): boolean;
 type ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;
 export declare function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig;
-export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string>;
-export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;
+export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root?: string): Promise<string>;
+export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root?: string): Promise<string>;

package/dist/cjs/embed/store.d.ts

@@ -4,5 +4,5 @@ export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir,
 export declare function modelPresent(cfg: Config): boolean;
 type ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;
 export declare function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig;
-export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string>;
-export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;
+export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root?: string): Promise<string>;
+export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root?: string): Promise<string>;

package/dist/cjs/embed/store.js

@@ -195,13 +195,15 @@ function localModelMissing(e) {
 }
 // `main` resolves to a sha once per model: a recorded ref pins until its snapshot is
 // removed, so only the first need for an id ever costs this HEAD request.
-function resolveSha(model) {
-    return _async_to_generator(function() {
-        var existing, url, res, err, sha;
+function resolveSha(_0) {
+    return _async_to_generator(function(model) {
+        var root, existing, url, res, err, sha;
+        var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    existing = (0, _identityts.readRef)(model);
+                    root = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : _identityts.DEFAULT_MODELS_ROOT;
+                    existing = (0, _identityts.readRef)(model, root);
                     if (existing) return [
                         2,
                         existing
@@ -234,19 +236,17 @@ function resolveSha(model) {
                 case 4:
                     sha = res.headers.get('x-repo-commit');
                     if (!sha) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(url, ": response carried no x-repo-commit header"));
-                    (0, _identityts.writeRef)(model, sha);
+                    (0, _identityts.writeRef)(model, sha, root);
                     return [
                         2,
                         sha
                     ];
             }
         });
-    })();
+    }).apply(this, arguments);
 }
-// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has
-// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an
-// array here). Tags are a fallback for cards that only tag languages, filtered to recognized
-// codes so an unrelated short tag (format markers etc.) is never mistaken for one.
+// cardData.language is the structured field; tags are a fallback for cards that only tag
+// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.
 function languagesFromCard(info) {
     var _info_tags;
     var _info_cardData;
@@ -258,21 +258,26 @@ function languagesFromCard(info) {
     var tagLangs = ((_info_tags = info.tags) !== null && _info_tags !== void 0 ? _info_tags : []).filter(_languagests.isKnownLanguageTag);
     return tagLangs.length > 0 ? tagLangs : undefined;
 }
-// Best-effort, cached forever once resolved (even to "none"): unlike model weights, missing or
-// unreachable metadata is not an error -- it just means the language-fit check stays off.
-function ensureLanguages(model) {
-    return _async_to_generator(function() {
-        var _languagesFromCard, measured, res, unused, info, unused1;
+// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:
+// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).
+function cardUnreadable(model) {
+    console.error("sense: could not read the model card for ".concat(model, "; the language-fit check is off until it resolves (retries on next run, or run `sense download`)"));
+}
+function ensureLanguages(_0) {
+    return _async_to_generator(function(model) {
+        var root, _languagesFromCard, measured, res, unused, info, unused1;
+        var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    if ((0, _nodefs.existsSync)((0, _identityts.languagesPath)(model))) return [
+                    root = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : _identityts.DEFAULT_MODELS_ROOT;
+                    if ((0, _nodefs.existsSync)((0, _identityts.languagesPath)(model, root))) return [
                         2
                     ];
                     // A measured entry needs no card and no network, and stays deterministic offline.
                     measured = _languagests.MEASURED_MODEL_LANGUAGES[model];
                     if (measured) {
-                        (0, _identityts.writeLanguages)(model, measured);
+                        (0, _identityts.writeLanguages)(model, measured, root);
                         return [
                             2
                         ];
@@ -297,13 +302,17 @@ function ensureLanguages(model) {
                     ];
                 case 3:
                     unused = _state.sent();
+                    cardUnreadable(model);
                     return [
                         2
                     ];
                 case 4:
-                    if (!res.ok) return [
-                        2
-                    ];
+                    if (!res.ok) {
+                        cardUnreadable(model);
+                        return [
+                            2
+                        ];
+                    }
                     _state.label = 5;
                 case 5:
                     _state.trys.push([
@@ -324,17 +333,18 @@ function ensureLanguages(model) {
                     ];
                 case 7:
                     unused1 = _state.sent();
+                    cardUnreadable(model);
                     return [
                         2
                     ];
                 case 8:
-                    (0, _identityts.writeLanguages)(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : []);
+                    (0, _identityts.writeLanguages)(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : [], root);
                     return [
                         2
                     ];
             }
         });
-    })();
+    }).apply(this, arguments);
 }
 function fetchToFile(url, dest) {
     return _async_to_generator(function() {
@@ -384,13 +394,15 @@ function fetchToFile(url, dest) {
         });
     })();
 }
-function downloadModelRevision(model, sha, onFile) {
-    return _async_to_generator(function() {
-        var dir, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, err;
+function downloadModelRevision(_0, _1, _2) {
+    return _async_to_generator(function(model, sha, onFile) {
+        var root, dir, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, err;
+        var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    dir = (0, _identityts.snapshotDir)(model, sha);
+                    root = _arguments.length > 3 && _arguments[3] !== void 0 ? _arguments[3] : _identityts.DEFAULT_MODELS_ROOT;
+                    dir = (0, _identityts.snapshotDir)(model, sha, root);
                     (0, _nodefs.mkdirSync)(dir, {
                         recursive: true
                     });
@@ -462,20 +474,22 @@ function downloadModelRevision(model, sha, onFile) {
                     ];
             }
         });
-    })();
+    }).apply(this, arguments);
 }
-function downloadModel(model, onFile) {
-    return _async_to_generator(function() {
-        var sha;
+function downloadModel(_0, _1) {
+    return _async_to_generator(function(model, onFile) {
+        var root, sha;
+        var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
+                    root = _arguments.length > 2 && _arguments[2] !== void 0 ? _arguments[2] : _identityts.DEFAULT_MODELS_ROOT;
                     if (!(0, _identityts.isDownloadable)(model)) {
                         throw new _errorsts.SenseError('EMBED_MODEL', 'embed.model "'.concat(model, '" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like "minishlab/potion-retrieval-32M"'));
                     }
                     return [
                         4,
-                        resolveSha(model)
+                        resolveSha(model, root)
                     ];
                 case 1:
                     sha = _state.sent();
@@ -483,16 +497,16 @@ function downloadModel(model, onFile) {
                     // construction and an explicit prefetch both end up with a cached language verdict.
                     return [
                         4,
-                        ensureLanguages(model)
+                        ensureLanguages(model, root)
                     ];
                 case 2:
                     _state.sent();
                     return [
                         2,
-                        downloadModelRevision(model, sha, onFile)
+                        downloadModelRevision(model, sha, onFile, root)
                     ];
             }
         });
-    })();
+    }).apply(this, arguments);
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }