sensemaking 0.15.3 → 0.16.0

package/dist/esm/embed/openai.d.ts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider>;

package/dist/esm/embed/openai.js

@@ -0,0 +1,49 @@
+import { SenseError } from '../errors.js';
+import { fetchWithRetry } from './http.js';
+const BATCH_CAP = 64;
+// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare
+// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.
+export async function openaiProvider(model, url, keyEnv) {
+    if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
+    const base = url.replace(/\/+$/, '');
+    const headers = {
+        'content-type': 'application/json'
+    };
+    const key = keyEnv ? process.env[keyEnv] : undefined;
+    if (key) headers.authorization = `Bearer ${key}`;
+    async function post(texts) {
+        const res = await fetchWithRetry(`${base}/embeddings`, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                model,
+                input: texts
+            })
+        });
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);
+        const body = await res.json();
+        // The schema carries `index` because response order is not promised, and the caller maps
+        // vectors back to chunks positionally: sort by it whenever every element has one.
+        const data = body.data.every((d)=>typeof d.index === 'number') ? [
+            ...body.data
+        ].sort((a, b)=>a.index - b.index) : body.data;
+        // A server that drops an input (empty or over-length strings are the usual cause) would
+        // otherwise shift every later vector onto the wrong chunk, silently.
+        if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);
+        return data.map((d)=>Float32Array.from(d.embedding));
+    }
+    const dims = (await post([
+        'dimension probe'
+    ]))[0].length;
+    // Symmetric for now: per-model prefixes (nomic's search_document/search_query) are a
+    // later workstream.
+    return {
+        id: `openai:${base}:${model}`,
+        dims,
+        batchCap: BATCH_CAP,
+        embedDocuments: post,
+        embedQuery: async (text)=>(await post([
+                text
+            ]))[0]
+    };
+}

package/dist/esm/embed/openai.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // Symmetric for now: per-model prefixes (nomic's search_document/search_query) are a\n  // later workstream.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["SenseError","fetchWithRetry","BATCH_CAP","openaiProvider","model","url","keyEnv","base","replace","headers","key","process","env","undefined","authorization","post","texts","res","method","body","JSON","stringify","input","ok","status","json","data","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,YAAY;AAG3C,MAAMC,YAAY;AAElB,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,IAAI,CAACD,KAAK,MAAM,IAAIL,WAAW,eAAe;IAC9C,MAAMO,OAAOF,IAAIG,OAAO,CAAC,QAAQ;IACjC,MAAMC,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMJ,SAASK,QAAQC,GAAG,CAACN,OAAO,GAAGO;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKC,KAAe;QACjC,MAAMC,MAAM,MAAMhB,eAAe,GAAGM,KAAK,WAAW,CAAC,EAAE;YAAEW,QAAQ;YAAQT;YAASU,MAAMC,KAAKC,SAAS,CAAC;gBAAEjB;gBAAOkB,OAAON;YAAM;QAAG;QAChI,IAAI,CAACC,IAAIM,EAAE,EAAE,MAAM,IAAIvB,WAAW,eAAe,GAAGO,KAAK,oBAAoB,EAAEU,IAAIO,MAAM,EAAE;QAC3F,MAAML,OAAQ,MAAMF,IAAIQ,IAAI;QAC5B,yFAAyF;QACzF,kFAAkF;QAClF,MAAMC,OAAOP,KAAKO,IAAI,CAACC,KAAK,CAAC,CAACC,IAAM,OAAOA,EAAEC,KAAK,KAAK,YAAY;eAAIV,KAAKO,IAAI;SAAC,CAACI,IAAI,CAAC,CAACC,GAAGC,IAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK,IAAeV,KAAKO,IAAI;QACvJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIA,KAAKO,MAAM,KAAKjB,MAAMiB,MAAM,EAAE,MAAM,IAAIjC,WAAW,eAAe,GAAGO,KAAK,qBAAqB,EAAEmB,KAAKO,MAAM,CAAC,gBAAgB,EAAEjB,MAAMiB,MAAM,CAAC,OAAO,CAAC;QACxJ,OAAOP,KAAKQ,GAAG,CAAC,CAACN,IAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;IACtD;IAEA,MAAMC,OAAO,AAAC,CAAA,MAAMvB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACkB,MAAM;IACxD,qFAAqF;IACrF,oBAAoB;IACpB,OAAO;QAAEM,IAAI,CAAC,OAAO,EAAEhC,KAAK,CAAC,EAAEH,OAAO;QAAEkC;QAAME,UAAUtC;QAAWuC,gBAAgB1B;QAAM2B,YAAY,OAAOC,OAAS,AAAC,CAAA,MAAM5B,KAAK;gBAAC4B;aAAK,CAAA,CAAE,CAAC,EAAE;IAAC;AAC/I"}

package/dist/esm/embed/query.d.ts

@@ -0,0 +1,21 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Config } from '../config/index.js';
+export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
+    v: Float32Array;
+    scale: number;
+};
+export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
+    path: string;
+    lines: string;
+    similarity: number;
+}>>;
+export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
+export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+    exclude: Set<string>;
+    allowed?: Set<string>;
+    k: number;
+}): Array<{
+    path: string;
+    similarity: number;
+}>;

package/dist/esm/embed/query.js

@@ -0,0 +1,181 @@
+import { join } from 'node:path';
+import { SenseError } from '../errors.js';
+import { embed } from '../features/embed.js';
+import { progress } from '../progress.js';
+import { parseFile } from '../scan.js';
+import { checkLanguageFit } from './langfit.js';
+import { getProvider } from './registry.js';
+// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is
+// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
+const STORE_DIMS = 256;
+// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.
+const TARGET_CHUNK_CAP = 16;
+// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for
+// benchmark/lib/embed.mjs, which scores the same lever math offline.
+export function toStore(full, dims, int8) {
+    const v = new Float32Array(dims);
+    let norm = 0;
+    for(let d = 0; d < dims; d++)norm += full[d] * full[d];
+    norm = Math.sqrt(norm) + 1e-32;
+    for(let d = 0; d < dims; d++)v[d] = full[d] / norm;
+    if (!int8) return {
+        v,
+        scale: 1
+    };
+    let max = 0;
+    for(let d = 0; d < dims; d++)max = Math.max(max, Math.abs(v[d]));
+    return {
+        v,
+        scale: max / 127 || 1
+    };
+}
+// Embed rows whose vector is NULL, re-deriving chunk text from the files through the
+// same parse + chunker that stored the rows.
+export async function embedPending(db, cfg, baseDir) {
+    const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table
+    const dirty = db.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk').all();
+    if (dirty.length === 0) return;
+    const storeDims = Math.min(STORE_DIMS, provider.dims);
+    const byPath = new Map();
+    for (const row of dirty){
+        var _byPath_get;
+        const list = (_byPath_get = byPath.get(row.path)) !== null && _byPath_get !== void 0 ? _byPath_get : [];
+        list.push(row.chunk);
+        byPath.set(row.path, list);
+    }
+    const jobs = [];
+    for (const [path, chunkIdxs] of byPath){
+        let chunks;
+        try {
+            // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already
+            // has embeddings rows means the tree had an embedding model at reconcile time.
+            chunks = parseFile({
+                relPath: path,
+                absPath: join(baseDir, path),
+                mtimeMs: 0,
+                ctimeMs: 0,
+                size: 0,
+                presets: [],
+                embed: true
+            }, [
+                embed
+            ]).doc.extracted.embed;
+        } catch  {
+            continue; // vanished since reconcile; the next reconcile removes its rows
+        }
+        for (const idx of chunkIdxs)if (chunks[idx]) jobs.push({
+            path,
+            chunk: idx,
+            text: chunks[idx].text
+        });
+    }
+    // Over the exact texts about to be embedded, before any of them are: a mismatch fails
+    // this run loudly instead of quietly storing vectors from the wrong model.
+    await checkLanguageFit(db, provider, jobs.map((j)=>j.text));
+    const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE "path" = ? AND chunk = ?');
+    // The lazy build is the one long silence a first search hits (measured 23s at
+    // 26k notes); progress makes it distinguishable from a hang.
+    const report = progress('embedding chunks', jobs.length);
+    for(let i = 0; i < jobs.length; i += provider.batchCap){
+        const batch = jobs.slice(i, i + provider.batchCap);
+        const vectors = await provider.embedDocuments(batch.map((j)=>j.text));
+        db.exec('BEGIN');
+        try {
+            batch.forEach((job, j)=>{
+                const { v, scale } = toStore(vectors[j], storeDims, true);
+                const q = new Int8Array(storeDims);
+                for(let d = 0; d < storeDims; d++)q[d] = Math.round(v[d] / scale);
+                update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);
+            });
+            db.exec('COMMIT');
+        } catch (err) {
+            db.exec('ROLLBACK');
+            throw err;
+        }
+        report.tick(Math.min(i + provider.batchCap, jobs.length));
+    }
+    report.finish();
+}
+// Dequantised int8 dot products land a little either side of a true cosine, so an identical
+// pair prints 1.001 and undermines a column whose whole job is being a bounded number.
+function asCosine(score) {
+    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
+}
+// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as
+// lexical syntax. Similarity comes back because the fused score cannot express match quality,
+// and nearest-neighbour search always returns a neighbour however far away.
+export async function semanticCandidates(db, cfg, terms, fetch, allowed) {
+    var _terms_match;
+    const baseDir = cfg.baseDir;
+    if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');
+    await embedPending(db, cfg, baseDir);
+    const provider = await getProvider(cfg);
+    const storeDims = Math.min(STORE_DIMS, provider.dims);
+    const text = ((_terms_match = terms.match(/[\p{L}\p{N}]+/gu)) !== null && _terms_match !== void 0 ? _terms_match : []).filter((t)=>![
+            'AND',
+            'OR',
+            'NOT',
+            'NEAR'
+        ].includes(t)).join(' ');
+    const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);
+    const rows = db.prepare('SELECT "path", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
+    const best = new Map();
+    for (const row of rows){
+        if (allowed && !allowed.has(row.path)) continue;
+        const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));
+        let dot = 0;
+        for(let d = 0; d < q.length; d++)dot += q[d] * qv[d];
+        const score = dot * row.scale;
+        const existing = best.get(row.path);
+        if (!existing || score > existing.score) best.set(row.path, {
+            score,
+            lines: `L${row.start_line}-${row.end_line}`
+        });
+    }
+    return [
+        ...best.entries()
+    ].sort((a, b)=>b[1].score - a[1].score).slice(0, fetch).map(([path, b])=>({
+            path,
+            lines: b.lines,
+            similarity: asCosine(b.score)
+        }));
+}
+// Whether a note has any chunk with a vector. Distinguishes "nothing is near this note" from
+// "this note has no text", which look the same in an empty result.
+export function hasEmbedding(db, path) {
+    const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE "path" = ? AND vector IS NOT NULL LIMIT 1').get(path);
+    return row !== undefined;
+}
+// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear
+// scan of stored vectors. Reads only what is stored, so it stays sync.
+export function similarNotes(db, _cfg, path, opts) {
+    // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus
+    // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.
+    const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk').all(path);
+    if (targetRows.length === 0) return [];
+    const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
+    const target = targetRows.filter((_, i)=>i % step === 0).map((row)=>({
+            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
+            scale: row.scale
+        }));
+    const rows = db.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
+    const best = new Map();
+    for (const row of rows){
+        if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
+        const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+        for (const t of target){
+            let dot = 0;
+            const len = Math.min(t.v.length, other.length);
+            for(let d = 0; d < len; d++)dot += t.v[d] * other[d];
+            const score = dot * t.scale * row.scale;
+            const existing = best.get(row.path);
+            if (existing === undefined || score > existing) best.set(row.path, score);
+        }
+    }
+    return [
+        ...best.entries()
+    ].sort((a, b)=>b[1] - a[1]).slice(0, opts.k).map(([p, score])=>({
+            path: p,
+            similarity: asCosine(score)
+        }));
+}

package/dist/esm/embed/query.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["join","SenseError","embed","progress","parseFile","checkLanguageFit","getProvider","STORE_DIMS","TARGET_CHUNK_CAP","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","db","cfg","baseDir","provider","dirty","prepare","all","length","storeDims","min","byPath","Map","row","list","get","path","push","chunk","set","jobs","chunkIdxs","chunks","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","extracted","idx","text","map","j","update","report","i","batchCap","batch","slice","vectors","embedDocuments","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","err","tick","finish","asCosine","score","semanticCandidates","terms","fetch","allowed","match","filter","t","includes","qv","embedQuery","rows","best","has","vector","byteOffset","byteLength","dot","existing","lines","start_line","end_line","entries","sort","a","b","similarity","hasEmbedding","undefined","similarNotes","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":"AAAA,SAASA,IAAI,QAAQ,YAAY;AAGjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,iBAAiB;AAC1C,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAE5C,6EAA6E;AAC7E,yEAAyE;AACzE,MAAMC,aAAa;AACnB,0FAA0F;AAC1F,MAAMC,mBAAmB;AAEzB,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAaC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC/E,MAAMC,WAAW,MAAMpB,YAAYkB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQJ,GAAGK,OAAO,CAAC,oFAAoFC,GAAG;IAChH,IAAIF,MAAMG,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IAEpD,MAAMsB,SAAS,IAAIC;IACnB,KAAK,MAAMC,OAAOR,MAAO;YACVM;QAAb,MAAMG,QAAOH,cAAAA,OAAOI,GAAG,CAACF,IAAIG,IAAI,eAAnBL,yBAAAA,cAAwB,EAAE;QACvCG,KAAKG,IAAI,CAACJ,IAAIK,KAAK;QACnBP,OAAOQ,GAAG,CAACN,IAAIG,IAAI,EAAEF;IACvB;IAEA,MAAMM,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACJ,MAAMK,UAAU,IAAIV,OAAQ;QACtC,IAAIW;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAASxC,UAAU;gBAAEyC,SAASP;gBAAMQ,SAAS9C,KAAKyB,SAASa;gBAAOS,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAEhD,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEiD,GAAG,CAACC,SAAS,CAAClD,KAAK;QAC7J,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMmD,OAAOV,UAAW,IAAIC,MAAM,CAACS,IAAI,EAAEX,KAAKH,IAAI,CAAC;YAAED;YAAME,OAAOa;YAAKC,MAAMV,MAAM,CAACS,IAAI,CAACC,IAAI;QAAC;IACrG;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAMjD,iBACJkB,IACAG,UACAgB,KAAKa,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;IAGxB,MAAMG,SAASlC,GAAGK,OAAO,CAAC;IAC1B,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAM8B,SAASvD,SAAS,oBAAoBuC,KAAKZ,MAAM;IACvD,IAAK,IAAI6B,IAAI,GAAGA,IAAIjB,KAAKZ,MAAM,EAAE6B,KAAKjC,SAASkC,QAAQ,CAAE;QACvD,MAAMC,QAAQnB,KAAKoB,KAAK,CAACH,GAAGA,IAAIjC,SAASkC,QAAQ;QACjD,MAAMG,UAAU,MAAMrC,SAASsC,cAAc,CAACH,MAAMN,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;QACrE/B,GAAG0C,IAAI,CAAC;QACR,IAAI;YACFJ,MAAMK,OAAO,CAAC,CAACC,KAAKX;gBAClB,MAAM,EAAE3C,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQsD,OAAO,CAACP,EAAE,EAAEzB,WAAW;gBACpD,MAAMqC,IAAI,IAAIC,UAAUtC;gBACxB,IAAK,IAAIf,IAAI,GAAGA,IAAIe,WAAWf,IAAKoD,CAAC,CAACpD,EAAE,GAAGC,KAAKqD,KAAK,CAACzD,CAAC,CAACG,EAAE,GAAGG;gBAC7DsC,OAAOc,GAAG,CAACpD,OAAOqD,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAI7B,IAAI,EAAE6B,IAAI3B,KAAK;YAC9D;YACAjB,GAAG0C,IAAI,CAAC;QACV,EAAE,OAAOU,KAAK;YACZpD,GAAG0C,IAAI,CAAC;YACR,MAAMU;QACR;QACAjB,OAAOkB,IAAI,CAAC3D,KAAKe,GAAG,CAAC2B,IAAIjC,SAASkC,QAAQ,EAAElB,KAAKZ,MAAM;IACzD;IACA4B,OAAOmB,MAAM;AACf;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAO9D,KAAKqD,KAAK,CAACrD,KAAKe,GAAG,CAAC,GAAGf,KAAKG,GAAG,CAAC,CAAC,GAAG2D,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmBzD,EAAgB,EAAEC,GAAW,EAAEyD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;QAO3GF;IANd,MAAMxD,UAAU,AAACD,IAAgCC,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIxB,WAAW,eAAe;IAClD,MAAMqB,aAAaC,IAAIC,KAAKC;IAE5B,MAAMC,WAAW,MAAMpB,YAAYkB;IACnC,MAAMO,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IACpD,MAAM2C,OAAO,EAAC2B,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAEI,MAAM,CAAC,CAACC,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAItF,IAAI,CAAC;IAClH,MAAM,EAAEa,GAAG2E,EAAE,EAAE,GAAG/E,QAAQ,MAAMiB,SAAS+D,UAAU,CAACnC,OAAOvB,WAAW;IAEtE,MAAM2D,OAAOnE,GAAGK,OAAO,CAAC,+FAA+FC,GAAG;IAQ1H,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIP,WAAW,CAACA,QAAQS,GAAG,CAACzD,IAAIG,IAAI,GAAG;QACvC,MAAM8B,IAAI,IAAIC,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE7E,KAAKe,GAAG,CAACD,WAAWI,IAAI0D,MAAM,CAACE,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIhF,IAAI,GAAGA,IAAIoD,EAAEtC,MAAM,EAAEd,IAAKgF,OAAO5B,CAAC,CAACpD,EAAE,GAAGwE,EAAE,CAACxE,EAAE;QACtD,MAAM+D,QAAQiB,MAAM7D,IAAIhB,KAAK;QAC7B,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;QAClC,IAAI,CAAC2D,YAAYlB,QAAQkB,SAASlB,KAAK,EAAEY,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAE;YAAEyC;YAAOmB,OAAO,CAAC,CAAC,EAAE/D,IAAIgE,UAAU,CAAC,CAAC,EAAEhE,IAAIiE,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK,EACtCjB,KAAK,CAAC,GAAGoB,OACT3B,GAAG,CAAC,CAAC,CAACjB,MAAMkE,EAAE,GAAM,CAAA;YAAElE;YAAM4D,OAAOM,EAAEN,KAAK;YAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;QAAE,CAAA;AAC/E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,SAAS2B,aAAanF,EAAgB,EAAEe,IAAY;IACzD,MAAMH,MAAMZ,GAAGK,OAAO,CAAC,kFAAkFS,GAAG,CAACC;IAC7G,OAAOH,QAAQwE;AACjB;AAEA,+FAA+F;AAC/F,uEAAuE;AACvE,OAAO,SAASC,aAAarF,EAAgB,EAAEsF,IAAY,EAAEvE,IAAY,EAAEwE,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,MAAMC,aAAaxF,GAAGK,OAAO,CAAC,+FAA+FC,GAAG,CAACS;IACjI,IAAIyE,WAAWjF,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMkF,OAAO/F,KAAKG,GAAG,CAAC,GAAGH,KAAKgG,IAAI,CAACF,WAAWjF,MAAM,GAAGtB;IACvD,MAAM0G,SAASH,WAAW1B,MAAM,CAAC,CAAC8B,GAAGxD,IAAMA,IAAIqD,SAAS,GAAGzD,GAAG,CAAC,CAACpB,MAAS,CAAA;YAAEtB,GAAG,IAAIwD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;YAAG5E,OAAOgB,IAAIhB,KAAK;QAAC,CAAA;IAE/K,MAAMuE,OAAOnE,GAAGK,OAAO,CAAC,yEAAyEC,GAAG;IACpG,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIvD,IAAIG,IAAI,KAAKA,QAAQwE,KAAKM,OAAO,CAACxB,GAAG,CAACzD,IAAIG,IAAI,KAAMwE,KAAK3B,OAAO,IAAI,CAAC2B,KAAK3B,OAAO,CAACS,GAAG,CAACzD,IAAIG,IAAI,GAAI;QACtG,MAAM+E,QAAQ,IAAIhD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;QAC3F,KAAK,MAAMT,KAAK4B,OAAQ;YACtB,IAAIlB,MAAM;YACV,MAAMsB,MAAMrG,KAAKe,GAAG,CAACsD,EAAEzE,CAAC,CAACiB,MAAM,EAAEuF,MAAMvF,MAAM;YAC7C,IAAK,IAAId,IAAI,GAAGA,IAAIsG,KAAKtG,IAAKgF,OAAOV,EAAEzE,CAAC,CAACG,EAAE,GAAGqG,KAAK,CAACrG,EAAE;YACtD,MAAM+D,QAAQiB,MAAMV,EAAEnE,KAAK,GAAGgB,IAAIhB,KAAK;YACvC,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;YAClC,IAAI2D,aAAaU,aAAa5B,QAAQkB,UAAUN,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAEyC;QACrE;IACF;IAEA,OAAO;WAAIY,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BzC,KAAK,CAAC,GAAGgD,KAAKS,CAAC,EACfhE,GAAG,CAAC,CAAC,CAACiE,GAAGzC,MAAM,GAAM,CAAA;YAAEzC,MAAMkF;YAAGf,YAAY3B,SAASC;QAAO,CAAA;AACjE"}

package/dist/esm/embed/registry.d.ts

@@ -0,0 +1,3 @@
+import type { Config } from '../config/index.js';
+import type { EmbedProvider } from './types.js';
+export declare function getProvider(cfg: Config): Promise<EmbedProvider>;

package/dist/esm/embed/registry.js

@@ -0,0 +1,45 @@
+import { embedConfig } from '../config/index.js';
+import { SenseError } from '../errors.js';
+import { cohereProvider } from './cohere.js';
+import { openaiProvider } from './openai.js';
+import { staticProvider } from './static.js';
+// One factory per wire protocol: adding a protocol is a new file plus an entry here, not
+// an edit to a dispatch ternary.
+const REGISTRY = {
+    static: (model)=>staticProvider(model),
+    openai: (model, url, key)=>openaiProvider(model, url, key),
+    cohere: (model, url, key)=>cohereProvider(model, url, key)
+};
+const providers = new Map();
+export function getProvider(cfg) {
+    var _e_url, _e_key, _e_languages;
+    const e = embedConfig(cfg);
+    if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an "embed" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');
+    const factory = REGISTRY[e.provider];
+    if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider "${e.provider}" has no provider implementation in this build`);
+    // Every input that shapes the provider is in the key: two trees sharing an endpoint and
+    // model but declaring different languages or reading different key env vars are two
+    // providers, not one, however long a process lives.
+    const sig = JSON.stringify([
+        e.provider,
+        e.model,
+        (_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : '',
+        (_e_key = e.key) !== null && _e_key !== void 0 ? _e_key : '',
+        (_e_languages = e.languages) !== null && _e_languages !== void 0 ? _e_languages : []
+    ]);
+    let p = providers.get(sig);
+    if (!p) {
+        p = factory(e.model, e.url, e.key);
+        // Owner-declared languages override a provider's own: the config is the one home for
+        // what no model card can state (http models have no fetchable card here).
+        if (e.languages) p = p.then((prov)=>({
+                ...prov,
+                languages: e.languages
+            }));
+        providers.set(sig, p);
+        // A rejected construction is evicted, so a transient failure retries instead of
+        // poisoning this config for the process lifetime.
+        p.catch(()=>providers.delete(sig));
+    }
+    return p;
+}

package/dist/esm/embed/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or reading different key env vars are two\n  // providers, not one, however long a process lives.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","getProvider","cfg","e","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQR,eAAeK,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,OAAO,SAASC,YAAYC,GAAW;QAQYC,QAAaA,QAAaA;IAP3E,MAAMA,IAAIhB,YAAYe;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIf,WAAW,kBAAkB;IAC/C,MAAMgB,UAAUZ,QAAQ,CAACW,EAAEE,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAIhB,WAAW,eAAe,CAAC,gBAAgB,EAAEe,EAAEE,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,wFAAwF;IACxF,oFAAoF;IACpF,oDAAoD;IACpD,MAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEE,QAAQ;QAAEF,EAAET,KAAK;SAAES,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,SAAAA,EAAEN,GAAG,cAALM,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIO,IAAIX,UAAUY,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAET,KAAK,EAAES,EAAEP,GAAG,EAAEO,EAAEN,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIM,EAAEM,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,CAACC,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEJ,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzEV,UAAUe,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC,IAAMhB,UAAUiB,MAAM,CAACV;IACjC;IACA,OAAOI;AACT"}

package/dist/esm/embed/static.d.ts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function staticProvider(model: string): Promise<EmbedProvider>;

package/dist/esm/embed/static.js

@@ -0,0 +1,72 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { SenseError } from '../errors.js';
+import { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.js';
+const BATCH_CAP = 64;
+// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no
+// special tokens, drop unk ids, mean-pool, L2-normalize.
+export async function staticProvider(model) {
+    var _ref;
+    var _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1;
+    let dir;
+    let languages;
+    if (isDownloadable(model)) {
+        // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.
+        dir = await downloadModel(model, (file, into)=>console.error(`sense: fetching ${model}/${file} into ${into}`));
+        // Cached beside the repo; a local model directory has no card to read, so it has none.
+        const cached = readLanguages(model);
+        languages = cached && cached.length > 0 ? cached : undefined;
+    } else {
+        dir = modelDir(model);
+        for (const file of MODEL_FILES){
+            if (!existsSync(join(dir, file))) {
+                throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);
+            }
+        }
+    }
+    const raw = readFileSync(join(dir, 'model.safetensors'));
+    const headerLen = Number(raw.readBigUInt64LE(0));
+    const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8'));
+    const entry = Object.entries(header).find(([k])=>k !== '__metadata__');
+    if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);
+    const spec = entry[1];
+    const dims = spec.shape[1];
+    const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];
+    const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));
+    const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));
+    // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.
+    const { Tokenizer } = await import('@huggingface/tokenizers');
+    const tok = new Tokenizer(tokenizerJson, {});
+    const unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
+    function one(text) {
+        // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an
+        // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the
+        // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.
+        const ids = tok.encode(text, {
+            add_special_tokens: false
+        }).ids.filter((id)=>Number.isInteger(id) && id !== unkId);
+        const v = new Float32Array(dims);
+        if (ids.length === 0) return v;
+        for (const id of ids){
+            const off = id * dims;
+            for(let d = 0; d < dims; d++)v[d] += matrix[off + d];
+        }
+        let norm = 0;
+        for(let d = 0; d < dims; d++){
+            v[d] /= ids.length;
+            norm += v[d] * v[d];
+        }
+        norm = Math.sqrt(norm) + 1e-32;
+        for(let d = 0; d < dims; d++)v[d] /= norm;
+        return v;
+    }
+    // Symmetric model: document and query embedding are the same call.
+    return {
+        id: `static:${model}`,
+        dims,
+        batchCap: BATCH_CAP,
+        languages,
+        embedDocuments: async (texts)=>texts.map(one),
+        embedQuery: async (text)=>one(text)
+    };
+}

package/dist/esm/embed/static.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","staticProvider","model","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeC,eAAeC,KAAa;;QAgCLC,sBAA7BA,4BAAAA;IA/Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIV,eAAeO,QAAQ;QACzB,qFAAqF;QACrFE,MAAM,MAAMV,cAAcQ,OAAO,CAACI,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAEP,MAAM,CAAC,EAAEI,KAAK,MAAM,EAAEC,MAAM;QAC9G,uFAAuF;QACvF,MAAMG,SAASX,cAAcG;QAC7BG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMN,SAASI;QACf,KAAK,MAAMI,QAAQT,YAAa;YAC9B,IAAI,CAACP,WAAWE,KAAKY,KAAKE,QAAQ;gBAChC,MAAM,IAAIb,WAAW,uBAAuB,CAAC,YAAY,EAAES,MAAM,6BAA6B,EAAEE,IAAI,YAAY,EAAER,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMiB,MAAMtB,aAAaC,KAAKY,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIlC,WAAW,eAAe,GAAGS,MAAM,oCAAoC,CAAC;IAC1H,MAAM0B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAC5B,aAAaC,KAAKY,KAAK,mBAAmB;IAC3E,oFAAoF;IACpF,MAAM,EAAEkC,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC;IACnC,MAAMC,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAE/C,OAAO;QAAE2B;QAAM4B,UAAUzD;QAAWK;QAAWqD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}

package/dist/esm/embed/store.d.ts

@@ -0,0 +1,8 @@
+import type { Config } from '../config/index.js';
+import { embedConfig } from '../config/index.js';
+export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.js';
+export declare function modelPresent(cfg: Config): boolean;
+type ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;
+export declare function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig;
+export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string>;
+export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;

package/dist/esm/embed/store.js

@@ -0,0 +1,123 @@
+import { existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { embedConfig } from '../config/index.js';
+import { SenseError } from '../errors.js';
+import { fetchWithRetry } from './http.js';
+import { hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.js';
+import { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.js';
+// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches
+// (downloads, sha resolution, language metadata). Re-exported here for callers that otherwise
+// only need store.ts's network surface.
+export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.js';
+// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an
+// EMBED_MODEL error at call time instead.
+export function modelPresent(cfg) {
+    const e = embedConfig(cfg);
+    if (!e) return false;
+    return e.provider !== 'static' || hasModelFiles(e.model);
+}
+// True when embed.model names a local path (not a Hugging Face id) that is missing its model
+// files -- nothing will ever fetch it for itself, unlike a downloadable id.
+export function localModelMissing(e) {
+    return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);
+}
+// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is
+// removed, so only the first need for an id ever costs this HEAD request.
+async function resolveSha(model) {
+    const existing = readRef(model);
+    if (existing) return existing;
+    const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;
+    // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN
+    // redirect would read the CDN's headers instead, which do not carry it.
+    let res;
+    try {
+        res = await fetchWithRetry(url, {
+            method: 'HEAD',
+            redirect: 'manual'
+        });
+    } catch (err) {
+        throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${err.message}`);
+    }
+    const sha = res.headers.get('x-repo-commit');
+    if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);
+    writeRef(model, sha);
+    return sha;
+}
+// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has
+// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an
+// array here). Tags are a fallback for cards that only tag languages, filtered to recognized
+// codes so an unrelated short tag (format markers etc.) is never mistaken for one.
+function languagesFromCard(info) {
+    var _info_tags;
+    var _info_cardData;
+    const lang = (_info_cardData = info.cardData) === null || _info_cardData === void 0 ? void 0 : _info_cardData.language;
+    if (typeof lang === 'string') return [
+        lang
+    ];
+    if (Array.isArray(lang) && lang.length > 0) return lang;
+    const tagLangs = ((_info_tags = info.tags) !== null && _info_tags !== void 0 ? _info_tags : []).filter(isKnownLanguageTag);
+    return tagLangs.length > 0 ? tagLangs : undefined;
+}
+// Best-effort, cached forever once resolved (even to "none"): unlike model weights, missing or
+// unreachable metadata is not an error -- it just means the language-fit check stays off.
+async function ensureLanguages(model) {
+    var _languagesFromCard;
+    if (existsSync(languagesPath(model))) return;
+    // A measured entry needs no card and no network, and stays deterministic offline.
+    const measured = MEASURED_MODEL_LANGUAGES[model];
+    if (measured) {
+        writeLanguages(model, measured);
+        return;
+    }
+    let res;
+    try {
+        res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});
+    } catch  {
+        return;
+    }
+    if (!res.ok) return;
+    let info;
+    try {
+        info = await res.json();
+    } catch  {
+        return;
+    }
+    writeLanguages(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : []);
+}
+async function fetchToFile(url, dest) {
+    let res;
+    try {
+        res = await fetchWithRetry(url, {});
+    } catch (err) {
+        throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${err.message}`);
+    }
+    if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);
+    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));
+    renameSync(`${dest}.part`, dest);
+}
+// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --
+// for downloadModel below, and for callers pinning an exact revision (the parity suite).
+export async function downloadModelRevision(model, sha, onFile) {
+    const dir = snapshotDir(model, sha);
+    mkdirSync(dir, {
+        recursive: true
+    });
+    for (const file of MODEL_FILES){
+        if (existsSync(join(dir, file))) continue;
+        onFile === null || onFile === void 0 ? void 0 : onFile(file, dir);
+        await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));
+    }
+    return dir;
+}
+// The only code path that resolves `main`: a static provider's first construction calls it
+// lazily, `sense download` as an explicit prefetch. Idempotent either way.
+export async function downloadModel(model, onFile) {
+    if (!isDownloadable(model)) {
+        throw new SenseError('EMBED_MODEL', `embed.model "${model}" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like "minishlab/potion-retrieval-32M"`);
+    }
+    const sha = await resolveSha(model);
+    // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first
+    // construction and an explicit prefetch both end up with a cached language verdict.
+    await ensureLanguages(model);
+    return downloadModelRevision(model, sha, onFile);
+}

package/dist/esm/embed/store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync, renameSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata). Re-exported here for callers that otherwise\n// only need store.ts's network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string): Promise<string> {\n  const existing = readRef(model);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has\n// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an\n// array here). Tags are a fallback for cards that only tag languages, filtered to recognized\n// codes so an unrelated short tag (format markers etc.) is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Best-effort, cached forever once resolved (even to \"none\"): unlike model weights, missing or\n// unreachable metadata is not an error -- it just means the language-fit check stays off.\nasync function ensureLanguages(model: string): Promise<void> {\n  if (existsSync(languagesPath(model))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    return;\n  }\n  if (!res.ok) return;\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? []);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n  renameSync(`${dest}.part`, dest);\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  const dir = snapshotDir(model, sha);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model);\n  return downloadModelRevision(model, sha, onFile);\n}\n"],"names":["existsSync","mkdirSync","renameSync","writeFileSync","join","embedConfig","SenseError","fetchWithRetry","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","existing","url","res","method","redirect","err","message","sha","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","undefined","ensureLanguages","measured","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,EAAEC,UAAU,EAAEC,aAAa,QAAQ,UAAU;AAC3E,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,YAAY;AAC3C,SAASC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC1I,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,8FAA8F;AAC9F,wCAAwC;AACxC,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAInB,YAAYkB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa;IACrC,MAAMG,WAAWjB,QAAQc;IACzB,IAAIG,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEJ,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,IAAIK;IACJ,IAAI;QACFA,MAAM,MAAMxB,eAAeuB,KAAK;YAAEE,QAAQ;YAAQC,UAAU;QAAS;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI5B,WAAW,eAAe,CAAC,gBAAgB,EAAEwB,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,MAAMC,MAAML,IAAIM,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAI9B,WAAW,eAAe,GAAGwB,IAAI,0CAA0C,CAAC;IAChGf,SAASW,OAAOU;IAChB,OAAOA;AACT;AAOA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,mFAAmF;AACnF,SAASG,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,MAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAACjC;IAC1C,OAAO+B,SAASD,MAAM,GAAG,IAAIC,WAAWG;AAC1C;AAEA,+FAA+F;AAC/F,0FAA0F;AAC1F,eAAeC,gBAAgBzB,KAAa;QAqBpBa;IApBtB,IAAIvC,WAAWU,cAAcgB,SAAS;IACtC,kFAAkF;IAClF,MAAM0B,WAAWnC,wBAAwB,CAACS,MAAM;IAChD,IAAI0B,UAAU;QACZtC,eAAeY,OAAO0B;QACtB;IACF;IACA,IAAIrB;IACJ,IAAI;QACFA,MAAM,MAAMxB,eAAe,CAAC,kCAAkC,EAAEmB,OAAO,EAAE,CAAC;IAC5E,EAAE,OAAM;QACN;IACF;IACA,IAAI,CAACK,IAAIsB,EAAE,EAAE;IACb,IAAIb;IACJ,IAAI;QACFA,OAAQ,MAAMT,IAAIuB,IAAI;IACxB,EAAE,OAAM;QACN;IACF;IACAxC,eAAeY,QAAOa,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE;AACrD;AAEA,eAAegB,YAAYzB,GAAW,EAAE0B,IAAY;IAClD,IAAIzB;IACJ,IAAI;QACFA,MAAM,MAAMxB,eAAeuB,KAAK,CAAC;IACnC,EAAE,OAAOI,KAAK;QACZ,MAAM,IAAI5B,WAAW,eAAe,CAAC,gBAAgB,EAAEwB,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,IAAI,CAACJ,IAAIsB,EAAE,EAAE,MAAM,IAAI/C,WAAW,eAAe,GAAGwB,IAAI,SAAS,EAAEC,IAAI0B,MAAM,EAAE;IAC/EtD,cAAc,GAAGqD,KAAK,KAAK,CAAC,EAAEE,OAAOC,IAAI,CAAC,MAAM5B,IAAI6B,WAAW;IAC/D1D,WAAW,GAAGsD,KAAK,KAAK,CAAC,EAAEA;AAC7B;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeK,sBAAsBnC,KAAa,EAAEU,GAAW,EAAE0B,MAA4C;IAClH,MAAMC,MAAMlD,YAAYa,OAAOU;IAC/BnC,UAAU8D,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQtD,YAAa;QAC9B,IAAIX,WAAWI,KAAK2D,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMR,YAAY,CAAC,uBAAuB,EAAE7B,MAAM,SAAS,EAAEU,IAAI,CAAC,EAAE6B,MAAM,EAAE7D,KAAK2D,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAcxC,KAAa,EAAEoC,MAA4C;IAC7F,IAAI,CAACrD,eAAeiB,QAAQ;QAC1B,MAAM,IAAIpB,WAAW,eAAe,CAAC,aAAa,EAAEoB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMU,MAAM,MAAMR,WAAWF;IAC7B,sFAAsF;IACtF,oFAAoF;IACpF,MAAMyB,gBAAgBzB;IACtB,OAAOmC,sBAAsBnC,OAAOU,KAAK0B;AAC3C"}