sensemaking 0.15.4 → 0.17.0

package/dist/esm/embed/cohere.js

@@ -0,0 +1,46 @@
+import { SenseError } from '../errors.js';
+import { fetchWithRetry } from './http.js';
+const BATCH_CAP = 96;
+// Cohere's native /v2/embed: input_type distinguishes doc vs query embeddings, which the
+// OpenAI-compatible shape cannot express.
+export async function cohereProvider(model, url, keyEnv) {
+    const base = (url !== null && url !== void 0 ? url : 'https://api.cohere.com').replace(/\/+$/, '');
+    const headers = {
+        'content-type': 'application/json'
+    };
+    const key = keyEnv ? process.env[keyEnv] : undefined;
+    if (key) headers.authorization = `Bearer ${key}`;
+    async function post(texts, inputType) {
+        const res = await fetchWithRetry(`${base}/v2/embed`, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                model,
+                input_type: inputType,
+                texts,
+                embedding_types: [
+                    'float'
+                ]
+            })
+        });
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/v2/embed -> HTTP ${res.status}`);
+        const body = await res.json();
+        // Cohere returns one vector per text in order, with no index field to re-sort by; the
+        // count is what a caller mapping positionally still has to be able to trust.
+        const vectors = body.embeddings.float;
+        if (vectors.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/v2/embed returned ${vectors.length} embeddings for ${texts.length} inputs`);
+        return vectors.map((v)=>Float32Array.from(v));
+    }
+    const dims = (await post([
+        'dimension probe'
+    ], 'search_query'))[0].length;
+    return {
+        id: `cohere:${base}:${model}`,
+        dims,
+        batchCap: BATCH_CAP,
+        embedDocuments: (texts)=>post(texts, 'search_document'),
+        embedQuery: async (text)=>(await post([
+                text
+            ], 'search_query'))[0]
+    };
+}

package/dist/esm/embed/cohere.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/cohere.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 96;\n\n// Cohere's native /v2/embed: input_type distinguishes doc vs query embeddings, which the\n// OpenAI-compatible shape cannot express.\nexport async function cohereProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  const base = (url ?? 'https://api.cohere.com').replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[], inputType: 'search_document' | 'search_query'): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/v2/embed`, { method: 'POST', headers, body: JSON.stringify({ model, input_type: inputType, texts, embedding_types: ['float'] }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/v2/embed -> HTTP ${res.status}`);\n    const body = (await res.json()) as { embeddings: { float: number[][] } };\n    // Cohere returns one vector per text in order, with no index field to re-sort by; the\n    // count is what a caller mapping positionally still has to be able to trust.\n    const vectors = body.embeddings.float;\n    if (vectors.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/v2/embed returned ${vectors.length} embeddings for ${texts.length} inputs`);\n    return vectors.map((v) => Float32Array.from(v));\n  }\n\n  const dims = (await post(['dimension probe'], 'search_query'))[0].length;\n  return { id: `cohere:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: (texts) => post(texts, 'search_document'), embedQuery: async (text) => (await post([text], 'search_query'))[0] };\n}\n"],"names":["SenseError","fetchWithRetry","BATCH_CAP","cohereProvider","model","url","keyEnv","base","replace","headers","key","process","env","undefined","authorization","post","texts","inputType","res","method","body","JSON","stringify","input_type","embedding_types","ok","status","json","vectors","embeddings","float","length","map","v","Float32Array","from","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,YAAY;AAG3C,MAAMC,YAAY;AAElB,yFAAyF;AACzF,0CAA0C;AAC1C,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,MAAMC,OAAO,CAACF,gBAAAA,iBAAAA,MAAO,0BAA0BG,OAAO,CAAC,QAAQ;IAC/D,MAAMC,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMJ,SAASK,QAAQC,GAAG,CAACN,OAAO,GAAGO;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKC,KAAe,EAAEC,SAA6C;QAChF,MAAMC,MAAM,MAAMjB,eAAe,GAAGM,KAAK,SAAS,CAAC,EAAE;YAAEY,QAAQ;YAAQV;YAASW,MAAMC,KAAKC,SAAS,CAAC;gBAAElB;gBAAOmB,YAAYN;gBAAWD;gBAAOQ,iBAAiB;oBAAC;iBAAQ;YAAC;QAAG;QAC1K,IAAI,CAACN,IAAIO,EAAE,EAAE,MAAM,IAAIzB,WAAW,eAAe,GAAGO,KAAK,kBAAkB,EAAEW,IAAIQ,MAAM,EAAE;QACzF,MAAMN,OAAQ,MAAMF,IAAIS,IAAI;QAC5B,sFAAsF;QACtF,6EAA6E;QAC7E,MAAMC,UAAUR,KAAKS,UAAU,CAACC,KAAK;QACrC,IAAIF,QAAQG,MAAM,KAAKf,MAAMe,MAAM,EAAE,MAAM,IAAI/B,WAAW,eAAe,GAAGO,KAAK,mBAAmB,EAAEqB,QAAQG,MAAM,CAAC,gBAAgB,EAAEf,MAAMe,MAAM,CAAC,OAAO,CAAC;QAC5J,OAAOH,QAAQI,GAAG,CAAC,CAACC,IAAMC,aAAaC,IAAI,CAACF;IAC9C;IAEA,MAAMG,OAAO,AAAC,CAAA,MAAMrB,KAAK;QAAC;KAAkB,EAAE,eAAc,CAAE,CAAC,EAAE,CAACgB,MAAM;IACxE,OAAO;QAAEM,IAAI,CAAC,OAAO,EAAE9B,KAAK,CAAC,EAAEH,OAAO;QAAEgC;QAAME,UAAUpC;QAAWqC,gBAAgB,CAACvB,QAAUD,KAAKC,OAAO;QAAoBwB,YAAY,OAAOC,OAAS,AAAC,CAAA,MAAM1B,KAAK;gBAAC0B;aAAK,EAAE,eAAc,CAAE,CAAC,EAAE;IAAC;AACpM"}

package/dist/esm/embed/http.d.ts

@@ -0,0 +1,2 @@
+export declare function fetchWithRetry(url: string, init: RequestInit, baseDelayMs?: number): Promise<Response>;
+export declare function probeReachable(url: string, timeoutMs?: number): Promise<boolean>;

package/dist/esm/embed/http.js

@@ -0,0 +1,33 @@
+const MAX_ATTEMPTS = 3;
+function sleep(ms) {
+    return new Promise((resolve)=>setTimeout(resolve, ms));
+}
+// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential
+// backoff honoring Retry-After; any other status returns immediately for the caller to handle.
+export async function fetchWithRetry(url, init, baseDelayMs = 200) {
+    for(let attempt = 0; attempt < MAX_ATTEMPTS; attempt++){
+        try {
+            const res = await fetch(url, init);
+            if (res.ok || res.status !== 429 && res.status < 500 || attempt === MAX_ATTEMPTS - 1) return res;
+            const retryAfter = Number(res.headers.get('retry-after'));
+            // Capped: a server naming a huge Retry-After must not stall an embed run for it.
+            await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10000) : baseDelayMs * 2 ** attempt);
+        } catch (err) {
+            if (attempt === MAX_ATTEMPTS - 1) throw err;
+            await sleep(baseDelayMs * 2 ** attempt);
+        }
+    }
+    throw new Error('unreachable');
+}
+// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as
+// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.
+export async function probeReachable(url, timeoutMs = 1500) {
+    try {
+        await fetch(url, {
+            signal: AbortSignal.timeout(timeoutMs)
+        });
+        return true;
+    } catch  {
+        return false;
+    }
+}

package/dist/esm/embed/http.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["const MAX_ATTEMPTS = 3;\n\nfunction sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\n// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential\n// backoff honoring Retry-After; any other status returns immediately for the caller to handle.\nexport async function fetchWithRetry(url: string, init: RequestInit, baseDelayMs = 200): Promise<Response> {\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    try {\n      const res = await fetch(url, init);\n      if (res.ok || (res.status !== 429 && res.status < 500) || attempt === MAX_ATTEMPTS - 1) return res;\n      const retryAfter = Number(res.headers.get('retry-after'));\n      // Capped: a server naming a huge Retry-After must not stall an embed run for it.\n      await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10_000) : baseDelayMs * 2 ** attempt);\n    } catch (err) {\n      if (attempt === MAX_ATTEMPTS - 1) throw err;\n      await sleep(baseDelayMs * 2 ** attempt);\n    }\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as\n// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  try {\n    await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n    return true;\n  } catch {\n    return false;\n  }\n}\n"],"names":["MAX_ATTEMPTS","sleep","ms","Promise","resolve","setTimeout","fetchWithRetry","url","init","baseDelayMs","attempt","res","fetch","ok","status","retryAfter","Number","headers","get","Math","min","err","Error","probeReachable","timeoutMs","signal","AbortSignal","timeout"],"mappings":"AAAA,MAAMA,eAAe;AAErB,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,2FAA2F;AAC3F,+FAA+F;AAC/F,OAAO,eAAeI,eAAeC,GAAW,EAAEC,IAAiB,EAAEC,cAAc,GAAG;IACpF,IAAK,IAAIC,UAAU,GAAGA,UAAUV,cAAcU,UAAW;QACvD,IAAI;YACF,MAAMC,MAAM,MAAMC,MAAML,KAAKC;YAC7B,IAAIG,IAAIE,EAAE,IAAKF,IAAIG,MAAM,KAAK,OAAOH,IAAIG,MAAM,GAAG,OAAQJ,YAAYV,eAAe,GAAG,OAAOW;YAC/F,MAAMI,aAAaC,OAAOL,IAAIM,OAAO,CAACC,GAAG,CAAC;YAC1C,iFAAiF;YACjF,MAAMjB,MAAMc,aAAa,IAAII,KAAKC,GAAG,CAACL,aAAa,MAAM,SAAUN,cAAc,KAAKC;QACxF,EAAE,OAAOW,KAAK;YACZ,IAAIX,YAAYV,eAAe,GAAG,MAAMqB;YACxC,MAAMpB,MAAMQ,cAAc,KAAKC;QACjC;IACF;IACA,MAAM,IAAIY,MAAM;AAClB;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,OAAO,eAAeC,eAAehB,GAAW,EAAEiB,YAAY,IAAI;IAChE,IAAI;QACF,MAAMZ,MAAML,KAAK;YAAEkB,QAAQC,YAAYC,OAAO,CAACH;QAAW;QAC1D,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF"}

package/dist/esm/embed/identity.d.ts

@@ -0,0 +1,13 @@
+export declare const MODEL_FILES: string[];
+export declare const MODEL_FILENAMES: string;
+export declare function isDownloadable(model: string): boolean;
+export declare function refsMainPath(model: string): string;
+export declare function readRef(model: string): string | undefined;
+export declare function writeRef(model: string, sha: string): void;
+export declare function snapshotDir(model: string, sha: string): string;
+export declare function languagesPath(model: string): string;
+export declare function readLanguages(model: string): string[] | undefined;
+export declare function writeLanguages(model: string, languages: string[]): void;
+export declare function modelDir(model: string): string;
+export declare function hasModelFiles(model: string): boolean;
+export declare function modelIdentity(model: string): string | undefined;

package/dist/esm/embed/identity.js

@@ -0,0 +1,92 @@
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+// HF cache layout, no network: ~/.sense/models/models--<org>--<name>/{refs/main,
+// snapshots/<sha>/<files>}, files stored directly (hf_hub's blobs/symlinks layer skipped).
+export const MODEL_FILES = [
+    'model.safetensors',
+    'tokenizer.json'
+];
+// The pair, for messages that name what a local model directory must contain.
+export const MODEL_FILENAMES = MODEL_FILES.join(' and ');
+// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather
+// than resolved against the HF cache.
+const HF_MODEL_ID = /^[A-Za-z0-9][\w.-]*\/[A-Za-z0-9][\w.-]*$/;
+export function isDownloadable(model) {
+    return HF_MODEL_ID.test(model);
+}
+function repoDir(model) {
+    const [org, name] = model.split('/');
+    return join(homedir(), '.sense', 'models', `models--${org}--${name}`);
+}
+export function refsMainPath(model) {
+    return join(repoDir(model), 'refs', 'main');
+}
+// What `main` last resolved to, read from disk -- no network. Undefined until some resolution
+// (a download or an explicit prefetch) has written it once.
+export function readRef(model) {
+    const p = refsMainPath(model);
+    if (!existsSync(p)) return undefined;
+    const sha = readFileSync(p, 'utf8').trim();
+    return sha.length > 0 ? sha : undefined;
+}
+// A recorded ref pins until its snapshot directory is removed; nothing here re-resolves it.
+export function writeRef(model, sha) {
+    mkdirSync(join(repoDir(model), 'refs'), {
+        recursive: true
+    });
+    writeFileSync(refsMainPath(model), sha);
+}
+export function snapshotDir(model, sha) {
+    return join(repoDir(model), 'snapshots', sha);
+}
+// Languages describe the repo, not one weight snapshot, so this lives beside refs/, not
+// inside a snapshot dir.
+export function languagesPath(model) {
+    return join(repoDir(model), 'languages.json');
+}
+// undefined means "never resolved yet"; an empty array is a resolved fact (checked, none
+// declared) and is cached the same way, so a languageless model is not re-fetched every run.
+export function readLanguages(model) {
+    const p = languagesPath(model);
+    if (!existsSync(p)) return undefined;
+    try {
+        return JSON.parse(readFileSync(p, 'utf8'));
+    } catch  {
+        return undefined;
+    }
+}
+export function writeLanguages(model, languages) {
+    mkdirSync(repoDir(model), {
+        recursive: true
+    });
+    writeFileSync(languagesPath(model), JSON.stringify(languages));
+}
+// A local directory the caller manages, or the resolved snapshot once `main` is pinned; an HF
+// id with no recorded ref yet has no known snapshot, so this names the (fileless) repo dir.
+export function modelDir(model) {
+    if (!isDownloadable(model)) return model;
+    const sha = readRef(model);
+    return sha ? snapshotDir(model, sha) : repoDir(model);
+}
+// Both files, so an interrupted download reads as absent and the next one resumes it.
+export function hasModelFiles(model) {
+    const dir = modelDir(model);
+    return MODEL_FILES.every((file)=>existsSync(join(dir, file)));
+}
+// Weight identity for the signature, no network: a HF id's resolved sha, or the two
+// files' size+mtime for a local path. Undefined when nothing is known yet.
+export function modelIdentity(model) {
+    if (isDownloadable(model)) return readRef(model);
+    const dir = modelDir(model);
+    const parts = [];
+    for (const file of MODEL_FILES){
+        try {
+            const st = statSync(join(dir, file));
+            parts.push(`${st.size}:${Math.trunc(st.mtimeMs)}`);
+        } catch  {
+            return undefined;
+        }
+    }
+    return parts.join(',');
+}

package/dist/esm/embed/identity.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/identity.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\n\n// HF cache layout, no network: ~/.sense/models/models--<org>--<name>/{refs/main,\n// snapshots/<sha>/<files>}, files stored directly (hf_hub's blobs/symlinks layer skipped).\n\nexport const MODEL_FILES = ['model.safetensors', 'tokenizer.json'];\n// The pair, for messages that name what a local model directory must contain.\nexport const MODEL_FILENAMES = MODEL_FILES.join(' and ');\n\n// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather\n// than resolved against the HF cache.\nconst HF_MODEL_ID = /^[A-Za-z0-9][\\w.-]*\\/[A-Za-z0-9][\\w.-]*$/;\n\nexport function isDownloadable(model: string): boolean {\n  return HF_MODEL_ID.test(model);\n}\n\nfunction repoDir(model: string): string {\n  const [org, name] = model.split('/');\n  return join(homedir(), '.sense', 'models', `models--${org}--${name}`);\n}\n\nexport function refsMainPath(model: string): string {\n  return join(repoDir(model), 'refs', 'main');\n}\n\n// What `main` last resolved to, read from disk -- no network. Undefined until some resolution\n// (a download or an explicit prefetch) has written it once.\nexport function readRef(model: string): string | undefined {\n  const p = refsMainPath(model);\n  if (!existsSync(p)) return undefined;\n  const sha = readFileSync(p, 'utf8').trim();\n  return sha.length > 0 ? sha : undefined;\n}\n\n// A recorded ref pins until its snapshot directory is removed; nothing here re-resolves it.\nexport function writeRef(model: string, sha: string): void {\n  mkdirSync(join(repoDir(model), 'refs'), { recursive: true });\n  writeFileSync(refsMainPath(model), sha);\n}\n\nexport function snapshotDir(model: string, sha: string): string {\n  return join(repoDir(model), 'snapshots', sha);\n}\n\n// Languages describe the repo, not one weight snapshot, so this lives beside refs/, not\n// inside a snapshot dir.\nexport function languagesPath(model: string): string {\n  return join(repoDir(model), 'languages.json');\n}\n\n// undefined means \"never resolved yet\"; an empty array is a resolved fact (checked, none\n// declared) and is cached the same way, so a languageless model is not re-fetched every run.\nexport function readLanguages(model: string): string[] | undefined {\n  const p = languagesPath(model);\n  if (!existsSync(p)) return undefined;\n  try {\n    return JSON.parse(readFileSync(p, 'utf8'));\n  } catch {\n    return undefined;\n  }\n}\n\nexport function writeLanguages(model: string, languages: string[]): void {\n  mkdirSync(repoDir(model), { recursive: true });\n  writeFileSync(languagesPath(model), JSON.stringify(languages));\n}\n\n// A local directory the caller manages, or the resolved snapshot once `main` is pinned; an HF\n// id with no recorded ref yet has no known snapshot, so this names the (fileless) repo dir.\nexport function modelDir(model: string): string {\n  if (!isDownloadable(model)) return model;\n  const sha = readRef(model);\n  return sha ? snapshotDir(model, sha) : repoDir(model);\n}\n\n// Both files, so an interrupted download reads as absent and the next one resumes it.\nexport function hasModelFiles(model: string): boolean {\n  const dir = modelDir(model);\n  return MODEL_FILES.every((file) => existsSync(join(dir, file)));\n}\n\n// Weight identity for the signature, no network: a HF id's resolved sha, or the two\n// files' size+mtime for a local path. Undefined when nothing is known yet.\nexport function modelIdentity(model: string): string | undefined {\n  if (isDownloadable(model)) return readRef(model);\n  const dir = modelDir(model);\n  const parts: string[] = [];\n  for (const file of MODEL_FILES) {\n    try {\n      const st = statSync(join(dir, file));\n      parts.push(`${st.size}:${Math.trunc(st.mtimeMs)}`);\n    } catch {\n      return undefined;\n    }\n  }\n  return parts.join(',');\n}\n"],"names":["existsSync","mkdirSync","readFileSync","statSync","writeFileSync","homedir","join","MODEL_FILES","MODEL_FILENAMES","HF_MODEL_ID","isDownloadable","model","test","repoDir","org","name","split","refsMainPath","readRef","p","undefined","sha","trim","length","writeRef","recursive","snapshotDir","languagesPath","readLanguages","JSON","parse","writeLanguages","languages","stringify","modelDir","hasModelFiles","dir","every","file","modelIdentity","parts","st","push","size","Math","trunc","mtimeMs"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,EAAEC,YAAY,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,UAAU;AACvF,SAASC,OAAO,QAAQ,UAAU;AAClC,SAASC,IAAI,QAAQ,YAAY;AAEjC,iFAAiF;AACjF,2FAA2F;AAE3F,OAAO,MAAMC,cAAc;IAAC;IAAqB;CAAiB,CAAC;AACnE,8EAA8E;AAC9E,OAAO,MAAMC,kBAAkBD,YAAYD,IAAI,CAAC,SAAS;AAEzD,+FAA+F;AAC/F,sCAAsC;AACtC,MAAMG,cAAc;AAEpB,OAAO,SAASC,eAAeC,KAAa;IAC1C,OAAOF,YAAYG,IAAI,CAACD;AAC1B;AAEA,SAASE,QAAQF,KAAa;IAC5B,MAAM,CAACG,KAAKC,KAAK,GAAGJ,MAAMK,KAAK,CAAC;IAChC,OAAOV,KAAKD,WAAW,UAAU,UAAU,CAAC,QAAQ,EAAES,IAAI,EAAE,EAAEC,MAAM;AACtE;AAEA,OAAO,SAASE,aAAaN,KAAa;IACxC,OAAOL,KAAKO,QAAQF,QAAQ,QAAQ;AACtC;AAEA,8FAA8F;AAC9F,4DAA4D;AAC5D,OAAO,SAASO,QAAQP,KAAa;IACnC,MAAMQ,IAAIF,aAAaN;IACvB,IAAI,CAACX,WAAWmB,IAAI,OAAOC;IAC3B,MAAMC,MAAMnB,aAAaiB,GAAG,QAAQG,IAAI;IACxC,OAAOD,IAAIE,MAAM,GAAG,IAAIF,MAAMD;AAChC;AAEA,4FAA4F;AAC5F,OAAO,SAASI,SAASb,KAAa,EAAEU,GAAW;IACjDpB,UAAUK,KAAKO,QAAQF,QAAQ,SAAS;QAAEc,WAAW;IAAK;IAC1DrB,cAAca,aAAaN,QAAQU;AACrC;AAEA,OAAO,SAASK,YAAYf,KAAa,EAAEU,GAAW;IACpD,OAAOf,KAAKO,QAAQF,QAAQ,aAAaU;AAC3C;AAEA,wFAAwF;AACxF,yBAAyB;AACzB,OAAO,SAASM,cAAchB,KAAa;IACzC,OAAOL,KAAKO,QAAQF,QAAQ;AAC9B;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,OAAO,SAASiB,cAAcjB,KAAa;IACzC,MAAMQ,IAAIQ,cAAchB;IACxB,IAAI,CAACX,WAAWmB,IAAI,OAAOC;IAC3B,IAAI;QACF,OAAOS,KAAKC,KAAK,CAAC5B,aAAaiB,GAAG;IACpC,EAAE,OAAM;QACN,OAAOC;IACT;AACF;AAEA,OAAO,SAASW,eAAepB,KAAa,EAAEqB,SAAmB;IAC/D/B,UAAUY,QAAQF,QAAQ;QAAEc,WAAW;IAAK;IAC5CrB,cAAcuB,cAAchB,QAAQkB,KAAKI,SAAS,CAACD;AACrD;AAEA,8FAA8F;AAC9F,4FAA4F;AAC5F,OAAO,SAASE,SAASvB,KAAa;IACpC,IAAI,CAACD,eAAeC,QAAQ,OAAOA;IACnC,MAAMU,MAAMH,QAAQP;IACpB,OAAOU,MAAMK,YAAYf,OAAOU,OAAOR,QAAQF;AACjD;AAEA,sFAAsF;AACtF,OAAO,SAASwB,cAAcxB,KAAa;IACzC,MAAMyB,MAAMF,SAASvB;IACrB,OAAOJ,YAAY8B,KAAK,CAAC,CAACC,OAAStC,WAAWM,KAAK8B,KAAKE;AAC1D;AAEA,oFAAoF;AACpF,2EAA2E;AAC3E,OAAO,SAASC,cAAc5B,KAAa;IACzC,IAAID,eAAeC,QAAQ,OAAOO,QAAQP;IAC1C,MAAMyB,MAAMF,SAASvB;IACrB,MAAM6B,QAAkB,EAAE;IAC1B,KAAK,MAAMF,QAAQ/B,YAAa;QAC9B,IAAI;YACF,MAAMkC,KAAKtC,SAASG,KAAK8B,KAAKE;YAC9BE,MAAME,IAAI,CAAC,GAAGD,GAAGE,IAAI,CAAC,CAAC,EAAEC,KAAKC,KAAK,CAACJ,GAAGK,OAAO,GAAG;QACnD,EAAE,OAAM;YACN,OAAO1B;QACT;IACF;IACA,OAAOoB,MAAMlC,IAAI,CAAC;AACpB"}

package/dist/esm/embed/langfit.d.ts

@@ -0,0 +1,6 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { EmbedProvider } from './types.js';
+type LangCounts = Record<string, number>;
+export declare function languageDistribution(db: DatabaseSync): LangCounts | undefined;
+export declare function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void>;
+export {};

package/dist/esm/embed/langfit.js

@@ -0,0 +1,44 @@
+import { franc } from 'franc-min';
+import { getMeta, setMeta } from '../db/shared.js';
+import { SenseError } from '../errors.js';
+import { toIso3 } from './languages.js';
+const META_KEY = 'embed_languages';
+// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a
+// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).
+const SAMPLE_CHARS = 300;
+// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule
+// never fires below this many classified chunks, declared languages or not.
+const MIN_CLASSIFIED = 10;
+export function languageDistribution(db) {
+    const raw = getMeta(db, META_KEY);
+    return raw ? JSON.parse(raw) : undefined;
+}
+function classify(text) {
+    const code = franc(text.slice(0, SAMPLE_CHARS));
+    return code === 'und' ? undefined : code;
+}
+// Merges chunk texts into the persisted distribution; throws (without persisting) if the model
+// declares languages and a clear majority classified so far is not among them.
+export async function checkLanguageFit(db, provider, texts) {
+    var _languageDistribution;
+    if (texts.length === 0) return;
+    const persisted = (_languageDistribution = languageDistribution(db)) !== null && _languageDistribution !== void 0 ? _languageDistribution : {};
+    const merged = {
+        ...persisted
+    };
+    for (const text of texts){
+        var _merged_code;
+        const code = classify(text);
+        if (code) merged[code] = ((_merged_code = merged[code]) !== null && _merged_code !== void 0 ? _merged_code : 0) + 1;
+    }
+    const total = Object.values(merged).reduce((a, b)=>a + b, 0);
+    if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {
+        const declared = new Set(provider.languages.map(toIso3));
+        const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b)=>b[1] - a[1])[0];
+        if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {
+            const pct = Math.round(majorityCount / total * 100);
+            throw new SenseError('EMBED_MODEL_MISMATCH', `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is "${majorityLang}" (ISO 639-3), which is not among them; choose a model for this tree's languages: \`sense init --model ...\`, see the sense-setup skill or INTEGRATIONS.md`);
+        }
+    }
+    setMeta(db, META_KEY, JSON.stringify(merged));
+}

package/dist/esm/embed/langfit.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { franc } from 'franc-min';\nimport { getMeta, setMeta } from '../db/shared.ts';\nimport { SenseError } from '../errors.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst META_KEY = 'embed_languages';\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\ntype LangCounts = Record<string, number>;\n\nexport function languageDistribution(db: DatabaseSync): LangCounts | undefined {\n  const raw = getMeta(db, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nfunction classify(text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  setMeta(db, META_KEY, JSON.stringify(merged));\n}\n"],"names":["franc","getMeta","setMeta","SenseError","toIso3","META_KEY","SAMPLE_CHARS","MIN_CLASSIFIED","languageDistribution","db","raw","JSON","parse","undefined","classify","text","code","slice","checkLanguageFit","provider","texts","length","persisted","merged","total","Object","values","reduce","a","b","languages","declared","Set","map","majorityLang","majorityCount","entries","sort","has","pct","Math","round","id","join","stringify"],"mappings":"AACA,SAASA,KAAK,QAAQ,YAAY;AAClC,SAASC,OAAO,EAAEC,OAAO,QAAQ,kBAAkB;AACnD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,MAAM,QAAQ,iBAAiB;AAGxC,MAAMC,WAAW;AACjB,+FAA+F;AAC/F,8FAA8F;AAC9F,MAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,MAAMC,iBAAiB;AAIvB,OAAO,SAASC,qBAAqBC,EAAgB;IACnD,MAAMC,MAAMT,QAAQQ,IAAIJ;IACxB,OAAOK,MAAMC,KAAKC,KAAK,CAACF,OAAOG;AACjC;AAEA,SAASC,SAASC,IAAY;IAC5B,MAAMC,OAAOhB,MAAMe,KAAKE,KAAK,CAAC,GAAGX;IACjC,OAAOU,SAAS,QAAQH,YAAYG;AACtC;AAEA,+FAA+F;AAC/F,+EAA+E;AAC/E,OAAO,eAAeE,iBAAiBT,EAAgB,EAAEU,QAAuB,EAAEC,KAAe;QAE7EZ;IADlB,IAAIY,MAAMC,MAAM,KAAK,GAAG;IACxB,MAAMC,aAAYd,wBAAAA,qBAAqBC,iBAArBD,mCAAAA,wBAA4B,CAAC;IAC/C,MAAMe,SAAS;QAAE,GAAGD,SAAS;IAAC;IAC9B,KAAK,MAAMP,QAAQK,MAAO;YAEEG;QAD1B,MAAMP,OAAOF,SAASC;QACtB,IAAIC,MAAMO,MAAM,CAACP,KAAK,GAAG,EAACO,eAAAA,MAAM,CAACP,KAAK,cAAZO,0BAAAA,eAAgB,KAAK;IACjD;IAEA,MAAMC,QAAQC,OAAOC,MAAM,CAACH,QAAQI,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG;IAC5D,IAAIV,SAASW,SAAS,IAAIX,SAASW,SAAS,CAACT,MAAM,GAAG,KAAKG,SAASjB,gBAAgB;QAClF,MAAMwB,WAAW,IAAIC,IAAIb,SAASW,SAAS,CAACG,GAAG,CAAC7B;QAChD,MAAM,CAAC8B,cAAcC,cAAc,GAAGV,OAAOW,OAAO,CAACb,QAAQc,IAAI,CAAC,CAACT,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC3F,IAAIO,gBAAgBX,SAAS,OAAO,CAACO,SAASO,GAAG,CAACJ,eAAe;YAC/D,MAAMK,MAAMC,KAAKC,KAAK,CAAC,AAACN,gBAAgBX,QAAS;YACjD,MAAM,IAAIrB,WACR,wBACA,CAAC,YAAY,EAAEgB,SAASuB,EAAE,CAAC,qBAAqB,EAAEvB,SAASW,SAAS,CAACa,IAAI,CAAC,MAAM,OAAO,EAAEJ,IAAI,qCAAqC,EAAEL,aAAa,0JAA0J,CAAC;QAEhT;IACF;IACAhC,QAAQO,IAAIJ,UAAUM,KAAKiC,SAAS,CAACrB;AACvC"}

package/dist/esm/embed/languages.d.ts

@@ -0,0 +1,3 @@
+export declare function isKnownLanguageTag(tag: string): boolean;
+export declare function toIso3(code: string): string;
+export declare const MEASURED_MODEL_LANGUAGES: Record<string, string[]>;

package/dist/esm/embed/languages.js

@@ -0,0 +1,80 @@
+// HF model cards declare languages as ISO 639-1 tags (e.g. "zh"); franc-min reports ISO 639-3
+// (e.g. "cmn"). This bridges the two for comparison. Limited to franc-min's own 82-language
+// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min
+// can actually emit; a declared tag with no entry here is compared as-is.
+const ISO_639_1_TO_3 = {
+    ar: 'arb',
+    am: 'amh',
+    az: 'azj',
+    be: 'bel',
+    bg: 'bul',
+    bn: 'ben',
+    bs: 'bos',
+    cs: 'ces',
+    de: 'deu',
+    el: 'ell',
+    en: 'eng',
+    es: 'spa',
+    fa: 'pes',
+    fr: 'fra',
+    gu: 'guj',
+    ha: 'hau',
+    hi: 'hin',
+    hr: 'hrv',
+    hu: 'hun',
+    id: 'ind',
+    ig: 'ibo',
+    it: 'ita',
+    ja: 'jpn',
+    jv: 'jav',
+    kk: 'kaz',
+    kn: 'kan',
+    ko: 'kor',
+    ml: 'mal',
+    mr: 'mar',
+    ms: 'zlm',
+    my: 'mya',
+    ne: 'npi',
+    nl: 'nld',
+    pa: 'pan',
+    pl: 'pol',
+    pt: 'por',
+    ro: 'ron',
+    ru: 'rus',
+    rw: 'kin',
+    si: 'sin',
+    so: 'som',
+    sr: 'srp',
+    su: 'sun',
+    sv: 'swe',
+    sw: 'swh',
+    ta: 'tam',
+    te: 'tel',
+    th: 'tha',
+    tl: 'tgl',
+    tr: 'tur',
+    uk: 'ukr',
+    ur: 'urd',
+    uz: 'uzn',
+    vi: 'vie',
+    yo: 'yor',
+    zh: 'cmn',
+    zu: 'zul'
+};
+export function isKnownLanguageTag(tag) {
+    return tag in ISO_639_1_TO_3;
+}
+// Normalizes a declared or detected language code to ISO 639-3 for comparison; strips a
+// region/script subtag ("zh-cn" -> "zh") first. Codes already in 639-3 form pass through.
+export function toIso3(code) {
+    var _ISO_639_1_TO_3_base;
+    const base = code.toLowerCase().split('-')[0];
+    return (_ISO_639_1_TO_3_base = ISO_639_1_TO_3[base]) !== null && _ISO_639_1_TO_3_base !== void 0 ? _ISO_639_1_TO_3_base : base;
+}
+// Languages this project measured for models whose cards declare none, so the fit check is
+// not silently off for the shipped default; asserted here because it was measured, not attested.
+export const MEASURED_MODEL_LANGUAGES = {
+    'minishlab/potion-retrieval-32M': [
+        'en'
+    ]
+};

package/dist/esm/embed/languages.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/languages.ts"],"sourcesContent":["// HF model cards declare languages as ISO 639-1 tags (e.g. \"zh\"); franc-min reports ISO 639-3\n// (e.g. \"cmn\"). This bridges the two for comparison. Limited to franc-min's own 82-language\n// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min\n// can actually emit; a declared tag with no entry here is compared as-is.\nconst ISO_639_1_TO_3: Record<string, string> = {\n  ar: 'arb',\n  am: 'amh',\n  az: 'azj',\n  be: 'bel',\n  bg: 'bul',\n  bn: 'ben',\n  bs: 'bos',\n  cs: 'ces',\n  de: 'deu',\n  el: 'ell',\n  en: 'eng',\n  es: 'spa',\n  fa: 'pes',\n  fr: 'fra',\n  gu: 'guj',\n  ha: 'hau',\n  hi: 'hin',\n  hr: 'hrv',\n  hu: 'hun',\n  id: 'ind',\n  ig: 'ibo',\n  it: 'ita',\n  ja: 'jpn',\n  jv: 'jav',\n  kk: 'kaz',\n  kn: 'kan',\n  ko: 'kor',\n  ml: 'mal',\n  mr: 'mar',\n  ms: 'zlm',\n  my: 'mya',\n  ne: 'npi',\n  nl: 'nld',\n  pa: 'pan',\n  pl: 'pol',\n  pt: 'por',\n  ro: 'ron',\n  ru: 'rus',\n  rw: 'kin',\n  si: 'sin',\n  so: 'som',\n  sr: 'srp',\n  su: 'sun',\n  sv: 'swe',\n  sw: 'swh',\n  ta: 'tam',\n  te: 'tel',\n  th: 'tha',\n  tl: 'tgl',\n  tr: 'tur',\n  uk: 'ukr',\n  ur: 'urd',\n  uz: 'uzn',\n  vi: 'vie',\n  yo: 'yor',\n  zh: 'cmn',\n  zu: 'zul',\n};\n\nexport function isKnownLanguageTag(tag: string): boolean {\n  return tag in ISO_639_1_TO_3;\n}\n\n// Normalizes a declared or detected language code to ISO 639-3 for comparison; strips a\n// region/script subtag (\"zh-cn\" -> \"zh\") first. Codes already in 639-3 form pass through.\nexport function toIso3(code: string): string {\n  const base = code.toLowerCase().split('-')[0];\n  return ISO_639_1_TO_3[base] ?? base;\n}\n\n// Languages this project measured for models whose cards declare none, so the fit check is\n// not silently off for the shipped default; asserted here because it was measured, not attested.\nexport const MEASURED_MODEL_LANGUAGES: Record<string, string[]> = {\n  'minishlab/potion-retrieval-32M': ['en'], // benchmark/reports/2026-08-27-embedding-model-selection.md\n};\n"],"names":["ISO_639_1_TO_3","ar","am","az","be","bg","bn","bs","cs","de","el","en","es","fa","fr","gu","ha","hi","hr","hu","id","ig","it","ja","jv","kk","kn","ko","ml","mr","ms","my","ne","nl","pa","pl","pt","ro","ru","rw","si","so","sr","su","sv","sw","ta","te","th","tl","tr","uk","ur","uz","vi","yo","zh","zu","isKnownLanguageTag","tag","toIso3","code","base","toLowerCase","split","MEASURED_MODEL_LANGUAGES"],"mappings":"AAAA,8FAA8F;AAC9F,4FAA4F;AAC5F,gGAAgG;AAChG,0EAA0E;AAC1E,MAAMA,iBAAyC;IAC7CC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;AACN;AAEA,OAAO,SAASC,mBAAmBC,GAAW;IAC5C,OAAOA,OAAO3D;AAChB;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,OAAO,SAAS4D,OAAOC,IAAY;QAE1B7D;IADP,MAAM8D,OAAOD,KAAKE,WAAW,GAAGC,KAAK,CAAC,IAAI,CAAC,EAAE;IAC7C,QAAOhE,uBAAAA,cAAc,CAAC8D,KAAK,cAApB9D,kCAAAA,uBAAwB8D;AACjC;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,OAAO,MAAMG,2BAAqD;IAChE,kCAAkC;QAAC;KAAK;AAC1C,EAAE"}

package/dist/esm/embed/openai.d.ts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider>;

package/dist/esm/embed/openai.js

@@ -0,0 +1,49 @@
+import { SenseError } from '../errors.js';
+import { fetchWithRetry } from './http.js';
+const BATCH_CAP = 64;
+// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare
+// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.
+export async function openaiProvider(model, url, keyEnv) {
+    if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
+    const base = url.replace(/\/+$/, '');
+    const headers = {
+        'content-type': 'application/json'
+    };
+    const key = keyEnv ? process.env[keyEnv] : undefined;
+    if (key) headers.authorization = `Bearer ${key}`;
+    async function post(texts) {
+        const res = await fetchWithRetry(`${base}/embeddings`, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                model,
+                input: texts
+            })
+        });
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);
+        const body = await res.json();
+        // The schema carries `index` because response order is not promised, and the caller maps
+        // vectors back to chunks positionally: sort by it whenever every element has one.
+        const data = body.data.every((d)=>typeof d.index === 'number') ? [
+            ...body.data
+        ].sort((a, b)=>a.index - b.index) : body.data;
+        // A server that drops an input (empty or over-length strings are the usual cause) would
+        // otherwise shift every later vector onto the wrong chunk, silently.
+        if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);
+        return data.map((d)=>Float32Array.from(d.embedding));
+    }
+    const dims = (await post([
+        'dimension probe'
+    ]))[0].length;
+    // Symmetric for now: per-model prefixes (nomic's search_document/search_query) are a
+    // later workstream.
+    return {
+        id: `openai:${base}:${model}`,
+        dims,
+        batchCap: BATCH_CAP,
+        embedDocuments: post,
+        embedQuery: async (text)=>(await post([
+                text
+            ]))[0]
+    };
+}

package/dist/esm/embed/openai.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // Symmetric for now: per-model prefixes (nomic's search_document/search_query) are a\n  // later workstream.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["SenseError","fetchWithRetry","BATCH_CAP","openaiProvider","model","url","keyEnv","base","replace","headers","key","process","env","undefined","authorization","post","texts","res","method","body","JSON","stringify","input","ok","status","json","data","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,YAAY;AAG3C,MAAMC,YAAY;AAElB,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,IAAI,CAACD,KAAK,MAAM,IAAIL,WAAW,eAAe;IAC9C,MAAMO,OAAOF,IAAIG,OAAO,CAAC,QAAQ;IACjC,MAAMC,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMJ,SAASK,QAAQC,GAAG,CAACN,OAAO,GAAGO;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKC,KAAe;QACjC,MAAMC,MAAM,MAAMhB,eAAe,GAAGM,KAAK,WAAW,CAAC,EAAE;YAAEW,QAAQ;YAAQT;YAASU,MAAMC,KAAKC,SAAS,CAAC;gBAAEjB;gBAAOkB,OAAON;YAAM;QAAG;QAChI,IAAI,CAACC,IAAIM,EAAE,EAAE,MAAM,IAAIvB,WAAW,eAAe,GAAGO,KAAK,oBAAoB,EAAEU,IAAIO,MAAM,EAAE;QAC3F,MAAML,OAAQ,MAAMF,IAAIQ,IAAI;QAC5B,yFAAyF;QACzF,kFAAkF;QAClF,MAAMC,OAAOP,KAAKO,IAAI,CAACC,KAAK,CAAC,CAACC,IAAM,OAAOA,EAAEC,KAAK,KAAK,YAAY;eAAIV,KAAKO,IAAI;SAAC,CAACI,IAAI,CAAC,CAACC,GAAGC,IAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK,IAAeV,KAAKO,IAAI;QACvJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIA,KAAKO,MAAM,KAAKjB,MAAMiB,MAAM,EAAE,MAAM,IAAIjC,WAAW,eAAe,GAAGO,KAAK,qBAAqB,EAAEmB,KAAKO,MAAM,CAAC,gBAAgB,EAAEjB,MAAMiB,MAAM,CAAC,OAAO,CAAC;QACxJ,OAAOP,KAAKQ,GAAG,CAAC,CAACN,IAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;IACtD;IAEA,MAAMC,OAAO,AAAC,CAAA,MAAMvB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACkB,MAAM;IACxD,qFAAqF;IACrF,oBAAoB;IACpB,OAAO;QAAEM,IAAI,CAAC,OAAO,EAAEhC,KAAK,CAAC,EAAEH,OAAO;QAAEkC;QAAME,UAAUtC;QAAWuC,gBAAgB1B;QAAM2B,YAAY,OAAOC,OAAS,AAAC,CAAA,MAAM5B,KAAK;gBAAC4B;aAAK,CAAA,CAAE,CAAC,EAAE;IAAC;AAC/I"}

package/dist/esm/embed/query.d.ts

@@ -0,0 +1,21 @@
+import type { DatabaseSync } from 'node:sqlite';
+import type { Config } from '../config/index.js';
+export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
+    v: Float32Array;
+    scale: number;
+};
+export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
+    path: string;
+    lines: string;
+    similarity: number;
+}>>;
+export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
+export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+    exclude: Set<string>;
+    allowed?: Set<string>;
+    k: number;
+}): Array<{
+    path: string;
+    similarity: number;
+}>;