sensemaking 0.15.3 → 0.16.0

package/dist/cjs/embed/query.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["embedPending","hasEmbedding","semanticCandidates","similarNotes","toStore","STORE_DIMS","TARGET_CHUNK_CAP","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","db","cfg","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","path","chunkIdxs","chunks","idx","update","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","chunk","err","tick","min","length","getProvider","prepare","all","Map","get","push","set","parseFile","relPath","absPath","join","mtimeMs","ctimeMs","size","presets","embed","doc","extracted","checkLanguageFit","progress","finish","asCosine","score","terms","fetch","allowed","qv","rows","best","dot","existing","SenseError","match","filter","t","includes","embedQuery","has","vector","byteOffset","byteLength","lines","start_line","end_line","entries","sort","a","b","similarity","undefined","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":";;;;;;;;;;;QAiCsBA;eAAAA;;QAwGNC;eAAAA;;QApCMC;eAAAA;;QA2CNC;eAAAA;;QA7HAC;eAAAA;;;wBAnBK;wBAGM;uBAEL;0BACG;sBACC;yBACO;0BACL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,0FAA0F;AAC1F,IAAMC,mBAAmB;AAIlB,SAASF,QAAQG,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAejB,aAAamB,EAAgB,EAAEC,GAAW,EAAEC,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOC,MAAMC,WACZC,QAQC,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,KAWPC,QAGAC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQR,KAAKU,KAAK,CAACH,GAAGA,IAAIb,SAASiB,QAAQ;oCACjC;;wCAAMjB,SAASkB,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChBlB,GAAGyB,IAAI,CAAC;oCACR,IAAI;wCACFR,MAAMS,OAAO,CAAC,SAACC,KAAKJ;4CAClB,IAAqBtC,WAAAA,QAAQiC,OAAO,CAACK,EAAE,EAAElB,WAAW,OAA5Cd,IAAaN,SAAbM,GAAGM,QAAUZ,SAAVY;4CACX,IAAM+B,IAAI,IAAIC,UAAUxB;4CACxB,IAAK,IAAIX,IAAI,GAAGA,IAAIW,WAAWX,IAAKkC,CAAC,CAAClC,EAAE,GAAGC,KAAKmC,KAAK,CAACvC,CAAC,CAACG,EAAE,GAAGG;4CAC7DiB,OAAOiB,GAAG,CAAClC,OAAOmC,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAIjB,IAAI,EAAEiB,IAAIQ,KAAK;wCAC9D;wCACAnC,GAAGyB,IAAI,CAAC;oCACV,EAAE,OAAOW,KAAK;wCACZpC,GAAGyB,IAAI,CAAC;wCACR,MAAMW;oCACR;oCACArB,OAAOsB,IAAI,CAAC1C,KAAK2C,GAAG,CAACtB,IAAIb,SAASiB,QAAQ,EAAEX,KAAK8B,MAAM;;;;;;oBACzD;oBAtDiB;;wBAAMC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXC,QAAQJ,GAAGyC,OAAO,CAAC,oFAAoFC,GAAG;oBAChH,IAAItC,MAAMmC,MAAM,KAAK,GAAG;;;oBAClBlC,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAE9CiB,SAAS,IAAIqC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAavC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOsC,GAAG,CAACrC,IAAIG,IAAI,eAAnBJ,yBAAAA;4BACbE,KAAKqC,IAAI,CAACtC,IAAI4B,KAAK;4BACnB7B,OAAOwC,GAAG,CAACvC,IAAIG,IAAI,EAAEF;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOI,uBAAMC;4BACZC,SAAAA,KAAAA;4BACJ,IAAI;gCACF,qFAAqF;gCACrF,+EAA+E;gCAC/EA,SAASmC,IAAAA,iBAAS,EAAC;oCAAEC,SAAStC;oCAAMuC,SAASC,IAAAA,cAAI,EAAChD,SAASQ;oCAAOyC,SAAS;oCAAGC,SAAS;oCAAGC,MAAM;oCAAGC,OAAO;oCAAMC,OAAO;gCAAK;oCAAIA,cAAK;mCAAGC,GAAG,CAACC,SAAS,CAACF,KAAK;4BAC7J,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa5C,gCAAb,8BAAA,SAAA,0BAAA;oCAAME,MAAN;oCAAwB,IAAID,MAAM,CAACC,IAAI,EAAEJ,KAAKoC,IAAI,CAAC;wCAAEnC,MAAAA;wCAAMyB,OAAOtB;wCAAKW,MAAMZ,MAAM,CAACC,IAAI,CAACW,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBAVK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAMkC,IAAAA,2BAAgB,EACpB1D,IACAG,UACAM,KAAKa,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMMV,SAASd,GAAGyC,OAAO,CAAC;oBAC1B,8EAA8E;oBAC9E,6DAA6D;oBACvD1B,SAAS4C,IAAAA,oBAAQ,EAAC,oBAAoBlD,KAAK8B,MAAM;oBAC9CvB,IAAI;;;yBAAGA,CAAAA,IAAIP,KAAK8B,MAAM,AAAD;;;;;;;;;;;;oBAAGvB,KAAKb,SAASiB,QAAQ;;;;;;oBAkBvDL,OAAO6C,MAAM;;;;;;IACf;;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAOnE,KAAKmC,KAAK,CAACnC,KAAK2C,GAAG,CAAC,GAAG3C,KAAKG,GAAG,CAAC,CAAC,GAAGgE,UAAU,QAAQ;AAC/D;AAKO,SAAe/E,mBAAmBiB,EAAgB,EAAEC,GAAW,EAAE8D,KAAa,EAAEC,KAAa,EAAEC,OAAqB;;YAO3GF,cANR7D,SAIAC,UACAE,WACAmB,MACYvC,UAAPiF,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM7D,KAEHqB,GACFyC,KACK3E,GACHoE,OACAQ;;;;oBAxBFpE,UAAU,AAACD,IAAgCC,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIqE,oBAAU,CAAC,eAAe;oBAClD;;wBAAM1F,aAAamB,IAAIC,KAAKC;;;oBAA5B;oBAEiB;;wBAAMsC,IAAAA,uBAAW,EAACvC;;;oBAA7BE,WAAW;oBACXE,YAAYV,KAAK2C,GAAG,CAACpD,YAAYiB,SAASd,IAAI;oBAC9CmC,OAAO,EAACuC,eAAAA,MAAMS,KAAK,CAAC,2naAAZT,0BAAAA,mBAAsCU,MAAM,CAAC,SAACC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIxB,IAAI,CAAC;oBACxF;;wBAAM/C,SAASyE,UAAU,CAACpD;;;oBAAlCvC,WAAAA;wBAAQ;wBAAiCoB;wBAAW;wBAA3D6D,KAAOjF,SAAVM;oBAEF4E,OAAOnE,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG;oBAQpH0B,OAAO,IAAIzB;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAawB,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAb5D,MAAN;4BACH,IAAI0D,WAAW,CAACA,QAAQY,GAAG,CAACtE,IAAIG,IAAI,GAAG;4BACjCkB,IAAI,IAAIC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAEpF,KAAK2C,GAAG,CAACjC,WAAWE,IAAIuE,MAAM,CAACE,UAAU;4BACvGX,MAAM;4BACV,IAAS3E,IAAI,GAAGA,IAAIkC,EAAEW,MAAM,EAAE7C,IAAK2E,OAAOzC,CAAC,CAAClC,EAAE,GAAGwE,EAAE,CAACxE,EAAE;4BAChDoE,QAAQO,MAAM9D,IAAIV,KAAK;4BACvByE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;4BAClC,IAAI,CAAC4D,YAAYR,QAAQQ,SAASR,KAAK,EAAEM,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAE;gCAAEoD,OAAAA;gCAAOmB,OAAO,AAAC,IAAqB1E,OAAlBA,IAAI2E,UAAU,EAAC,KAAgB,OAAb3E,IAAI4E,QAAQ;4BAAG;wBACnH;;wBARK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBASL;;wBAAQ,qBAAGf,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK;2BACtC3C,KAAK,CAAC,GAAG6C,OACT1C,GAAG,CAAC;qEAAEZ,kBAAM6E;mCAAQ;gCAAE7E,MAAAA;gCAAMuE,OAAOM,EAAEN,KAAK;gCAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;4BAAE;;;;;IAC/E;;AAIO,SAAShF,aAAakB,EAAgB,EAAEU,IAAY;IACzD,IAAMH,MAAMP,GAAGyC,OAAO,CAAC,kFAAkFG,GAAG,CAAClC;IAC7G,OAAOH,QAAQkF;AACjB;AAIO,SAASzG,aAAagB,EAAgB,EAAE0F,IAAY,EAAEhF,IAAY,EAAEiF,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,IAAMC,aAAa5F,GAAGyC,OAAO,CAAC,+FAA+FC,GAAG,CAAChC;IACjI,IAAIkF,WAAWrD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,IAAMsD,OAAOlG,KAAKG,GAAG,CAAC,GAAGH,KAAKmG,IAAI,CAACF,WAAWrD,MAAM,GAAGpD;IACvD,IAAM4G,SAASH,WAAWnB,MAAM,CAAC,SAACuB,GAAGhF;eAAMA,IAAI6E,SAAS;OAAGvE,GAAG,CAAC,SAACf;eAAS;YAAEhB,GAAG,IAAIsC,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;YAAGnF,OAAOU,IAAIV,KAAK;QAAC;;IAE/K,IAAMsE,OAAOnE,GAAGyC,OAAO,CAAC,yEAAyEC,GAAG;IACpG,IAAM0B,OAAO,IAAIzB;QACZ,kCAAA,2BAAA;;QAAL,QAAK,YAAawB,yBAAb,SAAA,6BAAA,QAAA,yBAAA,iCAAmB;YAAnB,IAAM5D,MAAN;YACH,IAAIA,IAAIG,IAAI,KAAKA,QAAQiF,KAAKM,OAAO,CAACpB,GAAG,CAACtE,IAAIG,IAAI,KAAMiF,KAAK1B,OAAO,IAAI,CAAC0B,KAAK1B,OAAO,CAACY,GAAG,CAACtE,IAAIG,IAAI,GAAI;YACtG,IAAMwF,QAAQ,IAAIrE,UAAUtB,IAAIuE,MAAM,CAAC5C,MAAM,EAAE3B,IAAIuE,MAAM,CAACC,UAAU,EAAExE,IAAIuE,MAAM,CAACE,UAAU;gBACtF,mCAAA,4BAAA;;gBAAL,QAAK,aAAWe,2BAAX,UAAA,8BAAA,SAAA,0BAAA,kCAAmB;oBAAnB,IAAMrB,IAAN;oBACH,IAAIL,MAAM;oBACV,IAAM8B,MAAMxG,KAAK2C,GAAG,CAACoC,EAAEnF,CAAC,CAACgD,MAAM,EAAE2D,MAAM3D,MAAM;oBAC7C,IAAK,IAAI7C,IAAI,GAAGA,IAAIyG,KAAKzG,IAAK2E,OAAOK,EAAEnF,CAAC,CAACG,EAAE,GAAGwG,KAAK,CAACxG,EAAE;oBACtD,IAAMoE,QAAQO,MAAMK,EAAE7E,KAAK,GAAGU,IAAIV,KAAK;oBACvC,IAAMyE,WAAWF,KAAKxB,GAAG,CAACrC,IAAIG,IAAI;oBAClC,IAAI4D,aAAamB,aAAa3B,QAAQQ,UAAUF,KAAKtB,GAAG,CAACvC,IAAIG,IAAI,EAAEoD;gBACrE;;gBAPK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAQP;;QAXK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAaL,OAAO,AAAC,qBAAGM,KAAKgB,OAAO,IACpBC,IAAI,CAAC,SAACC,GAAGC;eAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE;OAC1BnE,KAAK,CAAC,GAAGwE,KAAKS,CAAC,EACf9E,GAAG,CAAC;iDAAE+E,eAAGvC;eAAY;YAAEpD,MAAM2F;YAAGb,YAAY3B,SAASC;QAAO;;AACjE"}

package/dist/cjs/embed/registry.d.cts

@@ -0,0 +1,3 @@
+import type { Config } from '../config/index.js';
+import type { EmbedProvider } from './types.js';
+export declare function getProvider(cfg: Config): Promise<EmbedProvider>;

package/dist/cjs/embed/registry.d.ts

@@ -0,0 +1,3 @@
+import type { Config } from '../config/index.js';
+import type { EmbedProvider } from './types.js';
+export declare function getProvider(cfg: Config): Promise<EmbedProvider>;

package/dist/cjs/embed/registry.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "getProvider", {
+    enumerable: true,
+    get: function() {
+        return getProvider;
+    }
+});
+var _indexts = require("../config/index.js");
+var _errorsts = require("../errors.js");
+var _coherets = require("./cohere.js");
+var _openaits = require("./openai.js");
+var _staticts = require("./static.js");
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function ownKeys(object, enumerableOnly) {
+    var keys = Object.keys(object);
+    if (Object.getOwnPropertySymbols) {
+        var symbols = Object.getOwnPropertySymbols(object);
+        if (enumerableOnly) {
+            symbols = symbols.filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(object, sym).enumerable;
+            });
+        }
+        keys.push.apply(keys, symbols);
+    }
+    return keys;
+}
+function _object_spread_props(target, source) {
+    source = source != null ? source : {};
+    if (Object.getOwnPropertyDescriptors) Object.defineProperties(target, Object.getOwnPropertyDescriptors(source));
+    else {
+        ownKeys(Object(source)).forEach(function(key) {
+            Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key));
+        });
+    }
+    return target;
+}
+// One factory per wire protocol: adding a protocol is a new file plus an entry here, not
+// an edit to a dispatch ternary.
+var REGISTRY = {
+    static: function _static(model) {
+        return (0, _staticts.staticProvider)(model);
+    },
+    openai: function openai(model, url, key) {
+        return (0, _openaits.openaiProvider)(model, url, key);
+    },
+    cohere: function cohere(model, url, key) {
+        return (0, _coherets.cohereProvider)(model, url, key);
+    }
+};
+var providers = new Map();
+function getProvider(cfg) {
+    var _e_url, _e_key, _e_languages;
+    var e = (0, _indexts.embedConfig)(cfg);
+    if (!e) throw new _errorsts.SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an "embed" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');
+    var factory = REGISTRY[e.provider];
+    if (!factory) throw new _errorsts.SenseError('EMBED_MODEL', 'embed.provider "'.concat(e.provider, '" has no provider implementation in this build'));
+    // Every input that shapes the provider is in the key: two trees sharing an endpoint and
+    // model but declaring different languages or reading different key env vars are two
+    // providers, not one, however long a process lives.
+    var sig = JSON.stringify([
+        e.provider,
+        e.model,
+        (_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : '',
+        (_e_key = e.key) !== null && _e_key !== void 0 ? _e_key : '',
+        (_e_languages = e.languages) !== null && _e_languages !== void 0 ? _e_languages : []
+    ]);
+    var p = providers.get(sig);
+    if (!p) {
+        p = factory(e.model, e.url, e.key);
+        // Owner-declared languages override a provider's own: the config is the one home for
+        // what no model card can state (http models have no fetchable card here).
+        if (e.languages) p = p.then(function(prov) {
+            return _object_spread_props(_object_spread({}, prov), {
+                languages: e.languages
+            });
+        });
+        providers.set(sig, p);
+        // A rejected construction is evicted, so a transient failure retries instead of
+        // poisoning this config for the process lifetime.
+        p.catch(function() {
+            return providers.delete(sig);
+        });
+    }
+    return p;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or reading different key env vars are two\n  // providers, not one, however long a process lives.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["getProvider","REGISTRY","static","model","staticProvider","openai","url","key","openaiProvider","cohere","cohereProvider","providers","Map","cfg","e","embedConfig","SenseError","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":";;;;+BAoBgBA;;;eAAAA;;;uBAnBY;wBACD;wBACI;wBACA;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK/B,yFAAyF;AACzF,iCAAiC;AACjC,IAAMC,WAA4C;IAChDC,QAAQ,SAARA,QAASC;eAAUC,IAAAA,wBAAc,EAACD;;IAClCE,QAAQ,SAARA,OAASF,OAAOG,KAAKC;eAAQC,IAAAA,wBAAc,EAACL,OAAOG,KAAKC;;IACxDE,QAAQ,SAARA,OAASN,OAAOG,KAAKC;eAAQG,IAAAA,wBAAc,EAACP,OAAOG,KAAKC;;AAC1D;AAEA,IAAMI,YAAY,IAAIC;AAEf,SAASZ,YAAYa,GAAW;QAQYC,QAAaA,QAAaA;IAP3E,IAAMA,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIE,oBAAU,CAAC,kBAAkB;IAC/C,IAAMC,UAAUhB,QAAQ,CAACa,EAAEI,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAID,oBAAU,CAAC,eAAe,AAAC,mBAA6B,OAAXF,EAAEI,QAAQ,EAAC;IAChF,wFAAwF;IACxF,oFAAoF;IACpF,oDAAoD;IACpD,IAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACP,EAAEI,QAAQ;QAAEJ,EAAEX,KAAK;SAAEW,SAAAA,EAAER,GAAG,cAALQ,oBAAAA,SAAS;SAAIA,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,eAAAA,EAAEQ,SAAS,cAAXR,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIS,IAAIZ,UAAUa,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQH,EAAEX,KAAK,EAAEW,EAAER,GAAG,EAAEQ,EAAEP,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIO,EAAEQ,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,SAACC;mBAAU,wCAAKA;gBAAMJ,WAAWR,EAAEQ,SAAS;;;QACxEX,UAAUgB,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC;mBAAMjB,UAAUkB,MAAM,CAACV;;IACjC;IACA,OAAOI;AACT"}

package/dist/cjs/embed/static.d.cts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function staticProvider(model: string): Promise<EmbedProvider>;

package/dist/cjs/embed/static.d.ts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function staticProvider(model: string): Promise<EmbedProvider>;

package/dist/cjs/embed/static.js

@@ -0,0 +1,367 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "staticProvider", {
+    enumerable: true,
+    get: function() {
+        return staticProvider;
+    }
+});
+var _nodefs = require("node:fs");
+var _nodepath = require("node:path");
+var _errorsts = require("../errors.js");
+var _storets = require("./store.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _getRequireWildcardCache(nodeInterop) {
+    if (typeof WeakMap !== "function") return null;
+    var cacheBabelInterop = new WeakMap();
+    var cacheNodeInterop = new WeakMap();
+    return (_getRequireWildcardCache = function(nodeInterop) {
+        return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
+    })(nodeInterop);
+}
+function _interop_require_wildcard(obj, nodeInterop) {
+    if (!nodeInterop && obj && obj.__esModule) return obj;
+    if (obj === null || typeof obj !== "object" && typeof obj !== "function") return {
+        default: obj
+    };
+    var cache = _getRequireWildcardCache(nodeInterop);
+    if (cache && cache.has(obj)) return cache.get(obj);
+    var newObj = {
+        __proto__: null
+    };
+    var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
+    for(var key in obj){
+        if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
+            var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
+            if (desc && (desc.get || desc.set)) Object.defineProperty(newObj, key, desc);
+            else newObj[key] = obj[key];
+        }
+    }
+    newObj.default = obj;
+    if (cache) cache.set(obj, newObj);
+    return newObj;
+}
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var BATCH_CAP = 64;
+function staticProvider(model) {
+    return _async_to_generator(function() {
+        var _ref, _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1, dir, languages, cached, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, raw, headerLen, header, entry, spec, dims, dataStart, matrix, tokenizerJson, Tokenizer, tok, unkId;
+        function one(text) {
+            // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an
+            // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the
+            // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.
+            var ids = tok.encode(text, {
+                add_special_tokens: false
+            }).ids.filter(function(id) {
+                return Number.isInteger(id) && id !== unkId;
+            });
+            var v = new Float32Array(dims);
+            if (ids.length === 0) return v;
+            var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+            try {
+                for(var _iterator = ids[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                    var id = _step.value;
+                    var off = id * dims;
+                    for(var d = 0; d < dims; d++)v[d] += matrix[off + d];
+                }
+            } catch (err) {
+                _didIteratorError = true;
+                _iteratorError = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion && _iterator.return != null) {
+                        _iterator.return();
+                    }
+                } finally{
+                    if (_didIteratorError) {
+                        throw _iteratorError;
+                    }
+                }
+            }
+            var norm = 0;
+            for(var d1 = 0; d1 < dims; d1++){
+                v[d1] /= ids.length;
+                norm += v[d1] * v[d1];
+            }
+            norm = Math.sqrt(norm) + 1e-32;
+            for(var d2 = 0; d2 < dims; d2++)v[d2] /= norm;
+            return v;
+        }
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (!(0, _storets.isDownloadable)(model)) return [
+                        3,
+                        2
+                    ];
+                    return [
+                        4,
+                        (0, _storets.downloadModel)(model, function(file, into) {
+                            return console.error("sense: fetching ".concat(model, "/").concat(file, " into ").concat(into));
+                        })
+                    ];
+                case 1:
+                    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.
+                    dir = _state.sent();
+                    // Cached beside the repo; a local model directory has no card to read, so it has none.
+                    cached = (0, _storets.readLanguages)(model);
+                    languages = cached && cached.length > 0 ? cached : undefined;
+                    return [
+                        3,
+                        3
+                    ];
+                case 2:
+                    dir = (0, _storets.modelDir)(model);
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    try {
+                        for(_iterator = _storets.MODEL_FILES[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                            file = _step.value;
+                            if (!(0, _nodefs.existsSync)((0, _nodepath.join)(dir, file))) {
+                                throw new _errorsts.SenseError('EMBED_MODEL_MISSING', "embed model ".concat(model, " is not available (looked in ").concat(dir, "); expected ").concat(_storets.MODEL_FILENAMES, " in that directory"));
+                            }
+                        }
+                    } catch (err) {
+                        _didIteratorError = true;
+                        _iteratorError = err;
+                    } finally{
+                        try {
+                            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                _iterator.return();
+                            }
+                        } finally{
+                            if (_didIteratorError) {
+                                throw _iteratorError;
+                            }
+                        }
+                    }
+                    _state.label = 3;
+                case 3:
+                    raw = (0, _nodefs.readFileSync)((0, _nodepath.join)(dir, 'model.safetensors'));
+                    headerLen = Number(raw.readBigUInt64LE(0));
+                    header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8'));
+                    entry = Object.entries(header).find(function(param) {
+                        var _param = _sliced_to_array(param, 1), k = _param[0];
+                        return k !== '__metadata__';
+                    });
+                    if (!entry || entry[1].dtype !== 'F32') throw new _errorsts.SenseError('EMBED_MODEL', "".concat(model, ": expected an F32 safetensors matrix"));
+                    spec = entry[1];
+                    dims = spec.shape[1];
+                    dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];
+                    matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));
+                    tokenizerJson = JSON.parse((0, _nodefs.readFileSync)((0, _nodepath.join)(dir, 'tokenizer.json'), 'utf8'));
+                    return [
+                        4,
+                        Promise.resolve().then(function() {
+                            return /*#__PURE__*/ _interop_require_wildcard(require("@huggingface/tokenizers"));
+                        })
+                    ];
+                case 4:
+                    Tokenizer = _state.sent().Tokenizer;
+                    tok = new Tokenizer(tokenizerJson, {});
+                    unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
+                    // Symmetric model: document and query embedding are the same call.
+                    return [
+                        2,
+                        {
+                            id: "static:".concat(model),
+                            dims: dims,
+                            batchCap: BATCH_CAP,
+                            languages: languages,
+                            embedDocuments: function embedDocuments(texts) {
+                                return _async_to_generator(function() {
+                                    return _ts_generator(this, function(_state) {
+                                        return [
+                                            2,
+                                            texts.map(one)
+                                        ];
+                                    });
+                                })();
+                            },
+                            embedQuery: function embedQuery(text) {
+                                return _async_to_generator(function() {
+                                    return _ts_generator(this, function(_state) {
+                                        return [
+                                            2,
+                                            one(text)
+                                        ];
+                                    });
+                                })();
+                            }
+                        }
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/static.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["staticProvider","BATCH_CAP","model","tokenizerJson","dir","languages","cached","file","raw","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","text","ids","encode","add_special_tokens","filter","id","Number","isInteger","v","Float32Array","length","off","d","norm","Math","sqrt","isDownloadable","downloadModel","into","console","error","readLanguages","undefined","modelDir","MODEL_FILES","existsSync","join","SenseError","MODEL_FILENAMES","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","slice","vocab","unk_token","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":";;;;+BAUsBA;;;eAAAA;;;sBAVmB;wBACpB;wBACM;uBAC0E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGrG,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa;;kBAgCLC,sBAA7BA,4BAAAA,uBA/BVC,KACAC,WAKIC,QAID,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAOPC,KACAC,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAZ,eAEEa,WACFC,KACAC;QAEN,SAASC,IAAIC,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMC,MAAM,AAACJ,IAAIK,MAAM,CAACF,MAAM;gBAAEG,oBAAoB;YAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,SAACC;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOP;;YAC5H,IAAMU,IAAI,IAAIC,aAAahB;YAC3B,IAAIQ,IAAIS,MAAM,KAAK,GAAG,OAAOF;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYP,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMI,KAAN;oBACH,IAAMM,MAAMN,KAAKZ;oBACjB,IAAK,IAAImB,IAAI,GAAGA,IAAInB,MAAMmB,IAAKJ,CAAC,CAACI,EAAE,IAAIjB,MAAM,CAACgB,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAK;gBAC7BJ,CAAC,CAACI,GAAE,IAAIX,IAAIS,MAAM;gBAClBG,QAAQL,CAAC,CAACI,GAAE,GAAGJ,CAAC,CAACI,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAKJ,CAAC,CAACI,GAAE,IAAIC;YACvC,OAAOL;QACT;;;;yBAlDIQ,IAAAA,uBAAc,EAAClC,QAAfkC;;;;oBAEI;;wBAAMC,IAAAA,sBAAa,EAACnC,OAAO,SAACK,MAAM+B;mCAASC,QAAQC,KAAK,CAAC,AAAC,mBAA2BjC,OAATL,OAAM,KAAgBoC,OAAb/B,MAAK,UAAa,OAAL+B;;;;oBADxG,qFAAqF;oBACrFlC,MAAM;oBACN,uFAAuF;oBACjFE,SAASmC,IAAAA,sBAAa,EAACvC;oBAC7BG,YAAYC,UAAUA,OAAOwB,MAAM,GAAG,IAAIxB,SAASoC;;;;;;oBAEnDtC,MAAMuC,IAAAA,iBAAQ,EAACzC;oBACV,kCAAA,2BAAA;;wBAAL,IAAK,YAAc0C,oBAAW,uBAAzB,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBrC,OAAN;4BACH,IAAI,CAACsC,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAAC1C,KAAKG,QAAQ;gCAChC,MAAM,IAAIwC,oBAAU,CAAC,uBAAuB,AAAC,eAAmD3C,OAArCF,OAAM,iCAAiD8C,OAAlB5C,KAAI,gBAA8B,OAAhB4C,wBAAe,EAAC;4BACpI;wBACF;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAODxC,MAAMyC,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK;oBAC7BK,YAAYiB,OAAOlB,IAAI0C,eAAe,CAAC;oBACvCxC,SAASyC,KAAKC,KAAK,CAAC5C,IAAI6C,QAAQ,CAAC,GAAG,IAAI5C,WAAW6C,QAAQ,CAAC;oBAC5D3C,QAAQ4C,OAAOC,OAAO,CAAC9C,QAAQ+C,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC/C,SAASA,KAAK,CAAC,EAAE,CAACgD,KAAK,KAAK,OAAO,MAAM,IAAIZ,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAAN7C,OAAM;oBAC/EU,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKgD,KAAK,CAAC,EAAE;oBACpB9C,YAAYN,IAAIqD,UAAU,GAAG,IAAIpD,YAAYG,KAAKkD,YAAY,CAAC,EAAE;oBACjE/C,SAASD,YAAY,MAAM,IAAI,IAAIe,aAAarB,IAAIuD,MAAM,EAAEjD,WAAWF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,QAAQ,IAAIgB,aAAarB,IAAIuD,MAAM,CAACC,KAAK,CAAClD,WAAWA,YAAYF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,OAAO;oBAE/KV,gBAAgBgD,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BY,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUb,eAAe,CAAC;oBACpCe,iBAAQf,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqB8D,KAAK,cAA1B9D,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqB+D,SAAS,CAAC,uCAAI,CAAC;oBAuB/E,mEAAmE;oBACnE;;wBAAO;4BAAEzC,IAAI,AAAC,UAAe,OAANvB;4BAASW,MAAAA;4BAAMsD,UAAUlE;4BAAWI,WAAAA;4BAAW+D,gBAAgB,SAAhBA,eAAuBC;;;;;4CAAUA,MAAMC,GAAG,CAACnD;;;;;4BAAMoD,YAAY,SAAZA,WAAmBnD;;;;;4CAASD,IAAIC;;;;;wBAAM;;;;IAC/J"}

package/dist/cjs/embed/store.d.cts

@@ -0,0 +1,8 @@
+import type { Config } from '../config/index.js';
+import { embedConfig } from '../config/index.js';
+export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.js';
+export declare function modelPresent(cfg: Config): boolean;
+type ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;
+export declare function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig;
+export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string>;
+export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;

package/dist/cjs/embed/store.d.ts

@@ -0,0 +1,8 @@
+import type { Config } from '../config/index.js';
+import { embedConfig } from '../config/index.js';
+export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.js';
+export declare function modelPresent(cfg: Config): boolean;
+type ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;
+export declare function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig;
+export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string>;
+export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;