sensemaking 0.25.1 → 0.27.0

package/dist/esm/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","root","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa,EAAEC,IAAa;;QA+BpBC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIf,eAAeW,QAAQ;QACzB,qFAAqF;QACrFG,MAAM,MAAMf,cAAcY,OAAO,CAACK,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAER,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEC,MAAM,GAAGL;QACjH,uFAAuF;QACvF,MAAMQ,SAAShB,cAAcO,OAAOC;QACpCG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMX,SAASQ,OAAOC;QACtB,KAAK,MAAMI,QAAQd,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKiB,KAAKE,QAAQ;gBAChC,MAAM,IAAIlB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEG,IAAI,YAAY,EAAEb,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMsB,MAAM5B,aAAaE,KAAKiB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIvC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM2B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAClC,aAAaE,KAAKiB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAG1C,SAAS;IAC/B,MAAM2C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,0FAA0F;QAC1F,0FAA0F;QAC1F,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAEhD,OAAO;QAAE4B;QAAM4B,UAAU9D;QAAWU;QAAWqD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return {\n    id: `static:${model}`,\n    dims,\n    batchCap: BATCH_CAP,\n    languages,\n    embedDocuments: async (texts: string[], options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return texts.map(one);\n    },\n    embedQuery: async (text: string, options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return one(text);\n    },\n  };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","root","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","options","signal","throwIfAborted","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa,EAAEC,IAAa;;QA+BpBC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIf,eAAeW,QAAQ;QACzB,qFAAqF;QACrFG,MAAM,MAAMf,cAAcY,OAAO,CAACK,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAER,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEC,MAAM,GAAGL;QACjH,uFAAuF;QACvF,MAAMQ,SAAShB,cAAcO,OAAOC;QACpCG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMX,SAASQ,OAAOC;QACtB,KAAK,MAAMI,QAAQd,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKiB,KAAKE,QAAQ;gBAChC,MAAM,IAAIlB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEG,IAAI,YAAY,EAAEb,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMsB,MAAM5B,aAAaE,KAAKiB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIvC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM2B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAClC,aAAaE,KAAKiB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAG1C,SAAS;IAC/B,MAAM2C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,0FAA0F;QAC1F,0FAA0F;QAC1F,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QACLF,IAAI,CAAC,OAAO,EAAEhD,OAAO;QACrB4B;QACA4B,UAAU9D;QACVU;QACAqD,gBAAgB,OAAOC,OAAiBC;gBACtCA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOH,MAAMI,GAAG,CAACpB;QACnB;QACAqB,YAAY,OAAOpB,MAAcgB;gBAC/BA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOnB,IAAIC;QACb;IACF;AACF"}

package/dist/esm/embed/store.js

@@ -3,7 +3,7 @@ import { join } from 'node:path';
 import { embedConfig } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { writeFileAtomic } from '../lib/atomic-write.js';
-import { fetchWithRetry } from './http.js';
+import { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS, MODEL_FILE_ATTEMPT_TIMEOUT_MS } from './http.js';
 import { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.js';
 import { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.js';
 // identity.ts holds paths and cache identity, no network; store.ts holds the network fetches
@@ -29,20 +29,28 @@ async function resolveSha(model, root = DEFAULT_MODELS_ROOT) {
     const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;
     // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN
     // redirect would read the CDN's headers instead, which do not carry it.
-    let res;
-    try {
-        res = await fetchWithRetry(url, {
-            method: 'HEAD',
-            redirect: 'manual'
-        });
-    } catch (err) {
-        throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${err.message}`);
-    }
+    const res = await fetchWithRetry(url, {
+        method: 'HEAD',
+        redirect: 'manual'
+    }, {
+        attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS
+    });
     const sha = res.headers.get('x-repo-commit');
-    if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);
+    if (!sha) throw new SenseError('EMBED_MODEL', `${res.url}: response carried no x-repo-commit header`);
     writeRef(model, sha, root);
     return sha;
 }
+function isStringArray(value) {
+    return Array.isArray(value) && value.every((part)=>typeof part === 'string');
+}
+function isHfModelInfo(value) {
+    if (value === null || typeof value !== 'object') return false;
+    if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;
+    if (!('cardData' in value) || value.cardData === undefined) return true;
+    if (value.cardData === null || typeof value.cardData !== 'object') return false;
+    if (!('language' in value.cardData) || value.cardData.language === undefined) return true;
+    return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);
+}
 // cardData.language is the structured field; tags are a fallback for cards that only tag
 // languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.
 function languagesFromCard(info) {
@@ -72,7 +80,9 @@ async function ensureLanguages(model, root = DEFAULT_MODELS_ROOT) {
     }
     let res;
     try {
-        res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});
+        res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {}, {
+            attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS
+        });
     } catch  {
         cardUnreadable(model);
         return;
@@ -83,7 +93,9 @@ async function ensureLanguages(model, root = DEFAULT_MODELS_ROOT) {
     }
     let info;
     try {
-        info = await res.json();
+        const decoded = JSON.parse(new TextDecoder().decode(res.body));
+        if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');
+        info = decoded;
     } catch  {
         cardUnreadable(model);
         return;
@@ -91,14 +103,11 @@ async function ensureLanguages(model, root = DEFAULT_MODELS_ROOT) {
     writeLanguages(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : [], root);
 }
 async function fetchToFile(url, dest) {
-    let res;
-    try {
-        res = await fetchWithRetry(url, {});
-    } catch (err) {
-        throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${err.message}`);
-    }
-    if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);
-    writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));
+    const res = await fetchWithRetry(url, {}, {
+        attemptTimeoutMs: MODEL_FILE_ATTEMPT_TIMEOUT_MS
+    });
+    if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);
+    writeFileAtomic(dest, Buffer.from(res.body));
 }
 // Fetches an already-known sha straight into its snapshot dir, without touching refs/main --
 // for downloadModel below, and for callers pinning an exact revision (the parity suite).

package/dist/esm/embed/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","DEFAULT_MODELS_ROOT","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","root","existing","url","res","method","redirect","err","message","sha","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","undefined","cardUnreadable","console","error","ensureLanguages","measured","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAASC,cAAc,QAAQ,YAAY;AAC3C,SAASC,mBAAmB,EAAEC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC/J,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,4FAA4F;AAC5F,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIrB,YAAYoB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa,EAAEG,OAAetB,mBAAmB;IACzE,MAAMuB,WAAWlB,QAAQc,OAAOG;IAChC,IAAIC,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEL,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,IAAIM;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAeyB,KAAK;YAAEE,QAAQ;YAAQC,UAAU;QAAS;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI/B,WAAW,eAAe,CAAC,gBAAgB,EAAE2B,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,MAAMC,MAAML,IAAIM,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAIjC,WAAW,eAAe,GAAG2B,IAAI,0CAA0C,CAAC;IAChGhB,SAASW,OAAOW,KAAKR;IACrB,OAAOQ;AACT;AAOA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASG,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,MAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAAClC;IAC1C,OAAOgC,SAASD,MAAM,GAAG,IAAIC,WAAWG;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASC,eAAe1B,KAAa;IACnC2B,QAAQC,KAAK,CAAC,CAAC,yCAAyC,EAAE5B,MAAM,kGAAkG,CAAC;AACrK;AAEA,eAAe6B,gBAAgB7B,KAAa,EAAEG,OAAetB,mBAAmB;QA0BxDiC;IAzBtB,IAAIxC,WAAWU,cAAcgB,OAAOG,QAAQ;IAC5C,kFAAkF;IAClF,MAAM2B,WAAWvC,wBAAwB,CAACS,MAAM;IAChD,IAAI8B,UAAU;QACZ1C,eAAeY,OAAO8B,UAAU3B;QAChC;IACF;IACA,IAAIG;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAe,CAAC,kCAAkC,EAAEoB,OAAO,EAAE,CAAC;IAC5E,EAAE,OAAM;QACN0B,eAAe1B;QACf;IACF;IACA,IAAI,CAACM,IAAIyB,EAAE,EAAE;QACXL,eAAe1B;QACf;IACF;IACA,IAAIe;IACJ,IAAI;QACFA,OAAQ,MAAMT,IAAI0B,IAAI;IACxB,EAAE,OAAM;QACNN,eAAe1B;QACf;IACF;IACAZ,eAAeY,QAAOc,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE,EAAEX;AACvD;AAEA,eAAe8B,YAAY5B,GAAW,EAAE6B,IAAY;IAClD,IAAI5B;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAeyB,KAAK,CAAC;IACnC,EAAE,OAAOI,KAAK;QACZ,MAAM,IAAI/B,WAAW,eAAe,CAAC,gBAAgB,EAAE2B,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,IAAI,CAACJ,IAAIyB,EAAE,EAAE,MAAM,IAAIrD,WAAW,eAAe,GAAG2B,IAAI,SAAS,EAAEC,IAAI6B,MAAM,EAAE;IAC/ExD,gBAAgBuD,MAAME,OAAOC,IAAI,CAAC,MAAM/B,IAAIgC,WAAW;AACzD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeC,sBAAsBvC,KAAa,EAAEW,GAAW,EAAE6B,MAA4C,EAAErC,OAAetB,mBAAmB;IACtJ,MAAM4D,MAAMtD,YAAYa,OAAOW,KAAKR;IACpC5B,UAAUkE,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQ1D,YAAa;QAC9B,IAAIX,WAAWE,KAAKiE,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMR,YAAY,CAAC,uBAAuB,EAAEjC,MAAM,SAAS,EAAEW,IAAI,CAAC,EAAEgC,MAAM,EAAEnE,KAAKiE,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAc5C,KAAa,EAAEwC,MAA4C,EAAErC,OAAetB,mBAAmB;IACjI,IAAI,CAACE,eAAeiB,QAAQ;QAC1B,MAAM,IAAItB,WAAW,eAAe,CAAC,aAAa,EAAEsB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMW,MAAM,MAAMT,WAAWF,OAAOG;IACpC,sFAAsF;IACtF,oFAAoF;IACpF,MAAM0B,gBAAgB7B,OAAOG;IAC7B,OAAOoC,sBAAsBvC,OAAOW,KAAK6B,QAAQrC;AACnD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { type BufferedResponse, fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS, MODEL_FILE_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  const res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' }, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${res.url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\nfunction isStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.every((part) => typeof part === 'string');\n}\n\nfunction isHfModelInfo(value: unknown): value is HfModelInfo {\n  if (value === null || typeof value !== 'object') return false;\n  if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;\n  if (!('cardData' in value) || value.cardData === undefined) return true;\n  if (value.cardData === null || typeof value.cardData !== 'object') return false;\n  if (!('language' in value.cardData) || value.cardData.language === undefined) return true;\n  return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: BufferedResponse;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {}, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    const decoded: unknown = JSON.parse(new TextDecoder().decode(res.body));\n    if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');\n    info = decoded;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  const res = await fetchWithRetry(url, {}, { attemptTimeoutMs: MODEL_FILE_ATTEMPT_TIMEOUT_MS });\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(res.body));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","MODEL_FILE_ATTEMPT_TIMEOUT_MS","DEFAULT_MODELS_ROOT","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","root","existing","url","res","method","redirect","attemptTimeoutMs","sha","headers","get","isStringArray","value","Array","isArray","every","part","isHfModelInfo","tags","undefined","cardData","language","languagesFromCard","info","lang","length","tagLangs","filter","cardUnreadable","console","error","ensureLanguages","measured","ok","decoded","JSON","parse","TextDecoder","decode","body","Error","fetchToFile","dest","status","Buffer","from","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAAgCC,cAAc,EAAEC,uBAAuB,EAAEC,6BAA6B,QAAQ,YAAY;AAC1H,SAASC,mBAAmB,EAAEC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC/J,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,4FAA4F;AAC5F,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIvB,YAAYsB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa,EAAEG,OAAetB,mBAAmB;IACzE,MAAMuB,WAAWlB,QAAQc,OAAOG;IAChC,IAAIC,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEL,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,MAAMM,MAAM,MAAM5B,eAAe2B,KAAK;QAAEE,QAAQ;QAAQC,UAAU;IAAS,GAAG;QAAEC,kBAAkB9B;IAAwB;IAC1H,MAAM+B,MAAMJ,IAAIK,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAIlC,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,0CAA0C,CAAC;IACpGhB,SAASW,OAAOU,KAAKP;IACrB,OAAOO;AACT;AAOA,SAASG,cAAcC,KAAc;IACnC,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS;AACvE;AAEA,SAASC,cAAcL,KAAc;IACnC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,UAAU,OAAO;IACxD,IAAI,UAAUA,SAASA,MAAMM,IAAI,KAAKC,aAAa,CAACR,cAAcC,MAAMM,IAAI,GAAG,OAAO;IACtF,IAAI,CAAE,CAAA,cAAcN,KAAI,KAAMA,MAAMQ,QAAQ,KAAKD,WAAW,OAAO;IACnE,IAAIP,MAAMQ,QAAQ,KAAK,QAAQ,OAAOR,MAAMQ,QAAQ,KAAK,UAAU,OAAO;IAC1E,IAAI,CAAE,CAAA,cAAcR,MAAMQ,QAAQ,AAAD,KAAMR,MAAMQ,QAAQ,CAACC,QAAQ,KAAKF,WAAW,OAAO;IACrF,OAAO,OAAOP,MAAMQ,QAAQ,CAACC,QAAQ,KAAK,YAAYV,cAAcC,MAAMQ,QAAQ,CAACC,QAAQ;AAC7F;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASC,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKH,QAAQ,cAAbG,qCAAAA,eAAeF,QAAQ;IACpC,IAAI,OAAOG,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIX,MAAMC,OAAO,CAACU,SAASA,KAAKC,MAAM,GAAG,GAAG,OAAOD;IACnD,MAAME,WAAW,EAACH,aAAAA,KAAKL,IAAI,cAATK,wBAAAA,aAAa,EAAE,EAAEI,MAAM,CAACvC;IAC1C,OAAOsC,SAASD,MAAM,GAAG,IAAIC,WAAWP;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASS,eAAe9B,KAAa;IACnC+B,QAAQC,KAAK,CAAC,CAAC,yCAAyC,EAAEhC,MAAM,kGAAkG,CAAC;AACrK;AAEA,eAAeiC,gBAAgBjC,KAAa,EAAEG,OAAetB,mBAAmB;QA4BxD2C;IA3BtB,IAAIpD,WAAWY,cAAcgB,OAAOG,QAAQ;IAC5C,kFAAkF;IAClF,MAAM+B,WAAW3C,wBAAwB,CAACS,MAAM;IAChD,IAAIkC,UAAU;QACZ9C,eAAeY,OAAOkC,UAAU/B;QAChC;IACF;IACA,IAAIG;IACJ,IAAI;QACFA,MAAM,MAAM5B,eAAe,CAAC,kCAAkC,EAAEsB,OAAO,EAAE,CAAC,GAAG;YAAES,kBAAkB9B;QAAwB;IAC3H,EAAE,OAAM;QACNmD,eAAe9B;QACf;IACF;IACA,IAAI,CAACM,IAAI6B,EAAE,EAAE;QACXL,eAAe9B;QACf;IACF;IACA,IAAIyB;IACJ,IAAI;QACF,MAAMW,UAAmBC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAAClC,IAAImC,IAAI;QACrE,IAAI,CAACtB,cAAciB,UAAU,MAAM,IAAIM,MAAM;QAC7CjB,OAAOW;IACT,EAAE,OAAM;QACNN,eAAe9B;QACf;IACF;IACAZ,eAAeY,QAAOwB,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE,EAAErB;AACvD;AAEA,eAAewC,YAAYtC,GAAW,EAAEuC,IAAY;IAClD,MAAMtC,MAAM,MAAM5B,eAAe2B,KAAK,CAAC,GAAG;QAAEI,kBAAkB7B;IAA8B;IAC5F,IAAI,CAAC0B,IAAI6B,EAAE,EAAE,MAAM,IAAI3D,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,SAAS,EAAEC,IAAIuC,MAAM,EAAE;IACnFpE,gBAAgBmE,MAAME,OAAOC,IAAI,CAACzC,IAAImC,IAAI;AAC5C;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeO,sBAAsBhD,KAAa,EAAEU,GAAW,EAAEuC,MAA4C,EAAE9C,OAAetB,mBAAmB;IACtJ,MAAMqE,MAAM/D,YAAYa,OAAOU,KAAKP;IACpC9B,UAAU6E,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQnE,YAAa;QAC9B,IAAIb,WAAWE,KAAK4E,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMP,YAAY,CAAC,uBAAuB,EAAE3C,MAAM,SAAS,EAAEU,IAAI,CAAC,EAAE0C,MAAM,EAAE9E,KAAK4E,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAcrD,KAAa,EAAEiD,MAA4C,EAAE9C,OAAetB,mBAAmB;IACjI,IAAI,CAACE,eAAeiB,QAAQ;QAC1B,MAAM,IAAIxB,WAAW,eAAe,CAAC,aAAa,EAAEwB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMU,MAAM,MAAMR,WAAWF,OAAOG;IACpC,sFAAsF;IACtF,oFAAoF;IACpF,MAAM8B,gBAAgBjC,OAAOG;IAC7B,OAAO6C,sBAAsBhD,OAAOU,KAAKuC,QAAQ9C;AACnD"}

package/dist/esm/embed/types.d.ts

@@ -1,9 +1,12 @@
 export declare const STORE_DIMS = 256;
+export interface EmbedCallOptions {
+    signal?: AbortSignal;
+}
 export interface EmbedProvider {
     id: string;
     dims: number;
     batchCap: number;
     languages?: string[];
-    embedDocuments(texts: string[]): Promise<Float32Array[]>;
-    embedQuery(text: string): Promise<Float32Array>;
+    embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;
+    embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;
 }

package/dist/esm/embed/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;AAClG,OAAO,MAAMA,aAAa,IAAI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\nexport interface EmbedCallOptions {\n  signal?: AbortSignal;\n}\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;\n  embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;AAClG,OAAO,MAAMA,aAAa,IAAI"}

package/dist/esm/features/embed.js

@@ -1,9 +1,11 @@
 import { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.js';
 import { embedConfig } from '../config/index.js';
-import { stashChunkText } from '../embed/handoff.js';
+import { embedEndpointIdentity } from '../embed/endpoint.js';
+import { identifyChunkText, stashChunkText } from '../embed/handoff.js';
 import { modelIdentity } from '../embed/identity.js';
 import { countLines } from '../scan/frontmatter.js';
 import { appendRows } from '../store/shared.js';
+const CHUNK_LOOKUP_PAGE = 512;
 // The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so
 // `offset` shifts line numbers back onto raw and a range stays a direct Read range.
 function chunksOf(blocks, body, search, offset = 0, chunkTokens) {
@@ -24,7 +26,7 @@ function chunksOf(blocks, body, search, offset = 0, chunkTokens) {
 export const embed = {
     name: 'embed',
     async schema (db) {
-        await db.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))`);
+        await db.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))`);
     },
     extract (raw, body, search, _data, cfg, blocks) {
         var _cfg_embed;
@@ -32,39 +34,96 @@ export const embed = {
         // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.
         return chunksOf(blocks !== null && blocks !== void 0 ? blocks : parse(body), body, search, countLines(raw) - countLines(body), cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
     },
-    async remove (db, paths) {
-        if (paths.length === 0) return;
-        await db.runBatch('DELETE FROM embeddings WHERE "path" = ?', paths.map((p)=>[
-                p
+    async remove (db, paths, delta) {
+        const vanishedSet = new Set(delta.vanished);
+        const vanished = paths.filter((path)=>vanishedSet.has(path));
+        if (vanished.length === 0) return;
+        await db.runBatch('DELETE FROM embeddings WHERE "path" = ?', vanished.map((path)=>[
+                path
             ]));
     },
     // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file
     // filter skips it), so extracted is undefined here -- those docs contribute no rows.
-    async store (db, docs) {
+    async store (db, docs, delta) {
         const rows = [];
         const texts = new Map();
+        const added = new Set(delta.added);
+        // Keyset pages bound comparison metadata without reading native vector payloads.
+        let firstPage;
+        let nextPage;
         for (const { path, extracted } of docs){
-            if (!extracted) continue;
-            const chunks = extracted;
-            chunks.forEach((c, idx)=>rows.push([
+            const chunks = extracted !== null && extracted !== void 0 ? extracted : [];
+            if (chunks.length === 0) {
+                if (!added.has(path)) await db.runBatch('DELETE FROM embeddings WHERE "path" = ?', [
+                    [
+                        path
+                    ]
+                ]);
+                continue;
+            }
+            const identified = chunks.map((chunk)=>identifyChunkText(chunk.text));
+            const kept = new Set();
+            if (!added.has(path)) {
+                let cursor;
+                for(;;){
+                    let page;
+                    if (cursor === undefined) {
+                        firstPage !== null && firstPage !== void 0 ? firstPage : firstPage = await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE "path" = ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);
+                        page = await firstPage.all(path);
+                    } else {
+                        nextPage !== null && nextPage !== void 0 ? nextPage : nextPage = await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE "path" = ? AND chunk > ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);
+                        page = await nextPage.all(path, cursor);
+                    }
+                    const removed = [];
+                    const moved = [];
+                    for (const old of page){
+                        const index = Number(old.chunk);
+                        const chunk = chunks[index];
+                        if (!chunk || old.content_identity !== identified[index].identity) {
+                            removed.push([
+                                path,
+                                index
+                            ]);
+                        } else {
+                            kept.add(index);
+                            if (Number(old.start_line) !== chunk.startLine || Number(old.end_line) !== chunk.endLine) moved.push([
+                                chunk.startLine,
+                                chunk.endLine,
+                                path,
+                                index
+                            ]);
+                        }
+                    }
+                    if (removed.length > 0) await db.runBatch('DELETE FROM embeddings WHERE "path" = ? AND chunk = ?', removed);
+                    if (moved.length > 0) await db.runBatch('UPDATE embeddings SET start_line = ?, end_line = ? WHERE "path" = ? AND chunk = ?', moved);
+                    if (page.length < CHUNK_LOOKUP_PAGE) break;
+                    cursor = Number(page[page.length - 1].chunk);
+                }
+            }
+            const before = rows.length;
+            chunks.forEach((chunk, index)=>{
+                if (!kept.has(index)) rows.push([
                     path,
-                    idx,
-                    c.startLine,
-                    c.endLine
-                ]));
-            texts.set(path, chunks.map((c)=>c.text));
+                    index,
+                    chunk.startLine,
+                    chunk.endLine,
+                    identified[index].identity
+                ]);
+            });
+            // No new pending rows means no retained handoff. Matching pending rows can use
+            // embedPending's exact indexed-source fallback, including after repeated store calls.
+            if (rows.length > before) texts.set(path, identified);
         }
-        if (rows.length === 0) return;
         // scale and vector are written by the embed pass, not here, so they are not in the column list
         // and take the table's own default on the append path.
         await appendRows(db, 'embeddings', [
             'path',
             'chunk',
             'start_line',
-            'end_line'
-        ], 'INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);
-        // The text extract() just computed, handed to the embed pass that follows in this same
-        // process rather than re-derived from the file there (src/embed/handoff.ts).
+            'end_line',
+            'content_identity'
+        ], 'INSERT INTO embeddings ("path", chunk, start_line, end_line, content_identity, scale, vector) VALUES (?, ?, ?, ?, ?, NULL, NULL)', rows);
+        // Hand off only paths with new pending rows; missing entries use the indexed source.
         stashChunkText(db, texts);
     },
     enabledForFile (_cfg, file) {
@@ -81,6 +140,8 @@ export const embed = {
         // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker
         // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.
         const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;
-        return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;
+        const endpoint = embedEndpointIdentity(e.provider, e.url);
+        const endpointPart = endpoint === undefined ? '' : `:endpoint=${endpoint}`;
+        return `embed:${e.provider}:${e.model}${endpointPart}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;
     }
 };

package/dist/esm/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and build (or an authorised scoped CLI query) prepares them before semantic retrieval.\n\nexport type { Chunk };\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, string[]>();\n    for (const { path, extracted } of docs) {\n      if (!extracted) continue;\n      const chunks = extracted as Chunk[];\n      chunks.forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n      texts.set(\n        path,\n        chunks.map((c) => c.text)\n      );\n    }\n    if (rows.length === 0) return;\n    // scale and vector are written by the embed pass, not here, so they are not in the column list\n    // and take the table's own default on the append path.\n    await appendRows(db, 'embeddings', ['path', 'chunk', 'start_line', 'end_line'], 'INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n    // The text extract() just computed, handed to the embed pass that follows in this same\n    // process rather than re-derived from the file there (src/embed/handoff.ts).\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","stashChunkText","modelIdentity","countLines","appendRows","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","remove","paths","length","runBatch","p","store","docs","rows","texts","Map","path","extracted","chunks","forEach","idx","push","set","enabledForFile","_cfg","file","signature","e","identity","provider","model","chunkVersion"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,aAAa,QAAQ,uBAAuB;AACrD,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAQhD,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOlB,gBAAgBQ,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAChL;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUP,MAAMQ,OAAOA,MAAMC,QAAQL,WAAW2B,OAAO3B,WAAWI,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAClH;IACA,MAAMuB,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMR,GAAGS,QAAQ,CACf,2CACAF,MAAMf,GAAG,CAAC,CAACkB,IAAM;gBAACA;aAAE;IAExB;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMC,OAAMX,EAAE,EAAEY,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,MAAMC,QAAQ,IAAIC;QAClB,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIL,KAAM;YACtC,IAAI,CAACK,WAAW;YAChB,MAAMC,SAASD;YACfC,OAAOC,OAAO,CAAC,CAAC1B,GAAG2B,MAAQP,KAAKQ,IAAI,CAAC;oBAACL;oBAAMI;oBAAK3B,EAAEC,SAAS;oBAAED,EAAEE,OAAO;iBAAC;YACxEmB,MAAMQ,GAAG,CACPN,MACAE,OAAO1B,GAAG,CAAC,CAACC,IAAMA,EAAEG,IAAI;QAE5B;QACA,IAAIiB,KAAKL,MAAM,KAAK,GAAG;QACvB,+FAA+F;QAC/F,uDAAuD;QACvD,MAAM/B,WAAWuB,IAAI,cAAc;YAAC;YAAQ;YAAS;YAAc;SAAW,EAAE,+GAA+Ga;QAC/L,uFAAuF;QACvF,6EAA6E;QAC7EvC,eAAe0B,IAAIc;IACrB;IACAS,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAK5B,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9D6B,WAAUrB,GAAG;QACX,MAAMsB,IAAItD,YAAYgC;QACtB,IAAI,CAACsB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMC,WAAWD,EAAEE,QAAQ,KAAK,WAAWtD,cAAcoD,EAAEG,KAAK,IAAIxC;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMyC,eAAeJ,EAAE5C,WAAW,KAAKO,YAAY,GAAGpB,cAAc,CAAC,EAAEyD,EAAE5C,WAAW,EAAE,GAAGb;QACzF,OAAO,CAAC,MAAM,EAAEyD,EAAEE,QAAQ,CAAC,CAAC,EAAEF,EAAEG,KAAK,CAAC,CAAC,EAAEC,eAAeH,aAAatC,YAAY,CAAC,CAAC,EAAEsC,UAAU,GAAG,IAAI;IACxG;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { embedEndpointIdentity } from '../embed/endpoint.ts';\nimport { identifyChunkText, stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and build (or an authorised scoped CLI query) prepares them before semantic retrieval.\n\nexport type { Chunk };\n\nconst CHUNK_LOOKUP_PAGE = 512;\ninterface ChunkRow {\n  chunk: number;\n  start_line: number;\n  end_line: number;\n  content_identity: string;\n}\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths, delta) {\n    const vanishedSet = new Set(delta.vanished);\n    const vanished = paths.filter((path) => vanishedSet.has(path));\n    if (vanished.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      vanished.map((path) => [path])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs, delta) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, ReturnType<typeof identifyChunkText>[]>();\n    const added = new Set(delta.added);\n    // Keyset pages bound comparison metadata without reading native vector payloads.\n    let firstPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n    let nextPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n    for (const { path, extracted } of docs) {\n      const chunks = (extracted as Chunk[] | undefined) ?? [];\n      if (chunks.length === 0) {\n        if (!added.has(path)) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ?', [[path]]);\n        continue;\n      }\n      const identified = chunks.map((chunk) => identifyChunkText(chunk.text));\n      const kept = new Set<number>();\n      if (!added.has(path)) {\n        let cursor: number | undefined;\n        for (;;) {\n          let page: ChunkRow[];\n          if (cursor === undefined) {\n            firstPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n            page = (await firstPage.all(path)) as ChunkRow[];\n          } else {\n            nextPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? AND chunk > ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n            page = (await nextPage.all(path, cursor)) as ChunkRow[];\n          }\n          const removed: unknown[][] = [];\n          const moved: unknown[][] = [];\n          for (const old of page) {\n            const index = Number(old.chunk);\n            const chunk = chunks[index];\n            if (!chunk || old.content_identity !== identified[index].identity) {\n              removed.push([path, index]);\n            } else {\n              kept.add(index);\n              if (Number(old.start_line) !== chunk.startLine || Number(old.end_line) !== chunk.endLine) moved.push([chunk.startLine, chunk.endLine, path, index]);\n            }\n          }\n          if (removed.length > 0) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ? AND chunk = ?', removed);\n          if (moved.length > 0) await db.runBatch('UPDATE embeddings SET start_line = ?, end_line = ? WHERE \"path\" = ? AND chunk = ?', moved);\n          if (page.length < CHUNK_LOOKUP_PAGE) break;\n          cursor = Number(page[page.length - 1].chunk);\n        }\n      }\n      const before = rows.length;\n      chunks.forEach((chunk, index) => {\n        if (!kept.has(index)) rows.push([path, index, chunk.startLine, chunk.endLine, identified[index].identity]);\n      });\n      // No new pending rows means no retained handoff. Matching pending rows can use\n      // embedPending's exact indexed-source fallback, including after repeated store calls.\n      if (rows.length > before) texts.set(path, identified);\n    }\n    // scale and vector are written by the embed pass, not here, so they are not in the column list\n    // and take the table's own default on the append path.\n    await appendRows(db, 'embeddings', ['path', 'chunk', 'start_line', 'end_line', 'content_identity'], 'INSERT INTO embeddings (\"path\", chunk, start_line, end_line, content_identity, scale, vector) VALUES (?, ?, ?, ?, ?, NULL, NULL)', rows);\n    // Hand off only paths with new pending rows; missing entries use the indexed source.\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    const endpoint = embedEndpointIdentity(e.provider, e.url);\n    const endpointPart = endpoint === undefined ? '' : `:endpoint=${endpoint}`;\n    return `embed:${e.provider}:${e.model}${endpointPart}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","embedEndpointIdentity","identifyChunkText","stashChunkText","modelIdentity","countLines","appendRows","CHUNK_LOOKUP_PAGE","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","remove","paths","delta","vanishedSet","Set","vanished","path","has","length","runBatch","store","docs","rows","texts","Map","added","firstPage","nextPage","extracted","chunks","identified","chunk","kept","cursor","page","prepare","all","removed","moved","old","index","Number","content_identity","identity","push","add","start_line","end_line","before","forEach","set","enabledForFile","_cfg","file","signature","e","provider","model","chunkVersion","endpoint","url","endpointPart"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,iBAAiB,EAAEC,cAAc,QAAQ,sBAAsB;AACxE,SAASC,aAAa,QAAQ,uBAAuB;AACrD,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAQhD,MAAMC,oBAAoB;AAQ1B,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOrB,gBAAgBW,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8LAA8L,CAAC;IAChN;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUV,MAAMW,OAAOA,MAAMC,QAAQN,WAAW4B,OAAO5B,WAAWK,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAClH;IACA,MAAMuB,QAAON,EAAE,EAAEO,KAAK,EAAEC,KAAK;QAC3B,MAAMC,cAAc,IAAIC,IAAIF,MAAMG,QAAQ;QAC1C,MAAMA,WAAWJ,MAAMpB,MAAM,CAAC,CAACyB,OAASH,YAAYI,GAAG,CAACD;QACxD,IAAID,SAASG,MAAM,KAAK,GAAG;QAC3B,MAAMd,GAAGe,QAAQ,CACf,2CACAJ,SAASnB,GAAG,CAAC,CAACoB,OAAS;gBAACA;aAAK;IAEjC;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMI,OAAMhB,EAAE,EAAEiB,IAAI,EAAET,KAAK;QACzB,MAAMU,OAAoB,EAAE;QAC5B,MAAMC,QAAQ,IAAIC;QAClB,MAAMC,QAAQ,IAAIX,IAAIF,MAAMa,KAAK;QACjC,iFAAiF;QACjF,IAAIC;QACJ,IAAIC;QACJ,KAAK,MAAM,EAAEX,IAAI,EAAEY,SAAS,EAAE,IAAIP,KAAM;YACtC,MAAMQ,SAAUD,sBAAAA,uBAAAA,YAAqC,EAAE;YACvD,IAAIC,OAAOX,MAAM,KAAK,GAAG;gBACvB,IAAI,CAACO,MAAMR,GAAG,CAACD,OAAO,MAAMZ,GAAGe,QAAQ,CAAC,2CAA2C;oBAAC;wBAACH;qBAAK;iBAAC;gBAC3F;YACF;YACA,MAAMc,aAAaD,OAAOjC,GAAG,CAAC,CAACmC,QAAUvD,kBAAkBuD,MAAM/B,IAAI;YACrE,MAAMgC,OAAO,IAAIlB;YACjB,IAAI,CAACW,MAAMR,GAAG,CAACD,OAAO;gBACpB,IAAIiB;gBACJ,OAAS;oBACP,IAAIC;oBACJ,IAAID,WAAWvC,WAAW;wBACxBgC,sBAAAA,uBAAAA,YAAAA,YAAc,MAAMtB,GAAG+B,OAAO,CAAC,CAAC,2GAA2G,EAAEtD,mBAAmB;wBAChKqD,OAAQ,MAAMR,UAAUU,GAAG,CAACpB;oBAC9B,OAAO;wBACLW,qBAAAA,sBAAAA,WAAAA,WAAa,MAAMvB,GAAG+B,OAAO,CAAC,CAAC,yHAAyH,EAAEtD,mBAAmB;wBAC7KqD,OAAQ,MAAMP,SAASS,GAAG,CAACpB,MAAMiB;oBACnC;oBACA,MAAMI,UAAuB,EAAE;oBAC/B,MAAMC,QAAqB,EAAE;oBAC7B,KAAK,MAAMC,OAAOL,KAAM;wBACtB,MAAMM,QAAQC,OAAOF,IAAIR,KAAK;wBAC9B,MAAMA,QAAQF,MAAM,CAACW,MAAM;wBAC3B,IAAI,CAACT,SAASQ,IAAIG,gBAAgB,KAAKZ,UAAU,CAACU,MAAM,CAACG,QAAQ,EAAE;4BACjEN,QAAQO,IAAI,CAAC;gCAAC5B;gCAAMwB;6BAAM;wBAC5B,OAAO;4BACLR,KAAKa,GAAG,CAACL;4BACT,IAAIC,OAAOF,IAAIO,UAAU,MAAMf,MAAMjC,SAAS,IAAI2C,OAAOF,IAAIQ,QAAQ,MAAMhB,MAAMhC,OAAO,EAAEuC,MAAMM,IAAI,CAAC;gCAACb,MAAMjC,SAAS;gCAAEiC,MAAMhC,OAAO;gCAAEiB;gCAAMwB;6BAAM;wBACpJ;oBACF;oBACA,IAAIH,QAAQnB,MAAM,GAAG,GAAG,MAAMd,GAAGe,QAAQ,CAAC,yDAAyDkB;oBACnG,IAAIC,MAAMpB,MAAM,GAAG,GAAG,MAAMd,GAAGe,QAAQ,CAAC,qFAAqFmB;oBAC7H,IAAIJ,KAAKhB,MAAM,GAAGrC,mBAAmB;oBACrCoD,SAASQ,OAAOP,IAAI,CAACA,KAAKhB,MAAM,GAAG,EAAE,CAACa,KAAK;gBAC7C;YACF;YACA,MAAMiB,SAAS1B,KAAKJ,MAAM;YAC1BW,OAAOoB,OAAO,CAAC,CAAClB,OAAOS;gBACrB,IAAI,CAACR,KAAKf,GAAG,CAACuB,QAAQlB,KAAKsB,IAAI,CAAC;oBAAC5B;oBAAMwB;oBAAOT,MAAMjC,SAAS;oBAAEiC,MAAMhC,OAAO;oBAAE+B,UAAU,CAACU,MAAM,CAACG,QAAQ;iBAAC;YAC3G;YACA,+EAA+E;YAC/E,sFAAsF;YACtF,IAAIrB,KAAKJ,MAAM,GAAG8B,QAAQzB,MAAM2B,GAAG,CAAClC,MAAMc;QAC5C;QACA,+FAA+F;QAC/F,uDAAuD;QACvD,MAAMlD,WAAWwB,IAAI,cAAc;YAAC;YAAQ;YAAS;YAAc;YAAY;SAAmB,EAAE,oIAAoIkB;QACxO,qFAAqF;QACrF7C,eAAe2B,IAAImB;IACrB;IACA4B,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKpD,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9DqD,WAAU7C,GAAG;QACX,MAAM8C,IAAIjF,YAAYmC;QACtB,IAAI,CAAC8C,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMZ,WAAWY,EAAEC,QAAQ,KAAK,WAAW9E,cAAc6E,EAAEE,KAAK,IAAI/D;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMgE,eAAeH,EAAEpE,WAAW,KAAKO,YAAY,GAAGvB,cAAc,CAAC,EAAEoF,EAAEpE,WAAW,EAAE,GAAGhB;QACzF,MAAMwF,WAAWpF,sBAAsBgF,EAAEC,QAAQ,EAAED,EAAEK,GAAG;QACxD,MAAMC,eAAeF,aAAajE,YAAY,KAAK,CAAC,UAAU,EAAEiE,UAAU;QAC1E,OAAO,CAAC,MAAM,EAAEJ,EAAEC,QAAQ,CAAC,CAAC,EAAED,EAAEE,KAAK,GAAGI,aAAa,CAAC,EAAEH,eAAef,aAAajD,YAAY,CAAC,CAAC,EAAEiD,UAAU,GAAG,IAAI;IACvH;AACF,EAAE"}

package/dist/esm/graph/traverse.d.ts

@@ -1,8 +1,8 @@
 import type { Store } from '../store/types.js';
-interface FindPathOptions {
+export interface FindPathOptions {
     directed?: boolean;
     maxDepth?: number;
     allowed?: Set<string>;
 }
 export declare function findPath(store: Store, from: string, to: string, opts?: FindPathOptions): Promise<string[] | null>;
-export {};
+export declare function findPathInSnapshot(store: Store, from: string, to: string, opts: FindPathOptions): Promise<string[] | null>;

package/dist/esm/graph/traverse.js

@@ -53,9 +53,11 @@ async function reconstructPath(store, to) {
 // One BFS from `from`, tracking a predecessor per node. Undirected by default, matching
 // personalizedRank; null when `to` is unreached within `maxDepth`.
 export function findPath(store, from, to, opts = {}) {
-    return serialQuery(store, ()=>findIndexedPath(store, from, to, opts));
+    return serialQuery(store, ()=>store.transaction(()=>findPathInSnapshot(store, from, to, opts)));
 }
-async function findIndexedPath(store, from, to, opts) {
+// Internal entry for the scoped command wrapper, whose transaction also owns endpoint resolution
+// and scope reads. Direct graph callers use findPath() above and receive the same snapshot rule.
+export async function findPathInSnapshot(store, from, to, opts) {
     var _opts_maxDepth;
     if (from === to) return [
         from

package/dist/esm/graph/traverse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/graph/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\n\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\ninterface FindPathOptions {\n  directed?: boolean;\n  maxDepth?: number;\n  allowed?: Set<string>;\n}\n\nasync function resetVisited(store: Store): Promise<void> {\n  await store.exec('DROP TABLE IF EXISTS temp.visited');\n  await store.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');\n}\n\n// Materializes the caller-resolved scope Set into a temp table so it joins like any other\n// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.\nasync function setupAllowed(store: Store, allowed: Set<string>, extra?: string): Promise<void> {\n  const all = new Set(allowed);\n  if (extra !== undefined) all.add(extra);\n  await store.exec('DROP TABLE IF EXISTS temp.allowed_nodes');\n  await store.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');\n  await store.runBatch(\n    'INSERT INTO allowed_nodes (path) VALUES (?)',\n    [...all].map((p) => [p])\n  );\n}\n\n// Only resolved links (dst IS NOT NULL) are followed, in either direction.\nfunction candidateSelect(direction: Direction): string {\n  const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  if (direction === 'forward') return forward;\n  if (direction === 'reverse') return reverse;\n  return `${forward} UNION ALL ${reverse}`;\n}\n\n// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against\n// everything already visited, optionally filtered to the allowed set, inserted at the new depth (?2); RETURNING gives what this ring newly reached.\nfunction ringSql(direction: Direction, hasAllowed: boolean): string {\n  return `\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (${candidateSelect(direction)}) cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}\nGROUP BY cand.path\nORDER BY cand.path\nRETURNING path`;\n}\n\nasync function reconstructPath(store: Store, to: string): Promise<string[]> {\n  const lookup = await store.prepare('SELECT pred FROM visited WHERE path = ?');\n  const path: string[] = [];\n  let cur: string | null = to;\n  while (cur !== null) {\n    path.push(cur);\n    const row = (await lookup.get(cur)) as { pred: string | null } | undefined;\n    cur = row ? row.pred : null;\n  }\n  return path.reverse();\n}\n\n// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching\n// personalizedRank; null when `to` is unreached within `maxDepth`.\nexport function findPath(store: Store, from: string, to: string, opts: FindPathOptions = {}): Promise<string[] | null> {\n  return serialQuery(store, () => findIndexedPath(store, from, to, opts));\n}\n\nasync function findIndexedPath(store: Store, from: string, to: string, opts: FindPathOptions): Promise<string[] | null> {\n  if (from === to) return [from];\n\n  const direction: Direction = opts.directed ? 'forward' : 'both';\n  const maxDepth = opts.maxDepth ?? Infinity;\n  if (maxDepth <= 0) return null;\n\n  await resetVisited(store);\n  const insertStart = await store.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)');\n  await insertStart.run(from);\n  if (opts.allowed) await setupAllowed(store, opts.allowed, to);\n\n  const stmt = await store.prepare(ringSql(direction, !!opts.allowed));\n  for (let ring = 1; ring <= maxDepth; ring++) {\n    const rows = (await stmt.all(ring - 1, ring)) as Array<{ path: string }>;\n    if (rows.length === 0) break;\n    if (rows.some((row) => row.path === to)) return reconstructPath(store, to);\n  }\n  return null;\n}\n"],"names":["serialQuery","resetVisited","store","exec","setupAllowed","allowed","extra","all","Set","undefined","add","runBatch","map","p","candidateSelect","direction","forward","reverse","ringSql","hasAllowed","reconstructPath","to","lookup","prepare","path","cur","push","row","get","pred","findPath","from","opts","findIndexedPath","directed","maxDepth","Infinity","insertStart","run","stmt","ring","rows","length","some"],"mappings":"AAAA,2FAA2F;AAC3F,oEAAoE;AAEpE,SAASA,WAAW,QAAQ,yBAAyB;AAWrD,eAAeC,aAAaC,KAAY;IACtC,MAAMA,MAAMC,IAAI,CAAC;IACjB,MAAMD,MAAMC,IAAI,CAAC;AACnB;AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,eAAeC,aAAaF,KAAY,EAAEG,OAAoB,EAAEC,KAAc;IAC5E,MAAMC,MAAM,IAAIC,IAAIH;IACpB,IAAIC,UAAUG,WAAWF,IAAIG,GAAG,CAACJ;IACjC,MAAMJ,MAAMC,IAAI,CAAC;IACjB,MAAMD,MAAMC,IAAI,CAAC;IACjB,MAAMD,MAAMS,QAAQ,CAClB,+CACA;WAAIJ;KAAI,CAACK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;AAE3B;AAEA,2EAA2E;AAC3E,SAASC,gBAAgBC,SAAoB;IAC3C,MAAMC,UAAU;IAChB,MAAMC,UAAU;IAChB,IAAIF,cAAc,WAAW,OAAOC;IACpC,IAAID,cAAc,WAAW,OAAOE;IACpC,OAAO,GAAGD,QAAQ,WAAW,EAAEC,SAAS;AAC1C;AAEA,0FAA0F;AAC1F,oJAAoJ;AACpJ,SAASC,QAAQH,SAAoB,EAAEI,UAAmB;IACxD,OAAO,CAAC;;;MAGJ,EAAEL,gBAAgBC,WAAW;;AAEnC,EAAEI,aAAa,wEAAwE,GAAG;;;cAG5E,CAAC;AACf;AAEA,eAAeC,gBAAgBlB,KAAY,EAAEmB,EAAU;IACrD,MAAMC,SAAS,MAAMpB,MAAMqB,OAAO,CAAC;IACnC,MAAMC,OAAiB,EAAE;IACzB,IAAIC,MAAqBJ;IACzB,MAAOI,QAAQ,KAAM;QACnBD,KAAKE,IAAI,CAACD;QACV,MAAME,MAAO,MAAML,OAAOM,GAAG,CAACH;QAC9BA,MAAME,MAAMA,IAAIE,IAAI,GAAG;IACzB;IACA,OAAOL,KAAKP,OAAO;AACrB;AAEA,wFAAwF;AACxF,mEAAmE;AACnE,OAAO,SAASa,SAAS5B,KAAY,EAAE6B,IAAY,EAAEV,EAAU,EAAEW,OAAwB,CAAC,CAAC;IACzF,OAAOhC,YAAYE,OAAO,IAAM+B,gBAAgB/B,OAAO6B,MAAMV,IAAIW;AACnE;AAEA,eAAeC,gBAAgB/B,KAAY,EAAE6B,IAAY,EAAEV,EAAU,EAAEW,IAAqB;QAIzEA;IAHjB,IAAID,SAASV,IAAI,OAAO;QAACU;KAAK;IAE9B,MAAMhB,YAAuBiB,KAAKE,QAAQ,GAAG,YAAY;IACzD,MAAMC,YAAWH,iBAAAA,KAAKG,QAAQ,cAAbH,4BAAAA,iBAAiBI;IAClC,IAAID,YAAY,GAAG,OAAO;IAE1B,MAAMlC,aAAaC;IACnB,MAAMmC,cAAc,MAAMnC,MAAMqB,OAAO,CAAC;IACxC,MAAMc,YAAYC,GAAG,CAACP;IACtB,IAAIC,KAAK3B,OAAO,EAAE,MAAMD,aAAaF,OAAO8B,KAAK3B,OAAO,EAAEgB;IAE1D,MAAMkB,OAAO,MAAMrC,MAAMqB,OAAO,CAACL,QAAQH,WAAW,CAAC,CAACiB,KAAK3B,OAAO;IAClE,IAAK,IAAImC,OAAO,GAAGA,QAAQL,UAAUK,OAAQ;QAC3C,MAAMC,OAAQ,MAAMF,KAAKhC,GAAG,CAACiC,OAAO,GAAGA;QACvC,IAAIC,KAAKC,MAAM,KAAK,GAAG;QACvB,IAAID,KAAKE,IAAI,CAAC,CAAChB,MAAQA,IAAIH,IAAI,KAAKH,KAAK,OAAOD,gBAAgBlB,OAAOmB;IACzE;IACA,OAAO;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/graph/traverse.ts"],"sourcesContent":["// Frontier-by-frontier BFS through a temp visited table, one indexed anti-join per ring: a\n// local question touches a neighborhood, never the whole edge list.\n\nimport { serialQuery } from '../lib/serial-query.ts';\nimport type { Store } from '../store/types.ts';\n\ntype Direction = 'forward' | 'reverse' | 'both';\n\nexport interface FindPathOptions {\n  directed?: boolean;\n  maxDepth?: number;\n  allowed?: Set<string>;\n}\n\nasync function resetVisited(store: Store): Promise<void> {\n  await store.exec('DROP TABLE IF EXISTS temp.visited');\n  await store.exec('CREATE TEMP TABLE visited (path TEXT PRIMARY KEY, depth INTEGER NOT NULL, pred TEXT)');\n}\n\n// Materializes the caller-resolved scope Set into a temp table so it joins like any other\n// indexed column. `extra`, when given, is always in scope -- findPath's `to` endpoint.\nasync function setupAllowed(store: Store, allowed: Set<string>, extra?: string): Promise<void> {\n  const all = new Set(allowed);\n  if (extra !== undefined) all.add(extra);\n  await store.exec('DROP TABLE IF EXISTS temp.allowed_nodes');\n  await store.exec('CREATE TEMP TABLE allowed_nodes (path TEXT PRIMARY KEY)');\n  await store.runBatch(\n    'INSERT INTO allowed_nodes (path) VALUES (?)',\n    [...all].map((p) => [p])\n  );\n}\n\n// Only resolved links (dst IS NOT NULL) are followed, in either direction.\nfunction candidateSelect(direction: Direction): string {\n  const forward = 'SELECT l.dst AS path, l.src AS pred FROM links l JOIN visited v ON v.path = l.src AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  const reverse = 'SELECT l.src AS path, l.dst AS pred FROM links l JOIN visited v ON v.path = l.dst AND v.depth = ?1 WHERE l.dst IS NOT NULL';\n  if (direction === 'forward') return forward;\n  if (direction === 'reverse') return reverse;\n  return `${forward} UNION ALL ${reverse}`;\n}\n\n// One ring: candidates adjacent to the previous ring's frontier (?1), anti-joined against\n// everything already visited, optionally filtered to the allowed set, inserted at the new depth (?2); RETURNING gives what this ring newly reached.\nfunction ringSql(direction: Direction, hasAllowed: boolean): string {\n  return `\nINSERT INTO visited (path, depth, pred)\nSELECT cand.path, ?2, MIN(cand.pred)\nFROM (${candidateSelect(direction)}) cand\nWHERE NOT EXISTS (SELECT 1 FROM visited v2 WHERE v2.path = cand.path)\n${hasAllowed ? 'AND EXISTS (SELECT 1 FROM allowed_nodes a WHERE a.path = cand.path)' : ''}\nGROUP BY cand.path\nORDER BY cand.path\nRETURNING path`;\n}\n\nasync function reconstructPath(store: Store, to: string): Promise<string[]> {\n  const lookup = await store.prepare('SELECT pred FROM visited WHERE path = ?');\n  const path: string[] = [];\n  let cur: string | null = to;\n  while (cur !== null) {\n    path.push(cur);\n    const row = (await lookup.get(cur)) as { pred: string | null } | undefined;\n    cur = row ? row.pred : null;\n  }\n  return path.reverse();\n}\n\n// One BFS from `from`, tracking a predecessor per node. Undirected by default, matching\n// personalizedRank; null when `to` is unreached within `maxDepth`.\nexport function findPath(store: Store, from: string, to: string, opts: FindPathOptions = {}): Promise<string[] | null> {\n  return serialQuery(store, () => store.transaction(() => findPathInSnapshot(store, from, to, opts)));\n}\n\n// Internal entry for the scoped command wrapper, whose transaction also owns endpoint resolution\n// and scope reads. Direct graph callers use findPath() above and receive the same snapshot rule.\nexport async function findPathInSnapshot(store: Store, from: string, to: string, opts: FindPathOptions): Promise<string[] | null> {\n  if (from === to) return [from];\n\n  const direction: Direction = opts.directed ? 'forward' : 'both';\n  const maxDepth = opts.maxDepth ?? Infinity;\n  if (maxDepth <= 0) return null;\n\n  await resetVisited(store);\n  const insertStart = await store.prepare('INSERT INTO visited (path, depth, pred) VALUES (?, 0, NULL)');\n  await insertStart.run(from);\n  if (opts.allowed) await setupAllowed(store, opts.allowed, to);\n\n  const stmt = await store.prepare(ringSql(direction, !!opts.allowed));\n  for (let ring = 1; ring <= maxDepth; ring++) {\n    const rows = (await stmt.all(ring - 1, ring)) as Array<{ path: string }>;\n    if (rows.length === 0) break;\n    if (rows.some((row) => row.path === to)) return reconstructPath(store, to);\n  }\n  return null;\n}\n"],"names":["serialQuery","resetVisited","store","exec","setupAllowed","allowed","extra","all","Set","undefined","add","runBatch","map","p","candidateSelect","direction","forward","reverse","ringSql","hasAllowed","reconstructPath","to","lookup","prepare","path","cur","push","row","get","pred","findPath","from","opts","transaction","findPathInSnapshot","directed","maxDepth","Infinity","insertStart","run","stmt","ring","rows","length","some"],"mappings":"AAAA,2FAA2F;AAC3F,oEAAoE;AAEpE,SAASA,WAAW,QAAQ,yBAAyB;AAWrD,eAAeC,aAAaC,KAAY;IACtC,MAAMA,MAAMC,IAAI,CAAC;IACjB,MAAMD,MAAMC,IAAI,CAAC;AACnB;AAEA,0FAA0F;AAC1F,uFAAuF;AACvF,eAAeC,aAAaF,KAAY,EAAEG,OAAoB,EAAEC,KAAc;IAC5E,MAAMC,MAAM,IAAIC,IAAIH;IACpB,IAAIC,UAAUG,WAAWF,IAAIG,GAAG,CAACJ;IACjC,MAAMJ,MAAMC,IAAI,CAAC;IACjB,MAAMD,MAAMC,IAAI,CAAC;IACjB,MAAMD,MAAMS,QAAQ,CAClB,+CACA;WAAIJ;KAAI,CAACK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;AAE3B;AAEA,2EAA2E;AAC3E,SAASC,gBAAgBC,SAAoB;IAC3C,MAAMC,UAAU;IAChB,MAAMC,UAAU;IAChB,IAAIF,cAAc,WAAW,OAAOC;IACpC,IAAID,cAAc,WAAW,OAAOE;IACpC,OAAO,GAAGD,QAAQ,WAAW,EAAEC,SAAS;AAC1C;AAEA,0FAA0F;AAC1F,oJAAoJ;AACpJ,SAASC,QAAQH,SAAoB,EAAEI,UAAmB;IACxD,OAAO,CAAC;;;MAGJ,EAAEL,gBAAgBC,WAAW;;AAEnC,EAAEI,aAAa,wEAAwE,GAAG;;;cAG5E,CAAC;AACf;AAEA,eAAeC,gBAAgBlB,KAAY,EAAEmB,EAAU;IACrD,MAAMC,SAAS,MAAMpB,MAAMqB,OAAO,CAAC;IACnC,MAAMC,OAAiB,EAAE;IACzB,IAAIC,MAAqBJ;IACzB,MAAOI,QAAQ,KAAM;QACnBD,KAAKE,IAAI,CAACD;QACV,MAAME,MAAO,MAAML,OAAOM,GAAG,CAACH;QAC9BA,MAAME,MAAMA,IAAIE,IAAI,GAAG;IACzB;IACA,OAAOL,KAAKP,OAAO;AACrB;AAEA,wFAAwF;AACxF,mEAAmE;AACnE,OAAO,SAASa,SAAS5B,KAAY,EAAE6B,IAAY,EAAEV,EAAU,EAAEW,OAAwB,CAAC,CAAC;IACzF,OAAOhC,YAAYE,OAAO,IAAMA,MAAM+B,WAAW,CAAC,IAAMC,mBAAmBhC,OAAO6B,MAAMV,IAAIW;AAC9F;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,OAAO,eAAeE,mBAAmBhC,KAAY,EAAE6B,IAAY,EAAEV,EAAU,EAAEW,IAAqB;QAInFA;IAHjB,IAAID,SAASV,IAAI,OAAO;QAACU;KAAK;IAE9B,MAAMhB,YAAuBiB,KAAKG,QAAQ,GAAG,YAAY;IACzD,MAAMC,YAAWJ,iBAAAA,KAAKI,QAAQ,cAAbJ,4BAAAA,iBAAiBK;IAClC,IAAID,YAAY,GAAG,OAAO;IAE1B,MAAMnC,aAAaC;IACnB,MAAMoC,cAAc,MAAMpC,MAAMqB,OAAO,CAAC;IACxC,MAAMe,YAAYC,GAAG,CAACR;IACtB,IAAIC,KAAK3B,OAAO,EAAE,MAAMD,aAAaF,OAAO8B,KAAK3B,OAAO,EAAEgB;IAE1D,MAAMmB,OAAO,MAAMtC,MAAMqB,OAAO,CAACL,QAAQH,WAAW,CAAC,CAACiB,KAAK3B,OAAO;IAClE,IAAK,IAAIoC,OAAO,GAAGA,QAAQL,UAAUK,OAAQ;QAC3C,MAAMC,OAAQ,MAAMF,KAAKjC,GAAG,CAACkC,OAAO,GAAGA;QACvC,IAAIC,KAAKC,MAAM,KAAK,GAAG;QACvB,IAAID,KAAKE,IAAI,CAAC,CAACjB,MAAQA,IAAIH,IAAI,KAAKH,KAAK,OAAOD,gBAAgBlB,OAAOmB;IACzE;IACA,OAAO;AACT"}

package/dist/esm/index.d.ts

@@ -1,5 +1,5 @@
-export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.js';
-export { mapTree, peek, presetCoverage, search } from './commands/index.js';
+export type { PathOptions, Peek, PeekOptions, PeekSection, PresetCoverage, RelatedOptions, RelatedResult, SearchOptions, SearchResult, SearchResultVia, SearchSignalContribution, TreeMap, TreeMapField, TreeMapHub, TreeMapRecent } from './commands/index.js';
+export { findPath, mapTree, peek, presetCoverage, relatedNotes, search } from './commands/index.js';
 export type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
 export type { SenseErrorCode } from './errors.js';

package/dist/esm/index.js

@@ -1,6 +1,6 @@
 // Public library API. Deliberately small: every export is a stability promise;
 // internals (feature registry, graph, scan, meta) stay module-private.
-export { mapTree, peek, presetCoverage, search } from './commands/index.js';
+export { findPath, mapTree, peek, presetCoverage, relatedNotes, search } from './commands/index.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
 export { SenseError } from './errors.js';
 export { printRows } from './output/output.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { BuildOptions, BuildResult, OpenOptions, OpenResult, Stages } from './store/index.ts';\nexport { buildIndex as build, CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","buildIndex","build","CAPABILITY_NAMES","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,cAAcC,KAAK,EAAEC,gBAAgB,EAAEC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAIxG,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { PathOptions, Peek, PeekOptions, PeekSection, PresetCoverage, RelatedOptions, RelatedResult, SearchOptions, SearchResult, SearchResultVia, SearchSignalContribution, TreeMap, TreeMapField, TreeMapHub, TreeMapRecent } from './commands/index.ts';\nexport { findPath, mapTree, peek, presetCoverage, relatedNotes, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, LoadConfigOptions, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { BuildOptions, BuildResult, OpenOptions, OpenResult, Stages } from './store/index.ts';\nexport { buildIndex as build, CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["findPath","mapTree","peek","presetCoverage","relatedNotes","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","buildIndex","build","CAPABILITY_NAMES","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,QAAQ,EAAEC,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,YAAY,EAAEC,MAAM,QAAQ,sBAAsB;AAEpG,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,cAAcC,KAAK,EAAEC,gBAAgB,EAAEC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAIxG,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/output/output.d.ts

@@ -1,3 +1,5 @@
+import type { TreeMap } from '../commands/map.js';
+import type { Peek } from '../commands/peek.js';
 export type Row = Record<string, unknown>;
 export type Format = 'table' | 'json';
 export type RowFormat = Format | 'csv';
@@ -15,39 +17,5 @@ export declare function presetsLines(presets: Array<{
     embedded: number;
     signals: Record<string, number>;
 }>): string[];
-export declare function renderMap(result: {
-    docs: {
-        count: number;
-        bytes: number;
-    };
-    fields: Row[];
-    fieldsTotal: number;
-    features: {
-        on: string[];
-        off: string[];
-    };
-    presets: Array<{
-        name: string;
-        files: number;
-        embedded: number;
-        signals: Record<string, number>;
-    }>;
-    hubs: Row[];
-    recent: Row[];
-    recentCaveat: string | null;
-}): string;
-export declare function renderPeek(result: {
-    path: string;
-    tokens: number;
-    frontmatter: Row;
-    parseError?: string | null;
-    sections: Row[];
-    outbound: string[];
-    backlinks: string[];
-    unresolved: string[];
-    sectionsTotal: number;
-    outboundTotal: number;
-    backlinksTotal: number;
-    unresolvedTotal: number;
-    off: string[];
-}): string;
+export declare function renderMap(result: TreeMap): string;
+export declare function renderPeek(result: Peek): string;

package/dist/esm/output/output.js

@@ -1,4 +1,3 @@
-// rows -> table (default) | json | csv
 // Warned about on stderr (keeps --format json stdout machine-readable), never truncated.
 const OVERSIZE_BYTES = 50000;
 // Flatten embedded newlines so they can't break table column alignment; JSON output keeps them as-is.
@@ -139,7 +138,7 @@ function renderRows(rows, format, width = process.stdout.columns) {
     if (format === 'json') return stringifyJson(rows, 2);
     if (rows.length === 0) return '(0 rows)';
     const columns = Object.keys(rows[0]);
-    const linesByCell = rows.map((row)=>columns.map((col)=>cellLines(row[col])));
+    const linesByCell = rows.map((row)=>columns.map((col)=>cellLines(Reflect.get(row, col))));
     const heights = linesByCell.map((cellsInRow)=>Math.max(1, ...cellsInRow.map((lines)=>lines.length)));
     const natural = columns.map((col, i)=>Math.max(col.length, ...linesByCell.flatMap((cellsInRow)=>cellsInRow[i]).map((line)=>line.length)));
     const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;