sensemaking 0.27.0 → 0.28.0

package/dist/esm/embed/openai.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\ninterface OpenaiEmbedding {\n  embedding: number[];\n  index?: number;\n}\n\nfunction isEmbedding(value: unknown): value is OpenaiEmbedding {\n  if (value === null || typeof value !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every((part) => typeof part === 'number')) return false;\n  return !('index' in value) || value.index === undefined || typeof value.index === 'number';\n}\n\nfunction parseBody(body: ArrayBuffer, endpoint: string): OpenaiEmbedding[] {\n  let decoded: unknown;\n  try {\n    decoded = JSON.parse(new TextDecoder().decode(body));\n  } catch {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned invalid JSON`);\n  }\n  if (decoded === null || typeof decoded !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned an invalid embedding response`);\n  }\n  return decoded.data;\n}\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  const base = effectiveEmbedEndpoint('openai', url);\n  if (!base) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) }, { signal: options?.signal, attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n    options?.signal?.throwIfAborted();\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n    const body = parseBody(res.body, res.url);\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.every((d): d is OpenaiEmbedding & { index: number } => typeof d.index === 'number') ? [...body].sort((a, b) => a.index - b.index) : body;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${res.url} returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text, options) => (await post([text], options))[0] };\n}\n"],"names":["SenseError","effectiveEmbedEndpoint","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","BATCH_CAP","isEmbedding","value","Array","isArray","embedding","every","part","index","undefined","parseBody","body","endpoint","decoded","JSON","parse","TextDecoder","decode","data","openaiProvider","model","url","keyEnv","base","headers","key","process","env","authorization","post","texts","options","res","method","stringify","input","signal","attemptTimeoutMs","throwIfAborted","ok","status","d","sort","a","b","length","map","Float32Array","from","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,EAAEC,uBAAuB,QAAQ,YAAY;AAGpE,MAAMC,YAAY;AAOlB,SAASC,YAAYC,KAAc;IACjC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,YAAY,CAAE,CAAA,eAAeA,KAAI,KAAM,CAACC,MAAMC,OAAO,CAACF,MAAMG,SAAS,KAAK,CAACH,MAAMG,SAAS,CAACC,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS,WAAW,OAAO;IACpL,OAAO,CAAE,CAAA,WAAWL,KAAI,KAAMA,MAAMM,KAAK,KAAKC,aAAa,OAAOP,MAAMM,KAAK,KAAK;AACpF;AAEA,SAASE,UAAUC,IAAiB,EAAEC,QAAgB;IACpD,IAAIC;IACJ,IAAI;QACFA,UAAUC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAACN;IAChD,EAAE,OAAM;QACN,MAAM,IAAIf,WAAW,eAAe,GAAGgB,SAAS,sBAAsB,CAAC;IACzE;IACA,IAAIC,YAAY,QAAQ,OAAOA,YAAY,YAAY,CAAE,CAAA,UAAUA,OAAM,KAAM,CAACV,MAAMC,OAAO,CAACS,QAAQK,IAAI,KAAK,CAACL,QAAQK,IAAI,CAACZ,KAAK,CAACL,cAAc;QAC/I,MAAM,IAAIL,WAAW,eAAe,GAAGgB,SAAS,uCAAuC,CAAC;IAC1F;IACA,OAAOC,QAAQK,IAAI;AACrB;AAEA,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,MAAMC,OAAO1B,uBAAuB,UAAUwB;IAC9C,IAAI,CAACE,MAAM,MAAM,IAAI3B,WAAW,eAAe;IAC/C,MAAM4B,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASI,QAAQC,GAAG,CAACL,OAAO,GAAGb;IAC3C,IAAIgB,KAAKD,QAAQI,aAAa,GAAG,CAAC,OAAO,EAAEH,KAAK;IAEhD,eAAeI,KAAKC,KAAe,EAAEC,OAA0B;YAE7DA;QADA,MAAMC,MAAM,MAAMlC,eAAe,GAAGyB,KAAK,WAAW,CAAC,EAAE;YAAEU,QAAQ;YAAQT;YAASb,MAAMG,KAAKoB,SAAS,CAAC;gBAAEd;gBAAOe,OAAOL;YAAM;QAAG,GAAG;YAAEM,MAAM,EAAEL,oBAAAA,8BAAAA,QAASK,MAAM;YAAEC,kBAAkBtC;QAAwB;QACxMgC,oBAAAA,+BAAAA,kBAAAA,QAASK,MAAM,cAAfL,sCAAAA,gBAAiBO,cAAc;QAC/B,IAAI,CAACN,IAAIO,EAAE,EAAE,MAAM,IAAI3C,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,SAAS,EAAEW,IAAIQ,MAAM,EAAE;QACnF,MAAM7B,OAAOD,UAAUsB,IAAIrB,IAAI,EAAEqB,IAAIX,GAAG;QACxC,yFAAyF;QACzF,kFAAkF;QAClF,MAAMH,OAAOP,KAAKL,KAAK,CAAC,CAACmC,IAAgD,OAAOA,EAAEjC,KAAK,KAAK,YAAY;eAAIG;SAAK,CAAC+B,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEnC,KAAK,GAAGoC,EAAEpC,KAAK,IAAIG;QACtJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIO,KAAK2B,MAAM,KAAKf,MAAMe,MAAM,EAAE,MAAM,IAAIjD,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,UAAU,EAAEH,KAAK2B,MAAM,CAAC,gBAAgB,EAAEf,MAAMe,MAAM,CAAC,OAAO,CAAC;QAChJ,OAAO3B,KAAK4B,GAAG,CAAC,CAACL,IAAMM,aAAaC,IAAI,CAACP,EAAEpC,SAAS;IACtD;IAEA,MAAM4C,OAAO,AAAC,CAAA,MAAMpB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACgB,MAAM;IACxD,oFAAoF;IACpF,gDAAgD;IAChD,OAAO;QAAEK,IAAI,CAAC,OAAO,EAAE3B,KAAK,CAAC,EAAEH,OAAO;QAAE6B;QAAME,UAAUnD;QAAWoD,gBAAgBvB;QAAMwB,YAAY,OAAOC,MAAMvB,UAAY,AAAC,CAAA,MAAMF,KAAK;gBAACyB;aAAK,EAAEvB,QAAO,CAAE,CAAC,EAAE;IAAC;AACjK"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\ninterface OpenaiEmbedding {\n  embedding: number[];\n  index?: number;\n}\n\nfunction isEmbedding(value: unknown): value is OpenaiEmbedding {\n  if (value === null || typeof value !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every((part) => typeof part === 'number')) return false;\n  return !('index' in value) || value.index === undefined || typeof value.index === 'number';\n}\n\nfunction parseBody(body: ArrayBuffer, endpoint: string): OpenaiEmbedding[] {\n  let decoded: unknown;\n  try {\n    decoded = JSON.parse(new TextDecoder().decode(body));\n  } catch {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned invalid JSON`);\n  }\n  if (decoded === null || typeof decoded !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned an invalid embedding response`);\n  }\n  return decoded.data;\n}\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  const base = effectiveEmbedEndpoint('openai', url);\n  if (!base) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) }, { signal: options?.signal, attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n    options?.signal?.throwIfAborted();\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n    const body = parseBody(res.body, res.url);\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.every((d): d is OpenaiEmbedding & { index: number } => typeof d.index === 'number') ? [...body].sort((a, b) => a.index - b.index) : body;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${res.url} returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text, options) => (await post([text], options))[0] };\n}\n"],"names":["SenseError","effectiveEmbedEndpoint","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","BATCH_CAP","isEmbedding","value","Array","isArray","embedding","every","part","index","undefined","parseBody","body","endpoint","decoded","JSON","parse","TextDecoder","decode","data","openaiProvider","model","url","keyEnv","base","headers","key","process","env","authorization","post","texts","options","res","method","stringify","input","signal","attemptTimeoutMs","throwIfAborted","ok","status","d","sort","a","b","length","map","Float32Array","from","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,EAAEC,uBAAuB,QAAQ,YAAY;AAGpE,MAAMC,YAAY;AAOlB,SAASC,YAAYC,KAAc;IACjC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,YAAY,CAAE,CAAA,eAAeA,KAAI,KAAM,CAACC,MAAMC,OAAO,CAACF,MAAMG,SAAS,KAAK,CAACH,MAAMG,SAAS,CAACC,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS,WAAW,OAAO;IACpL,OAAO,CAAE,CAAA,WAAWL,KAAI,KAAMA,MAAMM,KAAK,KAAKC,aAAa,OAAOP,MAAMM,KAAK,KAAK;AACpF;AAEA,SAASE,UAAUC,IAAiB,EAAEC,QAAgB;IACpD,IAAIC;IACJ,IAAI;QACFA,UAAUC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAACN;IAChD,EAAE,OAAM;QACN,MAAM,IAAIf,WAAW,eAAe,GAAGgB,SAAS,sBAAsB,CAAC;IACzE;IACA,IAAIC,YAAY,QAAQ,OAAOA,YAAY,YAAY,CAAE,CAAA,UAAUA,OAAM,KAAM,CAACV,MAAMC,OAAO,CAACS,QAAQK,IAAI,KAAK,CAACL,QAAQK,IAAI,CAACZ,KAAK,CAACL,cAAc;QAC/I,MAAM,IAAIL,WAAW,eAAe,GAAGgB,SAAS,uCAAuC,CAAC;IAC1F;IACA,OAAOC,QAAQK,IAAI;AACrB;AAEA,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,MAAMC,OAAO1B,uBAAuB,UAAUwB;IAC9C,IAAI,CAACE,MAAM,MAAM,IAAI3B,WAAW,eAAe;IAC/C,MAAM4B,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASI,QAAQC,GAAG,CAACL,OAAO,GAAGb;IAC3C,IAAIgB,KAAKD,QAAQI,aAAa,GAAG,CAAC,OAAO,EAAEH,KAAK;IAEhD,eAAeI,KAAKC,KAAe,EAAEC,OAA0B;YAE7DA;QADA,MAAMC,MAAM,MAAMlC,eAAe,GAAGyB,KAAK,WAAW,CAAC,EAAE;YAAEU,QAAQ;YAAQT;YAASb,MAAMG,KAAKoB,SAAS,CAAC;gBAAEd;gBAAOe,OAAOL;YAAM;QAAG,GAAG;YAAEM,MAAM,EAAEL,oBAAAA,8BAAAA,QAASK,MAAM;YAAEC,kBAAkBtC;QAAwB;QACxMgC,oBAAAA,+BAAAA,kBAAAA,QAASK,MAAM,cAAfL,sCAAAA,gBAAiBO,cAAc;QAC/B,IAAI,CAACN,IAAIO,EAAE,EAAE,MAAM,IAAI3C,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,SAAS,EAAEW,IAAIQ,MAAM,EAAE;QACnF,MAAM7B,OAAOD,UAAUsB,IAAIrB,IAAI,EAAEqB,IAAIX,GAAG;QACxC,yFAAyF;QACzF,kFAAkF;QAClF,MAAMH,OAAOP,KAAKL,KAAK,CAAC,CAACmC,IAAgD,OAAOA,EAAEjC,KAAK,KAAK,YAAY;eAAIG;SAAK,CAAC+B,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEnC,KAAK,GAAGoC,EAAEpC,KAAK,IAAIG;QACtJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIO,KAAK2B,MAAM,KAAKf,MAAMe,MAAM,EAAE,MAAM,IAAIjD,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,UAAU,EAAEH,KAAK2B,MAAM,CAAC,gBAAgB,EAAEf,MAAMe,MAAM,CAAC,OAAO,CAAC;QAChJ,OAAO3B,KAAK4B,GAAG,CAAC,CAACL,IAAMM,aAAaC,IAAI,CAACP,EAAEpC,SAAS;IACtD;IAEA,MAAM4C,OAAO,AAAC,CAAA,MAAMpB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACgB,MAAM;IACxD,oFAAoF;IACpF,gDAAgD;IAChD,OAAO;QAAEK,IAAI,CAAC,OAAO,EAAE3B,KAAK,CAAC,EAAEH,OAAO;QAAE6B;QAAME,UAAUnD;QAAWoD,gBAAgBvB;QAAMwB,YAAY,OAAOC,MAAMvB,UAAY,AAAC,CAAA,MAAMF,KAAK;gBAACyB;aAAK,EAAEvB,QAAO,CAAE,CAAC,EAAE;IAAC;AACjK"}

package/dist/esm/embed/query.js

@@ -12,7 +12,7 @@ import { checkLanguageFit } from './langfit.js';
 import { getProvider } from './registry.js';
 import { STORE_DIMS } from './types.js';
 // Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for
-// benchmark/lib/embed.mjs, which scores the same lever math offline.
+// benchmark/lib/embed.ts, which scores the same lever math offline.
 export function toStore(full, dims, int8) {
     const v = new Float32Array(dims);
     let norm = 0;

package/dist/esm/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { type Config, featureSignature, type ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.ts';\nimport { getMeta } from '../store/shared.ts';\nimport { embeddingSignatureCompatible } from '../store/signature.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { identifyChunkText, takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { type EmbedCallOptions, STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nfunction staleEmbeddingWork(reason: string): SenseError {\n  return new SenseError('INDEX_NOT_READY', `${reason}; run \\`sense build\\` (or library build(config)) again to prepare the current index generation`);\n}\n\n// expectedSignature is captured by open() before provider work. Direct internal callers omit it\n// and capture the durable value here, before selecting chunks or constructing a provider.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options: EmbedCallOptions = {}): Promise<void> {\n  options.signal?.throwIfAborted();\n  const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;\n  options.signal?.throwIfAborted();\n  if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {\n    throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');\n  }\n\n  const pendingStmt = await store.prepare('SELECT \"path\", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  const dirty = ((await pendingStmt.all()) as Array<{ path: string; chunk: number; content_identity: string }>).filter((row) => allowed?.has(row.path) ?? true);\n  options.signal?.throwIfAborted();\n  if (dirty.length === 0) return;\n\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const submittedSignature = featureSignature(cfg, FEATURES);\n  if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {\n    throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');\n  }\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, typeof dirty>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string; identity: string }> = [];\n  for (const [path, pending] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && pending.every((row) => texts[row.chunk]?.identity === row.content_identity)) {\n      for (const row of pending) jobs.push({ path, chunk: row.chunk, ...texts[row.chunk] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    options.signal?.throwIfAborted();\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const row of pending) {\n      const chunk = chunks[row.chunk];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      const identified = identifyChunkText(chunk.text);\n      if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);\n      jobs.push({ path, chunk: row.chunk, ...identified });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  options.signal?.throwIfAborted();\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n  options.signal?.throwIfAborted();\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');\n    const vectors = await provider.embedDocuments(\n      batch.map((j) => j.text),\n      options\n    );\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');\n    const writes = batch.map((job, j) => {\n      const { v, scale } = toStore(vectors[j], storeDims, true);\n      const q = new Int8Array(storeDims);\n      for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n      return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n    });\n    options.signal?.throwIfAborted();\n    await store.transaction(async () => {\n      // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws\n      // inside this transaction and rolls every row in the provider batch back together.\n      options.signal?.throwIfAborted();\n      await store.vectors.writeVectors(writes);\n      options.signal?.throwIfAborted();\n      if ((await getMeta(store, FEATURE_SIGNATURE_META_KEY)) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');\n      const currentStmt = await store.prepare(`SELECT \"path\", chunk, content_identity FROM embeddings WHERE ${batch.map(() => '(\"path\" = ? AND chunk = ?)').join(' OR ')}`);\n      const current = (await currentStmt.all(...batch.flatMap((job) => [job.path, job.chunk]))) as Array<{ path: string; chunk: number; content_identity: string }>;\n      const identities = new Map(current.map((row) => [`${row.path}\\0${Number(row.chunk)}`, row.content_identity]));\n      if (batch.some((job) => identities.get(`${job.path}\\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');\n      options.signal?.throwIfAborted();\n    });\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\nexport interface PreparedSemanticQuery {\n  readonly vector: Float32Array;\n  readonly storeDims: number;\n}\n\n// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as\n// lexical syntax before the query is embedded, matching the prior semantic query behavior.\nexport async function prepareSemanticQuery(cfg: Config, terms: string, options: EmbedCallOptions = {}): Promise<PreparedSemanticQuery> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  options.signal?.throwIfAborted();\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const queryVector = await provider.embedQuery(text, options);\n  options.signal?.throwIfAborted();\n  const { v: qv } = toStore(queryVector, storeDims, false);\n  return { vector: qv, storeDims };\n}\n\n// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot\n// and the prepared vector, so this performs no provider work and only delegates to the store.\nexport function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["featureSignature","SenseError","embed","FEATURES","progress","parseSource","FEATURE_SIGNATURE_META_KEY","getMeta","embeddingSignatureCompatible","identifyChunkText","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","staleEmbeddingWork","reason","embedPending","store","cfg","allowed","expectedSignature","options","signal","throwIfAborted","expectedDurableSignature","undefined","pendingStmt","prepare","dirty","all","filter","row","has","path","length","provider","submittedSignature","storeDims","min","byPath","Map","list","get","push","set","textByPath","sourceStmt","jobs","pending","texts","every","chunk","identity","content_identity","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","text","presets","doc","extracted","identified","map","j","report","i","batchCap","batch","slice","vectors","embedDocuments","writes","job","q","Int8Array","round","vector","from","buffer","transaction","writeVectors","currentStmt","join","current","flatMap","identities","Number","some","tick","finish","ensureDocumentEmbeddings","_cfg","prepareSemanticQuery","terms","rootDir","baseDir","match","t","includes","queryVector","embedQuery","qv","semanticCandidates","prepared","fetch","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AAAA,SAAsBA,gBAAgB,QAA6B,qBAAqB;AACxF,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAC/C,SAASC,0BAA0B,QAAQ,wBAAwB;AACnE,SAASC,OAAO,QAAQ,qBAAqB;AAC7C,SAASC,4BAA4B,QAAQ,wBAAwB;AAErE,SAASC,iBAAiB,EAAEC,aAAa,QAAQ,eAAe;AAChE,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAAgCC,UAAU,QAAQ,aAAa;AAE/D,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,SAASE,mBAAmBC,MAAc;IACxC,OAAO,IAAI3B,WAAW,mBAAmB,GAAG2B,OAAO,8FAA8F,CAAC;AACpJ;AAEA,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,eAAeC,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B,EAAEC,iBAAiC,EAAEC,UAA4B,CAAC,CAAC;QA4BzIxB;QA3BnBwB,iBAEAA,kBAOAA,kBAIAA,kBAuCA,sFAAsF;IACtF,2EAA2E;IAC3EA,kBAMAA;KA5DAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMC,2BAA2BJ,sBAAsBK,YAAY,MAAM/B,QAAQuB,OAAOxB,8BAA8B2B;KACtHC,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAI,CAAC5B,6BAA6B6B,0BAA0BrC,iBAAiB+B,KAAK5B,YAAY;QAC5F,MAAMwB,mBAAmB;IAC3B;IAEA,MAAMY,cAAc,MAAMT,MAAMU,OAAO,CAAC;IACxC,MAAMC,QAAQ,AAAE,CAAA,MAAMF,YAAYG,GAAG,EAAC,EAAwEC,MAAM,CAAC,CAACC;;uBAAQZ,oBAAAA,8BAAAA,QAASa,GAAG,CAACD,IAAIE,IAAI,wCAAK;;KACxJZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAIK,MAAMM,MAAM,KAAK,GAAG;IAExB,MAAMC,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMa,qBAAqBjD,iBAAiB+B,KAAK5B;IACjD,IAAI,CAACK,6BAA6B6B,0BAA0BY,qBAAqB;QAC/E,MAAMtB,mBAAmB;IAC3B;IACA,MAAMuB,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IAEpD,MAAMoC,SAAS,IAAIC;IACnB,KAAK,MAAMT,OAAOH,MAAO;YACVW;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACX,IAAIE,IAAI,eAAnBM,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACZ;QACVQ,OAAOK,GAAG,CAACb,IAAIE,IAAI,EAAEQ;IACvB;IAEA,MAAMI,cAAahD,iBAAAA,cAAcoB,oBAAdpB,4BAAAA,iBAAwB,IAAI2C;IAC/C,MAAMM,aAAa,MAAM7B,MAAMU,OAAO,CAAC;IAEvC,MAAMoB,OAA+E,EAAE;IACvF,KAAK,MAAM,CAACd,MAAMe,QAAQ,IAAIT,OAAQ;YASpClB;QARA,MAAM4B,QAAQJ,WAAWH,GAAG,CAACT;QAC7B,IAAIgB,SAASD,QAAQE,KAAK,CAAC,CAACnB;gBAAQkB;mBAAAA,EAAAA,mBAAAA,KAAK,CAAClB,IAAIoB,KAAK,CAAC,cAAhBF,uCAAAA,iBAAkBG,QAAQ,MAAKrB,IAAIsB,gBAAgB;YAAG;YACxF,KAAK,MAAMtB,OAAOiB,QAASD,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGF,KAAK,CAAClB,IAAIoB,KAAK,CAAC;YAAC;YACnF;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAMG,SAAU,MAAMR,WAAWJ,GAAG,CAACT;SACrCZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAI,CAAC+B,QAAQ,MAAM,IAAIlE,WAAW,mBAAmB,CAAC,mCAAmC,EAAE6C,KAAK,kGAAkG,CAAC;QACnM,MAAMsB,SAAS/D,YAAY;YAAEgE,SAASvB;YAAMwB,SAASxB;YAAMyB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOS,IAAI;YAAGC,SAAS,EAAE;YAAE3E,OAAO;QAAK,GAAGiE,OAAOS,IAAI,EAAE;YAAC1E;SAAM,EAAE6B,KAAK+C,GAAG,CAACC,SAAS,CAAC7E,KAAK;QACnM,KAAK,MAAM0C,OAAOiB,QAAS;YACzB,MAAMG,QAAQI,MAAM,CAACxB,IAAIoB,KAAK,CAAC;YAC/B,IAAI,CAACA,OAAO,MAAM,IAAI/D,WAAW,mBAAmB,CAAC,4BAA4B,EAAE6C,KAAK,2GAA2G,CAAC;YACpM,MAAMkC,aAAavE,kBAAkBuD,MAAMY,IAAI;YAC/C,IAAII,WAAWf,QAAQ,KAAKrB,IAAIsB,gBAAgB,EAAE,MAAMvC,mBAAmB,CAAC,cAAc,EAAEmB,KAAK,CAAC,EAAEF,IAAIoB,KAAK,CAAC,yCAAyC,CAAC;YACxJJ,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGgB,UAAU;YAAC;QACpD;IACF;KAIA9C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMzB,iBACJmB,OACAkB,UACAY,KAAKqB,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI;KAExB1C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAE9B,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAM+C,SAAS/E,SAAS,oBAAoBwD,KAAKb,MAAM;IACvD,IAAK,IAAIqC,IAAI,GAAGA,IAAIxB,KAAKb,MAAM,EAAEqC,KAAKpC,SAASqC,QAAQ,CAAE;YAEvDnD,kBAMAA,kBAQAA;QAfA,MAAMoD,QAAQ1B,KAAK2B,KAAK,CAACH,GAAGA,IAAIpC,SAASqC,QAAQ;SACjDnD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM6D,UAAU,MAAMxC,SAASyC,cAAc,CAC3CH,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI,GACvB1C;SAEFA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM+D,SAASJ,MAAML,GAAG,CAAC,CAACU,KAAKT;YAC7B,MAAM,EAAEhE,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQ0E,OAAO,CAACN,EAAE,EAAEhC,WAAW;YACpD,MAAM0C,IAAI,IAAIC,UAAU3C;YACxB,IAAK,IAAI7B,IAAI,GAAGA,IAAI6B,WAAW7B,IAAKuE,CAAC,CAACvE,EAAE,GAAGC,KAAKwE,KAAK,CAAC5E,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEsB,MAAM6C,IAAI7C,IAAI;gBAAEkB,OAAO2B,IAAI3B,KAAK;gBAAExC;gBAAOuE,QAAQrB,OAAOsB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;SACA/D,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,MAAMN,MAAMoE,WAAW,CAAC;gBACtB,wFAAwF;YACxF,mFAAmF;YACnFhE,iBAEAA,kBAMAA;aARAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;YAC9B,MAAMN,MAAM0D,OAAO,CAACW,YAAY,CAACT;aACjCxD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;YAC9B,IAAI,AAAC,MAAM7B,QAAQuB,OAAOxB,gCAAiC+B,0BAA0B,MAAMV,mBAAmB;YAC9G,MAAMyE,cAAc,MAAMtE,MAAMU,OAAO,CAAC,CAAC,6DAA6D,EAAE8C,MAAML,GAAG,CAAC,IAAM,8BAA8BoB,IAAI,CAAC,SAAS;YACpK,MAAMC,UAAW,MAAMF,YAAY1D,GAAG,IAAI4C,MAAMiB,OAAO,CAAC,CAACZ,MAAQ;oBAACA,IAAI7C,IAAI;oBAAE6C,IAAI3B,KAAK;iBAAC;YACtF,MAAMwC,aAAa,IAAInD,IAAIiD,QAAQrB,GAAG,CAAC,CAACrC,MAAQ;oBAAC,GAAGA,IAAIE,IAAI,CAAC,EAAE,EAAE2D,OAAO7D,IAAIoB,KAAK,GAAG;oBAAEpB,IAAIsB,gBAAgB;iBAAC;YAC3G,IAAIoB,MAAMoB,IAAI,CAAC,CAACf,MAAQa,WAAWjD,GAAG,CAAC,GAAGoC,IAAI7C,IAAI,CAAC,EAAE,EAAE6C,IAAI3B,KAAK,EAAE,MAAM2B,IAAI1B,QAAQ,GAAG,MAAMtC,mBAAmB;aAChHO,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAChC;QACA+C,OAAOwB,IAAI,CAACrF,KAAK6B,GAAG,CAACiC,IAAIpC,SAASqC,QAAQ,EAAEzB,KAAKb,MAAM;IACzD;IACAoC,OAAOyB,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB/E,KAAY,EAAEgF,IAAY,EAAE9E,OAA4B;IACrG,MAAM6B,UAAU,AAAC,CAAA,MAAM/B,MAAM0D,OAAO,CAAC3B,OAAO,EAAC,EAAGlB,MAAM,CAAC,CAACC,MAAQZ,QAAQa,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIe,QAAQd,MAAM,GAAG,GAAG;QACtB,MAAM,IAAI9C,WAAW,mBAAmB,GAAG4D,QAAQd,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAOA,2FAA2F;AAC3F,2FAA2F;AAC3F,OAAO,eAAegE,qBAAqBhF,GAAW,EAAEiF,KAAa,EAAE9E,UAA4B,CAAC,CAAC;QACnF,cAEF8E;QACd9E,iBAEAA,kBAGAA;IARA,MAAM+E,WAAU,eAAA,AAAClF,IAAgCkF,OAAO,cAAxC,0BAAA,eAA4C,AAAClF,IAAgCmF,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIhH,WAAW,eAAe;IAClD,MAAM2E,OAAO,EAACoC,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAErE,MAAM,CAAC,CAACyE,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIf,IAAI,CAAC;KAClHnE,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMY,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMc,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IACpD,MAAMsG,cAAc,MAAMtE,SAASuE,UAAU,CAAC3C,MAAM1C;KACpDA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAM,EAAElB,GAAGsG,EAAE,EAAE,GAAG1G,QAAQwG,aAAapE,WAAW;IAClD,OAAO;QAAE6C,QAAQyB;QAAItE;IAAU;AACjC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,OAAO,SAASuE,mBAAmB3F,KAAY,EAAE4F,QAA+B,EAAEC,KAAa,EAAE3F,OAAqB;IACpH,OAAOF,MAAM0D,OAAO,CAACoC,UAAU,CAACF,SAAS3B,MAAM,EAAE2B,SAASxE,SAAS,EAAEyE,OAAO3F;AAC9E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAe6F,aAAa/F,KAAY,EAAEgB,IAAY;IAC3D,OAAOhB,MAAM0D,OAAO,CAACsC,SAAS,CAAChF;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiF,aAAajG,KAAY,EAAEgF,IAAY,EAAEhE,IAAY,EAAEkF,IAAgE;IAC3I,OAAOlG,MAAM0D,OAAO,CAACyC,OAAO,CAACnF,MAAMkF;AACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { type Config, featureSignature, type ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.ts';\nimport { getMeta } from '../store/shared.ts';\nimport { embeddingSignatureCompatible } from '../store/signature.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { identifyChunkText, takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { type EmbedCallOptions, STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.ts, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nfunction staleEmbeddingWork(reason: string): SenseError {\n  return new SenseError('INDEX_NOT_READY', `${reason}; run \\`sense build\\` (or library build(config)) again to prepare the current index generation`);\n}\n\n// expectedSignature is captured by open() before provider work. Direct internal callers omit it\n// and capture the durable value here, before selecting chunks or constructing a provider.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options: EmbedCallOptions = {}): Promise<void> {\n  options.signal?.throwIfAborted();\n  const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;\n  options.signal?.throwIfAborted();\n  if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {\n    throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');\n  }\n\n  const pendingStmt = await store.prepare('SELECT \"path\", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  const dirty = ((await pendingStmt.all()) as Array<{ path: string; chunk: number; content_identity: string }>).filter((row) => allowed?.has(row.path) ?? true);\n  options.signal?.throwIfAborted();\n  if (dirty.length === 0) return;\n\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const submittedSignature = featureSignature(cfg, FEATURES);\n  if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {\n    throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');\n  }\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, typeof dirty>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string; identity: string }> = [];\n  for (const [path, pending] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && pending.every((row) => texts[row.chunk]?.identity === row.content_identity)) {\n      for (const row of pending) jobs.push({ path, chunk: row.chunk, ...texts[row.chunk] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    options.signal?.throwIfAborted();\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const row of pending) {\n      const chunk = chunks[row.chunk];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      const identified = identifyChunkText(chunk.text);\n      if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);\n      jobs.push({ path, chunk: row.chunk, ...identified });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  options.signal?.throwIfAborted();\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n  options.signal?.throwIfAborted();\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');\n    const vectors = await provider.embedDocuments(\n      batch.map((j) => j.text),\n      options\n    );\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');\n    const writes = batch.map((job, j) => {\n      const { v, scale } = toStore(vectors[j], storeDims, true);\n      const q = new Int8Array(storeDims);\n      for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n      return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n    });\n    options.signal?.throwIfAborted();\n    await store.transaction(async () => {\n      // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws\n      // inside this transaction and rolls every row in the provider batch back together.\n      options.signal?.throwIfAborted();\n      await store.vectors.writeVectors(writes);\n      options.signal?.throwIfAborted();\n      if ((await getMeta(store, FEATURE_SIGNATURE_META_KEY)) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');\n      const currentStmt = await store.prepare(`SELECT \"path\", chunk, content_identity FROM embeddings WHERE ${batch.map(() => '(\"path\" = ? AND chunk = ?)').join(' OR ')}`);\n      const current = (await currentStmt.all(...batch.flatMap((job) => [job.path, job.chunk]))) as Array<{ path: string; chunk: number; content_identity: string }>;\n      const identities = new Map(current.map((row) => [`${row.path}\\0${Number(row.chunk)}`, row.content_identity]));\n      if (batch.some((job) => identities.get(`${job.path}\\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');\n      options.signal?.throwIfAborted();\n    });\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\nexport interface PreparedSemanticQuery {\n  readonly vector: Float32Array;\n  readonly storeDims: number;\n}\n\n// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as\n// lexical syntax before the query is embedded, matching the prior semantic query behavior.\nexport async function prepareSemanticQuery(cfg: Config, terms: string, options: EmbedCallOptions = {}): Promise<PreparedSemanticQuery> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  options.signal?.throwIfAborted();\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const queryVector = await provider.embedQuery(text, options);\n  options.signal?.throwIfAborted();\n  const { v: qv } = toStore(queryVector, storeDims, false);\n  return { vector: qv, storeDims };\n}\n\n// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot\n// and the prepared vector, so this performs no provider work and only delegates to the store.\nexport function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["featureSignature","SenseError","embed","FEATURES","progress","parseSource","FEATURE_SIGNATURE_META_KEY","getMeta","embeddingSignatureCompatible","identifyChunkText","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","staleEmbeddingWork","reason","embedPending","store","cfg","allowed","expectedSignature","options","signal","throwIfAborted","expectedDurableSignature","undefined","pendingStmt","prepare","dirty","all","filter","row","has","path","length","provider","submittedSignature","storeDims","min","byPath","Map","list","get","push","set","textByPath","sourceStmt","jobs","pending","texts","every","chunk","identity","content_identity","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","text","presets","doc","extracted","identified","map","j","report","i","batchCap","batch","slice","vectors","embedDocuments","writes","job","q","Int8Array","round","vector","from","buffer","transaction","writeVectors","currentStmt","join","current","flatMap","identities","Number","some","tick","finish","ensureDocumentEmbeddings","_cfg","prepareSemanticQuery","terms","rootDir","baseDir","match","t","includes","queryVector","embedQuery","qv","semanticCandidates","prepared","fetch","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AAAA,SAAsBA,gBAAgB,QAA6B,qBAAqB;AACxF,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAC/C,SAASC,0BAA0B,QAAQ,wBAAwB;AACnE,SAASC,OAAO,QAAQ,qBAAqB;AAC7C,SAASC,4BAA4B,QAAQ,wBAAwB;AAErE,SAASC,iBAAiB,EAAEC,aAAa,QAAQ,eAAe;AAChE,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAAgCC,UAAU,QAAQ,aAAa;AAE/D,yFAAyF;AACzF,oEAAoE;AACpE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,SAASE,mBAAmBC,MAAc;IACxC,OAAO,IAAI3B,WAAW,mBAAmB,GAAG2B,OAAO,8FAA8F,CAAC;AACpJ;AAEA,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,eAAeC,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B,EAAEC,iBAAiC,EAAEC,UAA4B,CAAC,CAAC;QA4BzIxB;QA3BnBwB,iBAEAA,kBAOAA,kBAIAA,kBAuCA,sFAAsF;IACtF,2EAA2E;IAC3EA,kBAMAA;KA5DAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMC,2BAA2BJ,sBAAsBK,YAAY,MAAM/B,QAAQuB,OAAOxB,8BAA8B2B;KACtHC,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAI,CAAC5B,6BAA6B6B,0BAA0BrC,iBAAiB+B,KAAK5B,YAAY;QAC5F,MAAMwB,mBAAmB;IAC3B;IAEA,MAAMY,cAAc,MAAMT,MAAMU,OAAO,CAAC;IACxC,MAAMC,QAAQ,AAAE,CAAA,MAAMF,YAAYG,GAAG,EAAC,EAAwEC,MAAM,CAAC,CAACC;;uBAAQZ,oBAAAA,8BAAAA,QAASa,GAAG,CAACD,IAAIE,IAAI,wCAAK;;KACxJZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAIK,MAAMM,MAAM,KAAK,GAAG;IAExB,MAAMC,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMa,qBAAqBjD,iBAAiB+B,KAAK5B;IACjD,IAAI,CAACK,6BAA6B6B,0BAA0BY,qBAAqB;QAC/E,MAAMtB,mBAAmB;IAC3B;IACA,MAAMuB,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IAEpD,MAAMoC,SAAS,IAAIC;IACnB,KAAK,MAAMT,OAAOH,MAAO;YACVW;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACX,IAAIE,IAAI,eAAnBM,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACZ;QACVQ,OAAOK,GAAG,CAACb,IAAIE,IAAI,EAAEQ;IACvB;IAEA,MAAMI,cAAahD,iBAAAA,cAAcoB,oBAAdpB,4BAAAA,iBAAwB,IAAI2C;IAC/C,MAAMM,aAAa,MAAM7B,MAAMU,OAAO,CAAC;IAEvC,MAAMoB,OAA+E,EAAE;IACvF,KAAK,MAAM,CAACd,MAAMe,QAAQ,IAAIT,OAAQ;YASpClB;QARA,MAAM4B,QAAQJ,WAAWH,GAAG,CAACT;QAC7B,IAAIgB,SAASD,QAAQE,KAAK,CAAC,CAACnB;gBAAQkB;mBAAAA,EAAAA,mBAAAA,KAAK,CAAClB,IAAIoB,KAAK,CAAC,cAAhBF,uCAAAA,iBAAkBG,QAAQ,MAAKrB,IAAIsB,gBAAgB;YAAG;YACxF,KAAK,MAAMtB,OAAOiB,QAASD,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGF,KAAK,CAAClB,IAAIoB,KAAK,CAAC;YAAC;YACnF;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAMG,SAAU,MAAMR,WAAWJ,GAAG,CAACT;SACrCZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAI,CAAC+B,QAAQ,MAAM,IAAIlE,WAAW,mBAAmB,CAAC,mCAAmC,EAAE6C,KAAK,kGAAkG,CAAC;QACnM,MAAMsB,SAAS/D,YAAY;YAAEgE,SAASvB;YAAMwB,SAASxB;YAAMyB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOS,IAAI;YAAGC,SAAS,EAAE;YAAE3E,OAAO;QAAK,GAAGiE,OAAOS,IAAI,EAAE;YAAC1E;SAAM,EAAE6B,KAAK+C,GAAG,CAACC,SAAS,CAAC7E,KAAK;QACnM,KAAK,MAAM0C,OAAOiB,QAAS;YACzB,MAAMG,QAAQI,MAAM,CAACxB,IAAIoB,KAAK,CAAC;YAC/B,IAAI,CAACA,OAAO,MAAM,IAAI/D,WAAW,mBAAmB,CAAC,4BAA4B,EAAE6C,KAAK,2GAA2G,CAAC;YACpM,MAAMkC,aAAavE,kBAAkBuD,MAAMY,IAAI;YAC/C,IAAII,WAAWf,QAAQ,KAAKrB,IAAIsB,gBAAgB,EAAE,MAAMvC,mBAAmB,CAAC,cAAc,EAAEmB,KAAK,CAAC,EAAEF,IAAIoB,KAAK,CAAC,yCAAyC,CAAC;YACxJJ,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGgB,UAAU;YAAC;QACpD;IACF;KAIA9C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMzB,iBACJmB,OACAkB,UACAY,KAAKqB,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI;KAExB1C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAE9B,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAM+C,SAAS/E,SAAS,oBAAoBwD,KAAKb,MAAM;IACvD,IAAK,IAAIqC,IAAI,GAAGA,IAAIxB,KAAKb,MAAM,EAAEqC,KAAKpC,SAASqC,QAAQ,CAAE;YAEvDnD,kBAMAA,kBAQAA;QAfA,MAAMoD,QAAQ1B,KAAK2B,KAAK,CAACH,GAAGA,IAAIpC,SAASqC,QAAQ;SACjDnD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM6D,UAAU,MAAMxC,SAASyC,cAAc,CAC3CH,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI,GACvB1C;SAEFA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM+D,SAASJ,MAAML,GAAG,CAAC,CAACU,KAAKT;YAC7B,MAAM,EAAEhE,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQ0E,OAAO,CAACN,EAAE,EAAEhC,WAAW;YACpD,MAAM0C,IAAI,IAAIC,UAAU3C;YACxB,IAAK,IAAI7B,IAAI,GAAGA,IAAI6B,WAAW7B,IAAKuE,CAAC,CAACvE,EAAE,GAAGC,KAAKwE,KAAK,CAAC5E,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEsB,MAAM6C,IAAI7C,IAAI;gBAAEkB,OAAO2B,IAAI3B,KAAK;gBAAExC;gBAAOuE,QAAQrB,OAAOsB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;SACA/D,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,MAAMN,MAAMoE,WAAW,CAAC;gBACtB,wFAAwF;YACxF,mFAAmF;YACnFhE,iBAEAA,kBAMAA;aARAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;YAC9B,MAAMN,MAAM0D,OAAO,CAACW,YAAY,CAACT;aACjCxD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;YAC9B,IAAI,AAAC,MAAM7B,QAAQuB,OAAOxB,gCAAiC+B,0BAA0B,MAAMV,mBAAmB;YAC9G,MAAMyE,cAAc,MAAMtE,MAAMU,OAAO,CAAC,CAAC,6DAA6D,EAAE8C,MAAML,GAAG,CAAC,IAAM,8BAA8BoB,IAAI,CAAC,SAAS;YACpK,MAAMC,UAAW,MAAMF,YAAY1D,GAAG,IAAI4C,MAAMiB,OAAO,CAAC,CAACZ,MAAQ;oBAACA,IAAI7C,IAAI;oBAAE6C,IAAI3B,KAAK;iBAAC;YACtF,MAAMwC,aAAa,IAAInD,IAAIiD,QAAQrB,GAAG,CAAC,CAACrC,MAAQ;oBAAC,GAAGA,IAAIE,IAAI,CAAC,EAAE,EAAE2D,OAAO7D,IAAIoB,KAAK,GAAG;oBAAEpB,IAAIsB,gBAAgB;iBAAC;YAC3G,IAAIoB,MAAMoB,IAAI,CAAC,CAACf,MAAQa,WAAWjD,GAAG,CAAC,GAAGoC,IAAI7C,IAAI,CAAC,EAAE,EAAE6C,IAAI3B,KAAK,EAAE,MAAM2B,IAAI1B,QAAQ,GAAG,MAAMtC,mBAAmB;aAChHO,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAChC;QACA+C,OAAOwB,IAAI,CAACrF,KAAK6B,GAAG,CAACiC,IAAIpC,SAASqC,QAAQ,EAAEzB,KAAKb,MAAM;IACzD;IACAoC,OAAOyB,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB/E,KAAY,EAAEgF,IAAY,EAAE9E,OAA4B;IACrG,MAAM6B,UAAU,AAAC,CAAA,MAAM/B,MAAM0D,OAAO,CAAC3B,OAAO,EAAC,EAAGlB,MAAM,CAAC,CAACC,MAAQZ,QAAQa,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIe,QAAQd,MAAM,GAAG,GAAG;QACtB,MAAM,IAAI9C,WAAW,mBAAmB,GAAG4D,QAAQd,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAOA,2FAA2F;AAC3F,2FAA2F;AAC3F,OAAO,eAAegE,qBAAqBhF,GAAW,EAAEiF,KAAa,EAAE9E,UAA4B,CAAC,CAAC;QACnF,cAEF8E;QACd9E,iBAEAA,kBAGAA;IARA,MAAM+E,WAAU,eAAA,AAAClF,IAAgCkF,OAAO,cAAxC,0BAAA,eAA4C,AAAClF,IAAgCmF,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIhH,WAAW,eAAe;IAClD,MAAM2E,OAAO,EAACoC,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAErE,MAAM,CAAC,CAACyE,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIf,IAAI,CAAC;KAClHnE,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMY,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMc,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IACpD,MAAMsG,cAAc,MAAMtE,SAASuE,UAAU,CAAC3C,MAAM1C;KACpDA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAM,EAAElB,GAAGsG,EAAE,EAAE,GAAG1G,QAAQwG,aAAapE,WAAW;IAClD,OAAO;QAAE6C,QAAQyB;QAAItE;IAAU;AACjC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,OAAO,SAASuE,mBAAmB3F,KAAY,EAAE4F,QAA+B,EAAEC,KAAa,EAAE3F,OAAqB;IACpH,OAAOF,MAAM0D,OAAO,CAACoC,UAAU,CAACF,SAAS3B,MAAM,EAAE2B,SAASxE,SAAS,EAAEyE,OAAO3F;AAC9E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAe6F,aAAa/F,KAAY,EAAEgB,IAAY;IAC3D,OAAOhB,MAAM0D,OAAO,CAACsC,SAAS,CAAChF;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiF,aAAajG,KAAY,EAAEgF,IAAY,EAAEhE,IAAY,EAAEkF,IAAgE;IAC3I,OAAOlG,MAAM0D,OAAO,CAACyC,OAAO,CAACnF,MAAMkF;AACrC"}

package/dist/esm/embed/registry.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction aborted(reason: unknown): Promise<never> {\n  return Promise.reject(reason);\n}\n\nfunction waitForProvider(provider: Promise<EmbedProvider>, signal: AbortSignal): Promise<EmbedProvider> {\n  if (signal.aborted) return aborted(signal.reason);\n  return new Promise((resolve, reject) => {\n    const cleanup = () => signal.removeEventListener('abort', onAbort);\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    provider.then(\n      (value) => {\n        cleanup();\n        resolve(value);\n      },\n      (err: unknown) => {\n        cleanup();\n        reject(err);\n      }\n    );\n    if (signal.aborted) onAbort();\n  });\n}\n\nexport function getProvider(cfg: Config, options: EmbedCallOptions = {}): Promise<EmbedProvider> {\n  // A caller that has already left must not start shared model initialization.\n  if (options.signal?.aborted) return aborted(options.signal.reason);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  const endpoint = effectiveEmbedEndpoint(e.provider, e.url);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, endpoint ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, endpoint, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return options.signal ? waitForProvider(p, options.signal) : p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","effectiveEmbedEndpoint","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","aborted","reason","Promise","reject","waitForProvider","provider","signal","resolve","cleanup","removeEventListener","onAbort","addEventListener","once","then","value","err","getProvider","cfg","options","e","factory","endpoint","sig","JSON","stringify","languages","p","get","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQT,eAAeM,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,SAASC,QAAQC,MAAe;IAC9B,OAAOC,QAAQC,MAAM,CAACF;AACxB;AAEA,SAASG,gBAAgBC,QAAgC,EAAEC,MAAmB;IAC5E,IAAIA,OAAON,OAAO,EAAE,OAAOA,QAAQM,OAAOL,MAAM;IAChD,OAAO,IAAIC,QAAQ,CAACK,SAASJ;QAC3B,MAAMK,UAAU,IAAMF,OAAOG,mBAAmB,CAAC,SAASC;QAC1D,MAAMA,UAAU;YACdF;YACAL,OAAOG,OAAOL,MAAM;QACtB;QACAK,OAAOK,gBAAgB,CAAC,SAASD,SAAS;YAAEE,MAAM;QAAK;QACvDP,SAASQ,IAAI,CACX,CAACC;YACCN;YACAD,QAAQO;QACV,GACA,CAACC;YACCP;YACAL,OAAOY;QACT;QAEF,IAAIT,OAAON,OAAO,EAAEU;IACtB;AACF;AAEA,OAAO,SAASM,YAAYC,GAAW,EAAEC,UAA4B,CAAC,CAAC;QAUJC,QAAaA;QAR1ED;IADJ,6EAA6E;IAC7E,KAAIA,kBAAAA,QAAQZ,MAAM,cAAdY,sCAAAA,gBAAgBlB,OAAO,EAAE,OAAOA,QAAQkB,QAAQZ,MAAM,CAACL,MAAM;IACjE,MAAMkB,IAAIlC,YAAYgC;IACtB,IAAI,CAACE,GAAG,MAAM,IAAIjC,WAAW,kBAAkB;IAC/C,MAAMkC,UAAU7B,QAAQ,CAAC4B,EAAEd,QAAQ,CAAC;IACpC,IAAI,CAACe,SAAS,MAAM,IAAIlC,WAAW,eAAe,CAAC,gBAAgB,EAAEiC,EAAEd,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,MAAMgB,WAAWjC,uBAAuB+B,EAAEd,QAAQ,EAAEc,EAAExB,GAAG;IACzD,wFAAwF;IACxF,sFAAsF;IACtF,MAAM2B,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEd,QAAQ;QAAEc,EAAE1B,KAAK;QAAE4B,qBAAAA,sBAAAA,WAAY;SAAIF,SAAAA,EAAEvB,GAAG,cAALuB,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAChG,IAAIO,IAAI5B,UAAU6B,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAE1B,KAAK,EAAE4B,UAAUF,EAAEvB,GAAG;QACpC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIuB,EAAEM,SAAS,EAAEC,IAAIA,EAAEb,IAAI,CAAC,CAACe,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEH,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzE3B,UAAU+B,GAAG,CAACP,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEI,KAAK,CAAC,IAAMhC,UAAUiC,MAAM,CAACT;IACjC;IACA,OAAOJ,QAAQZ,MAAM,GAAGF,gBAAgBsB,GAAGR,QAAQZ,MAAM,IAAIoB;AAC/D"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction aborted(reason: unknown): Promise<never> {\n  return Promise.reject(reason);\n}\n\nfunction waitForProvider(provider: Promise<EmbedProvider>, signal: AbortSignal): Promise<EmbedProvider> {\n  if (signal.aborted) return aborted(signal.reason);\n  return new Promise((resolve, reject) => {\n    const cleanup = () => signal.removeEventListener('abort', onAbort);\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    provider.then(\n      (value) => {\n        cleanup();\n        resolve(value);\n      },\n      (err: unknown) => {\n        cleanup();\n        reject(err);\n      }\n    );\n    if (signal.aborted) onAbort();\n  });\n}\n\nexport function getProvider(cfg: Config, options: EmbedCallOptions = {}): Promise<EmbedProvider> {\n  // A caller that has already left must not start shared model initialization.\n  if (options.signal?.aborted) return aborted(options.signal.reason);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  const endpoint = effectiveEmbedEndpoint(e.provider, e.url);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, endpoint ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, endpoint, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return options.signal ? waitForProvider(p, options.signal) : p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","effectiveEmbedEndpoint","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","aborted","reason","Promise","reject","waitForProvider","provider","signal","resolve","cleanup","removeEventListener","onAbort","addEventListener","once","then","value","err","getProvider","cfg","options","e","factory","endpoint","sig","JSON","stringify","languages","p","get","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQT,eAAeM,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,SAASC,QAAQC,MAAe;IAC9B,OAAOC,QAAQC,MAAM,CAACF;AACxB;AAEA,SAASG,gBAAgBC,QAAgC,EAAEC,MAAmB;IAC5E,IAAIA,OAAON,OAAO,EAAE,OAAOA,QAAQM,OAAOL,MAAM;IAChD,OAAO,IAAIC,QAAQ,CAACK,SAASJ;QAC3B,MAAMK,UAAU,IAAMF,OAAOG,mBAAmB,CAAC,SAASC;QAC1D,MAAMA,UAAU;YACdF;YACAL,OAAOG,OAAOL,MAAM;QACtB;QACAK,OAAOK,gBAAgB,CAAC,SAASD,SAAS;YAAEE,MAAM;QAAK;QACvDP,SAASQ,IAAI,CACX,CAACC;YACCN;YACAD,QAAQO;QACV,GACA,CAACC;YACCP;YACAL,OAAOY;QACT;QAEF,IAAIT,OAAON,OAAO,EAAEU;IACtB;AACF;AAEA,OAAO,SAASM,YAAYC,GAAW,EAAEC,UAA4B,CAAC,CAAC;QAUJC,QAAaA;QAR1ED;IADJ,6EAA6E;IAC7E,KAAIA,kBAAAA,QAAQZ,MAAM,cAAdY,sCAAAA,gBAAgBlB,OAAO,EAAE,OAAOA,QAAQkB,QAAQZ,MAAM,CAACL,MAAM;IACjE,MAAMkB,IAAIlC,YAAYgC;IACtB,IAAI,CAACE,GAAG,MAAM,IAAIjC,WAAW,kBAAkB;IAC/C,MAAMkC,UAAU7B,QAAQ,CAAC4B,EAAEd,QAAQ,CAAC;IACpC,IAAI,CAACe,SAAS,MAAM,IAAIlC,WAAW,eAAe,CAAC,gBAAgB,EAAEiC,EAAEd,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,MAAMgB,WAAWjC,uBAAuB+B,EAAEd,QAAQ,EAAEc,EAAExB,GAAG;IACzD,wFAAwF;IACxF,sFAAsF;IACtF,MAAM2B,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEd,QAAQ;QAAEc,EAAE1B,KAAK;QAAE4B,qBAAAA,sBAAAA,WAAY;SAAIF,SAAAA,EAAEvB,GAAG,cAALuB,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAChG,IAAIO,IAAI5B,UAAU6B,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAE1B,KAAK,EAAE4B,UAAUF,EAAEvB,GAAG;QACpC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIuB,EAAEM,SAAS,EAAEC,IAAIA,EAAEb,IAAI,CAAC,CAACe,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEH,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzE3B,UAAU+B,GAAG,CAACP,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEI,KAAK,CAAC,IAAMhC,UAAUiC,MAAM,CAACT;IACjC;IACA,OAAOJ,QAAQZ,MAAM,GAAGF,gBAAgBsB,GAAGR,QAAQZ,MAAM,IAAIoB;AAC/D"}

package/dist/esm/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return {\n    id: `static:${model}`,\n    dims,\n    batchCap: BATCH_CAP,\n    languages,\n    embedDocuments: async (texts: string[], options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return texts.map(one);\n    },\n    embedQuery: async (text: string, options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return one(text);\n    },\n  };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","root","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","options","signal","throwIfAborted","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa,EAAEC,IAAa;;QA+BpBC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIf,eAAeW,QAAQ;QACzB,qFAAqF;QACrFG,MAAM,MAAMf,cAAcY,OAAO,CAACK,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAER,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEC,MAAM,GAAGL;QACjH,uFAAuF;QACvF,MAAMQ,SAAShB,cAAcO,OAAOC;QACpCG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMX,SAASQ,OAAOC;QACtB,KAAK,MAAMI,QAAQd,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKiB,KAAKE,QAAQ;gBAChC,MAAM,IAAIlB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEG,IAAI,YAAY,EAAEb,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMsB,MAAM5B,aAAaE,KAAKiB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIvC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM2B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAClC,aAAaE,KAAKiB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAG1C,SAAS;IAC/B,MAAM2C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,0FAA0F;QAC1F,0FAA0F;QAC1F,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QACLF,IAAI,CAAC,OAAO,EAAEhD,OAAO;QACrB4B;QACA4B,UAAU9D;QACVU;QACAqD,gBAAgB,OAAOC,OAAiBC;gBACtCA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOH,MAAMI,GAAG,CAACpB;QACnB;QACAqB,YAAY,OAAOpB,MAAcgB;gBAC/BA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOnB,IAAIC;QACb;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return {\n    id: `static:${model}`,\n    dims,\n    batchCap: BATCH_CAP,\n    languages,\n    embedDocuments: async (texts: string[], options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return texts.map(one);\n    },\n    embedQuery: async (text: string, options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return one(text);\n    },\n  };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","root","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","options","signal","throwIfAborted","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa,EAAEC,IAAa;;QA+BpBC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIf,eAAeW,QAAQ;QACzB,qFAAqF;QACrFG,MAAM,MAAMf,cAAcY,OAAO,CAACK,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAER,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEC,MAAM,GAAGL;QACjH,uFAAuF;QACvF,MAAMQ,SAAShB,cAAcO,OAAOC;QACpCG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMX,SAASQ,OAAOC;QACtB,KAAK,MAAMI,QAAQd,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKiB,KAAKE,QAAQ;gBAChC,MAAM,IAAIlB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEG,IAAI,YAAY,EAAEb,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMsB,MAAM5B,aAAaE,KAAKiB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIvC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM2B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAClC,aAAaE,KAAKiB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAG1C,SAAS;IAC/B,MAAM2C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,0FAA0F;QAC1F,0FAA0F;QAC1F,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QACLF,IAAI,CAAC,OAAO,EAAEhD,OAAO;QACrB4B;QACA4B,UAAU9D;QACVU;QACAqD,gBAAgB,OAAOC,OAAiBC;gBACtCA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOH,MAAMI,GAAG,CAACpB;QACnB;QACAqB,YAAY,OAAOpB,MAAcgB;gBAC/BA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOnB,IAAIC;QACb;IACF;AACF"}

package/dist/esm/embed/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { type BufferedResponse, fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS, MODEL_FILE_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  const res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' }, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${res.url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\nfunction isStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.every((part) => typeof part === 'string');\n}\n\nfunction isHfModelInfo(value: unknown): value is HfModelInfo {\n  if (value === null || typeof value !== 'object') return false;\n  if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;\n  if (!('cardData' in value) || value.cardData === undefined) return true;\n  if (value.cardData === null || typeof value.cardData !== 'object') return false;\n  if (!('language' in value.cardData) || value.cardData.language === undefined) return true;\n  return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: BufferedResponse;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {}, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    const decoded: unknown = JSON.parse(new TextDecoder().decode(res.body));\n    if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');\n    info = decoded;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  const res = await fetchWithRetry(url, {}, { attemptTimeoutMs: MODEL_FILE_ATTEMPT_TIMEOUT_MS });\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(res.body));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","MODEL_FILE_ATTEMPT_TIMEOUT_MS","DEFAULT_MODELS_ROOT","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","root","existing","url","res","method","redirect","attemptTimeoutMs","sha","headers","get","isStringArray","value","Array","isArray","every","part","isHfModelInfo","tags","undefined","cardData","language","languagesFromCard","info","lang","length","tagLangs","filter","cardUnreadable","console","error","ensureLanguages","measured","ok","decoded","JSON","parse","TextDecoder","decode","body","Error","fetchToFile","dest","status","Buffer","from","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAAgCC,cAAc,EAAEC,uBAAuB,EAAEC,6BAA6B,QAAQ,YAAY;AAC1H,SAASC,mBAAmB,EAAEC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC/J,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,4FAA4F;AAC5F,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIvB,YAAYsB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa,EAAEG,OAAetB,mBAAmB;IACzE,MAAMuB,WAAWlB,QAAQc,OAAOG;IAChC,IAAIC,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEL,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,MAAMM,MAAM,MAAM5B,eAAe2B,KAAK;QAAEE,QAAQ;QAAQC,UAAU;IAAS,GAAG;QAAEC,kBAAkB9B;IAAwB;IAC1H,MAAM+B,MAAMJ,IAAIK,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAIlC,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,0CAA0C,CAAC;IACpGhB,SAASW,OAAOU,KAAKP;IACrB,OAAOO;AACT;AAOA,SAASG,cAAcC,KAAc;IACnC,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS;AACvE;AAEA,SAASC,cAAcL,KAAc;IACnC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,UAAU,OAAO;IACxD,IAAI,UAAUA,SAASA,MAAMM,IAAI,KAAKC,aAAa,CAACR,cAAcC,MAAMM,IAAI,GAAG,OAAO;IACtF,IAAI,CAAE,CAAA,cAAcN,KAAI,KAAMA,MAAMQ,QAAQ,KAAKD,WAAW,OAAO;IACnE,IAAIP,MAAMQ,QAAQ,KAAK,QAAQ,OAAOR,MAAMQ,QAAQ,KAAK,UAAU,OAAO;IAC1E,IAAI,CAAE,CAAA,cAAcR,MAAMQ,QAAQ,AAAD,KAAMR,MAAMQ,QAAQ,CAACC,QAAQ,KAAKF,WAAW,OAAO;IACrF,OAAO,OAAOP,MAAMQ,QAAQ,CAACC,QAAQ,KAAK,YAAYV,cAAcC,MAAMQ,QAAQ,CAACC,QAAQ;AAC7F;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASC,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKH,QAAQ,cAAbG,qCAAAA,eAAeF,QAAQ;IACpC,IAAI,OAAOG,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIX,MAAMC,OAAO,CAACU,SAASA,KAAKC,MAAM,GAAG,GAAG,OAAOD;IACnD,MAAME,WAAW,EAACH,aAAAA,KAAKL,IAAI,cAATK,wBAAAA,aAAa,EAAE,EAAEI,MAAM,CAACvC;IAC1C,OAAOsC,SAASD,MAAM,GAAG,IAAIC,WAAWP;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASS,eAAe9B,KAAa;IACnC+B,QAAQC,KAAK,CAAC,CAAC,yCAAyC,EAAEhC,MAAM,kGAAkG,CAAC;AACrK;AAEA,eAAeiC,gBAAgBjC,KAAa,EAAEG,OAAetB,mBAAmB;QA4BxD2C;IA3BtB,IAAIpD,WAAWY,cAAcgB,OAAOG,QAAQ;IAC5C,kFAAkF;IAClF,MAAM+B,WAAW3C,wBAAwB,CAACS,MAAM;IAChD,IAAIkC,UAAU;QACZ9C,eAAeY,OAAOkC,UAAU/B;QAChC;IACF;IACA,IAAIG;IACJ,IAAI;QACFA,MAAM,MAAM5B,eAAe,CAAC,kCAAkC,EAAEsB,OAAO,EAAE,CAAC,GAAG;YAAES,kBAAkB9B;QAAwB;IAC3H,EAAE,OAAM;QACNmD,eAAe9B;QACf;IACF;IACA,IAAI,CAACM,IAAI6B,EAAE,EAAE;QACXL,eAAe9B;QACf;IACF;IACA,IAAIyB;IACJ,IAAI;QACF,MAAMW,UAAmBC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAAClC,IAAImC,IAAI;QACrE,IAAI,CAACtB,cAAciB,UAAU,MAAM,IAAIM,MAAM;QAC7CjB,OAAOW;IACT,EAAE,OAAM;QACNN,eAAe9B;QACf;IACF;IACAZ,eAAeY,QAAOwB,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE,EAAErB;AACvD;AAEA,eAAewC,YAAYtC,GAAW,EAAEuC,IAAY;IAClD,MAAMtC,MAAM,MAAM5B,eAAe2B,KAAK,CAAC,GAAG;QAAEI,kBAAkB7B;IAA8B;IAC5F,IAAI,CAAC0B,IAAI6B,EAAE,EAAE,MAAM,IAAI3D,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,SAAS,EAAEC,IAAIuC,MAAM,EAAE;IACnFpE,gBAAgBmE,MAAME,OAAOC,IAAI,CAACzC,IAAImC,IAAI;AAC5C;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeO,sBAAsBhD,KAAa,EAAEU,GAAW,EAAEuC,MAA4C,EAAE9C,OAAetB,mBAAmB;IACtJ,MAAMqE,MAAM/D,YAAYa,OAAOU,KAAKP;IACpC9B,UAAU6E,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQnE,YAAa;QAC9B,IAAIb,WAAWE,KAAK4E,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMP,YAAY,CAAC,uBAAuB,EAAE3C,MAAM,SAAS,EAAEU,IAAI,CAAC,EAAE0C,MAAM,EAAE9E,KAAK4E,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAcrD,KAAa,EAAEiD,MAA4C,EAAE9C,OAAetB,mBAAmB;IACjI,IAAI,CAACE,eAAeiB,QAAQ;QAC1B,MAAM,IAAIxB,WAAW,eAAe,CAAC,aAAa,EAAEwB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMU,MAAM,MAAMR,WAAWF,OAAOG;IACpC,sFAAsF;IACtF,oFAAoF;IACpF,MAAM8B,gBAAgBjC,OAAOG;IAC7B,OAAO6C,sBAAsBhD,OAAOU,KAAKuC,QAAQ9C;AACnD"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { type BufferedResponse, fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS, MODEL_FILE_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  const res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' }, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${res.url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\nfunction isStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.every((part) => typeof part === 'string');\n}\n\nfunction isHfModelInfo(value: unknown): value is HfModelInfo {\n  if (value === null || typeof value !== 'object') return false;\n  if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;\n  if (!('cardData' in value) || value.cardData === undefined) return true;\n  if (value.cardData === null || typeof value.cardData !== 'object') return false;\n  if (!('language' in value.cardData) || value.cardData.language === undefined) return true;\n  return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: BufferedResponse;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {}, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    const decoded: unknown = JSON.parse(new TextDecoder().decode(res.body));\n    if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');\n    info = decoded;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  const res = await fetchWithRetry(url, {}, { attemptTimeoutMs: MODEL_FILE_ATTEMPT_TIMEOUT_MS });\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(res.body));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","MODEL_FILE_ATTEMPT_TIMEOUT_MS","DEFAULT_MODELS_ROOT","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","root","existing","url","res","method","redirect","attemptTimeoutMs","sha","headers","get","isStringArray","value","Array","isArray","every","part","isHfModelInfo","tags","undefined","cardData","language","languagesFromCard","info","lang","length","tagLangs","filter","cardUnreadable","console","error","ensureLanguages","measured","ok","decoded","JSON","parse","TextDecoder","decode","body","Error","fetchToFile","dest","status","Buffer","from","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAAgCC,cAAc,EAAEC,uBAAuB,EAAEC,6BAA6B,QAAQ,YAAY;AAC1H,SAASC,mBAAmB,EAAEC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC/J,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,4FAA4F;AAC5F,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIvB,YAAYsB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa,EAAEG,OAAetB,mBAAmB;IACzE,MAAMuB,WAAWlB,QAAQc,OAAOG;IAChC,IAAIC,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEL,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,MAAMM,MAAM,MAAM5B,eAAe2B,KAAK;QAAEE,QAAQ;QAAQC,UAAU;IAAS,GAAG;QAAEC,kBAAkB9B;IAAwB;IAC1H,MAAM+B,MAAMJ,IAAIK,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAIlC,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,0CAA0C,CAAC;IACpGhB,SAASW,OAAOU,KAAKP;IACrB,OAAOO;AACT;AAOA,SAASG,cAAcC,KAAc;IACnC,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS;AACvE;AAEA,SAASC,cAAcL,KAAc;IACnC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,UAAU,OAAO;IACxD,IAAI,UAAUA,SAASA,MAAMM,IAAI,KAAKC,aAAa,CAACR,cAAcC,MAAMM,IAAI,GAAG,OAAO;IACtF,IAAI,CAAE,CAAA,cAAcN,KAAI,KAAMA,MAAMQ,QAAQ,KAAKD,WAAW,OAAO;IACnE,IAAIP,MAAMQ,QAAQ,KAAK,QAAQ,OAAOR,MAAMQ,QAAQ,KAAK,UAAU,OAAO;IAC1E,IAAI,CAAE,CAAA,cAAcR,MAAMQ,QAAQ,AAAD,KAAMR,MAAMQ,QAAQ,CAACC,QAAQ,KAAKF,WAAW,OAAO;IACrF,OAAO,OAAOP,MAAMQ,QAAQ,CAACC,QAAQ,KAAK,YAAYV,cAAcC,MAAMQ,QAAQ,CAACC,QAAQ;AAC7F;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASC,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKH,QAAQ,cAAbG,qCAAAA,eAAeF,QAAQ;IACpC,IAAI,OAAOG,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIX,MAAMC,OAAO,CAACU,SAASA,KAAKC,MAAM,GAAG,GAAG,OAAOD;IACnD,MAAME,WAAW,EAACH,aAAAA,KAAKL,IAAI,cAATK,wBAAAA,aAAa,EAAE,EAAEI,MAAM,CAACvC;IAC1C,OAAOsC,SAASD,MAAM,GAAG,IAAIC,WAAWP;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASS,eAAe9B,KAAa;IACnC+B,QAAQC,KAAK,CAAC,CAAC,yCAAyC,EAAEhC,MAAM,kGAAkG,CAAC;AACrK;AAEA,eAAeiC,gBAAgBjC,KAAa,EAAEG,OAAetB,mBAAmB;QA4BxD2C;IA3BtB,IAAIpD,WAAWY,cAAcgB,OAAOG,QAAQ;IAC5C,kFAAkF;IAClF,MAAM+B,WAAW3C,wBAAwB,CAACS,MAAM;IAChD,IAAIkC,UAAU;QACZ9C,eAAeY,OAAOkC,UAAU/B;QAChC;IACF;IACA,IAAIG;IACJ,IAAI;QACFA,MAAM,MAAM5B,eAAe,CAAC,kCAAkC,EAAEsB,OAAO,EAAE,CAAC,GAAG;YAAES,kBAAkB9B;QAAwB;IAC3H,EAAE,OAAM;QACNmD,eAAe9B;QACf;IACF;IACA,IAAI,CAACM,IAAI6B,EAAE,EAAE;QACXL,eAAe9B;QACf;IACF;IACA,IAAIyB;IACJ,IAAI;QACF,MAAMW,UAAmBC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAAClC,IAAImC,IAAI;QACrE,IAAI,CAACtB,cAAciB,UAAU,MAAM,IAAIM,MAAM;QAC7CjB,OAAOW;IACT,EAAE,OAAM;QACNN,eAAe9B;QACf;IACF;IACAZ,eAAeY,QAAOwB,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE,EAAErB;AACvD;AAEA,eAAewC,YAAYtC,GAAW,EAAEuC,IAAY;IAClD,MAAMtC,MAAM,MAAM5B,eAAe2B,KAAK,CAAC,GAAG;QAAEI,kBAAkB7B;IAA8B;IAC5F,IAAI,CAAC0B,IAAI6B,EAAE,EAAE,MAAM,IAAI3D,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,SAAS,EAAEC,IAAIuC,MAAM,EAAE;IACnFpE,gBAAgBmE,MAAME,OAAOC,IAAI,CAACzC,IAAImC,IAAI;AAC5C;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeO,sBAAsBhD,KAAa,EAAEU,GAAW,EAAEuC,MAA4C,EAAE9C,OAAetB,mBAAmB;IACtJ,MAAMqE,MAAM/D,YAAYa,OAAOU,KAAKP;IACpC9B,UAAU6E,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQnE,YAAa;QAC9B,IAAIb,WAAWE,KAAK4E,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMP,YAAY,CAAC,uBAAuB,EAAE3C,MAAM,SAAS,EAAEU,IAAI,CAAC,EAAE0C,MAAM,EAAE9E,KAAK4E,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAcrD,KAAa,EAAEiD,MAA4C,EAAE9C,OAAetB,mBAAmB;IACjI,IAAI,CAACE,eAAeiB,QAAQ;QAC1B,MAAM,IAAIxB,WAAW,eAAe,CAAC,aAAa,EAAEwB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMU,MAAM,MAAMR,WAAWF,OAAOG;IACpC,sFAAsF;IACtF,oFAAoF;IACpF,MAAM8B,gBAAgBjC,OAAOG;IAC7B,OAAO6C,sBAAsBhD,OAAOU,KAAKuC,QAAQ9C;AACnD"}

package/dist/esm/embed/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\nexport interface EmbedCallOptions {\n  signal?: AbortSignal;\n}\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;\n  embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;AAClG,OAAO,MAAMA,aAAa,IAAI"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\nexport interface EmbedCallOptions {\n  signal?: AbortSignal;\n}\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;\n  embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;AAClG,OAAO,MAAMA,aAAa,IAAI"}

package/dist/esm/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_OPTION_INVALID'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED'\n  | 'STORE_UNKNOWN'\n  | 'STORE_DEPENDENCY_MISSING'\n  | 'STORE_CAPABILITY_MISSING'\n  | 'STORE_BUSY'\n  | 'INDEX_NOT_READY'\n  | 'LEXICAL_NULL_PATH';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AA0BrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_OPTION_INVALID'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED'\n  | 'STORE_UNKNOWN'\n  | 'STORE_DEPENDENCY_MISSING'\n  | 'STORE_CAPABILITY_MISSING'\n  | 'STORE_BUSY'\n  | 'INDEX_NOT_READY'\n  | 'LEXICAL_NULL_PATH';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AA0BrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}

package/dist/esm/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { embedEndpointIdentity } from '../embed/endpoint.ts';\nimport { identifyChunkText, stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and build (or an authorised scoped CLI query) prepares them before semantic retrieval.\n\nexport type { Chunk };\n\nconst CHUNK_LOOKUP_PAGE = 512;\ninterface ChunkRow {\n  chunk: number;\n  start_line: number;\n  end_line: number;\n  content_identity: string;\n}\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths, delta) {\n    const vanishedSet = new Set(delta.vanished);\n    const vanished = paths.filter((path) => vanishedSet.has(path));\n    if (vanished.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      vanished.map((path) => [path])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs, delta) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, ReturnType<typeof identifyChunkText>[]>();\n    const added = new Set(delta.added);\n    // Keyset pages bound comparison metadata without reading native vector payloads.\n    let firstPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n    let nextPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n    for (const { path, extracted } of docs) {\n      const chunks = (extracted as Chunk[] | undefined) ?? [];\n      if (chunks.length === 0) {\n        if (!added.has(path)) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ?', [[path]]);\n        continue;\n      }\n      const identified = chunks.map((chunk) => identifyChunkText(chunk.text));\n      const kept = new Set<number>();\n      if (!added.has(path)) {\n        let cursor: number | undefined;\n        for (;;) {\n          let page: ChunkRow[];\n          if (cursor === undefined) {\n            firstPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n            page = (await firstPage.all(path)) as ChunkRow[];\n          } else {\n            nextPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? AND chunk > ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n            page = (await nextPage.all(path, cursor)) as ChunkRow[];\n          }\n          const removed: unknown[][] = [];\n          const moved: unknown[][] = [];\n          for (const old of page) {\n            const index = Number(old.chunk);\n            const chunk = chunks[index];\n            if (!chunk || old.content_identity !== identified[index].identity) {\n              removed.push([path, index]);\n            } else {\n              kept.add(index);\n              if (Number(old.start_line) !== chunk.startLine || Number(old.end_line) !== chunk.endLine) moved.push([chunk.startLine, chunk.endLine, path, index]);\n            }\n          }\n          if (removed.length > 0) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ? AND chunk = ?', removed);\n          if (moved.length > 0) await db.runBatch('UPDATE embeddings SET start_line = ?, end_line = ? WHERE \"path\" = ? AND chunk = ?', moved);\n          if (page.length < CHUNK_LOOKUP_PAGE) break;\n          cursor = Number(page[page.length - 1].chunk);\n        }\n      }\n      const before = rows.length;\n      chunks.forEach((chunk, index) => {\n        if (!kept.has(index)) rows.push([path, index, chunk.startLine, chunk.endLine, identified[index].identity]);\n      });\n      // No new pending rows means no retained handoff. Matching pending rows can use\n      // embedPending's exact indexed-source fallback, including after repeated store calls.\n      if (rows.length > before) texts.set(path, identified);\n    }\n    // scale and vector are written by the embed pass, not here, so they are not in the column list\n    // and take the table's own default on the append path.\n    await appendRows(db, 'embeddings', ['path', 'chunk', 'start_line', 'end_line', 'content_identity'], 'INSERT INTO embeddings (\"path\", chunk, start_line, end_line, content_identity, scale, vector) VALUES (?, ?, ?, ?, ?, NULL, NULL)', rows);\n    // Hand off only paths with new pending rows; missing entries use the indexed source.\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    const endpoint = embedEndpointIdentity(e.provider, e.url);\n    const endpointPart = endpoint === undefined ? '' : `:endpoint=${endpoint}`;\n    return `embed:${e.provider}:${e.model}${endpointPart}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","embedEndpointIdentity","identifyChunkText","stashChunkText","modelIdentity","countLines","appendRows","CHUNK_LOOKUP_PAGE","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","remove","paths","delta","vanishedSet","Set","vanished","path","has","length","runBatch","store","docs","rows","texts","Map","added","firstPage","nextPage","extracted","chunks","identified","chunk","kept","cursor","page","prepare","all","removed","moved","old","index","Number","content_identity","identity","push","add","start_line","end_line","before","forEach","set","enabledForFile","_cfg","file","signature","e","provider","model","chunkVersion","endpoint","url","endpointPart"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,iBAAiB,EAAEC,cAAc,QAAQ,sBAAsB;AACxE,SAASC,aAAa,QAAQ,uBAAuB;AACrD,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAQhD,MAAMC,oBAAoB;AAQ1B,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOrB,gBAAgBW,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8LAA8L,CAAC;IAChN;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUV,MAAMW,OAAOA,MAAMC,QAAQN,WAAW4B,OAAO5B,WAAWK,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAClH;IACA,MAAMuB,QAAON,EAAE,EAAEO,KAAK,EAAEC,KAAK;QAC3B,MAAMC,cAAc,IAAIC,IAAIF,MAAMG,QAAQ;QAC1C,MAAMA,WAAWJ,MAAMpB,MAAM,CAAC,CAACyB,OAASH,YAAYI,GAAG,CAACD;QACxD,IAAID,SAASG,MAAM,KAAK,GAAG;QAC3B,MAAMd,GAAGe,QAAQ,CACf,2CACAJ,SAASnB,GAAG,CAAC,CAACoB,OAAS;gBAACA;aAAK;IAEjC;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMI,OAAMhB,EAAE,EAAEiB,IAAI,EAAET,KAAK;QACzB,MAAMU,OAAoB,EAAE;QAC5B,MAAMC,QAAQ,IAAIC;QAClB,MAAMC,QAAQ,IAAIX,IAAIF,MAAMa,KAAK;QACjC,iFAAiF;QACjF,IAAIC;QACJ,IAAIC;QACJ,KAAK,MAAM,EAAEX,IAAI,EAAEY,SAAS,EAAE,IAAIP,KAAM;YACtC,MAAMQ,SAAUD,sBAAAA,uBAAAA,YAAqC,EAAE;YACvD,IAAIC,OAAOX,MAAM,KAAK,GAAG;gBACvB,IAAI,CAACO,MAAMR,GAAG,CAACD,OAAO,MAAMZ,GAAGe,QAAQ,CAAC,2CAA2C;oBAAC;wBAACH;qBAAK;iBAAC;gBAC3F;YACF;YACA,MAAMc,aAAaD,OAAOjC,GAAG,CAAC,CAACmC,QAAUvD,kBAAkBuD,MAAM/B,IAAI;YACrE,MAAMgC,OAAO,IAAIlB;YACjB,IAAI,CAACW,MAAMR,GAAG,CAACD,OAAO;gBACpB,IAAIiB;gBACJ,OAAS;oBACP,IAAIC;oBACJ,IAAID,WAAWvC,WAAW;wBACxBgC,sBAAAA,uBAAAA,YAAAA,YAAc,MAAMtB,GAAG+B,OAAO,CAAC,CAAC,2GAA2G,EAAEtD,mBAAmB;wBAChKqD,OAAQ,MAAMR,UAAUU,GAAG,CAACpB;oBAC9B,OAAO;wBACLW,qBAAAA,sBAAAA,WAAAA,WAAa,MAAMvB,GAAG+B,OAAO,CAAC,CAAC,yHAAyH,EAAEtD,mBAAmB;wBAC7KqD,OAAQ,MAAMP,SAASS,GAAG,CAACpB,MAAMiB;oBACnC;oBACA,MAAMI,UAAuB,EAAE;oBAC/B,MAAMC,QAAqB,EAAE;oBAC7B,KAAK,MAAMC,OAAOL,KAAM;wBACtB,MAAMM,QAAQC,OAAOF,IAAIR,KAAK;wBAC9B,MAAMA,QAAQF,MAAM,CAACW,MAAM;wBAC3B,IAAI,CAACT,SAASQ,IAAIG,gBAAgB,KAAKZ,UAAU,CAACU,MAAM,CAACG,QAAQ,EAAE;4BACjEN,QAAQO,IAAI,CAAC;gCAAC5B;gCAAMwB;6BAAM;wBAC5B,OAAO;4BACLR,KAAKa,GAAG,CAACL;4BACT,IAAIC,OAAOF,IAAIO,UAAU,MAAMf,MAAMjC,SAAS,IAAI2C,OAAOF,IAAIQ,QAAQ,MAAMhB,MAAMhC,OAAO,EAAEuC,MAAMM,IAAI,CAAC;gCAACb,MAAMjC,SAAS;gCAAEiC,MAAMhC,OAAO;gCAAEiB;gCAAMwB;6BAAM;wBACpJ;oBACF;oBACA,IAAIH,QAAQnB,MAAM,GAAG,GAAG,MAAMd,GAAGe,QAAQ,CAAC,yDAAyDkB;oBACnG,IAAIC,MAAMpB,MAAM,GAAG,GAAG,MAAMd,GAAGe,QAAQ,CAAC,qFAAqFmB;oBAC7H,IAAIJ,KAAKhB,MAAM,GAAGrC,mBAAmB;oBACrCoD,SAASQ,OAAOP,IAAI,CAACA,KAAKhB,MAAM,GAAG,EAAE,CAACa,KAAK;gBAC7C;YACF;YACA,MAAMiB,SAAS1B,KAAKJ,MAAM;YAC1BW,OAAOoB,OAAO,CAAC,CAAClB,OAAOS;gBACrB,IAAI,CAACR,KAAKf,GAAG,CAACuB,QAAQlB,KAAKsB,IAAI,CAAC;oBAAC5B;oBAAMwB;oBAAOT,MAAMjC,SAAS;oBAAEiC,MAAMhC,OAAO;oBAAE+B,UAAU,CAACU,MAAM,CAACG,QAAQ;iBAAC;YAC3G;YACA,+EAA+E;YAC/E,sFAAsF;YACtF,IAAIrB,KAAKJ,MAAM,GAAG8B,QAAQzB,MAAM2B,GAAG,CAAClC,MAAMc;QAC5C;QACA,+FAA+F;QAC/F,uDAAuD;QACvD,MAAMlD,WAAWwB,IAAI,cAAc;YAAC;YAAQ;YAAS;YAAc;YAAY;SAAmB,EAAE,oIAAoIkB;QACxO,qFAAqF;QACrF7C,eAAe2B,IAAImB;IACrB;IACA4B,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKpD,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9DqD,WAAU7C,GAAG;QACX,MAAM8C,IAAIjF,YAAYmC;QACtB,IAAI,CAAC8C,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMZ,WAAWY,EAAEC,QAAQ,KAAK,WAAW9E,cAAc6E,EAAEE,KAAK,IAAI/D;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMgE,eAAeH,EAAEpE,WAAW,KAAKO,YAAY,GAAGvB,cAAc,CAAC,EAAEoF,EAAEpE,WAAW,EAAE,GAAGhB;QACzF,MAAMwF,WAAWpF,sBAAsBgF,EAAEC,QAAQ,EAAED,EAAEK,GAAG;QACxD,MAAMC,eAAeF,aAAajE,YAAY,KAAK,CAAC,UAAU,EAAEiE,UAAU;QAC1E,OAAO,CAAC,MAAM,EAAEJ,EAAEC,QAAQ,CAAC,CAAC,EAAED,EAAEE,KAAK,GAAGI,aAAa,CAAC,EAAEH,eAAef,aAAajD,YAAY,CAAC,CAAC,EAAEiD,UAAU,GAAG,IAAI;IACvH;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { embedEndpointIdentity } from '../embed/endpoint.ts';\nimport { identifyChunkText, stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and build (or an authorised scoped CLI query) prepares them before semantic retrieval.\n\nexport type { Chunk };\n\nconst CHUNK_LOOKUP_PAGE = 512;\ninterface ChunkRow {\n  chunk: number;\n  start_line: number;\n  end_line: number;\n  content_identity: string;\n}\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths, delta) {\n    const vanishedSet = new Set(delta.vanished);\n    const vanished = paths.filter((path) => vanishedSet.has(path));\n    if (vanished.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      vanished.map((path) => [path])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs, delta) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, ReturnType<typeof identifyChunkText>[]>();\n    const added = new Set(delta.added);\n    // Keyset pages bound comparison metadata without reading native vector payloads.\n    let firstPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n    let nextPage: Awaited<ReturnType<typeof db.prepare>> | undefined;\n    for (const { path, extracted } of docs) {\n      const chunks = (extracted as Chunk[] | undefined) ?? [];\n      if (chunks.length === 0) {\n        if (!added.has(path)) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ?', [[path]]);\n        continue;\n      }\n      const identified = chunks.map((chunk) => identifyChunkText(chunk.text));\n      const kept = new Set<number>();\n      if (!added.has(path)) {\n        let cursor: number | undefined;\n        for (;;) {\n          let page: ChunkRow[];\n          if (cursor === undefined) {\n            firstPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n            page = (await firstPage.all(path)) as ChunkRow[];\n          } else {\n            nextPage ??= await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE \"path\" = ? AND chunk > ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);\n            page = (await nextPage.all(path, cursor)) as ChunkRow[];\n          }\n          const removed: unknown[][] = [];\n          const moved: unknown[][] = [];\n          for (const old of page) {\n            const index = Number(old.chunk);\n            const chunk = chunks[index];\n            if (!chunk || old.content_identity !== identified[index].identity) {\n              removed.push([path, index]);\n            } else {\n              kept.add(index);\n              if (Number(old.start_line) !== chunk.startLine || Number(old.end_line) !== chunk.endLine) moved.push([chunk.startLine, chunk.endLine, path, index]);\n            }\n          }\n          if (removed.length > 0) await db.runBatch('DELETE FROM embeddings WHERE \"path\" = ? AND chunk = ?', removed);\n          if (moved.length > 0) await db.runBatch('UPDATE embeddings SET start_line = ?, end_line = ? WHERE \"path\" = ? AND chunk = ?', moved);\n          if (page.length < CHUNK_LOOKUP_PAGE) break;\n          cursor = Number(page[page.length - 1].chunk);\n        }\n      }\n      const before = rows.length;\n      chunks.forEach((chunk, index) => {\n        if (!kept.has(index)) rows.push([path, index, chunk.startLine, chunk.endLine, identified[index].identity]);\n      });\n      // No new pending rows means no retained handoff. Matching pending rows can use\n      // embedPending's exact indexed-source fallback, including after repeated store calls.\n      if (rows.length > before) texts.set(path, identified);\n    }\n    // scale and vector are written by the embed pass, not here, so they are not in the column list\n    // and take the table's own default on the append path.\n    await appendRows(db, 'embeddings', ['path', 'chunk', 'start_line', 'end_line', 'content_identity'], 'INSERT INTO embeddings (\"path\", chunk, start_line, end_line, content_identity, scale, vector) VALUES (?, ?, ?, ?, ?, NULL, NULL)', rows);\n    // Hand off only paths with new pending rows; missing entries use the indexed source.\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    const endpoint = embedEndpointIdentity(e.provider, e.url);\n    const endpointPart = endpoint === undefined ? '' : `:endpoint=${endpoint}`;\n    return `embed:${e.provider}:${e.model}${endpointPart}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","embedEndpointIdentity","identifyChunkText","stashChunkText","modelIdentity","countLines","appendRows","CHUNK_LOOKUP_PAGE","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","remove","paths","delta","vanishedSet","Set","vanished","path","has","length","runBatch","store","docs","rows","texts","Map","added","firstPage","nextPage","extracted","chunks","identified","chunk","kept","cursor","page","prepare","all","removed","moved","old","index","Number","content_identity","identity","push","add","start_line","end_line","before","forEach","set","enabledForFile","_cfg","file","signature","e","provider","model","chunkVersion","endpoint","url","endpointPart"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,iBAAiB,EAAEC,cAAc,QAAQ,sBAAsB;AACxE,SAASC,aAAa,QAAQ,uBAAuB;AACrD,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAQhD,MAAMC,oBAAoB;AAQ1B,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOrB,gBAAgBW,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8LAA8L,CAAC;IAChN;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUV,MAAMW,OAAOA,MAAMC,QAAQN,WAAW4B,OAAO5B,WAAWK,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAClH;IACA,MAAMuB,QAAON,EAAE,EAAEO,KAAK,EAAEC,KAAK;QAC3B,MAAMC,cAAc,IAAIC,IAAIF,MAAMG,QAAQ;QAC1C,MAAMA,WAAWJ,MAAMpB,MAAM,CAAC,CAACyB,OAASH,YAAYI,GAAG,CAACD;QACxD,IAAID,SAASG,MAAM,KAAK,GAAG;QAC3B,MAAMd,GAAGe,QAAQ,CACf,2CACAJ,SAASnB,GAAG,CAAC,CAACoB,OAAS;gBAACA;aAAK;IAEjC;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMI,OAAMhB,EAAE,EAAEiB,IAAI,EAAET,KAAK;QACzB,MAAMU,OAAoB,EAAE;QAC5B,MAAMC,QAAQ,IAAIC;QAClB,MAAMC,QAAQ,IAAIX,IAAIF,MAAMa,KAAK;QACjC,iFAAiF;QACjF,IAAIC;QACJ,IAAIC;QACJ,KAAK,MAAM,EAAEX,IAAI,EAAEY,SAAS,EAAE,IAAIP,KAAM;YACtC,MAAMQ,SAAUD,sBAAAA,uBAAAA,YAAqC,EAAE;YACvD,IAAIC,OAAOX,MAAM,KAAK,GAAG;gBACvB,IAAI,CAACO,MAAMR,GAAG,CAACD,OAAO,MAAMZ,GAAGe,QAAQ,CAAC,2CAA2C;oBAAC;wBAACH;qBAAK;iBAAC;gBAC3F;YACF;YACA,MAAMc,aAAaD,OAAOjC,GAAG,CAAC,CAACmC,QAAUvD,kBAAkBuD,MAAM/B,IAAI;YACrE,MAAMgC,OAAO,IAAIlB;YACjB,IAAI,CAACW,MAAMR,GAAG,CAACD,OAAO;gBACpB,IAAIiB;gBACJ,OAAS;oBACP,IAAIC;oBACJ,IAAID,WAAWvC,WAAW;wBACxBgC,sBAAAA,uBAAAA,YAAAA,YAAc,MAAMtB,GAAG+B,OAAO,CAAC,CAAC,2GAA2G,EAAEtD,mBAAmB;wBAChKqD,OAAQ,MAAMR,UAAUU,GAAG,CAACpB;oBAC9B,OAAO;wBACLW,qBAAAA,sBAAAA,WAAAA,WAAa,MAAMvB,GAAG+B,OAAO,CAAC,CAAC,yHAAyH,EAAEtD,mBAAmB;wBAC7KqD,OAAQ,MAAMP,SAASS,GAAG,CAACpB,MAAMiB;oBACnC;oBACA,MAAMI,UAAuB,EAAE;oBAC/B,MAAMC,QAAqB,EAAE;oBAC7B,KAAK,MAAMC,OAAOL,KAAM;wBACtB,MAAMM,QAAQC,OAAOF,IAAIR,KAAK;wBAC9B,MAAMA,QAAQF,MAAM,CAACW,MAAM;wBAC3B,IAAI,CAACT,SAASQ,IAAIG,gBAAgB,KAAKZ,UAAU,CAACU,MAAM,CAACG,QAAQ,EAAE;4BACjEN,QAAQO,IAAI,CAAC;gCAAC5B;gCAAMwB;6BAAM;wBAC5B,OAAO;4BACLR,KAAKa,GAAG,CAACL;4BACT,IAAIC,OAAOF,IAAIO,UAAU,MAAMf,MAAMjC,SAAS,IAAI2C,OAAOF,IAAIQ,QAAQ,MAAMhB,MAAMhC,OAAO,EAAEuC,MAAMM,IAAI,CAAC;gCAACb,MAAMjC,SAAS;gCAAEiC,MAAMhC,OAAO;gCAAEiB;gCAAMwB;6BAAM;wBACpJ;oBACF;oBACA,IAAIH,QAAQnB,MAAM,GAAG,GAAG,MAAMd,GAAGe,QAAQ,CAAC,yDAAyDkB;oBACnG,IAAIC,MAAMpB,MAAM,GAAG,GAAG,MAAMd,GAAGe,QAAQ,CAAC,qFAAqFmB;oBAC7H,IAAIJ,KAAKhB,MAAM,GAAGrC,mBAAmB;oBACrCoD,SAASQ,OAAOP,IAAI,CAACA,KAAKhB,MAAM,GAAG,EAAE,CAACa,KAAK;gBAC7C;YACF;YACA,MAAMiB,SAAS1B,KAAKJ,MAAM;YAC1BW,OAAOoB,OAAO,CAAC,CAAClB,OAAOS;gBACrB,IAAI,CAACR,KAAKf,GAAG,CAACuB,QAAQlB,KAAKsB,IAAI,CAAC;oBAAC5B;oBAAMwB;oBAAOT,MAAMjC,SAAS;oBAAEiC,MAAMhC,OAAO;oBAAE+B,UAAU,CAACU,MAAM,CAACG,QAAQ;iBAAC;YAC3G;YACA,+EAA+E;YAC/E,sFAAsF;YACtF,IAAIrB,KAAKJ,MAAM,GAAG8B,QAAQzB,MAAM2B,GAAG,CAAClC,MAAMc;QAC5C;QACA,+FAA+F;QAC/F,uDAAuD;QACvD,MAAMlD,WAAWwB,IAAI,cAAc;YAAC;YAAQ;YAAS;YAAc;YAAY;SAAmB,EAAE,oIAAoIkB;QACxO,qFAAqF;QACrF7C,eAAe2B,IAAImB;IACrB;IACA4B,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKpD,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9DqD,WAAU7C,GAAG;QACX,MAAM8C,IAAIjF,YAAYmC;QACtB,IAAI,CAAC8C,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMZ,WAAWY,EAAEC,QAAQ,KAAK,WAAW9E,cAAc6E,EAAEE,KAAK,IAAI/D;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMgE,eAAeH,EAAEpE,WAAW,KAAKO,YAAY,GAAGvB,cAAc,CAAC,EAAEoF,EAAEpE,WAAW,EAAE,GAAGhB;QACzF,MAAMwF,WAAWpF,sBAAsBgF,EAAEC,QAAQ,EAAED,EAAEK,GAAG;QACxD,MAAMC,eAAeF,aAAajE,YAAY,KAAK,CAAC,UAAU,EAAEiE,UAAU;QAC1E,OAAO,CAAC,MAAM,EAAEJ,EAAEC,QAAQ,CAAC,CAAC,EAAED,EAAEE,KAAK,GAAGI,aAAa,CAAC,EAAEH,eAAef,aAAajD,YAAY,CAAC,CAAC,EAAEiD,UAAU,GAAG,IAAI;IACvH;AACF,EAAE"}

package/dist/esm/features/fences.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column 0, no indent\n// allowance (diverges from CommonMark's up-to-3-space rule). Closer: same char, length >= opener's, trailing spaces only.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n\n// Masks <!-- ... --> spans that may cross line boundaries, one line at a time so callers can interleave their own per-line state.\n// An unclosed comment stays open forever unless closed externally; maskRegions() below calls close() at a paragraph blank line or the enclosing HTML block's end.\nexport interface CommentTracker {\n  mask(line: string): string;\n  close(): void; // force-ends an unclosed comment (paragraph blank line, or its HTML block ending)\n  readonly inComment: boolean;\n}\n\nexport function commentTracker(): CommentTracker {\n  let inComment = false;\n  return {\n    mask(line: string): string {\n      let out = '';\n      let i = 0;\n      while (i < line.length) {\n        if (inComment) {\n          const end = line.indexOf('-->', i);\n          if (end === -1) {\n            out += ' '.repeat(line.length - i);\n            i = line.length;\n          } else {\n            out += ' '.repeat(end + 3 - i);\n            i = end + 3;\n            inComment = false;\n          }\n          continue;\n        }\n        const start = line.indexOf('<!--', i);\n        if (start === -1) {\n          out += line.slice(i);\n          i = line.length;\n        } else {\n          out += `${line.slice(i, start)}    `;\n          i = start + 4;\n          inComment = true;\n        }\n      }\n      return out;\n    },\n    close() {\n      inComment = false;\n    },\n    get inComment() {\n      return inComment;\n    },\n  };\n}\n\n// CommonMark's HTML-block type-6 tag list (fixed by the spec). A line opening or closing one of these at\n// column 0 (leading whitespace/`>` tolerated) swallows following lines -- tags, links, comments alike -- until a blank line.\nconst HTML_BLOCK_TAGS = new Set([\n  'address',\n  'article',\n  'aside',\n  'base',\n  'basefont',\n  'blockquote',\n  'body',\n  'caption',\n  'center',\n  'col',\n  'colgroup',\n  'dd',\n  'details',\n  'dialog',\n  'dir',\n  'div',\n  'dl',\n  'dt',\n  'fieldset',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'frame',\n  'frameset',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'head',\n  'header',\n  'hr',\n  'html',\n  'iframe',\n  'legend',\n  'li',\n  'link',\n  'main',\n  'menu',\n  'menuitem',\n  'nav',\n  'noframes',\n  'ol',\n  'optgroup',\n  'option',\n  'p',\n  'param',\n  'search',\n  'section',\n  'summary',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'title',\n  'tr',\n  'track',\n  'ul',\n]);\n// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,\n// not on a blank line, and that line is the last one swallowed.\nconst HTML_PRE_TAGS = new Set(['script', 'pre', 'style', 'textarea']);\nconst HTML_PRE_CLOSE_RE = /<\\/(?:script|pre|style|textarea)>/i; // any of the four closers ends the block, not only the tag that opened it\n// An opening or closing tag at column 0, tag name captured for the lookups above.\nconst HTML_BLOCK_OPEN_RE = /^<\\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \\t]|\\/?>|$)/;\nconst BLANK_LINE_RE = /^[ \\t>]*$/; // whitespace/blockquote-markers only -- a paragraph break, and what ends a type-6 block\n\n// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines preserved, so line numbers and offsets survive), inline code spans masked too.\n// An unclosed comment dies at its block's end or the next paragraph blank line, never silently to end of file; bodies without a backtick/tilde/`<` fast-path out untouched.\nexport function maskRegions(body: string): string {\n  if (!/[`~<]/.test(body)) return body;\n  const fence = fenceTracker();\n  const comment = commentTracker();\n  let inHtmlBlock = false;\n  let htmlBlockClose: RegExp | null = null; // set while inside a type-1 (script/pre/style/textarea) block\n  return body\n    .split('\\n')\n    .map((line) => {\n      if (inHtmlBlock) {\n        if (htmlBlockClose) {\n          if (htmlBlockClose.test(line)) {\n            inHtmlBlock = false;\n            htmlBlockClose = null;\n          }\n          return ' '.repeat(line.length);\n        }\n        if (BLANK_LINE_RE.test(line)) {\n          // Block content never feeds the comment tracker, so an inner <!-- was only ever\n          // masked text; nothing to close here.\n          inHtmlBlock = false;\n          return line;\n        }\n        return ' '.repeat(line.length);\n      }\n      if (!comment.inComment) {\n        const isDelim = fence.feed(line);\n        if (isDelim || fence.inFence) return ' '.repeat(line.length);\n      }\n      const uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;\n      if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line\n      // Indented or blockquoted HTML blocks swallow their content in Obsidian too, so the\n      // opener test runs after stripping leading whitespace and > markers.\n      const stripped = uncommented.replace(/^[ \\t>]*/, '');\n      if (stripped[0] === '<') {\n        const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);\n        if (openMatch) {\n          const tagName = openMatch[1].toLowerCase();\n          const isClosingTag = stripped[1] === '/';\n          if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {\n            // The end condition can be met on the opening line itself: <script>x</script>\n            // is a one-line block and the next line parses (Obsidian-verified).\n            if (!HTML_PRE_CLOSE_RE.test(stripped.slice(openMatch[0].length))) {\n              inHtmlBlock = true;\n              htmlBlockClose = HTML_PRE_CLOSE_RE;\n            }\n            return ' '.repeat(line.length);\n          }\n          if (HTML_BLOCK_TAGS.has(tagName)) {\n            inHtmlBlock = true;\n            return ' '.repeat(line.length);\n          }\n        }\n      }\n      // Inline code spans hide their content too: `[[x]]` in prose is not a link.\n      return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;\n    })\n    .join('\\n');\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N (CommonMark code spans);\n// a shorter/longer run in between is literal text. Masked with spaces so column positions stay unaffected.\nexport function maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n"],"names":["BACKTICK_OPEN","TILDE_OPEN","fenceTracker","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim","commentTracker","inComment","mask","out","i","end","indexOf","repeat","start","slice","close","HTML_BLOCK_TAGS","Set","HTML_PRE_TAGS","HTML_PRE_CLOSE_RE","HTML_BLOCK_OPEN_RE","BLANK_LINE_RE","maskRegions","body","test","fence","comment","inHtmlBlock","htmlBlockClose","split","map","isDelim","uncommented","stripped","replace","openMatch","tagName","toLowerCase","isClosingTag","has","maskCodeSpans","join","j","n","k","closeStart","closeEnd"],"mappings":"AAAA,4GAA4G;AAC5G,0HAA0H;AAO1H,MAAMA,gBAAgB;AACtB,MAAMC,aAAa;AAEnB,OAAO,SAASC;IACd,IAAIC,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,MAAMK,KAAKR,cAAcS,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,MAAMC,KAAKX,WAAWQ,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,MAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,MAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAUA,OAAO,SAASa;IACd,IAAIC,YAAY;IAChB,OAAO;QACLC,MAAKX,IAAY;YACf,IAAIY,MAAM;YACV,IAAIC,IAAI;YACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;gBACtB,IAAIM,WAAW;oBACb,MAAMI,MAAMd,KAAKe,OAAO,CAAC,OAAOF;oBAChC,IAAIC,QAAQ,CAAC,GAAG;wBACdF,OAAO,IAAII,MAAM,CAAChB,KAAKI,MAAM,GAAGS;wBAChCA,IAAIb,KAAKI,MAAM;oBACjB,OAAO;wBACLQ,OAAO,IAAII,MAAM,CAACF,MAAM,IAAID;wBAC5BA,IAAIC,MAAM;wBACVJ,YAAY;oBACd;oBACA;gBACF;gBACA,MAAMO,QAAQjB,KAAKe,OAAO,CAAC,QAAQF;gBACnC,IAAII,UAAU,CAAC,GAAG;oBAChBL,OAAOZ,KAAKkB,KAAK,CAACL;oBAClBA,IAAIb,KAAKI,MAAM;gBACjB,OAAO;oBACLQ,OAAO,GAAGZ,KAAKkB,KAAK,CAACL,GAAGI,OAAO,IAAI,CAAC;oBACpCJ,IAAII,QAAQ;oBACZP,YAAY;gBACd;YACF;YACA,OAAOE;QACT;QACAO;YACET,YAAY;QACd;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAEA,yGAAyG;AACzG,6HAA6H;AAC7H,MAAMU,kBAAkB,IAAIC,IAAI;IAC9B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AACD,8FAA8F;AAC9F,gEAAgE;AAChE,MAAMC,gBAAgB,IAAID,IAAI;IAAC;IAAU;IAAO;IAAS;CAAW;AACpE,MAAME,oBAAoB,sCAAsC,0EAA0E;AAC1I,kFAAkF;AAClF,MAAMC,qBAAqB;AAC3B,MAAMC,gBAAgB,aAAa,wFAAwF;AAE3H,8KAA8K;AAC9K,4KAA4K;AAC5K,OAAO,SAASC,YAAYC,IAAY;IACtC,IAAI,CAAC,QAAQC,IAAI,CAACD,OAAO,OAAOA;IAChC,MAAME,QAAQlC;IACd,MAAMmC,UAAUrB;IAChB,IAAIsB,cAAc;IAClB,IAAIC,iBAAgC,MAAM,8DAA8D;IACxG,OAAOL,KACJM,KAAK,CAAC,MACNC,GAAG,CAAC,CAAClC;QACJ,IAAI+B,aAAa;YACf,IAAIC,gBAAgB;gBAClB,IAAIA,eAAeJ,IAAI,CAAC5B,OAAO;oBAC7B+B,cAAc;oBACdC,iBAAiB;gBACnB;gBACA,OAAO,IAAIhB,MAAM,CAAChB,KAAKI,MAAM;YAC/B;YACA,IAAIqB,cAAcG,IAAI,CAAC5B,OAAO;gBAC5B,gFAAgF;gBAChF,sCAAsC;gBACtC+B,cAAc;gBACd,OAAO/B;YACT;YACA,OAAO,IAAIgB,MAAM,CAAChB,KAAKI,MAAM;QAC/B;QACA,IAAI,CAAC0B,QAAQpB,SAAS,EAAE;YACtB,MAAMyB,UAAUN,MAAM9B,IAAI,CAACC;YAC3B,IAAImC,WAAWN,MAAMjC,OAAO,EAAE,OAAO,IAAIoB,MAAM,CAAChB,KAAKI,MAAM;QAC7D;QACA,MAAMgC,cAAcN,QAAQpB,SAAS,IAAIV,KAAKG,QAAQ,CAAC,UAAU2B,QAAQnB,IAAI,CAACX,QAAQA;QACtF,IAAI8B,QAAQpB,SAAS,IAAIe,cAAcG,IAAI,CAAC5B,OAAO8B,QAAQX,KAAK,IAAI,+CAA+C;QACnH,oFAAoF;QACpF,qEAAqE;QACrE,MAAMkB,WAAWD,YAAYE,OAAO,CAAC,YAAY;QACjD,IAAID,QAAQ,CAAC,EAAE,KAAK,KAAK;YACvB,MAAME,YAAYf,mBAAmBtB,IAAI,CAACmC;YAC1C,IAAIE,WAAW;gBACb,MAAMC,UAAUD,SAAS,CAAC,EAAE,CAACE,WAAW;gBACxC,MAAMC,eAAeL,QAAQ,CAAC,EAAE,KAAK;gBACrC,IAAI,CAACK,gBAAgBpB,cAAcqB,GAAG,CAACH,UAAU;oBAC/C,8EAA8E;oBAC9E,oEAAoE;oBACpE,IAAI,CAACjB,kBAAkBK,IAAI,CAACS,SAASnB,KAAK,CAACqB,SAAS,CAAC,EAAE,CAACnC,MAAM,IAAI;wBAChE2B,cAAc;wBACdC,iBAAiBT;oBACnB;oBACA,OAAO,IAAIP,MAAM,CAAChB,KAAKI,MAAM;gBAC/B;gBACA,IAAIgB,gBAAgBuB,GAAG,CAACH,UAAU;oBAChCT,cAAc;oBACd,OAAO,IAAIf,MAAM,CAAChB,KAAKI,MAAM;gBAC/B;YACF;QACF;QACA,4EAA4E;QAC5E,OAAOgC,YAAYjC,QAAQ,CAAC,OAAOyC,cAAcR,eAAeA;IAClE,GACCS,IAAI,CAAC;AACV;AAEA,6GAA6G;AAC7G,2GAA2G;AAC3G,OAAO,SAASD,cAAc5C,IAAY;IACxC,IAAIY,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;QACtB,IAAIJ,IAAI,CAACa,EAAE,KAAK,KAAK;YACnBD,OAAOZ,IAAI,CAACa,EAAE;YACdA;YACA;QACF;QACA,IAAIiC,IAAIjC;QACR,MAAOb,IAAI,CAAC8C,EAAE,KAAK,IAAKA;QACxB,MAAMC,IAAID,IAAIjC;QACd,IAAImC,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAIhD,KAAKI,MAAM,CAAE;YACtB,IAAIJ,IAAI,CAACgD,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAIzC,IAAIyC;YACR,MAAOhD,IAAI,CAACO,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIyC,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAW3C;gBACX;YACF;YACAyC,IAAIzC;QACN;QACA,IAAI0C,cAAc,GAAG;YACnBrC,OAAO,IAAII,MAAM,CAACkC,WAAWrC;YAC7BA,IAAIqC;QACN,OAAO;YACLtC,OAAOZ,KAAKkB,KAAK,CAACL,GAAGiC;YACrBjC,IAAIiC;QACN;IACF;IACA,OAAOlC;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/features/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column 0, no indent\n// allowance (diverges from CommonMark's up-to-3-space rule). Closer: same char, length >= opener's, trailing spaces only.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n\n// Masks <!-- ... --> spans that may cross line boundaries, one line at a time so callers can interleave their own per-line state.\n// An unclosed comment stays open forever unless closed externally; maskRegions() below calls close() at a paragraph blank line or the enclosing HTML block's end.\nexport interface CommentTracker {\n  mask(line: string): string;\n  close(): void; // force-ends an unclosed comment (paragraph blank line, or its HTML block ending)\n  readonly inComment: boolean;\n}\n\nexport function commentTracker(): CommentTracker {\n  let inComment = false;\n  return {\n    mask(line: string): string {\n      let out = '';\n      let i = 0;\n      while (i < line.length) {\n        if (inComment) {\n          const end = line.indexOf('-->', i);\n          if (end === -1) {\n            out += ' '.repeat(line.length - i);\n            i = line.length;\n          } else {\n            out += ' '.repeat(end + 3 - i);\n            i = end + 3;\n            inComment = false;\n          }\n          continue;\n        }\n        const start = line.indexOf('<!--', i);\n        if (start === -1) {\n          out += line.slice(i);\n          i = line.length;\n        } else {\n          out += `${line.slice(i, start)}    `;\n          i = start + 4;\n          inComment = true;\n        }\n      }\n      return out;\n    },\n    close() {\n      inComment = false;\n    },\n    get inComment() {\n      return inComment;\n    },\n  };\n}\n\n// CommonMark's HTML-block type-6 tag list (fixed by the spec). A line opening or closing one of these at\n// column 0 (leading whitespace/`>` tolerated) swallows following lines -- tags, links, comments alike -- until a blank line.\nconst HTML_BLOCK_TAGS = new Set([\n  'address',\n  'article',\n  'aside',\n  'base',\n  'basefont',\n  'blockquote',\n  'body',\n  'caption',\n  'center',\n  'col',\n  'colgroup',\n  'dd',\n  'details',\n  'dialog',\n  'dir',\n  'div',\n  'dl',\n  'dt',\n  'fieldset',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'frame',\n  'frameset',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'head',\n  'header',\n  'hr',\n  'html',\n  'iframe',\n  'legend',\n  'li',\n  'link',\n  'main',\n  'menu',\n  'menuitem',\n  'nav',\n  'noframes',\n  'ol',\n  'optgroup',\n  'option',\n  'p',\n  'param',\n  'search',\n  'section',\n  'summary',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'title',\n  'tr',\n  'track',\n  'ul',\n]);\n// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,\n// not on a blank line, and that line is the last one swallowed.\nconst HTML_PRE_TAGS = new Set(['script', 'pre', 'style', 'textarea']);\nconst HTML_PRE_CLOSE_RE = /<\\/(?:script|pre|style|textarea)>/i; // any of the four closers ends the block, not only the tag that opened it\n// An opening or closing tag at column 0, tag name captured for the lookups above.\nconst HTML_BLOCK_OPEN_RE = /^<\\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \\t]|\\/?>|$)/;\nconst BLANK_LINE_RE = /^[ \\t>]*$/; // whitespace/blockquote-markers only -- a paragraph break, and what ends a type-6 block\n\n// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines preserved, so line numbers and offsets survive), inline code spans masked too.\n// An unclosed comment dies at its block's end or the next paragraph blank line, never silently to end of file; bodies without a backtick/tilde/`<` fast-path out untouched.\nexport function maskRegions(body: string): string {\n  if (!/[`~<]/.test(body)) return body;\n  const fence = fenceTracker();\n  const comment = commentTracker();\n  let inHtmlBlock = false;\n  let htmlBlockClose: RegExp | null = null; // set while inside a type-1 (script/pre/style/textarea) block\n  return body\n    .split('\\n')\n    .map((line) => {\n      if (inHtmlBlock) {\n        if (htmlBlockClose) {\n          if (htmlBlockClose.test(line)) {\n            inHtmlBlock = false;\n            htmlBlockClose = null;\n          }\n          return ' '.repeat(line.length);\n        }\n        if (BLANK_LINE_RE.test(line)) {\n          // Block content never feeds the comment tracker, so an inner <!-- was only ever\n          // masked text; nothing to close here.\n          inHtmlBlock = false;\n          return line;\n        }\n        return ' '.repeat(line.length);\n      }\n      if (!comment.inComment) {\n        const isDelim = fence.feed(line);\n        if (isDelim || fence.inFence) return ' '.repeat(line.length);\n      }\n      const uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;\n      if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line\n      // Indented or blockquoted HTML blocks swallow their content in Obsidian too, so the\n      // opener test runs after stripping leading whitespace and > markers.\n      const stripped = uncommented.replace(/^[ \\t>]*/, '');\n      if (stripped[0] === '<') {\n        const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);\n        if (openMatch) {\n          const tagName = openMatch[1].toLowerCase();\n          const isClosingTag = stripped[1] === '/';\n          if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {\n            // The end condition can be met on the opening line itself: <script>x</script>\n            // is a one-line block and the next line parses (Obsidian-verified).\n            if (!HTML_PRE_CLOSE_RE.test(stripped.slice(openMatch[0].length))) {\n              inHtmlBlock = true;\n              htmlBlockClose = HTML_PRE_CLOSE_RE;\n            }\n            return ' '.repeat(line.length);\n          }\n          if (HTML_BLOCK_TAGS.has(tagName)) {\n            inHtmlBlock = true;\n            return ' '.repeat(line.length);\n          }\n        }\n      }\n      // Inline code spans hide their content too: `[[x]]` in prose is not a link.\n      return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;\n    })\n    .join('\\n');\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N (CommonMark code spans);\n// a shorter/longer run in between is literal text. Masked with spaces so column positions stay unaffected.\nexport function maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n"],"names":["BACKTICK_OPEN","TILDE_OPEN","fenceTracker","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim","commentTracker","inComment","mask","out","i","end","indexOf","repeat","start","slice","close","HTML_BLOCK_TAGS","Set","HTML_PRE_TAGS","HTML_PRE_CLOSE_RE","HTML_BLOCK_OPEN_RE","BLANK_LINE_RE","maskRegions","body","test","fence","comment","inHtmlBlock","htmlBlockClose","split","map","isDelim","uncommented","stripped","replace","openMatch","tagName","toLowerCase","isClosingTag","has","maskCodeSpans","join","j","n","k","closeStart","closeEnd"],"mappings":"AAAA,4GAA4G;AAC5G,0HAA0H;AAO1H,MAAMA,gBAAgB;AACtB,MAAMC,aAAa;AAEnB,OAAO,SAASC;IACd,IAAIC,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,MAAMK,KAAKR,cAAcS,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,MAAMC,KAAKX,WAAWQ,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,MAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,MAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAUA,OAAO,SAASa;IACd,IAAIC,YAAY;IAChB,OAAO;QACLC,MAAKX,IAAY;YACf,IAAIY,MAAM;YACV,IAAIC,IAAI;YACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;gBACtB,IAAIM,WAAW;oBACb,MAAMI,MAAMd,KAAKe,OAAO,CAAC,OAAOF;oBAChC,IAAIC,QAAQ,CAAC,GAAG;wBACdF,OAAO,IAAII,MAAM,CAAChB,KAAKI,MAAM,GAAGS;wBAChCA,IAAIb,KAAKI,MAAM;oBACjB,OAAO;wBACLQ,OAAO,IAAII,MAAM,CAACF,MAAM,IAAID;wBAC5BA,IAAIC,MAAM;wBACVJ,YAAY;oBACd;oBACA;gBACF;gBACA,MAAMO,QAAQjB,KAAKe,OAAO,CAAC,QAAQF;gBACnC,IAAII,UAAU,CAAC,GAAG;oBAChBL,OAAOZ,KAAKkB,KAAK,CAACL;oBAClBA,IAAIb,KAAKI,MAAM;gBACjB,OAAO;oBACLQ,OAAO,GAAGZ,KAAKkB,KAAK,CAACL,GAAGI,OAAO,IAAI,CAAC;oBACpCJ,IAAII,QAAQ;oBACZP,YAAY;gBACd;YACF;YACA,OAAOE;QACT;QACAO;YACET,YAAY;QACd;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAEA,yGAAyG;AACzG,6HAA6H;AAC7H,MAAMU,kBAAkB,IAAIC,IAAI;IAC9B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AACD,8FAA8F;AAC9F,gEAAgE;AAChE,MAAMC,gBAAgB,IAAID,IAAI;IAAC;IAAU;IAAO;IAAS;CAAW;AACpE,MAAME,oBAAoB,sCAAsC,0EAA0E;AAC1I,kFAAkF;AAClF,MAAMC,qBAAqB;AAC3B,MAAMC,gBAAgB,aAAa,wFAAwF;AAE3H,8KAA8K;AAC9K,4KAA4K;AAC5K,OAAO,SAASC,YAAYC,IAAY;IACtC,IAAI,CAAC,QAAQC,IAAI,CAACD,OAAO,OAAOA;IAChC,MAAME,QAAQlC;IACd,MAAMmC,UAAUrB;IAChB,IAAIsB,cAAc;IAClB,IAAIC,iBAAgC,MAAM,8DAA8D;IACxG,OAAOL,KACJM,KAAK,CAAC,MACNC,GAAG,CAAC,CAAClC;QACJ,IAAI+B,aAAa;YACf,IAAIC,gBAAgB;gBAClB,IAAIA,eAAeJ,IAAI,CAAC5B,OAAO;oBAC7B+B,cAAc;oBACdC,iBAAiB;gBACnB;gBACA,OAAO,IAAIhB,MAAM,CAAChB,KAAKI,MAAM;YAC/B;YACA,IAAIqB,cAAcG,IAAI,CAAC5B,OAAO;gBAC5B,gFAAgF;gBAChF,sCAAsC;gBACtC+B,cAAc;gBACd,OAAO/B;YACT;YACA,OAAO,IAAIgB,MAAM,CAAChB,KAAKI,MAAM;QAC/B;QACA,IAAI,CAAC0B,QAAQpB,SAAS,EAAE;YACtB,MAAMyB,UAAUN,MAAM9B,IAAI,CAACC;YAC3B,IAAImC,WAAWN,MAAMjC,OAAO,EAAE,OAAO,IAAIoB,MAAM,CAAChB,KAAKI,MAAM;QAC7D;QACA,MAAMgC,cAAcN,QAAQpB,SAAS,IAAIV,KAAKG,QAAQ,CAAC,UAAU2B,QAAQnB,IAAI,CAACX,QAAQA;QACtF,IAAI8B,QAAQpB,SAAS,IAAIe,cAAcG,IAAI,CAAC5B,OAAO8B,QAAQX,KAAK,IAAI,+CAA+C;QACnH,oFAAoF;QACpF,qEAAqE;QACrE,MAAMkB,WAAWD,YAAYE,OAAO,CAAC,YAAY;QACjD,IAAID,QAAQ,CAAC,EAAE,KAAK,KAAK;YACvB,MAAME,YAAYf,mBAAmBtB,IAAI,CAACmC;YAC1C,IAAIE,WAAW;gBACb,MAAMC,UAAUD,SAAS,CAAC,EAAE,CAACE,WAAW;gBACxC,MAAMC,eAAeL,QAAQ,CAAC,EAAE,KAAK;gBACrC,IAAI,CAACK,gBAAgBpB,cAAcqB,GAAG,CAACH,UAAU;oBAC/C,8EAA8E;oBAC9E,oEAAoE;oBACpE,IAAI,CAACjB,kBAAkBK,IAAI,CAACS,SAASnB,KAAK,CAACqB,SAAS,CAAC,EAAE,CAACnC,MAAM,IAAI;wBAChE2B,cAAc;wBACdC,iBAAiBT;oBACnB;oBACA,OAAO,IAAIP,MAAM,CAAChB,KAAKI,MAAM;gBAC/B;gBACA,IAAIgB,gBAAgBuB,GAAG,CAACH,UAAU;oBAChCT,cAAc;oBACd,OAAO,IAAIf,MAAM,CAAChB,KAAKI,MAAM;gBAC/B;YACF;QACF;QACA,4EAA4E;QAC5E,OAAOgC,YAAYjC,QAAQ,CAAC,OAAOyC,cAAcR,eAAeA;IAClE,GACCS,IAAI,CAAC;AACV;AAEA,6GAA6G;AAC7G,2GAA2G;AAC3G,OAAO,SAASD,cAAc5C,IAAY;IACxC,IAAIY,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;QACtB,IAAIJ,IAAI,CAACa,EAAE,KAAK,KAAK;YACnBD,OAAOZ,IAAI,CAACa,EAAE;YACdA;YACA;QACF;QACA,IAAIiC,IAAIjC;QACR,MAAOb,IAAI,CAAC8C,EAAE,KAAK,IAAKA;QACxB,MAAMC,IAAID,IAAIjC;QACd,IAAImC,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAIhD,KAAKI,MAAM,CAAE;YACtB,IAAIJ,IAAI,CAACgD,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAIzC,IAAIyC;YACR,MAAOhD,IAAI,CAACO,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIyC,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAW3C;gBACX;YACF;YACAyC,IAAIzC;QACN;QACA,IAAI0C,cAAc,GAAG;YACnBrC,OAAO,IAAII,MAAM,CAACkC,WAAWrC;YAC7BA,IAAIqC;QACN,OAAO;YACLtC,OAAOZ,KAAKkB,KAAK,CAACL,GAAGiC;YACrBjC,IAAIiC;QACN;IACF;IACA,OAAOlC;AACT"}

package/dist/esm/features/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/index.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { featureEnabled } from '../config/index.ts';\nimport { embed } from './embed.ts';\nimport { links } from './links.ts';\nimport { rank } from './rank.ts';\nimport { sections } from './sections.ts';\nimport { tags } from './tags.ts';\nimport type { Feature } from './types.ts';\n\n// Registry order matters: rank reads the links table in afterReconcile.\nexport const FEATURES: Feature[] = [links, sections, tags, rank, embed];\n\nexport function activeFeatures(cfg: Config): Feature[] {\n  return FEATURES.filter((feature) => featureEnabled(cfg, feature.name));\n}\n\nexport { LINK_EDGES_SQL, linkEdges, toEdges } from './links.ts';\nexport type { Section } from './sections.ts';\nexport type { Feature } from './types.ts';\n"],"names":["featureEnabled","embed","links","rank","sections","tags","FEATURES","activeFeatures","cfg","filter","feature","name","LINK_EDGES_SQL","linkEdges","toEdges"],"mappings":"AACA,SAASA,cAAc,QAAQ,qBAAqB;AACpD,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,QAAQ,QAAQ,gBAAgB;AACzC,SAASC,IAAI,QAAQ,YAAY;AAGjC,wEAAwE;AACxE,OAAO,MAAMC,WAAsB;IAACJ;IAAOE;IAAUC;IAAMF;IAAMF;CAAM,CAAC;AAExE,OAAO,SAASM,eAAeC,GAAW;IACxC,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAYV,eAAeQ,KAAKE,QAAQC,IAAI;AACtE;AAEA,SAASC,cAAc,EAAEC,SAAS,EAAEC,OAAO,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/features/index.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { featureEnabled } from '../config/index.ts';\nimport { embed } from './embed.ts';\nimport { links } from './links.ts';\nimport { rank } from './rank.ts';\nimport { sections } from './sections.ts';\nimport { tags } from './tags.ts';\nimport type { Feature } from './types.ts';\n\n// Registry order matters: rank reads the links table in afterReconcile.\nexport const FEATURES: Feature[] = [links, sections, tags, rank, embed];\n\nexport function activeFeatures(cfg: Config): Feature[] {\n  return FEATURES.filter((feature) => featureEnabled(cfg, feature.name));\n}\n\nexport { LINK_EDGES_SQL, linkEdges, toEdges } from './links.ts';\nexport type { Section } from './sections.ts';\nexport type { Feature } from './types.ts';\n"],"names":["featureEnabled","embed","links","rank","sections","tags","FEATURES","activeFeatures","cfg","filter","feature","name","LINK_EDGES_SQL","linkEdges","toEdges"],"mappings":"AACA,SAASA,cAAc,QAAQ,qBAAqB;AACpD,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,QAAQ,QAAQ,gBAAgB;AACzC,SAASC,IAAI,QAAQ,YAAY;AAGjC,wEAAwE;AACxE,OAAO,MAAMC,WAAsB;IAACJ;IAAOE;IAAUC;IAAMF;IAAMF;CAAM,CAAC;AAExE,OAAO,SAASM,eAAeC,GAAW;IACxC,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAYV,eAAeQ,KAAKE,QAAQC,IAAI;AACtE;AAEA,SAASC,cAAc,EAAEC,SAAS,EAAEC,OAAO,QAAQ,aAAa"}