sensemaking 0.7.2 → 0.8.0

package/dist/cjs/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic expansion needs features.embed in sense.config.json (e.g. \"features\": { \"embed\": true })');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0 }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n  }\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\n// Returns the cosine similarity alongside each candidate: stored and query vectors are both\n// L2-normalised before quantisation, so the dot product is cosine in [-1, 1]. `find` surfaces\n// it because the fused rank score cannot express match quality -- nearest-neighbour search\n// always returns a nearest neighbour, so without a magnitude an agent cannot tell a real hit\n// from the best of a bad lot.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: Math.round(b.score * 1000) / 1000 }));\n}\n"],"names":["embed","embedPending","semanticCandidates","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","join","chunks","forEach","start","end","text","slice","trim","startLine","endLine","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","fetchToFile","url","dest","fetch","then","res","ok","SenseError","status","Buffer","from","arrayBuffer","writeFileSync","renameSync","staticProvider","model","tokenizerJson","dir","file","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","ids","encode","add_special_tokens","id","Number","isInteger","v","Float32Array","off","d","norm","Math","sqrt","existsSync","homedir","replace","mkdirSync","recursive","console","error","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","vocab","unk_token","texts","map","apiProvider","keyEnv","base","headers","key","post","body","method","stringify","input","json","data","embedding","process","env","undefined","authorization","providers","Map","getProvider","cfg","e","embedConfig","sig","type","p","get","set","toStore","full","int8","scale","max","abs","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","chunkIdxs","update","batch","vectors","j","job","q","Int8Array","round","chunk","err","all","min","parseFile","relPath","absPath","mtimeMs","size","doc","terms","qv","rows","best","dot","score","existing","match","t","includes","vector","byteLength","start_line","end_line","sort","a","b","similarity"],"mappings":";;;;;;;;;;;QA6DaA;eAAAA;;QAmISC;eAAAA;;QAoDAC;eAAAA;;;sBApPyD;sBACvD;wBACH;wBAGO;wBACD;sBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG1B,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,IAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,IAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,IAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,IAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;KAAa,CAAd,OAAI,qBAAGA,WAAUA;IAC5E,IAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,IAAMC,SAAkB,EAAE;IAC1BP,OAAOQ,OAAO,CAAC,SAACC,OAAOb;QACrB,IAAMc,MAAMd,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,IAAMc,OAAOnB,MACVoB,KAAK,CAACH,QAAQ,GAAGC,KACjBJ,IAAI,CAAC,MACLO,IAAI;QACP,IAAIF,KAAKd,MAAM,GAAG,GAAGU,OAAOR,IAAI,CAAC;YAAEe,WAAWL;YAAOM,SAASL;YAAKC,MAAMV,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAS,OAALU,QAASA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEO,IAAMvB,QAAiB;IAC5BgC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;IACV;IACAC,SAAAA,SAAAA,QAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAAA,SAAAA,OAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACAG,OAAAA,SAAAA,MAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBnB,OAAO,CAAC,SAACqB,GAAGC;mBAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEf,SAAS,EAAEe,EAAEd,OAAO;;IACzF;AACF;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASgB,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,SAAOC;;;;;;wBAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,0BAAwCF,OAAfJ,KAAI,aAAsB,OAAXI,IAAIG,MAAM;;4BACrF,GAAO,OAALN,MAAK;;4BAAQO,OAAOC,IAAI;wBAAC;;4BAAML,IAAIM,WAAW;;;wBAA/DC,qBAAa;4BAAiBH,QAAAA;gCAAY;;;wBAC1CI,IAAAA,kBAAU,EAAC,AAAC,GAAO,OAALX,MAAK,UAAQA;;;;;;QAC7B;;AACF;AAEA,SAAeY,eAAeC,KAAa;;kBA2BEC,sBAA7BA,4BAAAA,uBA1BVC,SAIiB,OAARC,MAQP3D,KACA4D,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAT,eAEEU,WACFC,KACAC;QAEN,SAASC,IAAIjD,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMkD,MAAM,AAACH,IAAII,MAAM,CAACnD,MAAM;gBAAEoD,oBAAoB;YAAM,GAAGF,GAAG,CAAczD,MAAM,CAAC,SAAC4D;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOL;;YAC5H,IAAMQ,IAAI,IAAIC,aAAad;YAC3B,IAAIO,IAAIhE,MAAM,KAAK,GAAG,OAAOsE;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYN,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMG,KAAN;oBACH,IAAMK,MAAML,KAAKV;oBACjB,IAAK,IAAIgB,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKH,CAAC,CAACG,EAAE,IAAId,MAAM,CAACa,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK;gBAC7BH,CAAC,CAACG,GAAE,IAAIT,IAAIhE,MAAM;gBAClB0E,QAAQJ,CAAC,CAACG,GAAE,GAAGH,CAAC,CAACG,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,IAAIC;YACvC,OAAOJ;QACT;;;;oBA/CInB,MAAMF;yBACN,CAAC4B,IAAAA,kBAAU,EAACpE,IAAAA,cAAI,EAAC0C,KAAK,uBAAtB;;;;oBACFA,MAAM1C,IAAAA,cAAI,EAACqE,IAAAA,eAAO,KAAI,UAAU,eAAe,UAAU7B,MAAM8B,OAAO,CAAC,OAAO;oBAC9EC,IAAAA,iBAAS,EAAC7B,KAAK;wBAAE8B,WAAW;oBAAK;4BACd;wBAAC;wBAAqB;;;;+BAAtB;;;;oBAAR7B,OAAQ;yBACb,CAACyB,IAAAA,kBAAU,EAACpE,IAAAA,cAAI,EAAC0C,KAAKC,QAAtB;;;;oBACF8B,QAAQC,KAAK,CAAC,AAAC,YAAoB/B,OAATH,OAAM,KAAgBE,OAAbC,MAAK,UAAY,OAAJD,KAAI;oBACpD;;wBAAMjB,YAAY,AAAC,0BAA+CkB,OAAtBH,OAAM,kBAAqB,OAALG,OAAQ3C,IAAAA,cAAI,EAAC0C,KAAKC;;;oBAApF;;;;;;;;;oBAKA3D,MAAM2F,IAAAA,oBAAY,EAAC3E,IAAAA,cAAI,EAAC0C,KAAK;oBAC7BE,YAAYe,OAAO3E,IAAI4F,eAAe,CAAC;oBACvC/B,SAASgC,KAAKC,KAAK,CAAC9F,IAAI+F,QAAQ,CAAC,GAAG,IAAInC,WAAWoC,QAAQ,CAAC;oBAC5DlC,QAAQmC,OAAOC,OAAO,CAACrC,QAAQsC,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAACtC,SAASA,KAAK,CAAC,EAAE,CAACuC,KAAK,KAAK,OAAO,MAAM,IAAIrD,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAANQ,OAAM;oBAC/EO,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKuC,KAAK,CAAC,EAAE;oBACpBrC,YAAYjE,IAAIuG,UAAU,GAAG,IAAI3C,YAAYG,KAAKyC,YAAY,CAAC,EAAE;oBACjEtC,SAASD,YAAY,MAAM,IAAI,IAAIa,aAAa9E,IAAIyG,MAAM,EAAExC,WAAWF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,QAAQ,IAAIc,aAAa9E,IAAIyG,MAAM,CAACnF,KAAK,CAAC2C,WAAWA,YAAYF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,OAAO;oBAE/KP,gBAAgBoC,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAAC3E,IAAAA,cAAI,EAAC0C,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BS,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUV,eAAe,CAAC;oBACpCY,iBAAQZ,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBiD,KAAK,cAA1BjD,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBkD,SAAS,CAAC,uCAAI,CAAC;oBAuB/E;;wBAAO;4BAAEjC,IAAI,AAAC,UAAe,OAANlB;4BAASQ,MAAAA;4BAAMtE,OAAO,SAAPA,MAAckH;;;;;4CAAUA,MAAMC,GAAG,CAACvC;;;;;wBAAK;;;;IAC/E;;AAEA,gFAAgF;AAEhF,SAAewC,YAAYtD,KAAa,EAAEd,GAAuB,EAAEqE,MAA0B;;YAErFC,MACAC,SACAC,KAUAlD;QAPN,SAAemD,KAAKP,KAAe;;oBAC3B9D,KAEAsE;;;;4BAFM;;gCAAMxE,MAAM,AAAC,GAAO,OAALoE,MAAK,gBAAc;oCAAEK,QAAQ;oCAAQJ,SAAAA;oCAASG,MAAMvB,KAAKyB,SAAS,CAAC;wCAAE9D,OAAAA;wCAAO+D,OAAOX;oCAAM;gCAAG;;;4BAAjH9D,MAAM;4BACZ,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BF,OAA3BkE,MAAK,wBAAiC,OAAXlE,IAAIG,MAAM;4BAC3E;;gCAAMH,IAAI0E,IAAI;;;4BAAtBJ,OAAQ;4BACd;;gCAAOA,KAAKK,IAAI,CAACZ,GAAG,CAAC,SAAC7B;2CAAMF,aAAa3B,IAAI,CAAC6B,EAAE0C,SAAS;;;;;YAC3D;;;;;oBAXA,IAAI,CAAChF,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe;oBACxCgE,OAAOtE,IAAI4C,OAAO,CAAC,QAAQ;oBAC3B2B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAASY,QAAQC,GAAG,CAACb,OAAO,GAAGc;oBAC3C,IAAIX,KAAKD,QAAQa,aAAa,GAAG,AAAC,UAAa,OAAJZ;oBAS7B;;wBAAMC;4BAAM;;;;oBAApBnD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACzD,MAAM;oBACxD;;wBAAO;4BAAEmE,IAAI,AAAC,OAAclB,OAARwD,MAAK,KAAS,OAANxD;4BAASQ,MAAAA;4BAAMtE,OAAOyH;wBAAK;;;;IACzD;;AAEA,IAAMY,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,IAAMA,IAAIC,IAAAA,qBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAInF,oBAAU,CAAC,kBAAkB;IAC/C,IAAMqF,MAAM,AAxKd,AAwKe,GAAYF,OAAVA,EAAEG,IAAI,EAAC,YAAGH,EAAE3E,KAAK,EAAC,KAAe,QAAZ2E,SAAAA,EAAEzF,GAAG,cAALyF,oBAAAA,SAAS;IAC7C,IAAII,IAAIR,UAAUS,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIJ,EAAEG,IAAI,KAAK,QAAQxB,YAAYqB,EAAE3E,KAAK,EAAE2E,EAAEzF,GAAG,EAAEyF,EAAEjB,GAAG,IAAI3D,eAAe4E,EAAE3E,KAAK;QAClFuE,UAAUU,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAE3E,IAAY,EAAE4E,IAAa;IAC9D,IAAM/D,IAAI,IAAIC,aAAad;IAC3B,IAAIiB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKC,QAAQ0D,IAAI,CAAC3D,EAAE,GAAG2D,IAAI,CAAC3D,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,GAAG2D,IAAI,CAAC3D,GAAE,GAAGC;IAChD,IAAI,CAAC2D,MAAM,OAAO;QAAE/D,GAAAA;QAAGgE,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAI9D,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK8D,MAAM5D,KAAK4D,GAAG,CAACA,KAAK5D,KAAK6D,GAAG,CAAClE,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGgE,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAenJ,aAAaiC,EAAgB,EAAEsG,GAAW,EAAEc,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOtH,MAAMuH,WACZvI,QAMC,4BAAA,oBAAA,iBAAA,YAAA,QAAMuB,KAGPiH,QACGnJ;;;;;4BACDoJ,OACAC;;;;oCADAD,QAAQH,KAAKjI,KAAK,CAAChB,GAAGA,IAAIR;oCAChB;;wCAAMmJ,SAASvJ,KAAK,CAACgK,MAAM7C,GAAG,CAAC,SAAC+C;mDAAMA,EAAEvI,IAAI;;;;oCAAtDsI,UAAU;oCAChB/H,GAAGC,IAAI,CAAC;oCACR,IAAI;wCACF6H,MAAMxI,OAAO,CAAC,SAAC2I,KAAKD;4CAClB,IAAqBlB,WAAAA,QAAQiB,OAAO,CAACC,EAAE,EAAET,WAAW,OAA5CtE,IAAa6D,SAAb7D,GAAGgE,QAAUH,SAAVG;4CACX,IAAMiB,IAAI,IAAIC,UAAUZ;4CACxB,IAAK,IAAInE,IAAI,GAAGA,IAAImE,WAAWnE,IAAK8E,CAAC,CAAC9E,EAAE,GAAGE,KAAK8E,KAAK,CAACnF,CAAC,CAACG,EAAE,GAAG6D;4CAC7DY,OAAOtH,GAAG,CAAC0G,OAAO3F,OAAOC,IAAI,CAAC2G,EAAErD,MAAM,GAAGoD,IAAI5H,IAAI,EAAE4H,IAAII,KAAK;wCAC9D;wCACArI,GAAGC,IAAI,CAAC;oCACV,EAAE,OAAOqI,KAAK;wCACZtI,GAAGC,IAAI,CAAC;wCACR,MAAMqI;oCACR;;;;;;oBACF;oBAxCiB;;wBAAMjC,YAAYC;;;oBAA7Be,WAAW;oBACXC,QAAQtH,GAAGM,OAAO,CAAC,oFAAoFiI,GAAG;oBAChH,IAAIjB,MAAM3I,MAAM,KAAK,GAAG;;;oBAClB4I,YAAYjE,KAAKkF,GAAG,CAACvK,YAAYoJ,SAASjF,IAAI;oBAE9CoF,SAAS,IAAIpB;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAakB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOZ,GAAG,CAACa,IAAIpH,IAAI,eAAnBmH,yBAAAA;4BACbE,KAAK7I,IAAI,CAAC4I,IAAIY,KAAK;4BACnBb,OAAOX,GAAG,CAACY,IAAIpH,IAAI,EAAEqH;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOnH,uBAAMuH;4BACZvI,SAAAA,KAAAA;4BACJ,IAAI;gCACFA,SAASoJ,IAAAA,iBAAS,EAAC;oCAAEC,SAASrI;oCAAMsI,SAASvJ,IAAAA,cAAI,EAACgI,SAAS/G;oCAAOuI,SAAS;oCAAGC,MAAM;gCAAE;oCAAI/K;mCAAQgL,GAAG,CAACrI,SAAS,CAAC3C,KAAK;4BACvH,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa8J,gCAAb,8BAAA,SAAA,0BAAA;oCAAMhH,MAAN;oCAAwB,IAAIvB,MAAM,CAACuB,IAAI,EAAE+G,KAAK9I,IAAI,CAAC;wCAAEwB,MAAAA;wCAAMgI,OAAOzH;wCAAKnB,MAAMJ,MAAM,CAACuB,IAAI,CAACnB,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBARK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAUCoI,SAAS7H,GAAGM,OAAO,CAAC;oBACjB5B,IAAI;;;yBAAGA,CAAAA,IAAIiJ,KAAKhJ,MAAM,AAAD;;;;;;;;;;;;oBAAGD,KAAKR;;;;;;;;;;;IAiBxC;;AAUO,SAAeF,mBAAmBgC,EAAgB,EAAEsG,GAAW,EAAEyC,KAAa,EAAE/H,MAAa;;YAOpF+H,cANR3B,SAIAC,UACAE,WACA9H,MACYqH,UAAPkC,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMzB,KACHS,GACFiB,KACK/F,GACHgG,OACAC;;;;oBAvBFjC,UAAU,AAACd,IAAgCc,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIhG,oBAAU,CAAC,eAAe;oBAClD;;wBAAMrD,aAAaiC,IAAIsG,KAAKc;;;oBAA5B;oBAEiB;;wBAAMf,YAAYC;;;oBAA7Be,WAAW;oBACXE,YAAYjE,KAAKkF,GAAG,CAACvK,YAAYoJ,SAASjF,IAAI;oBAC9C3C,OAAO,EAACsJ,eAAAA,MAAMO,KAAK,CAAC,2naAAZP,0BAAAA,mBAAsC7J,MAAM,CAAC,SAACqK;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAInK,IAAI,CAAC;oBACvF;;wBAAMiI,SAASvJ,KAAK;4BAAE2B;;;;oBAA/BqH,WAAAA;wBAAS,aAA6B,CAAC,EAAE;wBAAES;wBAAW;wBAA7DyB,KAAOlC,SAAV7D;oBAEFgG,OAAOjJ,GAAGM,OAAO,CAAC,+FAA+FiI,GAAG;oBAQpHW,OAAO,IAAI9C;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAa6C,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAbxB,MAAN;4BACGS,IAAI,IAAIC,UAAUV,IAAIgC,MAAM,CAAC5E,MAAM,EAAE4C,IAAIgC,MAAM,CAAC9E,UAAU,EAAErB,KAAKkF,GAAG,CAACjB,WAAWE,IAAIgC,MAAM,CAACC,UAAU;4BACvGP,MAAM;4BACV,IAAS/F,IAAI,GAAGA,IAAI8E,EAAEvJ,MAAM,EAAEyE,IAAK+F,OAAOjB,CAAC,CAAC9E,EAAE,GAAG4F,EAAE,CAAC5F,EAAE;4BAChDgG,QAAQD,MAAM1B,IAAIR,KAAK;4BACvBoC,WAAWH,KAAKtC,GAAG,CAACa,IAAIpH,IAAI;4BAClC,IAAI,CAACgJ,YAAYD,QAAQC,SAASD,KAAK,EAAEF,KAAKrC,GAAG,CAACY,IAAIpH,IAAI,EAAE;gCAAE+I,OAAAA;gCAAO9K,OAAO,AAAC,IAAqBmJ,OAAlBA,IAAIkC,UAAU,EAAC,KAAgB,OAAblC,IAAImC,QAAQ;4BAAG;wBACnH;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAQL;;wBAAQ,qBAAGV,KAAK5E,OAAO,IACpBuF,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACX,KAAK,GAAGU,CAAC,CAAC,EAAE,CAACV,KAAK;2BACtC1J,KAAK,CAAC,GAAGsB,QACTiE,GAAG,CAAC;qEAAE5E,kBAAM0J;mCAAQ;gCAAE1J,MAAAA;gCAAM/B,OAAOyL,EAAEzL,KAAK;gCAAE0L,YAAY1G,KAAK8E,KAAK,CAAC2B,EAAEX,KAAK,GAAG,QAAQ;4BAAK;;;;;IAC/F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic expansion needs features.embed in sense.config.json (e.g. \"features\": { \"embed\": true })');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0 }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first --semantic query hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + BATCH, jobs.length));\n  }\n  report.finish();\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\n// Returns the cosine similarity alongside each candidate: stored and query vectors are both\n// L2-normalised before quantisation, so the dot product is cosine in [-1, 1]. `find` surfaces\n// it because the fused rank score cannot express match quality -- nearest-neighbour search\n// always returns a nearest neighbour, so without a magnitude an agent cannot tell a real hit\n// from the best of a bad lot.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: Math.round(b.score * 1000) / 1000 }));\n}\n"],"names":["embed","embedPending","semanticCandidates","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","join","chunks","forEach","start","end","text","slice","trim","startLine","endLine","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","fetchToFile","url","dest","fetch","then","res","ok","SenseError","status","Buffer","from","arrayBuffer","writeFileSync","renameSync","staticProvider","model","tokenizerJson","dir","file","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","ids","encode","add_special_tokens","id","Number","isInteger","v","Float32Array","off","d","norm","Math","sqrt","existsSync","homedir","replace","mkdirSync","recursive","console","error","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","vocab","unk_token","texts","map","apiProvider","keyEnv","base","headers","key","post","body","method","stringify","input","json","data","embedding","process","env","undefined","authorization","providers","Map","getProvider","cfg","e","embedConfig","sig","type","p","get","set","toStore","full","int8","scale","max","abs","baseDir","provider","dirty","storeDims","byPath","row","list","jobs","chunkIdxs","update","report","batch","vectors","j","job","q","Int8Array","round","chunk","err","tick","min","all","parseFile","relPath","absPath","mtimeMs","size","doc","progress","finish","terms","qv","rows","best","dot","score","existing","match","t","includes","vector","byteLength","start_line","end_line","sort","a","b","similarity"],"mappings":";;;;;;;;;;;QA8DaA;eAAAA;;QAmISC;eAAAA;;QAyDAC;eAAAA;;;sBA1PyD;sBACvD;wBACH;wBAGO;wBACD;0BACF;sBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG1B,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,IAAMC,aAAa;AACnB,IAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,IAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,IAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,IAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;KAAa,CAAd,OAAI,qBAAGA,WAAUA;IAC5E,IAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,IAAMC,SAAkB,EAAE;IAC1BP,OAAOQ,OAAO,CAAC,SAACC,OAAOb;QACrB,IAAMc,MAAMd,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,IAAMc,OAAOnB,MACVoB,KAAK,CAACH,QAAQ,GAAGC,KACjBJ,IAAI,CAAC,MACLO,IAAI;QACP,IAAIF,KAAKd,MAAM,GAAG,GAAGU,OAAOR,IAAI,CAAC;YAAEe,WAAWL;YAAOM,SAASL;YAAKC,MAAMV,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAS,OAALU,QAASA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEO,IAAMvB,QAAiB;IAC5BgC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC;IACV;IACAC,SAAAA,SAAAA,QAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAAA,SAAAA,OAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACAG,OAAAA,SAAAA,MAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBnB,OAAO,CAAC,SAACqB,GAAGC;mBAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEf,SAAS,EAAEe,EAAEd,OAAO;;IACzF;AACF;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASgB,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,SAAOC;;;;;;wBAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,0BAAwCF,OAAfJ,KAAI,aAAsB,OAAXI,IAAIG,MAAM;;4BACrF,GAAO,OAALN,MAAK;;4BAAQO,OAAOC,IAAI;wBAAC;;4BAAML,IAAIM,WAAW;;;wBAA/DC,qBAAa;4BAAiBH,QAAAA;gCAAY;;;wBAC1CI,IAAAA,kBAAU,EAAC,AAAC,GAAO,OAALX,MAAK,UAAQA;;;;;;QAC7B;;AACF;AAEA,SAAeY,eAAeC,KAAa;;kBA2BEC,sBAA7BA,4BAAAA,uBA1BVC,SAIiB,OAARC,MAQP3D,KACA4D,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAT,eAEEU,WACFC,KACAC;QAEN,SAASC,IAAIjD,IAAY;YACvB,mFAAmF;YACnF,uFAAuF;YACvF,yFAAyF;YACzF,IAAMkD,MAAM,AAACH,IAAII,MAAM,CAACnD,MAAM;gBAAEoD,oBAAoB;YAAM,GAAGF,GAAG,CAAczD,MAAM,CAAC,SAAC4D;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOL;;YAC5H,IAAMQ,IAAI,IAAIC,aAAad;YAC3B,IAAIO,IAAIhE,MAAM,KAAK,GAAG,OAAOsE;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYN,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMG,KAAN;oBACH,IAAMK,MAAML,KAAKV;oBACjB,IAAK,IAAIgB,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKH,CAAC,CAACG,EAAE,IAAId,MAAM,CAACa,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK;gBAC7BH,CAAC,CAACG,GAAE,IAAIT,IAAIhE,MAAM;gBAClB0E,QAAQJ,CAAC,CAACG,GAAE,GAAGH,CAAC,CAACG,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,IAAIC;YACvC,OAAOJ;QACT;;;;oBA/CInB,MAAMF;yBACN,CAAC4B,IAAAA,kBAAU,EAACpE,IAAAA,cAAI,EAAC0C,KAAK,uBAAtB;;;;oBACFA,MAAM1C,IAAAA,cAAI,EAACqE,IAAAA,eAAO,KAAI,UAAU,eAAe,UAAU7B,MAAM8B,OAAO,CAAC,OAAO;oBAC9EC,IAAAA,iBAAS,EAAC7B,KAAK;wBAAE8B,WAAW;oBAAK;4BACd;wBAAC;wBAAqB;;;;+BAAtB;;;;oBAAR7B,OAAQ;yBACb,CAACyB,IAAAA,kBAAU,EAACpE,IAAAA,cAAI,EAAC0C,KAAKC,QAAtB;;;;oBACF8B,QAAQC,KAAK,CAAC,AAAC,YAAoB/B,OAATH,OAAM,KAAgBE,OAAbC,MAAK,UAAY,OAAJD,KAAI;oBACpD;;wBAAMjB,YAAY,AAAC,0BAA+CkB,OAAtBH,OAAM,kBAAqB,OAALG,OAAQ3C,IAAAA,cAAI,EAAC0C,KAAKC;;;oBAApF;;;;;;;;;oBAKA3D,MAAM2F,IAAAA,oBAAY,EAAC3E,IAAAA,cAAI,EAAC0C,KAAK;oBAC7BE,YAAYe,OAAO3E,IAAI4F,eAAe,CAAC;oBACvC/B,SAASgC,KAAKC,KAAK,CAAC9F,IAAI+F,QAAQ,CAAC,GAAG,IAAInC,WAAWoC,QAAQ,CAAC;oBAC5DlC,QAAQmC,OAAOC,OAAO,CAACrC,QAAQsC,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAACtC,SAASA,KAAK,CAAC,EAAE,CAACuC,KAAK,KAAK,OAAO,MAAM,IAAIrD,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAANQ,OAAM;oBAC/EO,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKuC,KAAK,CAAC,EAAE;oBACpBrC,YAAYjE,IAAIuG,UAAU,GAAG,IAAI3C,YAAYG,KAAKyC,YAAY,CAAC,EAAE;oBACjEtC,SAASD,YAAY,MAAM,IAAI,IAAIa,aAAa9E,IAAIyG,MAAM,EAAExC,WAAWF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,QAAQ,IAAIc,aAAa9E,IAAIyG,MAAM,CAACnF,KAAK,CAAC2C,WAAWA,YAAYF,KAAKuC,KAAK,CAAC,EAAE,GAAGtC,OAAO;oBAE/KP,gBAAgBoC,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAAC3E,IAAAA,cAAI,EAAC0C,KAAK,mBAAmB;oBAErD;;wBAAM;2EAAA,QAAO;;;;oBAA3BS,YAAc,cAAdA;oBACFC,MAAM,IAAID,UAAUV,eAAe,CAAC;oBACpCY,iBAAQZ,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBiD,KAAK,cAA1BjD,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBkD,SAAS,CAAC,uCAAI,CAAC;oBAuB/E;;wBAAO;4BAAEjC,IAAI,AAAC,UAAe,OAANlB;4BAASQ,MAAAA;4BAAMtE,OAAO,SAAPA,MAAckH;;;;;4CAAUA,MAAMC,GAAG,CAACvC;;;;;wBAAK;;;;IAC/E;;AAEA,gFAAgF;AAEhF,SAAewC,YAAYtD,KAAa,EAAEd,GAAuB,EAAEqE,MAA0B;;YAErFC,MACAC,SACAC,KAUAlD;QAPN,SAAemD,KAAKP,KAAe;;oBAC3B9D,KAEAsE;;;;4BAFM;;gCAAMxE,MAAM,AAAC,GAAO,OAALoE,MAAK,gBAAc;oCAAEK,QAAQ;oCAAQJ,SAAAA;oCAASG,MAAMvB,KAAKyB,SAAS,CAAC;wCAAE9D,OAAAA;wCAAO+D,OAAOX;oCAAM;gCAAG;;;4BAAjH9D,MAAM;4BACZ,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BF,OAA3BkE,MAAK,wBAAiC,OAAXlE,IAAIG,MAAM;4BAC3E;;gCAAMH,IAAI0E,IAAI;;;4BAAtBJ,OAAQ;4BACd;;gCAAOA,KAAKK,IAAI,CAACZ,GAAG,CAAC,SAAC7B;2CAAMF,aAAa3B,IAAI,CAAC6B,EAAE0C,SAAS;;;;;YAC3D;;;;;oBAXA,IAAI,CAAChF,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe;oBACxCgE,OAAOtE,IAAI4C,OAAO,CAAC,QAAQ;oBAC3B2B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAASY,QAAQC,GAAG,CAACb,OAAO,GAAGc;oBAC3C,IAAIX,KAAKD,QAAQa,aAAa,GAAG,AAAC,UAAa,OAAJZ;oBAS7B;;wBAAMC;4BAAM;;;;oBAApBnD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACzD,MAAM;oBACxD;;wBAAO;4BAAEmE,IAAI,AAAC,OAAclB,OAARwD,MAAK,KAAS,OAANxD;4BAASQ,MAAAA;4BAAMtE,OAAOyH;wBAAK;;;;IACzD;;AAEA,IAAMY,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,IAAMA,IAAIC,IAAAA,qBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAInF,oBAAU,CAAC,kBAAkB;IAC/C,IAAMqF,MAAM,AAzKd,AAyKe,GAAYF,OAAVA,EAAEG,IAAI,EAAC,YAAGH,EAAE3E,KAAK,EAAC,KAAe,QAAZ2E,SAAAA,EAAEzF,GAAG,cAALyF,oBAAAA,SAAS;IAC7C,IAAII,IAAIR,UAAUS,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIJ,EAAEG,IAAI,KAAK,QAAQxB,YAAYqB,EAAE3E,KAAK,EAAE2E,EAAEzF,GAAG,EAAEyF,EAAEjB,GAAG,IAAI3D,eAAe4E,EAAE3E,KAAK;QAClFuE,UAAUU,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAE3E,IAAY,EAAE4E,IAAa;IAC9D,IAAM/D,IAAI,IAAIC,aAAad;IAC3B,IAAIiB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIhB,MAAMgB,IAAKC,QAAQ0D,IAAI,CAAC3D,EAAE,GAAG2D,IAAI,CAAC3D,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,KAAI,GAAGA,KAAIhB,MAAMgB,KAAKH,CAAC,CAACG,GAAE,GAAG2D,IAAI,CAAC3D,GAAE,GAAGC;IAChD,IAAI,CAAC2D,MAAM,OAAO;QAAE/D,GAAAA;QAAGgE,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAI9D,KAAI,GAAGA,KAAIhB,MAAMgB,KAAK8D,MAAM5D,KAAK4D,GAAG,CAACA,KAAK5D,KAAK6D,GAAG,CAAClE,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGgE,OAAOC,MAAM,OAAO;IAAE;AACpC;AAIO,SAAenJ,aAAaiC,EAAgB,EAAEsG,GAAW,EAAEc,OAAe;;mBACzEC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,MACD,4BAAA,oBAAA,iBAAA,YAAA,qBAAOtH,MAAMuH,WACZvI,QAMC,4BAAA,oBAAA,iBAAA,YAAA,QAAMuB,KAGPiH,QAGAC,QACGpJ;;;;;4BACDqJ,OACAC;;;;oCADAD,QAAQJ,KAAKjI,KAAK,CAAChB,GAAGA,IAAIR;oCAChB;;wCAAMmJ,SAASvJ,KAAK,CAACiK,MAAM9C,GAAG,CAAC,SAACgD;mDAAMA,EAAExI,IAAI;;;;oCAAtDuI,UAAU;oCAChBhI,GAAGC,IAAI,CAAC;oCACR,IAAI;wCACF8H,MAAMzI,OAAO,CAAC,SAAC4I,KAAKD;4CAClB,IAAqBnB,WAAAA,QAAQkB,OAAO,CAACC,EAAE,EAAEV,WAAW,OAA5CtE,IAAa6D,SAAb7D,GAAGgE,QAAUH,SAAVG;4CACX,IAAMkB,IAAI,IAAIC,UAAUb;4CACxB,IAAK,IAAInE,IAAI,GAAGA,IAAImE,WAAWnE,IAAK+E,CAAC,CAAC/E,EAAE,GAAGE,KAAK+E,KAAK,CAACpF,CAAC,CAACG,EAAE,GAAG6D;4CAC7DY,OAAOtH,GAAG,CAAC0G,OAAO3F,OAAOC,IAAI,CAAC4G,EAAEtD,MAAM,GAAGqD,IAAI7H,IAAI,EAAE6H,IAAII,KAAK;wCAC9D;wCACAtI,GAAGC,IAAI,CAAC;oCACV,EAAE,OAAOsI,KAAK;wCACZvI,GAAGC,IAAI,CAAC;wCACR,MAAMsI;oCACR;oCACAT,OAAOU,IAAI,CAAClF,KAAKmF,GAAG,CAAC/J,IAAIR,OAAOyJ,KAAKhJ,MAAM;;;;;;oBAC7C;oBA5CiB;;wBAAM0H,YAAYC;;;oBAA7Be,WAAW;oBACXC,QAAQtH,GAAGM,OAAO,CAAC,oFAAoFoI,GAAG;oBAChH,IAAIpB,MAAM3I,MAAM,KAAK,GAAG;;;oBAClB4I,YAAYjE,KAAKmF,GAAG,CAACxK,YAAYoJ,SAASjF,IAAI;oBAE9CoF,SAAS,IAAIpB;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAakB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOZ,GAAG,CAACa,IAAIpH,IAAI,eAAnBmH,yBAAAA;4BACbE,KAAK7I,IAAI,CAAC4I,IAAIa,KAAK;4BACnBd,OAAOX,GAAG,CAACY,IAAIpH,IAAI,EAAEqH;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC;oBACD,mCAAA,4BAAA;;wBAAL,IAAK,aAA2BH,6BAA3B,8BAAA,SAAA,0BAAA,kCAAmC;2DAAnC,kBAAOnH,uBAAMuH;4BACZvI,SAAAA,KAAAA;4BACJ,IAAI;gCACFA,SAASsJ,IAAAA,iBAAS,EAAC;oCAAEC,SAASvI;oCAAMwI,SAASzJ,IAAAA,cAAI,EAACgI,SAAS/G;oCAAOyI,SAAS;oCAAGC,MAAM;gCAAE;oCAAIjL;mCAAQkL,GAAG,CAACvI,SAAS,CAAC3C,KAAK;4BACvH,EAAE,eAAM;gCACN,UAAU,gEAAgE;4BAC5E;4BACK,mCAAA,4BAAA;;gCAAL,IAAK,aAAa8J,gCAAb,8BAAA,SAAA,0BAAA;oCAAMhH,MAAN;oCAAwB,IAAIvB,MAAM,CAACuB,IAAI,EAAE+G,KAAK9I,IAAI,CAAC;wCAAEwB,MAAAA;wCAAMiI,OAAO1H;wCAAKnB,MAAMJ,MAAM,CAACuB,IAAI,CAACnB,IAAI;oCAAC;;;gCAA9F;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;wBACP;;wBARK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAUCoI,SAAS7H,GAAGM,OAAO,CAAC;oBAC1B,wFAAwF;oBACxF,6DAA6D;oBACvDwH,SAASmB,IAAAA,oBAAQ,EAAC,oBAAoBtB,KAAKhJ,MAAM;oBAC9CD,IAAI;;;yBAAGA,CAAAA,IAAIiJ,KAAKhJ,MAAM,AAAD;;;;;;;;;;;;oBAAGD,KAAKR;;;;;;oBAkBtC4J,OAAOoB,MAAM;;;;;;IACf;;AAUO,SAAelL,mBAAmBgC,EAAgB,EAAEsG,GAAW,EAAE6C,KAAa,EAAEnI,MAAa;;YAOpFmI,cANR/B,SAIAC,UACAE,WACA9H,MACYqH,UAAPsC,IAELC,MAQAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM7B,KACHU,GACFoB,KACKnG,GACHoG,OACAC;;;;oBAvBFrC,UAAU,AAACd,IAAgCc,OAAO;oBACxD,IAAI,CAACA,SAAS,MAAM,IAAIhG,oBAAU,CAAC,eAAe;oBAClD;;wBAAMrD,aAAaiC,IAAIsG,KAAKc;;;oBAA5B;oBAEiB;;wBAAMf,YAAYC;;;oBAA7Be,WAAW;oBACXE,YAAYjE,KAAKmF,GAAG,CAACxK,YAAYoJ,SAASjF,IAAI;oBAC9C3C,OAAO,EAAC0J,eAAAA,MAAMO,KAAK,CAAC,2naAAZP,0BAAAA,mBAAsCjK,MAAM,CAAC,SAACyK;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIvK,IAAI,CAAC;oBACvF;;wBAAMiI,SAASvJ,KAAK;4BAAE2B;;;;oBAA/BqH,WAAAA;wBAAS,aAA6B,CAAC,EAAE;wBAAES;wBAAW;wBAA7D6B,KAAOtC,SAAV7D;oBAEFoG,OAAOrJ,GAAGM,OAAO,CAAC,+FAA+FoI,GAAG;oBAQpHY,OAAO,IAAIlD;oBACZ,kCAAA,2BAAA;;wBAAL,IAAK,YAAaiD,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;4BAAb5B,MAAN;4BACGU,IAAI,IAAIC,UAAUX,IAAIoC,MAAM,CAAChF,MAAM,EAAE4C,IAAIoC,MAAM,CAAClF,UAAU,EAAErB,KAAKmF,GAAG,CAAClB,WAAWE,IAAIoC,MAAM,CAACC,UAAU;4BACvGP,MAAM;4BACV,IAASnG,IAAI,GAAGA,IAAI+E,EAAExJ,MAAM,EAAEyE,IAAKmG,OAAOpB,CAAC,CAAC/E,EAAE,GAAGgG,EAAE,CAAChG,EAAE;4BAChDoG,QAAQD,MAAM9B,IAAIR,KAAK;4BACvBwC,WAAWH,KAAK1C,GAAG,CAACa,IAAIpH,IAAI;4BAClC,IAAI,CAACoJ,YAAYD,QAAQC,SAASD,KAAK,EAAEF,KAAKzC,GAAG,CAACY,IAAIpH,IAAI,EAAE;gCAAEmJ,OAAAA;gCAAOlL,OAAO,AAAC,IAAqBmJ,OAAlBA,IAAIsC,UAAU,EAAC,KAAgB,OAAbtC,IAAIuC,QAAQ;4BAAG;wBACnH;;wBAPK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAQL;;wBAAQ,qBAAGV,KAAKhF,OAAO,IACpB2F,IAAI,CAAC,SAACC,GAAGC;mCAAMA,CAAC,CAAC,EAAE,CAACX,KAAK,GAAGU,CAAC,CAAC,EAAE,CAACV,KAAK;2BACtC9J,KAAK,CAAC,GAAGsB,QACTiE,GAAG,CAAC;qEAAE5E,kBAAM8J;mCAAQ;gCAAE9J,MAAAA;gCAAM/B,OAAO6L,EAAE7L,KAAK;gCAAE8L,YAAY9G,KAAK+E,KAAK,CAAC8B,EAAEX,KAAK,GAAG,QAAQ;4BAAK;;;;;IAC/F"}

package/dist/cjs/features/links.js

@@ -47,7 +47,9 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// links(src, target, dst): target as written, dst the resolved path or NULL (a queryable dead link).
+// links(src, target, target_base, dst): target as written, target_base its baseKey
+// (indexed, for the incremental resolve below), dst the resolved path or NULL (a
+// queryable dead link).
 // Wikilinks ([[target]], [[target#anchor|alias]], embeds) plus relative markdown links to .md files.
 function extract(_raw, body) {
     var targets = new Set();
@@ -95,6 +97,9 @@ function extract(_raw, body) {
     }
     return _to_consumable_array(targets);
 }
+function cleanTarget(target) {
+    return target.replace(/\\/g, '/').replace(/^\.\//, '');
+}
 function baseKey(path) {
     return _posix.default.basename(path).replace(/\.md$/i, '').toLowerCase();
 }
@@ -103,7 +108,7 @@ function baseKey(path) {
 function resolveTarget(src, target, pathSet, byBase) {
     var _ref;
     var _byBase_get;
-    var clean = target.replace(/\\/g, '/').replace(/^\.\//, '');
+    var clean = cleanTarget(target);
     var fromSrc = _posix.default.normalize(_posix.default.join(_posix.default.dirname(src), clean));
     for(var _i = 0, _iter = [
         clean,
@@ -116,20 +121,18 @@ function resolveTarget(src, target, pathSet, byBase) {
     }
     return (_ref = (_byBase_get = byBase.get(baseKey(clean))) === null || _byBase_get === void 0 ? void 0 : _byBase_get[0]) !== null && _ref !== void 0 ? _ref : null;
 }
-// A new or deleted file can change any note's resolution, so re-resolve the whole table
-// whenever the tree changes. An in-memory probe per row; writes only on change.
-function resolveAll(db, files) {
-    var pathSet = new Set(files.map(function(f) {
-        return f.relPath;
-    }));
+function buildByBase(files) {
     var byBase = new Map();
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
     try {
-        for(var _iterator = _to_consumable_array(pathSet).sort()[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+        for(var _iterator = files.map(function(f) {
+            return f.relPath;
+        }).sort()[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
             var path = _step.value;
-            var list = byBase.get(baseKey(path));
+            var key = baseKey(path);
+            var list = byBase.get(key);
             if (list) list.push(path);
-            else byBase.set(baseKey(path), [
+            else byBase.set(key, [
                 path
             ]);
         }
@@ -147,14 +150,86 @@ function resolveAll(db, files) {
             }
         }
     }
-    var rows = db.prepare('SELECT src, target, dst FROM links').all();
+    return byBase;
+}
+// Applies resolveTarget to `rows`, writing only rows whose dst actually changes.
+function resolveRows(db, rows, pathSet, byBase) {
     var update = db.prepare('UPDATE links SET dst = ? WHERE src = ? AND target = ?');
-    var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+    var changed = false;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
     try {
-        for(var _iterator1 = rows[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
-            var row = _step1.value;
+        for(var _iterator = rows[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var row = _step.value;
             var dst = resolveTarget(row.src, row.target, pathSet, byBase);
-            if (dst !== row.dst) update.run(dst, row.src, row.target);
+            if (dst !== row.dst) {
+                update.run(dst, row.src, row.target);
+                changed = true;
+            }
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return changed;
+}
+// A new or deleted file can change any note's resolution, so re-resolve the whole table.
+// Fallback for cold builds and large deltas -- see afterReconcile's threshold.
+function resolveAll(db, files) {
+    var pathSet = new Set(files.map(function(f) {
+        return f.relPath;
+    }));
+    var byBase = buildByBase(files);
+    var rows = db.prepare('SELECT src, target, dst FROM links').all();
+    return resolveRows(db, rows, pathSet, byBase);
+}
+// Re-resolve only what this reconcile could have affected: (a) links whose src was just
+// re-stored (their rows are fresh, dst reset to NULL by store()), (b) links whose target's
+// basename matches a file that appeared or disappeared this reconcile (resolution and
+// lexicographic-tie ambiguity hinge on exactly that set), and (c) links whose dst was a
+// path that just vanished. All three are indexed lookups (src is the links PK's leading
+// column; target_base and dst each have their own index), so this reads only the rows this
+// reconcile could plausibly have changed, not the whole table.
+function resolveIncremental(db, delta) {
+    var pathSet = new Set(delta.files.map(function(f) {
+        return f.relPath;
+    }));
+    var byBase = buildByBase(delta.files);
+    var changedBasenames = new Set();
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = delta.added[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var p = _step.value;
+            changedBasenames.add(baseKey(p));
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+    try {
+        for(var _iterator1 = delta.vanished[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+            var p1 = _step1.value;
+            changedBasenames.add(baseKey(p1));
         }
     } catch (err) {
         _didIteratorError1 = true;
@@ -170,6 +245,46 @@ function resolveAll(db, files) {
             }
         }
     }
+    // \0 can't appear in a path or target, so the key never collides; a space can.
+    var candidates = new Map();
+    var collect = function collect(rows) {
+        var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+        try {
+            for(var _iterator = rows[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                var row = _step.value;
+                candidates.set("".concat(row.src, "\0").concat(row.target), row);
+            }
+        } catch (err) {
+            _didIteratorError = true;
+            _iteratorError = err;
+        } finally{
+            try {
+                if (!_iteratorNormalCompletion && _iterator.return != null) {
+                    _iterator.return();
+                }
+            } finally{
+                if (_didIteratorError) {
+                    throw _iteratorError;
+                }
+            }
+        }
+    };
+    // Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable
+    // limit (SQLITE_MAX_VARIABLE_NUMBER, 32766).
+    var collectIn = function collectIn(column, keys) {
+        for(var i = 0; i < keys.length; i += 500){
+            var _db_prepare;
+            var chunk = keys.slice(i, i + 500);
+            var placeholders = chunk.map(function() {
+                return '?';
+            }).join(', ');
+            collect((_db_prepare = db.prepare("SELECT src, target, dst FROM links WHERE ".concat(column, " IN (").concat(placeholders, ")"))).all.apply(_db_prepare, _to_consumable_array(chunk)));
+        }
+    };
+    collectIn('src', delta.reparsed);
+    collectIn('target_base', _to_consumable_array(changedBasenames));
+    collectIn('dst', delta.vanished);
+    return resolveRows(db, _to_consumable_array(candidates.values()), pathSet, byBase);
 }
 function linkEdges(db) {
     return db.prepare('SELECT src, dst FROM links WHERE dst IS NOT NULL').all().map(function(r) {
@@ -179,23 +294,70 @@ function linkEdges(db) {
         ];
     });
 }
+// A removed path's outbound link rows are already gone by the time afterReconcile runs
+// (remove() below deletes them for vanished files AND for reparsed existing files), so
+// afterReconcile can't tell from the links table alone whether deleted rows carried edges.
+// remove() records that here, keyed on the reconcile's own delta object -- state dies with
+// the reconcile, including on ROLLBACK, and distinct reconciles never share it.
+var removedWithLinks = new WeakMap();
+function recordRemoved(delta, path) {
+    var set = removedWithLinks.get(delta);
+    if (!set) {
+        set = new Set();
+        removedWithLinks.set(delta, set);
+    }
+    set.add(path);
+}
+// remove() runs per path; the vanished list is an array, so cache the Set per delta.
+var vanishedSets = new WeakMap();
+function vanishedSet(delta) {
+    var set = vanishedSets.get(delta);
+    if (!set) {
+        set = new Set(delta.vanished);
+        vanishedSets.set(delta, set);
+    }
+    return set;
+}
 var links = {
     name: 'links',
     schema: function schema(db) {
-        db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, dst TEXT, PRIMARY KEY (src, target))');
+        db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, PRIMARY KEY (src, target))');
         db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');
+        db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');
     },
     extract: extract,
-    remove: function remove(db, path) {
-        db.prepare('DELETE FROM links WHERE src = ?').run(path);
+    // Reparsed docs are diffed in store() below instead of wiped here: deleting and
+    // re-inserting resets dst to NULL, which made every touch-only reparse read as an edge
+    // change and recompute PageRank -- measured as most of the remaining update cost.
+    remove: function remove(db, path, delta) {
+        if (!vanishedSet(delta).has(path)) return;
+        var result = db.prepare('DELETE FROM links WHERE src = ?').run(path);
+        if (Number(result.changes) > 0) recordRemoved(delta, path);
     },
-    store: function store(db, path, extracted) {
-        var insert = db.prepare('INSERT OR REPLACE INTO links (src, target, dst) VALUES (?, ?, NULL)');
+    store: function store(db, path, extracted, delta) {
+        var targets = extracted;
+        // Stale rows (targets the new parse no longer contains) are real edge removals.
+        var stale;
+        if (targets.length === 0) {
+            stale = db.prepare('DELETE FROM links WHERE src = ?').run(path);
+        } else {
+            var _db_prepare;
+            var placeholders = targets.map(function() {
+                return '?';
+            }).join(', ');
+            stale = (_db_prepare = db.prepare("DELETE FROM links WHERE src = ? AND target NOT IN (".concat(placeholders, ")"))).run.apply(_db_prepare, [
+                path
+            ].concat(_to_consumable_array(targets)));
+        }
+        if (Number(stale.changes) > 0) recordRemoved(delta, path);
+        // Upsert preserves dst on surviving rows, so an unchanged link resolves to the same
+        // value and reports no change; only genuinely new rows start at NULL.
+        var insert = db.prepare('INSERT INTO links (src, target, target_base, dst) VALUES (?, ?, ?, NULL) ON CONFLICT(src, target) DO UPDATE SET target_base = excluded.target_base');
         var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
         try {
-            for(var _iterator = extracted[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            for(var _iterator = targets[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
                 var target = _step.value;
-                insert.run(path, target);
+                insert.run(path, target, baseKey(cleanTarget(target)));
             }
         } catch (err) {
             _didIteratorError = true;
@@ -212,6 +374,21 @@ var links = {
             }
         }
     },
-    afterReconcile: resolveAll
+    afterReconcile: function afterReconcile(db, delta) {
+        var _ref;
+        var _removedWithLinks_get;
+        // Any deleted rows that carried edges mean the edge set shrank -- whether the file
+        // vanished or was reparsed down to fewer/no links. dstChanged alone misses the
+        // reparsed-to-zero-links case (no surviving rows to re-resolve).
+        var removeHadLinks = ((_ref = (_removedWithLinks_get = removedWithLinks.get(delta)) === null || _removedWithLinks_get === void 0 ? void 0 : _removedWithLinks_get.size) !== null && _ref !== void 0 ? _ref : 0) > 0;
+        var churn = delta.reparsed.length + delta.vanished.length;
+        // Fallback rule: correctness over cleverness once too much of the tree moved in one
+        // reconcile -- a full pass is the one guaranteed to match resolveTarget's ambiguity
+        // rules exactly. A cold build reparses ~every file, so it clears this threshold on its
+        // own without a separate check.
+        var large = delta.files.length === 0 || churn > 0.2 * delta.files.length;
+        var dstChanged = large ? resolveAll(db, delta.files) : resolveIncremental(db, delta);
+        delta.linksChanged = dstChanged || removeHadLinks;
+    }
 };
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/features/links.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { FileStat } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// links(src, target, dst): target as written, dst the resolved path or NULL (a queryable dead link).\n\n// Wikilinks ([[target]], [[target#anchor|alias]], embeds) plus relative markdown links to .md files.\nfunction extract(_raw: string, body: string): string[] {\n  const targets = new Set<string>();\n  for (const m of body.matchAll(/\\[\\[([^\\]|#]+)(?:#[^\\]|]*)?(?:\\|[^\\]]*)?\\]\\]/g)) {\n    const target = m[1].trim();\n    if (target) targets.add(target);\n  }\n  for (const m of body.matchAll(/\\]\\(([^)]+\\.md)(?:#[^)]*)?\\)/g)) {\n    const target = m[1].trim();\n    if (target && !/^[a-z]+:\\/\\//i.test(target)) targets.add(target);\n  }\n  return [...targets];\n}\n\nfunction baseKey(path: string): string {\n  return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking\n// note's directory, then basename match (lexicographically first on ties).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n  const clean = target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n  const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n  for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n    if (pathSet.has(candidate)) return candidate;\n  }\n  return byBase.get(baseKey(clean))?.[0] ?? null;\n}\n\n// A new or deleted file can change any note's resolution, so re-resolve the whole table\n// whenever the tree changes. An in-memory probe per row; writes only on change.\nfunction resolveAll(db: DatabaseSync, files: FileStat[]): void {\n  const pathSet = new Set(files.map((f) => f.relPath));\n  const byBase = new Map<string, string[]>();\n  for (const path of [...pathSet].sort()) {\n    const list = byBase.get(baseKey(path));\n    if (list) list.push(path);\n    else byBase.set(baseKey(path), [path]);\n  }\n  const rows = db.prepare('SELECT src, target, dst FROM links').all() as Array<{ src: string; target: string; dst: string | null }>;\n  const update = db.prepare('UPDATE links SET dst = ? WHERE src = ? AND target = ?');\n  for (const row of rows) {\n    const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n    if (dst !== row.dst) update.run(dst, row.src, row.target);\n  }\n}\n\n// Resolved edges, for rank and for find's graph expansion.\nexport function linkEdges(db: DatabaseSync): [string, string][] {\n  return (db.prepare('SELECT src, dst FROM links WHERE dst IS NOT NULL').all() as Array<{ src: string; dst: string }>).map((r) => [r.src, r.dst]);\n}\n\nexport const links: Feature = {\n  name: 'links',\n  schema(db) {\n    db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, dst TEXT, PRIMARY KEY (src, target))');\n    db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n  },\n  extract,\n  remove(db, path) {\n    db.prepare('DELETE FROM links WHERE src = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const insert = db.prepare('INSERT OR REPLACE INTO links (src, target, dst) VALUES (?, ?, NULL)');\n    for (const target of extracted as string[]) insert.run(path, target);\n  },\n  afterReconcile: resolveAll,\n};\n"],"names":["linkEdges","links","extract","_raw","body","targets","Set","matchAll","m","target","trim","add","test","baseKey","path","posix","basename","replace","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","has","get","resolveAll","db","files","map","f","relPath","Map","sort","list","push","set","rows","prepare","all","update","row","dst","run","r","name","schema","exec","remove","store","extracted","insert","afterReconcile"],"mappings":";;;;;;;;;;;QAuDgBA;eAAAA;;QAIHC;eAAAA;;;4DA3DK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKlB,qGAAqG;AAErG,qGAAqG;AACrG,SAASC,QAAQC,IAAY,EAAEC,IAAY;IACzC,IAAMC,UAAU,IAAIC;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAWF,KAAKG,QAAQ,CAAC,qEAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAA2E;YAA3E,IAAMC,IAAN;YACH,IAAMC,SAASD,CAAC,CAAC,EAAE,CAACE,IAAI;YACxB,IAAID,QAAQJ,QAAQM,GAAG,CAACF;QAC1B;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QAIA,mCAAA,4BAAA;;QAAL,QAAK,aAAWL,KAAKG,QAAQ,CAAC,qDAAzB,UAAA,8BAAA,SAAA,0BAAA,kCAA2D;YAA3D,IAAMC,KAAN;YACH,IAAMC,UAASD,EAAC,CAAC,EAAE,CAACE,IAAI;YACxB,IAAID,WAAU,CAAC,gBAAgBG,IAAI,CAACH,UAASJ,QAAQM,GAAG,CAACF;QAC3D;;QAHK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAQ,qBAAGJ;AACb;AAEA,SAASQ,QAAQC,IAAY;IAC3B,OAAOC,cAAK,CAACC,QAAQ,CAACF,MAAMG,OAAO,CAAC,UAAU,IAAIC,WAAW;AAC/D;AAEA,uFAAuF;AACvF,2EAA2E;AAC3E,SAASC,cAAcC,GAAW,EAAEX,MAAc,EAAEY,OAAoB,EAAEC,MAA6B;;QAM9FA;IALP,IAAMC,QAAQd,OAAOQ,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;IAC1D,IAAMO,UAAUT,cAAK,CAACU,SAAS,CAACV,cAAK,CAACW,IAAI,CAACX,cAAK,CAACY,OAAO,CAACP,MAAMG;IAC/D,gBAAwB,QAAA;QAACA;QAAQ,GAAQ,OAANA,OAAM;QAAMC;QAAU,GAAU,OAARA,SAAQ;KAAK,OAAhD,mBAAkD;YAA/DI,YAAa;QACtB,IAAIP,QAAQQ,GAAG,CAACD,YAAY,OAAOA;IACrC;IACA,gBAAON,cAAAA,OAAOQ,GAAG,CAACjB,QAAQU,qBAAnBD,kCAAAA,WAA4B,CAAC,EAAE,uCAAI;AAC5C;AAEA,wFAAwF;AACxF,gFAAgF;AAChF,SAASS,WAAWC,EAAgB,EAAEC,KAAiB;IACrD,IAAMZ,UAAU,IAAIf,IAAI2B,MAAMC,GAAG,CAAC,SAACC;eAAMA,EAAEC,OAAO;;IAClD,IAAMd,SAAS,IAAIe;QACd,kCAAA,2BAAA;;QAAL,QAAK,YAAc,AAAC,qBAAGhB,SAASiB,IAAI,uBAA/B,SAAA,6BAAA,QAAA,yBAAA,iCAAmC;YAAnC,IAAMxB,OAAN;YACH,IAAMyB,OAAOjB,OAAOQ,GAAG,CAACjB,QAAQC;YAChC,IAAIyB,MAAMA,KAAKC,IAAI,CAAC1B;iBACfQ,OAAOmB,GAAG,CAAC5B,QAAQC,OAAO;gBAACA;aAAK;QACvC;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,IAAM4B,OAAOV,GAAGW,OAAO,CAAC,sCAAsCC,GAAG;IACjE,IAAMC,SAASb,GAAGW,OAAO,CAAC;QACrB,mCAAA,4BAAA;;QAAL,QAAK,aAAaD,yBAAb,UAAA,8BAAA,SAAA,0BAAA,kCAAmB;YAAnB,IAAMI,MAAN;YACH,IAAMC,MAAM5B,cAAc2B,IAAI1B,GAAG,EAAE0B,IAAIrC,MAAM,EAAEY,SAASC;YACxD,IAAIyB,QAAQD,IAAIC,GAAG,EAAEF,OAAOG,GAAG,CAACD,KAAKD,IAAI1B,GAAG,EAAE0B,IAAIrC,MAAM;QAC1D;;QAHK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;AAIP;AAGO,SAAST,UAAUgC,EAAgB;IACxC,OAAO,AAACA,GAAGW,OAAO,CAAC,oDAAoDC,GAAG,GAA2CV,GAAG,CAAC,SAACe;eAAM;YAACA,EAAE7B,GAAG;YAAE6B,EAAEF,GAAG;SAAC;;AAChJ;AAEO,IAAM9C,QAAiB;IAC5BiD,MAAM;IACNC,QAAAA,SAAAA,OAAOnB,EAAE;QACPA,GAAGoB,IAAI,CAAC;QACRpB,GAAGoB,IAAI,CAAC;IACV;IACAlD,SAAAA;IACAmD,QAAAA,SAAAA,OAAOrB,EAAE,EAAElB,IAAI;QACbkB,GAAGW,OAAO,CAAC,mCAAmCK,GAAG,CAAClC;IACpD;IACAwC,OAAAA,SAAAA,MAAMtB,EAAE,EAAElB,IAAI,EAAEyC,SAAS;QACvB,IAAMC,SAASxB,GAAGW,OAAO,CAAC;YACrB,kCAAA,2BAAA;;YAAL,QAAK,YAAgBY,8BAAhB,SAAA,6BAAA,QAAA,yBAAA;gBAAA,IAAM9C,SAAN;gBAAuC+C,OAAOR,GAAG,CAAClC,MAAML;;;YAAxD;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;IACP;IACAgD,gBAAgB1B;AAClB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { FileStat } from '../scan.ts';\nimport type { Feature, ReconcileDelta } from './types.ts';\n\n// links(src, target, target_base, dst): target as written, target_base its baseKey\n// (indexed, for the incremental resolve below), dst the resolved path or NULL (a\n// queryable dead link).\n\n// Wikilinks ([[target]], [[target#anchor|alias]], embeds) plus relative markdown links to .md files.\nfunction extract(_raw: string, body: string): string[] {\n  const targets = new Set<string>();\n  for (const m of body.matchAll(/\\[\\[([^\\]|#]+)(?:#[^\\]|]*)?(?:\\|[^\\]]*)?\\]\\]/g)) {\n    const target = m[1].trim();\n    if (target) targets.add(target);\n  }\n  for (const m of body.matchAll(/\\]\\(([^)]+\\.md)(?:#[^)]*)?\\)/g)) {\n    const target = m[1].trim();\n    if (target && !/^[a-z]+:\\/\\//i.test(target)) targets.add(target);\n  }\n  return [...targets];\n}\n\nfunction cleanTarget(target: string): string {\n  return target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n}\n\nfunction baseKey(path: string): string {\n  return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking\n// note's directory, then basename match (lexicographically first on ties).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n  const clean = cleanTarget(target);\n  const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n  for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n    if (pathSet.has(candidate)) return candidate;\n  }\n  return byBase.get(baseKey(clean))?.[0] ?? null;\n}\n\nfunction buildByBase(files: FileStat[]): Map<string, string[]> {\n  const byBase = new Map<string, string[]>();\n  for (const path of files.map((f) => f.relPath).sort()) {\n    const key = baseKey(path);\n    const list = byBase.get(key);\n    if (list) list.push(path);\n    else byBase.set(key, [path]);\n  }\n  return byBase;\n}\n\ninterface LinkRow {\n  src: string;\n  target: string;\n  dst: string | null;\n}\n\n// Applies resolveTarget to `rows`, writing only rows whose dst actually changes.\nfunction resolveRows(db: DatabaseSync, rows: LinkRow[], pathSet: Set<string>, byBase: Map<string, string[]>): boolean {\n  const update = db.prepare('UPDATE links SET dst = ? WHERE src = ? AND target = ?');\n  let changed = false;\n  for (const row of rows) {\n    const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n    if (dst !== row.dst) {\n      update.run(dst, row.src, row.target);\n      changed = true;\n    }\n  }\n  return changed;\n}\n\n// A new or deleted file can change any note's resolution, so re-resolve the whole table.\n// Fallback for cold builds and large deltas -- see afterReconcile's threshold.\nfunction resolveAll(db: DatabaseSync, files: FileStat[]): boolean {\n  const pathSet = new Set(files.map((f) => f.relPath));\n  const byBase = buildByBase(files);\n  const rows = db.prepare('SELECT src, target, dst FROM links').all() as unknown as LinkRow[];\n  return resolveRows(db, rows, pathSet, byBase);\n}\n\n// Re-resolve only what this reconcile could have affected: (a) links whose src was just\n// re-stored (their rows are fresh, dst reset to NULL by store()), (b) links whose target's\n// basename matches a file that appeared or disappeared this reconcile (resolution and\n// lexicographic-tie ambiguity hinge on exactly that set), and (c) links whose dst was a\n// path that just vanished. All three are indexed lookups (src is the links PK's leading\n// column; target_base and dst each have their own index), so this reads only the rows this\n// reconcile could plausibly have changed, not the whole table.\nfunction resolveIncremental(db: DatabaseSync, delta: ReconcileDelta): boolean {\n  const pathSet = new Set(delta.files.map((f) => f.relPath));\n  const byBase = buildByBase(delta.files);\n\n  const changedBasenames = new Set<string>();\n  for (const p of delta.added) changedBasenames.add(baseKey(p));\n  for (const p of delta.vanished) changedBasenames.add(baseKey(p));\n\n  // \\0 can't appear in a path or target, so the key never collides; a space can.\n  const candidates = new Map<string, LinkRow>();\n  const collect = (rows: LinkRow[]) => {\n    for (const row of rows) candidates.set(`${row.src}\\0${row.target}`, row);\n  };\n  // Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable\n  // limit (SQLITE_MAX_VARIABLE_NUMBER, 32766).\n  const collectIn = (column: string, keys: string[]) => {\n    for (let i = 0; i < keys.length; i += 500) {\n      const chunk = keys.slice(i, i + 500);\n      const placeholders = chunk.map(() => '?').join(', ');\n      collect(db.prepare(`SELECT src, target, dst FROM links WHERE ${column} IN (${placeholders})`).all(...chunk) as unknown as LinkRow[]);\n    }\n  };\n\n  collectIn('src', delta.reparsed);\n  collectIn('target_base', [...changedBasenames]);\n  collectIn('dst', delta.vanished);\n\n  return resolveRows(db, [...candidates.values()], pathSet, byBase);\n}\n\n// Resolved edges, for rank and for find's graph expansion.\nexport function linkEdges(db: DatabaseSync): [string, string][] {\n  return (db.prepare('SELECT src, dst FROM links WHERE dst IS NOT NULL').all() as Array<{ src: string; dst: string }>).map((r) => [r.src, r.dst]);\n}\n\n// A removed path's outbound link rows are already gone by the time afterReconcile runs\n// (remove() below deletes them for vanished files AND for reparsed existing files), so\n// afterReconcile can't tell from the links table alone whether deleted rows carried edges.\n// remove() records that here, keyed on the reconcile's own delta object -- state dies with\n// the reconcile, including on ROLLBACK, and distinct reconciles never share it.\nconst removedWithLinks = new WeakMap<ReconcileDelta, Set<string>>();\n\nfunction recordRemoved(delta: ReconcileDelta, path: string): void {\n  let set = removedWithLinks.get(delta);\n  if (!set) {\n    set = new Set();\n    removedWithLinks.set(delta, set);\n  }\n  set.add(path);\n}\n\n// remove() runs per path; the vanished list is an array, so cache the Set per delta.\nconst vanishedSets = new WeakMap<ReconcileDelta, Set<string>>();\nfunction vanishedSet(delta: ReconcileDelta): Set<string> {\n  let set = vanishedSets.get(delta);\n  if (!set) {\n    set = new Set(delta.vanished);\n    vanishedSets.set(delta, set);\n  }\n  return set;\n}\n\nexport const links: Feature = {\n  name: 'links',\n  schema(db) {\n    db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, PRIMARY KEY (src, target))');\n    db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n    db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');\n  },\n  extract,\n  // Reparsed docs are diffed in store() below instead of wiped here: deleting and\n  // re-inserting resets dst to NULL, which made every touch-only reparse read as an edge\n  // change and recompute PageRank -- measured as most of the remaining update cost.\n  remove(db, path, delta) {\n    if (!vanishedSet(delta).has(path)) return;\n    const result = db.prepare('DELETE FROM links WHERE src = ?').run(path);\n    if (Number(result.changes) > 0) recordRemoved(delta, path);\n  },\n  store(db, path, extracted, delta) {\n    const targets = extracted as string[];\n    // Stale rows (targets the new parse no longer contains) are real edge removals.\n    let stale: ReturnType<ReturnType<DatabaseSync['prepare']>['run']>;\n    if (targets.length === 0) {\n      stale = db.prepare('DELETE FROM links WHERE src = ?').run(path);\n    } else {\n      const placeholders = targets.map(() => '?').join(', ');\n      stale = db.prepare(`DELETE FROM links WHERE src = ? AND target NOT IN (${placeholders})`).run(path, ...targets);\n    }\n    if (Number(stale.changes) > 0) recordRemoved(delta, path);\n    // Upsert preserves dst on surviving rows, so an unchanged link resolves to the same\n    // value and reports no change; only genuinely new rows start at NULL.\n    const insert = db.prepare('INSERT INTO links (src, target, target_base, dst) VALUES (?, ?, ?, NULL) ON CONFLICT(src, target) DO UPDATE SET target_base = excluded.target_base');\n    for (const target of targets) insert.run(path, target, baseKey(cleanTarget(target)));\n  },\n  afterReconcile(db, delta) {\n    // Any deleted rows that carried edges mean the edge set shrank -- whether the file\n    // vanished or was reparsed down to fewer/no links. dstChanged alone misses the\n    // reparsed-to-zero-links case (no surviving rows to re-resolve).\n    const removeHadLinks = (removedWithLinks.get(delta)?.size ?? 0) > 0;\n\n    const churn = delta.reparsed.length + delta.vanished.length;\n    // Fallback rule: correctness over cleverness once too much of the tree moved in one\n    // reconcile -- a full pass is the one guaranteed to match resolveTarget's ambiguity\n    // rules exactly. A cold build reparses ~every file, so it clears this threshold on its\n    // own without a separate check.\n    const large = delta.files.length === 0 || churn > 0.2 * delta.files.length;\n\n    const dstChanged = large ? resolveAll(db, delta.files) : resolveIncremental(db, delta);\n    delta.linksChanged = dstChanged || removeHadLinks;\n  },\n};\n"],"names":["linkEdges","links","extract","_raw","body","targets","Set","matchAll","m","target","trim","add","test","cleanTarget","replace","baseKey","path","posix","basename","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","has","get","buildByBase","files","Map","map","f","relPath","sort","key","list","push","set","resolveRows","db","rows","update","prepare","changed","row","dst","run","resolveAll","all","resolveIncremental","delta","changedBasenames","added","p","vanished","candidates","collect","collectIn","column","keys","i","length","chunk","slice","placeholders","reparsed","values","r","removedWithLinks","WeakMap","recordRemoved","vanishedSets","vanishedSet","name","schema","exec","remove","result","Number","changes","store","extracted","stale","insert","afterReconcile","removeHadLinks","size","churn","large","dstChanged","linksChanged"],"mappings":";;;;;;;;;;;QAwHgBA;eAAAA;;QA+BHC;eAAAA;;;4DAvJK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKlB,mFAAmF;AACnF,iFAAiF;AACjF,wBAAwB;AAExB,qGAAqG;AACrG,SAASC,QAAQC,IAAY,EAAEC,IAAY;IACzC,IAAMC,UAAU,IAAIC;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAWF,KAAKG,QAAQ,CAAC,qEAAzB,SAAA,6BAAA,QAAA,yBAAA,iCAA2E;YAA3E,IAAMC,IAAN;YACH,IAAMC,SAASD,CAAC,CAAC,EAAE,CAACE,IAAI;YACxB,IAAID,QAAQJ,QAAQM,GAAG,CAACF;QAC1B;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QAIA,mCAAA,4BAAA;;QAAL,QAAK,aAAWL,KAAKG,QAAQ,CAAC,qDAAzB,UAAA,8BAAA,SAAA,0BAAA,kCAA2D;YAA3D,IAAMC,KAAN;YACH,IAAMC,UAASD,EAAC,CAAC,EAAE,CAACE,IAAI;YACxB,IAAID,WAAU,CAAC,gBAAgBG,IAAI,CAACH,UAASJ,QAAQM,GAAG,CAACF;QAC3D;;QAHK;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAQ,qBAAGJ;AACb;AAEA,SAASQ,YAAYJ,MAAc;IACjC,OAAOA,OAAOK,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;AACrD;AAEA,SAASC,QAAQC,IAAY;IAC3B,OAAOC,cAAK,CAACC,QAAQ,CAACF,MAAMF,OAAO,CAAC,UAAU,IAAIK,WAAW;AAC/D;AAEA,uFAAuF;AACvF,2EAA2E;AAC3E,SAASC,cAAcC,GAAW,EAAEZ,MAAc,EAAEa,OAAoB,EAAEC,MAA6B;;QAM9FA;IALP,IAAMC,QAAQX,YAAYJ;IAC1B,IAAMgB,UAAUR,cAAK,CAACS,SAAS,CAACT,cAAK,CAACU,IAAI,CAACV,cAAK,CAACW,OAAO,CAACP,MAAMG;IAC/D,gBAAwB,QAAA;QAACA;QAAQ,GAAQ,OAANA,OAAM;QAAMC;QAAU,GAAU,OAARA,SAAQ;KAAK,OAAhD,mBAAkD;YAA/DI,YAAa;QACtB,IAAIP,QAAQQ,GAAG,CAACD,YAAY,OAAOA;IACrC;IACA,gBAAON,cAAAA,OAAOQ,GAAG,CAAChB,QAAQS,qBAAnBD,kCAAAA,WAA4B,CAAC,EAAE,uCAAI;AAC5C;AAEA,SAASS,YAAYC,KAAiB;IACpC,IAAMV,SAAS,IAAIW;QACd,kCAAA,2BAAA;;QAAL,QAAK,YAAcD,MAAME,GAAG,CAAC,SAACC;mBAAMA,EAAEC,OAAO;WAAEC,IAAI,uBAA9C,SAAA,6BAAA,QAAA,yBAAA,iCAAkD;YAAlD,IAAMtB,OAAN;YACH,IAAMuB,MAAMxB,QAAQC;YACpB,IAAMwB,OAAOjB,OAAOQ,GAAG,CAACQ;YACxB,IAAIC,MAAMA,KAAKC,IAAI,CAACzB;iBACfO,OAAOmB,GAAG,CAACH,KAAK;gBAACvB;aAAK;QAC7B;;QALK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAML,OAAOO;AACT;AAQA,iFAAiF;AACjF,SAASoB,YAAYC,EAAgB,EAAEC,IAAe,EAAEvB,OAAoB,EAAEC,MAA6B;IACzG,IAAMuB,SAASF,GAAGG,OAAO,CAAC;IAC1B,IAAIC,UAAU;QACT,kCAAA,2BAAA;;QAAL,QAAK,YAAaH,yBAAb,SAAA,6BAAA,QAAA,yBAAA,iCAAmB;YAAnB,IAAMI,MAAN;YACH,IAAMC,MAAM9B,cAAc6B,IAAI5B,GAAG,EAAE4B,IAAIxC,MAAM,EAAEa,SAASC;YACxD,IAAI2B,QAAQD,IAAIC,GAAG,EAAE;gBACnBJ,OAAOK,GAAG,CAACD,KAAKD,IAAI5B,GAAG,EAAE4B,IAAIxC,MAAM;gBACnCuC,UAAU;YACZ;QACF;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,OAAOA;AACT;AAEA,yFAAyF;AACzF,+EAA+E;AAC/E,SAASI,WAAWR,EAAgB,EAAEX,KAAiB;IACrD,IAAMX,UAAU,IAAIhB,IAAI2B,MAAME,GAAG,CAAC,SAACC;eAAMA,EAAEC,OAAO;;IAClD,IAAMd,SAASS,YAAYC;IAC3B,IAAMY,OAAOD,GAAGG,OAAO,CAAC,sCAAsCM,GAAG;IACjE,OAAOV,YAAYC,IAAIC,MAAMvB,SAASC;AACxC;AAEA,wFAAwF;AACxF,2FAA2F;AAC3F,sFAAsF;AACtF,wFAAwF;AACxF,wFAAwF;AACxF,2FAA2F;AAC3F,+DAA+D;AAC/D,SAAS+B,mBAAmBV,EAAgB,EAAEW,KAAqB;IACjE,IAAMjC,UAAU,IAAIhB,IAAIiD,MAAMtB,KAAK,CAACE,GAAG,CAAC,SAACC;eAAMA,EAAEC,OAAO;;IACxD,IAAMd,SAASS,YAAYuB,MAAMtB,KAAK;IAEtC,IAAMuB,mBAAmB,IAAIlD;QACxB,kCAAA,2BAAA;;QAAL,QAAK,YAAWiD,MAAME,KAAK,qBAAtB,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMC,IAAN;YAAwBF,iBAAiB7C,GAAG,CAACI,QAAQ2C;;;QAArD;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAWH,MAAMI,QAAQ,qBAAzB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMD,KAAN;YAA2BF,iBAAiB7C,GAAG,CAACI,QAAQ2C;;;QAAxD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAEL,+EAA+E;IAC/E,IAAME,aAAa,IAAI1B;IACvB,IAAM2B,UAAU,iBAAChB;YACV,kCAAA,2BAAA;;YAAL,QAAK,YAAaA,yBAAb,SAAA,6BAAA,QAAA,yBAAA;gBAAA,IAAMI,MAAN;gBAAmBW,WAAWlB,GAAG,CAAC,AAAC,GAAcO,OAAZA,IAAI5B,GAAG,EAAC,MAAe,OAAX4B,IAAIxC,MAAM,GAAIwC;;;YAA/D;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;IACP;IACA,qFAAqF;IACrF,6CAA6C;IAC7C,IAAMa,YAAY,mBAACC,QAAgBC;QACjC,IAAK,IAAIC,IAAI,GAAGA,IAAID,KAAKE,MAAM,EAAED,KAAK,IAAK;gBAGjCrB;YAFR,IAAMuB,QAAQH,KAAKI,KAAK,CAACH,GAAGA,IAAI;YAChC,IAAMI,eAAeF,MAAMhC,GAAG,CAAC;uBAAM;eAAKR,IAAI,CAAC;YAC/CkC,QAAQjB,CAAAA,cAAAA,GAAGG,OAAO,CAAC,AAAC,4CAAyDsB,OAAdN,QAAO,SAAoB,OAAbM,cAAa,OAAIhB,GAAG,OAAzFT,aAA0F,qBAAGuB;QACvG;IACF;IAEAL,UAAU,OAAOP,MAAMe,QAAQ;IAC/BR,UAAU,eAAgB,qBAAGN;IAC7BM,UAAU,OAAOP,MAAMI,QAAQ;IAE/B,OAAOhB,YAAYC,IAAK,qBAAGgB,WAAWW,MAAM,KAAKjD,SAASC;AAC5D;AAGO,SAASvB,UAAU4C,EAAgB;IACxC,OAAO,AAACA,GAAGG,OAAO,CAAC,oDAAoDM,GAAG,GAA2ClB,GAAG,CAAC,SAACqC;eAAM;YAACA,EAAEnD,GAAG;YAAEmD,EAAEtB,GAAG;SAAC;;AAChJ;AAEA,uFAAuF;AACvF,uFAAuF;AACvF,2FAA2F;AAC3F,2FAA2F;AAC3F,gFAAgF;AAChF,IAAMuB,mBAAmB,IAAIC;AAE7B,SAASC,cAAcpB,KAAqB,EAAEvC,IAAY;IACxD,IAAI0B,MAAM+B,iBAAiB1C,GAAG,CAACwB;IAC/B,IAAI,CAACb,KAAK;QACRA,MAAM,IAAIpC;QACVmE,iBAAiB/B,GAAG,CAACa,OAAOb;IAC9B;IACAA,IAAI/B,GAAG,CAACK;AACV;AAEA,qFAAqF;AACrF,IAAM4D,eAAe,IAAIF;AACzB,SAASG,YAAYtB,KAAqB;IACxC,IAAIb,MAAMkC,aAAa7C,GAAG,CAACwB;IAC3B,IAAI,CAACb,KAAK;QACRA,MAAM,IAAIpC,IAAIiD,MAAMI,QAAQ;QAC5BiB,aAAalC,GAAG,CAACa,OAAOb;IAC1B;IACA,OAAOA;AACT;AAEO,IAAMzC,QAAiB;IAC5B6E,MAAM;IACNC,QAAAA,SAAAA,OAAOnC,EAAE;QACPA,GAAGoC,IAAI,CAAC;QACRpC,GAAGoC,IAAI,CAAC;QACRpC,GAAGoC,IAAI,CAAC;IACV;IACA9E,SAAAA;IACA,gFAAgF;IAChF,uFAAuF;IACvF,kFAAkF;IAClF+E,QAAAA,SAAAA,OAAOrC,EAAE,EAAE5B,IAAI,EAAEuC,KAAK;QACpB,IAAI,CAACsB,YAAYtB,OAAOzB,GAAG,CAACd,OAAO;QACnC,IAAMkE,SAAStC,GAAGG,OAAO,CAAC,mCAAmCI,GAAG,CAACnC;QACjE,IAAImE,OAAOD,OAAOE,OAAO,IAAI,GAAGT,cAAcpB,OAAOvC;IACvD;IACAqE,OAAAA,SAAAA,MAAMzC,EAAE,EAAE5B,IAAI,EAAEsE,SAAS,EAAE/B,KAAK;QAC9B,IAAMlD,UAAUiF;QAChB,gFAAgF;QAChF,IAAIC;QACJ,IAAIlF,QAAQ6D,MAAM,KAAK,GAAG;YACxBqB,QAAQ3C,GAAGG,OAAO,CAAC,mCAAmCI,GAAG,CAACnC;QAC5D,OAAO;gBAEG4B;YADR,IAAMyB,eAAehE,QAAQ8B,GAAG,CAAC;uBAAM;eAAKR,IAAI,CAAC;YACjD4D,QAAQ3C,CAAAA,cAAAA,GAAGG,OAAO,CAAC,AAAC,sDAAkE,OAAbsB,cAAa,OAAIlB,GAAG,OAArFP,aAAAA;gBAAsF5B;aAAiB,CAAvG4B,OAA4F,qBAAGvC;QACzG;QACA,IAAI8E,OAAOI,MAAMH,OAAO,IAAI,GAAGT,cAAcpB,OAAOvC;QACpD,oFAAoF;QACpF,sEAAsE;QACtE,IAAMwE,SAAS5C,GAAGG,OAAO,CAAC;YACrB,kCAAA,2BAAA;;YAAL,QAAK,YAAgB1C,4BAAhB,SAAA,6BAAA,QAAA,yBAAA;gBAAA,IAAMI,SAAN;gBAAyB+E,OAAOrC,GAAG,CAACnC,MAAMP,QAAQM,QAAQF,YAAYJ;;;YAAtE;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;IACP;IACAgF,gBAAAA,SAAAA,eAAe7C,EAAE,EAAEW,KAAK;;YAIEkB;QAHxB,mFAAmF;QACnF,+EAA+E;QAC/E,iEAAiE;QACjE,IAAMiB,iBAAiB,UAACjB,wBAAAA,iBAAiB1C,GAAG,CAACwB,oBAArBkB,4CAAAA,sBAA6BkB,IAAI,uCAAI,KAAK;QAElE,IAAMC,QAAQrC,MAAMe,QAAQ,CAACJ,MAAM,GAAGX,MAAMI,QAAQ,CAACO,MAAM;QAC3D,oFAAoF;QACpF,oFAAoF;QACpF,uFAAuF;QACvF,gCAAgC;QAChC,IAAM2B,QAAQtC,MAAMtB,KAAK,CAACiC,MAAM,KAAK,KAAK0B,QAAQ,MAAMrC,MAAMtB,KAAK,CAACiC,MAAM;QAE1E,IAAM4B,aAAaD,QAAQzC,WAAWR,IAAIW,MAAMtB,KAAK,IAAIqB,mBAAmBV,IAAIW;QAChFA,MAAMwC,YAAY,GAAGD,cAAcJ;IACrC;AACF"}

package/dist/cjs/features/rank.js

@@ -64,7 +64,14 @@ var rank = {
             return c.name === '_rank';
         })) db.exec('ALTER TABLE frontmatter ADD COLUMN "_rank" REAL');
     },
-    afterReconcile: function afterReconcile(db) {
+    afterReconcile: function afterReconcile(db, delta) {
+        // links.afterReconcile runs first (registry order) and sets delta.linksChanged;
+        // `undefined` (links off, or no opinion) recomputes, same as always -- only an
+        // explicit `false` allows a skip. Rank also depends on the node set (every note is a
+        // PageRank node, edges or not), so any add or vanish recomputes even with edges
+        // untouched -- otherwise a new linkless note keeps _rank NULL and every other rank
+        // stays normalized over the old count.
+        if (delta.linksChanged === false && delta.added.length === 0 && delta.vanished.length === 0) return;
         var nodes = db.prepare('SELECT "path" FROM frontmatter').all().map(function(r) {
             return r.path;
         });

package/dist/cjs/features/rank.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/rank.ts"],"sourcesContent":["import { pagerank } from '../graph.ts';\nimport { linkEdges } from './links.ts';\nimport type { Feature } from './types.ts';\n\n// frontmatter._rank: PageRank over resolved links -- a static \"how load-bearing is this\n// note\" prior. Depends on links (enforced in config.featureEnabled).\n\nexport const rank: Feature = {\n  name: 'rank',\n  schema(db) {\n    const columns = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n    if (!columns.some((c) => c.name === '_rank')) db.exec('ALTER TABLE frontmatter ADD COLUMN \"_rank\" REAL');\n  },\n  afterReconcile(db) {\n    const nodes = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n    const ranks = pagerank(nodes, linkEdges(db));\n    const update = db.prepare('UPDATE frontmatter SET \"_rank\" = ? WHERE \"path\" = ?');\n    for (const [path, value] of ranks) update.run(value, path);\n  },\n};\n"],"names":["rank","name","schema","db","columns","prepare","all","some","c","exec","afterReconcile","nodes","map","r","path","ranks","pagerank","linkEdges","update","value","run"],"mappings":";;;;+BAOaA;;;eAAAA;;;uBAPY;uBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAMnB,IAAMA,OAAgB;IAC3BC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACP,IAAMC,UAAUD,GAAGE,OAAO,CAAC,kCAAkCC,GAAG;QAChE,IAAI,CAACF,QAAQG,IAAI,CAAC,SAACC;mBAAMA,EAAEP,IAAI,KAAK;YAAUE,GAAGM,IAAI,CAAC;IACxD;IACAC,gBAAAA,SAAAA,eAAeP,EAAE;QACf,IAAMQ,QAAQ,AAACR,GAAGE,OAAO,CAAC,kCAAkCC,GAAG,GAA+BM,GAAG,CAAC,SAACC;mBAAMA,EAAEC,IAAI;;QAC/G,IAAMC,QAAQC,IAAAA,iBAAQ,EAACL,OAAOM,IAAAA,kBAAS,EAACd;QACxC,IAAMe,SAASf,GAAGE,OAAO,CAAC;YACrB,kCAAA,2BAAA;;YAAL,QAAK,YAAuBU,0BAAvB,SAAA,6BAAA,QAAA,yBAAA;gBAAA,mCAAA,iBAAOD,uBAAMK;gBAAiBD,OAAOE,GAAG,CAACD,OAAOL;;;YAAhD;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;IACP;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/rank.ts"],"sourcesContent":["import { pagerank } from '../graph.ts';\nimport { linkEdges } from './links.ts';\nimport type { Feature } from './types.ts';\n\n// frontmatter._rank: PageRank over resolved links -- a static \"how load-bearing is this\n// note\" prior. Depends on links (enforced in config.featureEnabled).\n\nexport const rank: Feature = {\n  name: 'rank',\n  schema(db) {\n    const columns = db.prepare('PRAGMA table_info(frontmatter)').all() as Array<{ name: string }>;\n    if (!columns.some((c) => c.name === '_rank')) db.exec('ALTER TABLE frontmatter ADD COLUMN \"_rank\" REAL');\n  },\n  afterReconcile(db, delta) {\n    // links.afterReconcile runs first (registry order) and sets delta.linksChanged;\n    // `undefined` (links off, or no opinion) recomputes, same as always -- only an\n    // explicit `false` allows a skip. Rank also depends on the node set (every note is a\n    // PageRank node, edges or not), so any add or vanish recomputes even with edges\n    // untouched -- otherwise a new linkless note keeps _rank NULL and every other rank\n    // stays normalized over the old count.\n    if (delta.linksChanged === false && delta.added.length === 0 && delta.vanished.length === 0) return;\n    const nodes = (db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path);\n    const ranks = pagerank(nodes, linkEdges(db));\n    const update = db.prepare('UPDATE frontmatter SET \"_rank\" = ? WHERE \"path\" = ?');\n    for (const [path, value] of ranks) update.run(value, path);\n  },\n};\n"],"names":["rank","name","schema","db","columns","prepare","all","some","c","exec","afterReconcile","delta","linksChanged","added","length","vanished","nodes","map","r","path","ranks","pagerank","linkEdges","update","value","run"],"mappings":";;;;+BAOaA;;;eAAAA;;;uBAPY;uBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAMnB,IAAMA,OAAgB;IAC3BC,MAAM;IACNC,QAAAA,SAAAA,OAAOC,EAAE;QACP,IAAMC,UAAUD,GAAGE,OAAO,CAAC,kCAAkCC,GAAG;QAChE,IAAI,CAACF,QAAQG,IAAI,CAAC,SAACC;mBAAMA,EAAEP,IAAI,KAAK;YAAUE,GAAGM,IAAI,CAAC;IACxD;IACAC,gBAAAA,SAAAA,eAAeP,EAAE,EAAEQ,KAAK;QACtB,gFAAgF;QAChF,+EAA+E;QAC/E,qFAAqF;QACrF,gFAAgF;QAChF,mFAAmF;QACnF,uCAAuC;QACvC,IAAIA,MAAMC,YAAY,KAAK,SAASD,MAAME,KAAK,CAACC,MAAM,KAAK,KAAKH,MAAMI,QAAQ,CAACD,MAAM,KAAK,GAAG;QAC7F,IAAME,QAAQ,AAACb,GAAGE,OAAO,CAAC,kCAAkCC,GAAG,GAA+BW,GAAG,CAAC,SAACC;mBAAMA,EAAEC,IAAI;;QAC/G,IAAMC,QAAQC,IAAAA,iBAAQ,EAACL,OAAOM,IAAAA,kBAAS,EAACnB;QACxC,IAAMoB,SAASpB,GAAGE,OAAO,CAAC;YACrB,kCAAA,2BAAA;;YAAL,QAAK,YAAuBe,0BAAvB,SAAA,6BAAA,QAAA,yBAAA;gBAAA,mCAAA,iBAAOD,uBAAMK;gBAAiBD,OAAOE,GAAG,CAACD,OAAOL;;;YAAhD;YAAA;;;qBAAA,6BAAA;oBAAA;;;oBAAA;0BAAA;;;;IACP;AACF"}

package/dist/cjs/features/types.d.cts

@@ -1,6 +1,13 @@
 import type { DatabaseSync } from 'node:sqlite';
 import type { FeatureName } from '../config.js';
 import type { FileStat } from '../scan.js';
+export interface ReconcileDelta {
+    files: FileStat[];
+    reparsed: string[];
+    added: string[];
+    vanished: string[];
+    linksChanged?: boolean;
+}
 export interface Feature {
     name: FeatureName;
     schema(db: DatabaseSync): void;
@@ -8,7 +15,7 @@ export interface Feature {
         title: string;
         summary: string;
     }): unknown;
-    remove?(db: DatabaseSync, path: string): void;
-    store?(db: DatabaseSync, path: string, extracted: unknown): void;
-    afterReconcile?(db: DatabaseSync, files: FileStat[]): void;
+    remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;
+    store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;
+    afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;
 }

package/dist/cjs/features/types.d.ts

@@ -1,6 +1,13 @@
 import type { DatabaseSync } from 'node:sqlite';
 import type { FeatureName } from '../config.js';
 import type { FileStat } from '../scan.js';
+export interface ReconcileDelta {
+    files: FileStat[];
+    reparsed: string[];
+    added: string[];
+    vanished: string[];
+    linksChanged?: boolean;
+}
 export interface Feature {
     name: FeatureName;
     schema(db: DatabaseSync): void;
@@ -8,7 +15,7 @@ export interface Feature {
         title: string;
         summary: string;
     }): unknown;
-    remove?(db: DatabaseSync, path: string): void;
-    store?(db: DatabaseSync, path: string, extracted: unknown): void;
-    afterReconcile?(db: DatabaseSync, files: FileStat[]): void;
+    remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;
+    store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;
+    afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;
 }

package/dist/cjs/index.d.cts

@@ -1,4 +1,6 @@
-export type { Config, EmbedConfig, FeatureName, ResolvedConfig } from './config.js';
+export type { FindOptions, Peek, TreeMap } from './commands.js';
+export { find, mapTree, peek } from './commands.js';
+export type { Config, EmbedConfig, FeatureName, ResolvedConfig, SavedFind } from './config.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export type { OpenResult } from './db.js';
 export { open, rebuild } from './db.js';
@@ -6,7 +8,5 @@ export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Row } from './output.js';
 export { printRows } from './output.js';
-export type { FindOptions, Peek, TreeMap } from './verbs.js';
-export { find, mapTree, peek } from './verbs.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/cjs/index.d.ts

@@ -1,4 +1,6 @@
-export type { Config, EmbedConfig, FeatureName, ResolvedConfig } from './config.js';
+export type { FindOptions, Peek, TreeMap } from './commands.js';
+export { find, mapTree, peek } from './commands.js';
+export type { Config, EmbedConfig, FeatureName, ResolvedConfig, SavedFind } from './config.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export type { OpenResult } from './db.js';
 export { open, rebuild } from './db.js';
@@ -6,7 +8,5 @@ export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Row } from './output.js';
 export { printRows } from './output.js';
-export type { FindOptions, Peek, TreeMap } from './verbs.js';
-export { find, mapTree, peek } from './verbs.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';