sensemaking 0.8.0 → 0.9.0

package/dist/esm/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  store(db, path, extracted) {\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic expansion needs features.embed in sense.config.json (e.g. \"features\": { \"embed\": true })');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0 }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first --semantic query hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + BATCH, jobs.length));\n  }\n  report.finish();\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\n// Returns the cosine similarity alongside each candidate: stored and query vectors are both\n// L2-normalised before quantisation, so the dot product is cosine in [-1, 1]. `find` surfaces\n// it because the fused rank score cannot express match quality -- nearest-neighbour search\n// always returns a nearest neighbour, so without a magnitude an agent cannot tell a real hit\n// from the best of a bad lot.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: Math.round(b.score * 1000) / 1000 }));\n}\n"],"names":["existsSync","mkdirSync","readFileSync","renameSync","writeFileSync","homedir","join","embedConfig","SenseError","progress","parseFile","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","chunks","forEach","start","end","text","slice","trim","startLine","endLine","embed","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","fetchToFile","url","dest","fetch","then","res","ok","status","Buffer","from","arrayBuffer","staticProvider","model","tokenizerJson","dir","replace","recursive","file","console","error","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","Tokenizer","tok","unkId","vocab","unk_token","one","ids","encode","add_special_tokens","id","isInteger","v","off","d","norm","Math","sqrt","texts","map","apiProvider","keyEnv","base","headers","key","process","env","undefined","authorization","post","method","body","stringify","input","json","data","embedding","providers","Map","getProvider","cfg","e","sig","type","p","get","set","toStore","full","int8","scale","max","abs","embedPending","baseDir","provider","dirty","all","storeDims","min","byPath","row","list","chunk","jobs","chunkIdxs","relPath","absPath","mtimeMs","size","doc","update","report","batch","vectors","j","job","q","Int8Array","round","err","tick","finish","semanticCandidates","terms","match","t","includes","qv","rows","best","vector","byteLength","dot","score","existing","start_line","end_line","sort","a","b","similarity"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,EAAEC,YAAY,EAAEC,UAAU,EAAEC,aAAa,QAAQ,UAAU;AACzF,SAASC,OAAO,QAAQ,UAAU;AAClC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,WAAW,QAAQ,eAAe;AAC3C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,iBAAiB;AAC1C,SAASC,SAAS,QAAQ,aAAa;AAGvC,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,MAAMC,aAAa;AACnB,MAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,MAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,MAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,MAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;WAAMA;KAAO,GAAGA;IAC5E,MAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAASvB,IAAI,CAAC;IACrE,MAAMwB,SAAkB,EAAE;IAC1BN,OAAOO,OAAO,CAAC,CAACC,OAAOZ;QACrB,MAAMa,MAAMb,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,MAAMa,OAAOlB,MACVmB,KAAK,CAACH,QAAQ,GAAGC,KACjB3B,IAAI,CAAC,MACL8B,IAAI;QACP,IAAIF,KAAKb,MAAM,GAAG,GAAGS,OAAOP,IAAI,CAAC;YAAEc,WAAWL;YAAOM,SAASL;YAAKC,MAAMT,SAAS,GAAGA,OAAO,EAAE,EAAES,MAAM,GAAGA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEA,OAAO,MAAMS,QAAiB;IAC5BC,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAC1K;IACAC,SAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACAG,OAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,MAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBpB,OAAO,CAAC,CAACsB,GAAGC,MAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEhB,SAAS,EAAEgB,EAAEf,OAAO;IACzF;AACF,EAAE;AAEF,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASiB,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,OAAOC;QAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIrD,WAAW,eAAe,CAAC,uBAAuB,EAAEgD,IAAI,SAAS,EAAEI,IAAIE,MAAM,EAAE;QACtG1D,cAAc,GAAGqD,KAAK,KAAK,CAAC,EAAEM,OAAOC,IAAI,CAAC,MAAMJ,IAAIK,WAAW;QAC/D9D,WAAW,GAAGsD,KAAK,KAAK,CAAC,EAAEA;IAC7B;AACF;AAEA,eAAeS,eAAeC,KAAa;;QA2BEC,sBAA7BA,4BAAAA;IA1Bd,IAAIC,MAAMF;IACV,IAAI,CAACnE,WAAWM,KAAK+D,KAAK,uBAAuB;QAC/CA,MAAM/D,KAAKD,WAAW,UAAU,eAAe,UAAU8D,MAAMG,OAAO,CAAC,OAAO;QAC9ErE,UAAUoE,KAAK;YAAEE,WAAW;QAAK;QACjC,KAAK,MAAMC,QAAQ;YAAC;YAAqB;SAAiB,CAAE;YAC1D,IAAI,CAACxE,WAAWM,KAAK+D,KAAKG,QAAQ;gBAChCC,QAAQC,KAAK,CAAC,CAAC,SAAS,EAAEP,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEH,IAAI,0BAA0B,CAAC;gBAC/E,MAAMd,YAAY,CAAC,uBAAuB,EAAEY,MAAM,cAAc,EAAEK,MAAM,EAAElE,KAAK+D,KAAKG;YACtF;QACF;IACF;IAEA,MAAM1D,MAAMZ,aAAaI,KAAK+D,KAAK;IACnC,MAAMM,YAAYC,OAAO9D,IAAI+D,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAAClE,IAAImE,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIhF,WAAW,eAAe,GAAG2D,MAAM,oCAAoC,CAAC;IAC1H,MAAMsB,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAY9E,IAAI+E,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAalF,IAAImF,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAalF,IAAImF,MAAM,CAAC9D,KAAK,CAACyD,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAMtB,gBAAgBW,KAAKC,KAAK,CAAC9E,aAAaI,KAAK+D,KAAK,mBAAmB;IAC3E,oFAAoF;IACpF,MAAM,EAAE6B,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC;IACnC,MAAMC,MAAM,IAAID,UAAU9B,eAAe,CAAC;IAC1C,MAAMgC,iBAAQhC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBiC,KAAK,cAA1BjC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBkC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIrE,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMsE,MAAM,AAACL,IAAIM,MAAM,CAACvE,MAAM;YAAEwE,oBAAoB;QAAM,GAAGF,GAAG,CAAc5E,MAAM,CAAC,CAAC+E,KAAO/B,OAAOgC,SAAS,CAACD,OAAOA,OAAOP;QAC5H,MAAMS,IAAI,IAAIb,aAAaN;QAC3B,IAAIc,IAAInF,MAAM,KAAK,GAAG,OAAOwF;QAC7B,KAAK,MAAMF,MAAMH,IAAK;YACpB,MAAMM,MAAMH,KAAKjB;YACjB,IAAK,IAAIqB,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIhB,MAAM,CAACe,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIP,IAAInF,MAAM;YAClB2F,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAExC,OAAO;QAAEuB;QAAMnD,OAAO,OAAO4E,QAAUA,MAAMC,GAAG,CAACb;IAAK;AAC/E;AAEA,gFAAgF;AAEhF,eAAec,YAAYlD,KAAa,EAAEX,GAAuB,EAAE8D,MAA0B;IAC3F,IAAI,CAAC9D,KAAK,MAAM,IAAIhD,WAAW,eAAe;IAC9C,MAAM+G,OAAO/D,IAAIc,OAAO,CAAC,QAAQ;IACjC,MAAMkD,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASI,QAAQC,GAAG,CAACL,OAAO,GAAGM;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKX,KAAe;QACjC,MAAMvD,MAAM,MAAMF,MAAM,GAAG6D,KAAK,WAAW,CAAC,EAAE;YAAEQ,QAAQ;YAAQP;YAASQ,MAAMjD,KAAKkD,SAAS,CAAC;gBAAE9D;gBAAO+D,OAAOf;YAAM;QAAG;QACvH,IAAI,CAACvD,IAAIC,EAAE,EAAE,MAAM,IAAIrD,WAAW,eAAe,GAAG+G,KAAK,oBAAoB,EAAE3D,IAAIE,MAAM,EAAE;QAC3F,MAAMkE,OAAQ,MAAMpE,IAAIuE,IAAI;QAC5B,OAAOH,KAAKI,IAAI,CAAChB,GAAG,CAAC,CAACL,IAAMf,aAAahC,IAAI,CAAC+C,EAAEsB,SAAS;IAC3D;IAEA,MAAM3C,OAAO,AAAC,CAAA,MAAMoC,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACzG,MAAM;IACxD,OAAO;QAAEsF,IAAI,CAAC,IAAI,EAAEY,KAAK,CAAC,EAAEpD,OAAO;QAAEuB;QAAMnD,OAAOuF;IAAK;AACzD;AAEA,MAAMQ,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,MAAMA,IAAInI,YAAYkI;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIlI,WAAW,kBAAkB;IAC/C,MAAMmI,MAAM,GAAGD,EAAEE,IAAI,CAAC,CAAC,EAAEF,EAAEvE,KAAK,CAAC,CAAC,GAAEuE,SAAAA,EAAElF,GAAG,cAALkF,oBAAAA,SAAS,IAAI;IACjD,IAAIG,IAAIP,UAAUQ,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIH,EAAEE,IAAI,KAAK,QAAQvB,YAAYqB,EAAEvE,KAAK,EAAEuE,EAAElF,GAAG,EAAEkF,EAAEjB,GAAG,IAAIvD,eAAewE,EAAEvE,KAAK;QAClFmE,UAAUS,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAEvD,IAAY,EAAEwD,IAAa;IAC9D,MAAMrC,IAAI,IAAIb,aAAaN;IAC3B,IAAIsB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKC,QAAQiC,IAAI,CAAClC,EAAE,GAAGkC,IAAI,CAAClC,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,GAAGkC,IAAI,CAAClC,EAAE,GAAGC;IAChD,IAAI,CAACkC,MAAM,OAAO;QAAErC;QAAGsC,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIrC,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKqC,MAAMnC,KAAKmC,GAAG,CAACA,KAAKnC,KAAKoC,GAAG,CAACxC,CAAC,CAACE,EAAE;IAChE,OAAO;QAAEF;QAAGsC,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAa5G,EAAgB,EAAE+F,GAAW,EAAEc,OAAe;IAC/E,MAAMC,WAAW,MAAMhB,YAAYC,MAAM,kDAAkD;IAC3F,MAAMgB,QAAQ/G,GAAGM,OAAO,CAAC,oFAAoF0G,GAAG;IAChH,IAAID,MAAMpI,MAAM,KAAK,GAAG;IACxB,MAAMsI,YAAY1C,KAAK2C,GAAG,CAACjJ,YAAY6I,SAAS9D,IAAI;IAEpD,MAAMmE,SAAS,IAAItB;IACnB,KAAK,MAAMuB,OAAOL,MAAO;YACVI;QAAb,MAAME,QAAOF,cAAAA,OAAOf,GAAG,CAACgB,IAAI/G,IAAI,eAAnB8G,yBAAAA,cAAwB,EAAE;QACvCE,KAAKxI,IAAI,CAACuI,IAAIE,KAAK;QACnBH,OAAOd,GAAG,CAACe,IAAI/G,IAAI,EAAEgH;IACvB;IAEA,MAAME,OAA6D,EAAE;IACrE,KAAK,MAAM,CAAClH,MAAMmH,UAAU,IAAIL,OAAQ;QACtC,IAAI/H;QACJ,IAAI;YACFA,SAASpB,UAAU;gBAAEyJ,SAASpH;gBAAMqH,SAAS9J,KAAKiJ,SAASxG;gBAAOsH,SAAS;gBAAGC,MAAM;YAAE,GAAG;gBAAC/H;aAAM,EAAEgI,GAAG,CAACpH,SAAS,CAACZ,KAAK;QACvH,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMe,OAAO4G,UAAW,IAAIpI,MAAM,CAACwB,IAAI,EAAE2G,KAAK1I,IAAI,CAAC;YAAEwB;YAAMiH,OAAO1G;YAAKpB,MAAMJ,MAAM,CAACwB,IAAI,CAACpB,IAAI;QAAC;IACrG;IAEA,MAAMsI,SAAS9H,GAAGM,OAAO,CAAC;IAC1B,wFAAwF;IACxF,6DAA6D;IAC7D,MAAMyH,SAAShK,SAAS,oBAAoBwJ,KAAK5I,MAAM;IACvD,IAAK,IAAID,IAAI,GAAGA,IAAI6I,KAAK5I,MAAM,EAAED,KAAKR,MAAO;QAC3C,MAAM8J,QAAQT,KAAK9H,KAAK,CAACf,GAAGA,IAAIR;QAChC,MAAM+J,UAAU,MAAMnB,SAASjH,KAAK,CAACmI,MAAMtD,GAAG,CAAC,CAACwD,IAAMA,EAAE1I,IAAI;QAC5DQ,GAAGC,IAAI,CAAC;QACR,IAAI;YACF+H,MAAM3I,OAAO,CAAC,CAAC8I,KAAKD;gBAClB,MAAM,EAAE/D,CAAC,EAAEsC,KAAK,EAAE,GAAGH,QAAQ2B,OAAO,CAACC,EAAE,EAAEjB,WAAW;gBACpD,MAAMmB,IAAI,IAAIC,UAAUpB;gBACxB,IAAK,IAAI5C,IAAI,GAAGA,IAAI4C,WAAW5C,IAAK+D,CAAC,CAAC/D,EAAE,GAAGE,KAAK+D,KAAK,CAACnE,CAAC,CAACE,EAAE,GAAGoC;gBAC7DqB,OAAOvH,GAAG,CAACkG,OAAOpF,OAAOC,IAAI,CAAC8G,EAAE7E,MAAM,GAAG4E,IAAI9H,IAAI,EAAE8H,IAAIb,KAAK;YAC9D;YACAtH,GAAGC,IAAI,CAAC;QACV,EAAE,OAAOsI,KAAK;YACZvI,GAAGC,IAAI,CAAC;YACR,MAAMsI;QACR;QACAR,OAAOS,IAAI,CAACjE,KAAK2C,GAAG,CAACxI,IAAIR,OAAOqJ,KAAK5I,MAAM;IAC7C;IACAoJ,OAAOU,MAAM;AACf;AAEA,oFAAoF;AACpF,wFAAwF;AACxF,oBAAoB;AACpB,4FAA4F;AAC5F,8FAA8F;AAC9F,2FAA2F;AAC3F,6FAA6F;AAC7F,8BAA8B;AAC9B,OAAO,eAAeC,mBAAmB1I,EAAgB,EAAE+F,GAAW,EAAE4C,KAAa,EAAE3H,MAAa;QAOpF2H;IANd,MAAM9B,UAAU,AAACd,IAAgCc,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAI/I,WAAW,eAAe;IAClD,MAAM8I,aAAa5G,IAAI+F,KAAKc;IAE5B,MAAMC,WAAW,MAAMhB,YAAYC;IACnC,MAAMkB,YAAY1C,KAAK2C,GAAG,CAACjJ,YAAY6I,SAAS9D,IAAI;IACpD,MAAMxD,OAAO,EAACmJ,eAAAA,MAAMC,KAAK,CAAC,gCAAZD,0BAAAA,eAAkC,EAAE,EAAEzJ,MAAM,CAAC,CAAC2J,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIjL,IAAI,CAAC;IAClH,MAAM,EAAEuG,GAAG4E,EAAE,EAAE,GAAGzC,QAAQ,AAAC,CAAA,MAAMQ,SAASjH,KAAK,CAAC;QAACL;KAAK,CAAA,CAAE,CAAC,EAAE,EAAEyH,WAAW;IAExE,MAAM+B,OAAOhJ,GAAGM,OAAO,CAAC,+FAA+F0G,GAAG;IAQ1H,MAAMiC,OAAO,IAAIpD;IACjB,KAAK,MAAMuB,OAAO4B,KAAM;QACtB,MAAMZ,IAAI,IAAIC,UAAUjB,IAAI8B,MAAM,CAAC3F,MAAM,EAAE6D,IAAI8B,MAAM,CAAC/F,UAAU,EAAEoB,KAAK2C,GAAG,CAACD,WAAWG,IAAI8B,MAAM,CAACC,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAI/E,IAAI,GAAGA,IAAI+D,EAAEzJ,MAAM,EAAE0F,IAAK+E,OAAOhB,CAAC,CAAC/D,EAAE,GAAG0E,EAAE,CAAC1E,EAAE;QACtD,MAAMgF,QAAQD,MAAMhC,IAAIX,KAAK;QAC7B,MAAM6C,WAAWL,KAAK7C,GAAG,CAACgB,IAAI/G,IAAI;QAClC,IAAI,CAACiJ,YAAYD,QAAQC,SAASD,KAAK,EAAEJ,KAAK5C,GAAG,CAACe,IAAI/G,IAAI,EAAE;YAAEgJ;YAAO/K,OAAO,CAAC,CAAC,EAAE8I,IAAImC,UAAU,CAAC,CAAC,EAAEnC,IAAIoC,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIP,KAAKtG,OAAO;KAAG,CACvB8G,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACN,KAAK,GAAGK,CAAC,CAAC,EAAE,CAACL,KAAK,EACtC5J,KAAK,CAAC,GAAGuB,QACT0D,GAAG,CAAC,CAAC,CAACrE,MAAMsJ,EAAE,GAAM,CAAA;YAAEtJ;YAAM/B,OAAOqL,EAAErL,KAAK;YAAEsL,YAAYrF,KAAK+D,KAAK,CAACqB,EAAEN,KAAK,GAAG,QAAQ;QAAK,CAAA;AAC/F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config.ts';\nimport { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport type { Feature } from './types.ts';\n\n// embeddings(path, chunk, start_line, end_line, scale, vector): heading-based chunks,\n// int8 vectors with a per-vector dequantization scale, vector NULL = not yet embedded.\n// Reconcile stores dirty rows inside its transaction; embedding tops up on the next\n// semantic query (embedPending), so staleness costs recall, never correctness.\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\nconst BATCH = 64;\n\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  embed(texts: string[]): Promise<Float32Array[]>;\n}\n\ninterface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic chunker used at reconcile (line ranges stored) and at embed time (text\n// re-derived from the file -- chunk text is never stored). Heading-delimited with the\n// preamble kept; whole file when no headings. The title/summary prefix mirrors the\n// bm25 column weighting.\nfunction chunksOf(raw: string, search?: { title: string; summary: string }): Chunk[] {\n  const lines = raw.split('\\n');\n  const starts: number[] = [];\n  let inFence = false;\n  for (let i = 0; i < lines.length; i++) {\n    if (/^(```|~~~)/.test(lines[i])) {\n      inFence = !inFence;\n      continue;\n    }\n    if (!inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    if (text.length > 0) chunks.push({ startLine: start, endLine: end, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, _body, search) {\n    return chunksOf(raw, search);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  // A doc covered only by semantic-false presets never had extract() run for it (db.ts's\n  // per-file filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.\n  store(db, path, extracted) {\n    if (!extracted) return;\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n};\n\n// --- static type: Model2Vec safetensors + pure-JS tokenizer, model cached in ~/.cache ---\n// Encode convention from model2vec/model.py: no special tokens, drop unk ids, mean-pool,\n// L2-normalize.\n\nfunction fetchToFile(url: string, dest: string): Promise<void> {\n  return fetch(url).then(async (res) => {\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `model download failed: ${url} -> HTTP ${res.status}`);\n    writeFileSync(`${dest}.part`, Buffer.from(await res.arrayBuffer()));\n    renameSync(`${dest}.part`, dest);\n  });\n}\n\nasync function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir = model;\n  if (!existsSync(join(dir, 'model.safetensors'))) {\n    dir = join(homedir(), '.cache', 'sensemaking', 'models', model.replace(/\\//g, '--'));\n    mkdirSync(dir, { recursive: true });\n    for (const file of ['model.safetensors', 'tokenizer.json']) {\n      if (!existsSync(join(dir, file))) {\n        console.error(`fetching ${model}/${file} into ${dir} (once; delete to refetch)`);\n        await fetchToFile(`https://huggingface.co/${model}/resolve/main/${file}`, join(dir, file));\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  return { id: `static:${model}`, dims, embed: async (texts) => texts.map(one) };\n}\n\n// --- api type: one POST against any OpenAI-compatible /embeddings endpoint ---\n\nasync function apiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'features.embed.type \"api\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetch(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[] }> };\n    return body.data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  return { id: `api:${base}:${model}`, dims, embed: post };\n}\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'semantic search needs at least one preset with vectors on (semantic !== false); every declared preset in sense.config.json has \"semantic\": false');\n  const sig = `${e.type}:${e.model}:${e.url ?? ''}`;\n  let p = providers.get(sig);\n  if (!p) {\n    p = e.type === 'api' ? apiProvider(e.model, e.url, e.key) : staticProvider(e.model);\n    providers.set(sig, p);\n  }\n  return p;\n}\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage.\nfunction toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means it was covered by a semantic-on preset at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first --semantic query hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += BATCH) {\n    const batch = jobs.slice(i, i + BATCH);\n    const vectors = await provider.embed(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + BATCH, jobs.length));\n  }\n  report.finish();\n}\n\n// Top candidates by cosine for a semantic find: best chunk per file, its line range\n// riding along. FTS5 operators in the terms are lexical syntax, not meaning -- stripped\n// before embedding.\n// Returns the cosine similarity alongside each candidate: stored and query vectors are both\n// L2-normalised before quantisation, so the dot product is cosine in [-1, 1]. `find` surfaces\n// it because the fused rank score cannot express match quality -- nearest-neighbour search\n// always returns a nearest neighbour, so without a magnitude an agent cannot tell a real hit\n// from the best of a bad lot.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore((await provider.embed([text]))[0], storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: Math.round(b.score * 1000) / 1000 }));\n}\n"],"names":["existsSync","mkdirSync","readFileSync","renameSync","writeFileSync","homedir","join","embedConfig","SenseError","progress","parseFile","STORE_DIMS","BATCH","chunksOf","raw","search","lines","split","starts","inFence","i","length","test","push","bounds","prefix","title","summary","filter","Boolean","chunks","forEach","start","end","text","slice","trim","startLine","endLine","embed","name","schema","db","exec","extract","_body","remove","path","prepare","run","store","extracted","insert","c","idx","enabledForFile","_cfg","file","fetchToFile","url","dest","fetch","then","res","ok","status","Buffer","from","arrayBuffer","staticProvider","model","tokenizerJson","dir","replace","recursive","console","error","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","Tokenizer","tok","unkId","vocab","unk_token","one","ids","encode","add_special_tokens","id","isInteger","v","off","d","norm","Math","sqrt","texts","map","apiProvider","keyEnv","base","headers","key","process","env","undefined","authorization","post","method","body","stringify","input","json","data","embedding","providers","Map","getProvider","cfg","e","sig","type","p","get","set","toStore","full","int8","scale","max","abs","embedPending","baseDir","provider","dirty","all","storeDims","min","byPath","row","list","chunk","jobs","chunkIdxs","relPath","absPath","mtimeMs","size","presets","doc","update","report","batch","vectors","j","job","q","Int8Array","round","err","tick","finish","semanticCandidates","terms","match","t","includes","qv","rows","best","vector","byteLength","dot","score","existing","start_line","end_line","sort","a","b","similarity"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,EAAEC,YAAY,EAAEC,UAAU,EAAEC,aAAa,QAAQ,UAAU;AACzF,SAASC,OAAO,QAAQ,UAAU;AAClC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,WAAW,QAAQ,eAAe;AAC3C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,QAAQ,QAAQ,iBAAiB;AAC1C,SAASC,SAAS,QAAQ,aAAa;AAGvC,sFAAsF;AACtF,uFAAuF;AACvF,oFAAoF;AACpF,+EAA+E;AAE/E,6EAA6E;AAC7E,yEAAyE;AACzE,MAAMC,aAAa;AACnB,MAAMC,QAAQ;AAcd,uFAAuF;AACvF,sFAAsF;AACtF,mFAAmF;AACnF,yBAAyB;AACzB,SAASC,SAASC,GAAW,EAAEC,MAA2C;IACxE,MAAMC,QAAQF,IAAIG,KAAK,CAAC;IACxB,MAAMC,SAAmB,EAAE;IAC3B,IAAIC,UAAU;IACd,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAI,aAAaE,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;YAC/BD,UAAU,CAACA;YACX;QACF;QACA,IAAI,CAACA,WAAW,YAAYG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAGF,OAAOK,IAAI,CAACH,IAAI;IAC9D;IACA,MAAMI,SAASN,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;WAAMA;KAAO,GAAGA;IAC5E,MAAMO,SAAS;QAACV,mBAAAA,6BAAAA,OAAQW,KAAK;QAAEX,mBAAAA,6BAAAA,OAAQY,OAAO;KAAC,CAACC,MAAM,CAACC,SAASvB,IAAI,CAAC;IACrE,MAAMwB,SAAkB,EAAE;IAC1BN,OAAOO,OAAO,CAAC,CAACC,OAAOZ;QACrB,MAAMa,MAAMb,IAAI,IAAII,OAAOH,MAAM,GAAGG,MAAM,CAACJ,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,MAAMa,OAAOlB,MACVmB,KAAK,CAACH,QAAQ,GAAGC,KACjB3B,IAAI,CAAC,MACL8B,IAAI;QACP,IAAIF,KAAKb,MAAM,GAAG,GAAGS,OAAOP,IAAI,CAAC;YAAEc,WAAWL;YAAOM,SAASL;YAAKC,MAAMT,SAAS,GAAGA,OAAO,EAAE,EAAES,MAAM,GAAGA;QAAK;IAChH;IACA,OAAOJ;AACT;AAEA,OAAO,MAAMS,QAAiB;IAC5BC,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAC1K;IACAC,SAAQ9B,GAAG,EAAE+B,KAAK,EAAE9B,MAAM;QACxB,OAAOF,SAASC,KAAKC;IACvB;IACA+B,QAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACA,uFAAuF;IACvF,4FAA4F;IAC5FG,OAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAI,CAACA,WAAW;QAChB,MAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBpB,OAAO,CAAC,CAACsB,GAAGC,MAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEhB,SAAS,EAAEgB,EAAEf,OAAO;IACzF;IACAiB,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKlB,KAAK;IACnB;AACF,EAAE;AAEF,2FAA2F;AAC3F,yFAAyF;AACzF,gBAAgB;AAEhB,SAASmB,YAAYC,GAAW,EAAEC,IAAY;IAC5C,OAAOC,MAAMF,KAAKG,IAAI,CAAC,OAAOC;QAC5B,IAAI,CAACA,IAAIC,EAAE,EAAE,MAAM,IAAIxD,WAAW,eAAe,CAAC,uBAAuB,EAAEmD,IAAI,SAAS,EAAEI,IAAIE,MAAM,EAAE;QACtG7D,cAAc,GAAGwD,KAAK,KAAK,CAAC,EAAEM,OAAOC,IAAI,CAAC,MAAMJ,IAAIK,WAAW;QAC/DjE,WAAW,GAAGyD,KAAK,KAAK,CAAC,EAAEA;IAC7B;AACF;AAEA,eAAeS,eAAeC,KAAa;;QA2BEC,sBAA7BA,4BAAAA;IA1Bd,IAAIC,MAAMF;IACV,IAAI,CAACtE,WAAWM,KAAKkE,KAAK,uBAAuB;QAC/CA,MAAMlE,KAAKD,WAAW,UAAU,eAAe,UAAUiE,MAAMG,OAAO,CAAC,OAAO;QAC9ExE,UAAUuE,KAAK;YAAEE,WAAW;QAAK;QACjC,KAAK,MAAMjB,QAAQ;YAAC;YAAqB;SAAiB,CAAE;YAC1D,IAAI,CAACzD,WAAWM,KAAKkE,KAAKf,QAAQ;gBAChCkB,QAAQC,KAAK,CAAC,CAAC,SAAS,EAAEN,MAAM,CAAC,EAAEb,KAAK,MAAM,EAAEe,IAAI,0BAA0B,CAAC;gBAC/E,MAAMd,YAAY,CAAC,uBAAuB,EAAEY,MAAM,cAAc,EAAEb,MAAM,EAAEnD,KAAKkE,KAAKf;YACtF;QACF;IACF;IAEA,MAAM3C,MAAMZ,aAAaI,KAAKkE,KAAK;IACnC,MAAMK,YAAYC,OAAOhE,IAAIiE,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACpE,IAAIqE,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIlF,WAAW,eAAe,GAAG8D,MAAM,oCAAoC,CAAC;IAC1H,MAAMqB,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYhF,IAAIiF,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAapF,IAAIqF,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAapF,IAAIqF,MAAM,CAAChE,KAAK,CAAC2D,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAMrB,gBAAgBU,KAAKC,KAAK,CAAChF,aAAaI,KAAKkE,KAAK,mBAAmB;IAC3E,oFAAoF;IACpF,MAAM,EAAE4B,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC;IACnC,MAAMC,MAAM,IAAID,UAAU7B,eAAe,CAAC;IAC1C,MAAM+B,iBAAQ/B,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBgC,KAAK,cAA1BhC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBiC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIvE,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMwE,MAAM,AAACL,IAAIM,MAAM,CAACzE,MAAM;YAAE0E,oBAAoB;QAAM,GAAGF,GAAG,CAAc9E,MAAM,CAAC,CAACiF,KAAO/B,OAAOgC,SAAS,CAACD,OAAOA,OAAOP;QAC5H,MAAMS,IAAI,IAAIb,aAAaN;QAC3B,IAAIc,IAAIrF,MAAM,KAAK,GAAG,OAAO0F;QAC7B,KAAK,MAAMF,MAAMH,IAAK;YACpB,MAAMM,MAAMH,KAAKjB;YACjB,IAAK,IAAIqB,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIhB,MAAM,CAACe,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIP,IAAIrF,MAAM;YAClB6F,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAEvC,OAAO;QAAEsB;QAAMrD,OAAO,OAAO8E,QAAUA,MAAMC,GAAG,CAACb;IAAK;AAC/E;AAEA,gFAAgF;AAEhF,eAAec,YAAYjD,KAAa,EAAEX,GAAuB,EAAE6D,MAA0B;IAC3F,IAAI,CAAC7D,KAAK,MAAM,IAAInD,WAAW,eAAe;IAC9C,MAAMiH,OAAO9D,IAAIc,OAAO,CAAC,QAAQ;IACjC,MAAMiD,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASI,QAAQC,GAAG,CAACL,OAAO,GAAGM;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKX,KAAe;QACjC,MAAMtD,MAAM,MAAMF,MAAM,GAAG4D,KAAK,WAAW,CAAC,EAAE;YAAEQ,QAAQ;YAAQP;YAASQ,MAAMjD,KAAKkD,SAAS,CAAC;gBAAE7D;gBAAO8D,OAAOf;YAAM;QAAG;QACvH,IAAI,CAACtD,IAAIC,EAAE,EAAE,MAAM,IAAIxD,WAAW,eAAe,GAAGiH,KAAK,oBAAoB,EAAE1D,IAAIE,MAAM,EAAE;QAC3F,MAAMiE,OAAQ,MAAMnE,IAAIsE,IAAI;QAC5B,OAAOH,KAAKI,IAAI,CAAChB,GAAG,CAAC,CAACL,IAAMf,aAAa/B,IAAI,CAAC8C,EAAEsB,SAAS;IAC3D;IAEA,MAAM3C,OAAO,AAAC,CAAA,MAAMoC,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAAC3G,MAAM;IACxD,OAAO;QAAEwF,IAAI,CAAC,IAAI,EAAEY,KAAK,CAAC,EAAEnD,OAAO;QAAEsB;QAAMrD,OAAOyF;IAAK;AACzD;AAEA,MAAMQ,YAAY,IAAIC;AAEtB,SAASC,YAAYC,GAAW;QAGMC;IAFpC,MAAMA,IAAIrI,YAAYoI;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIpI,WAAW,kBAAkB;IAC/C,MAAMqI,MAAM,GAAGD,EAAEE,IAAI,CAAC,CAAC,EAAEF,EAAEtE,KAAK,CAAC,CAAC,GAAEsE,SAAAA,EAAEjF,GAAG,cAALiF,oBAAAA,SAAS,IAAI;IACjD,IAAIG,IAAIP,UAAUQ,GAAG,CAACH;IACtB,IAAI,CAACE,GAAG;QACNA,IAAIH,EAAEE,IAAI,KAAK,QAAQvB,YAAYqB,EAAEtE,KAAK,EAAEsE,EAAEjF,GAAG,EAAEiF,EAAEjB,GAAG,IAAItD,eAAeuE,EAAEtE,KAAK;QAClFkE,UAAUS,GAAG,CAACJ,KAAKE;IACrB;IACA,OAAOA;AACT;AAEA,4EAA4E;AAC5E,SAASG,QAAQC,IAAkB,EAAEvD,IAAY,EAAEwD,IAAa;IAC9D,MAAMrC,IAAI,IAAIb,aAAaN;IAC3B,IAAIsB,OAAO;IACX,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKC,QAAQiC,IAAI,CAAClC,EAAE,GAAGkC,IAAI,CAAClC,EAAE;IACxDC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;IACzB,IAAK,IAAID,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKF,CAAC,CAACE,EAAE,GAAGkC,IAAI,CAAClC,EAAE,GAAGC;IAChD,IAAI,CAACkC,MAAM,OAAO;QAAErC;QAAGsC,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIrC,IAAI,GAAGA,IAAIrB,MAAMqB,IAAKqC,MAAMnC,KAAKmC,GAAG,CAACA,KAAKnC,KAAKoC,GAAG,CAACxC,CAAC,CAACE,EAAE;IAChE,OAAO;QAAEF;QAAGsC,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAa9G,EAAgB,EAAEiG,GAAW,EAAEc,OAAe;IAC/E,MAAMC,WAAW,MAAMhB,YAAYC,MAAM,kDAAkD;IAC3F,MAAMgB,QAAQjH,GAAGM,OAAO,CAAC,oFAAoF4G,GAAG;IAChH,IAAID,MAAMtI,MAAM,KAAK,GAAG;IACxB,MAAMwI,YAAY1C,KAAK2C,GAAG,CAACnJ,YAAY+I,SAAS9D,IAAI;IAEpD,MAAMmE,SAAS,IAAItB;IACnB,KAAK,MAAMuB,OAAOL,MAAO;YACVI;QAAb,MAAME,QAAOF,cAAAA,OAAOf,GAAG,CAACgB,IAAIjH,IAAI,eAAnBgH,yBAAAA,cAAwB,EAAE;QACvCE,KAAK1I,IAAI,CAACyI,IAAIE,KAAK;QACnBH,OAAOd,GAAG,CAACe,IAAIjH,IAAI,EAAEkH;IACvB;IAEA,MAAME,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACpH,MAAMqH,UAAU,IAAIL,OAAQ;QACtC,IAAIjI;QACJ,IAAI;YACF,qFAAqF;YACrF,sFAAsF;YACtFA,SAASpB,UAAU;gBAAE2J,SAAStH;gBAAMuH,SAAShK,KAAKmJ,SAAS1G;gBAAOwH,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAElI,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEmI,GAAG,CAACvH,SAAS,CAACZ,KAAK;QACjJ,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMe,OAAO8G,UAAW,IAAItI,MAAM,CAACwB,IAAI,EAAE6G,KAAK5I,IAAI,CAAC;YAAEwB;YAAMmH,OAAO5G;YAAKpB,MAAMJ,MAAM,CAACwB,IAAI,CAACpB,IAAI;QAAC;IACrG;IAEA,MAAMyI,SAASjI,GAAGM,OAAO,CAAC;IAC1B,wFAAwF;IACxF,6DAA6D;IAC7D,MAAM4H,SAASnK,SAAS,oBAAoB0J,KAAK9I,MAAM;IACvD,IAAK,IAAID,IAAI,GAAGA,IAAI+I,KAAK9I,MAAM,EAAED,KAAKR,MAAO;QAC3C,MAAMiK,QAAQV,KAAKhI,KAAK,CAACf,GAAGA,IAAIR;QAChC,MAAMkK,UAAU,MAAMpB,SAASnH,KAAK,CAACsI,MAAMvD,GAAG,CAAC,CAACyD,IAAMA,EAAE7I,IAAI;QAC5DQ,GAAGC,IAAI,CAAC;QACR,IAAI;YACFkI,MAAM9I,OAAO,CAAC,CAACiJ,KAAKD;gBAClB,MAAM,EAAEhE,CAAC,EAAEsC,KAAK,EAAE,GAAGH,QAAQ4B,OAAO,CAACC,EAAE,EAAElB,WAAW;gBACpD,MAAMoB,IAAI,IAAIC,UAAUrB;gBACxB,IAAK,IAAI5C,IAAI,GAAGA,IAAI4C,WAAW5C,IAAKgE,CAAC,CAAChE,EAAE,GAAGE,KAAKgE,KAAK,CAACpE,CAAC,CAACE,EAAE,GAAGoC;gBAC7DsB,OAAO1H,GAAG,CAACoG,OAAOnF,OAAOC,IAAI,CAAC8G,EAAE9E,MAAM,GAAG6E,IAAIjI,IAAI,EAAEiI,IAAId,KAAK;YAC9D;YACAxH,GAAGC,IAAI,CAAC;QACV,EAAE,OAAOyI,KAAK;YACZ1I,GAAGC,IAAI,CAAC;YACR,MAAMyI;QACR;QACAR,OAAOS,IAAI,CAAClE,KAAK2C,GAAG,CAAC1I,IAAIR,OAAOuJ,KAAK9I,MAAM;IAC7C;IACAuJ,OAAOU,MAAM;AACf;AAEA,oFAAoF;AACpF,wFAAwF;AACxF,oBAAoB;AACpB,4FAA4F;AAC5F,8FAA8F;AAC9F,2FAA2F;AAC3F,6FAA6F;AAC7F,8BAA8B;AAC9B,OAAO,eAAeC,mBAAmB7I,EAAgB,EAAEiG,GAAW,EAAE6C,KAAa,EAAE3H,MAAa;QAOpF2H;IANd,MAAM/B,UAAU,AAACd,IAAgCc,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIjJ,WAAW,eAAe;IAClD,MAAMgJ,aAAa9G,IAAIiG,KAAKc;IAE5B,MAAMC,WAAW,MAAMhB,YAAYC;IACnC,MAAMkB,YAAY1C,KAAK2C,GAAG,CAACnJ,YAAY+I,SAAS9D,IAAI;IACpD,MAAM1D,OAAO,EAACsJ,eAAAA,MAAMC,KAAK,CAAC,gCAAZD,0BAAAA,eAAkC,EAAE,EAAE5J,MAAM,CAAC,CAAC8J,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIpL,IAAI,CAAC;IAClH,MAAM,EAAEyG,GAAG6E,EAAE,EAAE,GAAG1C,QAAQ,AAAC,CAAA,MAAMQ,SAASnH,KAAK,CAAC;QAACL;KAAK,CAAA,CAAE,CAAC,EAAE,EAAE2H,WAAW;IAExE,MAAMgC,OAAOnJ,GAAGM,OAAO,CAAC,+FAA+F4G,GAAG;IAQ1H,MAAMkC,OAAO,IAAIrD;IACjB,KAAK,MAAMuB,OAAO6B,KAAM;QACtB,MAAMZ,IAAI,IAAIC,UAAUlB,IAAI+B,MAAM,CAAC5F,MAAM,EAAE6D,IAAI+B,MAAM,CAAChG,UAAU,EAAEoB,KAAK2C,GAAG,CAACD,WAAWG,IAAI+B,MAAM,CAACC,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIhF,IAAI,GAAGA,IAAIgE,EAAE5J,MAAM,EAAE4F,IAAKgF,OAAOhB,CAAC,CAAChE,EAAE,GAAG2E,EAAE,CAAC3E,EAAE;QACtD,MAAMiF,QAAQD,MAAMjC,IAAIX,KAAK;QAC7B,MAAM8C,WAAWL,KAAK9C,GAAG,CAACgB,IAAIjH,IAAI;QAClC,IAAI,CAACoJ,YAAYD,QAAQC,SAASD,KAAK,EAAEJ,KAAK7C,GAAG,CAACe,IAAIjH,IAAI,EAAE;YAAEmJ;YAAOlL,OAAO,CAAC,CAAC,EAAEgJ,IAAIoC,UAAU,CAAC,CAAC,EAAEpC,IAAIqC,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIP,KAAKvG,OAAO;KAAG,CACvB+G,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACN,KAAK,GAAGK,CAAC,CAAC,EAAE,CAACL,KAAK,EACtC/J,KAAK,CAAC,GAAG0B,QACTyD,GAAG,CAAC,CAAC,CAACvE,MAAMyJ,EAAE,GAAM,CAAA;YAAEzJ;YAAM/B,OAAOwL,EAAExL,KAAK;YAAEyL,YAAYtF,KAAKgE,KAAK,CAACqB,EAAEN,KAAK,GAAG,QAAQ;QAAK,CAAA;AAC/F"}

package/dist/esm/features/types.d.ts

@@ -1,5 +1,5 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { FeatureName } from '../config.js';
+import type { Config, FeatureName } from '../config.js';
 import type { FileStat } from '../scan.js';
 export interface ReconcileDelta {
     files: FileStat[];
@@ -18,4 +18,5 @@ export interface Feature {
     remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;
     store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;
     afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;
+    enabledForFile?(cfg: Config, file: FileStat): boolean;
 }

package/dist/esm/features/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/types.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { FeatureName } from '../config.ts';\nimport type { FileStat } from '../scan.ts';\n\n// What changed this reconcile, so afterReconcile hooks can act on the delta instead of\n// re-reading the whole tree. `reparsed`/`added`/`vanished` are relPaths; `added` is the\n// subset of `reparsed` that are brand-new files (not just modified). One delta object per\n// reconcile: hooks communicate through it (links sets `linksChanged` for rank), and any\n// state keyed on it dies with the reconcile -- including on ROLLBACK.\nexport interface ReconcileDelta {\n  files: FileStat[];\n  reparsed: string[];\n  added: string[];\n  vanished: string[];\n  // Set by the links feature: the resolved edge set may differ from before this reconcile.\n  linksChanged?: boolean;\n}\n\n// A feature owns its schema, per-file extraction/storage, and any whole-tree pass.\n// Adding a feature = one module here + one entry in the registry (index.ts).\nexport interface Feature {\n  name: FeatureName;\n  schema(db: DatabaseSync): void;\n  // Pure, per file: raw is the full file, body the prose after frontmatter,\n  // search the normalized title/summary strings.\n  extract?(raw: string, body: string, search?: { title: string; summary: string }): unknown;\n  remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;\n  store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;\n  // After all rows are current, inside the reconcile transaction (resolution, rank).\n  // Hooks run in registry order and communicate through fields on `delta`.\n  afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;\n}\n"],"names":[],"mappings":"AAkBA,mFAAmF;AACnF,6EAA6E;AAC7E,WAWC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/types.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config, FeatureName } from '../config.ts';\nimport type { FileStat } from '../scan.ts';\n\n// What changed this reconcile, so afterReconcile hooks can act on the delta instead of\n// re-reading the whole tree. `reparsed`/`added`/`vanished` are relPaths; `added` is the\n// subset of `reparsed` that are brand-new files (not just modified). One delta object per\n// reconcile: hooks communicate through it (links sets `linksChanged` for rank), and any\n// state keyed on it dies with the reconcile -- including on ROLLBACK.\nexport interface ReconcileDelta {\n  files: FileStat[];\n  reparsed: string[];\n  added: string[];\n  vanished: string[];\n  // Set by the links feature: the resolved edge set may differ from before this reconcile.\n  linksChanged?: boolean;\n}\n\n// A feature owns its schema, per-file extraction/storage, and any whole-tree pass.\n// Adding a feature = one module here + one entry in the registry (index.ts).\nexport interface Feature {\n  name: FeatureName;\n  schema(db: DatabaseSync): void;\n  // Pure, per file: raw is the full file, body the prose after frontmatter,\n  // search the normalized title/summary strings.\n  extract?(raw: string, body: string, search?: { title: string; summary: string }): unknown;\n  remove?(db: DatabaseSync, path: string, delta: ReconcileDelta): void;\n  store?(db: DatabaseSync, path: string, extracted: unknown, delta: ReconcileDelta): void;\n  // After all rows are current, inside the reconcile transaction (resolution, rank).\n  // Hooks run in registry order and communicate through fields on `delta`.\n  afterReconcile?(db: DatabaseSync, delta: ReconcileDelta): void;\n  // Per-file opt-out (currently only embed, via FileStat.embed): absent means the feature\n  // applies to every file it's otherwise on for. Checked per file before extract/store so a\n  // doc covered only by semantic-false presets gets no rows for it at all.\n  enabledForFile?(cfg: Config, file: FileStat): boolean;\n}\n"],"names":[],"mappings":"AAkBA,mFAAmF;AACnF,6EAA6E;AAC7E,WAeC"}

package/dist/esm/index.d.ts

@@ -1,6 +1,6 @@
-export type { FindOptions, Peek, TreeMap } from './commands.js';
-export { find, mapTree, peek } from './commands.js';
-export type { Config, EmbedConfig, FeatureName, ResolvedConfig, SavedFind } from './config.js';
+export type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands.js';
+export { mapTree, peek, presetCoverage, search } from './commands.js';
+export type { Config, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch } from './config.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export type { OpenResult } from './db.js';
 export { open, rebuild } from './db.js';

package/dist/esm/index.js

@@ -1,6 +1,6 @@
 // Public library API. Deliberately small: every export is a stability promise;
 // internals (feature registry, graph, scan, meta) stay module-private.
-export { find, mapTree, peek } from './commands.js';
+export { mapTree, peek, presetCoverage, search } from './commands.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.js';
 export { open, rebuild } from './db.js';
 export { SenseError } from './errors.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { FindOptions, Peek, TreeMap } from './commands.ts';\nexport { find, mapTree, peek } from './commands.ts';\nexport type { Config, EmbedConfig, FeatureName, ResolvedConfig, SavedFind } from './config.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\nexport type { OpenResult } from './db.ts';\nexport { open, rebuild } from './db.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["find","mapTree","peek","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","open","rebuild","SenseError","printRows","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,IAAI,EAAEC,OAAO,EAAEC,IAAI,QAAQ,gBAAgB;AAEpD,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAE1H,SAASC,IAAI,EAAEC,OAAO,QAAQ,UAAU;AAExC,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands.ts';\nexport type { Config, EmbedConfig, FeatureName, Preset, ResolvedConfig, SavedQuery, SavedSearch } from './config.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config.ts';\nexport type { OpenResult } from './db.ts';\nexport { open, rebuild } from './db.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Row } from './output.ts';\nexport { printRows } from './output.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","STATE_DIR","SUPPORTED_CONFIG_VERSION","open","rebuild","SenseError","printRows","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,gBAAgB;AAEtE,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,cAAc;AAE1H,SAASC,IAAI,EAAEC,OAAO,QAAQ,UAAU;AAExC,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,cAAc;AAGxC,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/output.d.ts

@@ -1,5 +1,15 @@
 export type Row = Record<string, unknown>;
 export declare function printRows(rows: Row[], format: 'table' | 'json'): void;
+export declare function featureOffHint(name: string): string;
+export declare function featuresLine(features: {
+    on: string[];
+    off: string[];
+}): string;
+export declare function presetsLines(presets: Array<{
+    name: string;
+    files: number;
+    embedded: number;
+}>): string[];
 export declare function renderMap(result: {
     docs: {
         count: number;
@@ -11,6 +21,11 @@ export declare function renderMap(result: {
         on: string[];
         off: string[];
     };
+    presets: Array<{
+        name: string;
+        files: number;
+        embedded: number;
+    }>;
     hubs: Row[];
     recent: Row[];
 }): string;

package/dist/esm/output.js

@@ -47,20 +47,34 @@ function renderRows(rows, format, width = process.stdout.columns) {
         ...cells.map(formatRow)
     ].join('\n');
 }
-// Text renderers for the layer commands; cli.ts prints what these return.
-function featuresLine(features) {
-    const off = features.off.length > 0 ? ` · off: ${features.off.map((name)=>`${name} (features.${name})`).join(', ')}` : '';
+// Text renderers for the top-level commands; cli.ts prints what these return.
+// `embed` is not a features-block key (it's derived from presets), so its off hint reads
+// differently from links/sections/rank's plain `features.<name>`.
+export function featureOffHint(name) {
+    return name === 'embed' ? 'no preset has semantic on' : `features.${name}`;
+}
+export function featuresLine(features) {
+    const off = features.off.length > 0 ? ` · off: ${features.off.map((name)=>`${name} (${featureOffHint(name)})`).join(', ')}` : '';
     return `features: ${features.on.join(', ')}${off}`;
 }
+export function presetsLines(presets) {
+    return [
+        'presets:',
+        ...presets.map((p)=>`  ${p.name}: ${p.files} file(s), ${p.embedded} embedded`)
+    ];
+}
 export function renderMap(result) {
     const parts = [
         `docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`,
-        `${featuresLine(result.features)}\n`,
+        featuresLine(result.features),
+        '',
+        ...presetsLines(result.presets),
+        '',
         renderRows(result.fields, 'table')
     ];
     if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);
     if (result.hubs.length > 0) parts.push('\nhubs (by link rank):', renderRows(result.hubs, 'table'));
-    else if (result.features.off.includes('rank')) parts.push('\nhubs: off (features.rank)');
+    else if (result.features.off.includes('rank')) parts.push(`\nhubs: off (${featureOffHint('rank')})`);
     parts.push('\nrecent:', renderRows(result.recent, 'table'));
     return parts.join('\n');
 }

package/dist/esm/output.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\n// Table output is for reading; a row wider than the terminal wraps into an unreadable\n// block (a `find` header measured 330 chars once `summary` is present, which drove one\n// user to pipe --format json through a script for every interactive query). Widest\n// columns give up space first, down to a floor, and only as far as the terminal needs.\n// --format json is never truncated: it is the machine-readable surface.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: 'table' | 'json', width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the layer commands; cli.ts prints what these return.\n\nfunction featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (features.${name})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, `${featuresLine(result.features)}\\n`, renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push('\\nhubs: off (features.rank)');\n  parts.push('\\nrecent:', renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","renderRows","console","log","length","warn","Math","round","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","process","stdout","columns","JSON","stringify","Object","keys","cells","map","row","col","natural","i","clip","w","padEnd","slice","formatRow","values","join","trimEnd","repeat","featuresLine","features","off","name","on","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,WAAWH,MAAMC;IAClCG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,sFAAsF;AACtF,uFAAuF;AACvF,mFAAmF;AACnF,uFAAuF;AACvF,wEAAwE;AACxE,MAAMI,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAOP,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMU,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACd,KAAKe,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASX,WAAWH,IAAW,EAAEC,MAAwB,EAAEuB,QAAQC,QAAQC,MAAM,CAACC,OAAO;IACvF,IAAI1B,WAAW,QAAQ,OAAO2B,KAAKC,SAAS,CAAC7B,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMqB,UAAUG,OAAOC,IAAI,CAAC/B,IAAI,CAAC,EAAE;IACnC,MAAMgC,QAAQhC,KAAKiC,GAAG,CAAC,CAACC,MAAQP,QAAQM,GAAG,CAAC,CAACE,MAAQxC,KAAKuC,GAAG,CAACC,IAAI;IAClE,MAAMC,UAAUT,QAAQM,GAAG,CAAC,CAACE,KAAKE,IAAM7B,KAAKe,GAAG,CAACY,IAAI7B,MAAM,KAAK0B,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAAC/B,MAAM;IAChG,MAAMO,SAASW,SAASA,QAAQd,aAAaE,UAAUwB,SAASZ,SAASY;IAEzE,MAAME,OAAO,CAAC1C,OAAe2C,IAAe3C,MAAMU,MAAM,IAAIiC,IAAI3C,MAAM4C,MAAM,CAACD,KAAK,GAAG3C,MAAM6C,KAAK,CAAC,GAAGF,IAAI,KAAK5B,WAAW;IACxH,MAAM+B,YAAY,CAACC,SACjBA,OACGV,GAAG,CAAC,CAACrC,OAAOyC,IAAMC,KAAK1C,OAAOiB,MAAM,CAACwB,EAAE,GACvCO,IAAI,CAAC,MACLC,OAAO;IAEZ,OAAO;QAACH,UAAUf;QAAUd,OAAOoB,GAAG,CAAC,CAACM,IAAM,IAAIO,MAAM,CAACP,IAAIK,IAAI,CAAC;WAAUZ,MAAMC,GAAG,CAACS;KAAW,CAACE,IAAI,CAAC;AACzG;AAEA,0EAA0E;AAE1E,SAASG,aAAaC,QAAyC;IAC7D,MAAMC,MAAMD,SAASC,GAAG,CAAC3C,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE0C,SAASC,GAAG,CAAChB,GAAG,CAAC,CAACiB,OAAS,GAAGA,KAAK,WAAW,EAAEA,KAAK,CAAC,CAAC,EAAEN,IAAI,CAAC,OAAO,GAAG;IACzH,OAAO,CAAC,UAAU,EAAEI,SAASG,EAAE,CAACP,IAAI,CAAC,QAAQK,KAAK;AACpD;AAEA,OAAO,SAASG,UAAUC,MAA6J;IACrL,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEhD,KAAKC,KAAK,CAAC4C,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,IAAI,CAAC;QAAE,GAAGV,aAAaM,OAAOL,QAAQ,EAAE,EAAE,CAAC;QAAE7C,WAAWkD,OAAOK,MAAM,EAAE;KAAS;IACnK,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACpD,MAAM,EAAEgD,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAACpD,MAAM,CAAC,aAAa,CAAC;IACvH,IAAI+C,OAAOQ,IAAI,CAACvD,MAAM,GAAG,GAAGgD,MAAMM,IAAI,CAAC,0BAA0BzD,WAAWkD,OAAOQ,IAAI,EAAE;SACpF,IAAIR,OAAOL,QAAQ,CAACC,GAAG,CAACa,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC;IAC1DN,MAAMM,IAAI,CAAC,aAAazD,WAAWkD,OAAOU,MAAM,EAAE;IAClD,OAAOT,MAAMV,IAAI,CAAC;AACpB;AAEA,OAAO,SAASoB,WAAWX,MAAwP;IACjR,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAKxE,MAAM,IAAIkC,OAAOuC,OAAO,CAAChB,OAAOiB,WAAW,EAAGL,MAAML,IAAI,CAAC,CAAC,EAAE,EAAEQ,IAAI,EAAE,EAAExE,OAAO;IAC9F,IAAIyD,OAAOkB,QAAQ,CAACjE,MAAM,GAAG,GAAG;QAC9B2D,MAAML,IAAI,CAAC,IAAI;QACf,KAAK,MAAMY,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAML,IAAI,CAAC,CAAC,EAAE,EAAE,IAAId,MAAM,CAAC0B,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAId,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAACjE,MAAM,EAAE2D,MAAML,IAAI,CAAC,CAAC,IAAI,EAAEP,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAACjE,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAI+C,OAAOJ,GAAG,CAACa,QAAQ,CAAC,aAAaG,MAAML,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOJ,GAAG,CAACa,QAAQ,CAAC,UAAU;QAChCG,MAAML,IAAI,CAAC,IAAI;QACf,OAAOK,MAAMrB,IAAI,CAAC;IACpB;IACA,MAAMkC,WAAW,CAACC,OAAeC,OAAiB/D;QAChD,IAAIA,QAAQ,GAAGgD,MAAML,IAAI,CAAC,GAAGmB,MAAM,EAAE,EAAE9D,MAAM,GAAG,EAAE+D,MAAMpC,IAAI,CAAC,QAAQ3B,QAAQ+D,MAAM1E,MAAM,GAAG,CAAC,GAAG,EAAEW,QAAQ+D,MAAM1E,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAI+C,OAAO4B,aAAa,GAAG5B,OAAO6B,eAAe,GAAG7B,OAAO8B,cAAc,GAAG,GAAGlB,MAAML,IAAI,CAAC;IAC1FkB,SAAS,aAAazB,OAAO+B,QAAQ,EAAE/B,OAAO4B,aAAa;IAC3DH,SAAS,cAAczB,OAAOgC,UAAU,EAAEhC,OAAO6B,eAAe;IAChEJ,SAAS,aAAazB,OAAOiC,SAAS,EAAEjC,OAAO8B,cAAc;IAC7D,OAAOlB,MAAMrB,IAAI,CAAC;AACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/output.ts"],"sourcesContent":["// rows -> table (default) | json\n\nexport type Row = Record<string, unknown>;\n\n// Warned about on stderr (keeps --format json stdout machine-readable), never truncated.\nconst OVERSIZE_BYTES = 50_000;\n\n// Flatten embedded newlines so they can't break table column alignment; JSON output is left alone.\nfunction cell(value: unknown): string {\n  return String(value ?? '').replace(/\\s*\\n\\s*/g, ' ');\n}\n\nexport function printRows(rows: Row[], format: 'table' | 'json'): void {\n  const rendered = renderRows(rows, format);\n  console.log(rendered);\n  if (rendered.length > OVERSIZE_BYTES) {\n    console.warn(`warning: result is ${Math.round(rendered.length / 1000)} KB across ${rows.length} row(s); consider a LIMIT, fewer columns, or snippet() instead of whole values`);\n  }\n}\n\n// Table output is for reading; a row wider than the terminal wraps into an unreadable\n// block (a `find` header measured 330 chars once `summary` is present, which drove one\n// user to pipe --format json through a script for every interactive query). Widest\n// columns give up space first, down to a floor, and only as far as the terminal needs.\n// --format json is never truncated: it is the machine-readable surface.\nconst MIN_COLUMN = 12;\nconst TRUNCATED = '…';\n\nfunction fitWidths(widths: number[], limit: number): number[] {\n  const gaps = (widths.length - 1) * 2;\n  const fitted = [...widths];\n  for (;;) {\n    const total = fitted.reduce((a, b) => a + b, 0) + gaps;\n    if (total <= limit) return fitted;\n    const widest = fitted.indexOf(Math.max(...fitted));\n    if (fitted[widest] <= MIN_COLUMN) return fitted;\n    fitted[widest] = Math.max(MIN_COLUMN, fitted[widest] - Math.max(1, total - limit));\n  }\n}\n\nfunction renderRows(rows: Row[], format: 'table' | 'json', width = process.stdout.columns): string {\n  if (format === 'json') return JSON.stringify(rows, null, 2);\n  if (rows.length === 0) return '(0 rows)';\n\n  const columns = Object.keys(rows[0]);\n  const cells = rows.map((row) => columns.map((col) => cell(row[col])));\n  const natural = columns.map((col, i) => Math.max(col.length, ...cells.map((row) => row[i].length)));\n  const widths = width && width > MIN_COLUMN ? fitWidths(natural, width) : natural;\n\n  const clip = (value: string, w: number) => (value.length <= w ? value.padEnd(w) : `${value.slice(0, w - 1)}${TRUNCATED}`);\n  const formatRow = (values: string[]) =>\n    values\n      .map((value, i) => clip(value, widths[i]))\n      .join('  ')\n      .trimEnd();\n\n  return [formatRow(columns), widths.map((w) => '-'.repeat(w)).join('  '), ...cells.map(formatRow)].join('\\n');\n}\n\n// Text renderers for the top-level commands; cli.ts prints what these return.\n\n// `embed` is not a features-block key (it's derived from presets), so its off hint reads\n// differently from links/sections/rank's plain `features.<name>`.\nexport function featureOffHint(name: string): string {\n  return name === 'embed' ? 'no preset has semantic on' : `features.${name}`;\n}\n\nexport function featuresLine(features: { on: string[]; off: string[] }): string {\n  const off = features.off.length > 0 ? ` · off: ${features.off.map((name) => `${name} (${featureOffHint(name)})`).join(', ')}` : '';\n  return `features: ${features.on.join(', ')}${off}`;\n}\n\nexport function presetsLines(presets: Array<{ name: string; files: number; embedded: number }>): string[] {\n  return ['presets:', ...presets.map((p) => `  ${p.name}: ${p.files} file(s), ${p.embedded} embedded`)];\n}\n\nexport function renderMap(result: { docs: { count: number; bytes: number }; fields: Row[]; fieldsTotal: number; features: { on: string[]; off: string[] }; presets: Array<{ name: string; files: number; embedded: number }>; hubs: Row[]; recent: Row[] }): string {\n  const parts = [`docs: ${result.docs.count} (${Math.round(result.docs.bytes / 1024)} KB)`, featuresLine(result.features), '', ...presetsLines(result.presets), '', renderRows(result.fields, 'table')];\n  if (result.fieldsTotal > result.fields.length) parts.push(`(+${result.fieldsTotal - result.fields.length} more fields)`);\n  if (result.hubs.length > 0) parts.push('\\nhubs (by link rank):', renderRows(result.hubs, 'table'));\n  else if (result.features.off.includes('rank')) parts.push(`\\nhubs: off (${featureOffHint('rank')})`);\n  parts.push('\\nrecent:', renderRows(result.recent, 'table'));\n  return parts.join('\\n');\n}\n\nexport function renderPeek(result: { path: string; tokens: number; frontmatter: Row; sections: Row[]; outbound: string[]; backlinks: string[]; unresolved: string[]; sectionsTotal: number; outboundTotal: number; backlinksTotal: number; unresolvedTotal: number; off: string[] }): string {\n  const lines = [`${result.path}  (~${result.tokens} tokens)`];\n  for (const [key, value] of Object.entries(result.frontmatter)) lines.push(`  ${key}: ${value}`);\n  if (result.sections.length > 0) {\n    lines.push('', 'sections:');\n    for (const s of result.sections) lines.push(`  ${'#'.repeat(s.level as number)} ${s.heading}  [L${s.start_line}-${s.end_line}, ~${s.tokens}t]`);\n    if (result.sectionsTotal > result.sections.length) lines.push(`  (+${result.sectionsTotal - result.sections.length} more sections -- sections table has all of them)`);\n  } else if (result.off.includes('sections')) lines.push('', 'sections: off (features.sections)');\n  if (result.off.includes('links')) {\n    lines.push('', 'links: off (features.links)');\n    return lines.join('\\n');\n  }\n  const linkLine = (label: string, shown: string[], total: number) => {\n    if (total > 0) lines.push(`${label} (${total}): ${shown.join(', ')}${total > shown.length ? `, +${total - shown.length} more` : ''}`);\n  };\n  if (result.outboundTotal + result.unresolvedTotal + result.backlinksTotal > 0) lines.push('');\n  linkLine('links out', result.outbound, result.outboundTotal);\n  linkLine('unresolved', result.unresolved, result.unresolvedTotal);\n  linkLine('backlinks', result.backlinks, result.backlinksTotal);\n  return lines.join('\\n');\n}\n"],"names":["OVERSIZE_BYTES","cell","value","String","replace","printRows","rows","format","rendered","renderRows","console","log","length","warn","Math","round","MIN_COLUMN","TRUNCATED","fitWidths","widths","limit","gaps","fitted","total","reduce","a","b","widest","indexOf","max","width","process","stdout","columns","JSON","stringify","Object","keys","cells","map","row","col","natural","i","clip","w","padEnd","slice","formatRow","values","join","trimEnd","repeat","featureOffHint","name","featuresLine","features","off","on","presetsLines","presets","p","files","embedded","renderMap","result","parts","docs","count","bytes","fields","fieldsTotal","push","hubs","includes","recent","renderPeek","lines","path","tokens","key","entries","frontmatter","sections","s","level","heading","start_line","end_line","sectionsTotal","linkLine","label","shown","outboundTotal","unresolvedTotal","backlinksTotal","outbound","unresolved","backlinks"],"mappings":"AAAA,iCAAiC;AAIjC,yFAAyF;AACzF,MAAMA,iBAAiB;AAEvB,mGAAmG;AACnG,SAASC,KAAKC,KAAc;IAC1B,OAAOC,OAAOD,kBAAAA,mBAAAA,QAAS,IAAIE,OAAO,CAAC,aAAa;AAClD;AAEA,OAAO,SAASC,UAAUC,IAAW,EAAEC,MAAwB;IAC7D,MAAMC,WAAWC,WAAWH,MAAMC;IAClCG,QAAQC,GAAG,CAACH;IACZ,IAAIA,SAASI,MAAM,GAAGZ,gBAAgB;QACpCU,QAAQG,IAAI,CAAC,CAAC,mBAAmB,EAAEC,KAAKC,KAAK,CAACP,SAASI,MAAM,GAAG,MAAM,WAAW,EAAEN,KAAKM,MAAM,CAAC,8EAA8E,CAAC;IAChL;AACF;AAEA,sFAAsF;AACtF,uFAAuF;AACvF,mFAAmF;AACnF,uFAAuF;AACvF,wEAAwE;AACxE,MAAMI,aAAa;AACnB,MAAMC,YAAY;AAElB,SAASC,UAAUC,MAAgB,EAAEC,KAAa;IAChD,MAAMC,OAAO,AAACF,CAAAA,OAAOP,MAAM,GAAG,CAAA,IAAK;IACnC,MAAMU,SAAS;WAAIH;KAAO;IAC1B,OAAS;QACP,MAAMI,QAAQD,OAAOE,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG,KAAKL;QAClD,IAAIE,SAASH,OAAO,OAAOE;QAC3B,MAAMK,SAASL,OAAOM,OAAO,CAACd,KAAKe,GAAG,IAAIP;QAC1C,IAAIA,MAAM,CAACK,OAAO,IAAIX,YAAY,OAAOM;QACzCA,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAACb,YAAYM,MAAM,CAACK,OAAO,GAAGb,KAAKe,GAAG,CAAC,GAAGN,QAAQH;IAC7E;AACF;AAEA,SAASX,WAAWH,IAAW,EAAEC,MAAwB,EAAEuB,QAAQC,QAAQC,MAAM,CAACC,OAAO;IACvF,IAAI1B,WAAW,QAAQ,OAAO2B,KAAKC,SAAS,CAAC7B,MAAM,MAAM;IACzD,IAAIA,KAAKM,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMqB,UAAUG,OAAOC,IAAI,CAAC/B,IAAI,CAAC,EAAE;IACnC,MAAMgC,QAAQhC,KAAKiC,GAAG,CAAC,CAACC,MAAQP,QAAQM,GAAG,CAAC,CAACE,MAAQxC,KAAKuC,GAAG,CAACC,IAAI;IAClE,MAAMC,UAAUT,QAAQM,GAAG,CAAC,CAACE,KAAKE,IAAM7B,KAAKe,GAAG,CAACY,IAAI7B,MAAM,KAAK0B,MAAMC,GAAG,CAAC,CAACC,MAAQA,GAAG,CAACG,EAAE,CAAC/B,MAAM;IAChG,MAAMO,SAASW,SAASA,QAAQd,aAAaE,UAAUwB,SAASZ,SAASY;IAEzE,MAAME,OAAO,CAAC1C,OAAe2C,IAAe3C,MAAMU,MAAM,IAAIiC,IAAI3C,MAAM4C,MAAM,CAACD,KAAK,GAAG3C,MAAM6C,KAAK,CAAC,GAAGF,IAAI,KAAK5B,WAAW;IACxH,MAAM+B,YAAY,CAACC,SACjBA,OACGV,GAAG,CAAC,CAACrC,OAAOyC,IAAMC,KAAK1C,OAAOiB,MAAM,CAACwB,EAAE,GACvCO,IAAI,CAAC,MACLC,OAAO;IAEZ,OAAO;QAACH,UAAUf;QAAUd,OAAOoB,GAAG,CAAC,CAACM,IAAM,IAAIO,MAAM,CAACP,IAAIK,IAAI,CAAC;WAAUZ,MAAMC,GAAG,CAACS;KAAW,CAACE,IAAI,CAAC;AACzG;AAEA,8EAA8E;AAE9E,yFAAyF;AACzF,kEAAkE;AAClE,OAAO,SAASG,eAAeC,IAAY;IACzC,OAAOA,SAAS,UAAU,8BAA8B,CAAC,SAAS,EAAEA,MAAM;AAC5E;AAEA,OAAO,SAASC,aAAaC,QAAyC;IACpE,MAAMC,MAAMD,SAASC,GAAG,CAAC7C,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE4C,SAASC,GAAG,CAAClB,GAAG,CAAC,CAACe,OAAS,GAAGA,KAAK,EAAE,EAAED,eAAeC,MAAM,CAAC,CAAC,EAAEJ,IAAI,CAAC,OAAO,GAAG;IAChI,OAAO,CAAC,UAAU,EAAEM,SAASE,EAAE,CAACR,IAAI,CAAC,QAAQO,KAAK;AACpD;AAEA,OAAO,SAASE,aAAaC,OAAiE;IAC5F,OAAO;QAAC;WAAeA,QAAQrB,GAAG,CAAC,CAACsB,IAAM,CAAC,EAAE,EAAEA,EAAEP,IAAI,CAAC,EAAE,EAAEO,EAAEC,KAAK,CAAC,UAAU,EAAED,EAAEE,QAAQ,CAAC,SAAS,CAAC;KAAE;AACvG;AAEA,OAAO,SAASC,UAAUC,MAAgO;IACxP,MAAMC,QAAQ;QAAC,CAAC,MAAM,EAAED,OAAOE,IAAI,CAACC,KAAK,CAAC,EAAE,EAAEtD,KAAKC,KAAK,CAACkD,OAAOE,IAAI,CAACE,KAAK,GAAG,MAAM,IAAI,CAAC;QAAEd,aAAaU,OAAOT,QAAQ;QAAG;WAAOG,aAAaM,OAAOL,OAAO;QAAG;QAAInD,WAAWwD,OAAOK,MAAM,EAAE;KAAS;IACrM,IAAIL,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC1D,MAAM,EAAEsD,MAAMM,IAAI,CAAC,CAAC,EAAE,EAAEP,OAAOM,WAAW,GAAGN,OAAOK,MAAM,CAAC1D,MAAM,CAAC,aAAa,CAAC;IACvH,IAAIqD,OAAOQ,IAAI,CAAC7D,MAAM,GAAG,GAAGsD,MAAMM,IAAI,CAAC,0BAA0B/D,WAAWwD,OAAOQ,IAAI,EAAE;SACpF,IAAIR,OAAOT,QAAQ,CAACC,GAAG,CAACiB,QAAQ,CAAC,SAASR,MAAMM,IAAI,CAAC,CAAC,aAAa,EAAEnB,eAAe,QAAQ,CAAC,CAAC;IACnGa,MAAMM,IAAI,CAAC,aAAa/D,WAAWwD,OAAOU,MAAM,EAAE;IAClD,OAAOT,MAAMhB,IAAI,CAAC;AACpB;AAEA,OAAO,SAAS0B,WAAWX,MAAwP;IACjR,MAAMY,QAAQ;QAAC,GAAGZ,OAAOa,IAAI,CAAC,IAAI,EAAEb,OAAOc,MAAM,CAAC,QAAQ,CAAC;KAAC;IAC5D,KAAK,MAAM,CAACC,KAAK9E,MAAM,IAAIkC,OAAO6C,OAAO,CAAChB,OAAOiB,WAAW,EAAGL,MAAML,IAAI,CAAC,CAAC,EAAE,EAAEQ,IAAI,EAAE,EAAE9E,OAAO;IAC9F,IAAI+D,OAAOkB,QAAQ,CAACvE,MAAM,GAAG,GAAG;QAC9BiE,MAAML,IAAI,CAAC,IAAI;QACf,KAAK,MAAMY,KAAKnB,OAAOkB,QAAQ,CAAEN,MAAML,IAAI,CAAC,CAAC,EAAE,EAAE,IAAIpB,MAAM,CAACgC,EAAEC,KAAK,EAAY,CAAC,EAAED,EAAEE,OAAO,CAAC,IAAI,EAAEF,EAAEG,UAAU,CAAC,CAAC,EAAEH,EAAEI,QAAQ,CAAC,GAAG,EAAEJ,EAAEL,MAAM,CAAC,EAAE,CAAC;QAC9I,IAAId,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAACvE,MAAM,EAAEiE,MAAML,IAAI,CAAC,CAAC,IAAI,EAAEP,OAAOwB,aAAa,GAAGxB,OAAOkB,QAAQ,CAACvE,MAAM,CAAC,iDAAiD,CAAC;IACvK,OAAO,IAAIqD,OAAOR,GAAG,CAACiB,QAAQ,CAAC,aAAaG,MAAML,IAAI,CAAC,IAAI;IAC3D,IAAIP,OAAOR,GAAG,CAACiB,QAAQ,CAAC,UAAU;QAChCG,MAAML,IAAI,CAAC,IAAI;QACf,OAAOK,MAAM3B,IAAI,CAAC;IACpB;IACA,MAAMwC,WAAW,CAACC,OAAeC,OAAiBrE;QAChD,IAAIA,QAAQ,GAAGsD,MAAML,IAAI,CAAC,GAAGmB,MAAM,EAAE,EAAEpE,MAAM,GAAG,EAAEqE,MAAM1C,IAAI,CAAC,QAAQ3B,QAAQqE,MAAMhF,MAAM,GAAG,CAAC,GAAG,EAAEW,QAAQqE,MAAMhF,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI;IACtI;IACA,IAAIqD,OAAO4B,aAAa,GAAG5B,OAAO6B,eAAe,GAAG7B,OAAO8B,cAAc,GAAG,GAAGlB,MAAML,IAAI,CAAC;IAC1FkB,SAAS,aAAazB,OAAO+B,QAAQ,EAAE/B,OAAO4B,aAAa;IAC3DH,SAAS,cAAczB,OAAOgC,UAAU,EAAEhC,OAAO6B,eAAe;IAChEJ,SAAS,aAAazB,OAAOiC,SAAS,EAAEjC,OAAO8B,cAAc;IAC7D,OAAOlB,MAAM3B,IAAI,CAAC;AACpB"}

package/dist/esm/scan.d.ts

@@ -6,12 +6,15 @@ export interface FileStat {
     absPath: string;
     mtimeMs: number;
     size: number;
+    presets: string[];
+    embed: boolean;
 }
 export declare function listFiles(cfg: Config, baseDir: string): FileStat[];
 export interface ParsedDoc {
     relPath: string;
     mtimeMs: number;
     size: number;
+    presets: string[];
     data: Record<string, string | number | bigint | null>;
     search: {
         title: string;

package/dist/esm/scan.js

@@ -3,6 +3,7 @@ import { join } from 'node:path';
 import fastGlob from 'fast-glob';
 import removeMarkdown from 'remove-markdown';
 import { parseDocument } from 'yaml';
+import { presetNames, presetSemanticEnabled } from './config.js';
 // Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.
 // Reserved: colliding with these would clash with the `frontmatter` table's own columns or
 // the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert
@@ -29,18 +30,44 @@ function stripText(value) {
     const withoutTables = withoutMarkdown.replace(/^\s*\|?[-\s|:]+\|\s*$/gm, '').replace(/\|/g, ' ');
     return normalizeText(withoutTables);
 }
+// Presets are the file selection: each preset's include/exclude picks its files (fast-glob,
+// tsconfig-standard semantics, resolved relative to baseDir). Unlike the old layer model,
+// presets are views, not partitions -- they may overlap freely, and a file's set of covering
+// presets (not a single owner) is what indexing and embedding derive from. A preset whose
+// globs match nothing is valid. Files matched by no preset at all are simply not indexed.
 export function listFiles(cfg, baseDir) {
-    const relPaths = fastGlob.sync(cfg.scan.include, {
-        cwd: baseDir
-    }).sort();
+    const coverage = new Map();
+    for (const name of presetNames(cfg)){
+        var _preset_exclude;
+        const preset = cfg.presets[name];
+        const matched = fastGlob.sync(preset.include, {
+            cwd: baseDir,
+            ignore: (_preset_exclude = preset.exclude) !== null && _preset_exclude !== void 0 ? _preset_exclude : []
+        });
+        for (const relPath of matched){
+            var _coverage_get;
+            const set = (_coverage_get = coverage.get(relPath)) !== null && _coverage_get !== void 0 ? _coverage_get : new Set();
+            set.add(name);
+            coverage.set(relPath, set);
+        }
+    }
+    const relPaths = [
+        ...coverage.keys()
+    ].sort();
     return relPaths.map((relPath)=>{
         const absPath = join(baseDir, relPath);
         const st = statSync(absPath);
+        const presets = [
+            ...coverage.get(relPath)
+        ].sort();
+        const embed = presets.some((name)=>presetSemanticEnabled(cfg, name));
         return {
             relPath,
             absPath,
             mtimeMs: st.mtimeMs,
-            size: st.size
+            size: st.size,
+            presets,
+            embed
         };
     });
 }
@@ -120,6 +147,7 @@ export function parseFile(file, extractors = []) {
             relPath: file.relPath,
             mtimeMs: file.mtimeMs,
             size: file.size,
+            presets: file.presets,
             data: mapped,
             search,
             extracted: Object.fromEntries(extractors.filter((f)=>f.extract).map((f)=>{

package/dist/esm/scan.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or\n// the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert\n// can tell a feature-owned column (e.g. rank's `_rank`) apart from a parsed frontmatter\n// column and leave the former alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n}\n\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const relPaths = fastGlob.sync(cfg.scan.include, { cwd: baseDir }).sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","relPaths","sync","scan","include","cwd","sort","map","relPath","absPath","st","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","keys","has","search","title","summary","text","extracted","fromEntries","filter","f","extract","name"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAIrC,6EAA6E;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,wFAAwF;AACxF,gDAAgD;AAChD,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW,EAAE;AAE9G,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBZ,eAAeW;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AASA,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAWlB,SAASmB,IAAI,CAACH,IAAII,IAAI,CAACC,OAAO,EAAE;QAAEC,KAAKL;IAAQ,GAAGM,IAAI;IACvE,OAAOL,SAASM,GAAG,CAAC,CAACC;QACnB,MAAMC,UAAU3B,KAAKkB,SAASQ;QAC9B,MAAME,KAAK7B,SAAS4B;QACpB,OAAO;YAAED;YAASC;YAASE,SAASD,GAAGC,OAAO;YAAEC,MAAMF,GAAGE,IAAI;QAAC;IAChE;AACF;AAaA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASC,SAASxB,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOyB,OAAOzB,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAO0B,OAAOC,aAAa,CAAC3B,SAASyB,OAAOzB,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAO4B,KAAKC,SAAS,CAAC7B;AACxB;AAEA,gEAAgE;AAChE,SAAS8B,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKvC,WAAW,OAAO;QAAEiC,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiBtB,OAAe,EAAEe,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM/C,cAAcsC;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,iBAAiB,EAAEwB,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,8BAA8B,EAAE,AAAC+B,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS/C,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO+C,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAE1B,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAO6B;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMxC,aAAa+D,KAAKlC,OAAO,EAAE;IACvC,MAAMsB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAKnC,OAAO,EAAEe,IAAIQ;IACnE,MAAMe,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACZ,MAAO;QACnC,IAAInD,iBAAiBgE,GAAG,CAACH,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAKnC,OAAO,CAAC,8BAA8B,EAAEuC,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGlC,SAASwB,IAAI,CAACU,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMI,SAAS;QAAEC,OAAOhE,cAAciD,KAAKe,KAAK;QAAGC,SAASjE,cAAciD,KAAKgB,OAAO;QAAGC,MAAM5D,UAAUmD;IAAS;IAElH,OAAO;QACLb,KAAK;YACHxB,SAASmC,KAAKnC,OAAO;YACrBG,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACfyB,MAAMS;YACNK;YACAI,WAAWP,OAAOQ,WAAW,CAACZ,WAAWa,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEpD,GAAG,CAAC,CAACmD;oBAAeA;uBAAT;oBAACA,EAAEE,IAAI;qBAAEF,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYtC,KAAKyB,SAASM;iBAAQ;;QAC1H;QACApB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan.ts"],"sourcesContent":["import { readFileSync, statSync } from 'node:fs';\nimport { join } from 'node:path';\nimport fastGlob from 'fast-glob';\nimport removeMarkdown from 'remove-markdown';\nimport { parseDocument } from 'yaml';\nimport type { Config } from './config.ts';\nimport { presetNames, presetSemanticEnabled } from './config.ts';\nimport type { Feature } from './features/types.ts';\n\n// Filesystem -> rows. Pure data in, data + warnings out; db.ts does the SQL.\n\n// Reserved: colliding with these would clash with the `frontmatter` table's own columns or\n// the other tables (`content`, `links`, `sections`). Exported so db.ts's frontmatter upsert\n// can tell a feature-owned column (e.g. rank's `_rank`) apart from a parsed frontmatter\n// column and leave the former alone on reparse.\nexport const RESERVED_COLUMNS = new Set(['path', '_mtime', '_size', '_rank', 'content', 'links', 'sections']);\n\nfunction normalizeText(value: unknown): string {\n  if (value === null || value === undefined) return '';\n  return String(value).replace(/\\s+/g, ' ').trim();\n}\n\n// Markdown is stripped at index time so snippets read as clean prose (also improves matching, e.g. `**bold**` indexes as `bold`).\n// remove-markdown doesn't cover Obsidian wikilinks or table scaffolding, so those are handled first/after.\nfunction stripText(value: string): string {\n  const withoutWikilinks = value.replace(/\\[\\[([^\\]|]+)\\|([^\\]]+)\\]\\]/g, '$2').replace(/\\[\\[([^\\]]+)\\]\\]/g, '$1');\n  const withoutMarkdown = removeMarkdown(withoutWikilinks);\n  const withoutTables = withoutMarkdown.replace(/^\\s*\\|?[-\\s|:]+\\|\\s*$/gm, '').replace(/\\|/g, ' ');\n  return normalizeText(withoutTables);\n}\n\nexport interface FileStat {\n  relPath: string;\n  absPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[]; // every declared preset covering this file (>= 1; union, overlap allowed)\n  embed: boolean; // true iff any covering preset has semantic !== false\n}\n\n// Presets are the file selection: each preset's include/exclude picks its files (fast-glob,\n// tsconfig-standard semantics, resolved relative to baseDir). Unlike the old layer model,\n// presets are views, not partitions -- they may overlap freely, and a file's set of covering\n// presets (not a single owner) is what indexing and embedding derive from. A preset whose\n// globs match nothing is valid. Files matched by no preset at all are simply not indexed.\nexport function listFiles(cfg: Config, baseDir: string): FileStat[] {\n  const coverage = new Map<string, Set<string>>();\n  for (const name of presetNames(cfg)) {\n    const preset = cfg.presets[name];\n    const matched = fastGlob.sync(preset.include, { cwd: baseDir, ignore: preset.exclude ?? [] });\n    for (const relPath of matched) {\n      const set = coverage.get(relPath) ?? new Set<string>();\n      set.add(name);\n      coverage.set(relPath, set);\n    }\n  }\n\n  const relPaths = [...coverage.keys()].sort();\n  return relPaths.map((relPath) => {\n    const absPath = join(baseDir, relPath);\n    const st = statSync(absPath);\n    const presets = [...(coverage.get(relPath) as Set<string>)].sort();\n    const embed = presets.some((name) => presetSemanticEnabled(cfg, name));\n    return { relPath, absPath, mtimeMs: st.mtimeMs, size: st.size, presets, embed };\n  });\n}\n\nexport interface ParsedDoc {\n  relPath: string;\n  mtimeMs: number;\n  size: number;\n  presets: string[];\n  data: Record<string, string | number | bigint | null>;\n  // title/summary are duplicated from frontmatter so bm25() can weight them above the body text.\n  search: { title: string; summary: string; text: string };\n  // Per-feature extraction results, keyed by feature name; features store them at reconcile.\n  extracted: Record<string, unknown>;\n}\n\n// SQLite storage classes follow the YAML scalar: strings -> TEXT, booleans and whole\n// numbers -> INTEGER, fractions -> REAL, everything else (lists, maps) -> JSON TEXT.\n// Booleans have no SQLite type of their own; 1/0 is the convention, so `WHERE flag = 1`\n// matches and `WHERE flag = 'true'` cannot -- `map` prints the observed type per field so\n// that mismatch is visible rather than a silent zero-row answer.\nfunction mapValue(value: unknown): string | number | bigint | null {\n  if (value === null || value === undefined) return null;\n  if (typeof value === 'boolean') return BigInt(value ? 1 : 0);\n  if (typeof value === 'number') return Number.isSafeInteger(value) ? BigInt(value) : value;\n  if (typeof value === 'string') return value;\n  return JSON.stringify(value);\n}\n\n// The delimiter split is all this package used gray-matter for.\nfunction splitFrontmatter(raw: string): { fm: string | null; body: string } {\n  const open = raw.match(/^---\\r?\\n/);\n  if (!open) return { fm: null, body: raw };\n  const rest = raw.slice(open[0].length);\n  const close = rest.match(/^---\\r?(\\n|$)/m);\n  if (!close || close.index === undefined) return { fm: null, body: raw };\n  return { fm: rest.slice(0, close.index), body: rest.slice(close.index + close[0].length) };\n}\n\n// Lenient by design: parseDocument collects syntax errors as data and still yields values,\n// so Obsidian-style frontmatter (e.g. an alias starting with @) survives with a warning.\nfunction parseFrontmatter(relPath: string, fm: string, warnings: string[]): Record<string, unknown> {\n  const doc = parseDocument(fm);\n  if (doc.errors.length > 0) {\n    warnings.push(`warning: ${relPath} frontmatter has ${doc.errors.length} syntax error(s) (${doc.errors[0].message.split('\\n')[0]}); parsed leniently`);\n  }\n  let data: unknown;\n  try {\n    data = doc.toJS();\n  } catch (err) {\n    warnings.push(`warning: ${relPath} has unparseable frontmatter (${(err as Error).message.split('\\n')[0]}); indexing without it`);\n    return {};\n  }\n  if (data === null || data === undefined) return {};\n  if (typeof data !== 'object' || Array.isArray(data)) {\n    warnings.push(`warning: ${relPath} frontmatter is not a key-value mapping; ignoring it`);\n    return {};\n  }\n  return data as Record<string, unknown>;\n}\n\nexport function parseFile(file: FileStat, extractors: Feature[] = []): { doc: ParsedDoc; warnings: string[] } {\n  const raw = readFileSync(file.absPath, 'utf8');\n  const warnings: string[] = [];\n\n  const { fm, body: content } = splitFrontmatter(raw);\n  const data = fm === null ? {} : parseFrontmatter(file.relPath, fm, warnings);\n  const mapped: Record<string, string | number | bigint | null> = {};\n\n  for (const key of Object.keys(data)) {\n    if (RESERVED_COLUMNS.has(key)) {\n      warnings.push(`warning: ${file.relPath} has a frontmatter key named \"${key}\", which is reserved; ignoring it`);\n      continue;\n    }\n    mapped[key] = mapValue(data[key]);\n  }\n\n  // title/summary are plain YAML strings -- whitespace-collapse only;\n  // the prose gets the full markdown strip.\n  const search = { title: normalizeText(data.title), summary: normalizeText(data.summary), text: stripText(content) };\n\n  return {\n    doc: {\n      relPath: file.relPath,\n      mtimeMs: file.mtimeMs,\n      size: file.size,\n      presets: file.presets,\n      data: mapped,\n      search,\n      extracted: Object.fromEntries(extractors.filter((f) => f.extract).map((f) => [f.name, f.extract?.(raw, content, search)])),\n    },\n    warnings,\n  };\n}\n"],"names":["readFileSync","statSync","join","fastGlob","removeMarkdown","parseDocument","presetNames","presetSemanticEnabled","RESERVED_COLUMNS","Set","normalizeText","value","undefined","String","replace","trim","stripText","withoutWikilinks","withoutMarkdown","withoutTables","listFiles","cfg","baseDir","coverage","Map","name","preset","presets","matched","sync","include","cwd","ignore","exclude","relPath","set","get","add","relPaths","keys","sort","map","absPath","st","embed","some","mtimeMs","size","mapValue","BigInt","Number","isSafeInteger","JSON","stringify","splitFrontmatter","raw","open","match","fm","body","rest","slice","length","close","index","parseFrontmatter","warnings","doc","errors","push","message","split","data","toJS","err","Array","isArray","parseFile","file","extractors","content","mapped","key","Object","has","search","title","summary","text","extracted","fromEntries","filter","f","extract"],"mappings":"AAAA,SAASA,YAAY,EAAEC,QAAQ,QAAQ,UAAU;AACjD,SAASC,IAAI,QAAQ,YAAY;AACjC,OAAOC,cAAc,YAAY;AACjC,OAAOC,oBAAoB,kBAAkB;AAC7C,SAASC,aAAa,QAAQ,OAAO;AAErC,SAASC,WAAW,EAAEC,qBAAqB,QAAQ,cAAc;AAGjE,6EAA6E;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,wFAAwF;AACxF,gDAAgD;AAChD,OAAO,MAAMC,mBAAmB,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAS;IAAS;IAAW;IAAS;CAAW,EAAE;AAE9G,SAASC,cAAcC,KAAc;IACnC,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,OAAOC,OAAOF,OAAOG,OAAO,CAAC,QAAQ,KAAKC,IAAI;AAChD;AAEA,kIAAkI;AAClI,2GAA2G;AAC3G,SAASC,UAAUL,KAAa;IAC9B,MAAMM,mBAAmBN,MAAMG,OAAO,CAAC,gCAAgC,MAAMA,OAAO,CAAC,qBAAqB;IAC1G,MAAMI,kBAAkBd,eAAea;IACvC,MAAME,gBAAgBD,gBAAgBJ,OAAO,CAAC,2BAA2B,IAAIA,OAAO,CAAC,OAAO;IAC5F,OAAOJ,cAAcS;AACvB;AAWA,4FAA4F;AAC5F,0FAA0F;AAC1F,6FAA6F;AAC7F,0FAA0F;AAC1F,0FAA0F;AAC1F,OAAO,SAASC,UAAUC,GAAW,EAAEC,OAAe;IACpD,MAAMC,WAAW,IAAIC;IACrB,KAAK,MAAMC,QAAQnB,YAAYe,KAAM;YAEmCK;QADtE,MAAMA,SAASL,IAAIM,OAAO,CAACF,KAAK;QAChC,MAAMG,UAAUzB,SAAS0B,IAAI,CAACH,OAAOI,OAAO,EAAE;YAAEC,KAAKT;YAASU,MAAM,GAAEN,kBAAAA,OAAOO,OAAO,cAAdP,6BAAAA,kBAAkB,EAAE;QAAC;QAC3F,KAAK,MAAMQ,WAAWN,QAAS;gBACjBL;YAAZ,MAAMY,OAAMZ,gBAAAA,SAASa,GAAG,CAACF,sBAAbX,2BAAAA,gBAAyB,IAAId;YACzC0B,IAAIE,GAAG,CAACZ;YACRF,SAASY,GAAG,CAACD,SAASC;QACxB;IACF;IAEA,MAAMG,WAAW;WAAIf,SAASgB,IAAI;KAAG,CAACC,IAAI;IAC1C,OAAOF,SAASG,GAAG,CAAC,CAACP;QACnB,MAAMQ,UAAUxC,KAAKoB,SAASY;QAC9B,MAAMS,KAAK1C,SAASyC;QACpB,MAAMf,UAAU;eAAKJ,SAASa,GAAG,CAACF;SAAyB,CAACM,IAAI;QAChE,MAAMI,QAAQjB,QAAQkB,IAAI,CAAC,CAACpB,OAASlB,sBAAsBc,KAAKI;QAChE,OAAO;YAAES;YAASQ;YAASI,SAASH,GAAGG,OAAO;YAAEC,MAAMJ,GAAGI,IAAI;YAAEpB;YAASiB;QAAM;IAChF;AACF;AAcA,qFAAqF;AACrF,qFAAqF;AACrF,wFAAwF;AACxF,0FAA0F;AAC1F,iEAAiE;AACjE,SAASI,SAASrC,KAAc;IAC9B,IAAIA,UAAU,QAAQA,UAAUC,WAAW,OAAO;IAClD,IAAI,OAAOD,UAAU,WAAW,OAAOsC,OAAOtC,QAAQ,IAAI;IAC1D,IAAI,OAAOA,UAAU,UAAU,OAAOuC,OAAOC,aAAa,CAACxC,SAASsC,OAAOtC,SAASA;IACpF,IAAI,OAAOA,UAAU,UAAU,OAAOA;IACtC,OAAOyC,KAAKC,SAAS,CAAC1C;AACxB;AAEA,gEAAgE;AAChE,SAAS2C,iBAAiBC,GAAW;IACnC,MAAMC,OAAOD,IAAIE,KAAK,CAAC;IACvB,IAAI,CAACD,MAAM,OAAO;QAAEE,IAAI;QAAMC,MAAMJ;IAAI;IACxC,MAAMK,OAAOL,IAAIM,KAAK,CAACL,IAAI,CAAC,EAAE,CAACM,MAAM;IACrC,MAAMC,QAAQH,KAAKH,KAAK,CAAC;IACzB,IAAI,CAACM,SAASA,MAAMC,KAAK,KAAKpD,WAAW,OAAO;QAAE8C,IAAI;QAAMC,MAAMJ;IAAI;IACtE,OAAO;QAAEG,IAAIE,KAAKC,KAAK,CAAC,GAAGE,MAAMC,KAAK;QAAGL,MAAMC,KAAKC,KAAK,CAACE,MAAMC,KAAK,GAAGD,KAAK,CAAC,EAAE,CAACD,MAAM;IAAE;AAC3F;AAEA,2FAA2F;AAC3F,yFAAyF;AACzF,SAASG,iBAAiB/B,OAAe,EAAEwB,EAAU,EAAEQ,QAAkB;IACvE,MAAMC,MAAM9D,cAAcqD;IAC1B,IAAIS,IAAIC,MAAM,CAACN,MAAM,GAAG,GAAG;QACzBI,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,iBAAiB,EAAEiC,IAAIC,MAAM,CAACN,MAAM,CAAC,kBAAkB,EAAEK,IAAIC,MAAM,CAAC,EAAE,CAACE,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,mBAAmB,CAAC;IACtJ;IACA,IAAIC;IACJ,IAAI;QACFA,OAAOL,IAAIM,IAAI;IACjB,EAAE,OAAOC,KAAK;QACZR,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,8BAA8B,EAAE,AAACwC,IAAcJ,OAAO,CAACC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,sBAAsB,CAAC;QAC/H,OAAO,CAAC;IACV;IACA,IAAIC,SAAS,QAAQA,SAAS5D,WAAW,OAAO,CAAC;IACjD,IAAI,OAAO4D,SAAS,YAAYG,MAAMC,OAAO,CAACJ,OAAO;QACnDN,SAASG,IAAI,CAAC,CAAC,SAAS,EAAEnC,QAAQ,oDAAoD,CAAC;QACvF,OAAO,CAAC;IACV;IACA,OAAOsC;AACT;AAEA,OAAO,SAASK,UAAUC,IAAc,EAAEC,aAAwB,EAAE;IAClE,MAAMxB,MAAMvD,aAAa8E,KAAKpC,OAAO,EAAE;IACvC,MAAMwB,WAAqB,EAAE;IAE7B,MAAM,EAAER,EAAE,EAAEC,MAAMqB,OAAO,EAAE,GAAG1B,iBAAiBC;IAC/C,MAAMiB,OAAOd,OAAO,OAAO,CAAC,IAAIO,iBAAiBa,KAAK5C,OAAO,EAAEwB,IAAIQ;IACnE,MAAMe,SAA0D,CAAC;IAEjE,KAAK,MAAMC,OAAOC,OAAO5C,IAAI,CAACiC,MAAO;QACnC,IAAIhE,iBAAiB4E,GAAG,CAACF,MAAM;YAC7BhB,SAASG,IAAI,CAAC,CAAC,SAAS,EAAES,KAAK5C,OAAO,CAAC,8BAA8B,EAAEgD,IAAI,iCAAiC,CAAC;YAC7G;QACF;QACAD,MAAM,CAACC,IAAI,GAAGlC,SAASwB,IAAI,CAACU,IAAI;IAClC;IAEA,oEAAoE;IACpE,0CAA0C;IAC1C,MAAMG,SAAS;QAAEC,OAAO5E,cAAc8D,KAAKc,KAAK;QAAGC,SAAS7E,cAAc8D,KAAKe,OAAO;QAAGC,MAAMxE,UAAUgE;IAAS;IAElH,OAAO;QACLb,KAAK;YACHjC,SAAS4C,KAAK5C,OAAO;YACrBY,SAASgC,KAAKhC,OAAO;YACrBC,MAAM+B,KAAK/B,IAAI;YACfpB,SAASmD,KAAKnD,OAAO;YACrB6C,MAAMS;YACNI;YACAI,WAAWN,OAAOO,WAAW,CAACX,WAAWY,MAAM,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEpD,GAAG,CAAC,CAACmD;oBAAeA;uBAAT;oBAACA,EAAEnE,IAAI;qBAAEmE,aAAAA,EAAEC,OAAO,cAATD,iCAAAA,gBAAAA,GAAYrC,KAAKyB,SAASK;iBAAQ;;QAC1H;QACAnB;IACF;AACF"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "sensemaking",
-  "version": "0.8.0",
-  "description": "SQL over a tree of markdown notes: filter by frontmatter, search the prose, follow the links, read the structure",
+  "version": "0.9.0",
+  "description": "Query and search a tree of markdown notes: SQL over frontmatter and links, ranked search over the prose — words, links, and meaning fused",
   "keywords": [
     "markdown",
     "frontmatter",
@@ -10,10 +10,16 @@
     "fts5",
     "query",
     "search",
-    "semantic",
+    "semantic-search",
+    "knowledge-base",
+    "rag",
     "obsidian",
     "notes",
     "agent",
+    "agent-memory",
+    "memory-consolidation",
+    "llm-wiki",
+    "dreaming",
     "cli"
   ],
   "homepage": "https://github.com/kmalakoff/sensemaking",