sensemaking 0.16.0 → 0.17.0

package/dist/esm/db/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/reconcile.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport type { Config } from '../config/index.ts';\nimport { contentTokenize } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../progress.ts';\nimport type { ParsedDoc } from '../scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../scan.ts';\nimport { segmentField } from '../segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from './shared.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so content population is\n// defined once. Assumes the frontmatter row for doc.relPath already exists (rowid lookup).\nfunction insertContentRow(insertBody: StatementSync, doc: ParsedDoc, segmenting: boolean): void {\n  insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(INSERT_CONTENT_SQL);\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertContentRow(insertBody, doc, segmenting);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport function rebuildContentTable(db: DatabaseSync, cfg: Config, baseDir: string): string[] {\n  const known = new Set((db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const insertBody = db.prepare(INSERT_CONTENT_SQL);\n  const warnings: string[] = [];\n  db.exec('BEGIN');\n  try {\n    for (const file of files) {\n      const { doc, warnings: fileWarnings } = parseFile(file);\n      warnings.push(...fileWarnings);\n      insertContentRow(insertBody, doc, segmenting);\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","segmentField","getColumns","getMeta","quoteIdent","setMeta","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","insertContentRow","insertBody","doc","segmenting","run","relPath","search","title","summary","text","reconcile","db","cfg","baseDir","files","currentSet","map","f","existingRows","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","feature","enabledForFile","fileWarnings","tick","push","key","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","txStart","Date","now","exec","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","undefined","values","ctimeMs","parseError","presetName","presets","store","extracted","name","afterReconcile","err","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","val","embedIdentityAdopted","embedPart","find","p","at","indexOf","signatureDiff","label","rebuildContentTable","known"],"mappings":"AAEA,SAASA,eAAe,QAAQ,qBAAqB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,iBAAiB;AAE1C,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,aAAa;AACpE,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,cAAc;AAEvE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,0FAA0F;AAC1F,2FAA2F;AAC3F,SAASC,iBAAiBC,UAAyB,EAAEC,GAAc,EAAEC,UAAmB;IACtFF,WAAWG,GAAG,CAACF,IAAIG,OAAO,EAAEH,IAAII,MAAM,CAACC,KAAK,EAAEL,IAAII,MAAM,CAACE,OAAO,EAAEN,IAAII,MAAM,CAACG,IAAI,EAAEP,IAAIG,OAAO,EAAEF,aAAaZ,aAAaW,IAAII,MAAM,CAACC,KAAK,IAAI,IAAIJ,aAAaZ,aAAaW,IAAII,MAAM,CAACE,OAAO,IAAI,IAAIL,aAAaZ,aAAaW,IAAII,MAAM,CAACG,IAAI,IAAI;AACrP;AAEA,OAAO,SAASC,UAAUC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtE,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIlB,IAAIiB,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAeP,GAAGQ,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMC,WAAW,IAAIC,IAAIJ,aAAaF,GAAG,CAAC,CAACO,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWP,aAAaQ,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGR,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYd,MAAMY,MAAM,CAAC,CAACT;QAC9B,MAAMY,MAAMR,SAASS,GAAG,CAACb,EAAEZ,OAAO;QAClC,OAAO,CAACwB,OAAOA,IAAIE,MAAM,KAAKd,EAAEe,OAAO,IAAIH,IAAII,KAAK,KAAKhB,EAAEiB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWpD,eAAe0B;IAChC,MAAM2B,cAAc/C,WAAWmB;IAC/B,MAAM6B,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAASvD,SAAS,mBAAmByC,UAAUO,MAAM;IAC3D,IAAIQ,cAAc;IAClB,KAAK,MAAMC,QAAQhB,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMiB,eAAeP,SAASZ,MAAM,CAAC,CAACoB,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACnC,KAAKgC;QACzG,MAAM,EAAE1C,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMG,OAAOC,OAAOC,IAAI,CAACnD,IAAIoD,IAAI,EAAG;YACvC,IAAI,CAACf,YAAYZ,GAAG,CAACwB,MAAM;gBACzBZ,YAAYgB,GAAG,CAACJ;gBAChBX,WAAWU,IAAI,CAACC;YAClB;QACF;QACAV,WAAWS,IAAI,CAAChD;IAClB;IACAwC,OAAOc,MAAM;IAEb,MAAMC,aAAa;WAAIlB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIkB,WAAWtB,MAAM,GAAGrC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAEwE,WAAWtB,MAAM,CAAC,0EAA0E,EAAErC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAM4D,kBAAkBD,WAAW/B,MAAM,CAAC,CAACiC,IAAM/D,yBAAyB+B,GAAG,CAACgC,MAAM,CAACrE,iBAAiBqC,GAAG,CAACgC;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB1C,GAAG,CAACtB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB1C,GAAG,CAAC,IAAM,KAAK6C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLhC,MAAM,CAAC,CAACiC,IAAMA,MAAM,QACpB3C,GAAG,CAAC,CAAC2C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQlC,UAAUF,MAAM,CAAC,CAACT,IAAM,CAACI,SAASM,GAAG,CAACV,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAM0D,QAAwB;QAAEjD;QAAOkD,UAAUvB,WAAWzB,GAAG,CAAC,CAACiD,IAAMA,EAAE5D,OAAO;QAAGyD;QAAOrC;IAAS;IAEnG,MAAMyC,UAAUC,KAAKC,GAAG;IACxBzD,GAAG0D,IAAI,CAAC;IACR,IAAI;YAkD8BvB;QAjDhC,KAAK,MAAMwB,OAAO9B,WAAY7B,GAAG0D,IAAI,CAAC,CAAC,mCAAmC,EAAE3E,WAAW4E,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAU5D,GAAGQ,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMqD,iBAAiB7D,GAAGQ,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMsD,mBAAmB9D,GAAGQ,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAIM,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMuC,MAAM/D,GAAGQ,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAMK,QAAQC,SAAU;oBAMKqB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrEyB,QAAQnE,GAAG,CAACoB;gBACZkD,IAAItE,GAAG,CAACoB;gBACRgD,eAAepE,GAAG,CAACoB;gBACnB,KAAK,MAAMsB,WAAWR,UAAUQ,kBAAAA,QAAQ6B,MAAM,cAAd7B,sCAAAA,qBAAAA,SAAiBnC,IAAIa,MAAMuC;YAC7D;QACF;QACA,IAAItB,WAAWN,MAAM,GAAG,GAAG;YACzB,MAAMyC,SAASjE,GAAGQ,OAAO,CAACyC;YAC1B,MAAM3D,aAAaU,GAAGQ,OAAO,CAACpB;YAC9B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMI,aAAanB,gBAAgB4B,SAASiE;YAC5C,KAAK,MAAM3E,OAAOuC,WAAY;oBAsBIK;gBArBhC,MAAMgC,SAASpB,gBAAgB1C,GAAG,CAAC,CAACsD;wBAO3BpE;oBANP,IAAIoE,QAAQ,QAAQ,OAAOpE,IAAIG,OAAO;oBACtC,IAAIiE,QAAQ,UAAU,OAAOpE,IAAI8B,OAAO;oBACxC,IAAIsC,QAAQ,UAAU,OAAOpE,IAAI6E,OAAO;oBACxC,IAAIT,QAAQ,SAAS,OAAOpE,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIoC,QAAQ,gBAAgB,OAAOpE,IAAI8E,UAAU;oBACjD,QAAO9E,gBAAAA,IAAIoD,IAAI,CAACgB,IAAI,cAAbpE,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0E,OAAOxE,GAAG,IAAI0E;gBACd,oFAAoF;gBACpF,IAAIzD,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAG;wBAEGyC;oBADhCyB,QAAQnE,GAAG,CAACF,IAAIG,OAAO;oBACvB,KAAK,MAAMyC,WAAWR,UAAUQ,mBAAAA,QAAQ6B,MAAM,cAAd7B,uCAAAA,sBAAAA,SAAiBnC,IAAIT,IAAIG,OAAO,EAAE0D;gBACpE;gBACA/D,iBAAiBC,YAAYC,KAAKC;gBAClC,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIkB,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAGmE,eAAepE,GAAG,CAACF,IAAIG,OAAO;gBAC7D,KAAK,MAAM4E,cAAc/E,IAAIgF,OAAO,CAAET,iBAAiBrE,GAAG,CAACF,IAAIG,OAAO,EAAE4E;gBACxE,KAAK,MAAMnC,WAAWR,UAAUQ,iBAAAA,QAAQqC,KAAK,cAAbrC,qCAAAA,oBAAAA,SAAgBnC,IAAIT,IAAIG,OAAO,EAAEH,IAAIkF,SAAS,CAACtC,QAAQuC,IAAI,CAAC,EAAEtB;YAChG;QACF;QACA,KAAK,MAAMjB,WAAWR,UAAUQ,0BAAAA,QAAQwC,cAAc,cAAtBxC,8CAAAA,6BAAAA,SAAyBnC,IAAIoD;QAC7DpD,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAarB,KAAKC,GAAG,KAAKF;IAChC,MAAMuB,UAAUhG,QAAQkB,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM+E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS/F,QAAQgB,IAAI,oBAAoBiF,OAAOJ;IAEjE,OAAO;QAAEpD,QAAQK,WAAWN,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASwD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGvC,IAAI,CAAC,OAAOoC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAIhF,IAAIgF,IAAIH,KAAK,CAAC,KAAKnF,GAAG,CAAC,CAACiF,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAI5G;IACpB,KAAK,MAAM,CAACsD,KAAKuD,IAAI,IAAIF,EAAG,IAAID,EAAEzE,GAAG,CAACqB,SAASuD,KAAKD,QAAQlD,GAAG,CAACJ;IAChE,KAAK,MAAMA,OAAOoD,EAAElD,IAAI,GAAI,IAAI,CAACmD,EAAE7E,GAAG,CAACwB,MAAMsD,QAAQlD,GAAG,CAACJ;IACzD,OAAOsD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASE,qBAAqBb,MAAc,EAAEC,KAAa;IAChE,MAAMa,YAAY,CAACN,MAAgBA,IAAIH,KAAK,CAAC,KAAKU,IAAI,CAAC,CAACC,IAAMA,EAAEZ,UAAU,CAAC;IAC3E,MAAMM,IAAII,UAAUd;IACpB,MAAMS,IAAIK,UAAUb;IACpB,IAAIS,MAAM3B,aAAa0B,MAAM1B,WAAW,OAAO;IAC/C,MAAMkC,KAAKR,EAAES,OAAO,CAAC;IACrB,OAAOR,EAAEQ,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKR,EAAEH,KAAK,CAAC,GAAGW,QAAQP;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASS,cAAcnB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMmB,QAAQ,CAAC/D,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAI+C,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAE/C,IAAIiD,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQvE,IAAI,KAAK,IAAI,aAAa;WAAIuE;KAAQ,CAACzF,GAAG,CAACkG,OAAOrD,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,SAASsD,oBAAoBxG,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAChF,MAAMuG,QAAQ,IAAIvH,IAAI,AAACc,GAAGQ,OAAO,CAAC,kCAAkCC,GAAG,GAA+BJ,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IACvH,MAAMV,QAAQ1B,UAAUwB,KAAKC,SAASa,MAAM,CAAC,CAACT,IAAMmG,MAAMzF,GAAG,CAACV,EAAEZ,OAAO;IACvE,MAAMF,aAAanB,gBAAgB4B,SAASiE;IAC5C,MAAM5E,aAAaU,GAAGQ,OAAO,CAACpB;IAC9B,MAAMsC,WAAqB,EAAE;IAC7B1B,GAAG0D,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAMzB,QAAQ9B,MAAO;YACxB,MAAM,EAAEZ,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD;YAClDP,SAASa,IAAI,IAAIF;YACjBhD,iBAAiBC,YAAYC,KAAKC;QACpC;QACAQ,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IACA,OAAOlD;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/reconcile.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport type { Config } from '../config/index.ts';\nimport { contentTokenize } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../scan/index.ts';\nimport { segmentField } from '../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from './shared.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so content population is\n// defined once. Assumes the frontmatter row for doc.relPath already exists (rowid lookup).\nfunction insertContentRow(insertBody: StatementSync, doc: ParsedDoc, segmenting: boolean): void {\n  insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures, cfg);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(INSERT_CONTENT_SQL);\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertContentRow(insertBody, doc, segmenting);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport function rebuildContentTable(db: DatabaseSync, cfg: Config, baseDir: string): string[] {\n  const known = new Set((db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const insertBody = db.prepare(INSERT_CONTENT_SQL);\n  const warnings: string[] = [];\n  db.exec('BEGIN');\n  try {\n    for (const file of files) {\n      const { doc, warnings: fileWarnings } = parseFile(file);\n      warnings.push(...fileWarnings);\n      insertContentRow(insertBody, doc, segmenting);\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","segmentField","getColumns","getMeta","quoteIdent","setMeta","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","insertContentRow","insertBody","doc","segmenting","run","relPath","search","title","summary","text","reconcile","db","cfg","baseDir","files","currentSet","map","f","existingRows","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","feature","enabledForFile","fileWarnings","tick","push","key","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","txStart","Date","now","exec","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","undefined","values","ctimeMs","parseError","presetName","presets","store","extracted","name","afterReconcile","err","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","val","embedIdentityAdopted","embedPart","find","p","at","indexOf","signatureDiff","label","rebuildContentTable","known"],"mappings":"AAEA,SAASA,eAAe,QAAQ,qBAAqB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AAEjD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAC1E,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,cAAc;AAEvE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,0FAA0F;AAC1F,2FAA2F;AAC3F,SAASC,iBAAiBC,UAAyB,EAAEC,GAAc,EAAEC,UAAmB;IACtFF,WAAWG,GAAG,CAACF,IAAIG,OAAO,EAAEH,IAAII,MAAM,CAACC,KAAK,EAAEL,IAAII,MAAM,CAACE,OAAO,EAAEN,IAAII,MAAM,CAACG,IAAI,EAAEP,IAAIG,OAAO,EAAEF,aAAaZ,aAAaW,IAAII,MAAM,CAACC,KAAK,IAAI,IAAIJ,aAAaZ,aAAaW,IAAII,MAAM,CAACE,OAAO,IAAI,IAAIL,aAAaZ,aAAaW,IAAII,MAAM,CAACG,IAAI,IAAI;AACrP;AAEA,OAAO,SAASC,UAAUC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtE,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIlB,IAAIiB,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAeP,GAAGQ,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMC,WAAW,IAAIC,IAAIJ,aAAaF,GAAG,CAAC,CAACO,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWP,aAAaQ,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGR,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYd,MAAMY,MAAM,CAAC,CAACT;QAC9B,MAAMY,MAAMR,SAASS,GAAG,CAACb,EAAEZ,OAAO;QAClC,OAAO,CAACwB,OAAOA,IAAIE,MAAM,KAAKd,EAAEe,OAAO,IAAIH,IAAII,KAAK,KAAKhB,EAAEiB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWpD,eAAe0B;IAChC,MAAM2B,cAAc/C,WAAWmB;IAC/B,MAAM6B,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAASvD,SAAS,mBAAmByC,UAAUO,MAAM;IAC3D,IAAIQ,cAAc;IAClB,KAAK,MAAMC,QAAQhB,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMiB,eAAeP,SAASZ,MAAM,CAAC,CAACoB,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACnC,KAAKgC;QACzG,MAAM,EAAE1C,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD,MAAMC,cAAcjC;QACtE8B,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMG,OAAOC,OAAOC,IAAI,CAACnD,IAAIoD,IAAI,EAAG;YACvC,IAAI,CAACf,YAAYZ,GAAG,CAACwB,MAAM;gBACzBZ,YAAYgB,GAAG,CAACJ;gBAChBX,WAAWU,IAAI,CAACC;YAClB;QACF;QACAV,WAAWS,IAAI,CAAChD;IAClB;IACAwC,OAAOc,MAAM;IAEb,MAAMC,aAAa;WAAIlB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIkB,WAAWtB,MAAM,GAAGrC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAEwE,WAAWtB,MAAM,CAAC,0EAA0E,EAAErC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAM4D,kBAAkBD,WAAW/B,MAAM,CAAC,CAACiC,IAAM/D,yBAAyB+B,GAAG,CAACgC,MAAM,CAACrE,iBAAiBqC,GAAG,CAACgC;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB1C,GAAG,CAACtB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB1C,GAAG,CAAC,IAAM,KAAK6C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLhC,MAAM,CAAC,CAACiC,IAAMA,MAAM,QACpB3C,GAAG,CAAC,CAAC2C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQlC,UAAUF,MAAM,CAAC,CAACT,IAAM,CAACI,SAASM,GAAG,CAACV,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAM0D,QAAwB;QAAEjD;QAAOkD,UAAUvB,WAAWzB,GAAG,CAAC,CAACiD,IAAMA,EAAE5D,OAAO;QAAGyD;QAAOrC;IAAS;IAEnG,MAAMyC,UAAUC,KAAKC,GAAG;IACxBzD,GAAG0D,IAAI,CAAC;IACR,IAAI;YAkD8BvB;QAjDhC,KAAK,MAAMwB,OAAO9B,WAAY7B,GAAG0D,IAAI,CAAC,CAAC,mCAAmC,EAAE3E,WAAW4E,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAU5D,GAAGQ,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMqD,iBAAiB7D,GAAGQ,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMsD,mBAAmB9D,GAAGQ,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAIM,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMuC,MAAM/D,GAAGQ,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAMK,QAAQC,SAAU;oBAMKqB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrEyB,QAAQnE,GAAG,CAACoB;gBACZkD,IAAItE,GAAG,CAACoB;gBACRgD,eAAepE,GAAG,CAACoB;gBACnB,KAAK,MAAMsB,WAAWR,UAAUQ,kBAAAA,QAAQ6B,MAAM,cAAd7B,sCAAAA,qBAAAA,SAAiBnC,IAAIa,MAAMuC;YAC7D;QACF;QACA,IAAItB,WAAWN,MAAM,GAAG,GAAG;YACzB,MAAMyC,SAASjE,GAAGQ,OAAO,CAACyC;YAC1B,MAAM3D,aAAaU,GAAGQ,OAAO,CAACpB;YAC9B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMI,aAAanB,gBAAgB4B,SAASiE;YAC5C,KAAK,MAAM3E,OAAOuC,WAAY;oBAsBIK;gBArBhC,MAAMgC,SAASpB,gBAAgB1C,GAAG,CAAC,CAACsD;wBAO3BpE;oBANP,IAAIoE,QAAQ,QAAQ,OAAOpE,IAAIG,OAAO;oBACtC,IAAIiE,QAAQ,UAAU,OAAOpE,IAAI8B,OAAO;oBACxC,IAAIsC,QAAQ,UAAU,OAAOpE,IAAI6E,OAAO;oBACxC,IAAIT,QAAQ,SAAS,OAAOpE,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIoC,QAAQ,gBAAgB,OAAOpE,IAAI8E,UAAU;oBACjD,QAAO9E,gBAAAA,IAAIoD,IAAI,CAACgB,IAAI,cAAbpE,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0E,OAAOxE,GAAG,IAAI0E;gBACd,oFAAoF;gBACpF,IAAIzD,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAG;wBAEGyC;oBADhCyB,QAAQnE,GAAG,CAACF,IAAIG,OAAO;oBACvB,KAAK,MAAMyC,WAAWR,UAAUQ,mBAAAA,QAAQ6B,MAAM,cAAd7B,uCAAAA,sBAAAA,SAAiBnC,IAAIT,IAAIG,OAAO,EAAE0D;gBACpE;gBACA/D,iBAAiBC,YAAYC,KAAKC;gBAClC,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIkB,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAGmE,eAAepE,GAAG,CAACF,IAAIG,OAAO;gBAC7D,KAAK,MAAM4E,cAAc/E,IAAIgF,OAAO,CAAET,iBAAiBrE,GAAG,CAACF,IAAIG,OAAO,EAAE4E;gBACxE,KAAK,MAAMnC,WAAWR,UAAUQ,iBAAAA,QAAQqC,KAAK,cAAbrC,qCAAAA,oBAAAA,SAAgBnC,IAAIT,IAAIG,OAAO,EAAEH,IAAIkF,SAAS,CAACtC,QAAQuC,IAAI,CAAC,EAAEtB;YAChG;QACF;QACA,KAAK,MAAMjB,WAAWR,UAAUQ,0BAAAA,QAAQwC,cAAc,cAAtBxC,8CAAAA,6BAAAA,SAAyBnC,IAAIoD;QAC7DpD,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAarB,KAAKC,GAAG,KAAKF;IAChC,MAAMuB,UAAUhG,QAAQkB,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM+E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS/F,QAAQgB,IAAI,oBAAoBiF,OAAOJ;IAEjE,OAAO;QAAEpD,QAAQK,WAAWN,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASwD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGvC,IAAI,CAAC,OAAOoC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAIhF,IAAIgF,IAAIH,KAAK,CAAC,KAAKnF,GAAG,CAAC,CAACiF,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAI5G;IACpB,KAAK,MAAM,CAACsD,KAAKuD,IAAI,IAAIF,EAAG,IAAID,EAAEzE,GAAG,CAACqB,SAASuD,KAAKD,QAAQlD,GAAG,CAACJ;IAChE,KAAK,MAAMA,OAAOoD,EAAElD,IAAI,GAAI,IAAI,CAACmD,EAAE7E,GAAG,CAACwB,MAAMsD,QAAQlD,GAAG,CAACJ;IACzD,OAAOsD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASE,qBAAqBb,MAAc,EAAEC,KAAa;IAChE,MAAMa,YAAY,CAACN,MAAgBA,IAAIH,KAAK,CAAC,KAAKU,IAAI,CAAC,CAACC,IAAMA,EAAEZ,UAAU,CAAC;IAC3E,MAAMM,IAAII,UAAUd;IACpB,MAAMS,IAAIK,UAAUb;IACpB,IAAIS,MAAM3B,aAAa0B,MAAM1B,WAAW,OAAO;IAC/C,MAAMkC,KAAKR,EAAES,OAAO,CAAC;IACrB,OAAOR,EAAEQ,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKR,EAAEH,KAAK,CAAC,GAAGW,QAAQP;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASS,cAAcnB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMmB,QAAQ,CAAC/D,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAI+C,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAE/C,IAAIiD,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQvE,IAAI,KAAK,IAAI,aAAa;WAAIuE;KAAQ,CAACzF,GAAG,CAACkG,OAAOrD,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,SAASsD,oBAAoBxG,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAChF,MAAMuG,QAAQ,IAAIvH,IAAI,AAACc,GAAGQ,OAAO,CAAC,kCAAkCC,GAAG,GAA+BJ,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IACvH,MAAMV,QAAQ1B,UAAUwB,KAAKC,SAASa,MAAM,CAAC,CAACT,IAAMmG,MAAMzF,GAAG,CAACV,EAAEZ,OAAO;IACvE,MAAMF,aAAanB,gBAAgB4B,SAASiE;IAC5C,MAAM5E,aAAaU,GAAGQ,OAAO,CAACpB;IAC9B,MAAMsC,WAAqB,EAAE;IAC7B1B,GAAG0D,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAMzB,QAAQ9B,MAAO;YACxB,MAAM,EAAEZ,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD;YAClDP,SAASa,IAAI,IAAIF;YACjBhD,iBAAiBC,YAAYC,KAAKC;QACpC;QACAQ,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IACA,OAAOlD;AACT"}

package/dist/esm/{sql-functions.js → db/sql-functions.js}

@@ -1,5 +1,5 @@
 import posix from 'node:path/posix';
-import { segmentMatch } from './segment.js';
+import { segmentMatch } from '../text/segment.js';
 // has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.
 export function registerFunctions(db, segmenting) {
     db.function('has', {

package/dist/esm/db/sql-functions.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/sql-functions.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { DatabaseSync } from 'node:sqlite';\nimport { segmentMatch } from '../text/segment.ts';\n\n// has(field, value): JSON-array field -> membership, string field -> substring, NULL -> false.\nexport function registerFunctions(db: DatabaseSync, segmenting: boolean): void {\n  db.function('has', { deterministic: true, varargs: false }, (field: unknown, value: unknown): number => {\n    if (field === null || field === undefined) return 0;\n\n    const needle = String(value);\n\n    if (typeof field === 'string') {\n      if (field.startsWith('[')) {\n        try {\n          const parsed = JSON.parse(field);\n          if (Array.isArray(parsed)) {\n            return parsed.some((item) => String(item) === needle) ? 1 : 0;\n          }\n        } catch {}\n      }\n      return field.includes(needle) ? 1 : 0;\n    }\n\n    return String(field).includes(needle) ? 1 : 0;\n  });\n\n  // Unix basename(path, suffix?): the filename, minus suffix when it ends with one. SQLite\n  // itself has no filename function, and the LIKE tricks that fake one also match folder names.\n  db.function('basename', { deterministic: true, varargs: true }, (path: unknown, suffix?: unknown): string | null => {\n    if (path === null || path === undefined) return null;\n    const name = posix.basename(String(path));\n    const tail = suffix === null || suffix === undefined ? '' : String(suffix);\n    // POSIX: a suffix identical to the whole name is not removed (node's own basename strips it).\n    return tail !== '' && tail !== name && name.endsWith(tail) ? name.slice(0, -tail.length) : name;\n  });\n\n  // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back, and\n  // matches nothing, so the same transform is available to hand-written SQL by name. `segmenting`\n  // is the same config predicate commands.ts's search() gates on (contentTokenize(cfg) ===\n  // undefined), passed in by open() rather than read back from persisted state.\n  db.function('segment', { deterministic: true, varargs: false }, (terms: unknown): string => {\n    const text = terms === null || terms === undefined ? '' : String(terms);\n    return segmenting ? segmentMatch(text) : text;\n  });\n}\n"],"names":["posix","segmentMatch","registerFunctions","db","segmenting","function","deterministic","varargs","field","value","undefined","needle","String","startsWith","parsed","JSON","parse","Array","isArray","some","item","includes","path","suffix","name","basename","tail","endsWith","slice","length","terms","text"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,SAASC,YAAY,QAAQ,qBAAqB;AAElD,+FAA+F;AAC/F,OAAO,SAASC,kBAAkBC,EAAgB,EAAEC,UAAmB;IACrED,GAAGE,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACC,OAAgBC;QAC3E,IAAID,UAAU,QAAQA,UAAUE,WAAW,OAAO;QAElD,MAAMC,SAASC,OAAOH;QAEtB,IAAI,OAAOD,UAAU,UAAU;YAC7B,IAAIA,MAAMK,UAAU,CAAC,MAAM;gBACzB,IAAI;oBACF,MAAMC,SAASC,KAAKC,KAAK,CAACR;oBAC1B,IAAIS,MAAMC,OAAO,CAACJ,SAAS;wBACzB,OAAOA,OAAOK,IAAI,CAAC,CAACC,OAASR,OAAOQ,UAAUT,UAAU,IAAI;oBAC9D;gBACF,EAAE,OAAM,CAAC;YACX;YACA,OAAOH,MAAMa,QAAQ,CAACV,UAAU,IAAI;QACtC;QAEA,OAAOC,OAAOJ,OAAOa,QAAQ,CAACV,UAAU,IAAI;IAC9C;IAEA,yFAAyF;IACzF,8FAA8F;IAC9FR,GAAGE,QAAQ,CAAC,YAAY;QAAEC,eAAe;QAAMC,SAAS;IAAK,GAAG,CAACe,MAAeC;QAC9E,IAAID,SAAS,QAAQA,SAASZ,WAAW,OAAO;QAChD,MAAMc,OAAOxB,MAAMyB,QAAQ,CAACb,OAAOU;QACnC,MAAMI,OAAOH,WAAW,QAAQA,WAAWb,YAAY,KAAKE,OAAOW;QACnE,8FAA8F;QAC9F,OAAOG,SAAS,MAAMA,SAASF,QAAQA,KAAKG,QAAQ,CAACD,QAAQF,KAAKI,KAAK,CAAC,GAAG,CAACF,KAAKG,MAAM,IAAIL;IAC7F;IAEA,0FAA0F;IAC1F,gGAAgG;IAChG,yFAAyF;IACzF,8EAA8E;IAC9ErB,GAAGE,QAAQ,CAAC,WAAW;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,CAACuB;QAC/D,MAAMC,OAAOD,UAAU,QAAQA,UAAUpB,YAAY,KAAKE,OAAOkB;QACjE,OAAO1B,aAAaH,aAAa8B,QAAQA;IAC3C;AACF"}

package/dist/esm/embed/langfit.js

@@ -1,3 +1,4 @@
+import { franc } from 'franc-min';
 import { getMeta, setMeta } from '../db/shared.js';
 import { SenseError } from '../errors.js';
 import { toIso3 } from './languages.js';
@@ -12,7 +13,7 @@ export function languageDistribution(db) {
     const raw = getMeta(db, META_KEY);
     return raw ? JSON.parse(raw) : undefined;
 }
-function classify(franc, text) {
+function classify(text) {
     const code = franc(text.slice(0, SAMPLE_CHARS));
     return code === 'und' ? undefined : code;
 }
@@ -21,14 +22,13 @@ function classify(franc, text) {
 export async function checkLanguageFit(db, provider, texts) {
     var _languageDistribution;
     if (texts.length === 0) return;
-    const { franc } = await import('franc-min');
     const persisted = (_languageDistribution = languageDistribution(db)) !== null && _languageDistribution !== void 0 ? _languageDistribution : {};
     const merged = {
         ...persisted
     };
     for (const text of texts){
         var _merged_code;
-        const code = classify(franc, text);
+        const code = classify(text);
         if (code) merged[code] = ((_merged_code = merged[code]) !== null && _merged_code !== void 0 ? _merged_code : 0) + 1;
     }
     const total = Object.values(merged).reduce((a, b)=>a + b, 0);

package/dist/esm/embed/langfit.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getMeta, setMeta } from '../db/shared.ts';\nimport { SenseError } from '../errors.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst META_KEY = 'embed_languages';\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\ntype LangCounts = Record<string, number>;\n\nexport function languageDistribution(db: DatabaseSync): LangCounts | undefined {\n  const raw = getMeta(db, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nfunction classify(franc: (text: string) => string, text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const { franc } = await import('franc-min');\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(franc, text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  setMeta(db, META_KEY, JSON.stringify(merged));\n}\n"],"names":["getMeta","setMeta","SenseError","toIso3","META_KEY","SAMPLE_CHARS","MIN_CLASSIFIED","languageDistribution","db","raw","JSON","parse","undefined","classify","franc","text","code","slice","checkLanguageFit","provider","texts","length","persisted","merged","total","Object","values","reduce","a","b","languages","declared","Set","map","majorityLang","majorityCount","entries","sort","has","pct","Math","round","id","join","stringify"],"mappings":"AACA,SAASA,OAAO,EAAEC,OAAO,QAAQ,kBAAkB;AACnD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,MAAM,QAAQ,iBAAiB;AAGxC,MAAMC,WAAW;AACjB,+FAA+F;AAC/F,8FAA8F;AAC9F,MAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,MAAMC,iBAAiB;AAIvB,OAAO,SAASC,qBAAqBC,EAAgB;IACnD,MAAMC,MAAMT,QAAQQ,IAAIJ;IACxB,OAAOK,MAAMC,KAAKC,KAAK,CAACF,OAAOG;AACjC;AAEA,SAASC,SAASC,KAA+B,EAAEC,IAAY;IAC7D,MAAMC,OAAOF,MAAMC,KAAKE,KAAK,CAAC,GAAGZ;IACjC,OAAOW,SAAS,QAAQJ,YAAYI;AACtC;AAEA,+FAA+F;AAC/F,+EAA+E;AAC/E,OAAO,eAAeE,iBAAiBV,EAAgB,EAAEW,QAAuB,EAAEC,KAAe;QAG7Eb;IAFlB,IAAIa,MAAMC,MAAM,KAAK,GAAG;IACxB,MAAM,EAAEP,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC;IAC/B,MAAMQ,aAAYf,wBAAAA,qBAAqBC,iBAArBD,mCAAAA,wBAA4B,CAAC;IAC/C,MAAMgB,SAAS;QAAE,GAAGD,SAAS;IAAC;IAC9B,KAAK,MAAMP,QAAQK,MAAO;YAEEG;QAD1B,MAAMP,OAAOH,SAASC,OAAOC;QAC7B,IAAIC,MAAMO,MAAM,CAACP,KAAK,GAAG,EAACO,eAAAA,MAAM,CAACP,KAAK,cAAZO,0BAAAA,eAAgB,KAAK;IACjD;IAEA,MAAMC,QAAQC,OAAOC,MAAM,CAACH,QAAQI,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG;IAC5D,IAAIV,SAASW,SAAS,IAAIX,SAASW,SAAS,CAACT,MAAM,GAAG,KAAKG,SAASlB,gBAAgB;QAClF,MAAMyB,WAAW,IAAIC,IAAIb,SAASW,SAAS,CAACG,GAAG,CAAC9B;QAChD,MAAM,CAAC+B,cAAcC,cAAc,GAAGV,OAAOW,OAAO,CAACb,QAAQc,IAAI,CAAC,CAACT,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC3F,IAAIO,gBAAgBX,SAAS,OAAO,CAACO,SAASO,GAAG,CAACJ,eAAe;YAC/D,MAAMK,MAAMC,KAAKC,KAAK,CAAC,AAACN,gBAAgBX,QAAS;YACjD,MAAM,IAAItB,WACR,wBACA,CAAC,YAAY,EAAEiB,SAASuB,EAAE,CAAC,qBAAqB,EAAEvB,SAASW,SAAS,CAACa,IAAI,CAAC,MAAM,OAAO,EAAEJ,IAAI,qCAAqC,EAAEL,aAAa,0JAA0J,CAAC;QAEhT;IACF;IACAjC,QAAQO,IAAIJ,UAAUM,KAAKkC,SAAS,CAACrB;AACvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { franc } from 'franc-min';\nimport { getMeta, setMeta } from '../db/shared.ts';\nimport { SenseError } from '../errors.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst META_KEY = 'embed_languages';\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\ntype LangCounts = Record<string, number>;\n\nexport function languageDistribution(db: DatabaseSync): LangCounts | undefined {\n  const raw = getMeta(db, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nfunction classify(text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  setMeta(db, META_KEY, JSON.stringify(merged));\n}\n"],"names":["franc","getMeta","setMeta","SenseError","toIso3","META_KEY","SAMPLE_CHARS","MIN_CLASSIFIED","languageDistribution","db","raw","JSON","parse","undefined","classify","text","code","slice","checkLanguageFit","provider","texts","length","persisted","merged","total","Object","values","reduce","a","b","languages","declared","Set","map","majorityLang","majorityCount","entries","sort","has","pct","Math","round","id","join","stringify"],"mappings":"AACA,SAASA,KAAK,QAAQ,YAAY;AAClC,SAASC,OAAO,EAAEC,OAAO,QAAQ,kBAAkB;AACnD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,MAAM,QAAQ,iBAAiB;AAGxC,MAAMC,WAAW;AACjB,+FAA+F;AAC/F,8FAA8F;AAC9F,MAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,MAAMC,iBAAiB;AAIvB,OAAO,SAASC,qBAAqBC,EAAgB;IACnD,MAAMC,MAAMT,QAAQQ,IAAIJ;IACxB,OAAOK,MAAMC,KAAKC,KAAK,CAACF,OAAOG;AACjC;AAEA,SAASC,SAASC,IAAY;IAC5B,MAAMC,OAAOhB,MAAMe,KAAKE,KAAK,CAAC,GAAGX;IACjC,OAAOU,SAAS,QAAQH,YAAYG;AACtC;AAEA,+FAA+F;AAC/F,+EAA+E;AAC/E,OAAO,eAAeE,iBAAiBT,EAAgB,EAAEU,QAAuB,EAAEC,KAAe;QAE7EZ;IADlB,IAAIY,MAAMC,MAAM,KAAK,GAAG;IACxB,MAAMC,aAAYd,wBAAAA,qBAAqBC,iBAArBD,mCAAAA,wBAA4B,CAAC;IAC/C,MAAMe,SAAS;QAAE,GAAGD,SAAS;IAAC;IAC9B,KAAK,MAAMP,QAAQK,MAAO;YAEEG;QAD1B,MAAMP,OAAOF,SAASC;QACtB,IAAIC,MAAMO,MAAM,CAACP,KAAK,GAAG,EAACO,eAAAA,MAAM,CAACP,KAAK,cAAZO,0BAAAA,eAAgB,KAAK;IACjD;IAEA,MAAMC,QAAQC,OAAOC,MAAM,CAACH,QAAQI,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG;IAC5D,IAAIV,SAASW,SAAS,IAAIX,SAASW,SAAS,CAACT,MAAM,GAAG,KAAKG,SAASjB,gBAAgB;QAClF,MAAMwB,WAAW,IAAIC,IAAIb,SAASW,SAAS,CAACG,GAAG,CAAC7B;QAChD,MAAM,CAAC8B,cAAcC,cAAc,GAAGV,OAAOW,OAAO,CAACb,QAAQc,IAAI,CAAC,CAACT,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC3F,IAAIO,gBAAgBX,SAAS,OAAO,CAACO,SAASO,GAAG,CAACJ,eAAe;YAC/D,MAAMK,MAAMC,KAAKC,KAAK,CAAC,AAACN,gBAAgBX,QAAS;YACjD,MAAM,IAAIrB,WACR,wBACA,CAAC,YAAY,EAAEgB,SAASuB,EAAE,CAAC,qBAAqB,EAAEvB,SAASW,SAAS,CAACa,IAAI,CAAC,MAAM,OAAO,EAAEJ,IAAI,qCAAqC,EAAEL,aAAa,0JAA0J,CAAC;QAEhT;IACF;IACAhC,QAAQO,IAAIJ,UAAUM,KAAKiC,SAAS,CAACrB;AACvC"}

package/dist/esm/embed/languages.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/languages.ts"],"sourcesContent":["// HF model cards declare languages as ISO 639-1 tags (e.g. \"zh\"); franc-min reports ISO 639-3\n// (e.g. \"cmn\"). This bridges the two for comparison. Limited to franc-min's own 82-language\n// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min\n// can actually emit; a declared tag with no entry here is compared as-is.\nconst ISO_639_1_TO_3: Record<string, string> = {\n  ar: 'arb',\n  am: 'amh',\n  az: 'azj',\n  be: 'bel',\n  bg: 'bul',\n  bn: 'ben',\n  bs: 'bos',\n  cs: 'ces',\n  de: 'deu',\n  el: 'ell',\n  en: 'eng',\n  es: 'spa',\n  fa: 'pes',\n  fr: 'fra',\n  gu: 'guj',\n  ha: 'hau',\n  hi: 'hin',\n  hr: 'hrv',\n  hu: 'hun',\n  id: 'ind',\n  ig: 'ibo',\n  it: 'ita',\n  ja: 'jpn',\n  jv: 'jav',\n  kk: 'kaz',\n  kn: 'kan',\n  ko: 'kor',\n  ml: 'mal',\n  mr: 'mar',\n  ms: 'zlm',\n  my: 'mya',\n  ne: 'npi',\n  nl: 'nld',\n  pa: 'pan',\n  pl: 'pol',\n  pt: 'por',\n  ro: 'ron',\n  ru: 'rus',\n  rw: 'kin',\n  si: 'sin',\n  so: 'som',\n  sr: 'srp',\n  su: 'sun',\n  sv: 'swe',\n  sw: 'swh',\n  ta: 'tam',\n  te: 'tel',\n  th: 'tha',\n  tl: 'tgl',\n  tr: 'tur',\n  uk: 'ukr',\n  ur: 'urd',\n  uz: 'uzn',\n  vi: 'vie',\n  yo: 'yor',\n  zh: 'cmn',\n  zu: 'zul',\n};\n\nexport function isKnownLanguageTag(tag: string): boolean {\n  return tag in ISO_639_1_TO_3;\n}\n\n// Normalizes a declared or detected language code to ISO 639-3 for comparison; strips a\n// region/script subtag (\"zh-cn\" -> \"zh\") first. Codes already in 639-3 form pass through.\nexport function toIso3(code: string): string {\n  const base = code.toLowerCase().split('-')[0];\n  return ISO_639_1_TO_3[base] ?? base;\n}\n\n// Languages this project measured for models whose cards declare none, so the fit check is\n// not silently off for the shipped default; asserted here because it was measured, not attested.\nexport const MEASURED_MODEL_LANGUAGES: Record<string, string[]> = {\n  'minishlab/potion-retrieval-32M': ['en'], // BENCHMARKING.md bake-off; token-loss measurements\n};\n"],"names":["ISO_639_1_TO_3","ar","am","az","be","bg","bn","bs","cs","de","el","en","es","fa","fr","gu","ha","hi","hr","hu","id","ig","it","ja","jv","kk","kn","ko","ml","mr","ms","my","ne","nl","pa","pl","pt","ro","ru","rw","si","so","sr","su","sv","sw","ta","te","th","tl","tr","uk","ur","uz","vi","yo","zh","zu","isKnownLanguageTag","tag","toIso3","code","base","toLowerCase","split","MEASURED_MODEL_LANGUAGES"],"mappings":"AAAA,8FAA8F;AAC9F,4FAA4F;AAC5F,gGAAgG;AAChG,0EAA0E;AAC1E,MAAMA,iBAAyC;IAC7CC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;AACN;AAEA,OAAO,SAASC,mBAAmBC,GAAW;IAC5C,OAAOA,OAAO3D;AAChB;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,OAAO,SAAS4D,OAAOC,IAAY;QAE1B7D;IADP,MAAM8D,OAAOD,KAAKE,WAAW,GAAGC,KAAK,CAAC,IAAI,CAAC,EAAE;IAC7C,QAAOhE,uBAAAA,cAAc,CAAC8D,KAAK,cAApB9D,kCAAAA,uBAAwB8D;AACjC;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,OAAO,MAAMG,2BAAqD;IAChE,kCAAkC;QAAC;KAAK;AAC1C,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/languages.ts"],"sourcesContent":["// HF model cards declare languages as ISO 639-1 tags (e.g. \"zh\"); franc-min reports ISO 639-3\n// (e.g. \"cmn\"). This bridges the two for comparison. Limited to franc-min's own 82-language\n// data (README table, https://github.com/wooorm/franc), so every value here is a code franc-min\n// can actually emit; a declared tag with no entry here is compared as-is.\nconst ISO_639_1_TO_3: Record<string, string> = {\n  ar: 'arb',\n  am: 'amh',\n  az: 'azj',\n  be: 'bel',\n  bg: 'bul',\n  bn: 'ben',\n  bs: 'bos',\n  cs: 'ces',\n  de: 'deu',\n  el: 'ell',\n  en: 'eng',\n  es: 'spa',\n  fa: 'pes',\n  fr: 'fra',\n  gu: 'guj',\n  ha: 'hau',\n  hi: 'hin',\n  hr: 'hrv',\n  hu: 'hun',\n  id: 'ind',\n  ig: 'ibo',\n  it: 'ita',\n  ja: 'jpn',\n  jv: 'jav',\n  kk: 'kaz',\n  kn: 'kan',\n  ko: 'kor',\n  ml: 'mal',\n  mr: 'mar',\n  ms: 'zlm',\n  my: 'mya',\n  ne: 'npi',\n  nl: 'nld',\n  pa: 'pan',\n  pl: 'pol',\n  pt: 'por',\n  ro: 'ron',\n  ru: 'rus',\n  rw: 'kin',\n  si: 'sin',\n  so: 'som',\n  sr: 'srp',\n  su: 'sun',\n  sv: 'swe',\n  sw: 'swh',\n  ta: 'tam',\n  te: 'tel',\n  th: 'tha',\n  tl: 'tgl',\n  tr: 'tur',\n  uk: 'ukr',\n  ur: 'urd',\n  uz: 'uzn',\n  vi: 'vie',\n  yo: 'yor',\n  zh: 'cmn',\n  zu: 'zul',\n};\n\nexport function isKnownLanguageTag(tag: string): boolean {\n  return tag in ISO_639_1_TO_3;\n}\n\n// Normalizes a declared or detected language code to ISO 639-3 for comparison; strips a\n// region/script subtag (\"zh-cn\" -> \"zh\") first. Codes already in 639-3 form pass through.\nexport function toIso3(code: string): string {\n  const base = code.toLowerCase().split('-')[0];\n  return ISO_639_1_TO_3[base] ?? base;\n}\n\n// Languages this project measured for models whose cards declare none, so the fit check is\n// not silently off for the shipped default; asserted here because it was measured, not attested.\nexport const MEASURED_MODEL_LANGUAGES: Record<string, string[]> = {\n  'minishlab/potion-retrieval-32M': ['en'], // benchmark/reports/2026-08-27-embedding-model-selection.md\n};\n"],"names":["ISO_639_1_TO_3","ar","am","az","be","bg","bn","bs","cs","de","el","en","es","fa","fr","gu","ha","hi","hr","hu","id","ig","it","ja","jv","kk","kn","ko","ml","mr","ms","my","ne","nl","pa","pl","pt","ro","ru","rw","si","so","sr","su","sv","sw","ta","te","th","tl","tr","uk","ur","uz","vi","yo","zh","zu","isKnownLanguageTag","tag","toIso3","code","base","toLowerCase","split","MEASURED_MODEL_LANGUAGES"],"mappings":"AAAA,8FAA8F;AAC9F,4FAA4F;AAC5F,gGAAgG;AAChG,0EAA0E;AAC1E,MAAMA,iBAAyC;IAC7CC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;IACJC,IAAI;AACN;AAEA,OAAO,SAASC,mBAAmBC,GAAW;IAC5C,OAAOA,OAAO3D;AAChB;AAEA,wFAAwF;AACxF,0FAA0F;AAC1F,OAAO,SAAS4D,OAAOC,IAAY;QAE1B7D;IADP,MAAM8D,OAAOD,KAAKE,WAAW,GAAGC,KAAK,CAAC,IAAI,CAAC,EAAE;IAC7C,QAAOhE,uBAAAA,cAAc,CAAC8D,KAAK,cAApB9D,kCAAAA,uBAAwB8D;AACjC;AAEA,2FAA2F;AAC3F,iGAAiG;AACjG,OAAO,MAAMG,2BAAqD;IAChE,kCAAkC;QAAC;KAAK;AAC1C,EAAE"}

package/dist/esm/embed/query.js

@@ -1,12 +1,12 @@
 import { join } from 'node:path';
 import { SenseError } from '../errors.js';
 import { embed } from '../features/embed.js';
-import { progress } from '../progress.js';
-import { parseFile } from '../scan.js';
+import { progress } from '../output/progress.js';
+import { parseFile } from '../scan/index.js';
 import { checkLanguageFit } from './langfit.js';
 import { getProvider } from './registry.js';
-// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is
-// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
+// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):
+// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
 const STORE_DIMS = 256;
 // Seed chunks that participate in a `related` scan; see similarNotes for the measurement.
 const TARGET_CHUNK_CAP = 16;
@@ -59,7 +59,7 @@ export async function embedPending(db, cfg, baseDir) {
                 embed: true
             }, [
                 embed
-            ]).doc.extracted.embed;
+            ], cfg).doc.extracted.embed;
         } catch  {
             continue; // vanished since reconcile; the next reconcile removes its rows
         }

package/dist/esm/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../progress.ts';\nimport { parseFile } from '../scan.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (BENCHMARKING.md): int8 at 256 dims is\n// quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed]).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["join","SenseError","embed","progress","parseFile","checkLanguageFit","getProvider","STORE_DIMS","TARGET_CHUNK_CAP","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","db","cfg","baseDir","provider","dirty","prepare","all","length","storeDims","min","byPath","Map","row","list","get","path","push","chunk","set","jobs","chunkIdxs","chunks","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","extracted","idx","text","map","j","update","report","i","batchCap","batch","slice","vectors","embedDocuments","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","err","tick","finish","asCosine","score","semanticCandidates","terms","fetch","allowed","match","filter","t","includes","qv","embedQuery","rows","best","has","vector","byteOffset","byteLength","dot","existing","lines","start_line","end_line","entries","sort","a","b","similarity","hasEmbedding","undefined","similarNotes","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":"AAAA,SAASA,IAAI,QAAQ,YAAY;AAGjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,iBAAiB;AAC1C,SAASC,SAAS,QAAQ,aAAa;AACvC,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAE5C,6EAA6E;AAC7E,yEAAyE;AACzE,MAAMC,aAAa;AACnB,0FAA0F;AAC1F,MAAMC,mBAAmB;AAEzB,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAaC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC/E,MAAMC,WAAW,MAAMpB,YAAYkB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQJ,GAAGK,OAAO,CAAC,oFAAoFC,GAAG;IAChH,IAAIF,MAAMG,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IAEpD,MAAMsB,SAAS,IAAIC;IACnB,KAAK,MAAMC,OAAOR,MAAO;YACVM;QAAb,MAAMG,QAAOH,cAAAA,OAAOI,GAAG,CAACF,IAAIG,IAAI,eAAnBL,yBAAAA,cAAwB,EAAE;QACvCG,KAAKG,IAAI,CAACJ,IAAIK,KAAK;QACnBP,OAAOQ,GAAG,CAACN,IAAIG,IAAI,EAAEF;IACvB;IAEA,MAAMM,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACJ,MAAMK,UAAU,IAAIV,OAAQ;QACtC,IAAIW;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAASxC,UAAU;gBAAEyC,SAASP;gBAAMQ,SAAS9C,KAAKyB,SAASa;gBAAOS,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAEhD,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEiD,GAAG,CAACC,SAAS,CAAClD,KAAK;QAC7J,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMmD,OAAOV,UAAW,IAAIC,MAAM,CAACS,IAAI,EAAEX,KAAKH,IAAI,CAAC;YAAED;YAAME,OAAOa;YAAKC,MAAMV,MAAM,CAACS,IAAI,CAACC,IAAI;QAAC;IACrG;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAMjD,iBACJkB,IACAG,UACAgB,KAAKa,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;IAGxB,MAAMG,SAASlC,GAAGK,OAAO,CAAC;IAC1B,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAM8B,SAASvD,SAAS,oBAAoBuC,KAAKZ,MAAM;IACvD,IAAK,IAAI6B,IAAI,GAAGA,IAAIjB,KAAKZ,MAAM,EAAE6B,KAAKjC,SAASkC,QAAQ,CAAE;QACvD,MAAMC,QAAQnB,KAAKoB,KAAK,CAACH,GAAGA,IAAIjC,SAASkC,QAAQ;QACjD,MAAMG,UAAU,MAAMrC,SAASsC,cAAc,CAACH,MAAMN,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;QACrE/B,GAAG0C,IAAI,CAAC;QACR,IAAI;YACFJ,MAAMK,OAAO,CAAC,CAACC,KAAKX;gBAClB,MAAM,EAAE3C,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQsD,OAAO,CAACP,EAAE,EAAEzB,WAAW;gBACpD,MAAMqC,IAAI,IAAIC,UAAUtC;gBACxB,IAAK,IAAIf,IAAI,GAAGA,IAAIe,WAAWf,IAAKoD,CAAC,CAACpD,EAAE,GAAGC,KAAKqD,KAAK,CAACzD,CAAC,CAACG,EAAE,GAAGG;gBAC7DsC,OAAOc,GAAG,CAACpD,OAAOqD,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAI7B,IAAI,EAAE6B,IAAI3B,KAAK;YAC9D;YACAjB,GAAG0C,IAAI,CAAC;QACV,EAAE,OAAOU,KAAK;YACZpD,GAAG0C,IAAI,CAAC;YACR,MAAMU;QACR;QACAjB,OAAOkB,IAAI,CAAC3D,KAAKe,GAAG,CAAC2B,IAAIjC,SAASkC,QAAQ,EAAElB,KAAKZ,MAAM;IACzD;IACA4B,OAAOmB,MAAM;AACf;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAO9D,KAAKqD,KAAK,CAACrD,KAAKe,GAAG,CAAC,GAAGf,KAAKG,GAAG,CAAC,CAAC,GAAG2D,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmBzD,EAAgB,EAAEC,GAAW,EAAEyD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;QAO3GF;IANd,MAAMxD,UAAU,AAACD,IAAgCC,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIxB,WAAW,eAAe;IAClD,MAAMqB,aAAaC,IAAIC,KAAKC;IAE5B,MAAMC,WAAW,MAAMpB,YAAYkB;IACnC,MAAMO,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IACpD,MAAM2C,OAAO,EAAC2B,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAEI,MAAM,CAAC,CAACC,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAItF,IAAI,CAAC;IAClH,MAAM,EAAEa,GAAG2E,EAAE,EAAE,GAAG/E,QAAQ,MAAMiB,SAAS+D,UAAU,CAACnC,OAAOvB,WAAW;IAEtE,MAAM2D,OAAOnE,GAAGK,OAAO,CAAC,+FAA+FC,GAAG;IAQ1H,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIP,WAAW,CAACA,QAAQS,GAAG,CAACzD,IAAIG,IAAI,GAAG;QACvC,MAAM8B,IAAI,IAAIC,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE7E,KAAKe,GAAG,CAACD,WAAWI,IAAI0D,MAAM,CAACE,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIhF,IAAI,GAAGA,IAAIoD,EAAEtC,MAAM,EAAEd,IAAKgF,OAAO5B,CAAC,CAACpD,EAAE,GAAGwE,EAAE,CAACxE,EAAE;QACtD,MAAM+D,QAAQiB,MAAM7D,IAAIhB,KAAK;QAC7B,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;QAClC,IAAI,CAAC2D,YAAYlB,QAAQkB,SAASlB,KAAK,EAAEY,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAE;YAAEyC;YAAOmB,OAAO,CAAC,CAAC,EAAE/D,IAAIgE,UAAU,CAAC,CAAC,EAAEhE,IAAIiE,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK,EACtCjB,KAAK,CAAC,GAAGoB,OACT3B,GAAG,CAAC,CAAC,CAACjB,MAAMkE,EAAE,GAAM,CAAA;YAAElE;YAAM4D,OAAOM,EAAEN,KAAK;YAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;QAAE,CAAA;AAC/E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,SAAS2B,aAAanF,EAAgB,EAAEe,IAAY;IACzD,MAAMH,MAAMZ,GAAGK,OAAO,CAAC,kFAAkFS,GAAG,CAACC;IAC7G,OAAOH,QAAQwE;AACjB;AAEA,+FAA+F;AAC/F,uEAAuE;AACvE,OAAO,SAASC,aAAarF,EAAgB,EAAEsF,IAAY,EAAEvE,IAAY,EAAEwE,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,MAAMC,aAAaxF,GAAGK,OAAO,CAAC,+FAA+FC,GAAG,CAACS;IACjI,IAAIyE,WAAWjF,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMkF,OAAO/F,KAAKG,GAAG,CAAC,GAAGH,KAAKgG,IAAI,CAACF,WAAWjF,MAAM,GAAGtB;IACvD,MAAM0G,SAASH,WAAW1B,MAAM,CAAC,CAAC8B,GAAGxD,IAAMA,IAAIqD,SAAS,GAAGzD,GAAG,CAAC,CAACpB,MAAS,CAAA;YAAEtB,GAAG,IAAIwD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;YAAG5E,OAAOgB,IAAIhB,KAAK;QAAC,CAAA;IAE/K,MAAMuE,OAAOnE,GAAGK,OAAO,CAAC,yEAAyEC,GAAG;IACpG,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIvD,IAAIG,IAAI,KAAKA,QAAQwE,KAAKM,OAAO,CAACxB,GAAG,CAACzD,IAAIG,IAAI,KAAMwE,KAAK3B,OAAO,IAAI,CAAC2B,KAAK3B,OAAO,CAACS,GAAG,CAACzD,IAAIG,IAAI,GAAI;QACtG,MAAM+E,QAAQ,IAAIhD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;QAC3F,KAAK,MAAMT,KAAK4B,OAAQ;YACtB,IAAIlB,MAAM;YACV,MAAMsB,MAAMrG,KAAKe,GAAG,CAACsD,EAAEzE,CAAC,CAACiB,MAAM,EAAEuF,MAAMvF,MAAM;YAC7C,IAAK,IAAId,IAAI,GAAGA,IAAIsG,KAAKtG,IAAKgF,OAAOV,EAAEzE,CAAC,CAACG,EAAE,GAAGqG,KAAK,CAACrG,EAAE;YACtD,MAAM+D,QAAQiB,MAAMV,EAAEnE,KAAK,GAAGgB,IAAIhB,KAAK;YACvC,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;YAClC,IAAI2D,aAAaU,aAAa5B,QAAQkB,UAAUN,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAEyC;QACrE;IACF;IAEA,OAAO;WAAIY,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BzC,KAAK,CAAC,GAAGgD,KAAKS,CAAC,EACfhE,GAAG,CAAC,CAAC,CAACiE,GAAGzC,MAAM,GAAM,CAAA;YAAEzC,MAAMkF;YAAGf,YAAY3B,SAASC;QAAO,CAAA;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["join","SenseError","embed","progress","parseFile","checkLanguageFit","getProvider","STORE_DIMS","TARGET_CHUNK_CAP","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","db","cfg","baseDir","provider","dirty","prepare","all","length","storeDims","min","byPath","Map","row","list","get","path","push","chunk","set","jobs","chunkIdxs","chunks","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","extracted","idx","text","map","j","update","report","i","batchCap","batch","slice","vectors","embedDocuments","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","err","tick","finish","asCosine","score","semanticCandidates","terms","fetch","allowed","match","filter","t","includes","qv","embedQuery","rows","best","has","vector","byteOffset","byteLength","dot","existing","lines","start_line","end_line","entries","sort","a","b","similarity","hasEmbedding","undefined","similarNotes","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":"AAAA,SAASA,IAAI,QAAQ,YAAY;AAGjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAE5C,8FAA8F;AAC9F,6FAA6F;AAC7F,MAAMC,aAAa;AACnB,0FAA0F;AAC1F,MAAMC,mBAAmB;AAEzB,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAaC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC/E,MAAMC,WAAW,MAAMpB,YAAYkB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQJ,GAAGK,OAAO,CAAC,oFAAoFC,GAAG;IAChH,IAAIF,MAAMG,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IAEpD,MAAMsB,SAAS,IAAIC;IACnB,KAAK,MAAMC,OAAOR,MAAO;YACVM;QAAb,MAAMG,QAAOH,cAAAA,OAAOI,GAAG,CAACF,IAAIG,IAAI,eAAnBL,yBAAAA,cAAwB,EAAE;QACvCG,KAAKG,IAAI,CAACJ,IAAIK,KAAK;QACnBP,OAAOQ,GAAG,CAACN,IAAIG,IAAI,EAAEF;IACvB;IAEA,MAAMM,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACJ,MAAMK,UAAU,IAAIV,OAAQ;QACtC,IAAIW;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAASxC,UAAU;gBAAEyC,SAASP;gBAAMQ,SAAS9C,KAAKyB,SAASa;gBAAOS,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAEhD,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEsB,KAAK2B,GAAG,CAACC,SAAS,CAAClD,KAAK;QAClK,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMmD,OAAOV,UAAW,IAAIC,MAAM,CAACS,IAAI,EAAEX,KAAKH,IAAI,CAAC;YAAED;YAAME,OAAOa;YAAKC,MAAMV,MAAM,CAACS,IAAI,CAACC,IAAI;QAAC;IACrG;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAMjD,iBACJkB,IACAG,UACAgB,KAAKa,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;IAGxB,MAAMG,SAASlC,GAAGK,OAAO,CAAC;IAC1B,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAM8B,SAASvD,SAAS,oBAAoBuC,KAAKZ,MAAM;IACvD,IAAK,IAAI6B,IAAI,GAAGA,IAAIjB,KAAKZ,MAAM,EAAE6B,KAAKjC,SAASkC,QAAQ,CAAE;QACvD,MAAMC,QAAQnB,KAAKoB,KAAK,CAACH,GAAGA,IAAIjC,SAASkC,QAAQ;QACjD,MAAMG,UAAU,MAAMrC,SAASsC,cAAc,CAACH,MAAMN,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;QACrE/B,GAAG0C,IAAI,CAAC;QACR,IAAI;YACFJ,MAAMK,OAAO,CAAC,CAACC,KAAKX;gBAClB,MAAM,EAAE3C,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQsD,OAAO,CAACP,EAAE,EAAEzB,WAAW;gBACpD,MAAMqC,IAAI,IAAIC,UAAUtC;gBACxB,IAAK,IAAIf,IAAI,GAAGA,IAAIe,WAAWf,IAAKoD,CAAC,CAACpD,EAAE,GAAGC,KAAKqD,KAAK,CAACzD,CAAC,CAACG,EAAE,GAAGG;gBAC7DsC,OAAOc,GAAG,CAACpD,OAAOqD,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAI7B,IAAI,EAAE6B,IAAI3B,KAAK;YAC9D;YACAjB,GAAG0C,IAAI,CAAC;QACV,EAAE,OAAOU,KAAK;YACZpD,GAAG0C,IAAI,CAAC;YACR,MAAMU;QACR;QACAjB,OAAOkB,IAAI,CAAC3D,KAAKe,GAAG,CAAC2B,IAAIjC,SAASkC,QAAQ,EAAElB,KAAKZ,MAAM;IACzD;IACA4B,OAAOmB,MAAM;AACf;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAO9D,KAAKqD,KAAK,CAACrD,KAAKe,GAAG,CAAC,GAAGf,KAAKG,GAAG,CAAC,CAAC,GAAG2D,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmBzD,EAAgB,EAAEC,GAAW,EAAEyD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;QAO3GF;IANd,MAAMxD,UAAU,AAACD,IAAgCC,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIxB,WAAW,eAAe;IAClD,MAAMqB,aAAaC,IAAIC,KAAKC;IAE5B,MAAMC,WAAW,MAAMpB,YAAYkB;IACnC,MAAMO,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IACpD,MAAM2C,OAAO,EAAC2B,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAEI,MAAM,CAAC,CAACC,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAItF,IAAI,CAAC;IAClH,MAAM,EAAEa,GAAG2E,EAAE,EAAE,GAAG/E,QAAQ,MAAMiB,SAAS+D,UAAU,CAACnC,OAAOvB,WAAW;IAEtE,MAAM2D,OAAOnE,GAAGK,OAAO,CAAC,+FAA+FC,GAAG;IAQ1H,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIP,WAAW,CAACA,QAAQS,GAAG,CAACzD,IAAIG,IAAI,GAAG;QACvC,MAAM8B,IAAI,IAAIC,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE7E,KAAKe,GAAG,CAACD,WAAWI,IAAI0D,MAAM,CAACE,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIhF,IAAI,GAAGA,IAAIoD,EAAEtC,MAAM,EAAEd,IAAKgF,OAAO5B,CAAC,CAACpD,EAAE,GAAGwE,EAAE,CAACxE,EAAE;QACtD,MAAM+D,QAAQiB,MAAM7D,IAAIhB,KAAK;QAC7B,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;QAClC,IAAI,CAAC2D,YAAYlB,QAAQkB,SAASlB,KAAK,EAAEY,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAE;YAAEyC;YAAOmB,OAAO,CAAC,CAAC,EAAE/D,IAAIgE,UAAU,CAAC,CAAC,EAAEhE,IAAIiE,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK,EACtCjB,KAAK,CAAC,GAAGoB,OACT3B,GAAG,CAAC,CAAC,CAACjB,MAAMkE,EAAE,GAAM,CAAA;YAAElE;YAAM4D,OAAOM,EAAEN,KAAK;YAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;QAAE,CAAA;AAC/E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,SAAS2B,aAAanF,EAAgB,EAAEe,IAAY;IACzD,MAAMH,MAAMZ,GAAGK,OAAO,CAAC,kFAAkFS,GAAG,CAACC;IAC7G,OAAOH,QAAQwE;AACjB;AAEA,+FAA+F;AAC/F,uEAAuE;AACvE,OAAO,SAASC,aAAarF,EAAgB,EAAEsF,IAAY,EAAEvE,IAAY,EAAEwE,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,MAAMC,aAAaxF,GAAGK,OAAO,CAAC,+FAA+FC,GAAG,CAACS;IACjI,IAAIyE,WAAWjF,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMkF,OAAO/F,KAAKG,GAAG,CAAC,GAAGH,KAAKgG,IAAI,CAACF,WAAWjF,MAAM,GAAGtB;IACvD,MAAM0G,SAASH,WAAW1B,MAAM,CAAC,CAAC8B,GAAGxD,IAAMA,IAAIqD,SAAS,GAAGzD,GAAG,CAAC,CAACpB,MAAS,CAAA;YAAEtB,GAAG,IAAIwD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;YAAG5E,OAAOgB,IAAIhB,KAAK;QAAC,CAAA;IAE/K,MAAMuE,OAAOnE,GAAGK,OAAO,CAAC,yEAAyEC,GAAG;IACpG,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIvD,IAAIG,IAAI,KAAKA,QAAQwE,KAAKM,OAAO,CAACxB,GAAG,CAACzD,IAAIG,IAAI,KAAMwE,KAAK3B,OAAO,IAAI,CAAC2B,KAAK3B,OAAO,CAACS,GAAG,CAACzD,IAAIG,IAAI,GAAI;QACtG,MAAM+E,QAAQ,IAAIhD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;QAC3F,KAAK,MAAMT,KAAK4B,OAAQ;YACtB,IAAIlB,MAAM;YACV,MAAMsB,MAAMrG,KAAKe,GAAG,CAACsD,EAAEzE,CAAC,CAACiB,MAAM,EAAEuF,MAAMvF,MAAM;YAC7C,IAAK,IAAId,IAAI,GAAGA,IAAIsG,KAAKtG,IAAKgF,OAAOV,EAAEzE,CAAC,CAACG,EAAE,GAAGqG,KAAK,CAACrG,EAAE;YACtD,MAAM+D,QAAQiB,MAAMV,EAAEnE,KAAK,GAAGgB,IAAIhB,KAAK;YACvC,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;YAClC,IAAI2D,aAAaU,aAAa5B,QAAQkB,UAAUN,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAEyC;QACrE;IACF;IAEA,OAAO;WAAIY,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BzC,KAAK,CAAC,GAAGgD,KAAKS,CAAC,EACfhE,GAAG,CAAC,CAAC,CAACiE,GAAGzC,MAAM,GAAM,CAAA;YAAEzC,MAAMkF;YAAGf,YAAY3B,SAASC;QAAO,CAAA;AACjE"}

package/dist/esm/embed/static.js

@@ -1,5 +1,6 @@
 import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
+import { Tokenizer } from '@huggingface/tokenizers';
 import { SenseError } from '../errors.js';
 import { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.js';
 const BATCH_CAP = 64;
@@ -34,8 +35,6 @@ export async function staticProvider(model) {
     const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];
     const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));
     const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));
-    // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.
-    const { Tokenizer } = await import('@huggingface/tokenizers');
     const tok = new Tokenizer(tokenizerJson, {});
     const unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
     function one(text) {

package/dist/esm/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  // Lazy import: the tokenizer loads only on the semantic path, never at CLI startup.\n  const { Tokenizer } = await import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","staticProvider","model","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeC,eAAeC,KAAa;;QAgCLC,sBAA7BA,4BAAAA;IA/Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIV,eAAeO,QAAQ;QACzB,qFAAqF;QACrFE,MAAM,MAAMV,cAAcQ,OAAO,CAACI,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAEP,MAAM,CAAC,EAAEI,KAAK,MAAM,EAAEC,MAAM;QAC9G,uFAAuF;QACvF,MAAMG,SAASX,cAAcG;QAC7BG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMN,SAASI;QACf,KAAK,MAAMI,QAAQT,YAAa;YAC9B,IAAI,CAACP,WAAWE,KAAKY,KAAKE,QAAQ;gBAChC,MAAM,IAAIb,WAAW,uBAAuB,CAAC,YAAY,EAAES,MAAM,6BAA6B,EAAEE,IAAI,YAAY,EAAER,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMiB,MAAMtB,aAAaC,KAAKY,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIlC,WAAW,eAAe,GAAGS,MAAM,oCAAoC,CAAC;IAC1H,MAAM0B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAC5B,aAAaC,KAAKY,KAAK,mBAAmB;IAC3E,oFAAoF;IACpF,MAAM,EAAEkC,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC;IACnC,MAAMC,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAE/C,OAAO;QAAE2B;QAAM4B,UAAUzD;QAAWK;QAAWqD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Tokenizer } from '@huggingface/tokenizers';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","join","Tokenizer","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","staticProvider","model","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,SAAS,QAAQ,0BAA0B;AACpD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeC,eAAeC,KAAa;;QA8BLC,sBAA7BA,4BAAAA;IA7Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIV,eAAeO,QAAQ;QACzB,qFAAqF;QACrFE,MAAM,MAAMV,cAAcQ,OAAO,CAACI,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAEP,MAAM,CAAC,EAAEI,KAAK,MAAM,EAAEC,MAAM;QAC9G,uFAAuF;QACvF,MAAMG,SAASX,cAAcG;QAC7BG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMN,SAASI;QACf,KAAK,MAAMI,QAAQT,YAAa;YAC9B,IAAI,CAACR,WAAWE,KAAKa,KAAKE,QAAQ;gBAChC,MAAM,IAAIb,WAAW,uBAAuB,CAAC,YAAY,EAAES,MAAM,6BAA6B,EAAEE,IAAI,YAAY,EAAER,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMiB,MAAMvB,aAAaC,KAAKa,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIlC,WAAW,eAAe,GAAGS,MAAM,oCAAoC,CAAC;IAC1H,MAAM0B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAC7B,aAAaC,KAAKa,KAAK,mBAAmB;IAC3E,MAAMkC,MAAM,IAAI9C,UAAUW,eAAe,CAAC;IAC1C,MAAMoC,iBAAQpC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBqC,KAAK,cAA1BrC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBsC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOjC,OAAOkC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIf,aAAaN;QAC3B,IAAIe,IAAIjC,MAAM,KAAK,GAAG,OAAOuC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKnB;YACjB,IAAK,IAAIuB,IAAI,GAAGA,IAAIvB,MAAMuB,IAAKF,CAAC,CAACE,EAAE,IAAIlB,MAAM,CAACiB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIvB,MAAMuB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIjC,MAAM;YAClB0C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIvB,MAAMuB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAE9C,OAAO;QAAE2B;QAAM2B,UAAUxD;QAAWK;QAAWoD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}

package/dist/esm/features/embed.d.ts

@@ -1,7 +1,4 @@
+import type { Chunk } from '../chunk/index.js';
 import type { Feature } from './types.js';
-export interface Chunk {
-    startLine: number;
-    endLine: number;
-    text: string;
-}
+export type { Chunk };
 export declare const embed: Feature;

package/dist/esm/features/embed.js

@@ -1,52 +1,40 @@
-import { fenceTracker } from '../fences.js';
-// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,
-// preamble kept, whole body when no headings; the title/summary prefix mirrors bm25 weighting.
+import { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.js';
+import { embedConfig } from '../config/index.js';
+import { modelIdentity } from '../embed/identity.js';
+// Deterministic, so embed time can re-derive text from stored line ranges; the title/summary
+// prefix mirrors bm25 field weighting, and an empty chunk is dropped before the prefix is added.
 //
 // Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw
 // file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw
 // put the frontmatter block in its own leading chunk on every note with a heading, which made
 // `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also
 // disagreed with FTS, which indexes the body alone.
-function chunksOf(body, search, offset = 0) {
-    const lines = body.split('\n');
-    const starts = [];
-    const fence = fenceTracker();
-    for(let i = 0; i < lines.length; i++){
-        if (fence.feed(lines[i])) continue;
-        if (!fence.inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);
-    }
-    const bounds = starts.length === 0 ? [
-        1
-    ] : starts[0] > 1 ? [
-        1,
-        ...starts
-    ] : starts;
+function chunksOf(blocks, body, search, offset = 0, chunkTokens) {
     const prefix = [
         search === null || search === void 0 ? void 0 : search.title,
         search === null || search === void 0 ? void 0 : search.summary
     ].filter(Boolean).join('\n');
-    const chunks = [];
-    bounds.forEach((start, i)=>{
-        const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;
-        const text = lines.slice(start - 1, end).join('\n').trim();
-        // A body with nothing in it yields no chunks at all, so a frontmatter-only note has no
-        // vectors rather than a vector of its own YAML.
-        if (text.length > 0) chunks.push({
-            startLine: start + offset,
-            endLine: end + offset,
-            text: prefix ? `${prefix}\n${text}` : text
-        });
-    });
-    return chunks;
+    // chunk()'s default text mode is 'raw' (D9), the shipped choice. A hardcoded option change
+    // here must bump src/chunk/version.ts's CHUNK_VERSION -- the signature for every chunker
+    // input not already carried in config.
+    return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? {
+        targetTokens: chunkTokens
+    } : undefined).map((c)=>({
+            startLine: c.startLine + offset,
+            endLine: c.endLine + offset,
+            text: prefix ? `${prefix}\n${c.text}` : c.text
+        }));
 }
 export const embed = {
     name: 'embed',
     schema (db) {
         db.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))`);
     },
-    extract (raw, body, search) {
+    extract (raw, body, search, _data, cfg, blocks) {
+        var _cfg_embed;
         // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.
-        return chunksOf(body, search, raw.split('\n').length - body.split('\n').length);
+        // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.
+        return chunksOf(blocks !== null && blocks !== void 0 ? blocks : parse(body), body, search, raw.split('\n').length - body.split('\n').length, cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
     },
     remove (db, path) {
         db.prepare('DELETE FROM embeddings WHERE "path" = ?').run(path);
@@ -60,5 +48,19 @@ export const embed = {
     },
     enabledForFile (_cfg, file) {
         return file.embed;
+    },
+    // Provider/model/chunk-version/identity, keyed so a rebuild notice can name it "embed
+    // settings". Driven by embedConfig(cfg) directly (a model can be configured with no preset
+    // yet using vectors), not by whether this feature is itself active.
+    signature (cfg) {
+        const e = embedConfig(cfg);
+        if (!e) return 'embed:off';
+        // Weight identity from identity.ts, no network: a static model's resolved sha or local
+        // size+mtime, appended once known so changed weights re-embed.
+        const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;
+        // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker
+        // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.
+        const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;
+        return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;
     }
 };

package/dist/esm/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import { fenceTracker } from '../fences.ts';\nimport type { Feature } from './types.ts';\n\n// Heading-based chunks, int8 vectors with a per-vector scale, NULL vector = not yet embedded.\n// Reconcile writes dirty rows; embedding tops up on the next search, so staleness costs recall.\n// Providers, the vector fill, and the search-time helpers live under src/embed/.\n\nexport interface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n\n// Deterministic, so embed time can re-derive text from stored line ranges. Heading-delimited,\n// preamble kept, whole body when no headings; the title/summary prefix mirrors bm25 weighting.\n//\n// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw\n// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw\n// put the frontmatter block in its own leading chunk on every note with a heading, which made\n// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also\n// disagreed with FTS, which indexes the body alone.\nfunction chunksOf(body: string, search?: { title: string; summary: string }, offset = 0): Chunk[] {\n  const lines = body.split('\\n');\n  const starts: number[] = [];\n  const fence = fenceTracker();\n  for (let i = 0; i < lines.length; i++) {\n    if (fence.feed(lines[i])) continue;\n    if (!fence.inFence && /^#{1,6} +/.test(lines[i])) starts.push(i + 1);\n  }\n  const bounds = starts.length === 0 ? [1] : starts[0] > 1 ? [1, ...starts] : starts;\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  const chunks: Chunk[] = [];\n  bounds.forEach((start, i) => {\n    const end = i + 1 < bounds.length ? bounds[i + 1] - 1 : lines.length;\n    const text = lines\n      .slice(start - 1, end)\n      .join('\\n')\n      .trim();\n    // A body with nothing in it yields no chunks at all, so a frontmatter-only note has no\n    // vectors rather than a vector of its own YAML.\n    if (text.length > 0) chunks.push({ startLine: start + offset, endLine: end + offset, text: prefix ? `${prefix}\\n${text}` : text });\n  });\n  return chunks;\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    return chunksOf(body, search, raw.split('\\n').length - body.split('\\n').length);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  // A tree with no embedding model never had extract() run for the doc (db.ts's per-file\n  // filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.\n  store(db, path, extracted) {\n    if (!extracted) return;\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n};\n"],"names":["fenceTracker","chunksOf","body","search","offset","lines","split","starts","fence","i","length","feed","inFence","test","push","bounds","prefix","title","summary","filter","Boolean","join","chunks","forEach","start","end","text","slice","trim","startLine","endLine","embed","name","schema","db","exec","extract","raw","remove","path","prepare","run","store","extracted","insert","c","idx","enabledForFile","_cfg","file"],"mappings":"AAAA,SAASA,YAAY,QAAQ,eAAe;AAa5C,8FAA8F;AAC9F,+FAA+F;AAC/F,EAAE;AACF,2FAA2F;AAC3F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,oDAAoD;AACpD,SAASC,SAASC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC;IACrF,MAAMC,QAAQH,KAAKI,KAAK,CAAC;IACzB,MAAMC,SAAmB,EAAE;IAC3B,MAAMC,QAAQR;IACd,IAAK,IAAIS,IAAI,GAAGA,IAAIJ,MAAMK,MAAM,EAAED,IAAK;QACrC,IAAID,MAAMG,IAAI,CAACN,KAAK,CAACI,EAAE,GAAG;QAC1B,IAAI,CAACD,MAAMI,OAAO,IAAI,YAAYC,IAAI,CAACR,KAAK,CAACI,EAAE,GAAGF,OAAOO,IAAI,CAACL,IAAI;IACpE;IACA,MAAMM,SAASR,OAAOG,MAAM,KAAK,IAAI;QAAC;KAAE,GAAGH,MAAM,CAAC,EAAE,GAAG,IAAI;QAAC;WAAMA;KAAO,GAAGA;IAC5E,MAAMS,SAAS;QAACb,mBAAAA,6BAAAA,OAAQc,KAAK;QAAEd,mBAAAA,6BAAAA,OAAQe,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,MAAMC,SAAkB,EAAE;IAC1BP,OAAOQ,OAAO,CAAC,CAACC,OAAOf;QACrB,MAAMgB,MAAMhB,IAAI,IAAIM,OAAOL,MAAM,GAAGK,MAAM,CAACN,IAAI,EAAE,GAAG,IAAIJ,MAAMK,MAAM;QACpE,MAAMgB,OAAOrB,MACVsB,KAAK,CAACH,QAAQ,GAAGC,KACjBJ,IAAI,CAAC,MACLO,IAAI;QACP,uFAAuF;QACvF,gDAAgD;QAChD,IAAIF,KAAKhB,MAAM,GAAG,GAAGY,OAAOR,IAAI,CAAC;YAAEe,WAAWL,QAAQpB;YAAQ0B,SAASL,MAAMrB;YAAQsB,MAAMV,SAAS,GAAGA,OAAO,EAAE,EAAEU,MAAM,GAAGA;QAAK;IAClI;IACA,OAAOJ;AACT;AAEA,OAAO,MAAMS,QAAiB;IAC5BC,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAC1K;IACAC,SAAQC,GAAG,EAAEnC,IAAI,EAAEC,MAAM;QACvB,mFAAmF;QACnF,OAAOF,SAASC,MAAMC,QAAQkC,IAAI/B,KAAK,CAAC,MAAMI,MAAM,GAAGR,KAAKI,KAAK,CAAC,MAAMI,MAAM;IAChF;IACA4B,QAAOJ,EAAE,EAAEK,IAAI;QACbL,GAAGM,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACA,uFAAuF;IACvF,mFAAmF;IACnFG,OAAMR,EAAE,EAAEK,IAAI,EAAEI,SAAS;QACvB,IAAI,CAACA,WAAW;QAChB,MAAMC,SAASV,GAAGM,OAAO,CAAC;QACzBG,UAAsBpB,OAAO,CAAC,CAACsB,GAAGC,MAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKD,EAAEhB,SAAS,EAAEgB,EAAEf,OAAO;IACzF;IACAiB,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKlB,KAAK;IACnB;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded. Reconcile writes dirty\n// rows; embedding tops up on the next search, so staleness costs recall. Providers, the vector\n// fill, and the search-time helpers live under src/embed/. Block/group structure (D1-D5) lives\n// in src/chunk; this keeps only the title/summary prefix and the empty-chunk drop (D6).\n\nexport type { Chunk };\n\n// Deterministic, so embed time can re-derive text from stored line ranges; the title/summary\n// prefix mirrors bm25 field weighting, and an empty chunk is dropped before the prefix is added.\n//\n// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw\n// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw\n// put the frontmatter block in its own leading chunk on every note with a heading, which made\n// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also\n// disagreed with FTS, which indexes the body alone.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // chunk()'s default text mode is 'raw' (D9), the shipped choice. A hardcoded option change\n  // here must bump src/chunk/version.ts's CHUNK_VERSION -- the signature for every chunker\n  // input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  schema(db) {\n    db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, raw.split('\\n').length - body.split('\\n').length, cfg?.embed?.chunkTokens);\n  },\n  remove(db, path) {\n    db.prepare('DELETE FROM embeddings WHERE \"path\" = ?').run(path);\n  },\n  // A tree with no embedding model never had extract() run for the doc (db.ts's per-file\n  // filter skips it), so extracted is undefined here -- store nothing, i.e. no rows.\n  store(db, path, extracted) {\n    if (!extracted) return;\n    const insert = db.prepare('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)');\n    (extracted as Chunk[]).forEach((c, idx) => insert.run(path, idx, c.startLine, c.endLine));\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Provider/model/chunk-version/identity, keyed so a rebuild notice can name it \"embed\n  // settings\". Driven by embedConfig(cfg) directly (a model can be configured with no preset\n  // yet using vectors), not by whether this feature is itself active.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","modelIdentity","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","split","length","remove","path","prepare","run","store","extracted","insert","forEach","idx","enabledForFile","_cfg","file","signature","e","identity","provider","model","chunkVersion"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,aAAa,QAAQ,uBAAuB;AAUrD,6FAA6F;AAC7F,iGAAiG;AACjG,EAAE;AACF,2FAA2F;AAC3F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,oDAAoD;AACpD,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,2FAA2F;IAC3F,yFAAyF;IACzF,uCAAuC;IACvC,OAAOf,gBAAgBK,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACNC,QAAOC,EAAE;QACPA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAC1K;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG4D0B;QAFvG,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUJ,MAAMK,OAAOA,MAAMC,QAAQsB,IAAIG,KAAK,CAAC,MAAMC,MAAM,GAAG3B,KAAK0B,KAAK,CAAC,MAAMC,MAAM,EAAEF,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAChI;IACAyB,QAAOR,EAAE,EAAES,IAAI;QACbT,GAAGU,OAAO,CAAC,2CAA2CC,GAAG,CAACF;IAC5D;IACA,uFAAuF;IACvF,mFAAmF;IACnFG,OAAMZ,EAAE,EAAES,IAAI,EAAEI,SAAS;QACvB,IAAI,CAACA,WAAW;QAChB,MAAMC,SAASd,GAAGU,OAAO,CAAC;QACzBG,UAAsBE,OAAO,CAAC,CAACtB,GAAGuB,MAAQF,OAAOH,GAAG,CAACF,MAAMO,KAAKvB,EAAEC,SAAS,EAAED,EAAEE,OAAO;IACzF;IACAsB,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKtB,KAAK;IACnB;IACA,sFAAsF;IACtF,2FAA2F;IAC3F,oEAAoE;IACpEuB,WAAUf,GAAG;QACX,MAAMgB,IAAI7C,YAAY6B;QACtB,IAAI,CAACgB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMC,WAAWD,EAAEE,QAAQ,KAAK,WAAW9C,cAAc4C,EAAEG,KAAK,IAAIlC;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMmC,eAAeJ,EAAEtC,WAAW,KAAKO,YAAY,GAAGjB,cAAc,CAAC,EAAEgD,EAAEtC,WAAW,EAAE,GAAGV;QACzF,OAAO,CAAC,MAAM,EAAEgD,EAAEE,QAAQ,CAAC,CAAC,EAAEF,EAAEG,KAAK,CAAC,CAAC,EAAEC,eAAeH,aAAahC,YAAY,CAAC,CAAC,EAAEgC,UAAU,GAAG,IAAI;IACxG;AACF,EAAE"}

package/dist/esm/{fences.js → features/fences.js}

@@ -1,6 +1,6 @@
 // Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column
 // 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see
-// test/unit/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself
+// test/unit/features/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself
 // contain a backtick (spec rule). A closer is a run of the SAME character, length >= the
 // opener's, holding nothing but trailing spaces after the run.
 const BACKTICK_OPEN = /^(`{3,})(.*)$/;

package/dist/esm/features/fences.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column\n// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see\n// test/unit/features/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself\n// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the\n// opener's, holding nothing but trailing spaces after the run.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n\n// Masks <!-- ... --> spans that may cross line boundaries, one line at a time so callers can\n// interleave it with their own per-line state. In isolation an unclosed <!-- stays open forever\n// (mask() has no notion of a line boundary that should kill it); maskRegions() below is what\n// calls close() at the point an unclosed comment actually dies -- a blank line at paragraph\n// level, or its enclosing HTML block's end.\nexport interface CommentTracker {\n  mask(line: string): string;\n  close(): void; // force-ends an unclosed comment (paragraph blank line, or its HTML block ending)\n  readonly inComment: boolean;\n}\n\nexport function commentTracker(): CommentTracker {\n  let inComment = false;\n  return {\n    mask(line: string): string {\n      let out = '';\n      let i = 0;\n      while (i < line.length) {\n        if (inComment) {\n          const end = line.indexOf('-->', i);\n          if (end === -1) {\n            out += ' '.repeat(line.length - i);\n            i = line.length;\n          } else {\n            out += ' '.repeat(end + 3 - i);\n            i = end + 3;\n            inComment = false;\n          }\n          continue;\n        }\n        const start = line.indexOf('<!--', i);\n        if (start === -1) {\n          out += line.slice(i);\n          i = line.length;\n        } else {\n          out += `${line.slice(i, start)}    `;\n          i = start + 4;\n          inComment = true;\n        }\n      }\n      return out;\n    },\n    close() {\n      inComment = false;\n    },\n    get inComment() {\n      return inComment;\n    },\n  };\n}\n\n// CommonMark's HTML-block type-6 list (fixed by the spec, not a drifting enumeration): a line\n// starting with an open or close tag of one of these, at column 0 (leading whitespace/`>`\n// blockquote markers tolerated), opens a block that swallows following lines -- tags, links,\n// and comments alike -- until a blank line closes it.\nconst HTML_BLOCK_TAGS = new Set([\n  'address',\n  'article',\n  'aside',\n  'base',\n  'basefont',\n  'blockquote',\n  'body',\n  'caption',\n  'center',\n  'col',\n  'colgroup',\n  'dd',\n  'details',\n  'dialog',\n  'dir',\n  'div',\n  'dl',\n  'dt',\n  'fieldset',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'frame',\n  'frameset',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'head',\n  'header',\n  'hr',\n  'html',\n  'iframe',\n  'legend',\n  'li',\n  'link',\n  'main',\n  'menu',\n  'menuitem',\n  'nav',\n  'noframes',\n  'ol',\n  'optgroup',\n  'option',\n  'p',\n  'param',\n  'search',\n  'section',\n  'summary',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'title',\n  'tr',\n  'track',\n  'ul',\n]);\n// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,\n// not on a blank line, and that line is the last one swallowed.\nconst HTML_PRE_TAGS = new Set(['script', 'pre', 'style', 'textarea']);\nconst HTML_PRE_CLOSE_RE = /<\\/(?:script|pre|style|textarea)>/i; // any of the four closers ends the block, not only the tag that opened it\n// An opening or closing tag at column 0, tag name captured for the lookups above.\nconst HTML_BLOCK_OPEN_RE = /^<\\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \\t]|\\/?>|$)/;\nconst BLANK_LINE_RE = /^[ \\t>]*$/; // whitespace/blockquote-markers only -- a paragraph break, and what ends a type-6 block\n\n// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines\n// preserved, so line numbers and offsets survive) plus inline code spans masked. The single\n// region masker for tags/links: fence and HTML-block delimiter+content lines are opaque, an\n// unclosed comment dies at its containing block's end (or the next blank line at paragraph\n// level -- never silently to end of file), and a closed comment masks exactly its span.\n// Bodies without a backtick/tilde/`<` fast-path out untouched.\nexport function maskRegions(body: string): string {\n  if (!/[`~<]/.test(body)) return body;\n  const fence = fenceTracker();\n  const comment = commentTracker();\n  let inHtmlBlock = false;\n  let htmlBlockClose: RegExp | null = null; // set while inside a type-1 (script/pre/style/textarea) block\n  return body\n    .split('\\n')\n    .map((line) => {\n      if (inHtmlBlock) {\n        if (htmlBlockClose) {\n          if (htmlBlockClose.test(line)) {\n            inHtmlBlock = false;\n            htmlBlockClose = null;\n          }\n          return ' '.repeat(line.length);\n        }\n        if (BLANK_LINE_RE.test(line)) {\n          // Block content never feeds the comment tracker, so an inner <!-- was only ever\n          // masked text; nothing to close here.\n          inHtmlBlock = false;\n          return line;\n        }\n        return ' '.repeat(line.length);\n      }\n      if (!comment.inComment) {\n        const isDelim = fence.feed(line);\n        if (isDelim || fence.inFence) return ' '.repeat(line.length);\n      }\n      const uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;\n      if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line\n      // Indented or blockquoted HTML blocks still swallow their content in Obsidian, so the\n      // opener test runs after stripping leading whitespace and > markers.\n      const stripped = uncommented.replace(/^[ \\t>]*/, '');\n      if (stripped[0] === '<') {\n        const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);\n        if (openMatch) {\n          const tagName = openMatch[1].toLowerCase();\n          const isClosingTag = stripped[1] === '/';\n          if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {\n            // The end condition can be met on the opening line itself: <script>x</script>\n            // is a one-line block and the next line parses (Obsidian-verified).\n            if (!HTML_PRE_CLOSE_RE.test(stripped.slice(openMatch[0].length))) {\n              inHtmlBlock = true;\n              htmlBlockClose = HTML_PRE_CLOSE_RE;\n            }\n            return ' '.repeat(line.length);\n          }\n          if (HTML_BLOCK_TAGS.has(tagName)) {\n            inHtmlBlock = true;\n            return ' '.repeat(line.length);\n          }\n        }\n      }\n      // Inline code spans hide their content too: `[[x]]` in prose is not a link.\n      return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;\n    })\n    .join('\\n');\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N -- a\n// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).\n// Masked with spaces so column positions and tag-boundary whitespace are unaffected.\nexport function maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n"],"names":["BACKTICK_OPEN","TILDE_OPEN","fenceTracker","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim","commentTracker","inComment","mask","out","i","end","indexOf","repeat","start","slice","close","HTML_BLOCK_TAGS","Set","HTML_PRE_TAGS","HTML_PRE_CLOSE_RE","HTML_BLOCK_OPEN_RE","BLANK_LINE_RE","maskRegions","body","test","fence","comment","inHtmlBlock","htmlBlockClose","split","map","isDelim","uncommented","stripped","replace","openMatch","tagName","toLowerCase","isClosingTag","has","maskCodeSpans","join","j","n","k","closeStart","closeEnd"],"mappings":"AAAA,+FAA+F;AAC/F,8FAA8F;AAC9F,wGAAwG;AACxG,yFAAyF;AACzF,+DAA+D;AAO/D,MAAMA,gBAAgB;AACtB,MAAMC,aAAa;AAEnB,OAAO,SAASC;IACd,IAAIC,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,MAAMK,KAAKR,cAAcS,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,MAAMC,KAAKX,WAAWQ,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,MAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,MAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAaA,OAAO,SAASa;IACd,IAAIC,YAAY;IAChB,OAAO;QACLC,MAAKX,IAAY;YACf,IAAIY,MAAM;YACV,IAAIC,IAAI;YACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;gBACtB,IAAIM,WAAW;oBACb,MAAMI,MAAMd,KAAKe,OAAO,CAAC,OAAOF;oBAChC,IAAIC,QAAQ,CAAC,GAAG;wBACdF,OAAO,IAAII,MAAM,CAAChB,KAAKI,MAAM,GAAGS;wBAChCA,IAAIb,KAAKI,MAAM;oBACjB,OAAO;wBACLQ,OAAO,IAAII,MAAM,CAACF,MAAM,IAAID;wBAC5BA,IAAIC,MAAM;wBACVJ,YAAY;oBACd;oBACA;gBACF;gBACA,MAAMO,QAAQjB,KAAKe,OAAO,CAAC,QAAQF;gBACnC,IAAII,UAAU,CAAC,GAAG;oBAChBL,OAAOZ,KAAKkB,KAAK,CAACL;oBAClBA,IAAIb,KAAKI,MAAM;gBACjB,OAAO;oBACLQ,OAAO,GAAGZ,KAAKkB,KAAK,CAACL,GAAGI,OAAO,IAAI,CAAC;oBACpCJ,IAAII,QAAQ;oBACZP,YAAY;gBACd;YACF;YACA,OAAOE;QACT;QACAO;YACET,YAAY;QACd;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,6FAA6F;AAC7F,sDAAsD;AACtD,MAAMU,kBAAkB,IAAIC,IAAI;IAC9B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AACD,8FAA8F;AAC9F,gEAAgE;AAChE,MAAMC,gBAAgB,IAAID,IAAI;IAAC;IAAU;IAAO;IAAS;CAAW;AACpE,MAAME,oBAAoB,sCAAsC,0EAA0E;AAC1I,kFAAkF;AAClF,MAAMC,qBAAqB;AAC3B,MAAMC,gBAAgB,aAAa,wFAAwF;AAE3H,+FAA+F;AAC/F,4FAA4F;AAC5F,4FAA4F;AAC5F,2FAA2F;AAC3F,wFAAwF;AACxF,+DAA+D;AAC/D,OAAO,SAASC,YAAYC,IAAY;IACtC,IAAI,CAAC,QAAQC,IAAI,CAACD,OAAO,OAAOA;IAChC,MAAME,QAAQlC;IACd,MAAMmC,UAAUrB;IAChB,IAAIsB,cAAc;IAClB,IAAIC,iBAAgC,MAAM,8DAA8D;IACxG,OAAOL,KACJM,KAAK,CAAC,MACNC,GAAG,CAAC,CAAClC;QACJ,IAAI+B,aAAa;YACf,IAAIC,gBAAgB;gBAClB,IAAIA,eAAeJ,IAAI,CAAC5B,OAAO;oBAC7B+B,cAAc;oBACdC,iBAAiB;gBACnB;gBACA,OAAO,IAAIhB,MAAM,CAAChB,KAAKI,MAAM;YAC/B;YACA,IAAIqB,cAAcG,IAAI,CAAC5B,OAAO;gBAC5B,gFAAgF;gBAChF,sCAAsC;gBACtC+B,cAAc;gBACd,OAAO/B;YACT;YACA,OAAO,IAAIgB,MAAM,CAAChB,KAAKI,MAAM;QAC/B;QACA,IAAI,CAAC0B,QAAQpB,SAAS,EAAE;YACtB,MAAMyB,UAAUN,MAAM9B,IAAI,CAACC;YAC3B,IAAImC,WAAWN,MAAMjC,OAAO,EAAE,OAAO,IAAIoB,MAAM,CAAChB,KAAKI,MAAM;QAC7D;QACA,MAAMgC,cAAcN,QAAQpB,SAAS,IAAIV,KAAKG,QAAQ,CAAC,UAAU2B,QAAQnB,IAAI,CAACX,QAAQA;QACtF,IAAI8B,QAAQpB,SAAS,IAAIe,cAAcG,IAAI,CAAC5B,OAAO8B,QAAQX,KAAK,IAAI,+CAA+C;QACnH,sFAAsF;QACtF,qEAAqE;QACrE,MAAMkB,WAAWD,YAAYE,OAAO,CAAC,YAAY;QACjD,IAAID,QAAQ,CAAC,EAAE,KAAK,KAAK;YACvB,MAAME,YAAYf,mBAAmBtB,IAAI,CAACmC;YAC1C,IAAIE,WAAW;gBACb,MAAMC,UAAUD,SAAS,CAAC,EAAE,CAACE,WAAW;gBACxC,MAAMC,eAAeL,QAAQ,CAAC,EAAE,KAAK;gBACrC,IAAI,CAACK,gBAAgBpB,cAAcqB,GAAG,CAACH,UAAU;oBAC/C,8EAA8E;oBAC9E,oEAAoE;oBACpE,IAAI,CAACjB,kBAAkBK,IAAI,CAACS,SAASnB,KAAK,CAACqB,SAAS,CAAC,EAAE,CAACnC,MAAM,IAAI;wBAChE2B,cAAc;wBACdC,iBAAiBT;oBACnB;oBACA,OAAO,IAAIP,MAAM,CAAChB,KAAKI,MAAM;gBAC/B;gBACA,IAAIgB,gBAAgBuB,GAAG,CAACH,UAAU;oBAChCT,cAAc;oBACd,OAAO,IAAIf,MAAM,CAAChB,KAAKI,MAAM;gBAC/B;YACF;QACF;QACA,4EAA4E;QAC5E,OAAOgC,YAAYjC,QAAQ,CAAC,OAAOyC,cAAcR,eAAeA;IAClE,GACCS,IAAI,CAAC;AACV;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,qFAAqF;AACrF,OAAO,SAASD,cAAc5C,IAAY;IACxC,IAAIY,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIb,KAAKI,MAAM,CAAE;QACtB,IAAIJ,IAAI,CAACa,EAAE,KAAK,KAAK;YACnBD,OAAOZ,IAAI,CAACa,EAAE;YACdA;YACA;QACF;QACA,IAAIiC,IAAIjC;QACR,MAAOb,IAAI,CAAC8C,EAAE,KAAK,IAAKA;QACxB,MAAMC,IAAID,IAAIjC;QACd,IAAImC,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAIhD,KAAKI,MAAM,CAAE;YACtB,IAAIJ,IAAI,CAACgD,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAIzC,IAAIyC;YACR,MAAOhD,IAAI,CAACO,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIyC,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAW3C;gBACX;YACF;YACAyC,IAAIzC;QACN;QACA,IAAI0C,cAAc,GAAG;YACnBrC,OAAO,IAAII,MAAM,CAACkC,WAAWrC;YAC7BA,IAAIqC;QACN,OAAO;YACLtC,OAAOZ,KAAKkB,KAAK,CAACL,GAAGiC;YACrBjC,IAAIiC;QACN;IACF;IACA,OAAOlC;AACT"}