sensemaking 0.18.1 → 0.18.2

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per\n// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain\n// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from\n// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one\n// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared\n// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the\n// only DuckDB type that can hold the different JS types mapValue() produces across files for\n// the same key).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity\n// fence anyway so a runaway frontmatter generator fails with a clear message instead of an\n// unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","reparseFiles","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","withTransaction","feature","rows","contentTouched","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":";;;;+BAmCsBA;;;eAAAA;;;wBAlCK;uBACI;0BAEN;wBAEmB;yBACf;wBACU;6BACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,4FAA4F;AAC5F,8FAA8F;AAC9F,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iBAAiB;AACjB,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,6FAA6F;AAC7F,2FAA2F;AAC3F,8BAA8B;AAC9B,IAAMC,0BAA0B;AAEzB,SAAeX,UAAUY,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAEAC,QAC6CC,eAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAIAC,iBACAC,WAKAC,OACAC,OACAC,UACAC;;;;oBApCArB,QAAQsB,IAAAA,mBAAS,EAACxB,KAAKC;oBACvBE,aAAa,IAAId,IAAIa,MAAMuB,GAAG,CAAC,SAACC;+BAAMA,EAAEjC,OAAO;;oBAEhC;;wBAAMM,KAAK4B,OAAO,CAAC;;;oBAAlCvB,eAAe;oBACC;;wBAAMA,aAAawB,GAAG;;;oBAAtCvB,eAAgB;oBAChBC,WAAW,IAAIuB,IAAIxB,aAAaoB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtDvB,WAAWF,aAAa2B,MAAM,CAAC,SAACF;+BAAM,CAAC3B,WAAW8B,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhFvB,YAAYN,MAAM8B,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM5B,SAAS6B,GAAG,CAACT,EAAEjC,OAAO;wBAClC,OAAO,CAACyC,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAIhC,SAASiC,MAAM,KAAK,KAAKhC,UAAUgC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG3B,QAAQ;wBAAK;;oBAEhFL,WAAWiC,IAAAA,uBAAc,EAAC1C;oBACZ;;wBAAM2C,IAAAA,oBAAU,EAAC5C;;;oBAA/BW,cAAc;oBAEdC,SAASiC,IAAAA,oBAAQ,EAAC,mBAAmBpC,UAAUgC,MAAM;oBACR5B,gBAAAA,IAAAA,uBAAY,EAACJ,WAAWC,UAAUT,KAAKU,aAAaC,OAAOkC,IAAI,GAApGhC,aAAqCD,cAA3CkC,MAAkBhC,WAAyBF,cAAzBE,UAAUC,aAAeH,cAAfG;oBACpCJ,OAAOoC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAahC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBN,YAAYsC,GAAG,CAAChC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGP;oBACvB,IAAIO,WAAWuB,MAAM,GAAG1C,yBAAyB;wBAC/C,MAAM,IAAImD,oBAAU,CAAC,gBAAgB,AAAC,0BAA2FnD,OAAlEmB,WAAWuB,MAAM,EAAC,kDAAwE,OAAxB1C,yBAAwB;oBAC3J;oBACMoB,kBAAkBD,WAAWe,MAAM,CAAC,SAACkB;+BAAM9D,yBAAyB6C,GAAG,CAACiB,MAAM,CAACC,0BAAgB,CAAClB,GAAG,CAACiB;;oBACpG/B,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBO,GAAG,CAAC2B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FnC,OAAhFA,gBAAgBO,GAAG,CAAC;+BAAM;uBAAK4B,IAAI,CAAC,OAAM,wCAGjI,OAHuKnC,gBAClLc,MAAM,CAAC,SAACkB;+BAAMA,MAAM;uBACpBzB,GAAG,CAAC,SAACyB;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFjC,QAAQZ,UAAUwB,MAAM,CAAC,SAACN;+BAAM,CAACpB,SAAS2B,GAAG,CAACP,EAAEjC,OAAO;uBAAGgC,GAAG,CAAC,SAACC;+BAAMA,EAAEjC,OAAO;;oBAC9E4B,QAAwB;wBAAEnB,OAAAA;wBAAOoD,UAAUzC,WAAWY,GAAG,CAAC,SAAC8B;mCAAMA,EAAE9D,OAAO;;wBAAG2B,OAAAA;wBAAOb,UAAAA;oBAAS;oBAC7Fe,WAAW,IAAIjC,IAAI+B;oBACnBG,mBAAmBV,WAAWY,GAAG,CAAC,SAAC8B;+BAAMA,EAAE9D,OAAO;uBAAEuC,MAAM,CAAC,SAACwB;+BAAM,CAAClC,SAASW,GAAG,CAACuB;;oBAEtF;;wBAAMC,IAAAA,8BAAe,EAAC1D,MAAM;;oCA8CyC2D,iBAK7BA,yBAhDjC,2BAAA,mBAAA,gBAAA,WAAA,OAAM1C,UAGH2C,MAaFC,gBAgBAC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAMtE,KAAwB,YAAA,QAAMuE,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAhDN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAa3C;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMjB,KAAKkE,IAAI,CAAC,AAAC,sCAAqD,OAAhBb,IAAAA,oBAAU,EAACpC,MAAK;;;4CAAtE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW2B,MAAM,GAAG,CAAA,GAApB3B;;;;4CACI8C,OAAO9C,WAAWY,GAAG,CAAC,SAACjC;uDAC3B0B,gBAAgBO,GAAG,CAAC,SAACT;wDAMZxB;oDALP,IAAIwB,QAAQ,QAAQ,OAAOxB,IAAIC,OAAO;oDACtC,IAAIuB,QAAQ,UAAU,OAAOxB,IAAI6C,OAAO;oDACxC,IAAIrB,QAAQ,UAAU,OAAOxB,IAAI0E,OAAO;oDACxC,IAAIlD,QAAQ,SAAS,OAAOxB,IAAI+C,IAAI;oDACpC,IAAIvB,QAAQ,gBAAgB,OAAOxB,IAAI2E,UAAU;oDACjD,QAAO3E,gBAAAA,IAAI4E,IAAI,CAACpD,IAAI,cAAbxB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMO,KAAKsE,QAAQ,CAAClD,WAAWwC;;;4CAA/B;;;4CAGIC,iBAAiB,AAAC,qBAAGrD,iBAAU,qBAAGgB;iDACpCqC,CAAAA,eAAepB,MAAM,GAAG,CAAA,GAAxBoB;;;;4CACF;;gDAAM7D,KAAKsE,QAAQ,CACjB,wCACAT,eAAenC,GAAG,CAAC,SAAC+B;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIE3C,CAAAA,WAAW2B,MAAM,GAAG,CAAA,GAApB3B;;;;4CAAuB;;gDAAMd,KAAKsE,QAAQ,CAAC/E,oBAAoBuB,WAAWY,GAAG,CAAClC;;;4CAAvD;;;iDAEvBgB,CAAAA,SAASiC,MAAM,GAAG,CAAA,GAAlBjC;;;;4CACF;;gDAAMR,KAAKsE,QAAQ,CACjB,4CACA9D,SAASkB,GAAG,CAAC,SAAC+B;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DK,gBAAgB,AAAC,qBAAGtD,iBAAU,qBAAGgB;iDACnCsC,CAAAA,cAAcrB,MAAM,GAAG,CAAA,GAAvBqB;;;;4CACF;;gDAAM9D,KAAKsE,QAAQ,CACjB,6CACAR,cAAcpC,GAAG,CAAC,SAAC+B;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIM;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAajD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMrB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAI8E,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMP,aAAN;4DAAiCD,WAAWS,IAAI;gEAAE/E,IAAIC,OAAO;gEAAEsE;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWtB,MAAM,GAAG,CAAA,GAApBsB;;;;4CAAuB;;gDAAM/D,KAAKsE,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAGzD,iBAAU,qBAAGgB;4CACJ,mCAAA,4BAAA;iDAA9ByC,CAAAA,aAAaxB,MAAM,GAAG,CAAA,GAAtBwB;;;;;;;;;;;;4CAA8B,aAAiBvD;;;mDAAjB,8BAAA,SAAA;;;;4CAAMiD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiB3D,MAAMiE,cAAc3C;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAMqC,SAEHA,gBADAe;;;;4DADGf,UAAN;4DACGe,iBAAiC5D,WAAWY,GAAG,CAAC,SAACjC;uEAAS;oEAAEuC,MAAMvC,IAAIC,OAAO;oEAAEiF,WAAWlF,IAAIkF,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgB3D,MAAM0E,gBAAgBpD;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBZ;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMiD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,UAAyB3D,MAAMsB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBApDA;oBAsDA;;wBAAO;4BAAEoB,QAAQ5B,WAAW2B,MAAM;4BAAE1B,UAAAA;wBAAS;;;;IAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per\n// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain\n// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from\n// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one\n// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared\n// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the\n// only DuckDB type that can hold the different JS types mapValue() produces across files for\n// the same key).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity\n// fence anyway so a runaway frontmatter generator fails with a clear message instead of an\n// unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","withTransaction","feature","rows","contentTouched","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":";;;;+BAmCsBA;;;eAAAA;;;wBAlCK;uBACI;0BAEN;wBAEmB;yBACf;wBACU;6BACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,4FAA4F;AAC5F,8FAA8F;AAC9F,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iBAAiB;AACjB,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,6FAA6F;AAC7F,2FAA2F;AAC3F,8BAA8B;AAC9B,IAAMC,0BAA0B;AAEzB,SAAeX,UAAUY,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAEAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAIAC,iBACAC,WAKAC,OACAC,OACAC,UACAC;;;;oBApCApB,QAAQqB,IAAAA,mBAAS,EAACvB,KAAKC;oBACvBE,aAAa,IAAId,IAAIa,MAAMsB,GAAG,CAAC,SAACC;+BAAMA,EAAEhC,OAAO;;oBAEhC;;wBAAMM,KAAK2B,OAAO,CAAC;;;oBAAlCtB,eAAe;oBACC;;wBAAMA,aAAauB,GAAG;;;oBAAtCtB,eAAgB;oBAChBC,WAAW,IAAIsB,IAAIvB,aAAamB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtDtB,WAAWF,aAAa0B,MAAM,CAAC,SAACF;+BAAM,CAAC1B,WAAW6B,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhFtB,YAAYN,MAAM6B,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM3B,SAAS4B,GAAG,CAACT,EAAEhC,OAAO;wBAClC,OAAO,CAACwC,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAI/B,SAASgC,MAAM,KAAK,KAAK/B,UAAU+B,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG3B,QAAQ;wBAAK;;oBAEhFJ,WAAWgC,IAAAA,uBAAc,EAACzC;oBACZ;;wBAAM0C,IAAAA,oBAAU,EAAC3C;;;oBAA/BW,cAAc;oBAEdC,SAASgC,IAAAA,oBAAQ,EAAC,mBAAmBnC,UAAU+B,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACpC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOkC,IAAI;;;oBAArE,OAAA,eAArCjC,aAAqC,KAA3CkC,MAAkBjC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOoC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAajC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAYsC,GAAG,CAACjC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,IAAIM,WAAWuB,MAAM,GAAGzC,yBAAyB;wBAC/C,MAAM,IAAImD,oBAAU,CAAC,gBAAgB,AAAC,0BAA2FnD,OAAlEkB,WAAWuB,MAAM,EAAC,kDAAwE,OAAxBzC,yBAAwB;oBAC3J;oBACMmB,kBAAkBD,WAAWe,MAAM,CAAC,SAACmB;+BAAM9D,yBAAyB4C,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBACpGhC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBO,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FpC,OAAhFA,gBAAgBO,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKpC,gBAClLc,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFlC,QAAQX,UAAUuB,MAAM,CAAC,SAACN;+BAAM,CAACnB,SAAS0B,GAAG,CAACP,EAAEhC,OAAO;uBAAG+B,GAAG,CAAC,SAACC;+BAAMA,EAAEhC,OAAO;;oBAC9E2B,QAAwB;wBAAElB,OAAAA;wBAAOoD,UAAU1C,WAAWY,GAAG,CAAC,SAAC+B;mCAAMA,EAAE9D,OAAO;;wBAAG0B,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIhC,IAAI8B;oBACnBG,mBAAmBV,WAAWY,GAAG,CAAC,SAAC+B;+BAAMA,EAAE9D,OAAO;uBAAEsC,MAAM,CAAC,SAACyB;+BAAM,CAACnC,SAASW,GAAG,CAACwB;;oBAEtF;;wBAAMC,IAAAA,8BAAe,EAAC1D,MAAM;;oCA8CyC2D,iBAK7BA,yBAhDjC,2BAAA,mBAAA,gBAAA,WAAA,OAAM3C,UAGH4C,MAaFC,gBAgBAC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAMtE,KAAwB,YAAA,QAAMuE,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAhDN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAa5C;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKkE,IAAI,CAAC,AAAC,sCAAqD,OAAhBb,IAAAA,oBAAU,EAACrC,MAAK;;;4CAAtE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW2B,MAAM,GAAG,CAAA,GAApB3B;;;;4CACI+C,OAAO/C,WAAWY,GAAG,CAAC,SAAChC;uDAC3ByB,gBAAgBO,GAAG,CAAC,SAACT;wDAMZvB;oDALP,IAAIuB,QAAQ,QAAQ,OAAOvB,IAAIC,OAAO;oDACtC,IAAIsB,QAAQ,UAAU,OAAOvB,IAAI4C,OAAO;oDACxC,IAAIrB,QAAQ,UAAU,OAAOvB,IAAI0E,OAAO;oDACxC,IAAInD,QAAQ,SAAS,OAAOvB,IAAI8C,IAAI;oDACpC,IAAIvB,QAAQ,gBAAgB,OAAOvB,IAAI2E,UAAU;oDACjD,QAAO3E,gBAAAA,IAAI4E,IAAI,CAACrD,IAAI,cAAbvB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMO,KAAKsE,QAAQ,CAACnD,WAAWyC;;;4CAA/B;;;4CAGIC,iBAAiB,AAAC,qBAAGrD,iBAAU,qBAAGe;iDACpCsC,CAAAA,eAAerB,MAAM,GAAG,CAAA,GAAxBqB;;;;4CACF;;gDAAM7D,KAAKsE,QAAQ,CACjB,wCACAT,eAAepC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIE5C,CAAAA,WAAW2B,MAAM,GAAG,CAAA,GAApB3B;;;;4CAAuB;;gDAAMb,KAAKsE,QAAQ,CAAC/E,oBAAoBsB,WAAWY,GAAG,CAACjC;;;4CAAvD;;;iDAEvBgB,CAAAA,SAASgC,MAAM,GAAG,CAAA,GAAlBhC;;;;4CACF;;gDAAMR,KAAKsE,QAAQ,CACjB,4CACA9D,SAASiB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DK,gBAAgB,AAAC,qBAAGtD,iBAAU,qBAAGe;iDACnCuC,CAAAA,cAActB,MAAM,GAAG,CAAA,GAAvBsB;;;;4CACF;;gDAAM9D,KAAKsE,QAAQ,CACjB,6CACAR,cAAcrC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIM;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAalD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMpB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAI8E,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMP,aAAN;4DAAiCD,WAAWS,IAAI;gEAAE/E,IAAIC,OAAO;gEAAEsE;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWvB,MAAM,GAAG,CAAA,GAApBuB;;;;4CAAuB;;gDAAM/D,KAAKsE,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAGzD,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9B0C,CAAAA,aAAazB,MAAM,GAAG,CAAA,GAAtByB;;;;;;;;;;;;4CAA8B,aAAiBvD;;;mDAAjB,8BAAA,SAAA;;;;4CAAMiD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiB3D,MAAMiE,cAAc5C;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAMsC,SAEHA,gBADAe;;;;4DADGf,UAAN;4DACGe,iBAAiC7D,WAAWY,GAAG,CAAC,SAAChC;uEAAS;oEAAEsC,MAAMtC,IAAIC,OAAO;oEAAEiF,WAAWlF,IAAIkF,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgB3D,MAAM0E,gBAAgBrD;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMiD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,UAAyB3D,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBApDA;oBAsDA;;wBAAO;4BAAEoB,QAAQ5B,WAAW2B,MAAM;4BAAE1B,UAAAA;wBAAS;;;;IAC/C"}

package/dist/cjs/store/sqlite/reconcile.js

@@ -269,7 +269,7 @@ function contentRow(doc, segmenting) {
 }
 function reconcile(conn, cfg, baseDir) {
     return _async_to_generator(function() {
-        var files, currentSet, existingStmt, existingRows, existing, vanished, toReparse, features, seenColumns, report, _reparseFiles, parsedDocs, warnings, newColumns, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, allColumns, writableColumns, insertSql, added, delta, addedSet, reparsedExisting, txStart, durationMs, prevRaw, prevMax;
+        var files, currentSet, existingStmt, existingRows, existing, vanished, toReparse, features, seenColumns, report, _ref, parsedDocs, warnings, newColumns, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, col, allColumns, writableColumns, insertSql, added, delta, addedSet, reparsedExisting, txStart, durationMs, prevRaw, prevMax;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -321,7 +321,12 @@ function reconcile(conn, cfg, baseDir) {
                     // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
                     // reconciles stay silent (progress() has a threshold).
                     report = (0, _progressts.progress)('reparsing files', toReparse.length);
-                    _reparseFiles = (0, _reparsets.reparseFiles)(toReparse, features, cfg, seenColumns, report.tick), parsedDocs = _reparseFiles.docs, warnings = _reparseFiles.warnings, newColumns = _reparseFiles.newColumns;
+                    return [
+                        4,
+                        (0, _reparsets.reparseFiles)(toReparse, features, cfg, seenColumns, report.tick)
+                    ];
+                case 4:
+                    _ref = _state.sent(), parsedDocs = _ref.docs, warnings = _ref.warnings, newColumns = _ref.newColumns;
                     report.finish();
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
@@ -814,7 +819,7 @@ function reconcile(conn, cfg, baseDir) {
                             })();
                         })
                     ];
-                case 4:
+                case 5:
                     _state.sent();
                     // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):
                     // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.
@@ -823,23 +828,23 @@ function reconcile(conn, cfg, baseDir) {
                         4,
                         (0, _sharedts.getMeta)(conn, 'reconcile_max_ms')
                     ];
-                case 5:
+                case 6:
                     prevRaw = _state.sent();
                     // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
                     // still gets recorded instead of losing to the "nothing recorded yet" default.
                     prevMax = prevRaw === null ? -1 : Number(prevRaw);
                     if (!(durationMs > prevMax)) return [
                         3,
-                        7
+                        8
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'reconcile_max_ms', String(durationMs))
                     ];
-                case 6:
-                    _state.sent();
-                    _state.label = 7;
                 case 7:
+                    _state.sent();
+                    _state.label = 8;
+                case 8:
                     return [
                         2,
                         {

package/dist/cjs/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on the frontmatter row it's coupled to, so every content\n    // delete/insert below must run after that row exists (vanished paths still have their\n    // frontmatter row at this point) and before it is removed (vanished frontmatter delete\n    // comes last).\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert into `content`, coupled to the frontmatter\n      // rowid (indexed via its PRIMARY KEY) rather than the UNINDEXED `path` column, which a\n      // per-row DELETE would otherwise scan the whole table to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","segmentField","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","reparseFiles","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","presetTouched","presetRows","presetName","removedPaths","exec","runBatch","ctimeMs","parseError","data","contentTokenize","undefined","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","getMeta","Number","setMeta","String","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedPart","find","at","indexOf","label","stmt","known","file","parseFile","fileWarnings"],"mappings":";;;;;;;;;;;QAyKgBA;eAAAA;;QAaAC;eAAAA;;QAsBMC;eAAAA;;QA7KAC;eAAAA;;QAiKNC;eAAAA;;;uBA/LgB;wBACL;wBACI;0BAEN;wBAE8B;yBAC1B;yBACA;wBAC4B;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,6FAA6F;AAC7F,4EAA4E;AAC5E,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,IAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,IAAMC,qBAAqB;AAE3B,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEO,SAAeb,UAAUe,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6CC,eAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAWAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SA4EAC,YACAC,SAGAC;;;;oBAjIAzB,QAAQ0B,IAAAA,mBAAS,EAAC5B,KAAKC;oBACvBE,aAAa,IAAIhB,IAAIe,MAAM2B,GAAG,CAAC,SAACC;+BAAMA,EAAErC,OAAO;;oBAEhC;;wBAAMM,KAAKgC,OAAO,CAAC;;;oBAAlC3B,eAAe;oBACC;;wBAAMA,aAAa4B,GAAG;;;oBAAtC3B,eAAgB;oBAChBC,WAAW,IAAI2B,IAAI5B,aAAawB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD3B,WAAWF,aAAa+B,MAAM,CAAC,SAACF;+BAAM,CAAC/B,WAAWkC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF3B,YAAYN,MAAMkC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAMhC,SAASiC,GAAG,CAACT,EAAErC,OAAO;wBAClC,OAAO,CAAC6C,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAIpC,SAASqC,MAAM,KAAK,KAAKpC,UAAUoC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFL,WAAWqC,IAAAA,wBAAc,EAAC9C;oBACZ;;wBAAM+C,IAAAA,oBAAU,EAAChD;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASqC,IAAAA,oBAAQ,EAAC,mBAAmBxC,UAAUoC,MAAM;oBACRhC,gBAAAA,IAAAA,uBAAY,EAACJ,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI,GAApGpC,aAAqCD,cAA3CsC,MAAkBpC,WAAyBF,cAAzBE,UAAUC,aAAeH,cAAfG;oBACpCJ,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAapC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBN,YAAY0C,GAAG,CAACpC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGP;oBACvB,uEAAuE;oBACvE,sGAAsG;oBACtG,IAAIO,WAAW2B,MAAM,GAAGxD,yBAAyB;wBAC/C,MAAM,IAAIiE,oBAAU,CAClB,gBACA,AAAC,0BAAuHjE,OAA9F6B,WAAW2B,MAAM,EAAC,8EAAoG,OAAxBxD,yBAAwB;oBAEpJ;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE8B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACkB;+BAAMpE,yBAAyBmD,GAAG,CAACiB,MAAM,CAACC,0BAAgB,CAAClB,GAAG,CAACiB;;oBAC1G,sFAAsF;oBACtF,gDAAgD;oBAC1CnC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC2B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FvC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK4B,IAAI,CAAC,OAAM,wCAGjI,OAHuKvC,gBAClLkB,MAAM,CAAC,SAACkB;+BAAMA,MAAM;uBACpBzB,GAAG,CAAC,SAACyB;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFrC,QAAQZ,UAAU4B,MAAM,CAAC,SAACN;+BAAM,CAACxB,SAAS+B,GAAG,CAACP,EAAErC,OAAO;uBAAGoC,GAAG,CAAC,SAACC;+BAAMA,EAAErC,OAAO;;oBAC9E4B,QAAwB;wBAAEnB,OAAAA;wBAAOwD,UAAU7C,WAAWgB,GAAG,CAAC,SAAC8B;mCAAMA,EAAElE,OAAO;;wBAAG2B,OAAAA;wBAAOb,UAAAA;oBAAS;oBAC7Fe,WAAW,IAAInC,IAAIiC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC8B;+BAAMA,EAAElE,OAAO;uBAAE2C,MAAM,CAAC,SAACwB;+BAAM,CAACtC,SAASe,GAAG,CAACuB;;oBAEhFpC,UAAUqC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EAAChE,MAAM;;oCAiEyCiE,iBAK7BA,yBArEjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMhD,UAuBHiD,MAeAzE,YAeF0E,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM5E,KAAwB,YAAA,QAAM6E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAML,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CArEN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAajD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMjB,KAAKuE,IAAI,CAAC,AAAC,sCAAqD,OAAhBd,IAAAA,oBAAU,EAACxC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAMDT,CAAAA,SAASqC,MAAM,GAAG,CAAA,GAAlBrC;;;;4CACF;;gDAAMR,KAAKwE,QAAQ,CACjB,sFACAhE,SAASsB,GAAG,CAAC,SAAC+B;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;iDAKErC,CAAAA,iBAAiBqB,MAAM,GAAG,CAAA,GAA1BrB;;;;4CACF,yFAAyF;4CACzF,uFAAuF;4CACvF,+DAA+D;4CAC/D;;gDAAMxB,KAAKwE,QAAQ,CACjB,sFACAhD,iBAAiBM,GAAG,CAAC,SAAC+B;2DAAM;wDAACA;qDAAE;;;;4CAFjC;;;iDAME/C,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIoD,OAAOpD,WAAWgB,GAAG,CAAC,SAACtC;uDAC3B2B,gBAAgBW,GAAG,CAAC,SAACb;wDAOZzB;oDANP,IAAIyB,QAAQ,QAAQ,OAAOzB,IAAIE,OAAO;oDACtC,IAAIuB,QAAQ,UAAU,OAAOzB,IAAIkD,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOzB,IAAIiF,OAAO;oDACxC,IAAIxD,QAAQ,SAAS,OAAOzB,IAAIoD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOzB,IAAIkF,UAAU;oDACjD,QAAOlF,gBAAAA,IAAImF,IAAI,CAAC1D,IAAI,cAAbzB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMQ,KAAKwE,QAAQ,CAACpD,WAAW8C;;;4CAA/B;4CAEA,wFAAwF;4CACxF,+EAA+E;4CACzEzE,aAAamF,IAAAA,wBAAe,EAAC3E,SAAS4E;4CAC5C;;gDAAM7E,KAAKwE,QAAQ,CACjBlF,oBACAwB,WAAWgB,GAAG,CAAC,SAACtC;2DAAQD,WAAWC,KAAKC;;;;4CAF1C;;;iDAMEe,CAAAA,SAASqC,MAAM,GAAG,CAAA,GAAlBrC;;;;4CACF;;gDAAMR,KAAKwE,QAAQ,CACjB,4CACAhE,SAASsB,GAAG,CAAC,SAAC+B;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DM,gBAAgB,AAAC,qBAAG3D,iBAAU,qBAAGgB;iDACnC2C,CAAAA,cAActB,MAAM,GAAG,CAAA,GAAvBsB;;;;4CACF;;gDAAMnE,KAAKwE,QAAQ,CACjB,6CACAL,cAAcrC,GAAG,CAAC,SAAC+B;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIO;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAatD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMtB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIsF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMT,aAAN;4DAAiCD,WAAWW,IAAI;gEAAEvF,IAAIE,OAAO;gEAAE2E;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWvB,MAAM,GAAG,CAAA,GAApBuB;;;;4CAAuB;;gDAAMpE,KAAKwE,QAAQ,CAAC,2DAA2DJ;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG9D,iBAAU,qBAAGgB;4CACJ,mCAAA,4BAAA;iDAA9B8C,CAAAA,aAAazB,MAAM,GAAG,CAAA,GAAtByB;;;;;;;;;;;;4CAA8B,aAAiB5D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBjE,MAAMsE,cAAchD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM2C,SAEHA,gBADAgB;;;;4DADGhB,UAAN;4DACGgB,iBAAiCnE,WAAWgB,GAAG,CAAC,SAACtC;uEAAS;oEAAE4C,MAAM5C,IAAIE,OAAO;oEAAEwF,WAAW1F,IAAI0F,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBjE,MAAMiF,gBAAgB3D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBZ;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,UAAyBjE,MAAMsB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBAvEA;oBAyEA,qFAAqF;oBACrF,8GAA8G;oBACxGI,aAAaoC,KAAKC,GAAG,KAAKtC;oBAChB;;wBAAM6D,IAAAA,iBAAO,EAACtF,MAAM;;;oBAA9B2B,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAI4D,OAAO5D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAM8D,IAAAA,iBAAO,EAACxF,MAAM,oBAAoByF,OAAO/D;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C;;AAIO,SAASjC,qBAAqB4G,MAAc,EAAEC,KAAa;IAChE,IAAMC,QAAQ,eAACC;eAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGtC,IAAI,CAAC,OAAOmC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAME,QAAQ,eAACC;eAAgB,IAAIhE,IAAIgE,IAAIH,KAAK,CAAC,KAAKjE,GAAG,CAAC,SAAC+D;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMM,IAAIF,MAAMP;IAChB,IAAMU,IAAIH,MAAMN;IAChB,IAAMU,UAAU,IAAIjH;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBgH,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOE,sBAAKC;YAAW,IAAIJ,EAAE3D,GAAG,CAAC8D,SAASC,KAAKF,QAAQhD,GAAG,CAACiD;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAaH,EAAEK,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMF,OAAN;YAAuB,IAAI,CAACF,EAAE9D,GAAG,CAACgE,OAAMD,QAAQhD,GAAG,CAACiD;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOD;AACT;AAIO,SAAStH,qBAAqB2G,MAAc,EAAEC,KAAa;IAChE,IAAMc,YAAY,mBAACP;eAAgBA,IAAIH,KAAK,CAAC,KAAKW,IAAI,CAAC,SAAC7C;mBAAMA,EAAEiC,UAAU,CAAC;;;IAC3E,IAAMM,IAAIK,UAAUf;IACpB,IAAMS,IAAIM,UAAUd;IACpB,IAAIS,MAAMvB,aAAasB,MAAMtB,WAAW,OAAO;IAC/C,IAAM8B,KAAKR,EAAES,OAAO,CAAC;IACrB,OAAOR,EAAEQ,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKR,EAAEH,KAAK,CAAC,GAAGW,QAAQP;AAClE;AAGO,SAASlH,cAAcwG,MAAc,EAAEC,KAAa;IACzD,IAAMU,UAAUvH,qBAAqB4G,QAAQC;IAC7C,IAAMkB,QAAQ,eAACP;eAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbQ,IAAIN,KAAK,CAAC,IAAG,OAAK;;IACzK,OAAOK,QAAQzD,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAGyD,SAASvE,GAAG,CAAC+E,OAAOnD,IAAI,CAAC;AACxE;AAQO,SAAe1E,oBAAoBgB,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YAChF4G,MACAC,UACA5G,OACAV,YACAsB,UACAmD,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM8C,MAETjG,WADwCkG,YAAhCzH,KAAe0H;;;;oBAPZ;;wBAAMlH,KAAKgC,OAAO,CAAC;;;oBAA1B8E,OAAO;wBACK1H;oBAAM;;wBAAM0H,KAAK7E,GAAG;;;oBAAhC8E,QAAQ,IAAA,CAAA,EAAA,MAAI3H;;wBAAM,cAA8C0C,GAAG,CAAC,SAACK;mCAAMA,EAAEC,IAAI;;sBAAC;oBAClFjC,QAAQ0B,IAAAA,mBAAS,EAAC5B,KAAKC,SAASmC,MAAM,CAAC,SAACN;+BAAMgF,MAAMzE,GAAG,CAACP,EAAErC,OAAO;;oBACjED,aAAamF,IAAAA,wBAAe,EAAC3E,SAAS4E;oBACtC9D;oBACAmD;oBACD,kCAAA,2BAAA;;wBAAL,IAAK,YAAc/D,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAf6G,OAAN;;4BACqCC,aAAAA,IAAAA,mBAAS,EAACD,OAA1CxH,MAAgCyH,WAAhCzH,KAAe0H,eAAiBD,WAA3BlG;4BACbA,CAAAA,YAAAA,UAASgE,IAAI,OAAbhE,WAAc,qBAAGmG;4BACjBhD,KAAKa,IAAI,CAACxF,WAAWC,KAAKC;wBAC5B;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKL;;wBAAMuE,IAAAA,8BAAe,EAAChE,MAAM;;;;;iDACtBkE,CAAAA,KAAKrB,MAAM,GAAG,CAAA,GAAdqB;;;;4CAAiB;;gDAAMlE,KAAKwE,QAAQ,CAAClF,oBAAoB4E;;;4CAAxC;;;;;;;;4BACvB;;;;oBAFA;oBAGA;;wBAAOnD;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on the frontmatter row it's coupled to, so every content\n    // delete/insert below must run after that row exists (vanished paths still have their\n    // frontmatter row at this point) and before it is removed (vanished frontmatter delete\n    // comes last).\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert into `content`, coupled to the frontmatter\n      // rowid (indexed via its PRIMARY KEY) rather than the UNINDEXED `path` column, which a\n      // per-row DELETE would otherwise scan the whole table to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","segmentField","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","presetTouched","presetRows","presetName","removedPaths","exec","runBatch","ctimeMs","parseError","data","contentTokenize","undefined","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","getMeta","Number","setMeta","String","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedPart","find","at","indexOf","label","stmt","known","file","parseFile","fileWarnings"],"mappings":";;;;;;;;;;;QAyKgBA;eAAAA;;QAaAC;eAAAA;;QAsBMC;eAAAA;;QA7KAC;eAAAA;;QAiKNC;eAAAA;;;uBA/LgB;wBACL;wBACI;0BAEN;wBAE8B;yBAC1B;yBACA;wBAC4B;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,6FAA6F;AAC7F,4EAA4E;AAC5E,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,IAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,IAAMC,qBAAqB;AAE3B,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEO,SAAeb,UAAUe,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAWAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SA4EAC,YACAC,SAGAC;;;;oBAjIAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAIhB,IAAIe,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAEpC,OAAO;;oBAEhC;;wBAAMM,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAEpC,OAAO;wBAClC,OAAO,CAAC4C,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,wBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,uEAAuE;oBACvE,sGAAsG;oBACtG,IAAIM,WAAW2B,MAAM,GAAGvD,yBAAyB;wBAC/C,MAAM,IAAIiE,oBAAU,CAClB,gBACA,AAAC,0BAAuHjE,OAA9F4B,WAAW2B,MAAM,EAAC,8EAAoG,OAAxBvD,yBAAwB;oBAEpJ;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE6B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMpE,yBAAyBkD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,sFAAsF;oBACtF,gDAAgD;oBAC1CpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAEpC,OAAO;uBAAGmC,GAAG,CAAC,SAACC;+BAAMA,EAAEpC,OAAO;;oBAC9E2B,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAElE,OAAO;;wBAAG0B,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIlC,IAAIgC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAElE,OAAO;uBAAE0C,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EAAChE,MAAM;;oCAiEyCiE,iBAK7BA,yBArEjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMjD,UAuBHkD,MAeAzE,YAeF0E,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM5E,KAAwB,YAAA,QAAM6E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAML,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CArEN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAalD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKuE,IAAI,CAAC,AAAC,sCAAqD,OAAhBd,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAMDR,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAKwE,QAAQ,CACjB,sFACAhE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;iDAKEtC,CAAAA,iBAAiBqB,MAAM,GAAG,CAAA,GAA1BrB;;;;4CACF,yFAAyF;4CACzF,uFAAuF;4CACvF,+DAA+D;4CAC/D;;gDAAMvB,KAAKwE,QAAQ,CACjB,sFACAjD,iBAAiBM,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFjC;;;iDAMEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIqD,OAAOrD,WAAWgB,GAAG,CAAC,SAACrC;uDAC3B0B,gBAAgBW,GAAG,CAAC,SAACb;wDAOZxB;oDANP,IAAIwB,QAAQ,QAAQ,OAAOxB,IAAIE,OAAO;oDACtC,IAAIsB,QAAQ,UAAU,OAAOxB,IAAIiD,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOxB,IAAIiF,OAAO;oDACxC,IAAIzD,QAAQ,SAAS,OAAOxB,IAAImD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOxB,IAAIkF,UAAU;oDACjD,QAAOlF,gBAAAA,IAAImF,IAAI,CAAC3D,IAAI,cAAbxB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMQ,KAAKwE,QAAQ,CAACrD,WAAW+C;;;4CAA/B;4CAEA,wFAAwF;4CACxF,+EAA+E;4CACzEzE,aAAamF,IAAAA,wBAAe,EAAC3E,SAAS4E;4CAC5C;;gDAAM7E,KAAKwE,QAAQ,CACjBlF,oBACAuB,WAAWgB,GAAG,CAAC,SAACrC;2DAAQD,WAAWC,KAAKC;;;;4CAF1C;;;iDAMEe,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAKwE,QAAQ,CACjB,4CACAhE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DM,gBAAgB,AAAC,qBAAG3D,iBAAU,qBAAGe;iDACnC4C,CAAAA,cAAcvB,MAAM,GAAG,CAAA,GAAvBuB;;;;4CACF;;gDAAMnE,KAAKwE,QAAQ,CACjB,6CACAL,cAActC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIO;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAavD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMrB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIsF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMT,aAAN;4DAAiCD,WAAWW,IAAI;gEAAEvF,IAAIE,OAAO;gEAAE2E;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWxB,MAAM,GAAG,CAAA,GAApBwB;;;;4CAAuB;;gDAAMpE,KAAKwE,QAAQ,CAAC,2DAA2DJ;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG9D,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9B+C,CAAAA,aAAa1B,MAAM,GAAG,CAAA,GAAtB0B;;;;;;;;;;;;4CAA8B,aAAiB5D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBjE,MAAMsE,cAAcjD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAgB;;;;4DADGhB,UAAN;4DACGgB,iBAAiCpE,WAAWgB,GAAG,CAAC,SAACrC;uEAAS;oEAAE2C,MAAM3C,IAAIE,OAAO;oEAAEwF,WAAW1F,IAAI0F,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBjE,MAAMiF,gBAAgB5D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBAvEA;oBAyEA,qFAAqF;oBACrF,8GAA8G;oBACxGI,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAM8D,IAAAA,iBAAO,EAACtF,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAI6D,OAAO7D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAM+D,IAAAA,iBAAO,EAACxF,MAAM,oBAAoByF,OAAOhE;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C;;AAIO,SAAShC,qBAAqB4G,MAAc,EAAEC,KAAa;IAChE,IAAMC,QAAQ,eAACC;eAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGtC,IAAI,CAAC,OAAOmC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;;IACxH,IAAME,QAAQ,eAACC;eAAgB,IAAIjE,IAAIiE,IAAIH,KAAK,CAAC,KAAKlE,GAAG,CAAC,SAACgE;mBAAS;gBAACD,MAAMC;gBAAOA;aAAK;;;IACvF,IAAMM,IAAIF,MAAMP;IAChB,IAAMU,IAAIH,MAAMN;IAChB,IAAMU,UAAU,IAAIjH;QACf,kCAAA,2BAAA;;QAAL,QAAK,YAAoBgH,sBAApB,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOE,sBAAKC;YAAW,IAAIJ,EAAE5D,GAAG,CAAC+D,SAASC,KAAKF,QAAQhD,GAAG,CAACiD;;;QAA3D;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACA,mCAAA,4BAAA;;QAAL,QAAK,aAAaH,EAAEK,IAAI,uBAAnB,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAMF,OAAN;YAAuB,IAAI,CAACF,EAAE/D,GAAG,CAACiE,OAAMD,QAAQhD,GAAG,CAACiD;;;QAApD;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOD;AACT;AAIO,SAAStH,qBAAqB2G,MAAc,EAAEC,KAAa;IAChE,IAAMc,YAAY,mBAACP;eAAgBA,IAAIH,KAAK,CAAC,KAAKW,IAAI,CAAC,SAAC7C;mBAAMA,EAAEiC,UAAU,CAAC;;;IAC3E,IAAMM,IAAIK,UAAUf;IACpB,IAAMS,IAAIM,UAAUd;IACpB,IAAIS,MAAMvB,aAAasB,MAAMtB,WAAW,OAAO;IAC/C,IAAM8B,KAAKR,EAAES,OAAO,CAAC;IACrB,OAAOR,EAAEQ,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKR,EAAEH,KAAK,CAAC,GAAGW,QAAQP;AAClE;AAGO,SAASlH,cAAcwG,MAAc,EAAEC,KAAa;IACzD,IAAMU,UAAUvH,qBAAqB4G,QAAQC;IAC7C,IAAMkB,QAAQ,eAACP;eAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,AAAC,WAAuB,OAAbQ,IAAIN,KAAK,CAAC,IAAG,OAAK;;IACzK,OAAOK,QAAQ1D,IAAI,KAAK,IAAI,aAAa,AAAC,qBAAG0D,SAASxE,GAAG,CAACgF,OAAOnD,IAAI,CAAC;AACxE;AAQO,SAAe1E,oBAAoBgB,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YAChF4G,MACAC,UACA5G,OACAV,YACAqB,UACAoD,MACD,2BAAA,mBAAA,gBAAA,WAAA,OAAM8C,MAETlG,WADwCmG,YAAhCzH,KAAe0H;;;;oBAPZ;;wBAAMlH,KAAK+B,OAAO,CAAC;;;oBAA1B+E,OAAO;wBACK1H;oBAAM;;wBAAM0H,KAAK9E,GAAG;;;oBAAhC+E,QAAQ,IAAA,CAAA,EAAA,MAAI3H;;wBAAM,cAA8CyC,GAAG,CAAC,SAACK;mCAAMA,EAAEC,IAAI;;sBAAC;oBAClFhC,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC,SAASkC,MAAM,CAAC,SAACN;+BAAMiF,MAAM1E,GAAG,CAACP,EAAEpC,OAAO;;oBACjED,aAAamF,IAAAA,wBAAe,EAAC3E,SAAS4E;oBACtC/D;oBACAoD;oBACD,kCAAA,2BAAA;;wBAAL,IAAK,YAAc/D,4BAAd,6BAAA,QAAA,yBAAA,iCAAqB;4BAAf6G,OAAN;;4BACqCC,aAAAA,IAAAA,mBAAS,EAACD,OAA1CxH,MAAgCyH,WAAhCzH,KAAe0H,eAAiBD,WAA3BnG;4BACbA,CAAAA,YAAAA,UAASiE,IAAI,OAAbjE,WAAc,qBAAGoG;4BACjBhD,KAAKa,IAAI,CAACxF,WAAWC,KAAKC;wBAC5B;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAKL;;wBAAMuE,IAAAA,8BAAe,EAAChE,MAAM;;;;;iDACtBkE,CAAAA,KAAKtB,MAAM,GAAG,CAAA,GAAdsB;;;;4CAAiB;;gDAAMlE,KAAKwE,QAAQ,CAAClF,oBAAoB4E;;;4CAAxC;;;;;;;;4BACvB;;;;oBAFA;oBAGA;;wBAAOpD;;;;IACT"}

package/dist/cjs/watch.js

@@ -144,7 +144,7 @@ var HEARTBEAT_INTERVAL_MS = 5000;
 var STALE_HEARTBEAT_MS = 15000;
 function runWatch(_0) {
     return _async_to_generator(function(cfg) {
-        var opts, _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs, onEvent, debounceMs, heartbeatIntervalMs, _ref, store, dbPath, initialWarnings, initialParsed, baseDir, existingHeartbeat, age, _tmp, stopping, touchHeartbeat, debounceTimer, scheduleReconcile, watcher, heartbeatTimer;
+        var opts, _opts_onEvent, _opts_debounceMs, _opts_heartbeatIntervalMs, onEvent, debounceMs, heartbeatIntervalMs, _ref, store, dbPath, initialWarnings, initialParsed, baseDir, existingHeartbeat, age, _tmp, stopping, touchHeartbeat, debounceTimer, inFlight, scheduleReconcile, watcher, heartbeatTimer;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
@@ -239,65 +239,77 @@ function runWatch(_0) {
                 case 7:
                     _state.sent();
                     debounceTimer = null;
+                    // A reconcile that has already started owns the store, and on a bulk reparse a live worker
+                    // pool as well. Shutdown waits on this rather than closing the connection underneath it.
+                    inFlight = null;
                     scheduleReconcile = function scheduleReconcile() {
                         if (debounceTimer) clearTimeout(debounceTimer);
                         debounceTimer = setTimeout(function() {
-                            return _async_to_generator(function() {
-                                var _ref, parsed, warnings, _tmp, err;
-                                return _ts_generator(this, function(_state) {
-                                    switch(_state.label){
-                                        case 0:
-                                            debounceTimer = null;
-                                            _state.label = 1;
-                                        case 1:
-                                            _state.trys.push([
-                                                1,
-                                                4,
-                                                ,
-                                                5
-                                            ]);
-                                            return [
-                                                4,
-                                                store.reconcile()
-                                            ];
-                                        case 2:
-                                            _ref = _state.sent(), parsed = _ref.parsed, warnings = _ref.warnings;
-                                            _tmp = {
-                                                type: 'reconciled',
-                                                parsed: parsed
-                                            };
-                                            return [
-                                                4,
-                                                (0, _indexts1.docCount)(store)
-                                            ];
-                                        case 3:
-                                            onEvent.apply(void 0, [
-                                                (_tmp.total = _state.sent(), _tmp.warnings = warnings, _tmp)
-                                            ]);
-                                            return [
-                                                3,
-                                                5
-                                            ];
-                                        case 4:
-                                            err = _state.sent();
-                                            onEvent({
-                                                type: 'reconcile-error',
-                                                message: err.message
-                                            });
-                                            return [
-                                                3,
-                                                5
-                                            ];
-                                        case 5:
-                                            return [
-                                                2
-                                            ];
-                                    }
-                                });
-                            })();
+                            debounceTimer = null;
+                            inFlight = function() {
+                                return _async_to_generator(function() {
+                                    var _ref, parsed, warnings, _tmp, err;
+                                    return _ts_generator(this, function(_state) {
+                                        switch(_state.label){
+                                            case 0:
+                                                _state.trys.push([
+                                                    0,
+                                                    3,
+                                                    4,
+                                                    5
+                                                ]);
+                                                return [
+                                                    4,
+                                                    store.reconcile()
+                                                ];
+                                            case 1:
+                                                _ref = _state.sent(), parsed = _ref.parsed, warnings = _ref.warnings;
+                                                _tmp = {
+                                                    type: 'reconciled',
+                                                    parsed: parsed
+                                                };
+                                                return [
+                                                    4,
+                                                    (0, _indexts1.docCount)(store)
+                                                ];
+                                            case 2:
+                                                onEvent.apply(void 0, [
+                                                    (_tmp.total = _state.sent(), _tmp.warnings = warnings, _tmp)
+                                                ]);
+                                                return [
+                                                    3,
+                                                    5
+                                                ];
+                                            case 3:
+                                                err = _state.sent();
+                                                onEvent({
+                                                    type: 'reconcile-error',
+                                                    message: err.message
+                                                });
+                                                return [
+                                                    3,
+                                                    5
+                                                ];
+                                            case 4:
+                                                inFlight = null;
+                                                return [
+                                                    7
+                                                ];
+                                            case 5:
+                                                return [
+                                                    2
+                                                ];
+                                        }
+                                    });
+                                })();
+                            }();
                         }, debounceMs);
                     };
-                    // Ignore our own state dir, or the heartbeat write would retrigger itself forever.
+                    // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event
+                    // whose filename the platform could not resolve (null, which fs.watch does deliver under
+                    // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs
+                    // less than missing a real edit. That is why the guard cannot promise zero reconciles, only
+                    // that an identified state-dir write is never one of them.
                     watcher = (0, _nodefs.watch)(baseDir, {
                         recursive: true
                     }, function(_event, filename) {
@@ -331,21 +343,27 @@ function runWatch(_0) {
                                                 watcher.close();
                                                 return [
                                                     4,
-                                                    (0, _indexts1.setMeta)(store, 'watch_heartbeat', null)
+                                                    inFlight
                                                 ];
                                             case 1:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    (0, _indexts1.setMeta)(store, 'watch_pid', null)
+                                                    (0, _indexts1.setMeta)(store, 'watch_heartbeat', null)
                                                 ];
                                             case 2:
                                                 _state.sent();
                                                 return [
                                                     4,
-                                                    store.close()
+                                                    (0, _indexts1.setMeta)(store, 'watch_pid', null)
                                                 ];
                                             case 3:
+                                                _state.sent();
+                                                return [
+                                                    4,
+                                                    store.close()
+                                                ];
+                                            case 4:
                                                 _state.sent();
                                                 resolveShutdown();
                                                 return [

package/dist/cjs/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(async () => {\n      debounceTimer = null;\n      try {\n        const { parsed, warnings } = await store.reconcile();\n        onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n      } catch (err) {\n        onEvent({ type: 'reconcile-error', message: (err as Error).message });\n      }\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","initialWarnings","initialParsed","baseDir","existingHeartbeat","age","stopping","touchHeartbeat","debounceTimer","scheduleReconcile","watcher","heartbeatTimer","requireWatchConcurrency","openStore","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","type","length","docCount","total","setMeta","toISOString","String","process","pid","clearTimeout","setTimeout","err","reconcile","message","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;uBAEP;wBACC;6BACC;wBACmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE/E,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAcpB,SAAeH;wCAASI,GAAmB;YAAEC,MAElCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBACsE,MAApEC,OAAOC,QAAkBC,iBAAyBC,eACpDC,SAEAC,mBAEEC,WAYJC,UACEC,gBAMFC,eACEC,mBAcAC,SAIAC;;;;;oBAhD4ChB,OAAAA,oEAAqB,CAAC;oBACxEiB,IAAAA,iCAAuB,EAAClB;oBAClBE,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBACoB;;wBAAMqB,IAAAA,mBAAS,EAACnB;;;oBAAhB,OAAA,eAApEK,QAAoE,KAApEA,OAAOC,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDa,UAAmCZ,gBAAkB,KAA1Ba;oBAC5CZ,UAAUT,IAAIS,OAAO;oBAED;;wBAAMa,IAAAA,iBAAO,EAACjB,OAAO;;;oBAAzCK,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACT,KAAKsB,KAAK,AAAD,GAA/Bb;;;;oBACIC,MAAMa,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAAChB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMZ,kBAAiB,GAAnCY;;;;oBACF;;wBAAMN,MAAMsB,KAAK;;;oBAAjB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACnB,MAAM,OAAM;;oBAI7GT,QAAQ;wBAAE6B,MAAM;wBAAWtB,SAAAA;wBAASH,QAAAA;oBAAO;yBACvCC,CAAAA,gBAAgByB,MAAM,GAAG,KAAKxB,gBAAgB,CAAA,GAA9CD;;;;;wBACQwB,MAAM;wBAAcV,QAAQb;;oBAAsB;;wBAAMyB,IAAAA,kBAAQ,EAAC5B;;;oBAA3EH;8BAAqDgC,QAAO,oBAAuBd,WAAUb;;;;oBAG3FK,WAAW;oBACTC,iBAAiB;;;;;wCACrB;;4CAAMsB,IAAAA,iBAAO,EAAC9B,OAAO,mBAAmB,IAAImB,OAAOY,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAAC9B,OAAO,aAAagC,OAAOC,QAAQC,GAAG;;;wCAApD;;;;;;wBACF;;oBACA;;wBAAM1B;;;oBAAN;oBAEIC,gBAAuC;oBACrCC,oBAAoB;wBACxB,IAAID,eAAe0B,aAAa1B;wBAChCA,gBAAgB2B,WAAW;;oCAGM,MAArBpB,QAAQD,gBAETsB;;;;4CAJT5B,gBAAgB;;;;;;;;;4CAEe;;gDAAMT,MAAMsC,SAAS;;;4CAArB,OAAA,eAArBtB,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;gDACNW,MAAM;gDAAcV,QAAAA;;4CAAe;;gDAAMY,IAAAA,kBAAQ,EAAC5B;;;4CAA5DH;sDAAsCgC,QAAO,oBAAuBd,WAAAA;;;;;;;4CAC7DsB;4CACPxC,QAAQ;gDAAE6B,MAAM;gDAAmBa,SAAS,AAACF,IAAcE,OAAO;4CAAC;;;;;;;;;;;4BAEvE;2BAAGzC;oBACL;oBAEA,mFAAmF;oBAC7Ea,UAAU6B,IAAAA,aAAO,EAACpC,SAAS;wBAAEqC,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpEnC;oBACF;oBACME,iBAAiBkC,YACrBC,IAAAA,0BAAW,EAACvC,gBAAgB;+BAAMD;wBAClCR;oBAGF;;wBAAO,IAAIiD,QAAc,SAACC;gCAmBpBrD,cACCA;4BAnBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAMsD,WAAW;;wCAKftD;;;;gDAJA,IAAIW,UAAU;;;gDACdA,WAAW;gDACX0B,QAAQkB,GAAG,CAAC,UAAUD;gDACtBjB,QAAQkB,GAAG,CAAC,WAAWD;iDACvBtD,eAAAA,KAAKwD,MAAM,cAAXxD,mCAAAA,aAAayD,mBAAmB,CAAC,SAASH;gDAC1CI,cAAc1C;gDACd,IAAIH,eAAe0B,aAAa1B;gDAChCE,QAAQW,KAAK;gDACb;;oDAAMQ,IAAAA,iBAAO,EAAC9B,OAAO,mBAAmB;;;gDAAxC;gDACA;;oDAAM8B,IAAAA,iBAAO,EAAC9B,OAAO,aAAa;;;gDAAlC;gDACA;;oDAAMA,MAAMsB,KAAK;;;gDAAjB;gDACA2B;;;;;;gCACF;;4BACAhB,QAAQsB,IAAI,CAAC,UAAUL;4BACvBjB,QAAQsB,IAAI,CAAC,WAAWL;4BACxB,KAAItD,eAAAA,KAAKwD,MAAM,cAAXxD,mCAAAA,aAAa4D,OAAO,EAAEN;kCACrBtD,gBAAAA,KAAKwD,MAAM,cAAXxD,oCAAAA,cAAa6D,gBAAgB,CAAC,SAASP,UAAU;gCAAEK,MAAM;4BAAK;wBACrE;;;;IACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event\n  // whose filename the platform could not resolve (null, which fs.watch does deliver under\n  // load) is attributed to nothing and so reconciles: one reconcile that parses nothing costs\n  // less than missing a real edit. That is why the guard cannot promise zero reconciles, only\n  // that an identified state-dir write is never one of them.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["runWatch","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","initialWarnings","initialParsed","baseDir","existingHeartbeat","age","stopping","touchHeartbeat","debounceTimer","inFlight","scheduleReconcile","watcher","heartbeatTimer","requireWatchConcurrency","openStore","warnings","parsed","getMeta","force","Date","now","parse","close","SenseError","Math","round","type","length","docCount","total","setMeta","toISOString","String","process","pid","clearTimeout","setTimeout","err","reconcile","message","fsWatch","recursive","_event","filename","startsWith","STATE_DIR","setInterval","guardedTick","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":";;;;+BAwBsBA;;;eAAAA;;;sBAxBW;uBAEP;wBACC;6BACC;wBACmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE/E,uJAAuJ;AACvJ,IAAMC,cAAc;AACpB,IAAMC,wBAAwB;AAC9B,IAAMC,qBAAqB;AAcpB,SAAeH;wCAASI,GAAmB;YAAEC,MAElCA,eACGA,kBACSA,2BAFtBC,SACAC,YACAC,qBACsE,MAApEC,OAAOC,QAAkBC,iBAAyBC,eACpDC,SAEAC,mBAEEC,WAYJC,UACEC,gBAMFC,eAGAC,UACEC,mBAsBAC,SAIAC;;;;;oBA3D4CjB,OAAAA,oEAAqB,CAAC;oBACxEkB,IAAAA,iCAAuB,EAACnB;oBAClBE,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,YAAO;oBAClCE,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBJ;oBAChCO,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BH;oBACoB;;wBAAMsB,IAAAA,mBAAS,EAACpB;;;oBAAhB,OAAA,eAApEK,QAAoE,KAApEA,OAAOC,SAA6D,KAA7DA,QAAkBC,kBAA2C,KAArDc,UAAmCb,gBAAkB,KAA1Bc;oBAC5Cb,UAAUT,IAAIS,OAAO;oBAED;;wBAAMc,IAAAA,iBAAO,EAAClB,OAAO;;;oBAAzCK,oBAAoB;yBACtBA,CAAAA,qBAAqB,CAACT,KAAKuB,KAAK,AAAD,GAA/Bd;;;;oBACIC,MAAMc,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACjB;yBAChCC,CAAAA,OAAO,KAAKA,MAAMZ,kBAAiB,GAAnCY;;;;oBACF;;wBAAMN,MAAMuB,KAAK;;;oBAAjB;oBACA,MAAM,IAAIC,oBAAU,CAAC,gBAAgB,AAAC,6CAAmE,OAAvBC,KAAKC,KAAK,CAACpB,MAAM,OAAM;;oBAI7GT,QAAQ;wBAAE8B,MAAM;wBAAWvB,SAAAA;wBAASH,QAAAA;oBAAO;yBACvCC,CAAAA,gBAAgB0B,MAAM,GAAG,KAAKzB,gBAAgB,CAAA,GAA9CD;;;;;wBACQyB,MAAM;wBAAcV,QAAQd;;oBAAsB;;wBAAM0B,IAAAA,kBAAQ,EAAC7B;;;oBAA3EH;8BAAqDiC,QAAO,oBAAuBd,WAAUd;;;;oBAG3FK,WAAW;oBACTC,iBAAiB;;;;;wCACrB;;4CAAMuB,IAAAA,iBAAO,EAAC/B,OAAO,mBAAmB,IAAIoB,OAAOY,WAAW;;;wCAA9D;wCACA;;4CAAMD,IAAAA,iBAAO,EAAC/B,OAAO,aAAaiC,OAAOC,QAAQC,GAAG;;;wCAApD;;;;;;wBACF;;oBACA;;wBAAM3B;;;oBAAN;oBAEIC,gBAAuC;oBAC3C,2FAA2F;oBAC3F,yFAAyF;oBACrFC,WAAiC;oBAC/BC,oBAAoB;wBACxB,IAAIF,eAAe2B,aAAa3B;wBAChCA,gBAAgB4B,WAAW;4BACzB5B,gBAAgB;4BAChBC,WAAW,AAAC;;wCAEqB,MAArBO,QAAQD,gBAETsB;;;;;;;;;;gDAFsB;;oDAAMtC,MAAMuC,SAAS;;;gDAArB,OAAA,eAArBtB,SAAqB,KAArBA,QAAQD,WAAa,KAAbA;;oDACNW,MAAM;oDAAcV,QAAAA;;gDAAe;;oDAAMY,IAAAA,kBAAQ,EAAC7B;;;gDAA5DH;0DAAsCiC,QAAO,oBAAuBd,WAAAA;;;;;;;gDAC7DsB;gDACPzC,QAAQ;oDAAE8B,MAAM;oDAAmBa,SAAS,AAACF,IAAcE,OAAO;gDAAC;;;;;;gDAEnE9B,WAAW;;;;;;;;;;gCAEf;;wBACF,GAAGZ;oBACL;oBAEA,4FAA4F;oBAC5F,yFAAyF;oBACzF,4FAA4F;oBAC5F,4FAA4F;oBAC5F,2DAA2D;oBACrDc,UAAU6B,IAAAA,aAAO,EAACrC,SAAS;wBAAEsC,WAAW;oBAAK,GAAG,SAACC,QAAQC;wBAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACC,kBAAS,GAAG;wBACpEnC;oBACF;oBACME,iBAAiBkC,YACrBC,IAAAA,0BAAW,EAACxC,gBAAgB;+BAAMD;wBAClCR;oBAGF;;wBAAO,IAAIkD,QAAc,SAACC;gCAoBpBtD,cACCA;4BApBL,oFAAoF;4BACpF,oFAAoF;4BACpF,IAAMuD,WAAW;;wCAKfvD;;;;gDAJA,IAAIW,UAAU;;;gDACdA,WAAW;gDACX2B,QAAQkB,GAAG,CAAC,UAAUD;gDACtBjB,QAAQkB,GAAG,CAAC,WAAWD;iDACvBvD,eAAAA,KAAKyD,MAAM,cAAXzD,mCAAAA,aAAa0D,mBAAmB,CAAC,SAASH;gDAC1CI,cAAc1C;gDACd,IAAIJ,eAAe2B,aAAa3B;gDAChCG,QAAQW,KAAK;gDACb;;oDAAMb;;;gDAAN;gDACA;;oDAAMqB,IAAAA,iBAAO,EAAC/B,OAAO,mBAAmB;;;gDAAxC;gDACA;;oDAAM+B,IAAAA,iBAAO,EAAC/B,OAAO,aAAa;;;gDAAlC;gDACA;;oDAAMA,MAAMuB,KAAK;;;gDAAjB;gDACA2B;;;;;;gCACF;;4BACAhB,QAAQsB,IAAI,CAAC,UAAUL;4BACvBjB,QAAQsB,IAAI,CAAC,WAAWL;4BACxB,KAAIvD,eAAAA,KAAKyD,MAAM,cAAXzD,mCAAAA,aAAa6D,OAAO,EAAEN;kCACrBvD,gBAAAA,KAAKyD,MAAM,cAAXzD,oCAAAA,cAAa8D,gBAAgB,CAAC,SAASP,UAAU;gCAAEK,MAAM;4BAAK;wBACrE;;;;IACF"}

package/dist/cjs/workers/parse.d.cts

@@ -0,0 +1,18 @@
+import type { Config, FeatureName } from '../config/index.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from '../scan/list.js';
+import type { WorkerErrorPayload } from '../scan/worker-error.js';
+export interface ParseWorkerData {
+    cfg: Config;
+    featureNames: FeatureName[];
+}
+export type ParseTask = FileStat;
+export type ParseTaskResult = {
+    ok: true;
+    doc: ParsedDoc;
+    warnings: string[];
+} | {
+    ok: false;
+    error: WorkerErrorPayload;
+};
+export default function parseTask(file: ParseTask): ParseTaskResult;

package/dist/cjs/workers/parse.d.ts

@@ -0,0 +1,18 @@
+import type { Config, FeatureName } from '../config/index.js';
+import type { ParsedDoc } from '../scan/index.js';
+import type { FileStat } from '../scan/list.js';
+import type { WorkerErrorPayload } from '../scan/worker-error.js';
+export interface ParseWorkerData {
+    cfg: Config;
+    featureNames: FeatureName[];
+}
+export type ParseTask = FileStat;
+export type ParseTaskResult = {
+    ok: true;
+    doc: ParsedDoc;
+    warnings: string[];
+} | {
+    ok: false;
+    error: WorkerErrorPayload;
+};
+export default function parseTask(file: ParseTask): ParseTaskResult;

package/dist/cjs/workers/parse.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "default", {
+    enumerable: true,
+    get: function() {
+        return parseTask;
+    }
+});
+var _tinypool = /*#__PURE__*/ _interop_require_default(require("tinypool"));
+var _indexts = require("../features/index.js");
+var _indexts1 = require("../scan/index.js");
+var _reparsets = require("../scan/reparse.js");
+var _workererrorts = require("../scan/worker-error.js");
+function _interop_require_default(obj) {
+    return obj && obj.__esModule ? obj : {
+        default: obj
+    };
+}
+// Read once per worker, not per task.
+var _Tinypool_workerData = _tinypool.default.workerData, cfg = _Tinypool_workerData.cfg, featureNames = _Tinypool_workerData.featureNames;
+// Filtering the registry (rather than mapping the names) keeps registry order, which is the
+// order `extracted` keys land in on the serial path.
+var selected = _indexts.FEATURES.filter(function(feature) {
+    return featureNames.includes(feature.name);
+});
+function parseTask(file) {
+    try {
+        var _parseFile = (0, _indexts1.parseFile)(file, (0, _reparsets.featuresForFile)(selected, cfg, file), cfg), doc = _parseFile.doc, warnings = _parseFile.warnings;
+        return {
+            ok: true,
+            doc: doc,
+            warnings: warnings
+        };
+    } catch (err) {
+        return {
+            ok: false,
+            error: (0, _workererrorts.serializeError)(err)
+        };
+    }
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/workers/parse.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task.\n// A Feature carries closures and cannot cross the thread boundary; its name can, and the\n// registry on this side resolves it back, which keeps the caller's selection intact.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[] } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    return { ok: true, doc, warnings };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["parseTask","Tinypool","workerData","cfg","featureNames","selected","FEATURES","filter","feature","includes","name","file","parseFile","featuresForFile","doc","warnings","ok","err","error","serializeError"],"mappings":";;;;+BA8BA;;;eAAwBA;;;+DA9BH;uBAEI;wBAEC;yBAEM;6BAED;;;;;;AAgB/B,sCAAsC;AACtC,IAA8BC,uBAAAA,iBAAQ,CAACC,UAAU,EAAzCC,MAAsBF,qBAAtBE,KAAKC,eAAiBH,qBAAjBG;AACb,4FAA4F;AAC5F,qDAAqD;AACrD,IAAMC,WAAWC,iBAAQ,CAACC,MAAM,CAAC,SAACC;WAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;;AAEjE,SAASV,UAAUW,IAAe;IAC/C,IAAI;QACF,IAA0BC,aAAAA,IAAAA,mBAAS,EAACD,MAAME,IAAAA,0BAAe,EAACR,UAAUF,KAAKQ,OAAOR,MAAxEW,MAAkBF,WAAlBE,KAAKC,WAAaH,WAAbG;QACb,OAAO;YAAEC,IAAI;YAAMF,KAAAA;YAAKC,UAAAA;QAAS;IACnC,EAAE,OAAOE,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOC,IAAAA,6BAAc,EAACF;QAAK;IACjD;AACF"}

package/dist/esm/scan/reparse.d.ts

@@ -7,4 +7,9 @@ export interface ReparseResult {
     warnings: string[];
     newColumns: string[];
 }
-export declare function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void): ReparseResult;
+export interface ReparseOptions {
+    threshold?: number;
+    maxWorkers?: number;
+}
+export declare function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[];
+export declare function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options?: ReparseOptions): Promise<ReparseResult>;