sensemaking 0.21.1 → 0.22.1

package/dist/cjs/features/sections.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    if (rows.length === 0) return;\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row.\n    await db.runBatch('INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["sections","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","countLines","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","estimateTokens","name","schema","db","exec","extract","_search","_data","_cfg","parse","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push"],"mappings":";;;;+BA8BaA;;;eAAAA;;;uBA7ByB;6BACX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc3B,iGAAiG;AACjG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,IAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,IAAMC,SAASF,SAASG,MAAM,GAAGC,IAAAA,yBAAU,EAACL;IAC5C,IAAMM,QAAmBR,OAAOS,MAAM,CAAC,SAACC;eAAMA,EAAEC,IAAI,KAAK;OAAWC,GAAG,CAAC,SAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGb;YAAQc,SAAShB,SAASG,MAAM;YAAEc,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMF,MAAM,EAAEe,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMF,MAAM,EAAEE,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,IAAMF,OAAOb,SAASmB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAACC,IAAAA,uBAAc,EAACV;IAC7C;IACA,OAAOR;AACT;AAEO,IAAMV,WAAoB;IAC/B6B,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQ9B,GAAG,EAAEC,IAAI,EAAE8B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAElC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUmC,IAAAA,cAAK,EAACjC,OAAOD,KAAKC;IACxD;IACMkC,QAAN,SAAMA,OAAOP,EAAE,EAAEQ,KAAK;;;;;wBACpB,IAAIA,MAAM/B,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMuB,GAAGS,QAAQ,CACf,yCACAD,MAAMzB,GAAG,CAAC,SAAC2B;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IACMC,OAAN,SAAMA,MAAMX,EAAE,EAAEY,IAAI;;gBACZC,MACD,2BAAA,mBAAA,uBAAA,WAAA;;;;wBADCA;wBACD,kCAAA,2BAAA;;;gCAAA,kBAAA,aAAQC,mBAAAA,MAAMC,wBAAAA;gCAAsBA,UAAwBC,OAAO,CAAC,SAACxB,GAAGyB;2CAAQJ,KAAKK,IAAI,CAAC;wCAACJ;wCAAMG;wCAAKzB,EAAER,KAAK;wCAAEQ,EAAEN,OAAO;wCAAEM,EAAEH,SAAS;wCAAEG,EAAEF,OAAO;wCAAEE,EAAED,MAAM;qCAAC;;;4BAAhK,IAAK,YAA6BqB,2BAA7B,6BAAA,QAAA,yBAAA;;4BAAA;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBACL,IAAIC,KAAKpC,MAAM,KAAK,GAAG;;;wBACvB,wFAAwF;wBACxF,0FAA0F;wBAC1F,qEAAqE;wBACrE;;4BAAMuB,GAAGS,QAAQ,CAAC,qJAAqJI;;;wBAAvK;;;;;;QACF;;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nconst SECTION_COLUMNS = ['path', 'idx', 'level', 'heading', 'start_line', 'end_line', 'tokens'];\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row. A store with an append\n    // path has no second writer, and remove() cleared every touched path above, so it skips the guard.\n    await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["sections","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","countLines","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","estimateTokens","SECTION_COLUMNS","name","schema","db","exec","extract","_search","_data","_cfg","parse","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push","appendRows"],"mappings":";;;;+BAiCaA;;;eAAAA;;;uBAhCyB;6BACX;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc3B,iGAAiG;AACjG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,IAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,IAAMC,SAASF,SAASG,MAAM,GAAGC,IAAAA,yBAAU,EAACL;IAC5C,IAAMM,QAAmBR,OAAOS,MAAM,CAAC,SAACC;eAAMA,EAAEC,IAAI,KAAK;OAAWC,GAAG,CAAC,SAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGb;YAAQc,SAAShB,SAASG,MAAM;YAAEc,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMF,MAAM,EAAEe,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMF,MAAM,EAAEE,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,IAAMF,OAAOb,SAASmB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAACC,IAAAA,uBAAc,EAACV;IAC7C;IACA,OAAOR;AACT;AAEA,IAAMmB,kBAAkB;IAAC;IAAQ;IAAO;IAAS;IAAW;IAAc;IAAY;CAAS;AAExF,IAAM7B,WAAoB;IAC/B8B,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQ/B,GAAG,EAAEC,IAAI,EAAE+B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAEnC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUoC,IAAAA,cAAK,EAAClC,OAAOD,KAAKC;IACxD;IACMmC,QAAN,SAAMA,OAAOP,EAAE,EAAEQ,KAAK;;;;;wBACpB,IAAIA,MAAMhC,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMwB,GAAGS,QAAQ,CACf,yCACAD,MAAM1B,GAAG,CAAC,SAAC4B;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IACMC,OAAN,SAAMA,MAAMX,EAAE,EAAEY,IAAI;;gBACZC,MACD,2BAAA,mBAAA,uBAAA,WAAA;;;;wBADCA;wBACD,kCAAA,2BAAA;;;gCAAA,kBAAA,aAAQC,mBAAAA,MAAMC,wBAAAA;gCAAsBA,UAAwBC,OAAO,CAAC,SAACzB,GAAG0B;2CAAQJ,KAAKK,IAAI,CAAC;wCAACJ;wCAAMG;wCAAK1B,EAAER,KAAK;wCAAEQ,EAAEN,OAAO;wCAAEM,EAAEH,SAAS;wCAAEG,EAAEF,OAAO;wCAAEE,EAAED,MAAM;qCAAC;;;4BAAhK,IAAK,YAA6BsB,2BAA7B,6BAAA,QAAA,yBAAA;;4BAAA;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBACL,wFAAwF;wBACxF,0FAA0F;wBAC1F,4FAA4F;wBAC5F,mGAAmG;wBACnG;;4BAAMO,IAAAA,oBAAU,EAACnB,IAAI,YAAYH,iBAAiB,qJAAqJgB;;;wBAAvM;;;;;;QACF;;AACF"}

package/dist/cjs/features/tags.js

@@ -8,6 +8,7 @@ Object.defineProperty(exports, "tags", {
         return tags;
     }
 });
+var _sharedts = require("../store/shared.js");
 var _fencests = require("./fences.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -372,12 +373,14 @@ var tags = {
                                 }
                             }
                         }
-                        if (rows.length === 0) return [
-                            2
-                        ];
+                        // OR IGNORE guards the same concurrent-reconcile race sections.ts documents; remove() cleared
+                        // every touched path above, so a store with an append path needs no guard.
                         return [
                             4,
-                            db.runBatch('INSERT OR IGNORE INTO tags ("path", tag) VALUES (?, ?)', rows)
+                            (0, _sharedts.appendRows)(db, 'tags', [
+                                'path',
+                                'tag'
+                            ], 'INSERT OR IGNORE INTO tags ("path", tag) VALUES (?, ?)', rows)
                         ];
                     case 1:
                         _state.sent();

package/dist/cjs/features/tags.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/tags.ts"],"sourcesContent":["import { maskRegions } from './fences.ts';\nimport type { Feature } from './types.ts';\n\n// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline\n// #tags, deduplicated, source not distinguished. Nested tags store full (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` matches the parent too.\n\n// Obsidian treats [[#Heading]] as a same-note link, not a tag.\nconst WIKILINK_RE = /\\[\\[.*?\\]\\]/g; // to the first ]], so a heading holding a lone ] still masks\n// Obsidian doesn't read tags inside HTML markup.\nconst HTML_TAG_RE = /<\\/?[a-zA-Z][^>]*>/g; // tag-shaped only: a comparison's `< 5` must not open a span\n// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.\nconst INLINE_TAG_RE = /(?:^|[\\s([])#([\\p{L}\\p{N}_/-]+)/gu;\n// A markdown link destination `](...)` -- `[text](#anchor)` is a same-page link, not a tag.\nconst LINK_DEST_RE = /\\]\\((?:[^()]|\\([^()]*\\))*\\)/g; // one paren-nesting level, as CommonMark destinations allow: (https://x/a_(b)#frag)\n\n// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an\n// all-digit result -- a tag needs at least one non-digit character.\nfunction normalizeTag(raw: string): string | null {\n  const stripped = raw.replace(/^#/, '').replace(/\\/+$/, '');\n  if (!stripped || /^\\d+$/.test(stripped)) return null;\n  return stripped;\n}\n\n// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string\n// members are skipped rather than throwing -- `tags:\\n  -` parses to [null].\nfunction frontmatterTags(data?: Record<string, unknown>): string[] {\n  const raw = data?.tags;\n  const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [];\n  const found: string[] = [];\n  for (const item of list) {\n    if (typeof item !== 'string') continue;\n    const tag = normalizeTag(item);\n    if (tag) found.push(tag);\n  }\n  return found;\n}\n\n// #tag tokens outside the regions maskRegions() masks (fences, code spans, wikilinks, HTML,\n// comments, link destinations); this loop only does per-line tag extraction.\nfunction inlineTags(body: string): string[] {\n  const found: string[] = [];\n  for (const line of maskRegions(body).split('\\n')) {\n    if (!line.includes('#')) continue; // most lines; skip the regex work\n    let cleaned = line;\n    if (cleaned.includes('[[')) cleaned = cleaned.replace(WIKILINK_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('](')) cleaned = cleaned.replace(LINK_DEST_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('<')) cleaned = cleaned.replace(HTML_TAG_RE, (m) => ' '.repeat(m.length));\n    for (const m of cleaned.matchAll(INLINE_TAG_RE)) {\n      const tag = normalizeTag(m[1]);\n      if (tag) found.push(tag);\n    }\n  }\n  return found;\n}\n\nfunction extract(_raw: string, body: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): string[] {\n  return [...new Set([...frontmatterTags(data), ...inlineTags(body)])].sort();\n}\n\nexport const tags: Feature = {\n  name: 'tags',\n  async schema(db) {\n    await db.exec('CREATE TABLE IF NOT EXISTS tags (\"path\" TEXT, tag TEXT, PRIMARY KEY (\"path\", tag))');\n    await db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');\n  },\n  extract,\n  // Blanket clear-then-reinsert (store() always writes the full found list back), so remove()\n  // covers vanished and reparsed files without per-file NOT IN diffing, which cannot be one statement shared across files.\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM tags WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) for (const tag of extracted as string[]) rows.push([path, tag]);\n    if (rows.length === 0) return;\n    await db.runBatch('INSERT OR IGNORE INTO tags (\"path\", tag) VALUES (?, ?)', rows);\n  },\n};\n"],"names":["tags","WIKILINK_RE","HTML_TAG_RE","INLINE_TAG_RE","LINK_DEST_RE","normalizeTag","raw","stripped","replace","test","frontmatterTags","data","list","Array","isArray","found","item","tag","push","inlineTags","body","maskRegions","split","line","includes","cleaned","m","repeat","length","matchAll","extract","_raw","_search","Set","sort","name","schema","db","exec","remove","paths","runBatch","map","p","store","docs","rows","path","extracted"],"mappings":";;;;+BA2DaA;;;eAAAA;;;wBA3De;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG5B,0FAA0F;AAC1F,kJAAkJ;AAElJ,+DAA+D;AAC/D,IAAMC,cAAc,gBAAgB,6DAA6D;AACjG,iDAAiD;AACjD,IAAMC,cAAc,uBAAuB,6DAA6D;AACxG,iGAAiG;AACjG,IAAMC,gBAAgB;AACtB,4FAA4F;AAC5F,IAAMC,eAAe,gCAAgC,oFAAoF;AAEzI,sFAAsF;AACtF,oEAAoE;AACpE,SAASC,aAAaC,GAAW;IAC/B,IAAMC,WAAWD,IAAIE,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,QAAQ;IACvD,IAAI,CAACD,YAAY,QAAQE,IAAI,CAACF,WAAW,OAAO;IAChD,OAAOA;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASG,gBAAgBC,IAA8B;IACrD,IAAML,MAAMK,iBAAAA,2BAAAA,KAAMX,IAAI;IACtB,IAAMY,OAAOC,MAAMC,OAAO,CAACR,OAAOA,MAAM,OAAOA,QAAQ,WAAW;QAACA;KAAI,GAAG,EAAE;IAC5E,IAAMS,QAAkB,EAAE;QACrB,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,yBAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAoB;YAApB,IAAMI,OAAN;YACH,IAAI,OAAOA,SAAS,UAAU;YAC9B,IAAMC,MAAMZ,aAAaW;YACzB,IAAIC,KAAKF,MAAMG,IAAI,CAACD;QACtB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOF;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASI,WAAWC,IAAY;IAC9B,IAAML,QAAkB,EAAE;QACrB,kCAAA,2BAAA;;QAAL,QAAK,YAAcM,IAAAA,qBAAW,EAACD,MAAME,KAAK,CAAC,0BAAtC,SAAA,6BAAA,QAAA,yBAAA,iCAA6C;YAA7C,IAAMC,OAAN;YACH,IAAI,CAACA,KAAKC,QAAQ,CAAC,MAAM,UAAU,kCAAkC;YACrE,IAAIC,UAAUF;YACd,IAAIE,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACP,aAAa,SAACyB;uBAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;;YAC7F,IAAIH,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACJ,cAAc,SAACsB;uBAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;;YAC9F,IAAIH,QAAQD,QAAQ,CAAC,MAAMC,UAAUA,QAAQjB,OAAO,CAACN,aAAa,SAACwB;uBAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;;gBACvF,mCAAA,4BAAA;;gBAAL,QAAK,aAAWH,QAAQI,QAAQ,CAAC1B,mCAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAA4C;oBAA5C,IAAMuB,IAAN;oBACH,IAAMT,MAAMZ,aAAaqB,CAAC,CAAC,EAAE;oBAC7B,IAAIT,KAAKF,MAAMG,IAAI,CAACD;gBACtB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAIP;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWL,OAAOF;AACT;AAEA,SAASe,QAAQC,IAAY,EAAEX,IAAY,EAAEY,OAA4C,EAAErB,IAA8B;IACvH,OAAO,AAAC,qBAAG,IAAIsB,IAAI,AAAC,qBAAGvB,gBAAgBC,cAAO,qBAAGQ,WAAWC,UAASc,IAAI;AAC3E;AAEO,IAAMlC,OAAgB;IAC3BmC,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;wBACA;;4BAAMD,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAR,SAAAA;IAGMS,QAFN,4FAA4F;IAC5F,yHAAyH;IACzH,SAAMA,OAAOF,EAAE,EAAEG,KAAK;;;;;wBACpB,IAAIA,MAAMZ,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMS,GAAGI,QAAQ,CACf,qCACAD,MAAME,GAAG,CAAC,SAACC;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IACMC,OAAN,SAAMA,MAAMP,EAAE,EAAEQ,IAAI;;gBACZC,MACuC,2BAAA,mBAAA,gBAAxC,4BAAA,oBAAA,iBAAA,WAAA,oBAAQC,MAAMC,WAA0B,YAAA,QAAM/B;;;;wBAD7C6B;wBACuC,kCAAA,2BAAA,4BAAxC,mCAAA,4BAAA;;4BAAL,IAAK,YAA6BD,2BAA7B,8BAAA,QAAA,yBAAA;8CAAA,aAAQE,mBAAAA,MAAMC,wBAAAA;;oCAAqB,IAAK,aAAaA,gCAAb,6BAAA,SAAA,0BAAA;wCAAM/B,MAAN;wCAAoC6B,KAAK5B,IAAI;4CAAE6B;4CAAM9B;;;;oCAArD;oCAAA;;;6CAAA,6BAAA;4CAAA;;;4CAAA;kDAAA;;;;;;4BAAxC;4BAAA;;;qCAAA,8BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBACL,IAAI6B,KAAKlB,MAAM,KAAK,GAAG;;;wBACvB;;4BAAMS,GAAGI,QAAQ,CAAC,0DAA0DK;;;wBAA5E;;;;;;QACF;;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/tags.ts"],"sourcesContent":["import { appendRows } from '../store/shared.ts';\nimport { maskRegions } from './fences.ts';\nimport type { Feature } from './types.ts';\n\n// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline\n// #tags, deduplicated, source not distinguished. Nested tags store full (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` matches the parent too.\n\n// Obsidian treats [[#Heading]] as a same-note link, not a tag.\nconst WIKILINK_RE = /\\[\\[.*?\\]\\]/g; // to the first ]], so a heading holding a lone ] still masks\n// Obsidian doesn't read tags inside HTML markup.\nconst HTML_TAG_RE = /<\\/?[a-zA-Z][^>]*>/g; // tag-shaped only: a comparison's `< 5` must not open a span\n// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.\nconst INLINE_TAG_RE = /(?:^|[\\s([])#([\\p{L}\\p{N}_/-]+)/gu;\n// A markdown link destination `](...)` -- `[text](#anchor)` is a same-page link, not a tag.\nconst LINK_DEST_RE = /\\]\\((?:[^()]|\\([^()]*\\))*\\)/g; // one paren-nesting level, as CommonMark destinations allow: (https://x/a_(b)#frag)\n\n// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an\n// all-digit result -- a tag needs at least one non-digit character.\nfunction normalizeTag(raw: string): string | null {\n  const stripped = raw.replace(/^#/, '').replace(/\\/+$/, '');\n  if (!stripped || /^\\d+$/.test(stripped)) return null;\n  return stripped;\n}\n\n// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string\n// members are skipped rather than throwing -- `tags:\\n  -` parses to [null].\nfunction frontmatterTags(data?: Record<string, unknown>): string[] {\n  const raw = data?.tags;\n  const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [];\n  const found: string[] = [];\n  for (const item of list) {\n    if (typeof item !== 'string') continue;\n    const tag = normalizeTag(item);\n    if (tag) found.push(tag);\n  }\n  return found;\n}\n\n// #tag tokens outside the regions maskRegions() masks (fences, code spans, wikilinks, HTML,\n// comments, link destinations); this loop only does per-line tag extraction.\nfunction inlineTags(body: string): string[] {\n  const found: string[] = [];\n  for (const line of maskRegions(body).split('\\n')) {\n    if (!line.includes('#')) continue; // most lines; skip the regex work\n    let cleaned = line;\n    if (cleaned.includes('[[')) cleaned = cleaned.replace(WIKILINK_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('](')) cleaned = cleaned.replace(LINK_DEST_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('<')) cleaned = cleaned.replace(HTML_TAG_RE, (m) => ' '.repeat(m.length));\n    for (const m of cleaned.matchAll(INLINE_TAG_RE)) {\n      const tag = normalizeTag(m[1]);\n      if (tag) found.push(tag);\n    }\n  }\n  return found;\n}\n\nfunction extract(_raw: string, body: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): string[] {\n  return [...new Set([...frontmatterTags(data), ...inlineTags(body)])].sort();\n}\n\nexport const tags: Feature = {\n  name: 'tags',\n  async schema(db) {\n    await db.exec('CREATE TABLE IF NOT EXISTS tags (\"path\" TEXT, tag TEXT, PRIMARY KEY (\"path\", tag))');\n    await db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');\n  },\n  extract,\n  // Blanket clear-then-reinsert (store() always writes the full found list back), so remove()\n  // covers vanished and reparsed files without per-file NOT IN diffing, which cannot be one statement shared across files.\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM tags WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) for (const tag of extracted as string[]) rows.push([path, tag]);\n    // OR IGNORE guards the same concurrent-reconcile race sections.ts documents; remove() cleared\n    // every touched path above, so a store with an append path needs no guard.\n    await appendRows(db, 'tags', ['path', 'tag'], 'INSERT OR IGNORE INTO tags (\"path\", tag) VALUES (?, ?)', rows);\n  },\n};\n"],"names":["tags","WIKILINK_RE","HTML_TAG_RE","INLINE_TAG_RE","LINK_DEST_RE","normalizeTag","raw","stripped","replace","test","frontmatterTags","data","list","Array","isArray","found","item","tag","push","inlineTags","body","maskRegions","split","line","includes","cleaned","m","repeat","length","matchAll","extract","_raw","_search","Set","sort","name","schema","db","exec","remove","paths","runBatch","map","p","store","docs","rows","path","extracted","appendRows"],"mappings":";;;;+BA4DaA;;;eAAAA;;;wBA5Dc;wBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG5B,0FAA0F;AAC1F,kJAAkJ;AAElJ,+DAA+D;AAC/D,IAAMC,cAAc,gBAAgB,6DAA6D;AACjG,iDAAiD;AACjD,IAAMC,cAAc,uBAAuB,6DAA6D;AACxG,iGAAiG;AACjG,IAAMC,gBAAgB;AACtB,4FAA4F;AAC5F,IAAMC,eAAe,gCAAgC,oFAAoF;AAEzI,sFAAsF;AACtF,oEAAoE;AACpE,SAASC,aAAaC,GAAW;IAC/B,IAAMC,WAAWD,IAAIE,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,QAAQ;IACvD,IAAI,CAACD,YAAY,QAAQE,IAAI,CAACF,WAAW,OAAO;IAChD,OAAOA;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASG,gBAAgBC,IAA8B;IACrD,IAAML,MAAMK,iBAAAA,2BAAAA,KAAMX,IAAI;IACtB,IAAMY,OAAOC,MAAMC,OAAO,CAACR,OAAOA,MAAM,OAAOA,QAAQ,WAAW;QAACA;KAAI,GAAG,EAAE;IAC5E,IAAMS,QAAkB,EAAE;QACrB,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,yBAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAoB;YAApB,IAAMI,OAAN;YACH,IAAI,OAAOA,SAAS,UAAU;YAC9B,IAAMC,MAAMZ,aAAaW;YACzB,IAAIC,KAAKF,MAAMG,IAAI,CAACD;QACtB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOF;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASI,WAAWC,IAAY;IAC9B,IAAML,QAAkB,EAAE;QACrB,kCAAA,2BAAA;;QAAL,QAAK,YAAcM,IAAAA,qBAAW,EAACD,MAAME,KAAK,CAAC,0BAAtC,SAAA,6BAAA,QAAA,yBAAA,iCAA6C;YAA7C,IAAMC,OAAN;YACH,IAAI,CAACA,KAAKC,QAAQ,CAAC,MAAM,UAAU,kCAAkC;YACrE,IAAIC,UAAUF;YACd,IAAIE,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACP,aAAa,SAACyB;uBAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;;YAC7F,IAAIH,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACJ,cAAc,SAACsB;uBAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;;YAC9F,IAAIH,QAAQD,QAAQ,CAAC,MAAMC,UAAUA,QAAQjB,OAAO,CAACN,aAAa,SAACwB;uBAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;;gBACvF,mCAAA,4BAAA;;gBAAL,QAAK,aAAWH,QAAQI,QAAQ,CAAC1B,mCAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAA4C;oBAA5C,IAAMuB,IAAN;oBACH,IAAMT,MAAMZ,aAAaqB,CAAC,CAAC,EAAE;oBAC7B,IAAIT,KAAKF,MAAMG,IAAI,CAACD;gBACtB;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAIP;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWL,OAAOF;AACT;AAEA,SAASe,QAAQC,IAAY,EAAEX,IAAY,EAAEY,OAA4C,EAAErB,IAA8B;IACvH,OAAO,AAAC,qBAAG,IAAIsB,IAAI,AAAC,qBAAGvB,gBAAgBC,cAAO,qBAAGQ,WAAWC,UAASc,IAAI;AAC3E;AAEO,IAAMlC,OAAgB;IAC3BmC,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;wBACA;;4BAAMD,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAR,SAAAA;IAGMS,QAFN,4FAA4F;IAC5F,yHAAyH;IACzH,SAAMA,OAAOF,EAAE,EAAEG,KAAK;;;;;wBACpB,IAAIA,MAAMZ,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMS,GAAGI,QAAQ,CACf,qCACAD,MAAME,GAAG,CAAC,SAACC;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IACMC,OAAN,SAAMA,MAAMP,EAAE,EAAEQ,IAAI;;gBACZC,MACuC,2BAAA,mBAAA,gBAAxC,4BAAA,oBAAA,iBAAA,WAAA,oBAAQC,MAAMC,WAA0B,YAAA,QAAM/B;;;;wBAD7C6B;wBACuC,kCAAA,2BAAA,4BAAxC,mCAAA,4BAAA;;4BAAL,IAAK,YAA6BD,2BAA7B,8BAAA,QAAA,yBAAA;8CAAA,aAAQE,mBAAAA,MAAMC,wBAAAA;;oCAAqB,IAAK,aAAaA,gCAAb,6BAAA,SAAA,0BAAA;wCAAM/B,MAAN;wCAAoC6B,KAAK5B,IAAI;4CAAE6B;4CAAM9B;;;;oCAArD;oCAAA;;;6CAAA,6BAAA;4CAAA;;;4CAAA;kDAAA;;;;;;4BAAxC;4BAAA;;;qCAAA,8BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBACL,8FAA8F;wBAC9F,2EAA2E;wBAC3E;;4BAAMgC,IAAAA,oBAAU,EAACZ,IAAI;gCAAS;gCAAQ;+BAAQ,0DAA0DS;;;wBAAxG;;;;;;QACF;;AACF"}

package/dist/cjs/index.d.cts

@@ -6,7 +6,7 @@ export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
-export type { OpenResult } from './store/index.js';
+export type { OpenResult, Stages } from './store/index.js';
 export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';

package/dist/cjs/index.d.ts

@@ -6,7 +6,7 @@ export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
-export type { OpenResult } from './store/index.js';
+export type { OpenResult, Stages } from './store/index.js';
 export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';

package/dist/cjs/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAW9DA;eAAAA,0BAAgB;;QANhBC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIQC;eAAAA,oBAAU;;QANXC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAMXC;eAAbC,mBAAS;;QAR9BC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAEsC;uBAIvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult, Stages } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["CAPABILITY_NAMES","CONFIG_FILENAME","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","clearCache","initConfig","loadConfig","mapTree","migrateConfig","open","openStore","peek","presetCoverage","printRows","runWatch","search"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;;;;;;;;;;;;QAW9DA;eAAAA,0BAAgB;;QANhBC;eAAAA,yBAAe;;QAAyCC;eAAAA,sBAAY;;QAAEC;eAAAA,mBAAS;;QAAEC;eAAAA,qBAAW;;QAAEC;eAAAA,kCAAwB;;QAEtHC;eAAAA,oBAAU;;QAIQC;eAAAA,oBAAU;;QANXC;eAAAA,oBAAU;;QAAEC;eAAAA,oBAAU;;QAFvCC;eAAAA,gBAAO;;QAEkCC;eAAAA,uBAAa;;QAMXC;eAAbC,mBAAS;;QAR9BC;eAAAA,aAAI;;QAAEC;eAAAA,uBAAc;;QAM7BC;eAAAA,mBAAS;;QAMTC;eAAAA,iBAAQ;;QAZuBC;eAAAA,eAAM;;;uBAAQ;wBAEiF;wBAE5G;wBAED;wBAEsC;uBAIvC"}

package/dist/cjs/scan/pool.d.cts

@@ -5,6 +5,7 @@ import type { FileStat } from './list.js';
 export type FileResult = {
     doc: ParsedDoc;
     warnings: string[];
+    parseMs?: number;
 };
 export declare class ParsePool {
     private pool;

package/dist/cjs/scan/pool.d.ts

@@ -5,6 +5,7 @@ import type { FileStat } from './list.js';
 export type FileResult = {
     doc: ParsedDoc;
     warnings: string[];
+    parseMs?: number;
 };
 export declare class ParsePool {
     private pool;

package/dist/cjs/scan/pool.js

@@ -222,7 +222,8 @@ var ParsePool = /*#__PURE__*/ function() {
                                             2,
                                             {
                                                 doc: result.doc,
-                                                warnings: result.warnings
+                                                warnings: result.warnings,
+                                                parseMs: result.parseMs
                                             }
                                         ];
                                 }

package/dist/cjs/scan/pool.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\nexport type FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["ParsePool","_require","require","createRequire","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","poolsCreated","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","run","files","features","cfg","onParsed","done","featureNames","map","feature","Promise","all","file","result","ok","reviveError","error","doc","warnings","close","undefined","destroy"],"mappings":";;;;+BAoCaA;;;eAAAA;;;0BApCiB;wBACA;6BASF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOH,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCI,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAMO,IAAA,AAAMZ,0BAAN;;aAAMA;gCAAAA;QAEX,2FAA2F;QAC3F,uGAAuG;aACvGa,eAAe;;iBAJJb;IAMX,OAAQc,MAOP,GAPD,SAAQA,OAAOC,UAA2B,EAAEC,UAAkB;QAC5D,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,IAAM,AAAEC,WAAajB,SAAS,YAAtBiB;YACR,IAAI,CAACD,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUd;gBAAqBe,YAAYJ;gBAAYK,YAAYL;gBAAYD,YAAAA;YAAW;YACrH,IAAI,CAACF,YAAY;QACnB;QACA,OAAO,IAAI,CAACI,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,OAAMK,GAgBL,GAhBD,SAAMA,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEV,UAAkB;;gBAGzHD,YACAE,MACFU;;gBAJJ,6FAA6F;gBAC7F,uFAAuF;gBACjFZ,aAA8B;oBAAEU,KAAAA;oBAAKG,cAAcJ,SAASK,GAAG,CAAC,SAACC;+BAAYA,QAAQtB,IAAI;;gBAAE;gBAC3FS,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;gBACjCW,OAAO;gBACX,0FAA0F;gBAC1F,4FAA4F;gBAC5F;;oBAAOI,QAAQC,GAAG,CAChBT,MAAMM,GAAG,CAAC,SAAOI;;gCACTC;;;;wCAAU;;4CAAMjB,KAAKK,GAAG,CAACW;;;wCAAzBC,SAAU;wCAChB,IAAI,CAACA,OAAOC,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACF,OAAOG,KAAK;wCAC9CX,qBAAAA,+BAAAA,SAAW,EAAEC;wCACb;;4CAAO;gDAAEW,KAAKJ,OAAOI,GAAG;gDAAEC,UAAUL,OAAOK,QAAQ;4CAAC;;;;wBACtD;;;;QAEJ;;IAEA,iDAAiD;IACjD,OAAMC,KAKL,GALD,SAAMA;;gBAEEvB;;;;wBADN,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;;;wBACVA,OAAO,IAAI,CAACA,IAAI;wBACtB,IAAI,CAACA,IAAI,GAAGwB;wBACZ;;4BAAMxB,KAAKyB,OAAO;;;wBAAlB;;;;;;QACF;;WAzCW1C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["ParsePool","_require","require","createRequire","workerFile","resolveWorkerFile","load","rel","name","join","dirname","resolve","Error","poolsCreated","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","run","files","features","cfg","onParsed","done","featureNames","map","feature","Promise","all","file","result","ok","reviveError","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":";;;;+BAqCaA;;;eAAAA;;;0BArCiB;wBACA;6BASF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE5B,8GAA8G;AAC9G,IAAMC,WAAW,OAAOC,YAAY,cAAcC,IAAAA,yBAAa,EAAC,uDAAmBD;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,IAAME,OAAOH,IAAAA,yBAAa,EAAC;IAC3B,gBAAkB,QAAA;QAAC;QAAM;QAAS;KAAW,OAA3B,mBAA6B;YAApCI,MAAO;QAChB,IAAI;YACF,IAAI,AAACD,KAAK,AAAC,GAAM,OAAJC,KAAI,kBAAsCC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaK,IAAAA,cAAI,EAACC,IAAAA,iBAAO,EAACJ,KAAKK,OAAO,CAAC,AAAC,GAAM,OAAJJ,KAAI,oBAAkB,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,eAAM,CAAC;IACX;IACA,MAAM,IAAIQ,MAAM;AAClB;AAOO,IAAA,AAAMZ,0BAAN;;aAAMA;gCAAAA;QAEX,2FAA2F;QAC3F,uGAAuG;aACvGa,eAAe;;iBAJJb;IAMX,OAAQc,MAOP,GAPD,SAAQA,OAAOC,UAA2B,EAAEC,UAAkB;QAC5D,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,IAAM,AAAEC,WAAajB,SAAS,YAAtBiB;YACR,IAAI,CAACD,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUd;gBAAqBe,YAAYJ;gBAAYK,YAAYL;gBAAYD,YAAAA;YAAW;YACrH,IAAI,CAACF,YAAY;QACnB;QACA,OAAO,IAAI,CAACI,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,OAAMK,GAgBL,GAhBD,SAAMA,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEV,UAAkB;;gBAGzHD,YACAE,MACFU;;gBAJJ,6FAA6F;gBAC7F,uFAAuF;gBACjFZ,aAA8B;oBAAEU,KAAAA;oBAAKG,cAAcJ,SAASK,GAAG,CAAC,SAACC;+BAAYA,QAAQtB,IAAI;;gBAAE;gBAC3FS,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;gBACjCW,OAAO;gBACX,0FAA0F;gBAC1F,4FAA4F;gBAC5F;;oBAAOI,QAAQC,GAAG,CAChBT,MAAMM,GAAG,CAAC,SAAOI;;gCACTC;;;;wCAAU;;4CAAMjB,KAAKK,GAAG,CAACW;;;wCAAzBC,SAAU;wCAChB,IAAI,CAACA,OAAOC,EAAE,EAAE,MAAMC,IAAAA,0BAAW,EAACF,OAAOG,KAAK;wCAC9CX,qBAAAA,+BAAAA,SAAW,EAAEC;wCACb;;4CAAO;gDAAEW,KAAKJ,OAAOI,GAAG;gDAAEC,UAAUL,OAAOK,QAAQ;gDAAEC,SAASN,OAAOM,OAAO;4CAAC;;;;wBAC/E;;;;QAEJ;;IAEA,iDAAiD;IACjD,OAAMC,KAKL,GALD,SAAMA;;gBAEExB;;;;wBADN,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;;;wBACVA,OAAO,IAAI,CAACA,IAAI;wBACtB,IAAI,CAACA,IAAI,GAAGyB;wBACZ;;4BAAMzB,KAAK0B,OAAO;;;wBAAlB;;;;;;QACF;;WAzCW3C"}

package/dist/cjs/scan/reparse.d.cts

@@ -7,6 +7,7 @@ export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
     newColumns: string[];
+    workerParseMs: number;
 }
 export interface ReparseOptions {
     threshold?: number;

package/dist/cjs/scan/reparse.d.ts

@@ -7,6 +7,7 @@ export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
     newColumns: string[];
+    workerParseMs: number;
 }
 export interface ReparseOptions {
     threshold?: number;

package/dist/cjs/scan/reparse.js

@@ -265,7 +265,7 @@ function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
 }
 function reparseFiles(_0, _1, _2, _3, _4) {
     return _async_to_generator(function(files, features, cfg, knownColumns, onParsed) {
-        var options, _options_threshold, _options_maxWorkers, threshold, maxWorkers, results, _tmp, docs, warnings, newColumns, seen, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, doc, fileWarnings, _warnings, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, key;
+        var options, _options_threshold, _options_maxWorkers, threshold, maxWorkers, results, _tmp, docs, warnings, newColumns, seen, workerParseMs, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, doc, fileWarnings, parseMs, _warnings, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, key;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
@@ -296,12 +296,14 @@ function reparseFiles(_0, _1, _2, _3, _4) {
                     warnings = [];
                     newColumns = [];
                     seen = new Set(knownColumns);
+                    workerParseMs = 0;
                     _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                     try {
                         for(_iterator = results[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                            _step_value = _step.value, doc = _step_value.doc, fileWarnings = _step_value.warnings;
+                            _step_value = _step.value, doc = _step_value.doc, fileWarnings = _step_value.warnings, parseMs = _step_value.parseMs;
                             ;
                             (_warnings = warnings).push.apply(_warnings, _to_consumable_array(fileWarnings));
+                            workerParseMs += parseMs !== null && parseMs !== void 0 ? parseMs : 0;
                             _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
                             try {
                                 for(_iterator1 = Object.keys(doc.data)[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
@@ -346,7 +348,8 @@ function reparseFiles(_0, _1, _2, _3, _4) {
                         {
                             docs: docs,
                             warnings: warnings,
-                            newColumns: newColumns
+                            newColumns: newColumns,
+                            workerParseMs: workerParseMs
                         }
                     ];
             }

package/dist/cjs/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["featuresForFile","reparseFiles","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","err","run","ParsePool","close","catch","knownColumns","options","threshold","docs","warnings","newColumns","seen","doc","fileWarnings","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAoCgBA;eAAAA;;QAkCMC;eAAAA;;;sBAtEe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsB1B,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAMC,oBAAoB;AAC1B,IAAMC,sBAAsB;AAIrB,SAASH,gBAAgBI,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAef,gBAAgBI,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WAEER,SAGCS;;;;oBANT,IAAIF,MAAM;;wBAAOA,KAAKG,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIG,iBAAS;;;;;;;;;oBAEb;;wBAAMH,UAAUE,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;oBAChB;;wBAAMQ,UAAUI,KAAK;;;oBAArB;oBACA;;wBAAOZ;;;oBACAS;oBACP,oFAAoF;oBACpF,8DAA8D;oBAC9D;;wBAAMD,UAAUI,KAAK,GAAGC,KAAK,CAAC,YAAO;;;oBAArC;oBACA,MAAMJ;;;;;;;IAEV;;AAIO,SAAerB;wCAAaU,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEsB,YAAiC,EAAEf,QAAiC;YAAEgB,SAC1IA,oBACCA,qBADbC,WACAV,YACAN,eAEAiB,MACAC,UACAC,YACAC,MACD,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAC1BJ,WACK,4BAAA,oBAAA,iBAAA,YAAA,QAAMK;;;;;oBAX+IR,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;oBACjCiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBS,KAAKC,GAAG,CAACnC,qBAAqBoC,IAAAA,4BAAoB;yBAC3E5B,CAAAA,MAAM6B,MAAM,IAAIX,SAAQ,GAAxBlB;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO,YAAYS,QAAQR,IAAI;;;2BAA5E;;;;;;2BAAgFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA1JC;oBAEAiB;oBACAC;oBACAC;oBACAC,OAAO,IAAIQ,IAAId;oBAChB,kCAAA,2BAAA;;wBAAL,IAAK,YAAyCd,8BAAzC,6BAAA,QAAA,yBAAA,iCAAkD;0CAAlD,aAAQqB,kBAAAA,KAAeC,2BAAVJ;;4BAChBA,CAAAA,YAAAA,UAASf,IAAI,OAAbe,WAAc,qBAAGI;4BACZ,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAACT,IAAIU,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACH,KAAKY,GAAG,CAACT,MAAM;wCAClBH,KAAKa,GAAG,CAACV;wCACTJ,WAAWhB,IAAI,CAACoB;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLN,KAAKd,IAAI,CAACkB;wBACZ;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWL;;wBAAO;4BAAEJ,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;wBAAW;;;;IACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n  // Sum of worker-side parseFile time (workers/parse.ts) across files; 0 on the serial path.\n  workerParseMs: number;\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  let workerParseMs = 0;\n  for (const { doc, warnings: fileWarnings, parseMs } of results) {\n    warnings.push(...fileWarnings);\n    workerParseMs += parseMs ?? 0;\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns, workerParseMs };\n}\n"],"names":["featuresForFile","reparseFiles","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","parseFile","reparsePooled","maxWorkers","pool","ephemeral","err","run","ParsePool","close","catch","knownColumns","options","threshold","docs","warnings","newColumns","seen","workerParseMs","doc","fileWarnings","parseMs","key","Math","min","availableParallelism","length","Set","Object","keys","data","has","add"],"mappings":";;;;;;;;;;;QAsCgBA;eAAAA;;QAkCMC;eAAAA;;;sBAxEe;uBAIX;sBAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwB1B,+FAA+F;AAC/F,+GAA+G;AAC/G,IAAMC,oBAAoB;AAC1B,IAAMC,sBAAsB;AAIrB,SAASH,gBAAgBI,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,SAACC;eAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,IAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;QACN,kCAAA,2BAAA;;QAAL,QAAK,YAAcH,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAML,OAAN;YACH,IAAMS,eAAef,gBAAgBI,UAAUC,KAAKC;YACpDO,QAAQG,IAAI,CAACC,IAAAA,kBAAS,EAACX,MAAMS,cAAcV;YAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;QACf;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,SAAeK,cAAcP,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEO,UAAkB,EAAEC,IAA2B;;YAEzKC,WAEER,SAGCS;;;;oBANT,IAAIF,MAAM;;wBAAOA,KAAKG,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;oBACpDE,YAAY,IAAIG,iBAAS;;;;;;;;;oBAEb;;wBAAMH,UAAUE,GAAG,CAACZ,OAAOP,UAAUC,KAAKO,UAAUO;;;oBAA9DN,UAAU;oBAChB;;wBAAMQ,UAAUI,KAAK;;;oBAArB;oBACA;;wBAAOZ;;;oBACAS;oBACP,oFAAoF;oBACpF,8DAA8D;oBAC9D;;wBAAMD,UAAUI,KAAK,GAAGC,KAAK,CAAC,YAAO;;;oBAArC;oBACA,MAAMJ;;;;;;;IAEV;;AAIO,SAAerB;wCAAaU,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEsB,YAAiC,EAAEf,QAAiC;YAAEgB,SAC1IA,oBACCA,qBADbC,WACAV,YACAN,eAEAiB,MACAC,UACAC,YACAC,MACFC,eACC,2BAAA,mBAAA,gBAAA,WAAA,oBAAQC,KAAeC,cAAcC,SACxCN,WAEK,4BAAA,oBAAA,iBAAA,YAAA,QAAMO;;;;;oBAb+IV,UAAAA,oEAA0B,CAAC;oBACjLC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;oBACjCiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBW,KAAKC,GAAG,CAACrC,qBAAqBsC,IAAAA,4BAAoB;yBAC3E9B,CAAAA,MAAM+B,MAAM,IAAIb,SAAQ,GAAxBlB;;;;oBAA4B;;wBAAMO,cAAcP,OAAOP,UAAUC,KAAKO,UAAUO,YAAYS,QAAQR,IAAI;;;2BAA5E;;;;;;2BAAgFV,cAAcC,OAAOP,UAAUC,KAAKO;;;oBAA1JC;oBAEAiB;oBACAC;oBACAC;oBACAC,OAAO,IAAIU,IAAIhB;oBACjBO,gBAAgB;oBACf,kCAAA,2BAAA;;wBAAL,IAAK,YAAkDrB,8BAAlD,6BAAA,QAAA,yBAAA,iCAA2D;0CAA3D,aAAQsB,kBAAAA,KAAeC,2BAAVL,UAAwBM,sBAAAA;;4BACxCN,CAAAA,YAAAA,UAASf,IAAI,OAAbe,WAAc,qBAAGK;4BACjBF,iBAAiBG,oBAAAA,qBAAAA,UAAW;4BACvB,mCAAA,4BAAA;;gCAAL,IAAK,aAAaO,OAAOC,IAAI,CAACV,IAAIW,IAAI,wBAAjC,8BAAA,SAAA,0BAAA,kCAAoC;oCAA9BR,MAAN;oCACH,IAAI,CAACL,KAAKc,GAAG,CAACT,MAAM;wCAClBL,KAAKe,GAAG,CAACV;wCACTN,WAAWhB,IAAI,CAACsB;oCAClB;gCACF;;gCALK;gCAAA;;;yCAAA,8BAAA;wCAAA;;;wCAAA;8CAAA;;;;4BAMLR,KAAKd,IAAI,CAACmB;wBACZ;;wBAVK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAYL;;wBAAO;4BAAEL,MAAAA;4BAAMC,UAAAA;4BAAUC,YAAAA;4BAAYE,eAAAA;wBAAc;;;;IACrD"}

package/dist/cjs/store/builder.d.cts

@@ -1,11 +1,13 @@
 import type { Config } from '../config/index.js';
 import type { EmbedChangeKind } from './embed-scope.js';
 import type { FeatureToggle } from './feature-scope.js';
+import type { Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export interface Builder {
     build(forcedPaths?: ReadonlySet<string>): Promise<{
         parsed: number;
         warnings: string[];
+        stages: Stages;
     }>;
     invalidate(kind: EmbedChangeKind): Promise<{
         parsed: number;

package/dist/cjs/store/builder.d.ts

@@ -1,11 +1,13 @@
 import type { Config } from '../config/index.js';
 import type { EmbedChangeKind } from './embed-scope.js';
 import type { FeatureToggle } from './feature-scope.js';
+import type { Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export interface Builder {
     build(forcedPaths?: ReadonlySet<string>): Promise<{
         parsed: number;
         warnings: string[];
+        stages: Stages;
     }>;
     invalidate(kind: EmbedChangeKind): Promise<{
         parsed: number;

package/dist/cjs/store/builder.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[] }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","nullEmbedVectors","conn","dialect","withTransaction","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","existingPaths","files","docs","paths","delta","prepare","Set","all","map","r","path","listFiles","filter","f","embed","has","relPath","length","reparseFiles","undefined","d","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","feature","docsForFeature","NARROW_FEATURE_TABLE","name","FEATURES","find","Error","turnedOn","afterReconcile","nullRank","columns","getColumns","rerunRank","rank","invalidateFeatureToggles","toggles","byName","linksToggle","rankToggle","Map","get","ParsePool","build","forcedPaths","reconcile","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":";;;;+BA6JgBA;;;eAAAA;;;uBA5JM;uBACG;sBACJ;wBAEK;sBACA;yBACG;8BAGQ;2BACX;wBACC;6BACK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBhC,6FAA6F;AAC7F,wEAAwE;AACxE,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;;;;;oBACzE;;wBAAMC,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAAsDF,QAAQG,SAAS;;;oBAAnH;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;;IACrB;;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,SAAeC,kBAAkBN,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe;;YACnHC,cACAC,kBAGAC,OAGEC,MACFC,OACAC;;;;oBATe;;wBAAMf,KAAKgB,OAAO,CAAC;;;oBAAlCN,eAAe;wBACKO;oBAAK;;wBAAMP,aAAaQ,GAAG;;;oBAA/CP,gBAAgB,IAAA,CAAA,EAAA,MAAIM;;wBAAK,cAA0BE,GAAG,CAAC,SAACC;mCAAM,AAACA,EAAuBC,IAAI;;sBAAC;oBACjG,8FAA8F;oBAC9F,4FAA4F;oBACtFT,QAAQU,IAAAA,mBAAS,EAACf,KAAKC,SAASe,MAAM,CAAC,SAACC;+BAAMA,EAAEC,KAAK,IAAId,cAAce,GAAG,CAACF,EAAEG,OAAO;;oBAC1F,IAAIf,MAAMgB,MAAM,KAAK,GAAG;;wBAAO;4BAAEvB,QAAQ;wBAAE;;oBAE1B;;wBAAMwB,IAAAA,uBAAY,EAACjB;4BAAQa,cAAK;2BAAGlB,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAA9EI,OAAS,cAATA;oBACFC,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACjCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,KAAK;wBAAMC,QAAQ;oBAAK;oBAChF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQyB,eACAA;;;;4CADN;;iDAAMA,gBAAAA,cAAK,CAACU,MAAM,cAAZV,oCAAAA,mBAAAA,cAAK,EAAUzB,MAAMc,OAAOC;;;4CAAlC;4CACA;;iDAAMU,eAAAA,cAAK,CAACW,KAAK,cAAXX,mCAAAA,kBAAAA,cAAK,EACTzB,MACAa,KAAKM,GAAG,CAAC,SAACY;2DAAO;wDAAEV,MAAMU,EAAEJ,OAAO;wDAAEU,WAAWN,EAAEM,SAAS,CAACZ,KAAK;oDAAC;oDACjEV;;;4CAHF;;;;;;4BAKF;2BACAd,QAAQG,SAAS;;;oBAVnB;oBAYA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,SAAeU,uBAAuBtC,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE8B,MAAqB;;YAC/IC,OACAC,SAQA7B,OACEC,MACF6B,gBACA5B,OAIAC;;;;oBAhBAyB,QAAQG,oCAAoB,CAACJ,OAAOK,IAAI,CAAC;oBACzCH,UAAUI,iBAAQ,CAACC,IAAI,CAAC,SAACtB;+BAAMA,EAAEoB,IAAI,KAAKL,OAAOK,IAAI;;oBAC3D,IAAI,CAACH,SAAS,MAAM,IAAIM,MAAM,AAAC,wDAAmE,OAAZR,OAAOK,IAAI,EAAC;yBAE9F,CAACL,OAAOS,QAAQ,EAAhB;;;;oBACF;;wBAAM9C,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;2BAAUvC,QAAQG,SAAS;;;oBAAtF;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;oBAGfO,QAAQU,IAAAA,mBAAS,EAACf,KAAKC;oBACZ;;wBAAMqB,IAAAA,uBAAY,EAACjB;4BAAQ6B;2BAAUlC,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAAhFI,OAAS,cAATA;oBACF6B,iBAAiB7B,KAAKM,GAAG,CAAC,SAACY;+BAAO;4BAAEV,MAAMU,EAAEJ,OAAO;4BAAEU,WAAWN,EAAEM,SAAS,CAACI,QAAQG,IAAI,CAAC;wBAAC;;oBAC1F9B,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACvC,gGAAgG;oBAChG,8FAA8F;oBAC9F,8CAA8C;oBACxCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,OAAOnB;wBAAOoB,QAAQ;oBAAK;oBACnF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCAEQyC,gBACAA;;;;4CAFN;;gDAAMzC,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;;;4CAA/B;4CACA;;iDAAMC,iBAAAA,QAAQL,KAAK,cAAbK,qCAAAA,oBAAAA,SAAgBzC,MAAM0C,gBAAgB3B;;;4CAA5C;4CACA;;iDAAM0B,0BAAAA,QAAQQ,cAAc,cAAtBR,8CAAAA,6BAAAA,SAAyBzC,MAAMe;;;4CAArC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBAPnB;oBASA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,SAAesB,SAASlD,IAAgB,EAAEC,OAAyB;;YAC3DkD;;;;oBAAU;;wBAAMC,IAAAA,oBAAU,EAACpD;;;oBAA3BmD,UAAU;oBAChB,IAAI,CAACA,QAAQzB,GAAG,CAAC,UAAU;;;oBAC3B;;wBAAMxB,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAA0CF,QAAQG,SAAS;;;oBAAvG;;;;;;IACF;;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,SAAeiD,UAAUrD,IAAgB,EAAEC,OAAyB;;YAC5Dc;;;;oBAAAA,QAAwB;wBAAEH,KAAK;wBAAMoB,QAAQ;wBAAMC,KAAK;wBAAMC,QAAQ;oBAAK;oBACjF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQsD;;;;4CAAN;;iDAAMA,uBAAAA,YAAI,CAACL,cAAc,cAAnBK,2CAAAA,0BAAAA,YAAI,EAAkBtD,MAAMe;;;4CAAlC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBALnB;;;;;;IAOF;;AAEA,SAAemD,yBAAyBvD,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE+C,OAAwB;;YACpJC,QACFpD,QAIEqD,aAWAC,YAMD,2BAAA,mBAAA,gBAAA,WAAA,OAAMpB;;;;oBAtBLkB,SAAS,IAAIG,IAAIJ,QAAQrC,GAAG,CAAC,SAACoB;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL;yBAAO;;oBAChElC,SAAS;oBAEb,4FAA4F;oBAC5F,mFAAmF;oBAC7EqD,cAAcD,OAAOI,GAAG,CAAC;yBAC3BH,aAAAA;;;;oBACS;;wBAAMpB,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAMiD;;;oBAA3ErD,SAAAA,SAAU,AAAC,cAA8EA,MAAM;yBAG3F,CAACqD,YAAYV,QAAQ,EAArB;;;;oBAAuB;;wBAAME,SAASlD,MAAMC;;;oBAArB;;;oBAG7B,wFAAwF;oBACxF,yFAAyF;oBACzF,8EAA8E;oBACxE0D,aAAaF,OAAOI,GAAG,CAAC;yBAC1BF,YAAAA;;;;yBACEA,WAAWX,QAAQ,EAAnBW;;;;oBAAqB;;wBAAMN,UAAUrD,MAAMC;;;oBAAtB;;;;;;oBACpB;;wBAAMiD,SAASlD,MAAMC;;;oBAArB;;;oBAGF,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAgBuD;;;2BAAhB,6BAAA,QAAA;;;;oBAAMjB,SAAN;oBACH,IAAIA,OAAOK,IAAI,KAAK,WAAWL,OAAOK,IAAI,KAAK,QAAQ;;;;oBAC5C;;wBAAMN,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAM8B;;;oBAA3ElC,SAAAA,SAAU,AAAC,cAAyEA,MAAM;;;oBAFvF;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAIL;;wBAAO;4BAAEA,QAAAA;wBAAO;;;;IAClB;;AAIO,SAASP,cAAcE,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB;IACrG,IAAMQ,OAAO,IAAIqD,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA,MAAQC;mBAAgBC,IAAAA,sBAAS,EAACjE,MAAMO,KAAKC,SAASP,SAASQ,MAAMuD;;QACrEE,YAAY,SAAZA,WAAaC;mBAAUA,SAAS,UAAUpE,iBAAiBC,MAAMC,WAAWK,kBAAkBN,MAAMO,KAAKC,SAASP,SAASQ;;QAC3H2D,oBAAoB,SAApBA,mBAAqBZ;mBAAYD,yBAAyBvD,MAAMO,KAAKC,SAASP,SAASQ,MAAM+C;;QAC7Fa,OAAO,SAAPA;mBAAa5D,KAAK4D,KAAK;;QACvB,IAAIC;YACF,OAAO7D,KAAK6D,YAAY;QAC1B;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport type { Stages } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","nullEmbedVectors","conn","dialect","withTransaction","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","existingPaths","files","docs","paths","delta","prepare","Set","all","map","r","path","listFiles","filter","f","embed","has","relPath","length","reparseFiles","undefined","d","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","feature","docsForFeature","NARROW_FEATURE_TABLE","name","FEATURES","find","Error","turnedOn","afterReconcile","nullRank","columns","getColumns","rerunRank","rank","invalidateFeatureToggles","toggles","byName","linksToggle","rankToggle","Map","get","ParsePool","build","forcedPaths","reconcile","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":";;;;+BA8JgBA;;;eAAAA;;;uBA7JM;uBACG;sBACJ;wBAEK;sBACA;yBACG;8BAGQ;2BACX;wBACC;6BAEK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBhC,6FAA6F;AAC7F,wEAAwE;AACxE,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;;;;;oBACzE;;wBAAMC,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAAsDF,QAAQG,SAAS;;;oBAAnH;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;;IACrB;;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,SAAeC,kBAAkBN,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe;;YACnHC,cACAC,kBAGAC,OAGEC,MACFC,OACAC;;;;oBATe;;wBAAMf,KAAKgB,OAAO,CAAC;;;oBAAlCN,eAAe;wBACKO;oBAAK;;wBAAMP,aAAaQ,GAAG;;;oBAA/CP,gBAAgB,IAAA,CAAA,EAAA,MAAIM;;wBAAK,cAA0BE,GAAG,CAAC,SAACC;mCAAM,AAACA,EAAuBC,IAAI;;sBAAC;oBACjG,8FAA8F;oBAC9F,4FAA4F;oBACtFT,QAAQU,IAAAA,mBAAS,EAACf,KAAKC,SAASe,MAAM,CAAC,SAACC;+BAAMA,EAAEC,KAAK,IAAId,cAAce,GAAG,CAACF,EAAEG,OAAO;;oBAC1F,IAAIf,MAAMgB,MAAM,KAAK,GAAG;;wBAAO;4BAAEvB,QAAQ;wBAAE;;oBAE1B;;wBAAMwB,IAAAA,uBAAY,EAACjB;4BAAQa,cAAK;2BAAGlB,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAA9EI,OAAS,cAATA;oBACFC,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACjCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,KAAK;wBAAMC,QAAQ;oBAAK;oBAChF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQyB,eACAA;;;;4CADN;;iDAAMA,gBAAAA,cAAK,CAACU,MAAM,cAAZV,oCAAAA,mBAAAA,cAAK,EAAUzB,MAAMc,OAAOC;;;4CAAlC;4CACA;;iDAAMU,eAAAA,cAAK,CAACW,KAAK,cAAXX,mCAAAA,kBAAAA,cAAK,EACTzB,MACAa,KAAKM,GAAG,CAAC,SAACY;2DAAO;wDAAEV,MAAMU,EAAEJ,OAAO;wDAAEU,WAAWN,EAAEM,SAAS,CAACZ,KAAK;oDAAC;oDACjEV;;;4CAHF;;;;;;4BAKF;2BACAd,QAAQG,SAAS;;;oBAVnB;oBAYA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,SAAeU,uBAAuBtC,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE8B,MAAqB;;YAC/IC,OACAC,SAQA7B,OACEC,MACF6B,gBACA5B,OAIAC;;;;oBAhBAyB,QAAQG,oCAAoB,CAACJ,OAAOK,IAAI,CAAC;oBACzCH,UAAUI,iBAAQ,CAACC,IAAI,CAAC,SAACtB;+BAAMA,EAAEoB,IAAI,KAAKL,OAAOK,IAAI;;oBAC3D,IAAI,CAACH,SAAS,MAAM,IAAIM,MAAM,AAAC,wDAAmE,OAAZR,OAAOK,IAAI,EAAC;yBAE9F,CAACL,OAAOS,QAAQ,EAAhB;;;;oBACF;;wBAAM9C,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;2BAAUvC,QAAQG,SAAS;;;oBAAtF;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;oBAGfO,QAAQU,IAAAA,mBAAS,EAACf,KAAKC;oBACZ;;wBAAMqB,IAAAA,uBAAY,EAACjB;4BAAQ6B;2BAAUlC,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAAhFI,OAAS,cAATA;oBACF6B,iBAAiB7B,KAAKM,GAAG,CAAC,SAACY;+BAAO;4BAAEV,MAAMU,EAAEJ,OAAO;4BAAEU,WAAWN,EAAEM,SAAS,CAACI,QAAQG,IAAI,CAAC;wBAAC;;oBAC1F9B,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACvC,gGAAgG;oBAChG,8FAA8F;oBAC9F,8CAA8C;oBACxCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,OAAOnB;wBAAOoB,QAAQ;oBAAK;oBACnF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCAEQyC,gBACAA;;;;4CAFN;;gDAAMzC,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;;;4CAA/B;4CACA;;iDAAMC,iBAAAA,QAAQL,KAAK,cAAbK,qCAAAA,oBAAAA,SAAgBzC,MAAM0C,gBAAgB3B;;;4CAA5C;4CACA;;iDAAM0B,0BAAAA,QAAQQ,cAAc,cAAtBR,8CAAAA,6BAAAA,SAAyBzC,MAAMe;;;4CAArC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBAPnB;oBASA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,SAAesB,SAASlD,IAAgB,EAAEC,OAAyB;;YAC3DkD;;;;oBAAU;;wBAAMC,IAAAA,oBAAU,EAACpD;;;oBAA3BmD,UAAU;oBAChB,IAAI,CAACA,QAAQzB,GAAG,CAAC,UAAU;;;oBAC3B;;wBAAMxB,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAA0CF,QAAQG,SAAS;;;oBAAvG;;;;;;IACF;;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,SAAeiD,UAAUrD,IAAgB,EAAEC,OAAyB;;YAC5Dc;;;;oBAAAA,QAAwB;wBAAEH,KAAK;wBAAMoB,QAAQ;wBAAMC,KAAK;wBAAMC,QAAQ;oBAAK;oBACjF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQsD;;;;4CAAN;;iDAAMA,uBAAAA,YAAI,CAACL,cAAc,cAAnBK,2CAAAA,0BAAAA,YAAI,EAAkBtD,MAAMe;;;4CAAlC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBALnB;;;;;;IAOF;;AAEA,SAAemD,yBAAyBvD,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE+C,OAAwB;;YACpJC,QACFpD,QAIEqD,aAWAC,YAMD,2BAAA,mBAAA,gBAAA,WAAA,OAAMpB;;;;oBAtBLkB,SAAS,IAAIG,IAAIJ,QAAQrC,GAAG,CAAC,SAACoB;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL;yBAAO;;oBAChElC,SAAS;oBAEb,4FAA4F;oBAC5F,mFAAmF;oBAC7EqD,cAAcD,OAAOI,GAAG,CAAC;yBAC3BH,aAAAA;;;;oBACS;;wBAAMpB,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAMiD;;;oBAA3ErD,SAAAA,SAAU,AAAC,cAA8EA,MAAM;yBAG3F,CAACqD,YAAYV,QAAQ,EAArB;;;;oBAAuB;;wBAAME,SAASlD,MAAMC;;;oBAArB;;;oBAG7B,wFAAwF;oBACxF,yFAAyF;oBACzF,8EAA8E;oBACxE0D,aAAaF,OAAOI,GAAG,CAAC;yBAC1BF,YAAAA;;;;yBACEA,WAAWX,QAAQ,EAAnBW;;;;oBAAqB;;wBAAMN,UAAUrD,MAAMC;;;oBAAtB;;;;;;oBACpB;;wBAAMiD,SAASlD,MAAMC;;;oBAArB;;;oBAGF,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAgBuD;;;2BAAhB,6BAAA,QAAA;;;;oBAAMjB,SAAN;oBACH,IAAIA,OAAOK,IAAI,KAAK,WAAWL,OAAOK,IAAI,KAAK,QAAQ;;;;oBAC5C;;wBAAMN,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAM8B;;;oBAA3ElC,SAAAA,SAAU,AAAC,cAAyEA,MAAM;;;oBAFvF;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAIL;;wBAAO;4BAAEA,QAAAA;wBAAO;;;;IAClB;;AAIO,SAASP,cAAcE,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB;IACrG,IAAMQ,OAAO,IAAIqD,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA,MAAQC;mBAAgBC,IAAAA,sBAAS,EAACjE,MAAMO,KAAKC,SAASP,SAASQ,MAAMuD;;QACrEE,YAAY,SAAZA,WAAaC;mBAAUA,SAAS,UAAUpE,iBAAiBC,MAAMC,WAAWK,kBAAkBN,MAAMO,KAAKC,SAASP,SAASQ;;QAC3H2D,oBAAoB,SAApBA,mBAAqBZ;mBAAYD,yBAAyBvD,MAAMO,KAAKC,SAASP,SAASQ,MAAM+C;;QAC7Fa,OAAO,SAAPA;mBAAa5D,KAAK4D,KAAK;;QACvB,IAAIC;YACF,OAAO7D,KAAK6D,YAAY;QAC1B;IACF;AACF"}

package/dist/cjs/store/duckdb/connection.js

@@ -19,8 +19,10 @@ _export(exports, {
         return storeValueToJs;
     }
 });
+var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _batchts = require("./batch.js");
+var _nativets = require("./native.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -610,6 +612,124 @@ function createConnection(duckdb) {
                 });
             })();
         },
+        appendRows: // The appender writes columnar vectors with no per-parameter binding, which is the cost
+        // runBatch pays: measured 7.2x on frontmatter at 6,566 rows. Alignment is against the table's
+        // own physical column order, read fresh, so a column the caller does not write (a
+        // feature-owned "_rank", say) takes appendDefault() rather than shifting every value one slot.
+        function appendRows(table, columns, rows) {
+            return _async_to_generator(function() {
+                var variantValue, infoStmt, physical, rowIndexOf;
+                return _ts_generator(this, function(_state) {
+                    switch(_state.label){
+                        case 0:
+                            if (rows.length === 0) return [
+                                2
+                            ];
+                            return [
+                                4,
+                                (0, _nativets.duckdbApi)()
+                            ];
+                        case 1:
+                            variantValue = _state.sent().variantValue;
+                            return [
+                                4,
+                                conn.prepare("PRAGMA table_info(".concat((0, _sharedts.quoteIdent)(table), ")"))
+                            ];
+                        case 2:
+                            infoStmt = _state.sent();
+                            return [
+                                4,
+                                infoStmt.all()
+                            ];
+                        case 3:
+                            physical = _state.sent();
+                            rowIndexOf = new Map(columns.map(function(name, i) {
+                                return [
+                                    name,
+                                    i
+                                ];
+                            }));
+                            return [
+                                4,
+                                (0, _transactionts.withTransaction)(conn, function() {
+                                    return _async_to_generator(function() {
+                                        var appender, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, column, idx, value;
+                                        return _ts_generator(this, function(_state) {
+                                            switch(_state.label){
+                                                case 0:
+                                                    return [
+                                                        4,
+                                                        duckdb.createAppender(table)
+                                                    ];
+                                                case 1:
+                                                    appender = _state.sent();
+                                                    try {
+                                                        _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                                        try {
+                                                            for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                                                row = _step.value;
+                                                                _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                                                                try {
+                                                                    for(_iterator1 = physical[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                                                                        column = _step1.value;
+                                                                        idx = rowIndexOf.get(column.name);
+                                                                        value = idx === undefined ? undefined : row[idx];
+                                                                        if (idx === undefined) appender.appendDefault();
+                                                                        else if (value === null || value === undefined) appender.appendNull();
+                                                                        else if (column.type === 'VARIANT') appender.appendVariant(variantValue(value));
+                                                                        else appender.appendValue(value);
+                                                                    }
+                                                                } catch (err) {
+                                                                    _didIteratorError1 = true;
+                                                                    _iteratorError1 = err;
+                                                                } finally{
+                                                                    try {
+                                                                        if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                                                                            _iterator1.return();
+                                                                        }
+                                                                    } finally{
+                                                                        if (_didIteratorError1) {
+                                                                            throw _iteratorError1;
+                                                                        }
+                                                                    }
+                                                                }
+                                                                appender.endRow();
+                                                            }
+                                                        } catch (err) {
+                                                            _didIteratorError = true;
+                                                            _iteratorError = err;
+                                                        } finally{
+                                                            try {
+                                                                if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                                                    _iterator.return();
+                                                                }
+                                                            } finally{
+                                                                if (_didIteratorError) {
+                                                                    throw _iteratorError;
+                                                                }
+                                                            }
+                                                        }
+                                                        appender.flushSync();
+                                                    } finally{
+                                                        appender.closeSync();
+                                                    }
+                                                    return [
+                                                        2
+                                                    ];
+                                            }
+                                        });
+                                    })();
+                                })
+                            ];
+                        case 4:
+                            _state.sent();
+                            return [
+                                2
+                            ];
+                    }
+                });
+            })();
+        },
         runBatch: // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),
         // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.
         function runBatch(sql, paramRows) {