sensemaking 0.22.0 → 0.22.1

package/dist/cjs/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n  // Inserts rows whose path cannot already exist in `table` (no ON CONFLICT needed), through a\n  // faster append-only path. Optional: sqlite/turso omit it and every row goes through the upsert.\n  insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB"}

package/dist/esm/features/embed.js

@@ -3,6 +3,7 @@ import { embedConfig } from '../config/index.js';
 import { stashChunkText } from '../embed/handoff.js';
 import { modelIdentity } from '../embed/identity.js';
 import { countLines } from '../scan/frontmatter.js';
+import { appendRows } from '../store/shared.js';
 // The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so
 // `offset` shifts line numbers back onto raw and a range stays a direct Read range.
 function chunksOf(blocks, body, search, offset = 0, chunkTokens) {
@@ -54,7 +55,14 @@ export const embed = {
             texts.set(path, chunks.map((c)=>c.text));
         }
         if (rows.length === 0) return;
-        await db.runBatch('INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);
+        // scale and vector are written by the embed pass, not here, so they are not in the column list
+        // and take the table's own default on the append path.
+        await appendRows(db, 'embeddings', [
+            'path',
+            'chunk',
+            'start_line',
+            'end_line'
+        ], 'INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);
         // The text extract() just computed, handed to the embed pass that follows in this same
         // process rather than re-derived from the file there (src/embed/handoff.ts).
         stashChunkText(db, texts);

package/dist/esm/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and embedding tops up on the next search, so staleness costs recall.\n\nexport type { Chunk };\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, string[]>();\n    for (const { path, extracted } of docs) {\n      if (!extracted) continue;\n      const chunks = extracted as Chunk[];\n      chunks.forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n      texts.set(\n        path,\n        chunks.map((c) => c.text)\n      );\n    }\n    if (rows.length === 0) return;\n    await db.runBatch('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n    // The text extract() just computed, handed to the embed pass that follows in this same\n    // process rather than re-derived from the file there (src/embed/handoff.ts).\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","stashChunkText","modelIdentity","countLines","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","remove","paths","length","runBatch","p","store","docs","rows","texts","Map","path","extracted","chunks","forEach","idx","push","set","enabledForFile","_cfg","file","signature","e","identity","provider","model","chunkVersion"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,aAAa,QAAQ,uBAAuB;AACrD,SAASC,UAAU,QAAQ,yBAAyB;AAQpD,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOjB,gBAAgBO,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAChL;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUN,MAAMO,OAAOA,MAAMC,QAAQJ,WAAW0B,OAAO1B,WAAWG,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAClH;IACA,MAAMuB,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMR,GAAGS,QAAQ,CACf,2CACAF,MAAMf,GAAG,CAAC,CAACkB,IAAM;gBAACA;aAAE;IAExB;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMC,OAAMX,EAAE,EAAEY,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,MAAMC,QAAQ,IAAIC;QAClB,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIL,KAAM;YACtC,IAAI,CAACK,WAAW;YAChB,MAAMC,SAASD;YACfC,OAAOC,OAAO,CAAC,CAAC1B,GAAG2B,MAAQP,KAAKQ,IAAI,CAAC;oBAACL;oBAAMI;oBAAK3B,EAAEC,SAAS;oBAAED,EAAEE,OAAO;iBAAC;YACxEmB,MAAMQ,GAAG,CACPN,MACAE,OAAO1B,GAAG,CAAC,CAACC,IAAMA,EAAEG,IAAI;QAE5B;QACA,IAAIiB,KAAKL,MAAM,KAAK,GAAG;QACvB,MAAMR,GAAGS,QAAQ,CAAC,+GAA+GI;QACjI,uFAAuF;QACvF,6EAA6E;QAC7EtC,eAAeyB,IAAIc;IACrB;IACAS,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAK5B,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9D6B,WAAUrB,GAAG;QACX,MAAMsB,IAAIrD,YAAY+B;QACtB,IAAI,CAACsB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMC,WAAWD,EAAEE,QAAQ,KAAK,WAAWrD,cAAcmD,EAAEG,KAAK,IAAIxC;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMyC,eAAeJ,EAAE5C,WAAW,KAAKO,YAAY,GAAGnB,cAAc,CAAC,EAAEwD,EAAE5C,WAAW,EAAE,GAAGZ;QACzF,OAAO,CAAC,MAAM,EAAEwD,EAAEE,QAAQ,CAAC,CAAC,EAAEF,EAAEG,KAAK,CAAC,CAAC,EAAEC,eAAeH,aAAatC,YAAY,CAAC,CAAC,EAAEsC,UAAU,GAAG,IAAI;IACxG;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and embedding tops up on the next search, so staleness costs recall.\n\nexport type { Chunk };\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, string[]>();\n    for (const { path, extracted } of docs) {\n      if (!extracted) continue;\n      const chunks = extracted as Chunk[];\n      chunks.forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n      texts.set(\n        path,\n        chunks.map((c) => c.text)\n      );\n    }\n    if (rows.length === 0) return;\n    // scale and vector are written by the embed pass, not here, so they are not in the column list\n    // and take the table's own default on the append path.\n    await appendRows(db, 'embeddings', ['path', 'chunk', 'start_line', 'end_line'], 'INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n    // The text extract() just computed, handed to the embed pass that follows in this same\n    // process rather than re-derived from the file there (src/embed/handoff.ts).\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","stashChunkText","modelIdentity","countLines","appendRows","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","remove","paths","length","runBatch","p","store","docs","rows","texts","Map","path","extracted","chunks","forEach","idx","push","set","enabledForFile","_cfg","file","signature","e","identity","provider","model","chunkVersion"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,aAAa,QAAQ,uBAAuB;AACrD,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAQhD,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOlB,gBAAgBQ,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAChL;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUP,MAAMQ,OAAOA,MAAMC,QAAQL,WAAW2B,OAAO3B,WAAWI,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAClH;IACA,MAAMuB,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMR,GAAGS,QAAQ,CACf,2CACAF,MAAMf,GAAG,CAAC,CAACkB,IAAM;gBAACA;aAAE;IAExB;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMC,OAAMX,EAAE,EAAEY,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,MAAMC,QAAQ,IAAIC;QAClB,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIL,KAAM;YACtC,IAAI,CAACK,WAAW;YAChB,MAAMC,SAASD;YACfC,OAAOC,OAAO,CAAC,CAAC1B,GAAG2B,MAAQP,KAAKQ,IAAI,CAAC;oBAACL;oBAAMI;oBAAK3B,EAAEC,SAAS;oBAAED,EAAEE,OAAO;iBAAC;YACxEmB,MAAMQ,GAAG,CACPN,MACAE,OAAO1B,GAAG,CAAC,CAACC,IAAMA,EAAEG,IAAI;QAE5B;QACA,IAAIiB,KAAKL,MAAM,KAAK,GAAG;QACvB,+FAA+F;QAC/F,uDAAuD;QACvD,MAAM/B,WAAWuB,IAAI,cAAc;YAAC;YAAQ;YAAS;YAAc;SAAW,EAAE,+GAA+Ga;QAC/L,uFAAuF;QACvF,6EAA6E;QAC7EvC,eAAe0B,IAAIc;IACrB;IACAS,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAK5B,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9D6B,WAAUrB,GAAG;QACX,MAAMsB,IAAItD,YAAYgC;QACtB,IAAI,CAACsB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMC,WAAWD,EAAEE,QAAQ,KAAK,WAAWtD,cAAcoD,EAAEG,KAAK,IAAIxC;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMyC,eAAeJ,EAAE5C,WAAW,KAAKO,YAAY,GAAGpB,cAAc,CAAC,EAAEyD,EAAE5C,WAAW,EAAE,GAAGb;QACzF,OAAO,CAAC,MAAM,EAAEyD,EAAEE,QAAQ,CAAC,CAAC,EAAEF,EAAEG,KAAK,CAAC,CAAC,EAAEC,eAAeH,aAAatC,YAAY,CAAC,CAAC,EAAEsC,UAAU,GAAG,IAAI;IACxG;AACF,EAAE"}

package/dist/esm/features/links.js

@@ -1,4 +1,5 @@
 import posix from 'node:path/posix';
+import { appendRows } from '../store/shared.js';
 import { maskRegions } from './fences.js';
 // links(src, target, target_base, dst, embed): target as written, target_base its indexed baseKey, dst the resolved path or NULL (a queryable dead link).
 // embed distinguishes `![[x]]`/`![](x.md)` from `[[x]]`/`[](x.md)` -- Obsidian's grain: a target both linked and embedded in one note is two rows.
@@ -185,6 +186,15 @@ export async function linkEdges(db) {
 // Records whether remove()/store() deleted any link row, since afterReconcile runs after both
 // and dstChanged alone misses the reparsed-to-zero-links case. Keyed on delta so state dies with it.
 const removedWithLinks = new WeakMap();
+// dst is written by resolution, not by store(), so it is absent from the column list and takes the
+// table's default on the append path.
+const LINK_COLUMNS = [
+    'src',
+    'target',
+    'target_base',
+    'embed'
+];
+const UPSERT_LINK_SQL = 'INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base';
 export const links = {
     name: 'links',
     async schema (db) {
@@ -230,18 +240,25 @@ export const links = {
         }
         // Upsert preserves dst on surviving rows; only new rows start at NULL. A same-note anchor
         // gets no target_base, so SQL's `IN` excludes it from resolveIncremental's lookup.
+        // Split, unlike the other features: this hook has no remove() pass behind it (see above), so
+        // only a path in `added` is guaranteed to have no row yet. The rest genuinely conflict and the
+        // upsert is what preserves their `dst`.
+        const appendable = [];
         const upsertRows = [];
         for (const [path, targets] of byPath){
-            for (const { target, embed } of targets)upsertRows.push([
-                path,
-                target,
-                target.startsWith('#') ? null : baseKey(cleanTarget(target)),
-                embed ? 1 : 0
-            ]);
-        }
-        if (upsertRows.length > 0) {
-            await db.runBatch('INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base', upsertRows);
+            for (const { target, embed } of targets){
+                const row = [
+                    path,
+                    target,
+                    target.startsWith('#') ? null : baseKey(cleanTarget(target)),
+                    embed ? 1 : 0
+                ];
+                if (addedSet.has(path)) appendable.push(row);
+                else upsertRows.push(row);
+            }
         }
+        await appendRows(db, 'links', LINK_COLUMNS, UPSERT_LINK_SQL, appendable);
+        if (upsertRows.length > 0) await db.runBatch(UPSERT_LINK_SQL, upsertRows);
     },
     async afterReconcile (db, delta) {
         var _removedWithLinks_get;

package/dist/esm/features/links.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { FileStat } from '../scan/index.ts';\nimport type { Connection } from '../store/types.ts';\nimport { maskRegions } from './fences.ts';\nimport type { ExtractedDoc, Feature, ReconcileDelta } from './types.ts';\n\n// links(src, target, target_base, dst, embed): target as written, target_base its indexed baseKey, dst the resolved path or NULL (a queryable dead link).\n// embed distinguishes `![[x]]`/`![](x.md)` from `[[x]]`/`[](x.md)` -- Obsidian's grain: a target both linked and embedded in one note is two rows.\n\n// Alias stripped after |; a leading # keeps the anchor as the target (self-link needs a\n// heading name), else the anchor splits off. Null = not a link.\nfunction parseWikilinkInner(inner: string): string | null {\n  const beforeAlias = inner.split('|')[0].trim();\n  if (beforeAlias.startsWith('#')) return beforeAlias.length > 1 ? beforeAlias : null;\n  const target = beforeAlias.split('#')[0].trim();\n  return target || null;\n}\n\n// A frontmatter value counts only when it is exactly a wikilink (\"[[X]]\") -- mid-string and\n// ![[...]] forms don't count there, per Obsidian's rule (probe-verified).\nfunction extract(_raw: string, rawBody: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): Array<{ target: string; embed: boolean }> {\n  const body = /\\[\\[|\\]\\(/.test(rawBody) ? maskRegions(rawBody) : rawBody;\n  const seen = new Set<string>();\n  const results: Array<{ target: string; embed: boolean }> = [];\n  const add = (target: string, embed: boolean) => {\n    const key = `${target}\\0${embed ? '1' : '0'}`;\n    if (seen.has(key)) return;\n    seen.add(key);\n    results.push({ target, embed });\n  };\n  // To the first ]], as Obsidian parses it, so a heading or alias holding a lone ] still\n  // matches; anchor and alias split off in parseWikilinkInner.\n  for (const m of body.matchAll(/\\[\\[(.*?)\\]\\]/g)) {\n    const target = parseWikilinkInner(m[1]);\n    if (target) add(target, body[(m.index ?? 0) - 1] === '!');\n  }\n  // Any internal destination, not only .md-suffixed: \\](Zektor) resolves like [[Zektor]] and\n  // \\](#anchor) is a self-edge. External URLs and titled links (dest stops at whitespace) skip.\n  for (const m of body.matchAll(/(!)?\\[[^\\]]*\\]\\(((?:[^()\\s]|\\([^()\\s]*\\))+)(?:\\s+(?:\"[^)]*\"|'[^)]*'))?\\)/g)) {\n    const dest = m[2].trim();\n    // External: a URI scheme prefix (mailto:, tel:, data:, https:...), protocol-relative\n    // //host, a www. shorthand, or a scheme anywhere (the malformed double-paren case).\n    if (!dest || dest.includes('://')) continue;\n    if (/^([a-z][a-z0-9+.-]*:|\\/\\/|www\\.)/i.test(dest)) continue;\n    const target = dest.startsWith('#') ? dest : dest.split('#')[0];\n    if (target) add(target, m[1] === '!');\n  }\n  for (const value of Object.values(data ?? {})) {\n    for (const item of Array.isArray(value) ? value : [value]) {\n      if (typeof item !== 'string') continue;\n      const m = /^\\[\\[(.*)\\]\\]$/.exec(item.trim());\n      if (!m) continue;\n      const target = parseWikilinkInner(m[1]);\n      if (target) add(target, false);\n    }\n  }\n  return results;\n}\n\nfunction cleanTarget(target: string): string {\n  return target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n}\n\nfunction baseKey(path: string): string {\n  return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking note's\n// directory, then basename match (shortest path wins ties, verified against a cold-loaded vault).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n  // [[#Heading]]: a same-note anchor, always the note itself -- never depends on other files.\n  if (target.startsWith('#')) return src;\n  const clean = cleanTarget(target);\n  const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n  for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n    if (pathSet.has(candidate)) return candidate;\n  }\n  const candidates = byBase.get(baseKey(clean));\n  if (!candidates) return null;\n  // The linking note itself wins a basename collision (Obsidian resolves to self before the\n  // shortest-path rule -- cold-load verified on the hub corpus).\n  return candidates.includes(src) ? src : candidates[0];\n}\n\n// Shortest path wins a basename collision; equal lengths fall back to lexicographic, our own\n// deterministic tiebreak for a case Obsidian itself leaves registration-order-dependent.\nfunction buildByBase(files: FileStat[]): Map<string, string[]> {\n  const byBase = new Map<string, string[]>();\n  const paths = files.map((f) => f.relPath).sort((a, b) => a.length - b.length || (a < b ? -1 : a > b ? 1 : 0));\n  for (const path of paths) {\n    const key = baseKey(path);\n    const list = byBase.get(key);\n    if (list) list.push(path);\n    else byBase.set(key, [path]);\n  }\n  return byBase;\n}\n\ninterface LinkRow {\n  src: string;\n  target: string;\n  dst: string | null;\n  embed: number;\n}\n\n// resolveTarget applied to `rows`, returning only the rows whose dst changed, so the caller\n// can write them in one runBatch call.\nfunction resolveRows(rows: LinkRow[], pathSet: Set<string>, byBase: Map<string, string[]>): LinkRow[] {\n  const changed: LinkRow[] = [];\n  for (const row of rows) {\n    const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n    if (dst !== row.dst) changed.push({ ...row, dst });\n  }\n  return changed;\n}\n\nasync function writeResolved(db: Connection, rows: LinkRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await db.runBatch(\n    'UPDATE links SET dst = ? WHERE src = ? AND target = ?',\n    rows.map((r) => [r.dst, r.src, r.target])\n  );\n}\n\n// A new or deleted file can change any note's resolution, so re-resolve the whole table.\n// Fallback for cold builds and large deltas -- see afterReconcile's threshold.\nasync function resolveAll(db: Connection, files: FileStat[]): Promise<boolean> {\n  const pathSet = new Set(files.map((f) => f.relPath));\n  const byBase = buildByBase(files);\n  const stmt = await db.prepare('SELECT src, target, dst, embed FROM links');\n  const rows = (await stmt.all()) as unknown as LinkRow[];\n  const changed = resolveRows(rows, pathSet, byBase);\n  await writeResolved(db, changed);\n  return changed.length > 0;\n}\n\n// Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable limit\n// (SQLITE_MAX_VARIABLE_NUMBER, 32766); harmless headroom on DuckDB, which has no such cap.\nasync function selectIn(db: Connection, column: string, keys: string[]): Promise<LinkRow[]> {\n  const rows: LinkRow[] = [];\n  for (let i = 0; i < keys.length; i += 500) {\n    const chunk = keys.slice(i, i + 500);\n    const stmt = await db.prepare(`SELECT src, target, dst, embed FROM links WHERE ${column} IN (${chunk.map(() => '?').join(', ')})`);\n    rows.push(...((await stmt.all(...chunk)) as unknown as LinkRow[]));\n  }\n  return rows;\n}\n\n// Re-resolves only what this reconcile could have touched: re-stored sources, links whose\n// target basename matches an added/vanished file, and links whose dst vanished -- all indexed.\nasync function resolveIncremental(db: Connection, delta: ReconcileDelta): Promise<boolean> {\n  const pathSet = new Set(delta.files.map((f) => f.relPath));\n  const byBase = buildByBase(delta.files);\n\n  const changedBasenames = new Set<string>();\n  for (const p of delta.added) changedBasenames.add(baseKey(p));\n  for (const p of delta.vanished) changedBasenames.add(baseKey(p));\n\n  // \\0 can't appear in a path or target, so the key never collides; a space can.\n  const candidates = new Map<string, LinkRow>();\n  const collect = (rows: LinkRow[]) => {\n    for (const row of rows) candidates.set(`${row.src}\\0${row.target}\\0${row.embed}`, row);\n  };\n  collect(await selectIn(db, 'src', delta.reparsed));\n  collect(await selectIn(db, 'target_base', [...changedBasenames]));\n  collect(await selectIn(db, 'dst', delta.vanished));\n\n  const changed = resolveRows([...candidates.values()], pathSet, byBase);\n  await writeResolved(db, changed);\n  return changed.length > 0;\n}\n\n// One edge per (src, target); an embed row is excluded only when the same pair also exists as\n// a link. Self-edges are dropped so PageRank mass is not self-recycled.\nexport const LINK_EDGES_SQL = `SELECT src, dst FROM links WHERE dst IS NOT NULL AND embed = 0 AND src != dst\n    UNION ALL\n    SELECT src, dst FROM links l WHERE dst IS NOT NULL AND embed = 1 AND src != dst\n      AND NOT EXISTS (SELECT 1 FROM links l0 WHERE l0.src = l.src AND l0.target = l.target AND l0.embed = 0)`;\n\nexport function toEdges(rows: Array<{ src: string; dst: string }>): [string, string][] {\n  return rows.map((r) => [r.src, r.dst]);\n}\n\n// Resolved edges, for rank (inside reconcile's afterReconcile pass).\nexport async function linkEdges(db: Connection): Promise<[string, string][]> {\n  const stmt = await db.prepare(LINK_EDGES_SQL);\n  return toEdges((await stmt.all()) as Array<{ src: string; dst: string }>);\n}\n\n// Records whether remove()/store() deleted any link row, since afterReconcile runs after both\n// and dstChanged alone misses the reparsed-to-zero-links case. Keyed on delta so state dies with it.\nconst removedWithLinks = new WeakMap<ReconcileDelta, boolean>();\n\nexport const links: Feature = {\n  name: 'links',\n  async schema(db) {\n    await db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, embed INTEGER, PRIMARY KEY (src, target, embed))');\n    await db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n    await db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');\n  },\n  extract,\n  // Reparsed docs are diffed in store() instead of wiped here: delete-and-reinsert resets dst\n  // to NULL, making every touch-only reparse look like an edge change and recompute PageRank.\n  async remove(db, paths, delta) {\n    const vanishedSet = new Set(delta.vanished);\n    const vanished = paths.filter((p) => vanishedSet.has(p));\n    if (vanished.length === 0) return;\n    const existingRows = await selectIn(db, 'src', vanished);\n    await db.runBatch(\n      'DELETE FROM links WHERE src = ?',\n      vanished.map((p) => [p])\n    );\n    if (existingRows.length > 0) removedWithLinks.set(delta, true);\n  },\n  async store(db, docs: ExtractedDoc[], delta) {\n    const byPath = new Map(docs.map((d) => [d.path, d.extracted as Array<{ target: string; embed: boolean }>]));\n    const addedSet = new Set(delta.added);\n    const reparsedExisting = docs.map((d) => d.path).filter((p) => !addedSet.has(p));\n\n    // Stale rows (target/embed pairs missing from the new parse) are diffed in JS from one\n    // bulk SELECT, so this pass costs one query plus one batched DELETE regardless of file count.\n    const existingRows = await selectIn(db, 'src', reparsedExisting);\n    const staleRows = existingRows.filter((row) => {\n      const targets = byPath.get(row.src) ?? [];\n      return !targets.some((t) => t.target === row.target && (t.embed ? 1 : 0) === row.embed);\n    });\n    if (staleRows.length > 0) {\n      await db.runBatch(\n        'DELETE FROM links WHERE src = ? AND target = ? AND embed = ?',\n        staleRows.map((r) => [r.src, r.target, r.embed])\n      );\n      removedWithLinks.set(delta, true);\n    }\n\n    // Upsert preserves dst on surviving rows; only new rows start at NULL. A same-note anchor\n    // gets no target_base, so SQL's `IN` excludes it from resolveIncremental's lookup.\n    const upsertRows: unknown[][] = [];\n    for (const [path, targets] of byPath) {\n      for (const { target, embed } of targets) upsertRows.push([path, target, target.startsWith('#') ? null : baseKey(cleanTarget(target)), embed ? 1 : 0]);\n    }\n    if (upsertRows.length > 0) {\n      await db.runBatch('INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base', upsertRows);\n    }\n  },\n  async afterReconcile(db, delta) {\n    // Any deleted rows mean the edge set shrank -- whether the file vanished or was reparsed\n    // down to fewer/no links.\n    const removeHadLinks = removedWithLinks.get(delta) ?? false;\n\n    // A cold build is the only case a full pass wins. Measured 2026-09-01 on 6,566 notes, resolve\n    // time alone: full is flat near 90ms at any churn, incremental 7ms at 10 files and 30ms at 2,000.\n    const large = delta.files.length === 0 || delta.added.length === delta.files.length;\n\n    const dstChanged = large ? await resolveAll(db, delta.files) : await resolveIncremental(db, delta);\n    delta.linksChanged = dstChanged || removeHadLinks;\n  },\n};\n"],"names":["posix","maskRegions","parseWikilinkInner","inner","beforeAlias","split","trim","startsWith","length","target","extract","_raw","rawBody","_search","data","body","test","seen","Set","results","add","embed","key","has","push","m","matchAll","index","dest","includes","value","Object","values","item","Array","isArray","exec","cleanTarget","replace","baseKey","path","basename","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","candidates","get","buildByBase","files","Map","paths","map","f","relPath","sort","a","b","list","set","resolveRows","rows","changed","row","dst","writeResolved","db","runBatch","r","resolveAll","stmt","prepare","all","selectIn","column","keys","i","chunk","slice","resolveIncremental","delta","changedBasenames","p","added","vanished","collect","reparsed","LINK_EDGES_SQL","toEdges","linkEdges","removedWithLinks","WeakMap","links","name","schema","remove","vanishedSet","filter","existingRows","store","docs","byPath","d","extracted","addedSet","reparsedExisting","staleRows","targets","some","t","upsertRows","afterReconcile","removeHadLinks","large","dstChanged","linksChanged"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAGpC,SAASC,WAAW,QAAQ,cAAc;AAG1C,0JAA0J;AAC1J,mJAAmJ;AAEnJ,wFAAwF;AACxF,gEAAgE;AAChE,SAASC,mBAAmBC,KAAa;IACvC,MAAMC,cAAcD,MAAME,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC5C,IAAIF,YAAYG,UAAU,CAAC,MAAM,OAAOH,YAAYI,MAAM,GAAG,IAAIJ,cAAc;IAC/E,MAAMK,SAASL,YAAYC,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC7C,OAAOG,UAAU;AACnB;AAEA,4FAA4F;AAC5F,0EAA0E;AAC1E,SAASC,QAAQC,IAAY,EAAEC,OAAe,EAAEC,OAA4C,EAAEC,IAA8B;IAC1H,MAAMC,OAAO,YAAYC,IAAI,CAACJ,WAAWX,YAAYW,WAAWA;IAChE,MAAMK,OAAO,IAAIC;IACjB,MAAMC,UAAqD,EAAE;IAC7D,MAAMC,MAAM,CAACX,QAAgBY;QAC3B,MAAMC,MAAM,GAAGb,OAAO,EAAE,EAAEY,QAAQ,MAAM,KAAK;QAC7C,IAAIJ,KAAKM,GAAG,CAACD,MAAM;QACnBL,KAAKG,GAAG,CAACE;QACTH,QAAQK,IAAI,CAAC;YAAEf;YAAQY;QAAM;IAC/B;IACA,uFAAuF;IACvF,6DAA6D;IAC7D,KAAK,MAAMI,KAAKV,KAAKW,QAAQ,CAAC,kBAAmB;YAEjBD;QAD9B,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;QACtC,IAAIhB,QAAQW,IAAIX,QAAQM,IAAI,CAAC,EAACU,WAAAA,EAAEE,KAAK,cAAPF,sBAAAA,WAAW,KAAK,EAAE,KAAK;IACvD;IACA,2FAA2F;IAC3F,8FAA8F;IAC9F,KAAK,MAAMA,KAAKV,KAAKW,QAAQ,CAAC,6EAA8E;QAC1G,MAAME,OAAOH,CAAC,CAAC,EAAE,CAACnB,IAAI;QACtB,qFAAqF;QACrF,oFAAoF;QACpF,IAAI,CAACsB,QAAQA,KAAKC,QAAQ,CAAC,QAAQ;QACnC,IAAI,oCAAoCb,IAAI,CAACY,OAAO;QACpD,MAAMnB,SAASmB,KAAKrB,UAAU,CAAC,OAAOqB,OAAOA,KAAKvB,KAAK,CAAC,IAAI,CAAC,EAAE;QAC/D,IAAII,QAAQW,IAAIX,QAAQgB,CAAC,CAAC,EAAE,KAAK;IACnC;IACA,KAAK,MAAMK,SAASC,OAAOC,MAAM,CAAClB,iBAAAA,kBAAAA,OAAQ,CAAC,GAAI;QAC7C,KAAK,MAAMmB,QAAQC,MAAMC,OAAO,CAACL,SAASA,QAAQ;YAACA;SAAM,CAAE;YACzD,IAAI,OAAOG,SAAS,UAAU;YAC9B,MAAMR,IAAI,iBAAiBW,IAAI,CAACH,KAAK3B,IAAI;YACzC,IAAI,CAACmB,GAAG;YACR,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;YACtC,IAAIhB,QAAQW,IAAIX,QAAQ;QAC1B;IACF;IACA,OAAOU;AACT;AAEA,SAASkB,YAAY5B,MAAc;IACjC,OAAOA,OAAO6B,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;AACrD;AAEA,SAASC,QAAQC,IAAY;IAC3B,OAAOxC,MAAMyC,QAAQ,CAACD,MAAMF,OAAO,CAAC,UAAU,IAAII,WAAW;AAC/D;AAEA,8FAA8F;AAC9F,kGAAkG;AAClG,SAASC,cAAcC,GAAW,EAAEnC,MAAc,EAAEoC,OAAoB,EAAEC,MAA6B;IACrG,4FAA4F;IAC5F,IAAIrC,OAAOF,UAAU,CAAC,MAAM,OAAOqC;IACnC,MAAMG,QAAQV,YAAY5B;IAC1B,MAAMuC,UAAUhD,MAAMiD,SAAS,CAACjD,MAAMkD,IAAI,CAAClD,MAAMmD,OAAO,CAACP,MAAMG;IAC/D,KAAK,MAAMK,aAAa;QAACL;QAAO,GAAGA,MAAM,GAAG,CAAC;QAAEC;QAAS,GAAGA,QAAQ,GAAG,CAAC;KAAC,CAAE;QACxE,IAAIH,QAAQtB,GAAG,CAAC6B,YAAY,OAAOA;IACrC;IACA,MAAMC,aAAaP,OAAOQ,GAAG,CAACf,QAAQQ;IACtC,IAAI,CAACM,YAAY,OAAO;IACxB,0FAA0F;IAC1F,+DAA+D;IAC/D,OAAOA,WAAWxB,QAAQ,CAACe,OAAOA,MAAMS,UAAU,CAAC,EAAE;AACvD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,SAASE,YAAYC,KAAiB;IACpC,MAAMV,SAAS,IAAIW;IACnB,MAAMC,QAAQF,MAAMG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEvD,MAAM,GAAGwD,EAAExD,MAAM,IAAKuD,CAAAA,IAAIC,IAAI,CAAC,IAAID,IAAIC,IAAI,IAAI,CAAA;IAC1G,KAAK,MAAMxB,QAAQkB,MAAO;QACxB,MAAMpC,MAAMiB,QAAQC;QACpB,MAAMyB,OAAOnB,OAAOQ,GAAG,CAAChC;QACxB,IAAI2C,MAAMA,KAAKzC,IAAI,CAACgB;aACfM,OAAOoB,GAAG,CAAC5C,KAAK;YAACkB;SAAK;IAC7B;IACA,OAAOM;AACT;AASA,4FAA4F;AAC5F,uCAAuC;AACvC,SAASqB,YAAYC,IAAe,EAAEvB,OAAoB,EAAEC,MAA6B;IACvF,MAAMuB,UAAqB,EAAE;IAC7B,KAAK,MAAMC,OAAOF,KAAM;QACtB,MAAMG,MAAM5B,cAAc2B,IAAI1B,GAAG,EAAE0B,IAAI7D,MAAM,EAAEoC,SAASC;QACxD,IAAIyB,QAAQD,IAAIC,GAAG,EAAEF,QAAQ7C,IAAI,CAAC;YAAE,GAAG8C,GAAG;YAAEC;QAAI;IAClD;IACA,OAAOF;AACT;AAEA,eAAeG,cAAcC,EAAc,EAAEL,IAAe;IAC1D,IAAIA,KAAK5D,MAAM,KAAK,GAAG;IACvB,MAAMiE,GAAGC,QAAQ,CACf,yDACAN,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAEJ,GAAG;YAAEI,EAAE/B,GAAG;YAAE+B,EAAElE,MAAM;SAAC;AAE5C;AAEA,yFAAyF;AACzF,+EAA+E;AAC/E,eAAemE,WAAWH,EAAc,EAAEjB,KAAiB;IACzD,MAAMX,UAAU,IAAI3B,IAAIsC,MAAMG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAClD,MAAMf,SAASS,YAAYC;IAC3B,MAAMqB,OAAO,MAAMJ,GAAGK,OAAO,CAAC;IAC9B,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAC5B,MAAMV,UAAUF,YAAYC,MAAMvB,SAASC;IAC3C,MAAM0B,cAAcC,IAAIJ;IACxB,OAAOA,QAAQ7D,MAAM,GAAG;AAC1B;AAEA,2FAA2F;AAC3F,2FAA2F;AAC3F,eAAewE,SAASP,EAAc,EAAEQ,MAAc,EAAEC,IAAc;IACpE,MAAMd,OAAkB,EAAE;IAC1B,IAAK,IAAIe,IAAI,GAAGA,IAAID,KAAK1E,MAAM,EAAE2E,KAAK,IAAK;QACzC,MAAMC,QAAQF,KAAKG,KAAK,CAACF,GAAGA,IAAI;QAChC,MAAMN,OAAO,MAAMJ,GAAGK,OAAO,CAAC,CAAC,gDAAgD,EAAEG,OAAO,KAAK,EAAEG,MAAMzB,GAAG,CAAC,IAAM,KAAKT,IAAI,CAAC,MAAM,CAAC,CAAC;QACjIkB,KAAK5C,IAAI,IAAM,MAAMqD,KAAKE,GAAG,IAAIK;IACnC;IACA,OAAOhB;AACT;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,eAAekB,mBAAmBb,EAAc,EAAEc,KAAqB;IACrE,MAAM1C,UAAU,IAAI3B,IAAIqE,MAAM/B,KAAK,CAACG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACxD,MAAMf,SAASS,YAAYgC,MAAM/B,KAAK;IAEtC,MAAMgC,mBAAmB,IAAItE;IAC7B,KAAK,MAAMuE,KAAKF,MAAMG,KAAK,CAAEF,iBAAiBpE,GAAG,CAACmB,QAAQkD;IAC1D,KAAK,MAAMA,KAAKF,MAAMI,QAAQ,CAAEH,iBAAiBpE,GAAG,CAACmB,QAAQkD;IAE7D,+EAA+E;IAC/E,MAAMpC,aAAa,IAAII;IACvB,MAAMmC,UAAU,CAACxB;QACf,KAAK,MAAME,OAAOF,KAAMf,WAAWa,GAAG,CAAC,GAAGI,IAAI1B,GAAG,CAAC,EAAE,EAAE0B,IAAI7D,MAAM,CAAC,EAAE,EAAE6D,IAAIjD,KAAK,EAAE,EAAEiD;IACpF;IACAsB,QAAQ,MAAMZ,SAASP,IAAI,OAAOc,MAAMM,QAAQ;IAChDD,QAAQ,MAAMZ,SAASP,IAAI,eAAe;WAAIe;KAAiB;IAC/DI,QAAQ,MAAMZ,SAASP,IAAI,OAAOc,MAAMI,QAAQ;IAEhD,MAAMtB,UAAUF,YAAY;WAAId,WAAWrB,MAAM;KAAG,EAAEa,SAASC;IAC/D,MAAM0B,cAAcC,IAAIJ;IACxB,OAAOA,QAAQ7D,MAAM,GAAG;AAC1B;AAEA,8FAA8F;AAC9F,wEAAwE;AACxE,OAAO,MAAMsF,iBAAiB,CAAC;;;4GAG6E,CAAC,CAAC;AAE9G,OAAO,SAASC,QAAQ3B,IAAyC;IAC/D,OAAOA,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAE/B,GAAG;YAAE+B,EAAEJ,GAAG;SAAC;AACvC;AAEA,qEAAqE;AACrE,OAAO,eAAeyB,UAAUvB,EAAc;IAC5C,MAAMI,OAAO,MAAMJ,GAAGK,OAAO,CAACgB;IAC9B,OAAOC,QAAS,MAAMlB,KAAKE,GAAG;AAChC;AAEA,8FAA8F;AAC9F,qGAAqG;AACrG,MAAMkB,mBAAmB,IAAIC;AAE7B,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAO5B,EAAE;QACb,MAAMA,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;IAChB;IACA1B;IACA,4FAA4F;IAC5F,4FAA4F;IAC5F,MAAM4F,QAAO7B,EAAE,EAAEf,KAAK,EAAE6B,KAAK;QAC3B,MAAMgB,cAAc,IAAIrF,IAAIqE,MAAMI,QAAQ;QAC1C,MAAMA,WAAWjC,MAAM8C,MAAM,CAAC,CAACf,IAAMc,YAAYhF,GAAG,CAACkE;QACrD,IAAIE,SAASnF,MAAM,KAAK,GAAG;QAC3B,MAAMiG,eAAe,MAAMzB,SAASP,IAAI,OAAOkB;QAC/C,MAAMlB,GAAGC,QAAQ,CACf,mCACAiB,SAAShC,GAAG,CAAC,CAAC8B,IAAM;gBAACA;aAAE;QAEzB,IAAIgB,aAAajG,MAAM,GAAG,GAAGyF,iBAAiB/B,GAAG,CAACqB,OAAO;IAC3D;IACA,MAAMmB,OAAMjC,EAAE,EAAEkC,IAAoB,EAAEpB,KAAK;QACzC,MAAMqB,SAAS,IAAInD,IAAIkD,KAAKhD,GAAG,CAAC,CAACkD,IAAM;gBAACA,EAAErE,IAAI;gBAAEqE,EAAEC,SAAS;aAA8C;QACzG,MAAMC,WAAW,IAAI7F,IAAIqE,MAAMG,KAAK;QACpC,MAAMsB,mBAAmBL,KAAKhD,GAAG,CAAC,CAACkD,IAAMA,EAAErE,IAAI,EAAEgE,MAAM,CAAC,CAACf,IAAM,CAACsB,SAASxF,GAAG,CAACkE;QAE7E,uFAAuF;QACvF,8FAA8F;QAC9F,MAAMgB,eAAe,MAAMzB,SAASP,IAAI,OAAOuC;QAC/C,MAAMC,YAAYR,aAAaD,MAAM,CAAC,CAAClC;gBACrBsC;YAAhB,MAAMM,WAAUN,cAAAA,OAAOtD,GAAG,CAACgB,IAAI1B,GAAG,eAAlBgE,yBAAAA,cAAuB,EAAE;YACzC,OAAO,CAACM,QAAQC,IAAI,CAAC,CAACC,IAAMA,EAAE3G,MAAM,KAAK6D,IAAI7D,MAAM,IAAI,AAAC2G,CAAAA,EAAE/F,KAAK,GAAG,IAAI,CAAA,MAAOiD,IAAIjD,KAAK;QACxF;QACA,IAAI4F,UAAUzG,MAAM,GAAG,GAAG;YACxB,MAAMiE,GAAGC,QAAQ,CACf,gEACAuC,UAAUtD,GAAG,CAAC,CAACgB,IAAM;oBAACA,EAAE/B,GAAG;oBAAE+B,EAAElE,MAAM;oBAAEkE,EAAEtD,KAAK;iBAAC;YAEjD4E,iBAAiB/B,GAAG,CAACqB,OAAO;QAC9B;QAEA,0FAA0F;QAC1F,mFAAmF;QACnF,MAAM8B,aAA0B,EAAE;QAClC,KAAK,MAAM,CAAC7E,MAAM0E,QAAQ,IAAIN,OAAQ;YACpC,KAAK,MAAM,EAAEnG,MAAM,EAAEY,KAAK,EAAE,IAAI6F,QAASG,WAAW7F,IAAI,CAAC;gBAACgB;gBAAM/B;gBAAQA,OAAOF,UAAU,CAAC,OAAO,OAAOgC,QAAQF,YAAY5B;gBAAUY,QAAQ,IAAI;aAAE;QACtJ;QACA,IAAIgG,WAAW7G,MAAM,GAAG,GAAG;YACzB,MAAMiE,GAAGC,QAAQ,CAAC,uKAAuK2C;QAC3L;IACF;IACA,MAAMC,gBAAe7C,EAAE,EAAEc,KAAK;YAGLU;QAFvB,yFAAyF;QACzF,0BAA0B;QAC1B,MAAMsB,kBAAiBtB,wBAAAA,iBAAiB3C,GAAG,CAACiC,oBAArBU,mCAAAA,wBAA+B;QAEtD,8FAA8F;QAC9F,kGAAkG;QAClG,MAAMuB,QAAQjC,MAAM/B,KAAK,CAAChD,MAAM,KAAK,KAAK+E,MAAMG,KAAK,CAAClF,MAAM,KAAK+E,MAAM/B,KAAK,CAAChD,MAAM;QAEnF,MAAMiH,aAAaD,QAAQ,MAAM5C,WAAWH,IAAIc,MAAM/B,KAAK,IAAI,MAAM8B,mBAAmBb,IAAIc;QAC5FA,MAAMmC,YAAY,GAAGD,cAAcF;IACrC;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/links.ts"],"sourcesContent":["import posix from 'node:path/posix';\nimport type { FileStat } from '../scan/index.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Connection } from '../store/types.ts';\nimport { maskRegions } from './fences.ts';\nimport type { ExtractedDoc, Feature, ReconcileDelta } from './types.ts';\n\n// links(src, target, target_base, dst, embed): target as written, target_base its indexed baseKey, dst the resolved path or NULL (a queryable dead link).\n// embed distinguishes `![[x]]`/`![](x.md)` from `[[x]]`/`[](x.md)` -- Obsidian's grain: a target both linked and embedded in one note is two rows.\n\n// Alias stripped after |; a leading # keeps the anchor as the target (self-link needs a\n// heading name), else the anchor splits off. Null = not a link.\nfunction parseWikilinkInner(inner: string): string | null {\n  const beforeAlias = inner.split('|')[0].trim();\n  if (beforeAlias.startsWith('#')) return beforeAlias.length > 1 ? beforeAlias : null;\n  const target = beforeAlias.split('#')[0].trim();\n  return target || null;\n}\n\n// A frontmatter value counts only when it is exactly a wikilink (\"[[X]]\") -- mid-string and\n// ![[...]] forms don't count there, per Obsidian's rule (probe-verified).\nfunction extract(_raw: string, rawBody: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): Array<{ target: string; embed: boolean }> {\n  const body = /\\[\\[|\\]\\(/.test(rawBody) ? maskRegions(rawBody) : rawBody;\n  const seen = new Set<string>();\n  const results: Array<{ target: string; embed: boolean }> = [];\n  const add = (target: string, embed: boolean) => {\n    const key = `${target}\\0${embed ? '1' : '0'}`;\n    if (seen.has(key)) return;\n    seen.add(key);\n    results.push({ target, embed });\n  };\n  // To the first ]], as Obsidian parses it, so a heading or alias holding a lone ] still\n  // matches; anchor and alias split off in parseWikilinkInner.\n  for (const m of body.matchAll(/\\[\\[(.*?)\\]\\]/g)) {\n    const target = parseWikilinkInner(m[1]);\n    if (target) add(target, body[(m.index ?? 0) - 1] === '!');\n  }\n  // Any internal destination, not only .md-suffixed: \\](Zektor) resolves like [[Zektor]] and\n  // \\](#anchor) is a self-edge. External URLs and titled links (dest stops at whitespace) skip.\n  for (const m of body.matchAll(/(!)?\\[[^\\]]*\\]\\(((?:[^()\\s]|\\([^()\\s]*\\))+)(?:\\s+(?:\"[^)]*\"|'[^)]*'))?\\)/g)) {\n    const dest = m[2].trim();\n    // External: a URI scheme prefix (mailto:, tel:, data:, https:...), protocol-relative\n    // //host, a www. shorthand, or a scheme anywhere (the malformed double-paren case).\n    if (!dest || dest.includes('://')) continue;\n    if (/^([a-z][a-z0-9+.-]*:|\\/\\/|www\\.)/i.test(dest)) continue;\n    const target = dest.startsWith('#') ? dest : dest.split('#')[0];\n    if (target) add(target, m[1] === '!');\n  }\n  for (const value of Object.values(data ?? {})) {\n    for (const item of Array.isArray(value) ? value : [value]) {\n      if (typeof item !== 'string') continue;\n      const m = /^\\[\\[(.*)\\]\\]$/.exec(item.trim());\n      if (!m) continue;\n      const target = parseWikilinkInner(m[1]);\n      if (target) add(target, false);\n    }\n  }\n  return results;\n}\n\nfunction cleanTarget(target: string): string {\n  return target.replace(/\\\\/g, '/').replace(/^\\.\\//, '');\n}\n\nfunction baseKey(path: string): string {\n  return posix.basename(path).replace(/\\.md$/i, '').toLowerCase();\n}\n\n// Obsidian-style: exact relative path (with/without .md), path relative to the linking note's\n// directory, then basename match (shortest path wins ties, verified against a cold-loaded vault).\nfunction resolveTarget(src: string, target: string, pathSet: Set<string>, byBase: Map<string, string[]>): string | null {\n  // [[#Heading]]: a same-note anchor, always the note itself -- never depends on other files.\n  if (target.startsWith('#')) return src;\n  const clean = cleanTarget(target);\n  const fromSrc = posix.normalize(posix.join(posix.dirname(src), clean));\n  for (const candidate of [clean, `${clean}.md`, fromSrc, `${fromSrc}.md`]) {\n    if (pathSet.has(candidate)) return candidate;\n  }\n  const candidates = byBase.get(baseKey(clean));\n  if (!candidates) return null;\n  // The linking note itself wins a basename collision (Obsidian resolves to self before the\n  // shortest-path rule -- cold-load verified on the hub corpus).\n  return candidates.includes(src) ? src : candidates[0];\n}\n\n// Shortest path wins a basename collision; equal lengths fall back to lexicographic, our own\n// deterministic tiebreak for a case Obsidian itself leaves registration-order-dependent.\nfunction buildByBase(files: FileStat[]): Map<string, string[]> {\n  const byBase = new Map<string, string[]>();\n  const paths = files.map((f) => f.relPath).sort((a, b) => a.length - b.length || (a < b ? -1 : a > b ? 1 : 0));\n  for (const path of paths) {\n    const key = baseKey(path);\n    const list = byBase.get(key);\n    if (list) list.push(path);\n    else byBase.set(key, [path]);\n  }\n  return byBase;\n}\n\ninterface LinkRow {\n  src: string;\n  target: string;\n  dst: string | null;\n  embed: number;\n}\n\n// resolveTarget applied to `rows`, returning only the rows whose dst changed, so the caller\n// can write them in one runBatch call.\nfunction resolveRows(rows: LinkRow[], pathSet: Set<string>, byBase: Map<string, string[]>): LinkRow[] {\n  const changed: LinkRow[] = [];\n  for (const row of rows) {\n    const dst = resolveTarget(row.src, row.target, pathSet, byBase);\n    if (dst !== row.dst) changed.push({ ...row, dst });\n  }\n  return changed;\n}\n\nasync function writeResolved(db: Connection, rows: LinkRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  await db.runBatch(\n    'UPDATE links SET dst = ? WHERE src = ? AND target = ?',\n    rows.map((r) => [r.dst, r.src, r.target])\n  );\n}\n\n// A new or deleted file can change any note's resolution, so re-resolve the whole table.\n// Fallback for cold builds and large deltas -- see afterReconcile's threshold.\nasync function resolveAll(db: Connection, files: FileStat[]): Promise<boolean> {\n  const pathSet = new Set(files.map((f) => f.relPath));\n  const byBase = buildByBase(files);\n  const stmt = await db.prepare('SELECT src, target, dst, embed FROM links');\n  const rows = (await stmt.all()) as unknown as LinkRow[];\n  const changed = resolveRows(rows, pathSet, byBase);\n  await writeResolved(db, changed);\n  return changed.length > 0;\n}\n\n// Chunked so a large-but-under-threshold delta never exceeds SQLite's bound-variable limit\n// (SQLITE_MAX_VARIABLE_NUMBER, 32766); harmless headroom on DuckDB, which has no such cap.\nasync function selectIn(db: Connection, column: string, keys: string[]): Promise<LinkRow[]> {\n  const rows: LinkRow[] = [];\n  for (let i = 0; i < keys.length; i += 500) {\n    const chunk = keys.slice(i, i + 500);\n    const stmt = await db.prepare(`SELECT src, target, dst, embed FROM links WHERE ${column} IN (${chunk.map(() => '?').join(', ')})`);\n    rows.push(...((await stmt.all(...chunk)) as unknown as LinkRow[]));\n  }\n  return rows;\n}\n\n// Re-resolves only what this reconcile could have touched: re-stored sources, links whose\n// target basename matches an added/vanished file, and links whose dst vanished -- all indexed.\nasync function resolveIncremental(db: Connection, delta: ReconcileDelta): Promise<boolean> {\n  const pathSet = new Set(delta.files.map((f) => f.relPath));\n  const byBase = buildByBase(delta.files);\n\n  const changedBasenames = new Set<string>();\n  for (const p of delta.added) changedBasenames.add(baseKey(p));\n  for (const p of delta.vanished) changedBasenames.add(baseKey(p));\n\n  // \\0 can't appear in a path or target, so the key never collides; a space can.\n  const candidates = new Map<string, LinkRow>();\n  const collect = (rows: LinkRow[]) => {\n    for (const row of rows) candidates.set(`${row.src}\\0${row.target}\\0${row.embed}`, row);\n  };\n  collect(await selectIn(db, 'src', delta.reparsed));\n  collect(await selectIn(db, 'target_base', [...changedBasenames]));\n  collect(await selectIn(db, 'dst', delta.vanished));\n\n  const changed = resolveRows([...candidates.values()], pathSet, byBase);\n  await writeResolved(db, changed);\n  return changed.length > 0;\n}\n\n// One edge per (src, target); an embed row is excluded only when the same pair also exists as\n// a link. Self-edges are dropped so PageRank mass is not self-recycled.\nexport const LINK_EDGES_SQL = `SELECT src, dst FROM links WHERE dst IS NOT NULL AND embed = 0 AND src != dst\n    UNION ALL\n    SELECT src, dst FROM links l WHERE dst IS NOT NULL AND embed = 1 AND src != dst\n      AND NOT EXISTS (SELECT 1 FROM links l0 WHERE l0.src = l.src AND l0.target = l.target AND l0.embed = 0)`;\n\nexport function toEdges(rows: Array<{ src: string; dst: string }>): [string, string][] {\n  return rows.map((r) => [r.src, r.dst]);\n}\n\n// Resolved edges, for rank (inside reconcile's afterReconcile pass).\nexport async function linkEdges(db: Connection): Promise<[string, string][]> {\n  const stmt = await db.prepare(LINK_EDGES_SQL);\n  return toEdges((await stmt.all()) as Array<{ src: string; dst: string }>);\n}\n\n// Records whether remove()/store() deleted any link row, since afterReconcile runs after both\n// and dstChanged alone misses the reparsed-to-zero-links case. Keyed on delta so state dies with it.\nconst removedWithLinks = new WeakMap<ReconcileDelta, boolean>();\n\n// dst is written by resolution, not by store(), so it is absent from the column list and takes the\n// table's default on the append path.\nconst LINK_COLUMNS = ['src', 'target', 'target_base', 'embed'];\nconst UPSERT_LINK_SQL = 'INSERT INTO links (src, target, target_base, dst, embed) VALUES (?, ?, ?, NULL, ?) ON CONFLICT(src, target, embed) DO UPDATE SET target_base = excluded.target_base';\n\nexport const links: Feature = {\n  name: 'links',\n  async schema(db) {\n    await db.exec('CREATE TABLE IF NOT EXISTS links (src TEXT, target TEXT, target_base TEXT, dst TEXT, embed INTEGER, PRIMARY KEY (src, target, embed))');\n    await db.exec('CREATE INDEX IF NOT EXISTS links_dst ON links(dst)');\n    await db.exec('CREATE INDEX IF NOT EXISTS links_target_base ON links(target_base)');\n  },\n  extract,\n  // Reparsed docs are diffed in store() instead of wiped here: delete-and-reinsert resets dst\n  // to NULL, making every touch-only reparse look like an edge change and recompute PageRank.\n  async remove(db, paths, delta) {\n    const vanishedSet = new Set(delta.vanished);\n    const vanished = paths.filter((p) => vanishedSet.has(p));\n    if (vanished.length === 0) return;\n    const existingRows = await selectIn(db, 'src', vanished);\n    await db.runBatch(\n      'DELETE FROM links WHERE src = ?',\n      vanished.map((p) => [p])\n    );\n    if (existingRows.length > 0) removedWithLinks.set(delta, true);\n  },\n  async store(db, docs: ExtractedDoc[], delta) {\n    const byPath = new Map(docs.map((d) => [d.path, d.extracted as Array<{ target: string; embed: boolean }>]));\n    const addedSet = new Set(delta.added);\n    const reparsedExisting = docs.map((d) => d.path).filter((p) => !addedSet.has(p));\n\n    // Stale rows (target/embed pairs missing from the new parse) are diffed in JS from one\n    // bulk SELECT, so this pass costs one query plus one batched DELETE regardless of file count.\n    const existingRows = await selectIn(db, 'src', reparsedExisting);\n    const staleRows = existingRows.filter((row) => {\n      const targets = byPath.get(row.src) ?? [];\n      return !targets.some((t) => t.target === row.target && (t.embed ? 1 : 0) === row.embed);\n    });\n    if (staleRows.length > 0) {\n      await db.runBatch(\n        'DELETE FROM links WHERE src = ? AND target = ? AND embed = ?',\n        staleRows.map((r) => [r.src, r.target, r.embed])\n      );\n      removedWithLinks.set(delta, true);\n    }\n\n    // Upsert preserves dst on surviving rows; only new rows start at NULL. A same-note anchor\n    // gets no target_base, so SQL's `IN` excludes it from resolveIncremental's lookup.\n    // Split, unlike the other features: this hook has no remove() pass behind it (see above), so\n    // only a path in `added` is guaranteed to have no row yet. The rest genuinely conflict and the\n    // upsert is what preserves their `dst`.\n    const appendable: unknown[][] = [];\n    const upsertRows: unknown[][] = [];\n    for (const [path, targets] of byPath) {\n      for (const { target, embed } of targets) {\n        const row = [path, target, target.startsWith('#') ? null : baseKey(cleanTarget(target)), embed ? 1 : 0];\n        if (addedSet.has(path)) appendable.push(row);\n        else upsertRows.push(row);\n      }\n    }\n    await appendRows(db, 'links', LINK_COLUMNS, UPSERT_LINK_SQL, appendable);\n    if (upsertRows.length > 0) await db.runBatch(UPSERT_LINK_SQL, upsertRows);\n  },\n  async afterReconcile(db, delta) {\n    // Any deleted rows mean the edge set shrank -- whether the file vanished or was reparsed\n    // down to fewer/no links.\n    const removeHadLinks = removedWithLinks.get(delta) ?? false;\n\n    // A cold build is the only case a full pass wins. Measured 2026-09-01 on 6,566 notes, resolve\n    // time alone: full is flat near 90ms at any churn, incremental 7ms at 10 files and 30ms at 2,000.\n    const large = delta.files.length === 0 || delta.added.length === delta.files.length;\n\n    const dstChanged = large ? await resolveAll(db, delta.files) : await resolveIncremental(db, delta);\n    delta.linksChanged = dstChanged || removeHadLinks;\n  },\n};\n"],"names":["posix","appendRows","maskRegions","parseWikilinkInner","inner","beforeAlias","split","trim","startsWith","length","target","extract","_raw","rawBody","_search","data","body","test","seen","Set","results","add","embed","key","has","push","m","matchAll","index","dest","includes","value","Object","values","item","Array","isArray","exec","cleanTarget","replace","baseKey","path","basename","toLowerCase","resolveTarget","src","pathSet","byBase","clean","fromSrc","normalize","join","dirname","candidate","candidates","get","buildByBase","files","Map","paths","map","f","relPath","sort","a","b","list","set","resolveRows","rows","changed","row","dst","writeResolved","db","runBatch","r","resolveAll","stmt","prepare","all","selectIn","column","keys","i","chunk","slice","resolveIncremental","delta","changedBasenames","p","added","vanished","collect","reparsed","LINK_EDGES_SQL","toEdges","linkEdges","removedWithLinks","WeakMap","LINK_COLUMNS","UPSERT_LINK_SQL","links","name","schema","remove","vanishedSet","filter","existingRows","store","docs","byPath","d","extracted","addedSet","reparsedExisting","staleRows","targets","some","t","appendable","upsertRows","afterReconcile","removeHadLinks","large","dstChanged","linksChanged"],"mappings":"AAAA,OAAOA,WAAW,kBAAkB;AAEpC,SAASC,UAAU,QAAQ,qBAAqB;AAEhD,SAASC,WAAW,QAAQ,cAAc;AAG1C,0JAA0J;AAC1J,mJAAmJ;AAEnJ,wFAAwF;AACxF,gEAAgE;AAChE,SAASC,mBAAmBC,KAAa;IACvC,MAAMC,cAAcD,MAAME,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC5C,IAAIF,YAAYG,UAAU,CAAC,MAAM,OAAOH,YAAYI,MAAM,GAAG,IAAIJ,cAAc;IAC/E,MAAMK,SAASL,YAAYC,KAAK,CAAC,IAAI,CAAC,EAAE,CAACC,IAAI;IAC7C,OAAOG,UAAU;AACnB;AAEA,4FAA4F;AAC5F,0EAA0E;AAC1E,SAASC,QAAQC,IAAY,EAAEC,OAAe,EAAEC,OAA4C,EAAEC,IAA8B;IAC1H,MAAMC,OAAO,YAAYC,IAAI,CAACJ,WAAWX,YAAYW,WAAWA;IAChE,MAAMK,OAAO,IAAIC;IACjB,MAAMC,UAAqD,EAAE;IAC7D,MAAMC,MAAM,CAACX,QAAgBY;QAC3B,MAAMC,MAAM,GAAGb,OAAO,EAAE,EAAEY,QAAQ,MAAM,KAAK;QAC7C,IAAIJ,KAAKM,GAAG,CAACD,MAAM;QACnBL,KAAKG,GAAG,CAACE;QACTH,QAAQK,IAAI,CAAC;YAAEf;YAAQY;QAAM;IAC/B;IACA,uFAAuF;IACvF,6DAA6D;IAC7D,KAAK,MAAMI,KAAKV,KAAKW,QAAQ,CAAC,kBAAmB;YAEjBD;QAD9B,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;QACtC,IAAIhB,QAAQW,IAAIX,QAAQM,IAAI,CAAC,EAACU,WAAAA,EAAEE,KAAK,cAAPF,sBAAAA,WAAW,KAAK,EAAE,KAAK;IACvD;IACA,2FAA2F;IAC3F,8FAA8F;IAC9F,KAAK,MAAMA,KAAKV,KAAKW,QAAQ,CAAC,6EAA8E;QAC1G,MAAME,OAAOH,CAAC,CAAC,EAAE,CAACnB,IAAI;QACtB,qFAAqF;QACrF,oFAAoF;QACpF,IAAI,CAACsB,QAAQA,KAAKC,QAAQ,CAAC,QAAQ;QACnC,IAAI,oCAAoCb,IAAI,CAACY,OAAO;QACpD,MAAMnB,SAASmB,KAAKrB,UAAU,CAAC,OAAOqB,OAAOA,KAAKvB,KAAK,CAAC,IAAI,CAAC,EAAE;QAC/D,IAAII,QAAQW,IAAIX,QAAQgB,CAAC,CAAC,EAAE,KAAK;IACnC;IACA,KAAK,MAAMK,SAASC,OAAOC,MAAM,CAAClB,iBAAAA,kBAAAA,OAAQ,CAAC,GAAI;QAC7C,KAAK,MAAMmB,QAAQC,MAAMC,OAAO,CAACL,SAASA,QAAQ;YAACA;SAAM,CAAE;YACzD,IAAI,OAAOG,SAAS,UAAU;YAC9B,MAAMR,IAAI,iBAAiBW,IAAI,CAACH,KAAK3B,IAAI;YACzC,IAAI,CAACmB,GAAG;YACR,MAAMhB,SAASP,mBAAmBuB,CAAC,CAAC,EAAE;YACtC,IAAIhB,QAAQW,IAAIX,QAAQ;QAC1B;IACF;IACA,OAAOU;AACT;AAEA,SAASkB,YAAY5B,MAAc;IACjC,OAAOA,OAAO6B,OAAO,CAAC,OAAO,KAAKA,OAAO,CAAC,SAAS;AACrD;AAEA,SAASC,QAAQC,IAAY;IAC3B,OAAOzC,MAAM0C,QAAQ,CAACD,MAAMF,OAAO,CAAC,UAAU,IAAII,WAAW;AAC/D;AAEA,8FAA8F;AAC9F,kGAAkG;AAClG,SAASC,cAAcC,GAAW,EAAEnC,MAAc,EAAEoC,OAAoB,EAAEC,MAA6B;IACrG,4FAA4F;IAC5F,IAAIrC,OAAOF,UAAU,CAAC,MAAM,OAAOqC;IACnC,MAAMG,QAAQV,YAAY5B;IAC1B,MAAMuC,UAAUjD,MAAMkD,SAAS,CAAClD,MAAMmD,IAAI,CAACnD,MAAMoD,OAAO,CAACP,MAAMG;IAC/D,KAAK,MAAMK,aAAa;QAACL;QAAO,GAAGA,MAAM,GAAG,CAAC;QAAEC;QAAS,GAAGA,QAAQ,GAAG,CAAC;KAAC,CAAE;QACxE,IAAIH,QAAQtB,GAAG,CAAC6B,YAAY,OAAOA;IACrC;IACA,MAAMC,aAAaP,OAAOQ,GAAG,CAACf,QAAQQ;IACtC,IAAI,CAACM,YAAY,OAAO;IACxB,0FAA0F;IAC1F,+DAA+D;IAC/D,OAAOA,WAAWxB,QAAQ,CAACe,OAAOA,MAAMS,UAAU,CAAC,EAAE;AACvD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,SAASE,YAAYC,KAAiB;IACpC,MAAMV,SAAS,IAAIW;IACnB,MAAMC,QAAQF,MAAMG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO,EAAEC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEvD,MAAM,GAAGwD,EAAExD,MAAM,IAAKuD,CAAAA,IAAIC,IAAI,CAAC,IAAID,IAAIC,IAAI,IAAI,CAAA;IAC1G,KAAK,MAAMxB,QAAQkB,MAAO;QACxB,MAAMpC,MAAMiB,QAAQC;QACpB,MAAMyB,OAAOnB,OAAOQ,GAAG,CAAChC;QACxB,IAAI2C,MAAMA,KAAKzC,IAAI,CAACgB;aACfM,OAAOoB,GAAG,CAAC5C,KAAK;YAACkB;SAAK;IAC7B;IACA,OAAOM;AACT;AASA,4FAA4F;AAC5F,uCAAuC;AACvC,SAASqB,YAAYC,IAAe,EAAEvB,OAAoB,EAAEC,MAA6B;IACvF,MAAMuB,UAAqB,EAAE;IAC7B,KAAK,MAAMC,OAAOF,KAAM;QACtB,MAAMG,MAAM5B,cAAc2B,IAAI1B,GAAG,EAAE0B,IAAI7D,MAAM,EAAEoC,SAASC;QACxD,IAAIyB,QAAQD,IAAIC,GAAG,EAAEF,QAAQ7C,IAAI,CAAC;YAAE,GAAG8C,GAAG;YAAEC;QAAI;IAClD;IACA,OAAOF;AACT;AAEA,eAAeG,cAAcC,EAAc,EAAEL,IAAe;IAC1D,IAAIA,KAAK5D,MAAM,KAAK,GAAG;IACvB,MAAMiE,GAAGC,QAAQ,CACf,yDACAN,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAEJ,GAAG;YAAEI,EAAE/B,GAAG;YAAE+B,EAAElE,MAAM;SAAC;AAE5C;AAEA,yFAAyF;AACzF,+EAA+E;AAC/E,eAAemE,WAAWH,EAAc,EAAEjB,KAAiB;IACzD,MAAMX,UAAU,IAAI3B,IAAIsC,MAAMG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAClD,MAAMf,SAASS,YAAYC;IAC3B,MAAMqB,OAAO,MAAMJ,GAAGK,OAAO,CAAC;IAC9B,MAAMV,OAAQ,MAAMS,KAAKE,GAAG;IAC5B,MAAMV,UAAUF,YAAYC,MAAMvB,SAASC;IAC3C,MAAM0B,cAAcC,IAAIJ;IACxB,OAAOA,QAAQ7D,MAAM,GAAG;AAC1B;AAEA,2FAA2F;AAC3F,2FAA2F;AAC3F,eAAewE,SAASP,EAAc,EAAEQ,MAAc,EAAEC,IAAc;IACpE,MAAMd,OAAkB,EAAE;IAC1B,IAAK,IAAIe,IAAI,GAAGA,IAAID,KAAK1E,MAAM,EAAE2E,KAAK,IAAK;QACzC,MAAMC,QAAQF,KAAKG,KAAK,CAACF,GAAGA,IAAI;QAChC,MAAMN,OAAO,MAAMJ,GAAGK,OAAO,CAAC,CAAC,gDAAgD,EAAEG,OAAO,KAAK,EAAEG,MAAMzB,GAAG,CAAC,IAAM,KAAKT,IAAI,CAAC,MAAM,CAAC,CAAC;QACjIkB,KAAK5C,IAAI,IAAM,MAAMqD,KAAKE,GAAG,IAAIK;IACnC;IACA,OAAOhB;AACT;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,eAAekB,mBAAmBb,EAAc,EAAEc,KAAqB;IACrE,MAAM1C,UAAU,IAAI3B,IAAIqE,MAAM/B,KAAK,CAACG,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACxD,MAAMf,SAASS,YAAYgC,MAAM/B,KAAK;IAEtC,MAAMgC,mBAAmB,IAAItE;IAC7B,KAAK,MAAMuE,KAAKF,MAAMG,KAAK,CAAEF,iBAAiBpE,GAAG,CAACmB,QAAQkD;IAC1D,KAAK,MAAMA,KAAKF,MAAMI,QAAQ,CAAEH,iBAAiBpE,GAAG,CAACmB,QAAQkD;IAE7D,+EAA+E;IAC/E,MAAMpC,aAAa,IAAII;IACvB,MAAMmC,UAAU,CAACxB;QACf,KAAK,MAAME,OAAOF,KAAMf,WAAWa,GAAG,CAAC,GAAGI,IAAI1B,GAAG,CAAC,EAAE,EAAE0B,IAAI7D,MAAM,CAAC,EAAE,EAAE6D,IAAIjD,KAAK,EAAE,EAAEiD;IACpF;IACAsB,QAAQ,MAAMZ,SAASP,IAAI,OAAOc,MAAMM,QAAQ;IAChDD,QAAQ,MAAMZ,SAASP,IAAI,eAAe;WAAIe;KAAiB;IAC/DI,QAAQ,MAAMZ,SAASP,IAAI,OAAOc,MAAMI,QAAQ;IAEhD,MAAMtB,UAAUF,YAAY;WAAId,WAAWrB,MAAM;KAAG,EAAEa,SAASC;IAC/D,MAAM0B,cAAcC,IAAIJ;IACxB,OAAOA,QAAQ7D,MAAM,GAAG;AAC1B;AAEA,8FAA8F;AAC9F,wEAAwE;AACxE,OAAO,MAAMsF,iBAAiB,CAAC;;;4GAG6E,CAAC,CAAC;AAE9G,OAAO,SAASC,QAAQ3B,IAAyC;IAC/D,OAAOA,KAAKT,GAAG,CAAC,CAACgB,IAAM;YAACA,EAAE/B,GAAG;YAAE+B,EAAEJ,GAAG;SAAC;AACvC;AAEA,qEAAqE;AACrE,OAAO,eAAeyB,UAAUvB,EAAc;IAC5C,MAAMI,OAAO,MAAMJ,GAAGK,OAAO,CAACgB;IAC9B,OAAOC,QAAS,MAAMlB,KAAKE,GAAG;AAChC;AAEA,8FAA8F;AAC9F,qGAAqG;AACrG,MAAMkB,mBAAmB,IAAIC;AAE7B,mGAAmG;AACnG,sCAAsC;AACtC,MAAMC,eAAe;IAAC;IAAO;IAAU;IAAe;CAAQ;AAC9D,MAAMC,kBAAkB;AAExB,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAO9B,EAAE;QACb,MAAMA,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;QACd,MAAMqC,GAAGrC,IAAI,CAAC;IAChB;IACA1B;IACA,4FAA4F;IAC5F,4FAA4F;IAC5F,MAAM8F,QAAO/B,EAAE,EAAEf,KAAK,EAAE6B,KAAK;QAC3B,MAAMkB,cAAc,IAAIvF,IAAIqE,MAAMI,QAAQ;QAC1C,MAAMA,WAAWjC,MAAMgD,MAAM,CAAC,CAACjB,IAAMgB,YAAYlF,GAAG,CAACkE;QACrD,IAAIE,SAASnF,MAAM,KAAK,GAAG;QAC3B,MAAMmG,eAAe,MAAM3B,SAASP,IAAI,OAAOkB;QAC/C,MAAMlB,GAAGC,QAAQ,CACf,mCACAiB,SAAShC,GAAG,CAAC,CAAC8B,IAAM;gBAACA;aAAE;QAEzB,IAAIkB,aAAanG,MAAM,GAAG,GAAGyF,iBAAiB/B,GAAG,CAACqB,OAAO;IAC3D;IACA,MAAMqB,OAAMnC,EAAE,EAAEoC,IAAoB,EAAEtB,KAAK;QACzC,MAAMuB,SAAS,IAAIrD,IAAIoD,KAAKlD,GAAG,CAAC,CAACoD,IAAM;gBAACA,EAAEvE,IAAI;gBAAEuE,EAAEC,SAAS;aAA8C;QACzG,MAAMC,WAAW,IAAI/F,IAAIqE,MAAMG,KAAK;QACpC,MAAMwB,mBAAmBL,KAAKlD,GAAG,CAAC,CAACoD,IAAMA,EAAEvE,IAAI,EAAEkE,MAAM,CAAC,CAACjB,IAAM,CAACwB,SAAS1F,GAAG,CAACkE;QAE7E,uFAAuF;QACvF,8FAA8F;QAC9F,MAAMkB,eAAe,MAAM3B,SAASP,IAAI,OAAOyC;QAC/C,MAAMC,YAAYR,aAAaD,MAAM,CAAC,CAACpC;gBACrBwC;YAAhB,MAAMM,WAAUN,cAAAA,OAAOxD,GAAG,CAACgB,IAAI1B,GAAG,eAAlBkE,yBAAAA,cAAuB,EAAE;YACzC,OAAO,CAACM,QAAQC,IAAI,CAAC,CAACC,IAAMA,EAAE7G,MAAM,KAAK6D,IAAI7D,MAAM,IAAI,AAAC6G,CAAAA,EAAEjG,KAAK,GAAG,IAAI,CAAA,MAAOiD,IAAIjD,KAAK;QACxF;QACA,IAAI8F,UAAU3G,MAAM,GAAG,GAAG;YACxB,MAAMiE,GAAGC,QAAQ,CACf,gEACAyC,UAAUxD,GAAG,CAAC,CAACgB,IAAM;oBAACA,EAAE/B,GAAG;oBAAE+B,EAAElE,MAAM;oBAAEkE,EAAEtD,KAAK;iBAAC;YAEjD4E,iBAAiB/B,GAAG,CAACqB,OAAO;QAC9B;QAEA,0FAA0F;QAC1F,mFAAmF;QACnF,6FAA6F;QAC7F,+FAA+F;QAC/F,wCAAwC;QACxC,MAAMgC,aAA0B,EAAE;QAClC,MAAMC,aAA0B,EAAE;QAClC,KAAK,MAAM,CAAChF,MAAM4E,QAAQ,IAAIN,OAAQ;YACpC,KAAK,MAAM,EAAErG,MAAM,EAAEY,KAAK,EAAE,IAAI+F,QAAS;gBACvC,MAAM9C,MAAM;oBAAC9B;oBAAM/B;oBAAQA,OAAOF,UAAU,CAAC,OAAO,OAAOgC,QAAQF,YAAY5B;oBAAUY,QAAQ,IAAI;iBAAE;gBACvG,IAAI4F,SAAS1F,GAAG,CAACiB,OAAO+E,WAAW/F,IAAI,CAAC8C;qBACnCkD,WAAWhG,IAAI,CAAC8C;YACvB;QACF;QACA,MAAMtE,WAAWyE,IAAI,SAAS0B,cAAcC,iBAAiBmB;QAC7D,IAAIC,WAAWhH,MAAM,GAAG,GAAG,MAAMiE,GAAGC,QAAQ,CAAC0B,iBAAiBoB;IAChE;IACA,MAAMC,gBAAehD,EAAE,EAAEc,KAAK;YAGLU;QAFvB,yFAAyF;QACzF,0BAA0B;QAC1B,MAAMyB,kBAAiBzB,wBAAAA,iBAAiB3C,GAAG,CAACiC,oBAArBU,mCAAAA,wBAA+B;QAEtD,8FAA8F;QAC9F,kGAAkG;QAClG,MAAM0B,QAAQpC,MAAM/B,KAAK,CAAChD,MAAM,KAAK,KAAK+E,MAAMG,KAAK,CAAClF,MAAM,KAAK+E,MAAM/B,KAAK,CAAChD,MAAM;QAEnF,MAAMoH,aAAaD,QAAQ,MAAM/C,WAAWH,IAAIc,MAAM/B,KAAK,IAAI,MAAM8B,mBAAmBb,IAAIc;QAC5FA,MAAMsC,YAAY,GAAGD,cAAcF;IACrC;AACF,EAAE"}

package/dist/esm/features/sections.js

@@ -1,5 +1,6 @@
 import { estimateTokens, parse } from '../chunk/index.js';
 import { countLines } from '../scan/frontmatter.js';
+import { appendRows } from '../store/shared.js';
 // Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;
 // a section runs to just before the next heading, or EOF.
 function sectionsFromBlocks(blocks, raw, body) {
@@ -22,6 +23,15 @@ function sectionsFromBlocks(blocks, raw, body) {
     }
     return found;
 }
+const SECTION_COLUMNS = [
+    'path',
+    'idx',
+    'level',
+    'heading',
+    'start_line',
+    'end_line',
+    'tokens'
+];
 export const sections = {
     name: 'sections',
     async schema (db) {
@@ -47,10 +57,10 @@ export const sections = {
                 s.endLine,
                 s.tokens
             ]));
-        if (rows.length === 0) return;
         // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this
         // write's lock, so a path called "added" here can already have this row from a concurrent
-        // reconcile that committed first -- same file, same parse, same row.
-        await db.runBatch('INSERT INTO sections ("path", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT("path", idx) DO NOTHING', rows);
+        // reconcile that committed first -- same file, same parse, same row. A store with an append
+        // path has no second writer, and remove() cleared every touched path above, so it skips the guard.
+        await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections ("path", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT("path", idx) DO NOTHING', rows);
     }
 };

package/dist/esm/features/sections.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    if (rows.length === 0) return;\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row.\n    await db.runBatch('INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["estimateTokens","parse","countLines","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","sections","name","schema","db","exec","extract","_search","_data","_cfg","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push"],"mappings":"AACA,SAASA,cAAc,EAAEC,KAAK,QAAQ,oBAAoB;AAC1D,SAASC,UAAU,QAAQ,yBAAyB;AAcpD,iGAAiG;AACjG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,MAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,MAAMC,SAASF,SAASG,MAAM,GAAGR,WAAWI;IAC5C,MAAMK,QAAmBP,OAAOQ,MAAM,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,WAAWC,GAAG,CAAC,CAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGZ;YAAQa,SAASf,SAASG,MAAM;YAAEa,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMD,MAAM,EAAEc,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMD,MAAM,EAAEC,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,MAAMF,OAAOZ,SAASkB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAAC5B,eAAemB;IAC7C;IACA,OAAOR;AACT;AAEA,OAAO,MAAMkB,WAAoB;IAC/BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,4KAA4K,CAAC;IAC9L;IACAC,SAAQ7B,GAAG,EAAEC,IAAI,EAAE6B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAEjC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUH,MAAMK,OAAOD,KAAKC;IACxD;IACA,MAAMgC,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAM7B,MAAM,KAAK,GAAG;QACxB,MAAMsB,GAAGQ,QAAQ,CACf,yCACAD,MAAMxB,GAAG,CAAC,CAAC0B,IAAM;gBAACA;aAAE;IAExB;IACA,MAAMC,OAAMV,EAAE,EAAEW,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM,AAACG,UAAwBC,OAAO,CAAC,CAACvB,GAAGwB,MAAQJ,KAAKK,IAAI,CAAC;gBAACJ;gBAAMG;gBAAKxB,EAAER,KAAK;gBAAEQ,EAAEN,OAAO;gBAAEM,EAAEH,SAAS;gBAAEG,EAAEF,OAAO;gBAAEE,EAAED,MAAM;aAAC;QAChK,IAAIqB,KAAKlC,MAAM,KAAK,GAAG;QACvB,wFAAwF;QACxF,0FAA0F;QAC1F,qEAAqE;QACrE,MAAMsB,GAAGQ,QAAQ,CAAC,qJAAqJI;IACzK;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/sections.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { estimateTokens, parse } from '../chunk/index.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport { appendRows } from '../store/shared.ts';\nimport type { Feature } from './types.ts';\n\n// sections(path, idx, level, heading, start_line, end_line, tokens): the heading outline,\n// 1-indexed over the raw file so a row is a direct Read range; tokens is estimateTokens (D5), CJK-aware.\n\nexport interface Section {\n  level: number;\n  heading: string;\n  startLine: number;\n  endLine: number;\n  tokens: number;\n}\n\n// Heading blocks parse() already found (mdast/CommonMark fences), offset back onto the raw file;\n// a section runs to just before the next heading, or EOF.\nfunction sectionsFromBlocks(blocks: Block[], raw: string, body: string): Section[] {\n  const rawLines = raw.split('\\n');\n  const offset = rawLines.length - countLines(body);\n  const found: Section[] = blocks.filter((b) => b.type === 'heading').map((b) => ({ level: b.depth ?? 1, heading: (b.text ?? '').trim(), startLine: b.startLine + offset, endLine: rawLines.length, tokens: 0 }));\n  for (let s = 0; s < found.length; s++) {\n    if (s + 1 < found.length) found[s].endLine = found[s + 1].startLine - 1;\n    const text = rawLines.slice(found[s].startLine - 1, found[s].endLine).join('\\n');\n    found[s].tokens = Math.ceil(estimateTokens(text));\n  }\n  return found;\n}\n\nconst SECTION_COLUMNS = ['path', 'idx', 'level', 'heading', 'start_line', 'end_line', 'tokens'];\n\nexport const sections: Feature = {\n  name: 'sections',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS sections (\"path\" TEXT, idx INTEGER, level INTEGER, heading TEXT, start_line INTEGER, end_line INTEGER, tokens INTEGER, PRIMARY KEY (\"path\", idx))`);\n  },\n  extract(raw, body, _search, _data, _cfg, blocks) {\n    return sectionsFromBlocks(blocks ?? parse(body), raw, body);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM sections WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) (extracted as Section[]).forEach((s, idx) => rows.push([path, idx, s.level, s.heading, s.startLine, s.endLine, s.tokens]));\n    // DO NOTHING, not a bare INSERT: reconcile's added/touched split is decided before this\n    // write's lock, so a path called \"added\" here can already have this row from a concurrent\n    // reconcile that committed first -- same file, same parse, same row. A store with an append\n    // path has no second writer, and remove() cleared every touched path above, so it skips the guard.\n    await appendRows(db, 'sections', SECTION_COLUMNS, 'INSERT INTO sections (\"path\", idx, level, heading, start_line, end_line, tokens) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(\"path\", idx) DO NOTHING', rows);\n  },\n};\n"],"names":["estimateTokens","parse","countLines","appendRows","sectionsFromBlocks","blocks","raw","body","rawLines","split","offset","length","found","filter","b","type","map","level","depth","heading","text","trim","startLine","endLine","tokens","s","slice","join","Math","ceil","SECTION_COLUMNS","sections","name","schema","db","exec","extract","_search","_data","_cfg","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push"],"mappings":"AACA,SAASA,cAAc,EAAEC,KAAK,QAAQ,oBAAoB;AAC1D,SAASC,UAAU,QAAQ,yBAAyB;AACpD,SAASC,UAAU,QAAQ,qBAAqB;AAchD,iGAAiG;AACjG,0DAA0D;AAC1D,SAASC,mBAAmBC,MAAe,EAAEC,GAAW,EAAEC,IAAY;IACpE,MAAMC,WAAWF,IAAIG,KAAK,CAAC;IAC3B,MAAMC,SAASF,SAASG,MAAM,GAAGT,WAAWK;IAC5C,MAAMK,QAAmBP,OAAOQ,MAAM,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,WAAWC,GAAG,CAAC,CAACF;YAAgBA,UAAwBA;eAAjC;YAAEG,KAAK,GAAEH,WAAAA,EAAEI,KAAK,cAAPJ,sBAAAA,WAAW;YAAGK,SAAS,EAACL,UAAAA,EAAEM,IAAI,cAANN,qBAAAA,UAAU,IAAIO,IAAI;YAAIC,WAAWR,EAAEQ,SAAS,GAAGZ;YAAQa,SAASf,SAASG,MAAM;YAAEa,QAAQ;QAAE;;IAC5M,IAAK,IAAIC,IAAI,GAAGA,IAAIb,MAAMD,MAAM,EAAEc,IAAK;QACrC,IAAIA,IAAI,IAAIb,MAAMD,MAAM,EAAEC,KAAK,CAACa,EAAE,CAACF,OAAO,GAAGX,KAAK,CAACa,IAAI,EAAE,CAACH,SAAS,GAAG;QACtE,MAAMF,OAAOZ,SAASkB,KAAK,CAACd,KAAK,CAACa,EAAE,CAACH,SAAS,GAAG,GAAGV,KAAK,CAACa,EAAE,CAACF,OAAO,EAAEI,IAAI,CAAC;QAC3Ef,KAAK,CAACa,EAAE,CAACD,MAAM,GAAGI,KAAKC,IAAI,CAAC7B,eAAeoB;IAC7C;IACA,OAAOR;AACT;AAEA,MAAMkB,kBAAkB;IAAC;IAAQ;IAAO;IAAS;IAAW;IAAc;IAAY;CAAS;AAE/F,OAAO,MAAMC,WAAoB;IAC/BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,4KAA4K,CAAC;IAC9L;IACAC,SAAQ9B,GAAG,EAAEC,IAAI,EAAE8B,OAAO,EAAEC,KAAK,EAAEC,IAAI,EAAElC,MAAM;QAC7C,OAAOD,mBAAmBC,mBAAAA,oBAAAA,SAAUJ,MAAMM,OAAOD,KAAKC;IACxD;IACA,MAAMiC,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAM9B,MAAM,KAAK,GAAG;QACxB,MAAMuB,GAAGQ,QAAQ,CACf,yCACAD,MAAMzB,GAAG,CAAC,CAAC2B,IAAM;gBAACA;aAAE;IAExB;IACA,MAAMC,OAAMV,EAAE,EAAEW,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM,AAACG,UAAwBC,OAAO,CAAC,CAACxB,GAAGyB,MAAQJ,KAAKK,IAAI,CAAC;gBAACJ;gBAAMG;gBAAKzB,EAAER,KAAK;gBAAEQ,EAAEN,OAAO;gBAAEM,EAAEH,SAAS;gBAAEG,EAAEF,OAAO;gBAAEE,EAAED,MAAM;aAAC;QAChK,wFAAwF;QACxF,0FAA0F;QAC1F,4FAA4F;QAC5F,mGAAmG;QACnG,MAAMrB,WAAW+B,IAAI,YAAYJ,iBAAiB,qJAAqJgB;IACzM;AACF,EAAE"}

package/dist/esm/features/tags.js

@@ -1,3 +1,4 @@
+import { appendRows } from '../store/shared.js';
 import { maskRegions } from './fences.js';
 // tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline
 // #tags, deduplicated, source not distinguished. Nested tags store full (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` matches the parent too.
@@ -77,7 +78,11 @@ export const tags = {
             path,
             tag
         ]);
-        if (rows.length === 0) return;
-        await db.runBatch('INSERT OR IGNORE INTO tags ("path", tag) VALUES (?, ?)', rows);
+        // OR IGNORE guards the same concurrent-reconcile race sections.ts documents; remove() cleared
+        // every touched path above, so a store with an append path needs no guard.
+        await appendRows(db, 'tags', [
+            'path',
+            'tag'
+        ], 'INSERT OR IGNORE INTO tags ("path", tag) VALUES (?, ?)', rows);
     }
 };

package/dist/esm/features/tags.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/tags.ts"],"sourcesContent":["import { maskRegions } from './fences.ts';\nimport type { Feature } from './types.ts';\n\n// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline\n// #tags, deduplicated, source not distinguished. Nested tags store full (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` matches the parent too.\n\n// Obsidian treats [[#Heading]] as a same-note link, not a tag.\nconst WIKILINK_RE = /\\[\\[.*?\\]\\]/g; // to the first ]], so a heading holding a lone ] still masks\n// Obsidian doesn't read tags inside HTML markup.\nconst HTML_TAG_RE = /<\\/?[a-zA-Z][^>]*>/g; // tag-shaped only: a comparison's `< 5` must not open a span\n// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.\nconst INLINE_TAG_RE = /(?:^|[\\s([])#([\\p{L}\\p{N}_/-]+)/gu;\n// A markdown link destination `](...)` -- `[text](#anchor)` is a same-page link, not a tag.\nconst LINK_DEST_RE = /\\]\\((?:[^()]|\\([^()]*\\))*\\)/g; // one paren-nesting level, as CommonMark destinations allow: (https://x/a_(b)#frag)\n\n// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an\n// all-digit result -- a tag needs at least one non-digit character.\nfunction normalizeTag(raw: string): string | null {\n  const stripped = raw.replace(/^#/, '').replace(/\\/+$/, '');\n  if (!stripped || /^\\d+$/.test(stripped)) return null;\n  return stripped;\n}\n\n// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string\n// members are skipped rather than throwing -- `tags:\\n  -` parses to [null].\nfunction frontmatterTags(data?: Record<string, unknown>): string[] {\n  const raw = data?.tags;\n  const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [];\n  const found: string[] = [];\n  for (const item of list) {\n    if (typeof item !== 'string') continue;\n    const tag = normalizeTag(item);\n    if (tag) found.push(tag);\n  }\n  return found;\n}\n\n// #tag tokens outside the regions maskRegions() masks (fences, code spans, wikilinks, HTML,\n// comments, link destinations); this loop only does per-line tag extraction.\nfunction inlineTags(body: string): string[] {\n  const found: string[] = [];\n  for (const line of maskRegions(body).split('\\n')) {\n    if (!line.includes('#')) continue; // most lines; skip the regex work\n    let cleaned = line;\n    if (cleaned.includes('[[')) cleaned = cleaned.replace(WIKILINK_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('](')) cleaned = cleaned.replace(LINK_DEST_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('<')) cleaned = cleaned.replace(HTML_TAG_RE, (m) => ' '.repeat(m.length));\n    for (const m of cleaned.matchAll(INLINE_TAG_RE)) {\n      const tag = normalizeTag(m[1]);\n      if (tag) found.push(tag);\n    }\n  }\n  return found;\n}\n\nfunction extract(_raw: string, body: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): string[] {\n  return [...new Set([...frontmatterTags(data), ...inlineTags(body)])].sort();\n}\n\nexport const tags: Feature = {\n  name: 'tags',\n  async schema(db) {\n    await db.exec('CREATE TABLE IF NOT EXISTS tags (\"path\" TEXT, tag TEXT, PRIMARY KEY (\"path\", tag))');\n    await db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');\n  },\n  extract,\n  // Blanket clear-then-reinsert (store() always writes the full found list back), so remove()\n  // covers vanished and reparsed files without per-file NOT IN diffing, which cannot be one statement shared across files.\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM tags WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) for (const tag of extracted as string[]) rows.push([path, tag]);\n    if (rows.length === 0) return;\n    await db.runBatch('INSERT OR IGNORE INTO tags (\"path\", tag) VALUES (?, ?)', rows);\n  },\n};\n"],"names":["maskRegions","WIKILINK_RE","HTML_TAG_RE","INLINE_TAG_RE","LINK_DEST_RE","normalizeTag","raw","stripped","replace","test","frontmatterTags","data","tags","list","Array","isArray","found","item","tag","push","inlineTags","body","line","split","includes","cleaned","m","repeat","length","matchAll","extract","_raw","_search","Set","sort","name","schema","db","exec","remove","paths","runBatch","map","p","store","docs","rows","path","extracted"],"mappings":"AAAA,SAASA,WAAW,QAAQ,cAAc;AAG1C,0FAA0F;AAC1F,kJAAkJ;AAElJ,+DAA+D;AAC/D,MAAMC,cAAc,gBAAgB,6DAA6D;AACjG,iDAAiD;AACjD,MAAMC,cAAc,uBAAuB,6DAA6D;AACxG,iGAAiG;AACjG,MAAMC,gBAAgB;AACtB,4FAA4F;AAC5F,MAAMC,eAAe,gCAAgC,oFAAoF;AAEzI,sFAAsF;AACtF,oEAAoE;AACpE,SAASC,aAAaC,GAAW;IAC/B,MAAMC,WAAWD,IAAIE,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,QAAQ;IACvD,IAAI,CAACD,YAAY,QAAQE,IAAI,CAACF,WAAW,OAAO;IAChD,OAAOA;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASG,gBAAgBC,IAA8B;IACrD,MAAML,MAAMK,iBAAAA,2BAAAA,KAAMC,IAAI;IACtB,MAAMC,OAAOC,MAAMC,OAAO,CAACT,OAAOA,MAAM,OAAOA,QAAQ,WAAW;QAACA;KAAI,GAAG,EAAE;IAC5E,MAAMU,QAAkB,EAAE;IAC1B,KAAK,MAAMC,QAAQJ,KAAM;QACvB,IAAI,OAAOI,SAAS,UAAU;QAC9B,MAAMC,MAAMb,aAAaY;QACzB,IAAIC,KAAKF,MAAMG,IAAI,CAACD;IACtB;IACA,OAAOF;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASI,WAAWC,IAAY;IAC9B,MAAML,QAAkB,EAAE;IAC1B,KAAK,MAAMM,QAAQtB,YAAYqB,MAAME,KAAK,CAAC,MAAO;QAChD,IAAI,CAACD,KAAKE,QAAQ,CAAC,MAAM,UAAU,kCAAkC;QACrE,IAAIC,UAAUH;QACd,IAAIG,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACP,aAAa,CAACyB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC7F,IAAIH,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACJ,cAAc,CAACsB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC9F,IAAIH,QAAQD,QAAQ,CAAC,MAAMC,UAAUA,QAAQjB,OAAO,CAACN,aAAa,CAACwB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC5F,KAAK,MAAMF,KAAKD,QAAQI,QAAQ,CAAC1B,eAAgB;YAC/C,MAAMe,MAAMb,aAAaqB,CAAC,CAAC,EAAE;YAC7B,IAAIR,KAAKF,MAAMG,IAAI,CAACD;QACtB;IACF;IACA,OAAOF;AACT;AAEA,SAASc,QAAQC,IAAY,EAAEV,IAAY,EAAEW,OAA4C,EAAErB,IAA8B;IACvH,OAAO;WAAI,IAAIsB,IAAI;eAAIvB,gBAAgBC;eAAUS,WAAWC;SAAM;KAAE,CAACa,IAAI;AAC3E;AAEA,OAAO,MAAMtB,OAAgB;IAC3BuB,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC;QACd,MAAMD,GAAGC,IAAI,CAAC;IAChB;IACAR;IACA,4FAA4F;IAC5F,yHAAyH;IACzH,MAAMS,QAAOF,EAAE,EAAEG,KAAK;QACpB,IAAIA,MAAMZ,MAAM,KAAK,GAAG;QACxB,MAAMS,GAAGI,QAAQ,CACf,qCACAD,MAAME,GAAG,CAAC,CAACC,IAAM;gBAACA;aAAE;IAExB;IACA,MAAMC,OAAMP,EAAE,EAAEQ,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM,KAAK,MAAM3B,OAAO8B,UAAuBF,KAAK3B,IAAI,CAAC;YAAC4B;YAAM7B;SAAI;QACtG,IAAI4B,KAAKlB,MAAM,KAAK,GAAG;QACvB,MAAMS,GAAGI,QAAQ,CAAC,0DAA0DK;IAC9E;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/tags.ts"],"sourcesContent":["import { appendRows } from '../store/shared.ts';\nimport { maskRegions } from './fences.ts';\nimport type { Feature } from './types.ts';\n\n// tags(path, tag): Obsidian's file.tags grain -- frontmatter list/string tags plus inline\n// #tags, deduplicated, source not distinguished. Nested tags store full (book/scifi); `tag = 'book' OR tag LIKE 'book/%'` matches the parent too.\n\n// Obsidian treats [[#Heading]] as a same-note link, not a tag.\nconst WIKILINK_RE = /\\[\\[.*?\\]\\]/g; // to the first ]], so a heading holding a lone ] still masks\n// Obsidian doesn't read tags inside HTML markup.\nconst HTML_TAG_RE = /<\\/?[a-zA-Z][^>]*>/g; // tag-shaped only: a comparison's `< 5` must not open a span\n// Anchors on start-of-line or a preceding whitespace/(/[ so `a#b` and URL fragments don't count.\nconst INLINE_TAG_RE = /(?:^|[\\s([])#([\\p{L}\\p{N}_/-]+)/gu;\n// A markdown link destination `](...)` -- `[text](#anchor)` is a same-page link, not a tag.\nconst LINK_DEST_RE = /\\]\\((?:[^()]|\\([^()]*\\))*\\)/g; // one paren-nesting level, as CommonMark destinations allow: (https://x/a_(b)#frag)\n\n// Strips a leading # (frontmatter entries may carry one) and a trailing /; rejects an\n// all-digit result -- a tag needs at least one non-digit character.\nfunction normalizeTag(raw: string): string | null {\n  const stripped = raw.replace(/^#/, '').replace(/\\/+$/, '');\n  if (!stripped || /^\\d+$/.test(stripped)) return null;\n  return stripped;\n}\n\n// data.tags: a YAML list (Obsidian also accepts a bare string). Null members and non-string\n// members are skipped rather than throwing -- `tags:\\n  -` parses to [null].\nfunction frontmatterTags(data?: Record<string, unknown>): string[] {\n  const raw = data?.tags;\n  const list = Array.isArray(raw) ? raw : typeof raw === 'string' ? [raw] : [];\n  const found: string[] = [];\n  for (const item of list) {\n    if (typeof item !== 'string') continue;\n    const tag = normalizeTag(item);\n    if (tag) found.push(tag);\n  }\n  return found;\n}\n\n// #tag tokens outside the regions maskRegions() masks (fences, code spans, wikilinks, HTML,\n// comments, link destinations); this loop only does per-line tag extraction.\nfunction inlineTags(body: string): string[] {\n  const found: string[] = [];\n  for (const line of maskRegions(body).split('\\n')) {\n    if (!line.includes('#')) continue; // most lines; skip the regex work\n    let cleaned = line;\n    if (cleaned.includes('[[')) cleaned = cleaned.replace(WIKILINK_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('](')) cleaned = cleaned.replace(LINK_DEST_RE, (m) => ' '.repeat(m.length));\n    if (cleaned.includes('<')) cleaned = cleaned.replace(HTML_TAG_RE, (m) => ' '.repeat(m.length));\n    for (const m of cleaned.matchAll(INLINE_TAG_RE)) {\n      const tag = normalizeTag(m[1]);\n      if (tag) found.push(tag);\n    }\n  }\n  return found;\n}\n\nfunction extract(_raw: string, body: string, _search?: { title: string; summary: string }, data?: Record<string, unknown>): string[] {\n  return [...new Set([...frontmatterTags(data), ...inlineTags(body)])].sort();\n}\n\nexport const tags: Feature = {\n  name: 'tags',\n  async schema(db) {\n    await db.exec('CREATE TABLE IF NOT EXISTS tags (\"path\" TEXT, tag TEXT, PRIMARY KEY (\"path\", tag))');\n    await db.exec('CREATE INDEX IF NOT EXISTS tags_tag ON tags(tag)');\n  },\n  extract,\n  // Blanket clear-then-reinsert (store() always writes the full found list back), so remove()\n  // covers vanished and reparsed files without per-file NOT IN diffing, which cannot be one statement shared across files.\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM tags WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) for (const tag of extracted as string[]) rows.push([path, tag]);\n    // OR IGNORE guards the same concurrent-reconcile race sections.ts documents; remove() cleared\n    // every touched path above, so a store with an append path needs no guard.\n    await appendRows(db, 'tags', ['path', 'tag'], 'INSERT OR IGNORE INTO tags (\"path\", tag) VALUES (?, ?)', rows);\n  },\n};\n"],"names":["appendRows","maskRegions","WIKILINK_RE","HTML_TAG_RE","INLINE_TAG_RE","LINK_DEST_RE","normalizeTag","raw","stripped","replace","test","frontmatterTags","data","tags","list","Array","isArray","found","item","tag","push","inlineTags","body","line","split","includes","cleaned","m","repeat","length","matchAll","extract","_raw","_search","Set","sort","name","schema","db","exec","remove","paths","runBatch","map","p","store","docs","rows","path","extracted"],"mappings":"AAAA,SAASA,UAAU,QAAQ,qBAAqB;AAChD,SAASC,WAAW,QAAQ,cAAc;AAG1C,0FAA0F;AAC1F,kJAAkJ;AAElJ,+DAA+D;AAC/D,MAAMC,cAAc,gBAAgB,6DAA6D;AACjG,iDAAiD;AACjD,MAAMC,cAAc,uBAAuB,6DAA6D;AACxG,iGAAiG;AACjG,MAAMC,gBAAgB;AACtB,4FAA4F;AAC5F,MAAMC,eAAe,gCAAgC,oFAAoF;AAEzI,sFAAsF;AACtF,oEAAoE;AACpE,SAASC,aAAaC,GAAW;IAC/B,MAAMC,WAAWD,IAAIE,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,QAAQ;IACvD,IAAI,CAACD,YAAY,QAAQE,IAAI,CAACF,WAAW,OAAO;IAChD,OAAOA;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASG,gBAAgBC,IAA8B;IACrD,MAAML,MAAMK,iBAAAA,2BAAAA,KAAMC,IAAI;IACtB,MAAMC,OAAOC,MAAMC,OAAO,CAACT,OAAOA,MAAM,OAAOA,QAAQ,WAAW;QAACA;KAAI,GAAG,EAAE;IAC5E,MAAMU,QAAkB,EAAE;IAC1B,KAAK,MAAMC,QAAQJ,KAAM;QACvB,IAAI,OAAOI,SAAS,UAAU;QAC9B,MAAMC,MAAMb,aAAaY;QACzB,IAAIC,KAAKF,MAAMG,IAAI,CAACD;IACtB;IACA,OAAOF;AACT;AAEA,4FAA4F;AAC5F,6EAA6E;AAC7E,SAASI,WAAWC,IAAY;IAC9B,MAAML,QAAkB,EAAE;IAC1B,KAAK,MAAMM,QAAQtB,YAAYqB,MAAME,KAAK,CAAC,MAAO;QAChD,IAAI,CAACD,KAAKE,QAAQ,CAAC,MAAM,UAAU,kCAAkC;QACrE,IAAIC,UAAUH;QACd,IAAIG,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACP,aAAa,CAACyB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC7F,IAAIH,QAAQD,QAAQ,CAAC,OAAOC,UAAUA,QAAQjB,OAAO,CAACJ,cAAc,CAACsB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC9F,IAAIH,QAAQD,QAAQ,CAAC,MAAMC,UAAUA,QAAQjB,OAAO,CAACN,aAAa,CAACwB,IAAM,IAAIC,MAAM,CAACD,EAAEE,MAAM;QAC5F,KAAK,MAAMF,KAAKD,QAAQI,QAAQ,CAAC1B,eAAgB;YAC/C,MAAMe,MAAMb,aAAaqB,CAAC,CAAC,EAAE;YAC7B,IAAIR,KAAKF,MAAMG,IAAI,CAACD;QACtB;IACF;IACA,OAAOF;AACT;AAEA,SAASc,QAAQC,IAAY,EAAEV,IAAY,EAAEW,OAA4C,EAAErB,IAA8B;IACvH,OAAO;WAAI,IAAIsB,IAAI;eAAIvB,gBAAgBC;eAAUS,WAAWC;SAAM;KAAE,CAACa,IAAI;AAC3E;AAEA,OAAO,MAAMtB,OAAgB;IAC3BuB,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC;QACd,MAAMD,GAAGC,IAAI,CAAC;IAChB;IACAR;IACA,4FAA4F;IAC5F,yHAAyH;IACzH,MAAMS,QAAOF,EAAE,EAAEG,KAAK;QACpB,IAAIA,MAAMZ,MAAM,KAAK,GAAG;QACxB,MAAMS,GAAGI,QAAQ,CACf,qCACAD,MAAME,GAAG,CAAC,CAACC,IAAM;gBAACA;aAAE;IAExB;IACA,MAAMC,OAAMP,EAAE,EAAEQ,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM,KAAK,MAAM3B,OAAO8B,UAAuBF,KAAK3B,IAAI,CAAC;YAAC4B;YAAM7B;SAAI;QACtG,8FAA8F;QAC9F,2EAA2E;QAC3E,MAAMnB,WAAWsC,IAAI,QAAQ;YAAC;YAAQ;SAAM,EAAE,0DAA0DS;IAC1G;AACF,EAAE"}

package/dist/esm/store/duckdb/connection.js

@@ -1,5 +1,7 @@
+import { quoteIdent } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { rewriteBatch } from './batch.js';
+import { duckdbApi } from './native.js';
 // getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small
 // ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.
 const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
@@ -66,6 +68,39 @@ export function createConnection(duckdb) {
         async prepare (sql) {
             return new DuckdbStatement(await duckdb.prepare(sql));
         },
+        // The appender writes columnar vectors with no per-parameter binding, which is the cost
+        // runBatch pays: measured 7.2x on frontmatter at 6,566 rows. Alignment is against the table's
+        // own physical column order, read fresh, so a column the caller does not write (a
+        // feature-owned "_rank", say) takes appendDefault() rather than shifting every value one slot.
+        async appendRows (table, columns, rows) {
+            if (rows.length === 0) return;
+            const { variantValue } = await duckdbApi();
+            const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);
+            const physical = await infoStmt.all();
+            const rowIndexOf = new Map(columns.map((name, i)=>[
+                    name,
+                    i
+                ]));
+            await withTransaction(conn, async ()=>{
+                const appender = await duckdb.createAppender(table);
+                try {
+                    for (const row of rows){
+                        for (const column of physical){
+                            const idx = rowIndexOf.get(column.name);
+                            const value = idx === undefined ? undefined : row[idx];
+                            if (idx === undefined) appender.appendDefault();
+                            else if (value === null || value === undefined) appender.appendNull();
+                            else if (column.type === 'VARIANT') appender.appendVariant(variantValue(value));
+                            else appender.appendValue(value);
+                        }
+                        appender.endRow();
+                    }
+                    appender.flushSync();
+                } finally{
+                    appender.closeSync();
+                }
+            });
+        },
         // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),
         // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.
         async runBatch (sql, paramRows) {

package/dist/esm/store/duckdb/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,gGAAgG;AAChG,yGAAyG;AACzG,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAQA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAyB;QAC7BD;QACA,MAAME,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,oHAAoH;QACpH,qIAAqI;QACrI,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMrB,gBAAgB8C,MAAM;gBAC1B,MAAMM,YAAYnD,aAAa+C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK/B,IAAI,CAAC6B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKpC,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMoC,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO0C,UAAW;wBAC3BE,KAAK/B,IAAI,CAACb;wBACV,MAAM4C,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKpC,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // The appender writes columnar vectors with no per-parameter binding, which is the cost\n    // runBatch pays: measured 7.2x on frontmatter at 6,566 rows. Alignment is against the table's\n    // own physical column order, read fresh, so a column the caller does not write (a\n    // feature-owned \"_rank\", say) takes appendDefault() rather than shifting every value one slot.\n    async appendRows(table: string, columns: string[], rows: unknown[][]): Promise<void> {\n      if (rows.length === 0) return;\n      const { variantValue } = await duckdbApi();\n      const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);\n      const physical = (await infoStmt.all()) as Array<{ name: string; type: string }>;\n      const rowIndexOf = new Map(columns.map((name, i) => [name, i]));\n\n      await withTransaction(conn, async () => {\n        const appender = await duckdb.createAppender(table);\n        try {\n          for (const row of rows) {\n            for (const column of physical) {\n              const idx = rowIndexOf.get(column.name);\n              const value = idx === undefined ? undefined : row[idx];\n              if (idx === undefined) appender.appendDefault();\n              else if (value === null || value === undefined) appender.appendNull();\n              // VARIANT is frontmatter's dynamic columns only; every other table is statically\n              // typed, and appendValue is what preserves those types.\n              else if (column.type === 'VARIANT') appender.appendVariant(variantValue(value as DuckDBValue));\n              else appender.appendValue(value as DuckDBValue);\n            }\n            appender.endRow();\n          }\n          appender.flushSync();\n        } finally {\n          appender.closeSync();\n        }\n      });\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["quoteIdent","withTransaction","rewriteBatch","duckdbApi","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","appendRows","table","variantValue","infoStmt","physical","rowIndexOf","Map","appender","createAppender","column","idx","undefined","appendDefault","appendNull","type","appendVariant","appendValue","endRow","flushSync","closeSync","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAC1C,SAASC,SAAS,QAAQ,cAAc;AAExC,gGAAgG;AAChG,yGAAyG;AACzG,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAQA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAyB;QAC7BD;QACA,MAAME,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,wFAAwF;QACxF,8FAA8F;QAC9F,kFAAkF;QAClF,+FAA+F;QAC/F,MAAME,YAAWC,KAAa,EAAEhB,OAAiB,EAAEP,IAAiB;YAClE,IAAIA,KAAKP,MAAM,KAAK,GAAG;YACvB,MAAM,EAAE+B,YAAY,EAAE,GAAG,MAAMnD;YAC/B,MAAMoD,WAAW,MAAMP,KAAKG,OAAO,CAAC,CAAC,kBAAkB,EAAEnD,WAAWqD,OAAO,CAAC,CAAC;YAC7E,MAAMG,WAAY,MAAMD,SAASxB,GAAG;YACpC,MAAM0B,aAAa,IAAIC,IAAIrB,QAAQF,GAAG,CAAC,CAACM,MAAMH,IAAM;oBAACG;oBAAMH;iBAAE;YAE7D,MAAMrC,gBAAgB+C,MAAM;gBAC1B,MAAMW,WAAW,MAAMZ,OAAOa,cAAc,CAACP;gBAC7C,IAAI;oBACF,KAAK,MAAM1C,OAAOmB,KAAM;wBACtB,KAAK,MAAM+B,UAAUL,SAAU;4BAC7B,MAAMM,MAAML,WAAW5B,GAAG,CAACgC,OAAOpB,IAAI;4BACtC,MAAMhC,QAAQqD,QAAQC,YAAYA,YAAYpD,GAAG,CAACmD,IAAI;4BACtD,IAAIA,QAAQC,WAAWJ,SAASK,aAAa;iCACxC,IAAIvD,UAAU,QAAQA,UAAUsD,WAAWJ,SAASM,UAAU;iCAG9D,IAAIJ,OAAOK,IAAI,KAAK,WAAWP,SAASQ,aAAa,CAACb,aAAa7C;iCACnEkD,SAASS,WAAW,CAAC3D;wBAC5B;wBACAkD,SAASU,MAAM;oBACjB;oBACAV,SAASW,SAAS;gBACpB,SAAU;oBACRX,SAASY,SAAS;gBACpB;YACF;QACF;QACA,oHAAoH;QACpH,qIAAqI;QACrI,MAAMC,UAAStB,GAAW,EAAEuB,SAAsB;YAChD,IAAIA,UAAUlD,MAAM,KAAK,GAAG;YAC5B,MAAMtB,gBAAgB+C,MAAM;gBAC1B,MAAM0B,YAAYxE,aAAagD,KAAKuB,UAAUlD,MAAM;gBACpD,IAAImD,WAAW;oBACb,MAAMC,OAAO,MAAM5B,OAAOI,OAAO,CAACuB,UAAUxB,GAAG;oBAC/C,IAAI;wBACFyB,KAAKnD,IAAI,CAACiD,UAAUG,IAAI;wBACxB,MAAMD,KAAKtD,GAAG;oBAChB,SAAU;wBACRsD,KAAKxD,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMwD,OAAO,MAAM5B,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO8D,UAAW;wBAC3BE,KAAKnD,IAAI,CAACb;wBACV,MAAMgE,KAAKtD,GAAG;oBAChB;gBACF,SAAU;oBACRsD,KAAKxD,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}

package/dist/esm/store/duckdb/reconcile.js

@@ -1,15 +1,19 @@
 import { SenseError } from '../../errors.js';
-import { CORE_FRONTMATTER_COLUMNS } from '../reconcile.js';
-import { quoteIdent } from '../shared.js';
+import { appendRows, quoteIdent } from '../shared.js';
 import { markContentStale } from './lexical.js';
-import { duckdbApi } from './native.js';
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
 // store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here
 // unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter
 // column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
 // No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):
 // `path` is content's own primary key, so this is a plain per-doc row.
-const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)`;
+const CONTENT_COLUMNS = [
+    'path',
+    'title',
+    'summary',
+    'text'
+];
+const INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;
 function contentRow(doc) {
     return [
         doc.relPath,
@@ -25,7 +29,9 @@ async function reconcileContent(conn, touched, docs, _delta) {
     if (touched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', touched.map((p)=>[
             p
         ]));
-    if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));
+    // The delete above cleared every touched path, and duckdb holds the cache file for its whole
+    // connection, so no second writer can have landed one of these rows: nothing can conflict.
+    await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));
     // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
     // not here (lexical.ts's FtsIndexState).
     markContentStale(conn);
@@ -39,36 +45,6 @@ async function addColumns(conn, names) {
     if (names.length === 0) return;
     await conn.exec(names.map((name)=>`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));
 }
-// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so
-// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. "_rank") that still exist on the table, and get appendDefault().
-async function insertNew(conn, table, columns, rows) {
-    if (rows.length === 0) return;
-    const { variantValue } = await duckdbApi();
-    const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);
-    const physicalColumns = (await infoStmt.all()).map((c)=>c.name);
-    const rowIndexOf = new Map(columns.map((name, i)=>[
-            name,
-            i
-        ]));
-    const native = conn.duckdb;
-    const appender = await native.createAppender(table);
-    try {
-        for (const row of rows){
-            for (const name of physicalColumns){
-                const idx = rowIndexOf.get(name);
-                const value = idx === undefined ? undefined : row[idx];
-                if (idx === undefined) appender.appendDefault();
-                else if (value === null || value === undefined) appender.appendNull();
-                else if (CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value);
-                else appender.appendVariant(variantValue(value));
-            }
-            appender.endRow();
-        }
-        appender.flushSync();
-    } finally{
-        appender.closeSync();
-    }
-}
 export const duckdbDialect = {
     beginMode: ()=>'BEGIN',
     checkColumnLimit (count) {
@@ -77,6 +53,5 @@ export const duckdbDialect = {
         }
     },
     addColumns,
-    reconcileContent,
-    insertNew
+    reconcileContent
 };