sensemaking 0.22.0 → 0.22.1

package/dist/cjs/store/duckdb/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["createConnection","storeRowToJs","storeValueToJs","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","value","row","out","Object","entries","key","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","register","run","params","result","length","bind","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","duckdb","conn","exec","sql","prepare","runBatch","paramRows","withTransaction","rewritten","stmt","rewriteBatch","flat"],"mappings":";;;;;;;;;;;QA0EgBA;eAAAA;;QA/DAC;eAAAA;;QAHAC;eAAAA;;;6BAPgB;uBAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7B,gGAAgG;AAChG,yGAAyG;AACzG,IAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AACxC,SAASJ,eAAeK,KAAc;IAC3C,OAAO,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAYA,SAAS,CAACJ,YAAYI,SAASJ,WAAWE,OAAOE,SAASA;AAChG;AACO,SAASN,aAAaO,GAA4B;IACvD,IAAMC,MAA+B,CAAC;QACjC,kCAAA,2BAAA;;QAAL,QAAK,YAAsBC,OAAOC,OAAO,CAACH,yBAArC,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOI,sBAAKL;YAA+BE,GAAG,CAACG,IAAI,GAAGV,eAAeK;;;QAArE;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOE;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,IAAMI,oBAAoB,IAAIC,qBAA8C,SAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,gCAAN;;aAAMA,gBAGQF,QAAiC;gCAHzCE;QAIF,IAAI,CAACF,QAAQ,GAAGA;QAChBF,kBAAkBK,QAAQ,CAAC,IAAI,EAAEH;;iBAL/BE;IAQJ,OAAME,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBC;;;;wBADN,IAAID,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACI,GAAG;;;wBAAhCE,SAAS;wBACf;;4BAAO;gCAAEG,SAASH,OAAOI,WAAW;gCAAEC,iBAAiB;4BAAE;;;;QAC3D;;IAEA,OAAMC,GAGL,GAHD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGP,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACtBQ;;;;wBAAO;;4BAAM,IAAI,CAACC,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAAzBQ,OAAO;wBACb;;4BAAOA,IAAI,CAAC,EAAE;;;;QAChB;;IAEA,OAAMC,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGT,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBU;;;;wBADN,IAAIV,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACgB,aAAa;;;wBAA1CD,SAAS;wBACf;;4BAAQA,OAAOE,eAAe,GAAsCC,GAAG,CAAChC;;;;QAC1E;;IAEA,OAAOiC,OAIN,GAJD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGd,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;;;;wBAG1B;;4BAAA,uBAAM,IAAI,CAACS,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAFzB,4FAA4F;wBAC5F,+DAA+D;wBAC/D;;uCAAA;;oCAAO;;;;;wBAAP;;;;;;QACF;;IAEAe,OAAAA,OAIC,GAJDA,SAAAA;QACE,IAAM1B,MAA+B,EAAE;QACvC,IAAK,IAAI2B,IAAI,GAAGA,IAAI,IAAI,CAACrB,QAAQ,CAACsB,WAAW,EAAED,IAAK3B,IAAI6B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACxB,QAAQ,CAACyB,UAAU,CAACJ;QAAG;QACjG,OAAO3B;IACT;IAEAgC,OAAAA,cAGC,GAHDA,SAAAA,eAAeC,QAAiB;IAC9B,4FAA4F;IAC5F,uFAAuF;IACzF;WAxCIzB;;AAiDC,SAASjB,iBAAiB2C,MAAwB;IACvD,IAAMC,OAAyB;QAC7BD,QAAAA;QACME,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMH,OAAOxB,GAAG,CAAC2B;;;4BAAjB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZ7B;4BAAgB;;gCAAM0B,OAAOI,OAAO,CAACD;;;4BAAhD;;gCAAO,IAAA,CAAA,EAAA,MAAI7B;;oCAAgB;kCAAyB;;;;YACtD;;QAGM+B,UAFN,oHAAoH;QACpH,qIAAqI;QACrI,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAU3B,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAM4B,IAAAA,8BAAe,EAACN,MAAM;;4CACpBO,WAEEC,MASFA,OAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAM5C;;;;oDAbP2C,YAAYE,IAAAA,qBAAY,EAACP,KAAKG,UAAU3B,MAAM;yDAChD6B,WAAAA;;;;oDACW;;wDAAMR,OAAOI,OAAO,CAACI,UAAUL,GAAG;;;oDAAzCM,OAAO;;;;;;;;;oDAEXA,KAAK7B,IAAI,CAAC0B,UAAUK,IAAI;oDACxB;;wDAAMF,KAAKjC,GAAG;;;oDAAd;;;;;;oDAEAiC,KAAKpC,WAAW;;;;;oDAElB;;;;oDAEW;;wDAAM2B,OAAOI,OAAO,CAACD;;;oDAA5BM,QAAO;;;;;;;;;oDAEN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaH;;;2DAAb,6BAAA,QAAA;;;;oDAAMzC,MAAN;oDACH4C,MAAK7B,IAAI,CAACf;oDACV;;wDAAM4C,MAAKjC,GAAG;;;oDAAd;;;oDAFG;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAKLiC,MAAKpC,WAAW;;;;;;;;;;oCAEpB;;;;4BArBA;;;;;;YAsBF;;IACF;IACA,OAAO4B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // The appender writes columnar vectors with no per-parameter binding, which is the cost\n    // runBatch pays: measured 7.2x on frontmatter at 6,566 rows. Alignment is against the table's\n    // own physical column order, read fresh, so a column the caller does not write (a\n    // feature-owned \"_rank\", say) takes appendDefault() rather than shifting every value one slot.\n    async appendRows(table: string, columns: string[], rows: unknown[][]): Promise<void> {\n      if (rows.length === 0) return;\n      const { variantValue } = await duckdbApi();\n      const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);\n      const physical = (await infoStmt.all()) as Array<{ name: string; type: string }>;\n      const rowIndexOf = new Map(columns.map((name, i) => [name, i]));\n\n      await withTransaction(conn, async () => {\n        const appender = await duckdb.createAppender(table);\n        try {\n          for (const row of rows) {\n            for (const column of physical) {\n              const idx = rowIndexOf.get(column.name);\n              const value = idx === undefined ? undefined : row[idx];\n              if (idx === undefined) appender.appendDefault();\n              else if (value === null || value === undefined) appender.appendNull();\n              // VARIANT is frontmatter's dynamic columns only; every other table is statically\n              // typed, and appendValue is what preserves those types.\n              else if (column.type === 'VARIANT') appender.appendVariant(variantValue(value as DuckDBValue));\n              else appender.appendValue(value as DuckDBValue);\n            }\n            appender.endRow();\n          }\n          appender.flushSync();\n        } finally {\n          appender.closeSync();\n        }\n      });\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["createConnection","storeRowToJs","storeValueToJs","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","value","row","out","Object","entries","key","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","register","run","params","result","length","bind","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","duckdb","conn","exec","sql","prepare","appendRows","table","variantValue","infoStmt","physical","rowIndexOf","duckdbApi","quoteIdent","Map","withTransaction","appender","column","idx","createAppender","undefined","appendDefault","appendNull","type","appendVariant","appendValue","endRow","flushSync","closeSync","runBatch","paramRows","rewritten","stmt","rewriteBatch","flat"],"mappings":";;;;;;;;;;;QA4EgBA;eAAAA;;QA/DAC;eAAAA;;QAHAC;eAAAA;;;wBATW;6BACK;uBAEH;wBACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,gGAAgG;AAChG,yGAAyG;AACzG,IAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AACxC,SAASJ,eAAeK,KAAc;IAC3C,OAAO,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAYA,SAAS,CAACJ,YAAYI,SAASJ,WAAWE,OAAOE,SAASA;AAChG;AACO,SAASN,aAAaO,GAA4B;IACvD,IAAMC,MAA+B,CAAC;QACjC,kCAAA,2BAAA;;QAAL,QAAK,YAAsBC,OAAOC,OAAO,CAACH,yBAArC,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOI,sBAAKL;YAA+BE,GAAG,CAACG,IAAI,GAAGV,eAAeK;;;QAArE;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOE;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,IAAMI,oBAAoB,IAAIC,qBAA8C,SAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,gCAAN;;aAAMA,gBAGQF,QAAiC;gCAHzCE;QAIF,IAAI,CAACF,QAAQ,GAAGA;QAChBF,kBAAkBK,QAAQ,CAAC,IAAI,EAAEH;;iBAL/BE;IAQJ,OAAME,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBC;;;;wBADN,IAAID,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACI,GAAG;;;wBAAhCE,SAAS;wBACf;;4BAAO;gCAAEG,SAASH,OAAOI,WAAW;gCAAEC,iBAAiB;4BAAE;;;;QAC3D;;IAEA,OAAMC,GAGL,GAHD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGP,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACtBQ;;;;wBAAO;;4BAAM,IAAI,CAACC,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAAzBQ,OAAO;wBACb;;4BAAOA,IAAI,CAAC,EAAE;;;;QAChB;;IAEA,OAAMC,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGT,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBU;;;;wBADN,IAAIV,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACgB,aAAa;;;wBAA1CD,SAAS;wBACf;;4BAAQA,OAAOE,eAAe,GAAsCC,GAAG,CAAChC;;;;QAC1E;;IAEA,OAAOiC,OAIN,GAJD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGd,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;;;;wBAG1B;;4BAAA,uBAAM,IAAI,CAACS,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAFzB,4FAA4F;wBAC5F,+DAA+D;wBAC/D;;uCAAA;;oCAAO;;;;;wBAAP;;;;;;QACF;;IAEAe,OAAAA,OAIC,GAJDA,SAAAA;QACE,IAAM1B,MAA+B,EAAE;QACvC,IAAK,IAAI2B,IAAI,GAAGA,IAAI,IAAI,CAACrB,QAAQ,CAACsB,WAAW,EAAED,IAAK3B,IAAI6B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACxB,QAAQ,CAACyB,UAAU,CAACJ;QAAG;QACjG,OAAO3B;IACT;IAEAgC,OAAAA,cAGC,GAHDA,SAAAA,eAAeC,QAAiB;IAC9B,4FAA4F;IAC5F,uFAAuF;IACzF;WAxCIzB;;AAiDC,SAASjB,iBAAiB2C,MAAwB;IACvD,IAAMC,OAAyB;QAC7BD,QAAAA;QACME,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMH,OAAOxB,GAAG,CAAC2B;;;4BAAjB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZ7B;4BAAgB;;gCAAM0B,OAAOI,OAAO,CAACD;;;4BAAhD;;gCAAO,IAAA,CAAA,EAAA,MAAI7B;;oCAAgB;kCAAyB;;;;YACtD;;QAKM+B,YAJN,wFAAwF;QACxF,8FAA8F;QAC9F,kFAAkF;QAClF,+FAA+F;QAC/F,SAAMA,WAAWC,KAAa,EAAEd,OAAiB,EAAEP,IAAiB;;oBAE1DsB,cACFC,UACAC,UACAC;;;;4BAJN,IAAIzB,KAAKN,MAAM,KAAK,GAAG;;;4BACE;;gCAAMgC,IAAAA,mBAAS;;;4BAAhCJ,eAAiB,cAAjBA;4BACS;;gCAAMN,KAAKG,OAAO,CAAC,AAAC,qBAAsC,OAAlBQ,IAAAA,oBAAU,EAACN,QAAO;;;4BAArEE,WAAW;4BACC;;gCAAMA,SAAStB,GAAG;;;4BAA9BuB,WAAY;4BACZC,aAAa,IAAIG,IAAIrB,QAAQF,GAAG,CAAC,SAACM,MAAMH;uCAAM;oCAACG;oCAAMH;iCAAE;;4BAE7D;;gCAAMqB,IAAAA,8BAAe,EAACb,MAAM;;4CACpBc,UAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMlD,KACJ,4BAAA,oBAAA,iBAAA,YAAA,QAAMmD,QACHC,KACArD;;;;oDALK;;wDAAMoC,OAAOkB,cAAc,CAACZ;;;oDAAvCS,WAAW;oDACjB,IAAI;wDACG,kCAAA,2BAAA;;4DAAL,IAAK,YAAa9B,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;gEAAbpB,MAAN;gEACE,mCAAA,4BAAA;;oEAAL,IAAK,aAAgB4C,+BAAhB,8BAAA,SAAA,0BAAA,kCAA0B;wEAApBO,SAAN;wEACGC,MAAMP,WAAW1B,GAAG,CAACgC,OAAOpB,IAAI;wEAChChC,QAAQqD,QAAQE,YAAYA,YAAYtD,GAAG,CAACoD,IAAI;wEACtD,IAAIA,QAAQE,WAAWJ,SAASK,aAAa;6EACxC,IAAIxD,UAAU,QAAQA,UAAUuD,WAAWJ,SAASM,UAAU;6EAG9D,IAAIL,OAAOM,IAAI,KAAK,WAAWP,SAASQ,aAAa,CAAChB,aAAa3C;6EACnEmD,SAASS,WAAW,CAAC5D;oEAC5B;;oEATK;oEAAA;;;6EAAA,8BAAA;4EAAA;;;4EAAA;kFAAA;;;;gEAULmD,SAASU,MAAM;4DACjB;;4DAZK;4DAAA;;;qEAAA,6BAAA;oEAAA;;;oEAAA;0EAAA;;;;wDAaLV,SAASW,SAAS;oDACpB,SAAU;wDACRX,SAASY,SAAS;oDACpB;;;;;;oCACF;;;;4BApBA;;;;;;YAqBF;;QAGMC,UAFN,oHAAoH;QACpH,qIAAqI;QACrI,SAAMA,SAASzB,GAAW,EAAE0B,SAAsB;;;;;4BAChD,IAAIA,UAAUlD,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMmC,IAAAA,8BAAe,EAACb,MAAM;;4CACpB6B,WAEEC,MASFA,OAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMlE;;;;oDAbPiE,YAAYE,IAAAA,qBAAY,EAAC7B,KAAK0B,UAAUlD,MAAM;yDAChDmD,WAAAA;;;;oDACW;;wDAAM9B,OAAOI,OAAO,CAAC0B,UAAU3B,GAAG;;;oDAAzC4B,OAAO;;;;;;;;;oDAEXA,KAAKnD,IAAI,CAACiD,UAAUI,IAAI;oDACxB;;wDAAMF,KAAKvD,GAAG;;;oDAAd;;;;;;oDAEAuD,KAAK1D,WAAW;;;;;oDAElB;;;;oDAEW;;wDAAM2B,OAAOI,OAAO,CAACD;;;oDAA5B4B,QAAO;;;;;;;;;oDAEN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaF;;;2DAAb,6BAAA,QAAA;;;;oDAAMhE,MAAN;oDACHkE,MAAKnD,IAAI,CAACf;oDACV;;wDAAMkE,MAAKvD,GAAG;;;oDAAd;;;oDAFG;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAKLuD,MAAK1D,WAAW;;;;;;;;;;oCAEpB;;;;4BArBA;;;;;;YAsBF;;IACF;IACA,OAAO4B;AACT"}

package/dist/cjs/store/duckdb/reconcile.js

@@ -9,10 +9,8 @@ Object.defineProperty(exports, "duckdbDialect", {
     }
 });
 var _errorsts = require("../../errors.js");
-var _reconcilets = require("../reconcile.js");
 var _sharedts = require("../shared.js");
 var _lexicalts = require("./lexical.js");
-var _nativets = require("./native.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -144,7 +142,13 @@ function _ts_generator(thisArg, body) {
 // column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
 // No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):
 // `path` is content's own primary key, so this is a plain per-doc row.
-var INSERT_CONTENT_SQL = 'INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)';
+var CONTENT_COLUMNS = [
+    'path',
+    'title',
+    'summary',
+    'text'
+];
+var INSERT_CONTENT_SQL = "INSERT INTO content (".concat(CONTENT_COLUMNS.map(_sharedts.quoteIdent).join(', '), ") VALUES (?, ?, ?, ?)");
 function contentRow(doc) {
     return [
         doc.relPath,
@@ -177,18 +181,14 @@ function reconcileContent(conn, touched, docs, _delta) {
                     _state.sent();
                     _state.label = 2;
                 case 2:
-                    if (!(docs.length > 0)) return [
-                        3,
-                        4
-                    ];
+                    // The delete above cleared every touched path, and duckdb holds the cache file for its whole
+                    // connection, so no second writer can have landed one of these rows: nothing can conflict.
                     return [
                         4,
-                        conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow))
+                        (0, _sharedts.appendRows)(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow))
                     ];
                 case 3:
                     _state.sent();
-                    _state.label = 4;
-                case 4:
                     // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
                     // not here (lexical.ts's FtsIndexState).
                     (0, _lexicalts.markContentStale)(conn);
@@ -227,107 +227,6 @@ function addColumns(conn, names) {
         });
     })();
 }
-// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so
-// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. "_rank") that still exist on the table, and get appendDefault().
-function insertNew(conn, table, columns, rows) {
-    return _async_to_generator(function() {
-        var variantValue, infoStmt, physicalColumns, rowIndexOf, native, appender, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, name, idx, value;
-        return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    if (rows.length === 0) return [
-                        2
-                    ];
-                    return [
-                        4,
-                        (0, _nativets.duckdbApi)()
-                    ];
-                case 1:
-                    variantValue = _state.sent().variantValue;
-                    return [
-                        4,
-                        conn.prepare("PRAGMA table_info(".concat((0, _sharedts.quoteIdent)(table), ")"))
-                    ];
-                case 2:
-                    infoStmt = _state.sent();
-                    return [
-                        4,
-                        infoStmt.all()
-                    ];
-                case 3:
-                    physicalColumns = _state.sent().map(function(c) {
-                        return c.name;
-                    });
-                    rowIndexOf = new Map(columns.map(function(name, i) {
-                        return [
-                            name,
-                            i
-                        ];
-                    }));
-                    native = conn.duckdb;
-                    return [
-                        4,
-                        native.createAppender(table)
-                    ];
-                case 4:
-                    appender = _state.sent();
-                    try {
-                        _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                        try {
-                            for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
-                                row = _step.value;
-                                _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
-                                try {
-                                    for(_iterator1 = physicalColumns[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
-                                        name = _step1.value;
-                                        idx = rowIndexOf.get(name);
-                                        value = idx === undefined ? undefined : row[idx];
-                                        if (idx === undefined) appender.appendDefault();
-                                        else if (value === null || value === undefined) appender.appendNull();
-                                        else if (_reconcilets.CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value);
-                                        else appender.appendVariant(variantValue(value));
-                                    }
-                                } catch (err) {
-                                    _didIteratorError1 = true;
-                                    _iteratorError1 = err;
-                                } finally{
-                                    try {
-                                        if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
-                                            _iterator1.return();
-                                        }
-                                    } finally{
-                                        if (_didIteratorError1) {
-                                            throw _iteratorError1;
-                                        }
-                                    }
-                                }
-                                appender.endRow();
-                            }
-                        } catch (err) {
-                            _didIteratorError = true;
-                            _iteratorError = err;
-                        } finally{
-                            try {
-                                if (!_iteratorNormalCompletion && _iterator.return != null) {
-                                    _iterator.return();
-                                }
-                            } finally{
-                                if (_didIteratorError) {
-                                    throw _iteratorError;
-                                }
-                            }
-                        }
-                        appender.flushSync();
-                    } finally{
-                        appender.closeSync();
-                    }
-                    return [
-                        2
-                    ];
-            }
-        });
-    })();
-}
 var duckdbDialect = {
     beginMode: function beginMode() {
         return 'BEGIN';
@@ -338,7 +237,6 @@ var duckdbDialect = {
         }
     },
     addColumns: addColumns,
-    reconcileContent: reconcileContent,
-    insertNew: insertNew
+    reconcileContent: reconcileContent
 };
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { DuckDBValue } from '@duckdb/node-api';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { CORE_FRONTMATTER_COLUMNS } from '../reconcile.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport type { DuckdbConnection } from './connection.ts';\nimport { markContentStale } from './lexical.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\n// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so\n// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. \"_rank\") that still exist on the table, and get appendDefault().\nasync function insertNew(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  const { variantValue } = await duckdbApi();\n  const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);\n  const physicalColumns = ((await infoStmt.all()) as Array<{ name: string }>).map((c) => c.name);\n  const rowIndexOf = new Map(columns.map((name, i) => [name, i]));\n\n  const native = (conn as DuckdbConnection).duckdb;\n  const appender = await native.createAppender(table);\n  try {\n    for (const row of rows) {\n      for (const name of physicalColumns) {\n        const idx = rowIndexOf.get(name);\n        const value = idx === undefined ? undefined : row[idx];\n        if (idx === undefined) appender.appendDefault();\n        else if (value === null || value === undefined) appender.appendNull();\n        else if (CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value as DuckDBValue);\n        else appender.appendVariant(variantValue(value as DuckDBValue));\n      }\n      appender.endRow();\n    }\n    appender.flushSync();\n  } finally {\n    appender.closeSync();\n  }\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n  insertNew,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","markContentStale","addColumns","names","exec","name","quoteIdent","join","insertNew","table","columns","rows","variantValue","infoStmt","physicalColumns","rowIndexOf","native","appender","row","idx","value","duckdbApi","prepare","all","c","Map","i","duckdb","createAppender","get","undefined","appendDefault","appendNull","CORE_FRONTMATTER_COLUMNS","has","appendValue","appendVariant","endRow","flushSync","closeSync","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BA+EaA;;;eAAAA;;;wBA9Ec;2BAGc;wBACd;yBAGM;wBACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;oBACrB,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMJ,GAAG,CAAC,SAACM;mCAAS,AAAC,sCAAsD,OAAjBC,IAAAA,oBAAU,EAACD,OAAM;2BAAWE,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEA,mGAAmG;AACnG,uLAAuL;AACvL,SAAeC,UAAUf,IAAgB,EAAEgB,KAAa,EAAEC,OAAiB,EAAEC,IAAiB;;YAEpFC,cACFC,UACAC,iBACAC,YAEAC,QACAC,UAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACJ,4BAAA,oBAAA,iBAAA,YAAA,QAAMb,MACHc,KACAC;;;;oBAZZ,IAAIT,KAAKd,MAAM,KAAK,GAAG;;;oBACE;;wBAAMwB,IAAAA,mBAAS;;;oBAAhCT,eAAiB,cAAjBA;oBACS;;wBAAMnB,KAAK6B,OAAO,CAAC,AAAC,qBAAsC,OAAlBhB,IAAAA,oBAAU,EAACG,QAAO;;;oBAArEI,WAAW;oBACS;;wBAAMA,SAASU,GAAG;;;oBAAtCT,kBAAkB,AAAE,cAAkDf,GAAG,CAAC,SAACyB;+BAAMA,EAAEnB,IAAI;;oBACvFU,aAAa,IAAIU,IAAIf,QAAQX,GAAG,CAAC,SAACM,MAAMqB;+BAAM;4BAACrB;4BAAMqB;yBAAE;;oBAEvDV,SAAS,AAACvB,KAA0BkC,MAAM;oBAC/B;;wBAAMX,OAAOY,cAAc,CAACnB;;;oBAAvCQ,WAAW;oBACjB,IAAI;wBACG,kCAAA,2BAAA;;4BAAL,IAAK,YAAaN,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;gCAAbO,MAAN;gCACE,mCAAA,4BAAA;;oCAAL,IAAK,aAAcJ,sCAAd,8BAAA,SAAA,0BAAA,kCAA+B;wCAAzBT,OAAN;wCACGc,MAAMJ,WAAWc,GAAG,CAACxB;wCACrBe,QAAQD,QAAQW,YAAYA,YAAYZ,GAAG,CAACC,IAAI;wCACtD,IAAIA,QAAQW,WAAWb,SAASc,aAAa;6CACxC,IAAIX,UAAU,QAAQA,UAAUU,WAAWb,SAASe,UAAU;6CAC9D,IAAIC,qCAAwB,CAACC,GAAG,CAAC7B,OAAOY,SAASkB,WAAW,CAACf;6CAC7DH,SAASmB,aAAa,CAACxB,aAAaQ;oCAC3C;;oCAPK;oCAAA;;;6CAAA,8BAAA;4CAAA;;;4CAAA;kDAAA;;;;gCAQLH,SAASoB,MAAM;4BACjB;;4BAVK;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBAWLpB,SAASqB,SAAS;oBACpB,SAAU;wBACRrB,SAASsB,SAAS;oBACpB;;;;;;IACF;;AAEO,IAAMzD,gBAAkC;IAC7C0D,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQnD,yBAAyB;YACnC,MAAM,IAAIoD,oBAAU,CAAC,gBAAgB,AAAC,0BAA+EpD,OAAtDmD,OAAM,kDAAwE,OAAxBnD,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;IACAgB,WAAAA;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","appendRows","markContentStale","addColumns","names","exec","name","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAiDaA;;;eAAAA;;;wBAjDc;wBAGY;yBAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAE9F,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQb,GAAG,CAAC,SAACkB;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,6FAA6F;oBAC7F,2FAA2F;oBAC3F;;wBAAMC,IAAAA,oBAAU,EAACP,MAAM,WAAWd,iBAAiBC,oBAAoBe,KAAKd,GAAG,CAACG;;;oBAAhF;oBACA,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMtB,GAAG,CAAC,SAACwB;mCAAS,AAAC,sCAAsD,OAAjBvB,IAAAA,oBAAU,EAACuB,OAAM;2BAAWtB,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C4B,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjB,yBAAyB;YACnC,MAAM,IAAIkB,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElB,OAAtDiB,OAAM,kDAAwE,OAAxBjB,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;AACF"}

package/dist/cjs/store/reconcile.js

@@ -467,8 +467,8 @@ function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
                                                     return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
                                                 });
                                             };
-                                            // A path in `added` has no existing frontmatter row, so it can never conflict: where a
-                                            // dialect offers a faster append-only path (insertNew), only the rest go through the upsert.
+                                            // A path in `added` has no existing frontmatter row, so it can never conflict; the rest
+                                            // genuinely can, and keep the upsert.
                                             return [
                                                 4,
                                                 stages.time('fm-upsert', function() {
@@ -477,53 +477,30 @@ function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
                                                         return _ts_generator(this, function(_state) {
                                                             switch(_state.label){
                                                                 case 0:
-                                                                    if (!dialect.insertNew) return [
-                                                                        3,
-                                                                        5
-                                                                    ];
                                                                     newDocs = parsedDocs.filter(function(d) {
                                                                         return addedSet.has(d.relPath);
                                                                     });
                                                                     updateDocs = parsedDocs.filter(function(d) {
                                                                         return !addedSet.has(d.relPath);
                                                                     });
-                                                                    if (!(newDocs.length > 0)) return [
-                                                                        3,
-                                                                        2
-                                                                    ];
                                                                     return [
                                                                         4,
-                                                                        dialect.insertNew(conn, 'frontmatter', writableColumns, newDocs.map(toRow))
+                                                                        (0, _sharedts.appendRows)(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow))
                                                                     ];
                                                                 case 1:
                                                                     _state.sent();
-                                                                    _state.label = 2;
-                                                                case 2:
                                                                     if (!(updateDocs.length > 0)) return [
                                                                         3,
-                                                                        4
+                                                                        3
                                                                     ];
                                                                     return [
                                                                         4,
                                                                         conn.runBatch(insertSql, updateDocs.map(toRow))
                                                                     ];
-                                                                case 3:
-                                                                    _state.sent();
-                                                                    _state.label = 4;
-                                                                case 4:
-                                                                    return [
-                                                                        3,
-                                                                        7
-                                                                    ];
-                                                                case 5:
-                                                                    return [
-                                                                        4,
-                                                                        conn.runBatch(insertSql, parsedDocs.map(toRow))
-                                                                    ];
-                                                                case 6:
+                                                                case 2:
                                                                     _state.sent();
-                                                                    _state.label = 7;
-                                                                case 7:
+                                                                    _state.label = 3;
+                                                                case 3:
                                                                     return [
                                                                         2
                                                                     ];
@@ -614,18 +591,18 @@ function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths) {
                                                                             }
                                                                         }
                                                                     }
-                                                                    if (!(presetRows.length > 0)) return [
-                                                                        3,
-                                                                        4
-                                                                    ];
+                                                                    // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken
+                                                                    // before this transaction's lock, so a path this process calls "added" can already have
+                                                                    // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.
                                                                     return [
                                                                         4,
-                                                                        conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows)
+                                                                        (0, _sharedts.appendRows)(conn, 'preset_files', [
+                                                                            'path',
+                                                                            'preset'
+                                                                        ], 'INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows)
                                                                     ];
                                                                 case 3:
                                                                     _state.sent();
-                                                                    _state.label = 4;
-                                                                case 4:
                                                                     return [
                                                                         2
                                                                     ];

package/dist/cjs/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict: where a\n        // dialect offers a faster append-only path (insertNew), only the rest go through the upsert.\n        await stages.time('fm-upsert', async () => {\n          if (dialect.insertNew) {\n            const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n            const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n            if (newDocs.length > 0) await dialect.insertNew(conn, 'frontmatter', writableColumns, newDocs.map(toRow));\n            if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n          } else {\n            await conn.runBatch(insertSql, parsedDocs.map(toRow));\n          }\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["CORE_FRONTMATTER_COLUMNS","reconcile","Set","conn","cfg","baseDir","dialect","pool","forcedPaths","start","features","stages","elapsed","files","currentSet","existingRows","existing","vanished","toReparse","seenColumns","report","parsedDocs","warnings","newColumns","workerParseMs","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","touched","txStart","durationMs","process","hrtime","bigint","activeFeatures","stageRecorder","map","f","name","Number","time","listFiles","relPath","existingStmt","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","take","getColumns","progress","reparseFiles","tick","docs","finish","add","checkColumnLimit","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","present","missingColumns","contentTouched","toRow","addColumns","currentPathsStmt","currentPaths","staleAdded","doc","ctimeMs","parseError","data","newDocs","updateDocs","insertNew","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","featureStage","remove","docsForFeature","extracted","store","afterReconcile","beginMode","recordLockWaitMs","recordDuration"],"mappings":";;;;;;;;;;;QAqBaA;eAAAA;;QAESC;eAAAA;;;uBAtBS;0BAEN;wBACmB;yBAEf;0BACI;wBACM;wBACkB;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAWzB,IAAMD,2BAA2B,IAAIE,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,SAAeD,UAAUE,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;;YACtJC,OACAC,UACAC,QACAC,SACAC,OACAC,YAEAC,cAIAC,UAIAC,UAIAC,WAOAC,aAIAC,QAE4D,MAApDC,YAAYC,UAAUC,YAAYC,eAE3C,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAMAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAGAC,SAEAC,SA2FAC;;;;oBArJA1B,QAAQ2B,QAAQC,MAAM,CAACC,MAAM;oBAC7B5B,WAAW6B,IAAAA,uBAAc,EAACnC;oBAC1BO,SAAS6B,IAAAA,uBAAa,EAAC9B,SAAS+B,GAAG,CAAC,SAACC;+BAAMA,EAAEC,IAAI;;oBACjD/B,UAAU;+BAAMgC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAK7B,SAAS;;oBAClD;;wBAAME,OAAOkC,IAAI,CAAC,QAAQ;mCAAMC,IAAAA,mBAAS,EAAC1C,KAAKC;;;;oBAAvDQ,QAAQ;oBACRC,aAAa,IAAIZ,IAAIW,MAAM4B,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAEhC;;wBAAMpC,OAAOkC,IAAI,CAAC,YAAY;;oCAC3CG;;;;4CAAe;;gDAAM7C,KAAK8C,OAAO,CAAC;;;4CAAlCD,eAAe;4CACb;;gDAAMA,aAAaE,GAAG;;;4CAA9B;;gDAAQ;;;;4BACV;;;;oBAHMnC,eAAe;oBAIfC,WAAW,IAAImC,IAAIpC,aAAa0B,GAAG,CAAC,SAACW;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBAC5D,+FAA+F;oBAC/F,+FAA+F;oBAC/F,gFAAgF;oBAC1EnC,WAAWF,aAAauC,MAAM,CAAC,SAACF;+BAAM,CAACtC,WAAWyC,GAAG,CAACH,EAAEC,IAAI;uBAAGZ,GAAG,CAAC,SAACW;+BAAMA,EAAEC,IAAI;;oBAEtF,6FAA6F;oBAC7F,0FAA0F;oBACpFnC,YAAYL,MAAMyC,MAAM,CAAC,SAACZ;;wBAC9B,IAAMc,MAAMxC,SAASyC,GAAG,CAACf,EAAEK,OAAO;wBAClC,OAAO,CAACS,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI,aAAKrD,wBAAAA,kCAAAA,YAAa+C,GAAG,CAACb,EAAEK,OAAO,wCAAK;oBACrG;oBAEA,IAAI9B,SAAS6C,MAAM,KAAK,KAAK5C,UAAU4C,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAGzC,QAAQ;4BAAMX,QAAQA,OAAOqD,IAAI,CAACpD,WAAW;wBAAG;;oBAErG;;wBAAMqD,IAAAA,oBAAU,EAAC9D;;;oBAA/BgB,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAAS8C,IAAAA,oBAAQ,EAAC,mBAAmBhD,UAAU4C,MAAM;oBAEO;;wBAAMnD,OAAOkC,IAAI,CAAC,SAAS;mCAAMsB,IAAAA,uBAAY,EAACjD,WAAWR,UAAUN,KAAKe,aAAaC,OAAOgD,IAAI,EAAE;gCAAE7D,MAAAA;4BAAK;;;;oBAAzG,OAAA,eAApDc,aAAoD,KAA1DgD,MAAkB/C,WAAwC,KAAxCA,UAAUC,aAA8B,KAA9BA,YAAYC,gBAAkB,KAAlBA;oBAChDJ,OAAOkD,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAa/C,iCAAb,6BAAA,QAAA,yBAAA;4BAAME,MAAN;4BAAyBN,YAAYoD,GAAG,CAAC9C;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGP;oBACvB,yFAAyF;oBACzF,iGAAiG;oBACjGb,QAAQkE,gBAAgB,CAAC9C,WAAWoC,MAAM;oBAC1C,0FAA0F;oBAC1F,sEAAsE;oBAChEnC,kBAAkBD,WAAW4B,MAAM,CAAC,SAACmB;+BAAMzE,yBAAyBuD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,yFAAyF;oBACzF,mDAAmD;oBAC7C7C,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBc,GAAG,CAACkC,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FjD,OAAhFA,gBAAgBc,GAAG,CAAC;+BAAM;uBAAKmC,IAAI,CAAC,OAAM,wCAGjI,OAHuKjD,gBAClL2B,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpBhC,GAAG,CAAC,SAACgC;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEF/C,QAAQX,UAAUoC,MAAM,CAAC,SAACZ;+BAAM,CAAC1B,SAASuC,GAAG,CAACb,EAAEK,OAAO;uBAAGN,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAC9EjB,QAAwB;wBAAEjB,OAAAA;wBAAOgE,UAAUxD,WAAWoB,GAAG,CAAC,SAACqC;mCAAMA,EAAE/B,OAAO;;wBAAGlB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAI7B,IAAI2B;oBACnBG,mBAAmBX,WAAWoB,GAAG,CAAC,SAACqC;+BAAMA,EAAE/B,OAAO;uBAAEO,MAAM,CAAC,SAACyB;+BAAM,CAAChD,SAASwB,GAAG,CAACwB;;oBACtF,+FAA+F;oBAC/F,0EAA0E;oBACpE9C,UAAU,AAAC,qBAAGhB,iBAAU,qBAAGe;oBAE3BE,UAAU8C,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EACnB/E,MACA;;oCAGQgF,SACAC,gBAQFC,gBAUIC,OAwDqB,2BAAA,mBAAA,uBAAA,WAAA,YACxB,4BAAA,oBAAA,yBAAA,YAAA,aAIA,4BAAA,oBAAA,yBAAA,YAAA;;;;4CAhFW;;gDAAMrB,IAAAA,oBAAU,EAAC9D;;;4CAA3BgF,UAAU;4CACVC,iBAAiB7D,WAAW+B,MAAM,CAAC,SAAC7B;uDAAQ,CAAC0D,QAAQ5B,GAAG,CAAC9B;;iDAC3D2D,CAAAA,eAAetB,MAAM,GAAG,CAAA,GAAxBsB;;;;4CAA2B;;gDAAMzE,OAAOkC,IAAI,CAAC,SAAS;2DAAMvC,QAAQiF,UAAU,CAACpF,MAAMiF;;;;4CAA1D;;;4CAE/B,yFAAyF;4CACzF,sFAAsF;4CACtF,sFAAsF;4CACtF,sFAAsF;4CACtF,4FAA4F;4CACxFC,iBAAiBpD;iDACjBJ,CAAAA,MAAMiC,MAAM,GAAG,CAAA,GAAfjC;;;;4CACe;;gDAAMlB,OAAOkC,IAAI,CAAC,iBAAiB;;4DAC5C2C,kBACAC,iBACAC;;;;oEAFmB;;wEAAMvF,KAAK8C,OAAO,CAAC;;;oEAAtCuC,mBAAmB;wEACAtF;oEAAM;;wEAAMsF,iBAAiBtC,GAAG;;;oEAAnDuC,eAAe,IAAA,CAAA,EAAA,MAAIvF;;wEAAM,cAA0DuC,GAAG,CAAC,SAACW;mFAAMA,EAAEC,IAAI;;sEAAC;oEACrGqC,aAAa7D,MAAMyB,MAAM,CAAC,SAACyB;+EAAMU,aAAalC,GAAG,CAACwB;;oEACxD;;wEAAOW,WAAW5B,MAAM,GAAG,IAAI,AAAC,qBAAG7B,gBAAS,qBAAGyD,eAAczD;;;;oDAC/D;;;;4CALAoD,iBAAiB;;;iDAOfhE,CAAAA,WAAWyC,MAAM,GAAG,CAAA,GAApBzC;;;;4CACIiE,QAAQ,eAACK;uDACbhE,gBAAgBc,GAAG,CAAC,SAAChB;wDAOZkE;oDANP,IAAIlE,QAAQ,QAAQ,OAAOkE,IAAI5C,OAAO;oDACtC,IAAItB,QAAQ,UAAU,OAAOkE,IAAIhC,OAAO;oDACxC,IAAIlC,QAAQ,UAAU,OAAOkE,IAAIC,OAAO;oDACxC,IAAInE,QAAQ,SAAS,OAAOkE,IAAI9B,IAAI;oDACpC,mFAAmF;oDACnF,IAAIpC,QAAQ,gBAAgB,OAAOkE,IAAIE,UAAU;oDACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAACrE,IAAI,cAAbkE,2BAAAA,gBAAiB;gDAC1B;;4CACF,uFAAuF;4CACvF,6FAA6F;4CAC7F;;gDAAMhF,OAAOkC,IAAI,CAAC,aAAa;;4DAErBkD,SACAC;;;;yEAFJ1F,QAAQ2F,SAAS,EAAjB3F;;;;oEACIyF,UAAU1E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;oEACzDiD,aAAa3E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM,CAAC/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;yEAC/DgD,CAAAA,QAAQjC,MAAM,GAAG,CAAA,GAAjBiC;;;;oEAAoB;;wEAAMzF,QAAQ2F,SAAS,CAAC9F,MAAM,eAAewB,iBAAiBoE,QAAQtD,GAAG,CAAC6C;;;oEAA1E;;;yEACpBU,CAAAA,WAAWlC,MAAM,GAAG,CAAA,GAApBkC;;;;oEAAuB;;wEAAM7F,KAAK+F,QAAQ,CAACtE,WAAWoE,WAAWvD,GAAG,CAAC6C;;;oEAA9C;;;;;;;;oEAE3B;;wEAAMnF,KAAK+F,QAAQ,CAACtE,WAAWP,WAAWoB,GAAG,CAAC6C;;;oEAA9C;;;;;;;;oDAEJ;;;;4CATA;;;4CAYF,wFAAwF;4CACxF,2FAA2F;4CAC3F;;gDAAM3E,OAAOkC,IAAI,CAAC,cAAc;2DAAMvC,QAAQ6F,gBAAgB,CAAChG,MAAMkF,gBAAgBhE,YAAYS,OAAO1B;;;;4CAAxG;4CAEA,0FAA0F;4CAC1F,kEAAkE;4CAClE;;gDAAMO,OAAOkC,IAAI,CAAC,WAAW;;4DAMrBuD,YAC6B,2BAAA,mBAAA,gBAA9B,4BAAA,oBAAA,iBAAA,WAAA,OAAMT,KAAwB,YAAA,QAAMU;;;;yEANrCpE,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;oEACF;;wEAAM9B,KAAK+F,QAAQ,CACjB,6CACAjE,QAAQQ,GAAG,CAAC,SAACsC;mFAAM;gFAACA;6EAAE;;;;oEAFxB;;;oEAIIqB;oEAC6B,kCAAA,2BAAA,4BAA9B,mCAAA,4BAAA;;wEAAL,IAAK,YAAa/E,iCAAb,8BAAA,QAAA,yBAAA;4EAAMsE,MAAN;;gFAAyB,IAAK,aAAoBA,IAAIW,OAAO,uBAA/B,6BAAA,SAAA,0BAAA;oFAAMD,aAAN;oFAAiCD,WAAWG,IAAI;wFAAEZ,IAAI5C,OAAO;wFAAEsD;;;;gFAA/D;gFAAA;;;yFAAA,6BAAA;wFAAA;;;wFAAA;8FAAA;;;;;;wEAA9B;wEAAA;;;iFAAA,8BAAA;gFAAA;;;gFAAA;sFAAA;;;;yEAIDD,CAAAA,WAAWtC,MAAM,GAAG,CAAA,GAApBsC;;;;oEAAuB;;wEAAMjG,KAAK+F,QAAQ,CAAC,kGAAkGE;;;oEAAtH;;;;;;;;oDAC7B;;;;4CAZA;iDAgBInF,CAAAA,SAAS6C,MAAM,GAAG,CAAA,GAAlB7C;;;;4CACF;;gDAAMN,OAAOkC,IAAI,CAAC,YAAY;2DAC5B1C,KAAK+F,QAAQ,CACX,4CACAjF,SAASwB,GAAG,CAAC,SAACsC;+DAAM;4DAACA;yDAAE;;;;;4CAH3B;;;4CAS2B,kCAAA,2BAAA;iDAAzB9C,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;;;;;;;;;;oDAA+BuE;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,WAAW;wEAAM6D;4EAAAA,kBAAAA,QAAQE,MAAM,cAAdF,sCAAAA,qBAAAA,SAAiBrG,MAAM8B,SAASH;;;;4DAA9F;;;;;;;4CAA3B,YAAiBpB;;;mDAAjB,6BAAA,QAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CACxB,mCAAA,4BAAA;;;;;;;;;;oDAAM8F,SACHG;;;;4DADGH,UAAN;4DACGG,iBAAiCtF,WAAWoB,GAAG,CAAC,SAACkD;uEAAS;oEAAEtC,MAAMsC,IAAI5C,OAAO;oEAAE6D,WAAWjB,IAAIiB,SAAS,CAACJ,QAAQ7D,IAAI,CAAC;gEAAC;;4DAC5H;;gEAAMhC,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,iBAAAA,QAAQK,KAAK,cAAbL,qCAAAA,oBAAAA,SAAgBrG,MAAMwG,gBAAgB7E;;;;4DAAnG;;;;;;4CACF;4CAHK,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;;oDAAM8F;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,0BAAAA,QAAQM,cAAc,cAAtBN,8CAAAA,6BAAAA,SAAyBrG,MAAM2B;;;;4DAA5F;;;;;;;4CAA3B,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;2BACAJ,QAAQyG,SAAS;;;oBAvFnB;oBA0FM5E,aAAa6C,KAAKC,GAAG,KAAK/C;oBAChC,2GAA2G;oBAC3G8E,IAAAA,4BAAgB,EAAC3G,SAAS8B;yBACtB7B,QAAQ2G,cAAc,EAAtB3G;;;;oBAAwB;;wBAAMK,OAAOkC,IAAI,CAAC,QAAQ;gCAAMvC;oCAAAA,0BAAAA,QAAQ2G,cAAc,cAAtB3G,8CAAAA,6BAAAA,SAAyBH,MAAMgC;;;;oBAA/D;;;oBAE5B;;wBAAO;4BAAE4B,QAAQ1C,WAAWyC,MAAM;4BAAExC,UAAAA;4BAAUX,QAAQA,OAAOqD,IAAI,CAACpD,WAAWuB,YAAYX;wBAAe;;;;IAC1G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["CORE_FRONTMATTER_COLUMNS","reconcile","Set","conn","cfg","baseDir","dialect","pool","forcedPaths","start","features","stages","elapsed","files","currentSet","existingRows","existing","vanished","toReparse","seenColumns","report","parsedDocs","warnings","newColumns","workerParseMs","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","touched","txStart","durationMs","process","hrtime","bigint","activeFeatures","stageRecorder","map","f","name","Number","time","listFiles","relPath","existingStmt","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","take","getColumns","progress","reparseFiles","tick","docs","finish","add","checkColumnLimit","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","present","missingColumns","contentTouched","toRow","addColumns","currentPathsStmt","currentPaths","staleAdded","doc","ctimeMs","parseError","data","newDocs","updateDocs","appendRows","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","featureStage","remove","docsForFeature","extracted","store","afterReconcile","beginMode","recordLockWaitMs","recordDuration"],"mappings":";;;;;;;;;;;QAqBaA;eAAAA;;QAESC;eAAAA;;;uBAtBS;0BAEN;wBACmB;yBAEf;0BACI;wBACkB;wBACM;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAWzB,IAAMD,2BAA2B,IAAIE,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,SAAeD,UAAUE,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;;YACtJC,OACAC,UACAC,QACAC,SACAC,OACAC,YAEAC,cAIAC,UAIAC,UAIAC,WAOAC,aAIAC,QAE4D,MAApDC,YAAYC,UAAUC,YAAYC,eAE3C,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAMAC,iBAGAC,WAKAC,OACAC,OACAC,UACAC,kBAGAC,SAEAC,SAuFAC;;;;oBAjJA1B,QAAQ2B,QAAQC,MAAM,CAACC,MAAM;oBAC7B5B,WAAW6B,IAAAA,uBAAc,EAACnC;oBAC1BO,SAAS6B,IAAAA,uBAAa,EAAC9B,SAAS+B,GAAG,CAAC,SAACC;+BAAMA,EAAEC,IAAI;;oBACjD/B,UAAU;+BAAMgC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAK7B,SAAS;;oBAClD;;wBAAME,OAAOkC,IAAI,CAAC,QAAQ;mCAAMC,IAAAA,mBAAS,EAAC1C,KAAKC;;;;oBAAvDQ,QAAQ;oBACRC,aAAa,IAAIZ,IAAIW,MAAM4B,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAEhC;;wBAAMpC,OAAOkC,IAAI,CAAC,YAAY;;oCAC3CG;;;;4CAAe;;gDAAM7C,KAAK8C,OAAO,CAAC;;;4CAAlCD,eAAe;4CACb;;gDAAMA,aAAaE,GAAG;;;4CAA9B;;gDAAQ;;;;4BACV;;;;oBAHMnC,eAAe;oBAIfC,WAAW,IAAImC,IAAIpC,aAAa0B,GAAG,CAAC,SAACW;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBAC5D,+FAA+F;oBAC/F,+FAA+F;oBAC/F,gFAAgF;oBAC1EnC,WAAWF,aAAauC,MAAM,CAAC,SAACF;+BAAM,CAACtC,WAAWyC,GAAG,CAACH,EAAEC,IAAI;uBAAGZ,GAAG,CAAC,SAACW;+BAAMA,EAAEC,IAAI;;oBAEtF,6FAA6F;oBAC7F,0FAA0F;oBACpFnC,YAAYL,MAAMyC,MAAM,CAAC,SAACZ;;wBAC9B,IAAMc,MAAMxC,SAASyC,GAAG,CAACf,EAAEK,OAAO;wBAClC,OAAO,CAACS,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI,aAAKrD,wBAAAA,kCAAAA,YAAa+C,GAAG,CAACb,EAAEK,OAAO,wCAAK;oBACrG;oBAEA,IAAI9B,SAAS6C,MAAM,KAAK,KAAK5C,UAAU4C,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAGzC,QAAQ;4BAAMX,QAAQA,OAAOqD,IAAI,CAACpD,WAAW;wBAAG;;oBAErG;;wBAAMqD,IAAAA,oBAAU,EAAC9D;;;oBAA/BgB,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAAS8C,IAAAA,oBAAQ,EAAC,mBAAmBhD,UAAU4C,MAAM;oBAEO;;wBAAMnD,OAAOkC,IAAI,CAAC,SAAS;mCAAMsB,IAAAA,uBAAY,EAACjD,WAAWR,UAAUN,KAAKe,aAAaC,OAAOgD,IAAI,EAAE;gCAAE7D,MAAAA;4BAAK;;;;oBAAzG,OAAA,eAApDc,aAAoD,KAA1DgD,MAAkB/C,WAAwC,KAAxCA,UAAUC,aAA8B,KAA9BA,YAAYC,gBAAkB,KAAlBA;oBAChDJ,OAAOkD,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAa/C,iCAAb,6BAAA,QAAA,yBAAA;4BAAME,MAAN;4BAAyBN,YAAYoD,GAAG,CAAC9C;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGP;oBACvB,yFAAyF;oBACzF,iGAAiG;oBACjGb,QAAQkE,gBAAgB,CAAC9C,WAAWoC,MAAM;oBAC1C,0FAA0F;oBAC1F,sEAAsE;oBAChEnC,kBAAkBD,WAAW4B,MAAM,CAAC,SAACmB;+BAAMzE,yBAAyBuD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,yFAAyF;oBACzF,mDAAmD;oBAC7C7C,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBc,GAAG,CAACkC,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FjD,OAAhFA,gBAAgBc,GAAG,CAAC;+BAAM;uBAAKmC,IAAI,CAAC,OAAM,wCAGjI,OAHuKjD,gBAClL2B,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpBhC,GAAG,CAAC,SAACgC;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEF/C,QAAQX,UAAUoC,MAAM,CAAC,SAACZ;+BAAM,CAAC1B,SAASuC,GAAG,CAACb,EAAEK,OAAO;uBAAGN,GAAG,CAAC,SAACC;+BAAMA,EAAEK,OAAO;;oBAC9EjB,QAAwB;wBAAEjB,OAAAA;wBAAOgE,UAAUxD,WAAWoB,GAAG,CAAC,SAACqC;mCAAMA,EAAE/B,OAAO;;wBAAGlB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAI7B,IAAI2B;oBACnBG,mBAAmBX,WAAWoB,GAAG,CAAC,SAACqC;+BAAMA,EAAE/B,OAAO;uBAAEO,MAAM,CAAC,SAACyB;+BAAM,CAAChD,SAASwB,GAAG,CAACwB;;oBACtF,+FAA+F;oBAC/F,0EAA0E;oBACpE9C,UAAU,AAAC,qBAAGhB,iBAAU,qBAAGe;oBAE3BE,UAAU8C,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EACnB/E,MACA;;oCAGQgF,SACAC,gBAQFC,gBAUIC,OAoDqB,2BAAA,mBAAA,uBAAA,WAAA,YACxB,4BAAA,oBAAA,yBAAA,YAAA,aAIA,4BAAA,oBAAA,yBAAA,YAAA;;;;4CA5EW;;gDAAMrB,IAAAA,oBAAU,EAAC9D;;;4CAA3BgF,UAAU;4CACVC,iBAAiB7D,WAAW+B,MAAM,CAAC,SAAC7B;uDAAQ,CAAC0D,QAAQ5B,GAAG,CAAC9B;;iDAC3D2D,CAAAA,eAAetB,MAAM,GAAG,CAAA,GAAxBsB;;;;4CAA2B;;gDAAMzE,OAAOkC,IAAI,CAAC,SAAS;2DAAMvC,QAAQiF,UAAU,CAACpF,MAAMiF;;;;4CAA1D;;;4CAE/B,yFAAyF;4CACzF,sFAAsF;4CACtF,sFAAsF;4CACtF,sFAAsF;4CACtF,4FAA4F;4CACxFC,iBAAiBpD;iDACjBJ,CAAAA,MAAMiC,MAAM,GAAG,CAAA,GAAfjC;;;;4CACe;;gDAAMlB,OAAOkC,IAAI,CAAC,iBAAiB;;4DAC5C2C,kBACAC,iBACAC;;;;oEAFmB;;wEAAMvF,KAAK8C,OAAO,CAAC;;;oEAAtCuC,mBAAmB;wEACAtF;oEAAM;;wEAAMsF,iBAAiBtC,GAAG;;;oEAAnDuC,eAAe,IAAA,CAAA,EAAA,MAAIvF;;wEAAM,cAA0DuC,GAAG,CAAC,SAACW;mFAAMA,EAAEC,IAAI;;sEAAC;oEACrGqC,aAAa7D,MAAMyB,MAAM,CAAC,SAACyB;+EAAMU,aAAalC,GAAG,CAACwB;;oEACxD;;wEAAOW,WAAW5B,MAAM,GAAG,IAAI,AAAC,qBAAG7B,gBAAS,qBAAGyD,eAAczD;;;;oDAC/D;;;;4CALAoD,iBAAiB;;;iDAOfhE,CAAAA,WAAWyC,MAAM,GAAG,CAAA,GAApBzC;;;;4CACIiE,QAAQ,eAACK;uDACbhE,gBAAgBc,GAAG,CAAC,SAAChB;wDAOZkE;oDANP,IAAIlE,QAAQ,QAAQ,OAAOkE,IAAI5C,OAAO;oDACtC,IAAItB,QAAQ,UAAU,OAAOkE,IAAIhC,OAAO;oDACxC,IAAIlC,QAAQ,UAAU,OAAOkE,IAAIC,OAAO;oDACxC,IAAInE,QAAQ,SAAS,OAAOkE,IAAI9B,IAAI;oDACpC,mFAAmF;oDACnF,IAAIpC,QAAQ,gBAAgB,OAAOkE,IAAIE,UAAU;oDACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAACrE,IAAI,cAAbkE,2BAAAA,gBAAiB;gDAC1B;;4CACF,wFAAwF;4CACxF,sCAAsC;4CACtC;;gDAAMhF,OAAOkC,IAAI,CAAC,aAAa;;4DACvBkD,SACAC;;;;oEADAD,UAAU1E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;oEACzDiD,aAAa3E,WAAWiC,MAAM,CAAC,SAACwB;+EAAM,CAAC/C,SAASwB,GAAG,CAACuB,EAAE/B,OAAO;;oEACnE;;wEAAMkD,IAAAA,oBAAU,EAAC9F,MAAM,eAAewB,iBAAiBC,WAAWmE,QAAQtD,GAAG,CAAC6C;;;oEAA9E;yEACIU,CAAAA,WAAWlC,MAAM,GAAG,CAAA,GAApBkC;;;;oEAAuB;;wEAAM7F,KAAK+F,QAAQ,CAACtE,WAAWoE,WAAWvD,GAAG,CAAC6C;;;oEAA9C;;;;;;;;oDAC7B;;;;4CALA;;;4CAQF,wFAAwF;4CACxF,2FAA2F;4CAC3F;;gDAAM3E,OAAOkC,IAAI,CAAC,cAAc;2DAAMvC,QAAQ6F,gBAAgB,CAAChG,MAAMkF,gBAAgBhE,YAAYS,OAAO1B;;;;4CAAxG;4CAEA,0FAA0F;4CAC1F,kEAAkE;4CAClE;;gDAAMO,OAAOkC,IAAI,CAAC,WAAW;;4DAMrBuD,YAC6B,2BAAA,mBAAA,gBAA9B,4BAAA,oBAAA,iBAAA,WAAA,OAAMT,KAAwB,YAAA,QAAMU;;;;yEANrCpE,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;oEACF;;wEAAM9B,KAAK+F,QAAQ,CACjB,6CACAjE,QAAQQ,GAAG,CAAC,SAACsC;mFAAM;gFAACA;6EAAE;;;;oEAFxB;;;oEAIIqB;oEAC6B,kCAAA,2BAAA,4BAA9B,mCAAA,4BAAA;;wEAAL,IAAK,YAAa/E,iCAAb,8BAAA,QAAA,yBAAA;4EAAMsE,MAAN;;gFAAyB,IAAK,aAAoBA,IAAIW,OAAO,uBAA/B,6BAAA,SAAA,0BAAA;oFAAMD,aAAN;oFAAiCD,WAAWG,IAAI;wFAAEZ,IAAI5C,OAAO;wFAAEsD;;;;gFAA/D;gFAAA;;;yFAAA,6BAAA;wFAAA;;;wFAAA;8FAAA;;;;;;wEAA9B;wEAAA;;;iFAAA,8BAAA;gFAAA;;;gFAAA;sFAAA;;;;oEACL,uFAAuF;oEACvF,wFAAwF;oEACxF,uGAAuG;oEACvG;;wEAAMJ,IAAAA,oBAAU,EAAC9F,MAAM;4EAAiB;4EAAQ;2EAAW,kGAAkGiG;;;oEAA7J;;;;;;oDACF;;;;4CAZA;iDAgBInF,CAAAA,SAAS6C,MAAM,GAAG,CAAA,GAAlB7C;;;;4CACF;;gDAAMN,OAAOkC,IAAI,CAAC,YAAY;2DAC5B1C,KAAK+F,QAAQ,CACX,4CACAjF,SAASwB,GAAG,CAAC,SAACsC;+DAAM;4DAACA;yDAAE;;;;;4CAH3B;;;4CAS2B,kCAAA,2BAAA;iDAAzB9C,CAAAA,QAAQ6B,MAAM,GAAG,CAAA,GAAjB7B;;;;;;;;;;;;;oDAA+BuE;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,WAAW;wEAAM6D;4EAAAA,kBAAAA,QAAQE,MAAM,cAAdF,sCAAAA,qBAAAA,SAAiBrG,MAAM8B,SAASH;;;;4DAA9F;;;;;;;4CAA3B,YAAiBpB;;;mDAAjB,6BAAA,QAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CACxB,mCAAA,4BAAA;;;;;;;;;;oDAAM8F,SACHG;;;;4DADGH,UAAN;4DACGG,iBAAiCtF,WAAWoB,GAAG,CAAC,SAACkD;uEAAS;oEAAEtC,MAAMsC,IAAI5C,OAAO;oEAAE6D,WAAWjB,IAAIiB,SAAS,CAACJ,QAAQ7D,IAAI,CAAC;gEAAC;;4DAC5H;;gEAAMhC,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,iBAAAA,QAAQK,KAAK,cAAbL,qCAAAA,oBAAAA,SAAgBrG,MAAMwG,gBAAgB7E;;;;4DAAnG;;;;;;4CACF;4CAHK,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;;oDAAM8F;;;;4DAAAA,UAAN;4DAA2B;;gEAAM7F,OAAOkC,IAAI,CAAC4D,IAAAA,sBAAY,EAACD,QAAQ7D,IAAI,EAAE,UAAU;wEAAM6D;4EAAAA,0BAAAA,QAAQM,cAAc,cAAtBN,8CAAAA,6BAAAA,SAAyBrG,MAAM2B;;;;4DAA5F;;;;;;;4CAA3B,aAAiBpB;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;2BACAJ,QAAQyG,SAAS;;;oBAnFnB;oBAsFM5E,aAAa6C,KAAKC,GAAG,KAAK/C;oBAChC,2GAA2G;oBAC3G8E,IAAAA,4BAAgB,EAAC3G,SAAS8B;yBACtB7B,QAAQ2G,cAAc,EAAtB3G;;;;oBAAwB;;wBAAMK,OAAOkC,IAAI,CAAC,QAAQ;gCAAMvC;oCAAAA,0BAAAA,QAAQ2G,cAAc,cAAtB3G,8CAAAA,6BAAAA,SAAyBH,MAAMgC;;;;oBAA/D;;;oBAE5B;;wBAAO;4BAAE4B,QAAQ1C,WAAWyC,MAAM;4BAAExC,UAAAA;4BAAUX,QAAQA,OAAOqD,IAAI,CAACpD,WAAWuB,YAAYX;wBAAe;;;;IAC1G"}

package/dist/cjs/store/shared.d.cts

@@ -1,4 +1,5 @@
 import type { Connection } from './types.js';
+export declare function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void>;
 export declare function quoteIdent(name: string): string;
 export declare function getColumns(conn: Connection): Promise<Set<string>>;
 export declare function getMeta(conn: Connection, key: string): Promise<string | null>;

package/dist/cjs/store/shared.d.ts

@@ -1,4 +1,5 @@
 import type { Connection } from './types.js';
+export declare function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void>;
 export declare function quoteIdent(name: string): string;
 export declare function getColumns(conn: Connection): Promise<Set<string>>;
 export declare function getMeta(conn: Connection, key: string): Promise<string | null>;

package/dist/cjs/store/shared.js

@@ -9,6 +9,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get appendRows () {
+        return appendRows;
+    },
     get getColumns () {
         return getColumns;
     },
@@ -150,6 +153,44 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
+function appendRows(conn, table, columns, conflictSql, rows) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (rows.length === 0) return [
+                        2
+                    ];
+                    if (!conn.appendRows) return [
+                        3,
+                        2
+                    ];
+                    return [
+                        4,
+                        conn.appendRows(table, columns, rows)
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        3,
+                        4
+                    ];
+                case 2:
+                    return [
+                        4,
+                        conn.runBatch(conflictSql, rows)
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 function quoteIdent(name) {
     return '"'.concat(name.split('"').join('""'), '"');
 }

package/dist/cjs/store/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["getColumns","getMeta","quoteIdent","recordReconcileDuration","setMeta","name","split","join","conn","stmt","rows","prepare","all","Set","map","r","key","row","get","value","run","ms","prevRaw","prevMax","Number","String"],"mappings":";;;;;;;;;;;QASsBA;eAAAA;;QAMAC;eAAAA;;QAVNC;eAAAA;;QA4BMC;eAAAA;;QAZAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAhBf,SAASF,WAAWG,IAAY;IACrC,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEO,SAAeP,WAAWQ,IAAgB;;YACzCC,MACAC;;;;oBADO;;wBAAMF,KAAKG,OAAO,CAAC;;;oBAA1BF,OAAO;oBACC;;wBAAMA,KAAKG,GAAG;;;oBAAtBF,OAAQ;oBACd;;wBAAO,IAAIG,IAAIH,KAAKI,GAAG,CAAC,SAACC;mCAAMA,EAAEV,IAAI;;;;;IACvC;;AAEO,SAAeJ,QAAQO,IAAgB,EAAEQ,GAAW;;YACnDP,MACAQ;;;;oBADO;;wBAAMT,KAAKG,OAAO,CAAC;;;oBAA1BF,OAAO;oBACA;;wBAAMA,KAAKS,GAAG,CAACF;;;oBAAtBC,MAAO;oBACb;;wBAAOA,MAAMA,IAAIE,KAAK,GAAG;;;;IAC3B;;AAEO,SAAef,QAAQI,IAAgB,EAAEQ,GAAW,EAAEG,KAAoB;;YAEvEV,MAIFA;;;;yBALFU,CAAAA,UAAU,IAAG,GAAbA;;;;oBACW;;wBAAMX,KAAKG,OAAO,CAAC;;;oBAA1BF,OAAO;oBACb;;wBAAMA,KAAKW,GAAG,CAACJ;;;oBAAf;oBACA;;;;oBAEW;;wBAAMR,KAAKG,OAAO,CAAC;;;oBAA1BF,QAAO;oBACb;;wBAAMA,MAAKW,GAAG,CAACJ,KAAKG;;;oBAApB;;;;;;IACF;;AAIO,SAAehB,wBAAwBK,IAAgB,EAAEa,EAAU;;YAClEC,SAGAC;;;;oBAHU;;wBAAMtB,QAAQO,MAAM;;;oBAA9Bc,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;yBAC3CD,CAAAA,KAAKE,OAAM,GAAXF;;;;oBAAc;;wBAAMjB,QAAQI,MAAM,oBAAoBiB,OAAOJ;;;oBAA/C;;;;;;;;IACpB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\n// Rows that cannot conflict, at whatever speed this connection offers: the append path where the\n// store has one, else `conflictSql` bound the ordinary way. `conflictSql` stays the caller's own\n// guarded statement, so a store without appendRows behaves exactly as it did before.\nexport async function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  if (conn.appendRows) await conn.appendRows(table, columns, rows);\n  else await conn.runBatch(conflictSql, rows);\n}\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["appendRows","getColumns","getMeta","quoteIdent","recordReconcileDuration","setMeta","conn","table","columns","conflictSql","rows","length","runBatch","name","split","join","stmt","prepare","all","Set","map","r","key","row","get","value","run","ms","prevRaw","prevMax","Number","String"],"mappings":";;;;;;;;;;;QAQsBA;eAAAA;;QAUAC;eAAAA;;QAMAC;eAAAA;;QAVNC;eAAAA;;QA4BMC;eAAAA;;QAZAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAtBf,SAAeL,WAAWM,IAAgB,EAAEC,KAAa,EAAEC,OAAiB,EAAEC,WAAmB,EAAEC,IAAiB;;;;;oBACzH,IAAIA,KAAKC,MAAM,KAAK,GAAG;;;yBACnBL,KAAKN,UAAU,EAAfM;;;;oBAAiB;;wBAAMA,KAAKN,UAAU,CAACO,OAAOC,SAASE;;;oBAAtC;;;;;;oBAChB;;wBAAMJ,KAAKM,QAAQ,CAACH,aAAaC;;;oBAAjC;;;;;;;;IACP;;AAEO,SAASP,WAAWU,IAAY;IACrC,OAAO,AAAC,IAA8B,OAA3BA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,OAAM;AACxC;AAEO,SAAed,WAAWK,IAAgB;;YACzCU,MACAN;;;;oBADO;;wBAAMJ,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACC;;wBAAMA,KAAKE,GAAG;;;oBAAtBR,OAAQ;oBACd;;wBAAO,IAAIS,IAAIT,KAAKU,GAAG,CAAC,SAACC;mCAAMA,EAAER,IAAI;;;;;IACvC;;AAEO,SAAeX,QAAQI,IAAgB,EAAEgB,GAAW;;YACnDN,MACAO;;;;oBADO;;wBAAMjB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACA;;wBAAMA,KAAKQ,GAAG,CAACF;;;oBAAtBC,MAAO;oBACb;;wBAAOA,MAAMA,IAAIE,KAAK,GAAG;;;;IAC3B;;AAEO,SAAepB,QAAQC,IAAgB,EAAEgB,GAAW,EAAEG,KAAoB;;YAEvET,MAIFA;;;;yBALFS,CAAAA,UAAU,IAAG,GAAbA;;;;oBACW;;wBAAMnB,KAAKW,OAAO,CAAC;;;oBAA1BD,OAAO;oBACb;;wBAAMA,KAAKU,GAAG,CAACJ;;;oBAAf;oBACA;;;;oBAEW;;wBAAMhB,KAAKW,OAAO,CAAC;;;oBAA1BD,QAAO;oBACb;;wBAAMA,MAAKU,GAAG,CAACJ,KAAKG;;;oBAApB;;;;;;IACF;;AAIO,SAAerB,wBAAwBE,IAAgB,EAAEqB,EAAU;;YAClEC,SAGAC;;;;oBAHU;;wBAAM3B,QAAQI,MAAM;;;oBAA9BsB,UAAU;oBAChB,yFAAyF;oBACzF,+EAA+E;oBACzEC,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;yBAC3CD,CAAAA,KAAKE,OAAM,GAAXF;;;;oBAAc;;wBAAMtB,QAAQC,MAAM,oBAAoByB,OAAOJ;;;oBAA/C;;;;;;;;IACpB"}

package/dist/cjs/store/types.d.cts

@@ -21,6 +21,7 @@ export interface Connection {
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
+    appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface ReconcileDialect {
     beginMode(): string;
@@ -28,7 +29,6 @@ export interface ReconcileDialect {
     addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
-    insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface OpenDialect<Handle> {
     filename: string;

package/dist/cjs/store/types.d.ts

@@ -21,6 +21,7 @@ export interface Connection {
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
+    appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface ReconcileDialect {
     beginMode(): string;
@@ -28,7 +29,6 @@ export interface ReconcileDialect {
     addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
-    insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface OpenDialect<Handle> {
     filename: string;