sensemaking 0.22.0 → 0.22.2

package/dist/esm/store/duckdb/connection.js

@@ -1,5 +1,7 @@
+import { quoteIdent } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { rewriteBatch } from './batch.js';
+import { duckdbApi } from './native.js';
 // getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small
 // ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.
 const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
@@ -66,6 +68,39 @@ export function createConnection(duckdb) {
         async prepare (sql) {
             return new DuckdbStatement(await duckdb.prepare(sql));
         },
+        // The appender writes columnar vectors with no per-parameter binding, which is the cost
+        // runBatch pays: measured 7.2x on frontmatter at 6,566 rows. Alignment is against the table's
+        // own physical column order, read fresh, so a column the caller does not write (a
+        // feature-owned "_rank", say) takes appendDefault() rather than shifting every value one slot.
+        async appendRows (table, columns, rows) {
+            if (rows.length === 0) return;
+            const { variantValue } = await duckdbApi();
+            const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);
+            const physical = await infoStmt.all();
+            const rowIndexOf = new Map(columns.map((name, i)=>[
+                    name,
+                    i
+                ]));
+            await withTransaction(conn, async ()=>{
+                const appender = await duckdb.createAppender(table);
+                try {
+                    for (const row of rows){
+                        for (const column of physical){
+                            const idx = rowIndexOf.get(column.name);
+                            const value = idx === undefined ? undefined : row[idx];
+                            if (idx === undefined) appender.appendDefault();
+                            else if (value === null || value === undefined) appender.appendNull();
+                            else if (column.type === 'VARIANT') appender.appendVariant(variantValue(value));
+                            else appender.appendValue(value);
+                        }
+                        appender.endRow();
+                    }
+                    appender.flushSync();
+                } finally{
+                    appender.closeSync();
+                }
+            });
+        },
         // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),
         // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.
         async runBatch (sql, paramRows) {

package/dist/esm/store/duckdb/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,gGAAgG;AAChG,yGAAyG;AACzG,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAQA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAyB;QAC7BD;QACA,MAAME,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,oHAAoH;QACpH,qIAAqI;QACrI,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMrB,gBAAgB8C,MAAM;gBAC1B,MAAMM,YAAYnD,aAAa+C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK/B,IAAI,CAAC6B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKpC,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMoC,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO0C,UAAW;wBAC3BE,KAAK/B,IAAI,CAACb;wBACV,MAAM4C,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKpC,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // The appender writes columnar vectors with no per-parameter binding, which is the cost\n    // runBatch pays: measured 7.2x on frontmatter at 6,566 rows. Alignment is against the table's\n    // own physical column order, read fresh, so a column the caller does not write (a\n    // feature-owned \"_rank\", say) takes appendDefault() rather than shifting every value one slot.\n    async appendRows(table: string, columns: string[], rows: unknown[][]): Promise<void> {\n      if (rows.length === 0) return;\n      const { variantValue } = await duckdbApi();\n      const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);\n      const physical = (await infoStmt.all()) as Array<{ name: string; type: string }>;\n      const rowIndexOf = new Map(columns.map((name, i) => [name, i]));\n\n      await withTransaction(conn, async () => {\n        const appender = await duckdb.createAppender(table);\n        try {\n          for (const row of rows) {\n            for (const column of physical) {\n              const idx = rowIndexOf.get(column.name);\n              const value = idx === undefined ? undefined : row[idx];\n              if (idx === undefined) appender.appendDefault();\n              else if (value === null || value === undefined) appender.appendNull();\n              // VARIANT is frontmatter's dynamic columns only; every other table is statically\n              // typed, and appendValue is what preserves those types.\n              else if (column.type === 'VARIANT') appender.appendVariant(variantValue(value as DuckDBValue));\n              else appender.appendValue(value as DuckDBValue);\n            }\n            appender.endRow();\n          }\n          appender.flushSync();\n        } finally {\n          appender.closeSync();\n        }\n      });\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["quoteIdent","withTransaction","rewriteBatch","duckdbApi","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","appendRows","table","variantValue","infoStmt","physical","rowIndexOf","Map","appender","createAppender","column","idx","undefined","appendDefault","appendNull","type","appendVariant","appendValue","endRow","flushSync","closeSync","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAC1C,SAASC,SAAS,QAAQ,cAAc;AAExC,gGAAgG;AAChG,yGAAyG;AACzG,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAQA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAyB;QAC7BD;QACA,MAAME,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,wFAAwF;QACxF,8FAA8F;QAC9F,kFAAkF;QAClF,+FAA+F;QAC/F,MAAME,YAAWC,KAAa,EAAEhB,OAAiB,EAAEP,IAAiB;YAClE,IAAIA,KAAKP,MAAM,KAAK,GAAG;YACvB,MAAM,EAAE+B,YAAY,EAAE,GAAG,MAAMnD;YAC/B,MAAMoD,WAAW,MAAMP,KAAKG,OAAO,CAAC,CAAC,kBAAkB,EAAEnD,WAAWqD,OAAO,CAAC,CAAC;YAC7E,MAAMG,WAAY,MAAMD,SAASxB,GAAG;YACpC,MAAM0B,aAAa,IAAIC,IAAIrB,QAAQF,GAAG,CAAC,CAACM,MAAMH,IAAM;oBAACG;oBAAMH;iBAAE;YAE7D,MAAMrC,gBAAgB+C,MAAM;gBAC1B,MAAMW,WAAW,MAAMZ,OAAOa,cAAc,CAACP;gBAC7C,IAAI;oBACF,KAAK,MAAM1C,OAAOmB,KAAM;wBACtB,KAAK,MAAM+B,UAAUL,SAAU;4BAC7B,MAAMM,MAAML,WAAW5B,GAAG,CAACgC,OAAOpB,IAAI;4BACtC,MAAMhC,QAAQqD,QAAQC,YAAYA,YAAYpD,GAAG,CAACmD,IAAI;4BACtD,IAAIA,QAAQC,WAAWJ,SAASK,aAAa;iCACxC,IAAIvD,UAAU,QAAQA,UAAUsD,WAAWJ,SAASM,UAAU;iCAG9D,IAAIJ,OAAOK,IAAI,KAAK,WAAWP,SAASQ,aAAa,CAACb,aAAa7C;iCACnEkD,SAASS,WAAW,CAAC3D;wBAC5B;wBACAkD,SAASU,MAAM;oBACjB;oBACAV,SAASW,SAAS;gBACpB,SAAU;oBACRX,SAASY,SAAS;gBACpB;YACF;QACF;QACA,oHAAoH;QACpH,qIAAqI;QACrI,MAAMC,UAAStB,GAAW,EAAEuB,SAAsB;YAChD,IAAIA,UAAUlD,MAAM,KAAK,GAAG;YAC5B,MAAMtB,gBAAgB+C,MAAM;gBAC1B,MAAM0B,YAAYxE,aAAagD,KAAKuB,UAAUlD,MAAM;gBACpD,IAAImD,WAAW;oBACb,MAAMC,OAAO,MAAM5B,OAAOI,OAAO,CAACuB,UAAUxB,GAAG;oBAC/C,IAAI;wBACFyB,KAAKnD,IAAI,CAACiD,UAAUG,IAAI;wBACxB,MAAMD,KAAKtD,GAAG;oBAChB,SAAU;wBACRsD,KAAKxD,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMwD,OAAO,MAAM5B,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO8D,UAAW;wBAC3BE,KAAKnD,IAAI,CAACb;wBACV,MAAMgE,KAAKtD,GAAG;oBAChB;gBACF,SAAU;oBACRsD,KAAKxD,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}

package/dist/esm/store/duckdb/reconcile.js

@@ -1,15 +1,19 @@
 import { SenseError } from '../../errors.js';
-import { CORE_FRONTMATTER_COLUMNS } from '../reconcile.js';
-import { quoteIdent } from '../shared.js';
+import { appendRows, quoteIdent } from '../shared.js';
 import { markContentStale } from './lexical.js';
-import { duckdbApi } from './native.js';
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
 // store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here
 // unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter
 // column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
 // No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):
 // `path` is content's own primary key, so this is a plain per-doc row.
-const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)`;
+const CONTENT_COLUMNS = [
+    'path',
+    'title',
+    'summary',
+    'text'
+];
+const INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;
 function contentRow(doc) {
     return [
         doc.relPath,
@@ -25,7 +29,9 @@ async function reconcileContent(conn, touched, docs, _delta) {
     if (touched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', touched.map((p)=>[
             p
         ]));
-    if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));
+    // The delete above cleared every touched path, and duckdb holds the cache file for its whole
+    // connection, so no second writer can have landed one of these rows: nothing can conflict.
+    await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));
     // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,
     // not here (lexical.ts's FtsIndexState).
     markContentStale(conn);
@@ -39,36 +45,6 @@ async function addColumns(conn, names) {
     if (names.length === 0) return;
     await conn.exec(names.map((name)=>`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));
 }
-// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so
-// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. "_rank") that still exist on the table, and get appendDefault().
-async function insertNew(conn, table, columns, rows) {
-    if (rows.length === 0) return;
-    const { variantValue } = await duckdbApi();
-    const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);
-    const physicalColumns = (await infoStmt.all()).map((c)=>c.name);
-    const rowIndexOf = new Map(columns.map((name, i)=>[
-            name,
-            i
-        ]));
-    const native = conn.duckdb;
-    const appender = await native.createAppender(table);
-    try {
-        for (const row of rows){
-            for (const name of physicalColumns){
-                const idx = rowIndexOf.get(name);
-                const value = idx === undefined ? undefined : row[idx];
-                if (idx === undefined) appender.appendDefault();
-                else if (value === null || value === undefined) appender.appendNull();
-                else if (CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value);
-                else appender.appendVariant(variantValue(value));
-            }
-            appender.endRow();
-        }
-        appender.flushSync();
-    } finally{
-        appender.closeSync();
-    }
-}
 export const duckdbDialect = {
     beginMode: ()=>'BEGIN',
     checkColumnLimit (count) {
@@ -77,6 +53,5 @@ export const duckdbDialect = {
         }
     },
     addColumns,
-    reconcileContent,
-    insertNew
+    reconcileContent
 };

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { DuckDBValue } from '@duckdb/node-api';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { CORE_FRONTMATTER_COLUMNS } from '../reconcile.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport type { DuckdbConnection } from './connection.ts';\nimport { markContentStale } from './lexical.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\n// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so\n// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. \"_rank\") that still exist on the table, and get appendDefault().\nasync function insertNew(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  const { variantValue } = await duckdbApi();\n  const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);\n  const physicalColumns = ((await infoStmt.all()) as Array<{ name: string }>).map((c) => c.name);\n  const rowIndexOf = new Map(columns.map((name, i) => [name, i]));\n\n  const native = (conn as DuckdbConnection).duckdb;\n  const appender = await native.createAppender(table);\n  try {\n    for (const row of rows) {\n      for (const name of physicalColumns) {\n        const idx = rowIndexOf.get(name);\n        const value = idx === undefined ? undefined : row[idx];\n        if (idx === undefined) appender.appendDefault();\n        else if (value === null || value === undefined) appender.appendNull();\n        else if (CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value as DuckDBValue);\n        else appender.appendVariant(variantValue(value as DuckDBValue));\n      }\n      appender.endRow();\n    }\n    appender.flushSync();\n  } finally {\n    appender.closeSync();\n  }\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n  insertNew,\n};\n"],"names":["SenseError","CORE_FRONTMATTER_COLUMNS","quoteIdent","markContentStale","duckdbApi","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","addColumns","names","exec","name","join","insertNew","table","columns","rows","variantValue","infoStmt","prepare","physicalColumns","all","c","rowIndexOf","Map","i","native","duckdb","appender","createAppender","row","idx","get","value","undefined","appendDefault","appendNull","has","appendValue","appendVariant","endRow","flushSync","closeSync","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,wBAAwB,QAAQ,kBAAkB;AAC3D,SAASC,UAAU,QAAQ,eAAe;AAG1C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,SAAS,QAAQ,cAAc;AAExC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIL,KAAKE,MAAM,GAAG,GAAG,MAAMJ,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;IACtE,6FAA6F;IAC7F,yCAAyC;IACzCH,iBAAiBY;AACnB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeQ,WAAWR,IAAgB,EAAES,KAAe;IACzD,IAAIA,MAAML,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKU,IAAI,CAACD,MAAMH,GAAG,CAAC,CAACK,OAAS,CAAC,mCAAmC,EAAExB,WAAWwB,MAAM,QAAQ,CAAC,EAAEC,IAAI,CAAC;AAC7G;AAEA,mGAAmG;AACnG,uLAAuL;AACvL,eAAeC,UAAUb,IAAgB,EAAEc,KAAa,EAAEC,OAAiB,EAAEC,IAAiB;IAC5F,IAAIA,KAAKZ,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEa,YAAY,EAAE,GAAG,MAAM5B;IAC/B,MAAM6B,WAAW,MAAMlB,KAAKmB,OAAO,CAAC,CAAC,kBAAkB,EAAEhC,WAAW2B,OAAO,CAAC,CAAC;IAC7E,MAAMM,kBAAkB,AAAE,CAAA,MAAMF,SAASG,GAAG,EAAC,EAA+Bf,GAAG,CAAC,CAACgB,IAAMA,EAAEX,IAAI;IAC7F,MAAMY,aAAa,IAAIC,IAAIT,QAAQT,GAAG,CAAC,CAACK,MAAMc,IAAM;YAACd;YAAMc;SAAE;IAE7D,MAAMC,SAAS,AAAC1B,KAA0B2B,MAAM;IAChD,MAAMC,WAAW,MAAMF,OAAOG,cAAc,CAACf;IAC7C,IAAI;QACF,KAAK,MAAMgB,OAAOd,KAAM;YACtB,KAAK,MAAML,QAAQS,gBAAiB;gBAClC,MAAMW,MAAMR,WAAWS,GAAG,CAACrB;gBAC3B,MAAMsB,QAAQF,QAAQG,YAAYA,YAAYJ,GAAG,CAACC,IAAI;gBACtD,IAAIA,QAAQG,WAAWN,SAASO,aAAa;qBACxC,IAAIF,UAAU,QAAQA,UAAUC,WAAWN,SAASQ,UAAU;qBAC9D,IAAIlD,yBAAyBmD,GAAG,CAAC1B,OAAOiB,SAASU,WAAW,CAACL;qBAC7DL,SAASW,aAAa,CAACtB,aAAagB;YAC3C;YACAL,SAASY,MAAM;QACjB;QACAZ,SAASa,SAAS;IACpB,SAAU;QACRb,SAASc,SAAS;IACpB;AACF;AAEA,OAAO,MAAMC,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhD,yBAAyB;YACnC,MAAM,IAAIb,WAAW,gBAAgB,CAAC,uBAAuB,EAAE6D,MAAM,8CAA8C,EAAEhD,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAU;IACAT;IACAc;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  // The delete above cleared every touched path, and duckdb holds the cache file for its whole\n  // connection, so no second writer can have landed one of these rows: nothing can conflict.\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","appendRows","quoteIdent","markContentStale","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","join","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","p","addColumns","names","exec","name","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AAEtD,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,MAAMC,qBAAqB,CAAC,qBAAqB,EAAED,gBAAgBE,GAAG,CAACJ,YAAYK,IAAI,CAAC,MAAM,qBAAqB,CAAC;AAEpH,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQZ,GAAG,CAAC,CAACiB,IAAM;YAACA;SAAE;IAE1B,6FAA6F;IAC7F,2FAA2F;IAC3F,MAAMtB,WAAWgB,MAAM,WAAWb,iBAAiBC,oBAAoBc,KAAKb,GAAG,CAACE;IAChF,6FAA6F;IAC7F,yCAAyC;IACzCL,iBAAiBc;AACnB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeO,WAAWP,IAAgB,EAAEQ,KAAe;IACzD,IAAIA,MAAMJ,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKS,IAAI,CAACD,MAAMnB,GAAG,CAAC,CAACqB,OAAS,CAAC,mCAAmC,EAAEzB,WAAWyB,MAAM,QAAQ,CAAC,EAAEpB,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMqB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhB,yBAAyB;YACnC,MAAM,IAAIf,WAAW,gBAAgB,CAAC,uBAAuB,EAAE+B,MAAM,8CAA8C,EAAEhB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAS;IACAR;AACF,EAAE"}

package/dist/esm/store/reconcile.js

@@ -3,7 +3,7 @@ import { progress } from '../output/progress.js';
 import { listFiles, RESERVED_COLUMNS } from '../scan/index.js';
 import { reparseFiles } from '../scan/reparse.js';
 import { recordLockWaitMs } from './lock-wait.js';
-import { getColumns, quoteIdent } from './shared.js';
+import { appendRows, getColumns, quoteIdent } from './shared.js';
 import { featureStage, stageRecorder } from './stages.js';
 import { withTransaction } from './transaction.js';
 // One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect
@@ -121,17 +121,13 @@ export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths)
                     if (col === '_parse_error') return doc.parseError;
                     return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
                 });
-            // A path in `added` has no existing frontmatter row, so it can never conflict: where a
-            // dialect offers a faster append-only path (insertNew), only the rest go through the upsert.
+            // A path in `added` has no existing frontmatter row, so it can never conflict; the rest
+            // genuinely can, and keep the upsert.
             await stages.time('fm-upsert', async ()=>{
-                if (dialect.insertNew) {
-                    const newDocs = parsedDocs.filter((d)=>addedSet.has(d.relPath));
-                    const updateDocs = parsedDocs.filter((d)=>!addedSet.has(d.relPath));
-                    if (newDocs.length > 0) await dialect.insertNew(conn, 'frontmatter', writableColumns, newDocs.map(toRow));
-                    if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));
-                } else {
-                    await conn.runBatch(insertSql, parsedDocs.map(toRow));
-                }
+                const newDocs = parsedDocs.filter((d)=>addedSet.has(d.relPath));
+                const updateDocs = parsedDocs.filter((d)=>!addedSet.has(d.relPath));
+                await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));
+                if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));
             });
         }
         // After the upsert, so every doc already has the frontmatter row sqlite's content rowid
@@ -151,7 +147,10 @@ export async function reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths)
             // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken
             // before this transaction's lock, so a path this process calls "added" can already have
             // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.
-            if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows);
+            await appendRows(conn, 'preset_files', [
+                'path',
+                'preset'
+            ], 'INSERT INTO preset_files ("path", preset) VALUES (?, ?) ON CONFLICT("path", preset) DO NOTHING', presetRows);
         });
         // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's
         // node set, so a lingering vanished row would dilute rank mass across every surviving note.

package/dist/esm/store/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict: where a\n        // dialect offers a faster append-only path (insertNew), only the rest go through the upsert.\n        await stages.time('fm-upsert', async () => {\n          if (dialect.insertNew) {\n            const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n            const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n            if (newDocs.length > 0) await dialect.insertNew(conn, 'frontmatter', writableColumns, newDocs.map(toRow));\n            if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n          } else {\n            await conn.runBatch(insertSql, parsedDocs.map(toRow));\n          }\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","pool","forcedPaths","start","process","hrtime","bigint","features","stages","map","f","name","elapsed","Number","files","time","currentSet","relPath","existingRows","existingStmt","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","take","seenColumns","report","docs","parsedDocs","newColumns","workerParseMs","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","toRow","doc","ctimeMs","parseError","data","insertNew","newDocs","updateDocs","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","remove","docsForFeature","extracted","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACrD,SAASC,YAAY,EAAeC,aAAa,QAAQ,cAAc;AACvE,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AAEvG,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAC5J,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,MAAMC,WAAWxB,eAAee;IAChC,MAAMU,SAAShB,cAAce,SAASE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvD,MAAMC,UAAU,IAAMC,OAAOT,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;IAChE,MAAMW,QAAQ,MAAMN,OAAOO,IAAI,CAAC,QAAQ,IAAM9B,UAAUa,KAAKC;IAC7D,MAAMiB,aAAa,IAAIrB,IAAImB,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IAErD,MAAMC,eAAe,MAAMV,OAAOO,IAAI,CAAC,YAAY;QACjD,MAAMI,eAAe,MAAMtB,KAAKuB,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,MAAMC,WAAW,IAAIC,IAAIL,aAAaT,GAAG,CAAC,CAACe,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWR,aAAaS,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGhB,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACjB;;QAC9B,MAAMoB,MAAMR,SAASS,GAAG,CAACrB,EAAEO,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKtB,EAAEuB,OAAO,IAAIH,IAAII,KAAK,KAAKxB,EAAEyB,IAAI,aAAKjC,wBAAAA,kCAAAA,YAAa0B,GAAG,CAAClB,EAAEO,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;QAAE9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAW;IAAG;IAEzH,MAAM4B,cAAc,MAAMnD,WAAWQ;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM4C,SAASzD,SAAS,mBAAmB6C,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAEC,aAAa,EAAE,GAAG,MAAMrC,OAAOO,IAAI,CAAC,SAAS,IAAM5B,aAAa0C,WAAWtB,UAAUT,KAAK0C,aAAaC,OAAOK,IAAI,EAAE;YAAE7C;QAAK;IAC3KwC,OAAOM,MAAM;IACb,KAAK,MAAMC,OAAOJ,WAAYJ,YAAYS,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIV;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjGxC,QAAQmD,gBAAgB,CAACD,WAAWd,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMgB,kBAAkBF,WAAWvB,MAAM,CAAC,CAAC0B,IAAM3D,yBAAyBkC,GAAG,CAACyB,MAAM,CAACnE,iBAAiB0C,GAAG,CAACyB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB3C,GAAG,CAACnB,YAAYiE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB3C,GAAG,CAAC,IAAM,KAAK8C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLzB,MAAM,CAAC,CAAC0B,IAAMA,MAAM,QACpB5C,GAAG,CAAC,CAAC4C,IAAM,GAAG/D,WAAW+D,GAAG,YAAY,EAAE/D,WAAW+D,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ3B,UAAUF,MAAM,CAAC,CAACjB,IAAM,CAACY,SAASM,GAAG,CAAClB,EAAEO,OAAO,GAAGR,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IACpF,MAAMwC,QAAwB;QAAE3C;QAAO4C,UAAUf,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO;QAAGuC;QAAO9B;IAAS;IACnG,MAAMkC,WAAW,IAAIjE,IAAI6D;IACzB,MAAMK,mBAAmBlB,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO,EAAEU,MAAM,CAAC,CAACmC,IAAM,CAACF,SAAShC,GAAG,CAACkC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIrC;WAAamC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAMzE,gBACJI,MACA;QACE,iGAAiG;QACjG,sFAAsF;QACtF,MAAMsE,UAAU,MAAM9E,WAAWQ;QACjC,MAAMuE,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACqB,MAAQ,CAACmB,QAAQvC,GAAG,CAACoB;QAC/D,IAAIoB,eAAehC,MAAM,GAAG,GAAG,MAAM5B,OAAOO,IAAI,CAAC,SAAS,IAAMf,QAAQqE,UAAU,CAACxE,MAAMuE;QAEzF,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBP;QACrB,IAAIP,MAAMpB,MAAM,GAAG,GACjBkC,iBAAiB,MAAM9D,OAAOO,IAAI,CAAC,iBAAiB;YAClD,MAAMwD,mBAAmB,MAAM1E,KAAKuB,OAAO,CAAC;YAC5C,MAAMoD,eAAe,IAAI7E,IAAI,AAAE,CAAA,MAAM4E,iBAAiBlD,GAAG,EAAC,EAA+BZ,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAajB,MAAM7B,MAAM,CAAC,CAACmC,IAAMU,aAAa5C,GAAG,CAACkC;YACxD,OAAOW,WAAWrC,MAAM,GAAG,IAAI;mBAAI2B;mBAAYU;aAAW,GAAGV;QAC/D;QAEF,IAAIpB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,QAAQ,CAACC,MACbvB,gBAAgB3C,GAAG,CAAC,CAACuC;wBAOZ2B;oBANP,IAAI3B,QAAQ,QAAQ,OAAO2B,IAAI1D,OAAO;oBACtC,IAAI+B,QAAQ,UAAU,OAAO2B,IAAI1C,OAAO;oBACxC,IAAIe,QAAQ,UAAU,OAAO2B,IAAIC,OAAO;oBACxC,IAAI5B,QAAQ,SAAS,OAAO2B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIa,QAAQ,gBAAgB,OAAO2B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC9B,IAAI,cAAb2B,2BAAAA,gBAAiB;gBAC1B;YACF,uFAAuF;YACvF,6FAA6F;YAC7F,MAAMnE,OAAOO,IAAI,CAAC,aAAa;gBAC7B,IAAIf,QAAQ+E,SAAS,EAAE;oBACrB,MAAMC,UAAUrC,WAAWhB,MAAM,CAAC,CAACgC,IAAMC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;oBAC/D,MAAMgE,aAAatC,WAAWhB,MAAM,CAAC,CAACgC,IAAM,CAACC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;oBACnE,IAAI+D,QAAQ5C,MAAM,GAAG,GAAG,MAAMpC,QAAQ+E,SAAS,CAAClF,MAAM,eAAeuD,iBAAiB4B,QAAQvE,GAAG,CAACiE;oBAClG,IAAIO,WAAW7C,MAAM,GAAG,GAAG,MAAMvC,KAAKqF,QAAQ,CAAC5B,WAAW2B,WAAWxE,GAAG,CAACiE;gBAC3E,OAAO;oBACL,MAAM7E,KAAKqF,QAAQ,CAAC5B,WAAWX,WAAWlC,GAAG,CAACiE;gBAChD;YACF;QACF;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMlE,OAAOO,IAAI,CAAC,cAAc,IAAMf,QAAQmF,gBAAgB,CAACtF,MAAMyE,gBAAgB3B,YAAYc,OAAO3D;QAExG,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMU,OAAOO,IAAI,CAAC,WAAW;YAC3B,IAAIgD,QAAQ3B,MAAM,GAAG,GACnB,MAAMvC,KAAKqF,QAAQ,CACjB,6CACAnB,QAAQtD,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;YAE1B,MAAMsB,aAA0B,EAAE;YAClC,KAAK,MAAMT,OAAOhC,WAAY,KAAK,MAAM0C,cAAcV,IAAIW,OAAO,CAAEF,WAAWG,IAAI,CAAC;gBAACZ,IAAI1D,OAAO;gBAAEoE;aAAW;YAC7G,uFAAuF;YACvF,wFAAwF;YACxF,uGAAuG;YACvG,IAAID,WAAWhD,MAAM,GAAG,GAAG,MAAMvC,KAAKqF,QAAQ,CAAC,kGAAkGE;QACnJ;QAEA,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAI1D,SAASU,MAAM,GAAG,GACpB,MAAM5B,OAAOO,IAAI,CAAC,YAAY,IAC5BlB,KAAKqF,QAAQ,CACX,4CACAxD,SAASjB,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;QAI7B,uFAAuF;QACvF,0FAA0F;QAC1F,IAAIC,QAAQ3B,MAAM,GAAG,GAAG,KAAK,MAAMoD,WAAWjF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAaiG,QAAQ7E,IAAI,EAAE,WAAW;gBAAM6E;oBAAAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,qBAAAA,SAAiB3F,MAAMkE,SAASN;;QACtJ,KAAK,MAAM+B,WAAWjF,SAAU;YAC9B,MAAMmF,iBAAiC/C,WAAWlC,GAAG,CAAC,CAACkE,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAE0E,WAAWhB,IAAIgB,SAAS,CAACH,QAAQ7E,IAAI,CAAC;gBAAC,CAAA;YAC5H,MAAMH,OAAOO,IAAI,CAACxB,aAAaiG,QAAQ7E,IAAI,EAAE,UAAU;oBAAM6E;wBAAAA,iBAAAA,QAAQI,KAAK,cAAbJ,qCAAAA,oBAAAA,SAAgB3F,MAAM6F,gBAAgBjC;;QACrG;QACA,KAAK,MAAM+B,WAAWjF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAaiG,QAAQ7E,IAAI,EAAE,UAAU;gBAAM6E;oBAAAA,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyB3F,MAAM4D;;IAC9H,GACAzD,QAAQ8F,SAAS;IAGnB,MAAMC,aAAa9B,KAAKC,GAAG,KAAKF;IAChC,2GAA2G;IAC3G5E,iBAAiBW,SAASgG;IAC1B,IAAI/F,QAAQgG,cAAc,EAAE,MAAMxF,OAAOO,IAAI,CAAC,QAAQ;YAAMf;gBAAAA,0BAAAA,QAAQgG,cAAc,cAAtBhG,8CAAAA,6BAAAA,SAAyBH,MAAMkG;;IAE3F,OAAO;QAAE1D,QAAQM,WAAWP,MAAM;QAAEE;QAAU9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAWmF,YAAYlD;IAAe;AAC1G"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport type { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { recordLockWaitMs } from './lock-wait.ts';\nimport { appendRows, getColumns, quoteIdent } from './shared.ts';\nimport { featureStage, type Stages, stageRecorder } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nexport const CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool?: ParsePool, forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }> {\n  const start = process.hrtime.bigint();\n  const features = activeFeatures(cfg);\n  const stages = stageRecorder(features.map((f) => f.name));\n  const elapsed = () => Number(process.hrtime.bigint() - start) / 1e6;\n  const files = await stages.time('list', () => listFiles(cfg, baseDir));\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = await stages.time('existing', async () => {\n    const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n    return (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  });\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  // A path whose coverage moved between presets (forcedPaths) but is no longer covered at all is\n  // already caught below by !currentSet.has, since it can only be forced by having existed under\n  // an old preset's match, which means it was reconciled into `existing` already.\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  // forcedPaths treats an unchanged file as touched because its preset coverage moved, not its\n  // stamp -- reconcile still owns add/update/remove and every cross-feature cascade for it.\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size || (forcedPaths?.has(f.relPath) ?? false);\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [], stages: stages.take(elapsed(), 0) };\n\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  // Pool wall time, dispatch to drain, so this stage shares a clock with every other one.\n  const { docs: parsedDocs, warnings, newColumns, workerParseMs } = await stages.time('parse', () => reparseFiles(toReparse, features, cfg, seenColumns, report.tick, { pool }));\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      const missingColumns = newColumns.filter((col) => !present.has(col));\n      if (missingColumns.length > 0) await stages.time('alter', () => dialect.addColumns(conn, missingColumns));\n\n      // Revalidated once the lock is held, before this process's own frontmatter upsert below:\n      // `added` came from a path read taken before this transaction's lock, so a path still\n      // called \"added\" here may already have a content row from a concurrent reconcile that\n      // committed while this one waited. One SELECT, not a DELETE per added file -- with no\n      // contention it returns the same set `existing` already ruled out, so nothing extra clears.\n      let contentTouched = touched;\n      if (added.length > 0)\n        contentTouched = await stages.time('added-recheck', async () => {\n          const currentPathsStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n          const currentPaths = new Set(((await currentPathsStmt.all()) as Array<{ path: string }>).map((r) => r.path));\n          const staleAdded = added.filter((p) => currentPaths.has(p));\n          return staleAdded.length > 0 ? [...touched, ...staleAdded] : touched;\n        });\n\n      if (parsedDocs.length > 0) {\n        const toRow = (doc: (typeof parsedDocs)[number]) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          });\n        // A path in `added` has no existing frontmatter row, so it can never conflict; the rest\n        // genuinely can, and keep the upsert.\n        await stages.time('fm-upsert', async () => {\n          const newDocs = parsedDocs.filter((d) => addedSet.has(d.relPath));\n          const updateDocs = parsedDocs.filter((d) => !addedSet.has(d.relPath));\n          await appendRows(conn, 'frontmatter', writableColumns, insertSql, newDocs.map(toRow));\n          if (updateDocs.length > 0) await conn.runBatch(insertSql, updateDocs.map(toRow));\n        });\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await stages.time('text-index', () => dialect.reconcileContent(conn, contentTouched, parsedDocs, delta, cfg));\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      await stages.time('presets', async () => {\n        if (touched.length > 0)\n          await conn.runBatch(\n            'DELETE FROM preset_files WHERE \"path\" = ?',\n            touched.map((p) => [p])\n          );\n        const presetRows: unknown[][] = [];\n        for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n        // DO NOTHING, not a bare INSERT: the added/touched split above comes from a read taken\n        // before this transaction's lock, so a path this process calls \"added\" can already have\n        // its (path, preset) row committed by a concurrent reconcile -- the row would be identical either way.\n        await appendRows(conn, 'preset_files', ['path', 'preset'], 'INSERT INTO preset_files (\"path\", preset) VALUES (?, ?) ON CONFLICT(\"path\", preset) DO NOTHING', presetRows);\n      });\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await stages.time('vanished', () =>\n          conn.runBatch(\n            'DELETE FROM frontmatter WHERE \"path\" = ?',\n            vanished.map((p) => [p])\n          )\n        );\n\n      // Timed per feature per hook, so link resolution and PageRank are named stages without\n      // links.ts or rank.ts knowing anything about this, and a new feature is visible for free.\n      if (touched.length > 0) for (const feature of features) await stages.time(featureStage(feature.name, 'remove'), () => feature.remove?.(conn, touched, delta));\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await stages.time(featureStage(feature.name, 'store'), () => feature.store?.(conn, docsForFeature, delta));\n      }\n      for (const feature of features) await stages.time(featureStage(feature.name, 'after'), () => feature.afterReconcile?.(conn, delta));\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  // Every store, not just the ones with a PRAGMA to derive: connectUnlocked's lock-wait budget needs it too.\n  recordLockWaitMs(baseDir, durationMs);\n  if (dialect.recordDuration) await stages.time('meta', () => dialect.recordDuration?.(conn, durationMs));\n\n  return { parsed: parsedDocs.length, warnings, stages: stages.take(elapsed(), durationMs, workerParseMs) };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","recordLockWaitMs","appendRows","getColumns","quoteIdent","featureStage","stageRecorder","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","pool","forcedPaths","start","process","hrtime","bigint","features","stages","map","f","name","elapsed","Number","files","time","currentSet","relPath","existingRows","existingStmt","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","take","seenColumns","report","docs","parsedDocs","newColumns","workerParseMs","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","present","missingColumns","addColumns","contentTouched","currentPathsStmt","currentPaths","staleAdded","toRow","doc","ctimeMs","parseError","data","newDocs","updateDocs","runBatch","reconcileContent","presetRows","presetName","presets","push","feature","remove","docsForFeature","extracted","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAE/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,gBAAgB,QAAQ,iBAAiB;AAClD,SAASC,UAAU,EAAEC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACjE,SAASC,YAAY,EAAeC,aAAa,QAAQ,cAAc;AACvE,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe,EAAE;AAEvG,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB,EAAEC,IAAgB,EAAEC,WAAiC;IAC5J,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;IACnC,MAAMC,WAAWzB,eAAegB;IAChC,MAAMU,SAAShB,cAAce,SAASE,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvD,MAAMC,UAAU,IAAMC,OAAOT,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;IAChE,MAAMW,QAAQ,MAAMN,OAAOO,IAAI,CAAC,QAAQ,IAAM/B,UAAUc,KAAKC;IAC7D,MAAMiB,aAAa,IAAIrB,IAAImB,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IAErD,MAAMC,eAAe,MAAMV,OAAOO,IAAI,CAAC,YAAY;QACjD,MAAMI,eAAe,MAAMtB,KAAKuB,OAAO,CAAC;QACxC,OAAQ,MAAMD,aAAaE,GAAG;IAChC;IACA,MAAMC,WAAW,IAAIC,IAAIL,aAAaT,GAAG,CAAC,CAACe,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAME,WAAWR,aAAaS,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGhB,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;IAEtF,6FAA6F;IAC7F,0FAA0F;IAC1F,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACjB;;QAC9B,MAAMoB,MAAMR,SAASS,GAAG,CAACrB,EAAEO,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKtB,EAAEuB,OAAO,IAAIH,IAAII,KAAK,KAAKxB,EAAEyB,IAAI,aAAKjC,wBAAAA,kCAAAA,YAAa0B,GAAG,CAAClB,EAAEO,OAAO,wCAAK;IACrG;IAEA,IAAIS,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;QAAE9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAW;IAAG;IAEzH,MAAM4B,cAAc,MAAMnD,WAAWQ;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM4C,SAAS1D,SAAS,mBAAmB8C,UAAUO,MAAM;IAC3D,wFAAwF;IACxF,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAEC,aAAa,EAAE,GAAG,MAAMrC,OAAOO,IAAI,CAAC,SAAS,IAAM7B,aAAa2C,WAAWtB,UAAUT,KAAK0C,aAAaC,OAAOK,IAAI,EAAE;YAAE7C;QAAK;IAC3KwC,OAAOM,MAAM;IACb,KAAK,MAAMC,OAAOJ,WAAYJ,YAAYS,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIV;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjGxC,QAAQmD,gBAAgB,CAACD,WAAWd,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMgB,kBAAkBF,WAAWvB,MAAM,CAAC,CAAC0B,IAAM3D,yBAAyBkC,GAAG,CAACyB,MAAM,CAACpE,iBAAiB2C,GAAG,CAACyB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB3C,GAAG,CAACnB,YAAYiE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB3C,GAAG,CAAC,IAAM,KAAK8C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLzB,MAAM,CAAC,CAAC0B,IAAMA,MAAM,QACpB5C,GAAG,CAAC,CAAC4C,IAAM,GAAG/D,WAAW+D,GAAG,YAAY,EAAE/D,WAAW+D,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ3B,UAAUF,MAAM,CAAC,CAACjB,IAAM,CAACY,SAASM,GAAG,CAAClB,EAAEO,OAAO,GAAGR,GAAG,CAAC,CAACC,IAAMA,EAAEO,OAAO;IACpF,MAAMwC,QAAwB;QAAE3C;QAAO4C,UAAUf,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO;QAAGuC;QAAO9B;IAAS;IACnG,MAAMkC,WAAW,IAAIjE,IAAI6D;IACzB,MAAMK,mBAAmBlB,WAAWlC,GAAG,CAAC,CAACkD,IAAMA,EAAE1C,OAAO,EAAEU,MAAM,CAAC,CAACmC,IAAM,CAACF,SAAShC,GAAG,CAACkC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIrC;WAAamC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAMzE,gBACJI,MACA;QACE,iGAAiG;QACjG,sFAAsF;QACtF,MAAMsE,UAAU,MAAM9E,WAAWQ;QACjC,MAAMuE,iBAAiBxB,WAAWjB,MAAM,CAAC,CAACqB,MAAQ,CAACmB,QAAQvC,GAAG,CAACoB;QAC/D,IAAIoB,eAAehC,MAAM,GAAG,GAAG,MAAM5B,OAAOO,IAAI,CAAC,SAAS,IAAMf,QAAQqE,UAAU,CAACxE,MAAMuE;QAEzF,yFAAyF;QACzF,sFAAsF;QACtF,sFAAsF;QACtF,sFAAsF;QACtF,4FAA4F;QAC5F,IAAIE,iBAAiBP;QACrB,IAAIP,MAAMpB,MAAM,GAAG,GACjBkC,iBAAiB,MAAM9D,OAAOO,IAAI,CAAC,iBAAiB;YAClD,MAAMwD,mBAAmB,MAAM1E,KAAKuB,OAAO,CAAC;YAC5C,MAAMoD,eAAe,IAAI7E,IAAI,AAAE,CAAA,MAAM4E,iBAAiBlD,GAAG,EAAC,EAA+BZ,GAAG,CAAC,CAACe,IAAMA,EAAEC,IAAI;YAC1G,MAAMgD,aAAajB,MAAM7B,MAAM,CAAC,CAACmC,IAAMU,aAAa5C,GAAG,CAACkC;YACxD,OAAOW,WAAWrC,MAAM,GAAG,IAAI;mBAAI2B;mBAAYU;aAAW,GAAGV;QAC/D;QAEF,IAAIpB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMsC,QAAQ,CAACC,MACbvB,gBAAgB3C,GAAG,CAAC,CAACuC;wBAOZ2B;oBANP,IAAI3B,QAAQ,QAAQ,OAAO2B,IAAI1D,OAAO;oBACtC,IAAI+B,QAAQ,UAAU,OAAO2B,IAAI1C,OAAO;oBACxC,IAAIe,QAAQ,UAAU,OAAO2B,IAAIC,OAAO;oBACxC,IAAI5B,QAAQ,SAAS,OAAO2B,IAAIxC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIa,QAAQ,gBAAgB,OAAO2B,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC9B,IAAI,cAAb2B,2BAAAA,gBAAiB;gBAC1B;YACF,wFAAwF;YACxF,sCAAsC;YACtC,MAAMnE,OAAOO,IAAI,CAAC,aAAa;gBAC7B,MAAMgE,UAAUpC,WAAWhB,MAAM,CAAC,CAACgC,IAAMC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBAC/D,MAAM+D,aAAarC,WAAWhB,MAAM,CAAC,CAACgC,IAAM,CAACC,SAAShC,GAAG,CAAC+B,EAAE1C,OAAO;gBACnE,MAAM7B,WAAWS,MAAM,eAAeuD,iBAAiBE,WAAWyB,QAAQtE,GAAG,CAACiE;gBAC9E,IAAIM,WAAW5C,MAAM,GAAG,GAAG,MAAMvC,KAAKoF,QAAQ,CAAC3B,WAAW0B,WAAWvE,GAAG,CAACiE;YAC3E;QACF;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMlE,OAAOO,IAAI,CAAC,cAAc,IAAMf,QAAQkF,gBAAgB,CAACrF,MAAMyE,gBAAgB3B,YAAYc,OAAO3D;QAExG,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMU,OAAOO,IAAI,CAAC,WAAW;YAC3B,IAAIgD,QAAQ3B,MAAM,GAAG,GACnB,MAAMvC,KAAKoF,QAAQ,CACjB,6CACAlB,QAAQtD,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;YAE1B,MAAMqB,aAA0B,EAAE;YAClC,KAAK,MAAMR,OAAOhC,WAAY,KAAK,MAAMyC,cAAcT,IAAIU,OAAO,CAAEF,WAAWG,IAAI,CAAC;gBAACX,IAAI1D,OAAO;gBAAEmE;aAAW;YAC7G,uFAAuF;YACvF,wFAAwF;YACxF,uGAAuG;YACvG,MAAMhG,WAAWS,MAAM,gBAAgB;gBAAC;gBAAQ;aAAS,EAAE,kGAAkGsF;QAC/J;QAEA,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAIzD,SAASU,MAAM,GAAG,GACpB,MAAM5B,OAAOO,IAAI,CAAC,YAAY,IAC5BlB,KAAKoF,QAAQ,CACX,4CACAvD,SAASjB,GAAG,CAAC,CAACqD,IAAM;oBAACA;iBAAE;QAI7B,uFAAuF;QACvF,0FAA0F;QAC1F,IAAIC,QAAQ3B,MAAM,GAAG,GAAG,KAAK,MAAMmD,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,WAAW;gBAAM4E;oBAAAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,qBAAAA,SAAiB1F,MAAMkE,SAASN;;QACtJ,KAAK,MAAM8B,WAAWhF,SAAU;YAC9B,MAAMkF,iBAAiC9C,WAAWlC,GAAG,CAAC,CAACkE,MAAS,CAAA;oBAAElD,MAAMkD,IAAI1D,OAAO;oBAAEyE,WAAWf,IAAIe,SAAS,CAACH,QAAQ5E,IAAI,CAAC;gBAAC,CAAA;YAC5H,MAAMH,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,UAAU;oBAAM4E;wBAAAA,iBAAAA,QAAQI,KAAK,cAAbJ,qCAAAA,oBAAAA,SAAgB1F,MAAM4F,gBAAgBhC;;QACrG;QACA,KAAK,MAAM8B,WAAWhF,SAAU,MAAMC,OAAOO,IAAI,CAACxB,aAAagG,QAAQ5E,IAAI,EAAE,UAAU;gBAAM4E;oBAAAA,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyB1F,MAAM4D;;IAC9H,GACAzD,QAAQ6F,SAAS;IAGnB,MAAMC,aAAa7B,KAAKC,GAAG,KAAKF;IAChC,2GAA2G;IAC3G7E,iBAAiBY,SAAS+F;IAC1B,IAAI9F,QAAQ+F,cAAc,EAAE,MAAMvF,OAAOO,IAAI,CAAC,QAAQ;YAAMf;gBAAAA,0BAAAA,QAAQ+F,cAAc,cAAtB/F,8CAAAA,6BAAAA,SAAyBH,MAAMiG;;IAE3F,OAAO;QAAEzD,QAAQM,WAAWP,MAAM;QAAEE;QAAU9B,QAAQA,OAAO+B,IAAI,CAAC3B,WAAWkF,YAAYjD;IAAe;AAC1G"}

package/dist/esm/store/shared.d.ts

@@ -1,4 +1,5 @@
 import type { Connection } from './types.js';
+export declare function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void>;
 export declare function quoteIdent(name: string): string;
 export declare function getColumns(conn: Connection): Promise<Set<string>>;
 export declare function getMeta(conn: Connection, key: string): Promise<string | null>;

package/dist/esm/store/shared.js

@@ -1,5 +1,13 @@
 // SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both
 // stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).
+// Rows that cannot conflict, at whatever speed this connection offers: the append path where the
+// store has one, else `conflictSql` bound the ordinary way. `conflictSql` stays the caller's own
+// guarded statement, so a store without appendRows behaves exactly as it did before.
+export async function appendRows(conn, table, columns, conflictSql, rows) {
+    if (rows.length === 0) return;
+    if (conn.appendRows) await conn.appendRows(table, columns, rows);
+    else await conn.runBatch(conflictSql, rows);
+}
 export function quoteIdent(name) {
     return `"${name.split('"').join('""')}"`;
 }

package/dist/esm/store/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["quoteIdent","name","split","join","getColumns","conn","stmt","prepare","rows","all","Set","map","r","getMeta","key","row","get","value","setMeta","run","recordReconcileDuration","ms","prevRaw","prevMax","Number","String"],"mappings":"AAEA,2FAA2F;AAC3F,sFAAsF;AAEtF,OAAO,SAASA,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,eAAeC,WAAWC,IAAgB;IAC/C,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMC,OAAQ,MAAMF,KAAKG,GAAG;IAC5B,OAAO,IAAIC,IAAIF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEX,IAAI;AACvC;AAEA,OAAO,eAAeY,QAAQR,IAAgB,EAAES,GAAW;IACzD,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMQ,MAAO,MAAMT,KAAKU,GAAG,CAACF;IAC5B,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQb,IAAgB,EAAES,GAAW,EAAEG,KAAoB;IAC/E,IAAIA,UAAU,MAAM;QAClB,MAAMX,OAAO,MAAMD,KAAKE,OAAO,CAAC;QAChC,MAAMD,KAAKa,GAAG,CAACL;QACf;IACF;IACA,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMD,KAAKa,GAAG,CAACL,KAAKG;AACtB;AAEA,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,eAAeG,wBAAwBf,IAAgB,EAAEgB,EAAU;IACxE,MAAMC,UAAU,MAAMT,QAAQR,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAMkB,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,KAAKE,SAAS,MAAML,QAAQb,MAAM,oBAAoBoB,OAAOJ;AACnE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\n// Rows that cannot conflict, at whatever speed this connection offers: the append path where the\n// store has one, else `conflictSql` bound the ordinary way. `conflictSql` stays the caller's own\n// guarded statement, so a store without appendRows behaves exactly as it did before.\nexport async function appendRows(conn: Connection, table: string, columns: string[], conflictSql: string, rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  if (conn.appendRows) await conn.appendRows(table, columns, rows);\n  else await conn.runBatch(conflictSql, rows);\n}\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["appendRows","conn","table","columns","conflictSql","rows","length","runBatch","quoteIdent","name","split","join","getColumns","stmt","prepare","all","Set","map","r","getMeta","key","row","get","value","setMeta","run","recordReconcileDuration","ms","prevRaw","prevMax","Number","String"],"mappings":"AAEA,2FAA2F;AAC3F,sFAAsF;AAEtF,iGAAiG;AACjG,iGAAiG;AACjG,qFAAqF;AACrF,OAAO,eAAeA,WAAWC,IAAgB,EAAEC,KAAa,EAAEC,OAAiB,EAAEC,WAAmB,EAAEC,IAAiB;IACzH,IAAIA,KAAKC,MAAM,KAAK,GAAG;IACvB,IAAIL,KAAKD,UAAU,EAAE,MAAMC,KAAKD,UAAU,CAACE,OAAOC,SAASE;SACtD,MAAMJ,KAAKM,QAAQ,CAACH,aAAaC;AACxC;AAEA,OAAO,SAASG,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,eAAeC,WAAWX,IAAgB;IAC/C,MAAMY,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMT,OAAQ,MAAMQ,KAAKE,GAAG;IAC5B,OAAO,IAAIC,IAAIX,KAAKY,GAAG,CAAC,CAACC,IAAMA,EAAET,IAAI;AACvC;AAEA,OAAO,eAAeU,QAAQlB,IAAgB,EAAEmB,GAAW;IACzD,MAAMP,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMO,MAAO,MAAMR,KAAKS,GAAG,CAACF;IAC5B,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQvB,IAAgB,EAAEmB,GAAW,EAAEG,KAAoB;IAC/E,IAAIA,UAAU,MAAM;QAClB,MAAMV,OAAO,MAAMZ,KAAKa,OAAO,CAAC;QAChC,MAAMD,KAAKY,GAAG,CAACL;QACf;IACF;IACA,MAAMP,OAAO,MAAMZ,KAAKa,OAAO,CAAC;IAChC,MAAMD,KAAKY,GAAG,CAACL,KAAKG;AACtB;AAEA,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,eAAeG,wBAAwBzB,IAAgB,EAAE0B,EAAU;IACxE,MAAMC,UAAU,MAAMT,QAAQlB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4B,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,KAAKE,SAAS,MAAML,QAAQvB,MAAM,oBAAoB8B,OAAOJ;AACnE"}

package/dist/esm/store/turso/connection.d.ts

@@ -1,3 +1,4 @@
 import type { Database } from '@tursodatabase/database';
 import type { Connection } from '../types.js';
+export declare function checkpointWal(db: Database): Promise<void>;
 export declare function createConnection(db: Database): Connection;

package/dist/esm/store/turso/connection.js

@@ -1,6 +1,6 @@
 import { BEGIN_WRITE, withTransaction } from '../transaction.js';
-// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this
-// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.
+// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%
+// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.
 let TursoStatementWrapper = class TursoStatementWrapper {
     async run(...params) {
         return this.stmt.run(...params);
@@ -24,6 +24,15 @@ let TursoStatementWrapper = class TursoStatementWrapper {
         this.stmt = stmt;
     }
 };
+// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last
+// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.
+export async function checkpointWal(db) {
+    try {
+        await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+    } catch (err) {
+        console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${err.message}`);
+    }
+}
 export function createConnection(db) {
     const conn = {
         async exec (sql) {
@@ -32,15 +41,19 @@ export function createConnection(db) {
         async prepare (sql) {
             return new TursoStatementWrapper(await db.prepare(sql));
         },
-        // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,
-        // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.
+        // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as
+        // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.
         async runBatch (sql, paramRows) {
             if (paramRows.length === 0) return;
             await withTransaction(conn, async ()=>{
-                await db.batch(paramRows.map((args)=>({
-                        sql,
-                        args
-                    })));
+                const stmt = await db.prepare(sql);
+                // Finalized here because nothing else will: open() hands back a connection the caller
+                // can hold across many batches, and the db.batch() this replaced finalized its own.
+                try {
+                    for (const row of paramRows)await stmt.run(...row);
+                } finally{
+                    await stmt.close();
+                }
             }, BEGIN_WRITE);
         }
     };

package/dist/esm/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n    // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          await db.batch(paramRows.map((args) => ({ sql, args })));\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","batch","map","args"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAOjE,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASQ,iBAAiBC,EAAY;IAC3C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,GAAGE,IAAI,CAACC;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIf,sBAAsB,MAAMY,GAAGI,OAAO,CAACD;QACpD;QACA,2FAA2F;QAC3F,uLAAuL;QACvL,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMpB,gBACJc,MACA;gBACE,MAAMD,GAAGQ,KAAK,CAACF,UAAUG,GAAG,CAAC,CAACC,OAAU,CAAA;wBAAEP;wBAAKO;oBAAK,CAAA;YACtD,GACAxB;QAEJ;IACF;IACA,OAAOe;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as\n    // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = await db.prepare(sql);\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","createConnection","conn","sql","prepare","runBatch","paramRows","length","row","close"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAOjE,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,OAAO,SAASC,iBAAiBN,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKO,GAAW;YACpB,MAAMR,GAAGC,IAAI,CAACO;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIpB,sBAAsB,MAAMY,GAAGS,OAAO,CAACD;QACpD;QACA,+FAA+F;QAC/F,wKAAwK;QACxK,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMzB,gBACJoB,MACA;gBACE,MAAMhB,OAAO,MAAMS,GAAGS,OAAO,CAACD;gBAC9B,sFAAsF;gBACtF,oFAAoF;gBACpF,IAAI;oBACF,KAAK,MAAMK,OAAOF,UAAW,MAAMpB,KAAKF,GAAG,IAAIwB;gBACjD,SAAU;oBACR,MAAMtB,KAAKuB,KAAK;gBAClB;YACF,GACA5B;QAEJ;IACF;IACA,OAAOqB;AACT"}

package/dist/esm/store/turso/open.js

@@ -4,7 +4,7 @@ import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
-import { createConnection } from './connection.js';
+import { checkpointWal, createConnection } from './connection.js';
 import { TURSO_PACKAGE, tursoApi } from './native.js';
 import { CONTENT_FTS_DDL, tursoDialect } from './reconcile.js';
 import { createStore } from './store.js';
@@ -33,6 +33,7 @@ async function ensureSchema(_handle, conn, cfg) {
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
 async function close(handle) {
+    await checkpointWal(handle);
     await handle.close();
 }
 async function setDerivedBusyTimeout(_handle, conn, ms) {

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWnB,eAAegB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEnB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMqB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,eAAeqB,MAAMC,MAAgB;IACnC,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAe/B,YAAY,MAAM+B;QACrC,MAAM,IAAI/B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEO,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI/B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE4B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGiB,UAAU,CAACT,MAAQ,iEAAiEU,IAAI,CAACV,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAe0B,UAAUzB,GAAmB;IACjD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","checkpointWal","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,aAAa,EAAEC,gBAAgB,QAAQ,kBAAkB;AAClE,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,eAAesB,MAAMC,MAAgB;IACnC,MAAMnB,cAAcmB;IACpB,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,6BAA6B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGiB,UAAU,CAACT,MAAQ,iEAAiEU,IAAI,CAACV,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAe0B,UAAUzB,GAAmB;IACjD,OAAOf,gBAAgBe,KAAKmB;AAC9B"}

package/dist/esm/store/turso/store.js

@@ -2,6 +2,7 @@ import { STORE_DIMS } from '../../embed/types.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
+import { checkpointWal } from './connection.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
@@ -72,6 +73,7 @@ export function createStore(db, conn) {
             }
         },
         async close () {
+            await checkpointWal(db);
             await db.close();
         }
     };