sensemaking 0.22.2 → 0.23.0

package/dist/cjs/store/sqlite/open.js

@@ -160,7 +160,7 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.db';
-var SCHEMA_VERSION = '19';
+var SCHEMA_VERSION = '20';
 // unicode61 splits on spaces, so a language written without them indexes a whole run as one
 // token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.
 var TOKENIZE = 'porter unicode61';

package/dist/cjs/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '19';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openSqlite","sqliteOpenDialect","TOKENIZE","createContentTable","conn","exec","ensureSchemaTables","cfg","feature","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","DatabaseSync","registerFunctions","createConnection","handle","close","ensureSchema","_handle","withTransaction","BEGIN_WRITE","setDerivedBusyTimeout","_conn","ms","filename","schemaVersion","reconcileDialect","sqliteDialect","createStore","openWithDialect"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAepFA;eAAAA;;QAGAC;eAAAA;;QA6FSC;eAAAA;;QAXTC;eAAAA;;;0BAnGgB;uBAEI;wBACQ;sBAET;wBACC;6BACY;4BAEZ;2BACH;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,+FAA+F;AAC/F,IAAMG,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,SAAeC,mBAAmBC,IAAgB;;;;;oBAChD;;wBAAMA,KAAKC,IAAI,CAAC,AAAC,6IAAqJ,OAATH,UAAS;;;oBAAtK;;;;;;IACF;;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAeI,mBAAmBF,IAAgB,EAAEG,GAAW;;YAOxD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBANX;;wBAAMJ,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMF,mBAAmBC;;;oBAAzB;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBI,IAAAA,wBAAc,EAACF;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACN;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMY,IAAAA,iBAAO,EAACP,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM,YAAYS,IAAAA,yBAAgB,EAACN,KAAKO,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASC,cAAcC,EAAgB;IACrC,IAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGX,IAAI,CAAC;YACR;QACF,EAAE,OAAOe,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YACnDb,IASEZ;;YATFY,KAAK,IAAIc,wBAAY,CAACF;YAC5B,IAAI;gBACF,6FAA6F;gBAC7F,yFAAyF;gBACzFZ,GAAGX,IAAI,CAAC;gBACR,yFAAyF;gBACzF,wEAAwE;gBACxEU,cAAcC;gBACde,IAAAA,iCAAiB,EAACf;gBACZZ,OAAO4B,IAAAA,8BAAgB,EAAChB;gBAC9B;;oBAAO;wBAAEiB,QAAQ;4BAAEjB,IAAAA;wBAAG;wBAAGZ,MAAAA;oBAAK;;YAChC,EAAE,OAAOgB,KAAK;gBACZ,4FAA4F;gBAC5F,0CAA0C;gBAC1CJ,GAAGkB,KAAK;gBACR,MAAMd;YACR;;;;;IACF;;AAEA,SAAec,MAAMD,MAAoB;;;YACvCA,OAAOjB,EAAE,CAACkB,KAAK;;;;;IACjB;;AAEA,SAAeC,aAAaC,OAAqB,EAAEhC,IAAgB,EAAEG,GAAW;;;;;oBAC9E,+FAA+F;oBAC/F,+EAA+E;oBAC/E;;wBAAM8B,IAAAA,8BAAe,EAACjC,MAAM;mCAAME,mBAAmBF,MAAMG;2BAAM+B,0BAAW;;;oBAA5E;;;;;;IACF;;AAEA,SAAeC,sBAAsBN,MAAoB,EAAEO,KAAiB,EAAEC,EAAU;;;YACtFR,OAAOjB,EAAE,CAACX,IAAI,CAAC,AAAC,yBAA2B,OAAHoC;;;;;IAC1C;;AAGO,IAAMxC,oBAA+C;IAC1DyC,UAAU5C;IACV6C,eAAe5C;IACf6C,kBAAkBC,0BAAa;IAC/BlB,SAAAA;IACAO,OAAAA;IACAC,cAAAA;IACAI,uBAAAA;IACAO,aAAa,SAAbA,YAAcb,QAAQ7B;eAAS0C,IAAAA,oBAAW,EAACb,OAAOjB,EAAE,EAAEZ;;AACxD;AAEO,SAAeJ,WAAWO,GAAmB;;;YAClD;;gBAAOwC,IAAAA,uBAAe,EAACxC,KAAKN;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '20';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openSqlite","sqliteOpenDialect","TOKENIZE","createContentTable","conn","exec","ensureSchemaTables","cfg","feature","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","DatabaseSync","registerFunctions","createConnection","handle","close","ensureSchema","_handle","withTransaction","BEGIN_WRITE","setDerivedBusyTimeout","_conn","ms","filename","schemaVersion","reconcileDialect","sqliteDialect","createStore","openWithDialect"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAepFA;eAAAA;;QAGAC;eAAAA;;QA6FSC;eAAAA;;QAXTC;eAAAA;;;0BAnGgB;uBAEI;wBACQ;sBAET;wBACC;6BACY;4BAEZ;2BACH;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,+FAA+F;AAC/F,IAAMG,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,SAAeC,mBAAmBC,IAAgB;;;;;oBAChD;;wBAAMA,KAAKC,IAAI,CAAC,AAAC,6IAAqJ,OAATH,UAAS;;;oBAAtK;;;;;;IACF;;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAeI,mBAAmBF,IAAgB,EAAEG,GAAW;;YAOxD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBANX;;wBAAMJ,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMF,mBAAmBC;;;oBAAzB;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBI,IAAAA,wBAAc,EAACF;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACN;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMY,IAAAA,iBAAO,EAACP,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM,YAAYS,IAAAA,yBAAgB,EAACN,KAAKO,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASC,cAAcC,EAAgB;IACrC,IAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGX,IAAI,CAAC;YACR;QACF,EAAE,OAAOe,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YACnDb,IASEZ;;YATFY,KAAK,IAAIc,wBAAY,CAACF;YAC5B,IAAI;gBACF,6FAA6F;gBAC7F,yFAAyF;gBACzFZ,GAAGX,IAAI,CAAC;gBACR,yFAAyF;gBACzF,wEAAwE;gBACxEU,cAAcC;gBACde,IAAAA,iCAAiB,EAACf;gBACZZ,OAAO4B,IAAAA,8BAAgB,EAAChB;gBAC9B;;oBAAO;wBAAEiB,QAAQ;4BAAEjB,IAAAA;wBAAG;wBAAGZ,MAAAA;oBAAK;;YAChC,EAAE,OAAOgB,KAAK;gBACZ,4FAA4F;gBAC5F,0CAA0C;gBAC1CJ,GAAGkB,KAAK;gBACR,MAAMd;YACR;;;;;IACF;;AAEA,SAAec,MAAMD,MAAoB;;;YACvCA,OAAOjB,EAAE,CAACkB,KAAK;;;;;IACjB;;AAEA,SAAeC,aAAaC,OAAqB,EAAEhC,IAAgB,EAAEG,GAAW;;;;;oBAC9E,+FAA+F;oBAC/F,+EAA+E;oBAC/E;;wBAAM8B,IAAAA,8BAAe,EAACjC,MAAM;mCAAME,mBAAmBF,MAAMG;2BAAM+B,0BAAW;;;oBAA5E;;;;;;IACF;;AAEA,SAAeC,sBAAsBN,MAAoB,EAAEO,KAAiB,EAAEC,EAAU;;;YACtFR,OAAOjB,EAAE,CAACX,IAAI,CAAC,AAAC,yBAA2B,OAAHoC;;;;;IAC1C;;AAGO,IAAMxC,oBAA+C;IAC1DyC,UAAU5C;IACV6C,eAAe5C;IACf6C,kBAAkBC,0BAAa;IAC/BlB,SAAAA;IACAO,OAAAA;IACAC,cAAAA;IACAI,uBAAAA;IACAO,aAAa,SAAbA,YAAcb,QAAQ7B;eAAS0C,IAAAA,oBAAW,EAACb,OAAOjB,EAAE,EAAEZ;;AACxD;AAEO,SAAeJ,WAAWO,GAAmB;;;YAClD;;gBAAOwC,IAAAA,uBAAe,EAACxC,KAAKN;;;IAC9B"}

package/dist/cjs/store/turso/open.d.cts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "4";
+export declare const SCHEMA_VERSION = "5";
 export type { OpenResult };
 export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "4";
+export declare const SCHEMA_VERSION = "5";
 export type { OpenResult };
 export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.js

@@ -164,7 +164,7 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.turso.db';
-var SCHEMA_VERSION = '4';
+var SCHEMA_VERSION = '5';
 // This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
 // extra native state to thread, unlike duckdb's separate instance/connection pair.
 function ensureSchema(_handle, conn, cfg) {

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","checkpointWal","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAwESC;eAAAA;;QAfTC;eAAAA;;;uBAzEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEe;wBACR;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMC,IAAAA,2BAAa,EAACD;;;oBAApB;oBACA;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeG,sBAAsBlB,OAAiB,EAAEC,IAAgB,EAAEkB,EAAU;;;;;oBAClF;;wBAAMlB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHc;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEb,QAAQS;4BAAIxB,MAAM+B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM3B,mBAA0C;IACrDmC,UAAUtC;IACVuC,eAAetC;IACfuC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAL,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGsB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG9B,cAAAA;IACAmB,uBAAAA;IACAqB,aAAa,SAAbA,YAAcvB,QAAQf;eAASsC,IAAAA,oBAAW,EAACvB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOsC,IAAAA,uBAAe,EAACtC,KAAKJ;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","checkpointWal","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAwESC;eAAAA;;QAfTC;eAAAA;;;uBAzEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEe;wBACR;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMC,IAAAA,2BAAa,EAACD;;;oBAApB;oBACA;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeG,sBAAsBlB,OAAiB,EAAEC,IAAgB,EAAEkB,EAAU;;;;;oBAClF;;wBAAMlB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHc;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEb,QAAQS;4BAAIxB,MAAM+B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM3B,mBAA0C;IACrDmC,UAAUtC;IACVuC,eAAetC;IACfuC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAL,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGsB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG9B,cAAAA;IACAmB,uBAAAA;IACAqB,aAAa,SAAbA,YAAcvB,QAAQf;eAASsC,IAAAA,oBAAW,EAACvB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOsC,IAAAA,uBAAe,EAACtC,KAAKJ;;;IAC9B"}

package/dist/cjs/text/strip.js

@@ -19,9 +19,7 @@ _export(exports, {
 var _indexts = require("../chunk/index.js");
 var _frontmatterts = require("../scan/frontmatter.js");
 function textFromBlocks(blocks) {
-    return (0, _frontmatterts.normalizeText)((0, _indexts.extractTexts)(blocks.map(function(block) {
-        return block.node;
-    })));
+    return (0, _frontmatterts.normalizeText)((0, _indexts.extractTexts)(blocks));
 }
 function stripText(value) {
     return textFromBlocks((0, _indexts.parse)(value));

package/dist/cjs/text/strip.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks.map((block) => block.node)));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["stripText","textFromBlocks","blocks","normalizeText","extractTexts","map","block","node","value","parse"],"mappings":";;;;;;;;;;;QAWgBA;eAAAA;;QALAC;eAAAA;;;uBALoB;6BACN;AAIvB,SAASA,eAAeC,MAAe;IAC5C,OAAOC,IAAAA,4BAAa,EAACC,IAAAA,qBAAY,EAACF,OAAOG,GAAG,CAAC,SAACC;eAAUA,MAAMC,IAAI;;AACpE;AAGO,SAASP,UAAUQ,KAAa;IACrC,OAAOP,eAAeQ,IAAAA,cAAK,EAACD;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["stripText","textFromBlocks","blocks","normalizeText","extractTexts","value","parse"],"mappings":";;;;;;;;;;;QAWgBA;eAAAA;;QALAC;eAAAA;;;uBALoB;6BACN;AAIvB,SAASA,eAAeC,MAAe;IAC5C,OAAOC,IAAAA,4BAAa,EAACC,IAAAA,qBAAY,EAACF;AACpC;AAGO,SAASF,UAAUK,KAAa;IACrC,OAAOJ,eAAeK,IAAAA,cAAK,EAACD;AAC9B"}

package/dist/cjs/workers/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\n// parseMs is the worker's own hrtime for parseFile, excluding dispatch and the return clone --\n// what separates it from the pool's dispatch-to-drain `parse` stage.\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[]; parseMs: number } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const start = process.hrtime.bigint();\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    const parseMs = Number(process.hrtime.bigint() - start) / 1e6;\n    return { ok: true, doc, warnings, parseMs };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["parseTask","Tinypool","workerData","cfg","featureNames","selected","FEATURES","filter","feature","includes","name","file","start","process","hrtime","bigint","parseFile","featuresForFile","doc","warnings","parseMs","Number","ok","err","error","serializeError"],"mappings":";;;;+BA+BA;;;eAAwBA;;;+DA/BH;uBAEI;wBAEC;yBAEM;6BAED;;;;;;AAiB/B,sCAAsC;AACtC,IAA8BC,uBAAAA,iBAAQ,CAACC,UAAU,EAAzCC,MAAsBF,qBAAtBE,KAAKC,eAAiBH,qBAAjBG;AACb,4FAA4F;AAC5F,qDAAqD;AACrD,IAAMC,WAAWC,iBAAQ,CAACC,MAAM,CAAC,SAACC;WAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;;AAEjE,SAASV,UAAUW,IAAe;IAC/C,IAAI;QACF,IAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;QACnC,IAA0BC,aAAAA,IAAAA,mBAAS,EAACL,MAAMM,IAAAA,0BAAe,EAACZ,UAAUF,KAAKQ,OAAOR,MAAxEe,MAAkBF,WAAlBE,KAAKC,WAAaH,WAAbG;QACb,IAAMC,UAAUC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QAC1D,OAAO;YAAEU,IAAI;YAAMJ,KAAAA;YAAKC,UAAAA;YAAUC,SAAAA;QAAQ;IAC5C,EAAE,OAAOG,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOC,IAAAA,6BAAc,EAACF;QAAK;IACjD;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the token tree.\nexport type ParseTask = FileStat;\n\n// parseMs is the worker's own hrtime for parseFile, excluding dispatch and the return clone --\n// what separates it from the pool's dispatch-to-drain `parse` stage.\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[]; parseMs: number } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const start = process.hrtime.bigint();\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    const parseMs = Number(process.hrtime.bigint() - start) / 1e6;\n    return { ok: true, doc, warnings, parseMs };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["parseTask","Tinypool","workerData","cfg","featureNames","selected","FEATURES","filter","feature","includes","name","file","start","process","hrtime","bigint","parseFile","featuresForFile","doc","warnings","parseMs","Number","ok","err","error","serializeError"],"mappings":";;;;+BA+BA;;;eAAwBA;;;+DA/BH;uBAEI;wBAEC;yBAEM;6BAED;;;;;;AAiB/B,sCAAsC;AACtC,IAA8BC,uBAAAA,iBAAQ,CAACC,UAAU,EAAzCC,MAAsBF,qBAAtBE,KAAKC,eAAiBH,qBAAjBG;AACb,4FAA4F;AAC5F,qDAAqD;AACrD,IAAMC,WAAWC,iBAAQ,CAACC,MAAM,CAAC,SAACC;WAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;;AAEjE,SAASV,UAAUW,IAAe;IAC/C,IAAI;QACF,IAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;QACnC,IAA0BC,aAAAA,IAAAA,mBAAS,EAACL,MAAMM,IAAAA,0BAAe,EAACZ,UAAUF,KAAKQ,OAAOR,MAAxEe,MAAkBF,WAAlBE,KAAKC,WAAaH,WAAbG;QACb,IAAMC,UAAUC,OAAOR,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QAC1D,OAAO;YAAEU,IAAI;YAAMJ,KAAAA;YAAKC,UAAAA;YAAUC,SAAAA;QAAQ;IAC5C,EAAE,OAAOG,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOC,IAAAA,6BAAc,EAACF;QAAK;IACjD;AACF"}

package/dist/esm/chunk/extract.d.ts

@@ -1,3 +1,3 @@
-import type { Nodes } from 'mdast';
-export declare function extractText(node: Nodes): string;
-export declare function extractTexts(nodes: Nodes[]): string;
+import type { Block } from './types.js';
+export declare function extractText(block: Block): string;
+export declare function extractTexts(blocks: Block[]): string;

package/dist/esm/chunk/extract.js

@@ -1,5 +1,6 @@
-// mdast has no wikilink, embed, or %%comment%% node -- literal text resolved by regex passes
-// over non-code text (code/inlineCode is placeholder-held and spliced back verbatim, below).
+import { parser } from './parser.js';
+// markdown-it, like mdast, has no wikilink, embed, or %%comment%% token -- literal text resolved
+// by regex passes over non-code text (code/inline code is placeholder-held and spliced back verbatim).
 const COMMENT_PAIR = /%%[\s\S]*?%%/g;
 // An embed always yields its target, whatever follows a pipe (a resize suffix, never an alias);
 // must run before the wikilink regex below, bang included, or that regex matches first.
@@ -11,7 +12,7 @@ const BLOCK_ID = / \^[A-Za-z0-9-]+$/gm;
 // A callout marker is only ever the first thing in a blockquote's text (Obsidian's grammar),
 // so this anchors to blockquote output alone -- never applied to prose in general.
 const CALLOUT_MARKER = /^\[!\w[\w-]*\][+-]?[ \t]?/;
-// An html node's value is raw HTML, block or inline: a full <!-- --> comment is dropped, any
+// An html token's content is raw HTML, block or inline: a full <!-- --> comment is dropped, any
 // remaining tags are stripped, and the text a browser would still render survives.
 const HTML_COMMENT = /<!--[\s\S]*?-->/g;
 const HTML_TAG = /<[^>]+>/g;
@@ -38,85 +39,297 @@ function embedText(inner) {
 function resolveFlavor(text) {
     return text.replace(HTML_COMMENT, '').replace(COMMENT_PAIR, '').replace(EMBED, (_, inner)=>embedText(inner)).replace(WIKILINK, (_, inner)=>wikilinkText(inner)).replace(BLOCK_ID, '');
 }
-// Sibling blocks (list items, blockquote content, footnote bodies) joined one per line;
-// empties dropped so a blank child never widens the gap between its neighbors.
-function joinBlocks(nodes, code) {
-    return nodes.map((n)=>extractNode(n, code)).filter((s)=>s.length > 0).join('\n');
-}
-// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the
-// source text nodes already carry whatever spacing separates them.
-function joinInline(nodes, code) {
-    return nodes.map((n)=>extractNode(n, code)).join('');
-}
 // U+E000 (Private Use Area) never appears in real markdown text, so it is a collision-free
 // placeholder delimiter, spliced back to the literal code value after flavor resolution.
 function codePlaceholder(value, code) {
     const idx = code.push(value) - 1;
     return `\uE000${idx}\uE000`;
 }
-// A GFM autolink's display text is its own target, missing only the scheme
-// mdast-util-gfm-autolink-literal fills into node.url (http:// for www., mailto: for an email).
-const SCHEME_PREFIX = /^[a-z][a-z0-9+.-]*:\/{0,2}/i;
-function isAutolink(url, text) {
-    return url === text || url.replace(SCHEME_PREFIX, '') === text;
-}
-function extractNode(node, code) {
-    switch(node.type){
-        case 'root':
-        case 'list':
-        case 'listItem':
-        case 'footnoteDefinition':
-            return joinBlocks(node.children, code);
-        case 'blockquote':
-            return joinBlocks(node.children, code).replace(CALLOUT_MARKER, '');
-        case 'link':
+// GFM's autolink literal ends where its path machine stops: a trail of punctuation is a genuine
+// end (link stops before it) only when trailEndsAt accepts it, else the marks are part of the link.
+const PUNCT = new Set([
+    '!',
+    '"',
+    '&',
+    "'",
+    ')',
+    '*',
+    ',',
+    '.',
+    ';',
+    ':',
+    '<',
+    '?',
+    '_',
+    '~',
+    ']'
+]);
+const TRAIL_PUNCT = new Set([
+    '!',
+    '"',
+    "'",
+    ')',
+    '*',
+    ',',
+    '.',
+    ';',
+    ':',
+    '?',
+    '_',
+    '~'
+]);
+const isAlpha = (c)=>c !== undefined && (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z');
+function trailEndsAt(s, j) {
+    for(;;){
+        const c = s[j];
+        if (c === undefined) return true;
+        if (TRAIL_PUNCT.has(c)) {
+            j += 1;
+            continue;
+        }
+        if (c === '&') {
+            j += 1;
+            if (!isAlpha(s[j])) return false;
+            j += 1;
+            while(isAlpha(s[j]))j += 1;
+            if (s[j] !== ';') return false;
+            j += 1;
+            continue;
+        }
+        if (c === ']') {
+            j += 1;
+            const d = s[j];
+            return d === undefined || d === '(' || d === '[' || /\s/u.test(d !== null && d !== void 0 ? d : ' ');
+        }
+        if (c === '<') return true;
+        if (c !== undefined && /^\s$/u.test(c)) return true;
+        return false;
+    }
+}
+// Where the GFM autolink ends in S (the linkifier's text plus the following text), old-style:
+// punctuation that fails the trail test, or a ) with closes <= opens, extends the link.
+function gfmAutolinkEnd(s) {
+    let open = 0;
+    let close = 0;
+    for(let i = 0; i < s.length; i++){
+        const c = s[i];
+        if (c === '(') {
+            open += 1;
+            continue;
+        }
+        if (c === ')' && close < open) {
+            close += 1;
+            continue;
+        }
+        if (PUNCT.has(c) && trailEndsAt(s, i)) return i;
+        if (c !== undefined && /^\s$/u.test(c)) return i;
+    }
+    return s.length;
+}
+// linkify-it links more than GFM's autolink literal (bare domains, ftp://, //host); only the
+// GFM forms (http(s)://, www., email) drop their text.
+function isGfmUrl(text) {
+    return /^https?:\/\//i.test(text) || /^www\./i.test(text);
+}
+// Only a bare address is GFM's email autolink: a url keeps its @, since a path can hold one.
+function isGfmEmail(text) {
+    return !isGfmUrl(text) && text.includes('@');
+}
+// The task-lists plugin injects its checkbox as the item's first inline child.
+function isTaskCheckbox(token) {
+    return (token === null || token === void 0 ? void 0 : token.type) === 'html_inline' && /^<input\b/i.test(token.content);
+}
+// A linkify span can swallow the & of a following entity (span ...c& + text amp; z); the
+// re-emitted lead is then a reference the entity rule never saw, so decode it old-style.
+const LEAD_ENTITY = /^&[a-z#][a-z0-9]{1,31};/i;
+// Split a flat token range into its top-level blocks: nesting 1 opens, nesting 0 is one block.
+function topLevelBlocks(tokens) {
+    const out = [];
+    let i = 0;
+    while(i < tokens.length){
+        const token = tokens[i];
+        if (token.nesting === 1) {
+            let depth = 1;
+            let j = i + 1;
+            while(j < tokens.length && depth > 0){
+                depth += tokens[j].nesting === 1 ? 1 : tokens[j].nesting === -1 ? -1 : 0;
+                j += 1;
+            }
+            out.push(tokens.slice(i, j));
+            i = j;
+        } else if (token.nesting === 0) {
+            out.push(tokens.slice(i, i + 1));
+            i += 1;
+        } else {
+            i += 1;
+        }
+    }
+    return out;
+}
+// A first line with no source content (a bare task checkbox, whitespace) contributes no line to
+// the value, so its softbreak is dropped; a first line whose content extraction removes (an
+// autolink, an image) still leaves its line ending behind.
+function paragraphText(children, code) {
+    var _children_k;
+    let k = 0;
+    const first = children[0];
+    if (isTaskCheckbox(first)) k = 1;
+    while(k < children.length && children[k].type === 'text' && children[k].content.trim() === '')k += 1;
+    if (k > 0 && ((_children_k = children[k]) === null || _children_k === void 0 ? void 0 : _children_k.type) === 'softbreak') return inlineText(children.slice(k + 1), code);
+    return inlineText(children, code);
+}
+function blockText(tokens, code) {
+    const first = tokens[0];
+    if (!first) return '';
+    switch(first.type){
+        case 'heading_open':
+        case 'paragraph_open':
             {
-                const text = joinInline(node.children, code);
-                return isAutolink(node.url, text) ? '' : text;
+                var _inline_children;
+                const inline = tokens.find((t)=>t.type === 'inline');
+                return inline ? paragraphText((_inline_children = inline.children) !== null && _inline_children !== void 0 ? _inline_children : [], code) : '';
             }
-        case 'heading':
-        case 'paragraph':
-        case 'linkReference':
-        case 'emphasis':
-        case 'strong':
-        case 'delete':
-            return joinInline(node.children, code);
-        case 'table':
-            return node.children.map((n)=>extractNode(n, code)).join('\n');
-        case 'tableRow':
-            return node.children.map((n)=>extractNode(n, code)).join(' ');
-        case 'tableCell':
-            return joinInline(node.children, code);
-        case 'image':
-        case 'imageReference':
-            var _node_alt;
-            return (_node_alt = node.alt) !== null && _node_alt !== void 0 ? _node_alt : '';
-        case 'text':
-            return node.value;
-        case 'inlineCode':
-        case 'code':
-            return codePlaceholder(node.value, code);
-        case 'html':
-            return stripHtml(node.value);
-        case 'break':
-            return ' ';
+        case 'fence':
+        case 'code_block':
+            // Fence and indented code tokens carry the closing newline their lines were joined with.
+            return codePlaceholder(first.content.replace(/\n$/, ''), code);
+        case 'table_open':
+            return tableText(tokens, code);
+        case 'ordered_list_open':
+        case 'bullet_list_open':
+            return listItemsText(tokens, code);
+        case 'blockquote_open':
+            return blocksIn(tokens.slice(1, -1), code).replace(CALLOUT_MARKER, '');
+        case 'html_block':
+            // A non-final-line html block carries its last line's terminator, which the mdast value lacked.
+            return stripHtml(first.content.replace(/\n$/, ''));
+        case 'footnote_reference_open':
+            return blocksIn(tokens.slice(1, -1), code);
         default:
             return '';
     }
 }
+// Sibling blocks (list items, blockquote children, footnote bodies) joined one per line;
+// empties dropped so a blank child never widens the gap between its neighbors.
+function blocksIn(tokens, code) {
+    return topLevelBlocks(tokens).map((b)=>blockText(b, code)).filter((s)=>s.length > 0).join('\n');
+}
+// The task-lists plugin slices 3 of the marker's 4 chars, leaving its trailing space on the text
+// after the checkbox token. Returns a copy with that space dropped, never a mutation: a block is
+// extracted more than once (parse pre-extracts headings, group.ts extracts again).
+function withoutTaskMarker(item) {
+    for(let i = 0; i < item.length; i++){
+        var _token_children;
+        const token = item[i];
+        // The checkbox, if any, sits in the item's first inline only.
+        if (token.type !== 'inline' || !((_token_children = token.children) === null || _token_children === void 0 ? void 0 : _token_children.length)) continue;
+        const second = token.children[1];
+        if (!isTaskCheckbox(token.children[0]) || (second === null || second === void 0 ? void 0 : second.type) !== 'text' || !second.content.startsWith(' ')) return item;
+        const children = token.children.slice();
+        children[1] = {
+            ...second,
+            content: second.content.slice(1)
+        };
+        const copy = item.slice();
+        copy[i] = {
+            ...token,
+            children
+        };
+        return copy;
+    }
+    return item;
+}
+function listItemsText(tokens, code) {
+    const items = [];
+    for (const item of topLevelBlocks(tokens.slice(1, -1))){
+        items.push(blocksIn(withoutTaskMarker(item).slice(1, -1), code));
+    }
+    return items.filter((s)=>s.length > 0).join('\n');
+}
+// Table rows joined by newline, cells by space (mdast's tableRow/tableCell joiners).
+function tableText(tokens, code) {
+    const rows = [];
+    let cells = [];
+    for (const token of tokens){
+        var _token_children;
+        if (token.type === 'tr_open') cells = [];
+        else if (token.type === 'inline') cells.push(inlineText((_token_children = token.children) !== null && _token_children !== void 0 ? _token_children : [], code));
+        else if (token.type === 'tr_close') rows.push(cells.join(' '));
+    }
+    return rows.join('\n');
+}
+// Inline content (headings, paragraphs, emphasis, links) concatenated with no separator: the
+// source text tokens already carry whatever spacing separates them.
+function inlineText(tokens, code, inAlt = false) {
+    let out = '';
+    for(let i = 0; i < tokens.length; i++){
+        const token = tokens[i];
+        if (token.type === 'link_open') {
+            let skip = 0;
+            let j = i + 1;
+            let depth = 1;
+            while(j < tokens.length && depth > 0){
+                if (tokens[j].type === 'link_open') depth += 1;
+                else if (tokens[j].type === 'link_close') depth -= 1;
+                j += 1;
+            }
+            const text = inlineText(tokens.slice(i + 1, j - 1), code, inAlt);
+            const markup = token.markup;
+            const next = tokens[j];
+            if (inAlt) {
+                // An image's alt is plain text: mdast kept every link's own text there, autolinks included.
+                out += text;
+            } else if (markup === 'linkify' && isGfmUrl(text)) {
+                // GFM drop: the next text token resumes at the old link's boundary, re-emitting any
+                // trimmed trail. <www.x> has no trail to absorb (the brackets bound it), keep the >.
+                // Emails keep their whole remainder: GFM's email machine has no trail, so the span
+                // already ends at the old boundary and the entity rule has decoded the rest.
+                const prev = tokens[i - 1];
+                const bracketed = (prev === null || prev === void 0 ? void 0 : prev.type) === 'text' && prev.content.endsWith('<') && (next === null || next === void 0 ? void 0 : next.type) === 'text' && next.content.startsWith('>');
+                const tail = (next === null || next === void 0 ? void 0 : next.type) === 'text' ? text + next.content : text;
+                const k = bracketed ? text.length : gfmAutolinkEnd(tail);
+                out += tail.slice(k).replace(LEAD_ENTITY, (m)=>parser().utils.unescapeAll(m));
+                // The trail came out of the next text token, so that token is consumed here, not re-read.
+                if ((next === null || next === void 0 ? void 0 : next.type) === 'text') skip = 1;
+            } else if (markup === 'linkify' && isGfmEmail(text)) {
+            // Email drop: nothing after the span to trim, so the span alone is dropped.
+            } else if (markup === 'autolink') {
+            // <...> leaf: dropped whole, its brackets live inside the token so nothing follows to trim.
+            } else {
+                out += text; // user link or non-GFM linkify target: keep the display text
+            }
+            i = j - 1 + skip;
+        } else if (token.type === 'text') {
+            out += token.content;
+        } else if (token.type === 'softbreak') {
+            out += '\n';
+        } else if (token.type === 'hardbreak') {
+            out += ' ';
+        } else if (token.type === 'code_inline') {
+            out += codePlaceholder(token.content, code);
+        } else if (token.type === 'html_inline') {
+            out += stripHtml(token.content);
+        } else if (token.type === 'image') {
+            var _token_children;
+            out += inlineText((_token_children = token.children) !== null && _token_children !== void 0 ? _token_children : [], code, true);
+        }
+    // Emphasis, strikethrough and footnote markers carry no text of their own.
+    }
+    return out;
+}
 const CODE_PLACEHOLDER = /\uE000(\d+)\uE000/g;
-// Plain text of one mdast node (or a whole tree): heading/list/table structure is kept as text,
-// markup (emphasis, link targets, task and callout markers) is dropped. Pure, synchronous.
-export function extractText(node) {
+// Plain text of one parsed block: heading/list/table structure is kept as text, markup (emphasis,
+// link targets, task and callout markers) is dropped. Pure, synchronous.
+export function extractText(block) {
     return extractTexts([
-        node
+        block
     ]);
 }
 // Blocks resolved together: a blank line inside %%...%% splits it across blocks, and a per-block
 // strip then never sees the closing %%. Code stays placeholder-held across all of them.
-export function extractTexts(nodes) {
+export function extractTexts(blocks) {
     const code = [];
-    const joined = nodes.map((n)=>extractNode(n, code)).filter((s)=>s.length > 0).join('\n');
+    const joined = blocks.map((b)=>blockText(b.node, code)).filter((s)=>s.length > 0).join('\n');
     const resolved = resolveFlavor(joined);
     return code.length === 0 ? resolved : resolved.replace(CODE_PLACEHOLDER, (_, i)=>code[Number(i)]);
 }