sensemaking 0.22.2 → 0.23.0

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","feature","name","schema","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","message","create","storage_compatibility_version","duckdbOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openDuckdb"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAWlC,8LAA8L;AAC9L,gGAAgG;AAChG,eAAeC,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;IAC9E,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,iFAAiF;AACjF,eAAeoB,MAAMC,MAAoB;IACvCA,OAAOC,MAAM,CAACC,cAAc;IAC5BF,OAAOG,QAAQ,CAACC,SAAS;AAC3B;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,gIAAgI;IAChI,qLAAqL;IACrL,IAAIC;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMvB,WAAU;IACxC,EAAE,OAAOwB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACyB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIT;IACJ,IAAIE;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMK,eAAeG,MAAM,CAACL,QAAQ;YAAEM,+BAA+B;QAAS;QACzFX,SAAS,MAAME,SAASE,OAAO;IACjC,EAAE,OAAOI,KAAK;QACZ,wFAAwF;QACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;QACnB,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,8BAA8B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACvH;IACA,4GAA4G;IAC5G,IAAI;QACF,MAAMvB,kBAAkBc;QACxB,MAAMR,OAAOV,iBAAiBkB;QAC9B,OAAO;YAAED,QAAQ;gBAAEG;gBAAUF;YAAO;YAAGR;QAAK;IAC9C,EAAE,OAAOgB,KAAK;QACZ,MAAMV,MAAM;YAAEI;YAAUF;QAAO;QAC/B,MAAMQ;IACR;AACF;AAEA,+FAA+F;AAC/F,OAAO,MAAMI,oBAA+C;IAC1DC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB9B;IAClBmB;IACAN;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFkB,UAAU,CAACR,MAAQ,2DAA2DS,IAAI,CAACT,IAAIC,OAAO;IAC9FnB;IACAH,aAAa,CAACY,QAAQP,OAASL,YAAYY,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAER;AAC7E,EAAE;AAEF,OAAO,eAAe0B,WAAWzB,GAAmB;IAClD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}

package/dist/esm/store/sqlite/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "19";
+export declare const SCHEMA_VERSION = "20";
 export type { OpenResult };
 interface SqliteHandle {
     db: DatabaseSync;

package/dist/esm/store/sqlite/open.js

@@ -13,7 +13,7 @@ import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '19';
+export const SCHEMA_VERSION = '20';
 // unicode61 splits on spaces, so a language written without them indexes a whole run as one
 // token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.
 const TOKENIZE = 'porter unicode61';

package/dist/esm/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '19';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DatabaseSync","featureSignature","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","TOKENIZE","createContentTable","conn","exec","ensureSchemaTables","cfg","feature","schema","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","handle","close","ensureSchema","_handle","setDerivedBusyTimeout","_conn","ms","sqliteOpenDialect","filename","schemaVersion","reconcileDialect","openSqlite"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAInC,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMC,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeC,mBAAmBC,IAAgB;IAChD,MAAMA,KAAKC,IAAI,CAAC,CAAC,0IAA0I,EAAEH,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeI,mBAAmBF,IAAgB,EAAEG,GAAW;IAC7D,MAAMH,KAAKC,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,mBAAmBC;IACzB,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMA,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,KAAK,MAAMG,WAAWnB,eAAekB,KAAM,MAAMC,QAAQC,MAAM,CAACL;IAChE,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYhB,iBAAiBmB,KAAKjB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASoB,cAAcC,EAAgB;IACrC,MAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGN,IAAI,CAAC;YACR;QACF,EAAE,OAAOU,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,MAAMb,KAAK,IAAIxB,aAAaoC;IAC5B,IAAI;QACF,6FAA6F;QAC7F,yFAAyF;QACzFZ,GAAGN,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEK,cAAcC;QACdb,kBAAkBa;QAClB,MAAMP,OAAOR,iBAAiBe;QAC9B,OAAO;YAAEc,QAAQ;gBAAEd;YAAG;YAAGP;QAAK;IAChC,EAAE,OAAOW,KAAK;QACZ,4FAA4F;QAC5F,0CAA0C;QAC1CJ,GAAGe,KAAK;QACR,MAAMX;IACR;AACF;AAEA,eAAeW,MAAMD,MAAoB;IACvCA,OAAOd,EAAE,CAACe,KAAK;AACjB;AAEA,eAAeC,aAAaC,OAAqB,EAAExB,IAAgB,EAAEG,GAAW;IAC9E,+FAA+F;IAC/F,+EAA+E;IAC/E,MAAMZ,gBAAgBS,MAAM,IAAME,mBAAmBF,MAAMG,MAAMb;AACnE;AAEA,eAAemC,sBAAsBJ,MAAoB,EAAEK,KAAiB,EAAEC,EAAU;IACtFN,OAAOd,EAAE,CAACN,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,IAAI;AAC9C;AAEA,+FAA+F;AAC/F,OAAO,MAAMC,oBAA+C;IAC1DC,UAAUjC;IACVkC,eAAejC;IACfkC,kBAAkBtC;IAClByB;IACAI;IACAC;IACAE;IACA9B,aAAa,CAAC0B,QAAQrB,OAASL,YAAY0B,OAAOd,EAAE,EAAEP;AACxD,EAAE;AAEF,OAAO,eAAegC,WAAW7B,GAAmB;IAClD,OAAOhB,gBAAgBgB,KAAKyB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '20';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DatabaseSync","featureSignature","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","TOKENIZE","createContentTable","conn","exec","ensureSchemaTables","cfg","feature","schema","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","handle","close","ensureSchema","_handle","setDerivedBusyTimeout","_conn","ms","sqliteOpenDialect","filename","schemaVersion","reconcileDialect","openSqlite"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAInC,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMC,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeC,mBAAmBC,IAAgB;IAChD,MAAMA,KAAKC,IAAI,CAAC,CAAC,0IAA0I,EAAEH,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeI,mBAAmBF,IAAgB,EAAEG,GAAW;IAC7D,MAAMH,KAAKC,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,mBAAmBC;IACzB,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMA,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,KAAK,MAAMG,WAAWnB,eAAekB,KAAM,MAAMC,QAAQC,MAAM,CAACL;IAChE,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYhB,iBAAiBmB,KAAKjB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASoB,cAAcC,EAAgB;IACrC,MAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGN,IAAI,CAAC;YACR;QACF,EAAE,OAAOU,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,MAAMb,KAAK,IAAIxB,aAAaoC;IAC5B,IAAI;QACF,6FAA6F;QAC7F,yFAAyF;QACzFZ,GAAGN,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEK,cAAcC;QACdb,kBAAkBa;QAClB,MAAMP,OAAOR,iBAAiBe;QAC9B,OAAO;YAAEc,QAAQ;gBAAEd;YAAG;YAAGP;QAAK;IAChC,EAAE,OAAOW,KAAK;QACZ,4FAA4F;QAC5F,0CAA0C;QAC1CJ,GAAGe,KAAK;QACR,MAAMX;IACR;AACF;AAEA,eAAeW,MAAMD,MAAoB;IACvCA,OAAOd,EAAE,CAACe,KAAK;AACjB;AAEA,eAAeC,aAAaC,OAAqB,EAAExB,IAAgB,EAAEG,GAAW;IAC9E,+FAA+F;IAC/F,+EAA+E;IAC/E,MAAMZ,gBAAgBS,MAAM,IAAME,mBAAmBF,MAAMG,MAAMb;AACnE;AAEA,eAAemC,sBAAsBJ,MAAoB,EAAEK,KAAiB,EAAEC,EAAU;IACtFN,OAAOd,EAAE,CAACN,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,IAAI;AAC9C;AAEA,+FAA+F;AAC/F,OAAO,MAAMC,oBAA+C;IAC1DC,UAAUjC;IACVkC,eAAejC;IACfkC,kBAAkBtC;IAClByB;IACAI;IACAC;IACAE;IACA9B,aAAa,CAAC0B,QAAQrB,OAASL,YAAY0B,OAAOd,EAAE,EAAEP;AACxD,EAAE;AAEF,OAAO,eAAegC,WAAW7B,GAAmB;IAClD,OAAOhB,gBAAgBgB,KAAKyB;AAC9B"}

package/dist/esm/store/turso/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "4";
+export declare const SCHEMA_VERSION = "5";
 export type { OpenResult };
 export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/esm/store/turso/open.js

@@ -11,7 +11,7 @@ import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.turso.db';
 // Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves
 // separately. Covers the FTS indexes, the "_ngram" sidecar columns, and embeddings.vector's width.
-export const SCHEMA_VERSION = '4';
+export const SCHEMA_VERSION = '5';
 // This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
 // extra native state to thread, unlike duckdb's separate instance/connection pair.
 async function ensureSchema(_handle, conn, cfg) {

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","checkpointWal","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,aAAa,EAAEC,gBAAgB,QAAQ,kBAAkB;AAClE,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,eAAesB,MAAMC,MAAgB;IACnC,MAAMnB,cAAcmB;IACpB,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,6BAA6B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGiB,UAAU,CAACT,MAAQ,iEAAiEU,IAAI,CAACV,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAe0B,UAAUzB,GAAmB;IACjD,OAAOf,gBAAgBe,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","checkpointWal","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,aAAa,EAAEC,gBAAgB,QAAQ,kBAAkB;AAClE,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,eAAesB,MAAMC,MAAgB;IACnC,MAAMnB,cAAcmB;IACpB,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,6BAA6B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGiB,UAAU,CAACT,MAAQ,iEAAiEU,IAAI,CAACV,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAe0B,UAAUzB,GAAmB;IACjD,OAAOf,gBAAgBe,KAAKmB;AAC9B"}

package/dist/esm/text/strip.js

@@ -3,7 +3,7 @@ import { normalizeText } from '../scan/frontmatter.js';
 // Flat prose from blocks a caller already parsed, via the same extractor the chunker uses
 // (src/chunk) -- words and vectors read one definition of "the prose of this note".
 export function textFromBlocks(blocks) {
-    return normalizeText(extractTexts(blocks.map((block)=>block.node)));
+    return normalizeText(extractTexts(blocks));
 }
 // Thin parse + delegate, for callers with no blocks of their own already.
 export function stripText(value) {

package/dist/esm/text/strip.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks.map((block) => block.node)));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractTexts","parse","normalizeText","textFromBlocks","blocks","map","block","node","stripText","value"],"mappings":"AACA,SAASA,YAAY,EAAEC,KAAK,QAAQ,oBAAoB;AACxD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,0FAA0F;AAC1F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,OAAOF,cAAcF,aAAaI,OAAOC,GAAG,CAAC,CAACC,QAAUA,MAAMC,IAAI;AACpE;AAEA,0EAA0E;AAC1E,OAAO,SAASC,UAAUC,KAAa;IACrC,OAAON,eAAeF,MAAMQ;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractTexts","parse","normalizeText","textFromBlocks","blocks","stripText","value"],"mappings":"AACA,SAASA,YAAY,EAAEC,KAAK,QAAQ,oBAAoB;AACxD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,0FAA0F;AAC1F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,OAAOF,cAAcF,aAAaI;AACpC;AAEA,0EAA0E;AAC1E,OAAO,SAASC,UAAUC,KAAa;IACrC,OAAOH,eAAeF,MAAMK;AAC9B"}

package/dist/esm/workers/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\n// parseMs is the worker's own hrtime for parseFile, excluding dispatch and the return clone --\n// what separates it from the pool's dispatch-to-drain `parse` stage.\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[]; parseMs: number } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const start = process.hrtime.bigint();\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    const parseMs = Number(process.hrtime.bigint() - start) / 1e6;\n    return { ok: true, doc, warnings, parseMs };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["Tinypool","FEATURES","parseFile","featuresForFile","serializeError","cfg","featureNames","workerData","selected","filter","feature","includes","name","parseTask","file","start","process","hrtime","bigint","doc","warnings","parseMs","Number","ok","err","error"],"mappings":"AAAA,OAAOA,cAAc,WAAW;AAEhC,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,eAAe,QAAQ,qBAAqB;AAErD,SAASC,cAAc,QAAQ,0BAA0B;AAiBzD,sCAAsC;AACtC,MAAM,EAAEC,GAAG,EAAEC,YAAY,EAAE,GAAGN,SAASO,UAAU;AACjD,4FAA4F;AAC5F,qDAAqD;AACrD,MAAMC,WAAWP,SAASQ,MAAM,CAAC,CAACC,UAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;AAEhF,eAAe,SAASC,UAAUC,IAAe;IAC/C,IAAI;QACF,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;QACnC,MAAM,EAAEC,GAAG,EAAEC,QAAQ,EAAE,GAAGlB,UAAUY,MAAMX,gBAAgBK,UAAUH,KAAKS,OAAOT;QAChF,MAAMgB,UAAUC,OAAON,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QAC1D,OAAO;YAAEQ,IAAI;YAAMJ;YAAKC;YAAUC;QAAQ;IAC5C,EAAE,OAAOG,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOrB,eAAeoB;QAAK;IACjD;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the token tree.\nexport type ParseTask = FileStat;\n\n// parseMs is the worker's own hrtime for parseFile, excluding dispatch and the return clone --\n// what separates it from the pool's dispatch-to-drain `parse` stage.\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[]; parseMs: number } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const start = process.hrtime.bigint();\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    const parseMs = Number(process.hrtime.bigint() - start) / 1e6;\n    return { ok: true, doc, warnings, parseMs };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["Tinypool","FEATURES","parseFile","featuresForFile","serializeError","cfg","featureNames","workerData","selected","filter","feature","includes","name","parseTask","file","start","process","hrtime","bigint","doc","warnings","parseMs","Number","ok","err","error"],"mappings":"AAAA,OAAOA,cAAc,WAAW;AAEhC,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,eAAe,QAAQ,qBAAqB;AAErD,SAASC,cAAc,QAAQ,0BAA0B;AAiBzD,sCAAsC;AACtC,MAAM,EAAEC,GAAG,EAAEC,YAAY,EAAE,GAAGN,SAASO,UAAU;AACjD,4FAA4F;AAC5F,qDAAqD;AACrD,MAAMC,WAAWP,SAASQ,MAAM,CAAC,CAACC,UAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;AAEhF,eAAe,SAASC,UAAUC,IAAe;IAC/C,IAAI;QACF,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;QACnC,MAAM,EAAEC,GAAG,EAAEC,QAAQ,EAAE,GAAGlB,UAAUY,MAAMX,gBAAgBK,UAAUH,KAAKS,OAAOT;QAChF,MAAMgB,UAAUC,OAAON,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QAC1D,OAAO;YAAEQ,IAAI;YAAMJ;YAAKC;YAAUC;QAAQ;IAC5C,EAAE,OAAOG,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOrB,eAAeoB;QAAK;IACjD;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.22.2",
+  "version": "0.23.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -84,24 +84,16 @@
     "@huggingface/tokenizers": "^0.1.3",
     "franc-min": "^6.2.0",
     "install-module-linked": "^2.0.0",
-    "mdast-util-from-markdown": "^2.0.3",
-    "mdast-util-gfm-autolink-literal": "^2.0.0",
-    "mdast-util-gfm-footnote": "^2.0.0",
-    "mdast-util-gfm-strikethrough": "^2.0.0",
-    "mdast-util-gfm-table": "^2.0.0",
-    "mdast-util-gfm-task-list-item": "^2.0.0",
-    "micromark-extension-gfm-autolink-literal": "^2.0.0",
-    "micromark-extension-gfm-footnote": "^2.0.0",
-    "micromark-extension-gfm-strikethrough": "^2.0.0",
-    "micromark-extension-gfm-table": "^2.0.0",
-    "micromark-extension-gfm-task-list-item": "^2.0.0",
+    "markdown-it": "^15.0.1",
+    "markdown-it-footnote": "^4.0.0",
+    "markdown-it-task-lists": "^2.1.1",
     "tinypool": "^2.1.2",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "*",
     "@tursodatabase/database": "*",
-    "@types/mdast": "^4.0.4",
+    "@types/markdown-it-footnote": "^3.0.4",
     "@types/mocha": "*",
     "@types/node": "*",
     "cr": "^0.1.0",