sensemaking 0.18.4 → 0.19.1

package/dist/esm/store/turso/lexical.js

@@ -0,0 +1,128 @@
+// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and
+// quoted phrases, plus a second ngram-tokenized index over "_ngram" sidecars for unspaced-script runs.
+import { SenseError } from '../../errors.js';
+import { hasUnspacedRun } from '../../text/segment.js';
+const FIELDS = [
+    'title',
+    'summary',
+    'text'
+];
+const NGRAM_FIELDS = [
+    'title_ngram',
+    'summary_ngram',
+    'text_ngram'
+];
+// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:
+// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.
+const FTS5_OPERATORS = [
+    {
+        label: 'prefix query',
+        re: /[\p{L}\p{N}_]+\*(?=\s|$)/u
+    },
+    {
+        label: 'boolean operator',
+        re: /(?:^|\s)(?:AND|OR|NOT)(?=\s|$)/
+    },
+    {
+        label: 'NEAR operator',
+        re: /(?:^|\s)NEAR\b/
+    },
+    {
+        label: 'boost/initial-token operator',
+        re: /\S*\^\S*/
+    },
+    {
+        label: 'column filter',
+        re: /(?:^|\s)[\p{L}_]\w*\s*:/u
+    }
+];
+function unsupportedOperator(terms) {
+    const withoutPhrases = terms.replace(/"[^"]*"/g, ' ');
+    for (const { label, re } of FTS5_OPERATORS){
+        const m = withoutPhrases.match(re);
+        if (m) return {
+            label,
+            token: m[0].trim()
+        };
+    }
+    return null;
+}
+// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer
+// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.
+function splitTerms(terms) {
+    const words = [];
+    const phrases = [];
+    const unspaced = [];
+    const withoutPhrases = terms.replace(/"([^"]*)"/g, (_m, inner)=>{
+        const phrase = inner.trim();
+        if (phrase.length === 0) return ' ';
+        if (hasUnspacedRun(phrase)) unspaced.push(phrase);
+        else phrases.push(phrase);
+        return ' ';
+    });
+    for (const tok of withoutPhrases.split(/\s+/)){
+        if (tok.length === 0) continue;
+        if (hasUnspacedRun(tok)) unspaced.push(tok);
+        else words.push(tok);
+    }
+    return {
+        words,
+        phrases,
+        unspaced
+    };
+}
+// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays
+// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).
+function escapeFtsLiteral(query) {
+    return query.replace(/'/g, "''");
+}
+// Terms are quoted ("O'Brien's" is a parse error bare) and AND-joined: Tantivy's default for
+// bare terms is disjunctive, so space-joining would loosen "apple banana" into "apple OR banana".
+function buildQueries(words, phrases, unspaced) {
+    const defaultParts = [
+        ...words,
+        ...phrases
+    ].map((t)=>`"${t}"`);
+    const unspacedParts = unspaced.map((t)=>`"${t}"`);
+    return {
+        defaultQuery: defaultParts.join(' AND '),
+        ngramQuery: unspacedParts.join(' AND ')
+    };
+}
+// fts_score() is real only as a SELECT's bare projected expression, and only for the index that
+// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.
+function ftsBranch(cols, query, opts) {
+    const colList = cols.map((c)=>`content.${c}`).join(', ');
+    const lit = escapeFtsLiteral(query);
+    return `SELECT content."path" AS path, fts_score(${colList}, '${lit}') AS score
+    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;
+}
+// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's
+// queryLexical take). `hit` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS excerpt fallback (commands/search.ts).
+export async function queryLexical(conn, terms, opts) {
+    const unsupported = unsupportedOperator(terms);
+    if (unsupported !== null) {
+        throw new SenseError('STORE_CAPABILITY_MISSING', `store "turso" does not implement FTS5's ${unsupported.label} ("${unsupported.token}") in this build; rephrase "${terms.trim()}" without it, or set "store" to "sqlite" in this tree's config to search it as written`);
+    }
+    const { words, phrases, unspaced } = splitTerms(terms);
+    const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);
+    if (defaultQuery === '' && ngramQuery === '') return [];
+    const branches = [];
+    if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));
+    if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));
+    // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,
+    // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.
+    const sql = `SELECT path, NULL AS hit FROM (
+    SELECT path, SUM(score) AS score FROM (
+      ${branches.join(' UNION ALL ')}
+    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}
+  ) sq ORDER BY score DESC, path LIMIT ?`;
+    const stmt = await conn.prepare(sql);
+    const rows = await stmt.all(opts.limit);
+    // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,
+    // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).
+    if (rows.some((r)=>r.path === null)) {
+        throw new SenseError('LEXICAL_NULL_PATH', 'store "turso" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');
+    }
+    return rows;
+}

package/dist/esm/store/turso/lexical.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: string[]; unspaced: string[] } {\n  const words: string[] = [];\n  const phrases: string[] = [];\n  const unspaced: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    if (hasUnspacedRun(phrase)) unspaced.push(phrase);\n    else phrases.push(phrase);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) unspaced.push(tok);\n    else words.push(tok);\n  }\n  return { words, phrases, unspaced };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: string[], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const defaultParts = [...words, ...phrases].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `hit` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced } = splitTerms(terms);\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(opts.limit)) as unknown as LexicalHit[];\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  return rows;\n}\n"],"names":["SenseError","hasUnspacedRun","FIELDS","NGRAM_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","_m","inner","phrase","length","push","tok","split","escapeFtsLiteral","query","buildQueries","defaultParts","map","t","unspacedParts","defaultQuery","join","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","queryLexical","conn","unsupported","branches","sql","stmt","prepare","rows","all","limit","some","r","path"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;AACvG,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,wBAAwB;AAGvD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAC3C,MAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AAEnE,uFAAuF;AACvF,+FAA+F;AAC/F,MAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,UAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMT,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACS,IAAIC;QACtD,MAAMC,SAASD,MAAMN,IAAI;QACzB,IAAIO,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChC,IAAIrB,eAAeoB,SAASH,SAASK,IAAI,CAACF;aACrCJ,QAAQM,IAAI,CAACF;QAClB,OAAO;IACT;IACA,KAAK,MAAMG,OAAOf,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAID,IAAIF,MAAM,KAAK,GAAG;QACtB,IAAIrB,eAAeuB,MAAMN,SAASK,IAAI,CAACC;aAClCR,MAAMO,IAAI,CAACC;IAClB;IACA,OAAO;QAAER;QAAOC;QAASC;IAAS;AACpC;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASQ,iBAAiBC,KAAa;IACrC,OAAOA,MAAMjB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASkB,aAAaZ,KAAe,EAAEC,OAAiB,EAAEC,QAAkB;IAC1E,MAAMW,eAAe;WAAIb;WAAUC;KAAQ,CAACa,GAAG,CAAC,CAACC,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAC/D,MAAMC,gBAAgBd,SAASY,GAAG,CAAC,CAACC,IAAM,CAAC,CAAC,EAAEA,EAAE,CAAC,CAAC;IAClD,OAAO;QAAEE,cAAcJ,aAAaK,IAAI,CAAC;QAAUC,YAAYH,cAAcE,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASE,UAAUC,IAAuB,EAAEV,KAAa,EAAEW,IAAyB;IAClF,MAAMC,UAAUF,KAAKP,GAAG,CAAC,CAACU,IAAM,CAAC,QAAQ,EAAEA,GAAG,EAAEN,IAAI,CAAC;IACrD,MAAMO,MAAMf,iBAAiBC;IAC7B,OAAO,CAAC,yCAAyC,EAAEY,QAAQ,GAAG,EAAEE,IAAI;iBACrD,EAAEH,KAAKI,SAAS,CAAC,iBAAiB,EAAEH,QAAQ,GAAG,EAAEE,IAAI,GAAG,EAAEH,KAAKK,SAAS,CAAC,CAAC,EAAEL,KAAKM,SAAS,EAAE;AAC7G;AAEA,uFAAuF;AACvF,gMAAgM;AAChM,OAAO,eAAeC,aAAaC,IAAgB,EAAEtC,KAAa,EAAE8B,IAAyB;IAC3F,MAAMS,cAAcxC,oBAAoBC;IACxC,IAAIuC,gBAAgB,MAAM;QACxB,MAAM,IAAI/C,WAAW,4BAA4B,CAAC,wCAAwC,EAAE+C,YAAY1C,KAAK,CAAC,GAAG,EAAE0C,YAAYlC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IACzQ;IACA,MAAM,EAAEE,KAAK,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGH,WAAWP;IAChD,MAAM,EAAEyB,YAAY,EAAEE,UAAU,EAAE,GAAGP,aAAaZ,OAAOC,SAASC;IAClE,IAAIe,iBAAiB,MAAME,eAAe,IAAI,OAAO,EAAE;IAEvD,MAAMa,WAAqB,EAAE;IAC7B,IAAIf,iBAAiB,IAAIe,SAASzB,IAAI,CAACa,UAAUlC,QAAQ+B,cAAcK;IACvE,IAAIH,eAAe,IAAIa,SAASzB,IAAI,CAACa,UAAUjC,cAAcgC,YAAYG;IAEzE,8FAA8F;IAC9F,mFAAmF;IACnF,MAAMW,MAAM,CAAC;;MAET,EAAED,SAASd,IAAI,CAAC,eAAe;4CACO,EAAEc,SAAS1B,MAAM,CAAC;wCACtB,CAAC;IACvC,MAAM4B,OAAO,MAAMJ,KAAKK,OAAO,CAACF;IAChC,MAAMG,OAAQ,MAAMF,KAAKG,GAAG,CAACf,KAAKgB,KAAK;IACvC,+FAA+F;IAC/F,yFAAyF;IACzF,IAAIF,KAAKG,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,OAAO;QACrC,MAAM,IAAIzD,WAAW,qBAAqB;IAC5C;IACA,OAAOoD;AACT"}

package/dist/esm/store/turso/native.js

@@ -6,13 +6,8 @@ const TURSO = {
     sizeHint: '~16MB'
 };
 let tursoApiPromise;
-// The one accessor every consumer of @tursodatabase/database (open.ts) must go through, rather
-// than each doing its own `import('@tursodatabase/database')`: once a fresh install has happened
-// in this process, Node keeps returning that pre-install "not found" miss to any NEW
-// bare-specifier lookup of the same path, so a second independent import elsewhere in the
-// process fails even though the package is now genuinely on disk (the same stale-cache problem
-// loadOrInstall works around for its own retry). Caching the resolved module here means only the
-// first caller ever resolves the bare specifier at all.
+// One accessor every consumer must go through, not each its own bare-specifier import: Node
+// keeps returning a pre-install "not found" miss to any new lookup of the same path afterward.
 export function tursoApi() {
     if (!tursoApiPromise) tursoApiPromise = loadOrInstall(TURSO, packageNodeModules());
     return tursoApiPromise;

package/dist/esm/store/turso/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// The one accessor every consumer of @tursodatabase/database (open.ts) must go through, rather\n// than each doing its own `import('@tursodatabase/database')`: once a fresh install has happened\n// in this process, Node keeps returning that pre-install \"not found\" miss to any NEW\n// bare-specifier lookup of the same path, so a second independent import elsewhere in the\n// process fails even though the package is now genuinely on disk (the same stale-cache problem\n// loadOrInstall works around for its own retry). Caching the resolved module here means only the\n// first caller ever resolves the bare specifier at all.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","TURSO_PACKAGE","TURSO","store","pkg","sizeHint","tursoApiPromise","tursoApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,gBAAgB,0BAA0B;AAEvD,MAAMC,QAA0B;IAAEC,OAAO;IAASC,KAAKH;IAAeI,UAAU;AAAQ;AAExF,IAAIC;AAEJ,+FAA+F;AAC/F,iGAAiG;AACjG,qFAAqF;AACrF,0FAA0F;AAC1F,+FAA+F;AAC/F,iGAAiG;AACjG,wDAAwD;AACxD,OAAO,SAASC;IACd,IAAI,CAACD,iBAAiBA,kBAAkBP,cAAwDG,OAAOF;IACvG,OAAOM;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// One accessor every consumer must go through, not each its own bare-specifier import: Node\n// keeps returning a pre-install \"not found\" miss to any new lookup of the same path afterward.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","TURSO_PACKAGE","TURSO","store","pkg","sizeHint","tursoApiPromise","tursoApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,gBAAgB,0BAA0B;AAEvD,MAAMC,QAA0B;IAAEC,OAAO;IAASC,KAAKH;IAAeI,UAAU;AAAQ;AAExF,IAAIC;AAEJ,4FAA4F;AAC5F,+FAA+F;AAC/F,OAAO,SAASC;IACd,IAAI,CAACD,iBAAiBA,kBAAkBP,cAAwDG,OAAOF;IACvG,OAAOM;AACT"}

package/dist/esm/store/turso/open.d.ts

@@ -1,7 +1,7 @@
 import type { ResolvedConfig } from '../../config/index.js';
 import type { Store } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "1";
+export declare const SCHEMA_VERSION = "3";
 export interface OpenResult {
     store: Store;
     cfg: ResolvedConfig;
@@ -9,4 +9,6 @@ export interface OpenResult {
     parsed: number;
     warnings: string[];
 }
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/esm/store/turso/open.js

@@ -1,28 +1,47 @@
 import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { featureSignature, STATE_DIR } from '../../config/index.js';
+import { rekeyChunkText } from '../../embed/handoff.js';
+import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
+import { clearCache } from '../cache.js';
 import { getMeta, setMeta } from '../shared.js';
-import { clearCache } from '../sqlite/open.js';
 import { createConnection } from './connection.js';
 import { TURSO_PACKAGE, tursoApi } from './native.js';
 import { reconcile } from './reconcile.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.turso.db';
-// Independent of sqlite's SCHEMA_VERSION and duckdb's -- each store's cache shape evolves
-// separately, and the store name already joins the feature signature (see featureSignature), so
-// a config's `store` key switching cleanly rebuilds rather than reusing another engine's cache.
-export const SCHEMA_VERSION = '1';
-// content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is
-// phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect
-// accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.
+// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves
+// separately. Covers the FTS indexes, the "_ngram" sidecar columns, and embeddings.vector's width.
+export const SCHEMA_VERSION = '3';
+// The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
+// columns makes a bare substring match a whole word, defeating the prefix-query rejection.
+// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:
+// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.
+export const CONTENT_FTS_DDL = [
+    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,
+    `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`
+];
+export const CONTENT_FTS_NAMES = [
+    'content_fts',
+    'content_fts_ngram'
+];
 async function ensureSchema(conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
-    await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);
+    await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);
+    for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
-    for (const feature of activeFeatures(cfg))await feature.schema(conn);
+    for (const feature of activeFeatures(cfg)){
+        // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`
+        // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.
+        if (feature.name === 'embed') {
+            await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY ("path", chunk))`);
+            continue;
+        }
+        await feature.schema(conn);
+    }
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
@@ -32,10 +51,8 @@ async function connect(cfg) {
         recursive: true
     });
     const dbPath = join(stateDir, DB_FILENAME);
-    // Dynamic, not a top-level import: a sqlite or duckdb tree must never even attempt to resolve
-    // this optional dependency, so nothing in this store's module graph imports it as a value until
-    // a turso tree is actually opened (type-only imports elsewhere are erased and cost nothing
-    // either way). Installed on first use if missing (see native.ts's tursoApi).
+    // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this
+    // optional dependency until a turso tree is actually opened. Installed on first use if missing.
     let turso;
     try {
         turso = await tursoApi();
@@ -45,20 +62,19 @@ async function connect(cfg) {
     }
     let db;
     try {
-        // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this
-        // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --
-        // which depends on this cache's own recorded reconcile_max_ms, readable only once connected
-        // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same
-        // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).
+        // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;
+        // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).
         db = await turso.connect(dbPath, {
-            timeout: 30000
+            timeout: 30000,
+            experimental: [
+                'index_method'
+            ]
         });
     } catch (err) {
         throw new SenseError('STORE_DEPENDENCY_MISSING', `store "turso" failed to open ${dbPath}: ${err.message}`);
     }
-    // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL
-    // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and
-    // recurses, so the catch never double-closes.
+    // A throw below must release this handle, or the leaked WAL makes the db file undeletable on
+    // Windows. closed guards the catch against double-closing.
     let closed = false;
     try {
         var _ref;
@@ -68,11 +84,8 @@ async function connect(cfg) {
         const features = await getMeta(conn, 'features');
         const wantFeatures = featureSignature(cfg, FEATURES);
         if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
-            // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
-            // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is
-            // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for
-            // those): turso has no content.tokenize knob to special-case, and the grace is an
-            // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.
+            // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched
+            // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.
             const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
             console.error(`sense: ${reason}; rebuilding the index`);
             closed = true;
@@ -103,8 +116,11 @@ async function connect(cfg) {
 // connection in the Store interface.
 export async function openTurso(cfg) {
     const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
+    const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);
+    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+    rekeyChunkText(conn, store);
     return {
-        store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir),
+        store: store,
         cfg: resolvedCfg,
         dbPath,
         parsed,

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { clearCache } from '../sqlite/open.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's SCHEMA_VERSION and duckdb's -- each store's cache shape evolves\n// separately, and the store name already joins the feature signature (see featureSignature), so\n// a config's `store` key switching cleanly rebuilds rather than reusing another engine's cache.\nexport const SCHEMA_VERSION = '1';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is\n// phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect\n// accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never even attempt to resolve\n  // this optional dependency, so nothing in this store's module graph imports it as a value until\n  // a turso tree is actually opened (type-only imports elsewhere are erased and cost nothing\n  // either way). Installed on first use if missing (see native.ts's tursoApi).\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this\n    // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --\n    // which depends on this cache's own recorded reconcile_max_ms, readable only once connected\n    // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same\n    // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).\n    db = await turso.connect(dbPath, { timeout: 30_000 });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL\n  // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and\n  // recurses, so the catch never double-closes.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is\n      // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for\n      // those): turso has no content.tokenize knob to special-case, and the grace is an\n      // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","SenseError","activeFeatures","FEATURES","getMeta","setMeta","clearCache","createConnection","TURSO_PACKAGE","tursoApi","reconcile","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","schema","connect","stateDir","baseDir","recursive","dbPath","turso","err","message","db","timeout","closed","version","features","wantFeatures","reason","console","error","close","recordedMaxMs","Number","Math","min","max","parsed","warnings","openTurso","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,UAAU,QAAQ,oBAAoB;AAE/C,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,0FAA0F;AAC1F,gGAAgG;AAChG,gGAAgG;AAChG,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,6FAA6F;AAC7F,+FAA+F;AAC/F,4FAA4F;AAC5F,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWhB,eAAec,KAAM,MAAME,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMX,QAAQW,MAAM,sBAAuB,MAAM,MAAMV,QAAQU,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMT,QAAQW,MAAM,gBAAiB,MAAM,MAAMV,QAAQU,MAAM,YAAYhB,iBAAiBiB,KAAKb;AACxG;AAWA,eAAeiB,QAAQJ,GAAmB;IACxC,MAAMK,WAAWvB,KAAKkB,IAAIM,OAAO,EAAEtB;IACnCH,UAAUwB,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS1B,KAAKuB,UAAUT;IAE9B,8FAA8F;IAC9F,gGAAgG;IAChG,2FAA2F;IAC3F,6EAA6E;IAC7E,IAAIa;IACJ,IAAI;QACFA,QAAQ,MAAMhB;IAChB,EAAE,OAAOiB,KAAK;QACZ,IAAIA,eAAezB,YAAY,MAAMyB;QACrC,MAAM,IAAIzB,WAAW,4BAA4B,CAAC,wBAAwB,EAAEO,cAAc,UAAU,EAAE,AAACkB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,wFAAwF;QACxF,yFAAyF;QACzF,4FAA4F;QAC5F,6FAA6F;QAC7F,+FAA+F;QAC/FA,KAAK,MAAMH,MAAML,OAAO,CAACI,QAAQ;YAAEK,SAAS;QAAO;IACrD,EAAE,OAAOH,KAAK;QACZ,MAAM,IAAIzB,WAAW,4BAA4B,CAAC,6BAA6B,EAAEuB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACtH;IACA,+FAA+F;IAC/F,4FAA4F;IAC5F,8CAA8C;IAC9C,IAAIG,SAAS;IACb,IAAI;YA0B4B;QAzB9B,MAAMf,OAAOR,iBAAiBqB;QAE9B,MAAMb,KAAKE,IAAI,CAAC;QAEhB,MAAMc,UAAU,MAAM3B,QAAQW,MAAM;QACpC,MAAMiB,WAAW,MAAM5B,QAAQW,MAAM;QACrC,MAAMkB,eAAelC,iBAAiBiB,KAAKb;QAC3C,IAAI,AAAC4B,YAAY,QAAQA,YAAYlB,kBAAoBmB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,4FAA4F;YAC5F,0FAA0F;YAC1F,kFAAkF;YAClF,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYlB,iBAAiB,mDAAmD;YACnHsB,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtDJ,SAAS;YACT,MAAMF,GAAGS,KAAK;YACd/B,WAAWU;YACX,OAAOI,QAAQJ;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMsB,gBAAgBC,QAAQ,OAAA,MAAMnC,QAAQW,MAAM,iCAApB,kBAAA,OAA4C;QAC1E,MAAMA,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEuB,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAEhG,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUK,MAAMC,KAAKA,IAAIM,OAAO;QAEnE,OAAO;YAAEM;YAAIb;YAAMC;YAAKQ;YAAQmB;YAAQC;QAAS;IACnD,EAAE,OAAOlB,KAAK;QACZ,IAAI,CAACI,QAAQ,MAAMF,GAAGS,KAAK;QAC3B,MAAMX;IACR;AACF;AAEA,gGAAgG;AAChG,qCAAqC;AACrC,OAAO,eAAemB,UAAU7B,GAAmB;IACjD,MAAM,EAAEY,EAAE,EAAEb,IAAI,EAAEC,KAAK8B,WAAW,EAAEtB,MAAM,EAAEmB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMxB,QAAQJ;IAC/E,OAAO;QAAE+B,OAAOpC,YAAYiB,IAAIb,MAAM+B,aAAaA,YAAYxB,OAAO;QAAGN,KAAK8B;QAAatB;QAAQmB;QAAQC;IAAS;AACtH"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","reconcile","createStore","DB_FILENAME","SCHEMA_VERSION","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ensureSchema","conn","cfg","exec","ddl","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","turso","err","message","db","timeout","experimental","closed","version","features","wantFeatures","reason","console","error","close","recordedMaxMs","Number","Math","min","max","parsed","warnings","openTurso","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,+FAA+F;AAC/F,2FAA2F;AAC3F,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAON,gBAAiB,MAAMG,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWnB,eAAegB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEnB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMqB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBJ;IAC5F,IAAI,AAAC,MAAMR,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYpB,iBAAiBqB,KAAKf;AACxG;AAWA,eAAeqB,QAAQN,GAAmB;IACxC,MAAMO,WAAW7B,KAAKsB,IAAIQ,OAAO,EAAE5B;IACnCH,UAAU8B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAShC,KAAK6B,UAAUb;IAE9B,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIiB;IACJ,IAAI;QACFA,QAAQ,MAAMpB;IAChB,EAAE,OAAOqB,KAAK;QACZ,IAAIA,eAAe7B,YAAY,MAAM6B;QACrC,MAAM,IAAI7B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEO,cAAc,UAAU,EAAE,AAACsB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAML,OAAO,CAACI,QAAQ;YAAEK,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI7B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE2B,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACtH;IACA,6FAA6F;IAC7F,2DAA2D;IAC3D,IAAII,SAAS;IACb,IAAI;YAuB4B;QAtB9B,MAAMlB,OAAOV,iBAAiByB;QAE9B,MAAMf,KAAKE,IAAI,CAAC;QAEhB,MAAMiB,UAAU,MAAM/B,QAAQY,MAAM;QACpC,MAAMoB,WAAW,MAAMhC,QAAQY,MAAM;QACrC,MAAMqB,eAAezC,iBAAiBqB,KAAKf;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;YACxG,yFAAyF;YACzF,iGAAiG;YACjG,MAAMC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;YACnH2B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtDJ,SAAS;YACT,MAAMH,GAAGU,KAAK;YACdtC,WAAWc;YACX,OAAOM,QAAQN;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMyB,gBAAgBC,QAAQ,OAAA,MAAMvC,QAAQY,MAAM,iCAApB,kBAAA,OAA4C;QAC1E,MAAMA,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAEhG,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMvC,UAAUO,MAAMC,KAAKA,IAAIQ,OAAO;QAEnE,OAAO;YAAEM;YAAIf;YAAMC;YAAKU;YAAQoB;YAAQC;QAAS;IACnD,EAAE,OAAOnB,KAAK;QACZ,IAAI,CAACK,QAAQ,MAAMH,GAAGU,KAAK;QAC3B,MAAMZ;IACR;AACF;AAEA,gGAAgG;AAChG,qCAAqC;AACrC,OAAO,eAAeoB,UAAUhC,GAAmB;IACjD,MAAM,EAAEc,EAAE,EAAEf,IAAI,EAAEC,KAAKiC,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQN;IAC/E,MAAMkC,QAAQzC,YAAYqB,IAAIf,MAAMkC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1F3B,eAAekB,MAAMmC;IACrB,OAAO;QAAEA,OAAOA;QAAOlC,KAAKiC;QAAavB;QAAQoB;QAAQC;IAAS;AACpE"}

package/dist/esm/store/turso/reconcile.js

@@ -3,15 +3,12 @@ import { activeFeatures } from '../../features/index.js';
 import { progress } from '../../output/progress.js';
 import { listFiles, RESERVED_COLUMNS } from '../../scan/index.js';
 import { reparseFiles } from '../../scan/reparse.js';
+import { hasUnspacedRun } from '../../text/segment.js';
 import { getColumns, getMeta, quoteIdent, setMeta } from '../shared.js';
-import { withTransaction } from '../transaction.js';
-// Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not
-// duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb
-// dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)
-// is derived from exactly that recorded value -- forking duckdb's file would lose it silently.
-// content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index
-// (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not
-// FTS5-shaped).
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
+import { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.js';
+// Fork of sqlite/reconcile.ts, not duckdb's, so this store keeps reconcile_max_ms bookkeeping
+// (open.ts's busy timeout derives from it). content is a plain table with "_ngram" sidecars for lexical.ts.
 const CORE_FRONTMATTER_COLUMNS = new Set([
     'path',
     '_mtime',
@@ -19,20 +16,28 @@ const CORE_FRONTMATTER_COLUMNS = new Set([
     '_size',
     '_parse_error'
 ]);
-// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no
-// error). The real fence is a SELECT projecting more than this many result columns, which fails
-// to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table
-// would pass every write silently and only break the first time something selects across it.
-// Enforced here, before ALTER, the same shape as sqlite's compile-time fence.
+// '' when the field has no unspaced-script run (the common case), so the ngram index carries
+// nothing for it -- same "pay for nothing when absent" shape as sqlite's segmentField sidecars.
+function ngramSidecar(text) {
+    return hasUnspacedRun(text) ? text : '';
+}
+// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with
+// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.
 const MAX_FRONTMATTER_COLUMNS = 2000;
+// Changed files above which rebuilding the FTS index beats maintaining it per row (see reconcile).
+const FTS_REBUILD_THRESHOLD = 250;
 // No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
-const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)`;
+const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;
 function contentRow(doc) {
+    const { title, summary, text } = doc.search;
     return [
         doc.relPath,
-        doc.search.title,
-        doc.search.summary,
-        doc.search.text
+        title,
+        summary,
+        text,
+        ngramSidecar(title),
+        ngramSidecar(summary),
+        ngramSidecar(text)
     ];
 }
 export async function reconcile(conn, cfg, baseDir) {
@@ -70,7 +75,7 @@ export async function reconcile(conn, cfg, baseDir) {
     // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a
     // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).
     const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
-    // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.
+    // ON CONFLICT UPDATE, rather than the delete-and-insert `OR` clause, keeps the row's identity stable across reparses.
     const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
     const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
     const delta = {
@@ -98,6 +103,16 @@ export async function reconcile(conn, cfg, baseDir) {
                 }));
             await conn.runBatch(insertSql, rows);
         }
+        // Tantivy maintains the FTS index per inserted row and the cost grows with the batch, so a
+        // large insert is superlinear: 6,566 notes took 945s with the index live, 3.3s built after.
+        // Measured crossover against a full rebuild, in changed files: ~250 on a 6,566-note tree
+        // (100 -> 895ms, 300 -> 4.1s, 1,300 -> 45.5s, rebuild 3.35s), ~295 on 13,132 (250 -> 5.2s,
+        // 400 -> 10.0s, rebuild 6.7s). Doubling the corpus moved it 250 to 295, so a constant fits
+        // where a ratio does not: 20% would allow 1,313 files here, costing 45s to save 3.3s.
+        // Erring low is the cheap direction, since a needless rebuild is linear and predictable.
+        const churn = delta.reparsed.length + delta.vanished.length;
+        const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;
+        if (bulk) for (const name of CONTENT_FTS_NAMES)await conn.exec(`DROP INDEX IF EXISTS ${name}`);
         // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5
         // content), so vanished and reparsed docs delete in one pass.
         const contentTouched = [
@@ -108,6 +123,7 @@ export async function reconcile(conn, cfg, baseDir) {
                 p
             ]));
         if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));
+        if (bulk) for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
         if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
                 p
             ]));
@@ -140,15 +156,13 @@ export async function reconcile(conn, cfg, baseDir) {
             await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
         }
         for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
-    });
-    // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the
-    // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait
-    // against (the same mechanism sqlite's reconcile records, and the one this store's busy
-    // timeout is derived from).
+    }, BEGIN_WRITE);
+    // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: the observed
+    // max is what the next open bounds its wait against.
     const durationMs = Date.now() - txStart;
     const prevRaw = await getMeta(conn, 'reconcile_max_ms');
     // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
-    // still gets recorded instead of losing to the "nothing recorded yet" default.
+    // gets recorded rather than losing to the "nothing recorded yet" default.
     const prevMax = prevRaw === null ? -1 : Number(prevRaw);
     if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));
     return {

package/dist/esm/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Fork of sqlite/reconcile.ts (house precedent: each store owns its engine-specific SQL), not\n// duckdb's, even though duckdb's content handling is the structurally closer of the two. duckdb\n// dropped sqlite's reconcile_max_ms bookkeeping entirely, and this store's busy timeout (open.ts)\n// is derived from exactly that recorded value -- forking duckdb's file would lose it silently.\n// content itself follows duckdb's shape instead: a plain table, no rowid coupling, no FTS index\n// (phase 2's job, not reconcile's), and no `_seg` sidecars (turso's unspaced-script path is not\n// FTS5-shaped).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with no\n// error). The real fence is a SELECT projecting more than this many result columns, which fails\n// to prepare (spike-measured, both `SELECT *` and an explicit list), so a wide frontmatter table\n// would pass every write silently and only break the first time something selects across it.\n// Enforced here, before ALTER, the same shape as sqlite's compile-time fence.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n    // content), so vanished and reparsed docs delete in one pass.\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n  // observed max so a big watcher reconcile's lock hold is what the next open bounds its wait\n  // against (the same mechanism sqlite's reconcile records, and the one this store's busy\n  // timeout is derived from).\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,8FAA8F;AAC9F,gGAAgG;AAChG,kGAAkG;AAClG,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,gBAAgB;AAChB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,iGAAiG;AACjG,gGAAgG;AAChG,iGAAiG;AACjG,6FAA6F;AAC7F,8EAA8E;AAC9E,MAAMC,0BAA0B;AAEhC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQvB,UAAUqB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWlD,eAAeuB;IAChC,MAAM4B,cAAc,MAAM9C,WAAWiB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAASnD,SAAS,mBAAmBuC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMnD,aAAaoC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAGnC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAE6D,WAAWb,MAAM,CAAC,2DAA2D,EAAEnC,wBAAwB,gPAAgP,CAAC;IAEtX;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAAC3D,iBAAiBoC,GAAG,CAACuB;IAC1G,uFAAuF;IACvF,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACpB,YAAYyD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGvD,WAAWuD,GAAG,YAAY,EAAEvD,WAAWuD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMjE,gBAAgBa,MAAM;YA+CyCqD,iBAK7BA;QAnDtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAErE,WAAWmD,MAAM;QAErG,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM8B,OAAOvB,WAAW3B,GAAG,CAAC,CAACZ,MAC3B8C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ3C;oBANP,IAAI2C,QAAQ,QAAQ,OAAO3C,IAAIC,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO3C,IAAI6B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO3C,IAAI+D,OAAO;oBACxC,IAAIpB,QAAQ,SAAS,OAAO3C,IAAI+B,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO3C,IAAIgE,UAAU;oBACjD,QAAOhE,gBAAAA,IAAIiE,IAAI,CAACtB,IAAI,cAAb3C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMO,KAAK2D,QAAQ,CAAClB,WAAWc;QACjC;QAEA,0FAA0F;QAC1F,8DAA8D;QAC9D,MAAMK,iBAAiB;eAAI7C;eAAaiC;SAAiB;QACzD,IAAIY,eAAenC,MAAM,GAAG,GAC1B,MAAMzB,KAAK2D,QAAQ,CACjB,wCACAC,eAAevD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAACpE,oBAAoByC,WAAW3B,GAAG,CAACb;QAElF,IAAIuB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAK2D,QAAQ,CACjB,4CACA5C,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAK2D,QAAQ,CACjB,6CACAE,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMrE,OAAOuC,WAAY,KAAK,MAAM+B,cAActE,IAAIuE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACxE,IAAIC,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACZ,MAAS,CAAA;oBAAEqB,MAAMrB,IAAIC,OAAO;oBAAE2E,WAAW5E,IAAI4E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,0FAA0F;IAC1F,4FAA4F;IAC5F,wFAAwF;IACxF,4BAA4B;IAC5B,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM1F,QAAQgB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAMzF,QAAQc,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\nimport { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.ts';\n\n// Fork of sqlite/reconcile.ts, not duckdb's, so this store keeps reconcile_max_ms bookkeeping\n// (open.ts's busy timeout derives from it). content is a plain table with \"_ngram\" sidecars for lexical.ts.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row (see reconcile).\nconst FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE, rather than the delete-and-insert `OR` clause, keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n      }\n\n      // Tantivy maintains the FTS index per inserted row and the cost grows with the batch, so a\n      // large insert is superlinear: 6,566 notes took 945s with the index live, 3.3s built after.\n      // Measured crossover against a full rebuild, in changed files: ~250 on a 6,566-note tree\n      // (100 -> 895ms, 300 -> 4.1s, 1,300 -> 45.5s, rebuild 3.35s), ~295 on 13,132 (250 -> 5.2s,\n      // 400 -> 10.0s, rebuild 6.7s). Doubling the corpus moved it 250 to 295, so a constant fits\n      // where a ratio does not: 20% would allow 1,313 files here, costing 45s to save 3.3s.\n      // Erring low is the cheap direction, since a needless rebuild is linear and predictable.\n      const churn = delta.reparsed.length + delta.vanished.length;\n      const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n      if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n      // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n      // content), so vanished and reparsed docs delete in one pass.\n      const contentTouched = [...vanished, ...reparsedExisting];\n      if (contentTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM content WHERE \"path\" = ?',\n          contentTouched.map((p) => [p])\n        );\n      if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n      if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      const presetTouched = [...vanished, ...reparsedExisting];\n      if (presetTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          presetTouched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      const removedPaths = [...vanished, ...reparsedExisting];\n      if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    BEGIN_WRITE\n  );\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: the observed\n  // max is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // gets recorded rather than losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","hasUnspacedRun","getColumns","getMeta","quoteIdent","setMeta","BEGIN_WRITE","withTransaction","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","CORE_FRONTMATTER_COLUMNS","Set","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","rows","ctimeMs","parseError","data","runBatch","churn","bulk","name","contentTouched","ddl","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,eAAe,EAAEC,iBAAiB,QAAQ,YAAY;AAE/D,8FAA8F;AAC9F,4GAA4G;AAC5G,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOZ,eAAeY,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,mGAAmG;AACnG,MAAMC,wBAAwB;AAE9B,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,uHAAuH,CAAC;AAEpJ,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AAC5G;AAEA,OAAO,eAAeU,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ7B,UAAU2B,KAAKC;IAC7B,MAAME,aAAa,IAAIjB,IAAIgB,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAER,OAAO;IAErD,MAAMS,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAER,OAAO;QAClC,OAAO,CAACqB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWxD,eAAe6B;IAChC,MAAM4B,cAAc,MAAMnD,WAAWsB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAASzD,SAAS,mBAAmB6C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMzD,aAAa0C,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAGnC,yBAAyB;QAC/C,MAAM,IAAInB,WACR,gBACA,CAAC,uBAAuB,EAAEmE,WAAWb,MAAM,CAAC,2DAA2D,EAAEnC,wBAAwB,gPAAgP,CAAC;IAEtX;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMiD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMtD,yBAAyB+B,GAAG,CAACuB,MAAM,CAACjE,iBAAiB0C,GAAG,CAACuB;IAC1G,sHAAsH;IACtH,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACzB,YAAY8D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAG5D,WAAW4D,GAAG,YAAY,EAAE5D,WAAW4D,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAER,OAAO,GAAGO,GAAG,CAAC,CAACC,IAAMA,EAAER,OAAO;IACpF,MAAM8C,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEhD,OAAO;QAAG6C;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI5D,IAAIwD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEhD,OAAO,EAAEkB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMrE,gBACJiB,MACA;YA2DqEqD,iBAK7BA;QA/DtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAE1E,WAAWwD,MAAM;QAErG,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM8B,OAAOvB,WAAW3B,GAAG,CAAC,CAACX,MAC3B6C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ1C;oBANP,IAAI0C,QAAQ,QAAQ,OAAO1C,IAAII,OAAO;oBACtC,IAAIsC,QAAQ,UAAU,OAAO1C,IAAI4B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO1C,IAAI8D,OAAO;oBACxC,IAAIpB,QAAQ,SAAS,OAAO1C,IAAI8B,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO1C,IAAI+D,UAAU;oBACjD,QAAO/D,gBAAAA,IAAIgE,IAAI,CAACtB,IAAI,cAAb1C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMM,KAAK2D,QAAQ,CAAClB,WAAWc;QACjC;QAEA,2FAA2F;QAC3F,4FAA4F;QAC5F,yFAAyF;QACzF,2FAA2F;QAC3F,2FAA2F;QAC3F,sFAAsF;QACtF,yFAAyF;QACzF,MAAMK,QAAQhB,MAAMC,QAAQ,CAACpB,MAAM,GAAGmB,MAAM7B,QAAQ,CAACU,MAAM;QAC3D,MAAMoC,OAAOjB,MAAMzC,KAAK,CAACsB,MAAM,KAAK,KAAKmC,QAAQrE;QACjD,IAAIsE,MAAM,KAAK,MAAMC,QAAQ7E,kBAAmB,MAAMe,KAAKsD,IAAI,CAAC,CAAC,qBAAqB,EAAEQ,MAAM;QAE9F,0FAA0F;QAC1F,8DAA8D;QAC9D,MAAMC,iBAAiB;eAAIhD;eAAaiC;SAAiB;QACzD,IAAIe,eAAetC,MAAM,GAAG,GAC1B,MAAMzB,KAAK2D,QAAQ,CACjB,wCACAI,eAAe1D,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAACnE,oBAAoBwC,WAAW3B,GAAG,CAACZ;QAClF,IAAIoE,MAAM,KAAK,MAAMG,OAAOhF,gBAAiB,MAAMgB,KAAKsD,IAAI,CAACU;QAE7D,IAAIjD,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAK2D,QAAQ,CACjB,4CACA5C,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMgB,gBAAgB;eAAIlD;eAAaiC;SAAiB;QACxD,IAAIiB,cAAcxC,MAAM,GAAG,GACzB,MAAMzB,KAAK2D,QAAQ,CACjB,6CACAM,cAAc5D,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMiB,aAA0B,EAAE;QAClC,KAAK,MAAMxE,OAAOsC,WAAY,KAAK,MAAMmC,cAAczE,IAAI0E,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAAC3E,IAAII,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWzC,MAAM,GAAG,GAAG,MAAMzB,KAAK2D,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAIvD;eAAaiC;SAAiB;QACvD,IAAIsB,aAAa7C,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQkB,MAAM,cAAdlB,sCAAAA,qBAAAA,SAAiBrD,MAAMsE,cAAc1B;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMmB,iBAAiCxC,WAAW3B,GAAG,CAAC,CAACX,MAAS,CAAA;oBAAEoB,MAAMpB,IAAII,OAAO;oBAAE2E,WAAW/E,IAAI+E,SAAS,CAACpB,QAAQS,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMT,iBAAAA,QAAQqB,KAAK,cAAbrB,qCAAAA,oBAAAA,SAAgBrD,MAAMwE,gBAAgB5B;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQsB,cAAc,cAAtBtB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE,GACA9D;IAGF,8FAA8F;IAC9F,qDAAqD;IACrD,MAAM8F,aAAazB,KAAKC,GAAG,KAAKF;IAChC,MAAM2B,UAAU,MAAMlG,QAAQqB,MAAM;IACpC,yFAAyF;IACzF,0EAA0E;IAC1E,MAAM8E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAMjG,QAAQmB,MAAM,oBAAoBgF,OAAOJ;IAEzE,OAAO;QAAElD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}