sensemaking 0.18.4 → 0.19.1

package/dist/cjs/store/turso/lexical.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: string[]; unspaced: string[] } {\n  const words: string[] = [];\n  const phrases: string[] = [];\n  const unspaced: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    if (hasUnspacedRun(phrase)) unspaced.push(phrase);\n    else phrases.push(phrase);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) unspaced.push(tok);\n    else words.push(tok);\n  }\n  return { words, phrases, unspaced };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: string[], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const defaultParts = [...words, ...phrases].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `hit` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced } = splitTerms(terms);\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(opts.limit)) as unknown as LexicalHit[];\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  return rows;\n}\n"],"names":["queryLexical","FIELDS","NGRAM_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","_m","inner","phrase","length","hasUnspacedRun","push","split","tok","escapeFtsLiteral","query","buildQueries","defaultParts","map","t","unspacedParts","defaultQuery","join","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","conn","unsupported","branches","sql","stmt","rows","SenseError","prepare","all","limit","some","r","path"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;;;;;+BAyEjFA;;;eAAAA;;;wBAxEK;yBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAC3C,IAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AAEnE,uFAAuF;AACvF,+FAA+F;AAC/F,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,UAAoB,EAAE;IAC5B,IAAMC,WAAqB,EAAE;IAC7B,IAAMT,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACS,IAAIC;QACtD,IAAMC,SAASD,MAAMN,IAAI;QACzB,IAAIO,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChC,IAAIC,IAAAA,yBAAc,EAACF,SAASH,SAASM,IAAI,CAACH;aACrCJ,QAAQO,IAAI,CAACH;QAClB,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaZ,eAAegB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;YACtB,IAAIC,IAAAA,yBAAc,EAACG,MAAMR,SAASM,IAAI,CAACE;iBAClCV,MAAMQ,IAAI,CAACE;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,SAAAA;QAASC,UAAAA;IAAS;AACpC;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASS,iBAAiBC,KAAa;IACrC,OAAOA,MAAMlB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASmB,aAAab,KAAe,EAAEC,OAAiB,EAAEC,QAAkB;IAC1E,IAAMY,eAAe,AAAC,qBAAGd,cAAO,qBAAGC,UAASc,GAAG,CAAC,SAACC;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAC7D,IAAMC,gBAAgBf,SAASa,GAAG,CAAC,SAACC;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAChD,OAAO;QAAEE,cAAcJ,aAAaK,IAAI,CAAC;QAAUC,YAAYH,cAAcE,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASE,UAAUC,IAAuB,EAAEV,KAAa,EAAEW,IAAyB;IAClF,IAAMC,UAAUF,KAAKP,GAAG,CAAC,SAACU;eAAM,AAAC,WAAY,OAAFA;OAAKN,IAAI,CAAC;IACrD,IAAMO,MAAMf,iBAAiBC;IAC7B,OAAO,AAAC,4CAAwDc,OAAbF,SAAQ,OAC1CD,OAD+CG,KAAI,kCACjBF,OAAlCD,KAAKI,SAAS,EAAC,qBAAgCD,OAAbF,SAAQ,OAAcD,OAATG,KAAI,OAAuBH,OAAlBA,KAAKK,SAAS,EAAC,KAAkB,OAAfL,KAAKM,SAAS;AAC3G;AAIO,SAAe5C,aAAa6C,IAAgB,EAAEtC,KAAa,EAAE+B,IAAyB;;YACrFQ,aAI+BhC,aAA7BC,OAAOC,SAASC,UACaW,eAA7BK,cAAcE,YAGhBY,UAMAC,KAKAC,MACAC;;;;oBApBAJ,cAAcxC,oBAAoBC;oBACxC,IAAIuC,gBAAgB,MAAM;wBACxB,MAAM,IAAIK,oBAAU,CAAC,4BAA4B,AAAC,4CAAiEL,OAAvBA,YAAY1C,KAAK,EAAC,OAAqDG,OAAhDuC,YAAYlC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBAClL;oBACqCC,cAAAA,WAAWP,QAAxCQ,QAA6BD,YAA7BC,OAAOC,UAAsBF,YAAtBE,SAASC,WAAaH,YAAbG;oBACaW,gBAAAA,aAAab,OAAOC,SAASC,WAA1DgB,eAA6BL,cAA7BK,cAAcE,aAAeP,cAAfO;oBACtB,IAAIF,iBAAiB,MAAME,eAAe,IAAI;;;;oBAExCY;oBACN,IAAId,iBAAiB,IAAIc,SAASxB,IAAI,CAACa,UAAUnC,QAAQgC,cAAcK;oBACvE,IAAIH,eAAe,IAAIY,SAASxB,IAAI,CAACa,UAAUlC,cAAciC,YAAYG;oBAEzE,8FAA8F;oBAC9F,mFAAmF;oBAC7EU,MAAM,AAAC,uFAG+BD,OADtCA,SAASb,IAAI,CAAC,gBAAe,kDACyB,OAAhBa,SAAS1B,MAAM,EAAC;oBAE/C;;wBAAMwB,KAAKO,OAAO,CAACJ;;;oBAA1BC,OAAO;oBACC;;wBAAMA,KAAKI,GAAG,CAACf,KAAKgB,KAAK;;;oBAAjCJ,OAAQ;oBACd,+FAA+F;oBAC/F,yFAAyF;oBACzF,IAAIA,KAAKK,IAAI,CAAC,SAACC;+BAAMA,EAAEC,IAAI,KAAK;wBAAO;wBACrC,MAAM,IAAIN,oBAAU,CAAC,qBAAqB;oBAC5C;oBACA;;wBAAOD;;;;IACT"}

package/dist/cjs/store/turso/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// The one accessor every consumer of @tursodatabase/database (open.ts) must go through, rather\n// than each doing its own `import('@tursodatabase/database')`: once a fresh install has happened\n// in this process, Node keeps returning that pre-install \"not found\" miss to any NEW\n// bare-specifier lookup of the same path, so a second independent import elsewhere in the\n// process fails even though the package is now genuinely on disk (the same stale-cache problem\n// loadOrInstall works around for its own retry). Caching the resolved module here means only the\n// first caller ever resolves the bare specifier at all.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["TURSO_PACKAGE","tursoApi","TURSO","store","pkg","sizeHint","tursoApiPromise","loadOrInstall","packageNodeModules"],"mappings":";;;;;;;;;;;QAEaA;eAAAA;;QAaGC;eAAAA;;;wBAfyD;AAElE,IAAMD,gBAAgB;AAE7B,IAAME,QAA0B;IAAEC,OAAO;IAASC,KAAKJ;IAAeK,UAAU;AAAQ;AAExF,IAAIC;AASG,SAASL;IACd,IAAI,CAACK,iBAAiBA,kBAAkBC,IAAAA,uBAAa,EAA2CL,OAAOM,IAAAA,4BAAkB;IACzH,OAAOF;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// One accessor every consumer must go through, not each its own bare-specifier import: Node\n// keeps returning a pre-install \"not found\" miss to any new lookup of the same path afterward.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["TURSO_PACKAGE","tursoApi","TURSO","store","pkg","sizeHint","tursoApiPromise","loadOrInstall","packageNodeModules"],"mappings":";;;;;;;;;;;QAEaA;eAAAA;;QAQGC;eAAAA;;;wBAVyD;AAElE,IAAMD,gBAAgB;AAE7B,IAAME,QAA0B;IAAEC,OAAO;IAASC,KAAKJ;IAAeK,UAAU;AAAQ;AAExF,IAAIC;AAIG,SAASL;IACd,IAAI,CAACK,iBAAiBA,kBAAkBC,IAAAA,uBAAa,EAA2CL,OAAOM,IAAAA,4BAAkB;IACzH,OAAOF;AACT"}

package/dist/cjs/store/turso/open.d.cts

@@ -1,7 +1,7 @@
 import type { ResolvedConfig } from '../../config/index.js';
 import type { Store } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "1";
+export declare const SCHEMA_VERSION = "3";
 export interface OpenResult {
     store: Store;
     cfg: ResolvedConfig;
@@ -9,4 +9,6 @@ export interface OpenResult {
     parsed: number;
     warnings: string[];
 }
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.d.ts

@@ -1,7 +1,7 @@
 import type { ResolvedConfig } from '../../config/index.js';
 import type { Store } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "1";
+export declare const SCHEMA_VERSION = "3";
 export interface OpenResult {
     store: Store;
     cfg: ResolvedConfig;
@@ -9,4 +9,6 @@ export interface OpenResult {
     parsed: number;
     warnings: string[];
 }
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.js

@@ -9,6 +9,12 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get CONTENT_FTS_DDL () {
+        return CONTENT_FTS_DDL;
+    },
+    get CONTENT_FTS_NAMES () {
+        return CONTENT_FTS_NAMES;
+    },
     get DB_FILENAME () {
         return DB_FILENAME;
     },
@@ -22,10 +28,12 @@ _export(exports, {
 var _nodefs = require("node:fs");
 var _nodepath = require("node:path");
 var _indexts = require("../../config/index.js");
+var _handoffts = require("../../embed/handoff.js");
+var _typests = require("../../embed/types.js");
 var _errorsts = require("../../errors.js");
 var _indexts1 = require("../../features/index.js");
+var _cachets = require("../cache.js");
 var _sharedts = require("../shared.js");
-var _opents = require("../sqlite/open.js");
 var _connectionts = require("./connection.js");
 var _nativets = require("./native.js");
 var _reconcilets = require("./reconcile.js");
@@ -162,13 +170,18 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.turso.db';
-var SCHEMA_VERSION = '1';
-// content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is
-// phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect
-// accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.
+var SCHEMA_VERSION = '3';
+var CONTENT_FTS_DDL = [
+    "CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')",
+    "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"
+];
+var CONTENT_FTS_NAMES = [
+    'content_fts',
+    'content_fts_ngram'
+];
 function ensureSchema(conn, cfg) {
     return _async_to_generator(function() {
-        var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
+        var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, ddl, err, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, feature, err;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -180,113 +193,185 @@ function ensureSchema(conn, cfg) {
                     _state.sent();
                     return [
                         4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)')
+                        conn.exec('CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)')
                     ];
                 case 2:
                     _state.sent();
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 3;
+                case 3:
+                    _state.trys.push([
+                        3,
+                        8,
+                        9,
+                        10
+                    ]);
+                    _iterator = CONTENT_FTS_DDL[Symbol.iterator]();
+                    _state.label = 4;
+                case 4:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        7
+                    ];
+                    ddl = _step.value;
+                    return [
+                        4,
+                        conn.exec(ddl)
+                    ];
+                case 5:
+                    _state.sent();
+                    _state.label = 6;
+                case 6:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        4
+                    ];
+                case 7:
+                    return [
+                        3,
+                        10
+                    ];
+                case 8:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        10
+                    ];
+                case 9:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 10:
                     return [
                         4,
                         conn.exec('CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))')
                     ];
-                case 3:
+                case 11:
                     _state.sent();
                     return [
                         4,
                         conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)')
                     ];
-                case 4:
+                case 12:
                     _state.sent();
-                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                    _state.label = 5;
-                case 5:
+                    _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                    _state.label = 13;
+                case 13:
                     _state.trys.push([
-                        5,
-                        10,
-                        11,
-                        12
+                        13,
+                        20,
+                        21,
+                        22
                     ]);
-                    _iterator = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
-                    _state.label = 6;
-                case 6:
-                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                    _iterator1 = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
+                    _state.label = 14;
+                case 14:
+                    if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
                         3,
-                        9
+                        19
+                    ];
+                    feature = _step1.value;
+                    if (!(feature.name === 'embed')) return [
+                        3,
+                        16
                     ];
-                    feature = _step.value;
+                    return [
+                        4,
+                        conn.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB('.concat(_typests.STORE_DIMS, '), PRIMARY KEY ("path", chunk))'))
+                    ];
+                case 15:
+                    _state.sent();
+                    return [
+                        3,
+                        18
+                    ];
+                case 16:
                     return [
                         4,
                         feature.schema(conn)
                     ];
-                case 7:
+                case 17:
                     _state.sent();
-                    _state.label = 8;
-                case 8:
-                    _iteratorNormalCompletion = true;
+                    _state.label = 18;
+                case 18:
+                    _iteratorNormalCompletion1 = true;
                     return [
                         3,
-                        6
+                        14
                     ];
-                case 9:
+                case 19:
                     return [
                         3,
-                        12
+                        22
                     ];
-                case 10:
+                case 20:
                     err = _state.sent();
-                    _didIteratorError = true;
-                    _iteratorError = err;
+                    _didIteratorError1 = true;
+                    _iteratorError1 = err;
                     return [
                         3,
-                        12
+                        22
                     ];
-                case 11:
+                case 21:
                     try {
-                        if (!_iteratorNormalCompletion && _iterator.return != null) {
-                            _iterator.return();
+                        if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                            _iterator1.return();
                         }
                     } finally{
-                        if (_didIteratorError) {
-                            throw _iteratorError;
+                        if (_didIteratorError1) {
+                            throw _iteratorError1;
                         }
                     }
                     return [
                         7
                     ];
-                case 12:
+                case 22:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'schema_version')
                     ];
-                case 13:
+                case 23:
                     if (!(_state.sent() === null)) return [
                         3,
-                        15
+                        25
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'schema_version', SCHEMA_VERSION)
                     ];
-                case 14:
+                case 24:
                     _state.sent();
-                    _state.label = 15;
-                case 15:
+                    _state.label = 25;
+                case 25:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'features')
                     ];
-                case 16:
+                case 26:
                     if (!(_state.sent() === null)) return [
                         3,
-                        18
+                        28
                     ];
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES))
                     ];
-                case 17:
+                case 27:
                     _state.sent();
-                    _state.label = 18;
-                case 18:
+                    _state.label = 28;
+                case 28:
                     return [
                         2
                     ];
@@ -337,15 +422,15 @@ function connect(cfg) {
                     return [
                         4,
                         turso.connect(dbPath, {
-                            timeout: 30000
+                            timeout: 30000,
+                            experimental: [
+                                'index_method'
+                            ]
                         })
                     ];
                 case 5:
-                    // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this
-                    // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --
-                    // which depends on this cache's own recorded reconcile_max_ms, readable only once connected
-                    // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same
-                    // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).
+                    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;
+                    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).
                     db = _state.sent();
                     return [
                         3,
@@ -355,9 +440,8 @@ function connect(cfg) {
                     err1 = _state.sent();
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "turso" failed to open '.concat(dbPath, ": ").concat(err1.message));
                 case 7:
-                    // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL
-                    // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and
-                    // recurses, so the catch never double-closes.
+                    // A throw below must release this handle, or the leaked WAL makes the db file undeletable on
+                    // Windows. closed guards the catch against double-closing.
                     closed = false;
                     _state.label = 8;
                 case 8:
@@ -391,11 +475,8 @@ function connect(cfg) {
                         3,
                         13
                     ];
-                    // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --
-                    // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is
-                    // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for
-                    // those): turso has no content.tokenize knob to special-case, and the grace is an
-                    // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.
+                    // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched
+                    // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.
                     reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
                     console.error("sense: ".concat(reason, "; rebuilding the index"));
                     closed = true;
@@ -405,7 +486,7 @@ function connect(cfg) {
                     ];
                 case 12:
                     _state.sent();
-                    (0, _opents.clearCache)(cfg);
+                    (0, _cachets.clearCache)(cfg);
                     return [
                         2,
                         connect(cfg)
@@ -473,7 +554,7 @@ function connect(cfg) {
 }
 function openTurso(cfg) {
     return _async_to_generator(function() {
-        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings;
+        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings, store;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -483,10 +564,13 @@ function openTurso(cfg) {
                     ];
                 case 1:
                     _ref = _state.sent(), db = _ref.db, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
+                    store = (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir);
+                    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
+                    (0, _handoffts.rekeyChunkText)(conn, store);
                     return [
                         2,
                         {
-                            store: (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir),
+                            store: store,
                             cfg: resolvedCfg,
                             dbPath: dbPath,
                             parsed: parsed,

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { clearCache } from '../sqlite/open.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's SCHEMA_VERSION and duckdb's -- each store's cache shape evolves\n// separately, and the store name already joins the feature signature (see featureSignature), so\n// a config's `store` key switching cleanly rebuilds rather than reusing another engine's cache.\nexport const SCHEMA_VERSION = '1';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// content is a plain table (not FTS-virtual): phase 1 creates no FTS index at all -- that is\n// phase 2's job (see the plan). Untyped ALTER TABLE ADD COLUMN, like sqlite -- turso's dialect\n// accepts it with no declared type (spike-confirmed), unlike duckdb which requires VARIANT.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never even attempt to resolve\n  // this optional dependency, so nothing in this store's module graph imports it as a value until\n  // a turso tree is actually opened (type-only imports elsewhere are erased and cost nothing\n  // either way). Installed on first use if missing (see native.ts's tursoApi).\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is a connect-time option in this\n    // client (no synchronous PRAGMA-at-open-time escape hatch), so the real derived value --\n    // which depends on this cache's own recorded reconcile_max_ms, readable only once connected\n    // -- is installed via a runtime PRAGMA below, right before the open-time reconcile, the same\n    // lever sqlite's open() uses (spike-confirmed: raw PRAGMA busy_timeout works as a setter too).\n    db = await turso.connect(dbPath, { timeout: 30_000 });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) must release this handle, or the leaked WAL\n  // makes the db file undeletable on Windows. closed marks the rebuild branch that closes and\n  // recurses, so the catch never double-closes.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's and duckdb's open() (see their comments for why this is\n      // announced). No tokenize-only or embed-identity-adopted partial path (sqlite's grace for\n      // those): turso has no content.tokenize knob to special-case, and the grace is an\n      // optimization, not a correctness requirement -- any mismatch rebuilds, duckdb's shape.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","feature","exec","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","resolvedCfg","store","createStore"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIAC;eAAAA;;QAwGSC;eAAAA;;;sBA3HI;wBACL;uBAGuB;wBACjB;wBACc;wBACR;sBACN;4BAEM;wBACO;2BACd;uBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAIpB,IAAMC,iBAAiB;AAU9B,6FAA6F;AAC7F,+FAA+F;AAC/F,4FAA4F;AAC5F,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAKlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;oBAAsC;;wBAAMA,QAAQG,MAAM,CAACL;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMS,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,YAAYQ,IAAAA,yBAAgB,EAACP,KAAKQ,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQT,GAAmB;;YAClCU,UAEAC,QAMFC,OAGKC,KAKLC,IAQKD,MAMLE,QA2B4B,MAzBxBhB,MAIAiB,SACAC,UACAC,cAOEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBA/DHH,WAAWa,IAAAA,cAAI,EAACvB,IAAIwB,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUf;;;;;;;;;oBAQpB;;wBAAMiC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAUxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;wBAAO;;;oBALnD,wFAAwF;oBACxF,yFAAyF;oBACzF,4FAA4F;oBAC5F,6FAA6F;oBAC7F,+FAA+F;oBAC/FlB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,+FAA+F;oBAC/F,4FAA4F;oBAC5F,8CAA8C;oBAC1ChB,SAAS;;;;;;;;;oBAELhB,OAAOkC,IAAAA,8BAAgB,EAACnB;oBAE9B;;wBAAMf,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMG,IAAAA,iBAAO,EAACN,MAAM;;;oBAA9BiB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACN,MAAM;;;oBAA/BkB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACP,KAAKQ,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYpB,kBAAoBqB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,4FAA4F;oBAC5F,4FAA4F;oBAC5F,0FAA0F;oBAC1F,kFAAkF;oBAClF,wFAAwF;oBAClFC,SAASH,YAAY,QAAQA,YAAYpB,iBAAiB,mDAAmD;oBACnHsC,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPhB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGsB,KAAK;;;oBAAd;oBACAC,IAAAA,kBAAU,EAACrC;oBACX;;wBAAOS,QAAQT;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMK,IAAAA,iBAAO,EAACN,MAAM;;;oBAA5CqB,gBAAgBkB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAMvC,KAAKG,IAAI,CAAC,AAAC,yBAA8E,OAAtDqC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIrB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMsB,IAAAA,sBAAS,EAAC3C,MAAMC,KAAKA,IAAIwB,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIf,MAAAA;4BAAMC,KAAAA;4BAAKW,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGsB,KAAK;;;oBAAd;;;oBACb,MAAMvB;;;;;;;IAEV;;AAIO,SAAehB,UAAUG,GAAmB;;YACgB,MAAzDc,IAAIf,MAAW4C,aAAahC,QAAQU,QAAQC;;;;oBAAa;;wBAAMb,QAAQT;;;oBAAd,OAAA,eAAzDc,KAAyD,KAAzDA,IAAIf,OAAqD,KAArDA,MAAW4C,cAA0C,KAA/C3C,KAAkBW,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBACpD;;wBAAO;4BAAEsB,OAAOC,IAAAA,oBAAW,EAAC/B,IAAIf,MAAM4C,aAAaA,YAAYnB,OAAO;4BAAGxB,KAAK2C;4BAAahC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACtH"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","ddl","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAkCaA;eAAAA;;QAIAC;eAAAA;;QArBAC;eAAAA;;QAGAC;eAAAA;;QA+GSC;eAAAA;;;sBAnII;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;wBACM;4BAEA;wBACO;2BACd;uBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAcvB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,SAAeI,aAAaC,IAAgB,EAAEC,GAAW;;YAGlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaV;;;2BAAb,6BAAA,QAAA;;;;oBAAMQ,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBC,IAAAA,wBAAc,EAACJ;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXG,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACR;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMY,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,YAAYW,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQZ,GAAmB;;YAClCa,UAEAC,QAIFC,OAGKC,KAKLC,IAKKD,MAKLE,QAwB4B,MAtBxBnB,MAIAoB,SACAC,UACAC,cAIEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBAtDHH,WAAWa,IAAAA,cAAI,EAAC1B,IAAI2B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUlB;;;;;;;;;oBAMpB;;wBAAMoC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHnB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,6FAA6F;oBAC7F,2DAA2D;oBACvDhB,SAAS;;;;;;;;;oBAELnB,OAAOsC,IAAAA,8BAAgB,EAACpB;oBAE9B;;wBAAMlB,KAAKI,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACT,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACT,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,yFAAyF;oBACzF,iGAAiG;oBAC3FC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;oBACnH0C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPjB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGuB,KAAK;;;oBAAd;oBACAC,IAAAA,mBAAU,EAACzC;oBACX;;wBAAOY,QAAQZ;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMQ,IAAAA,iBAAO,EAACT,MAAM;;;oBAA5CwB,gBAAgBmB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAM3C,KAAKI,IAAI,CAAC,AAAC,yBAA8E,OAAtDwC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAItB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMuB,IAAAA,sBAAS,EAAC/C,MAAMC,KAAKA,IAAI2B,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIlB,MAAAA;4BAAMC,KAAAA;4BAAKc,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGuB,KAAK;;;oBAAd;;;oBACb,MAAMxB;;;;;;;IAEV;;AAIO,SAAenB,UAAUG,GAAmB;;YACgB,MAAzDiB,IAAIlB,MAAWgD,aAAajC,QAAQU,QAAQC,UAC9CuB;;;;oBAD2D;;wBAAMpC,QAAQZ;;;oBAAd,OAAA,eAAzDiB,KAAyD,KAAzDA,IAAIlB,OAAqD,KAArDA,MAAWgD,cAA0C,KAA/C/C,KAAkBc,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC9CuB,QAAQC,IAAAA,oBAAW,EAAChC,IAAIlB,MAAMgD,aAAaA,YAAYpB,OAAO;oBACpE,0FAA0F;oBAC1FuB,IAAAA,yBAAc,EAACnD,MAAMiD;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAOhD,KAAK+C;4BAAajC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}