sensemaking 0.23.1 → 0.24.0

package/dist/cjs/store/turso/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: string[]; unspaced: string[] } {\n  const words: string[] = [];\n  const phrases: string[] = [];\n  const unspaced: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    if (hasUnspacedRun(phrase)) unspaced.push(phrase);\n    else phrases.push(phrase);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (hasUnspacedRun(tok)) unspaced.push(tok);\n    else words.push(tok);\n  }\n  return { words, phrases, unspaced };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: string[], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const defaultParts = [...words, ...phrases].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `hit` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS excerpt fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced } = splitTerms(terms);\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC.\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(opts.limit)) as unknown as LexicalHit[];\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  return rows;\n}\n"],"names":["queryLexical","FIELDS","NGRAM_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","_m","inner","phrase","length","hasUnspacedRun","push","split","tok","escapeFtsLiteral","query","buildQueries","defaultParts","map","t","unspacedParts","defaultQuery","join","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","conn","unsupported","branches","sql","stmt","rows","SenseError","prepare","all","limit","some","r","path"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;;;;;+BAyEjFA;;;eAAAA;;;wBAxEK;yBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAC3C,IAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AAEnE,uFAAuF;AACvF,+FAA+F;AAC/F,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,UAAoB,EAAE;IAC5B,IAAMC,WAAqB,EAAE;IAC7B,IAAMT,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACS,IAAIC;QACtD,IAAMC,SAASD,MAAMN,IAAI;QACzB,IAAIO,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChC,IAAIC,IAAAA,yBAAc,EAACF,SAASH,SAASM,IAAI,CAACH;aACrCJ,QAAQO,IAAI,CAACH;QAClB,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAaZ,eAAegB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIJ,MAAM,KAAK,GAAG;YACtB,IAAIC,IAAAA,yBAAc,EAACG,MAAMR,SAASM,IAAI,CAACE;iBAClCV,MAAMQ,IAAI,CAACE;QAClB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAO;QAAEV,OAAAA;QAAOC,SAAAA;QAASC,UAAAA;IAAS;AACpC;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASS,iBAAiBC,KAAa;IACrC,OAAOA,MAAMlB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASmB,aAAab,KAAe,EAAEC,OAAiB,EAAEC,QAAkB;IAC1E,IAAMY,eAAe,AAAC,qBAAGd,cAAO,qBAAGC,UAASc,GAAG,CAAC,SAACC;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAC7D,IAAMC,gBAAgBf,SAASa,GAAG,CAAC,SAACC;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAChD,OAAO;QAAEE,cAAcJ,aAAaK,IAAI,CAAC;QAAUC,YAAYH,cAAcE,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASE,UAAUC,IAAuB,EAAEV,KAAa,EAAEW,IAAyB;IAClF,IAAMC,UAAUF,KAAKP,GAAG,CAAC,SAACU;eAAM,AAAC,WAAY,OAAFA;OAAKN,IAAI,CAAC;IACrD,IAAMO,MAAMf,iBAAiBC;IAC7B,OAAO,AAAC,4CAAwDc,OAAbF,SAAQ,OAC1CD,OAD+CG,KAAI,kCACjBF,OAAlCD,KAAKI,SAAS,EAAC,qBAAgCD,OAAbF,SAAQ,OAAcD,OAATG,KAAI,OAAuBH,OAAlBA,KAAKK,SAAS,EAAC,KAAkB,OAAfL,KAAKM,SAAS;AAC3G;AAIO,SAAe5C,aAAa6C,IAAgB,EAAEtC,KAAa,EAAE+B,IAAyB;;YACrFQ,aAI+BhC,aAA7BC,OAAOC,SAASC,UACaW,eAA7BK,cAAcE,YAGhBY,UAMAC,KAKAC,MACAC;;;;oBApBAJ,cAAcxC,oBAAoBC;oBACxC,IAAIuC,gBAAgB,MAAM;wBACxB,MAAM,IAAIK,oBAAU,CAAC,4BAA4B,AAAC,4CAAiEL,OAAvBA,YAAY1C,KAAK,EAAC,OAAqDG,OAAhDuC,YAAYlC,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBAClL;oBACqCC,cAAAA,WAAWP,QAAxCQ,QAA6BD,YAA7BC,OAAOC,UAAsBF,YAAtBE,SAASC,WAAaH,YAAbG;oBACaW,gBAAAA,aAAab,OAAOC,SAASC,WAA1DgB,eAA6BL,cAA7BK,cAAcE,aAAeP,cAAfO;oBACtB,IAAIF,iBAAiB,MAAME,eAAe,IAAI;;;;oBAExCY;oBACN,IAAId,iBAAiB,IAAIc,SAASxB,IAAI,CAACa,UAAUnC,QAAQgC,cAAcK;oBACvE,IAAIH,eAAe,IAAIY,SAASxB,IAAI,CAACa,UAAUlC,cAAciC,YAAYG;oBAEzE,8FAA8F;oBAC9F,mFAAmF;oBAC7EU,MAAM,AAAC,uFAG+BD,OADtCA,SAASb,IAAI,CAAC,gBAAe,kDACyB,OAAhBa,SAAS1B,MAAM,EAAC;oBAE/C;;wBAAMwB,KAAKO,OAAO,CAACJ;;;oBAA1BC,OAAO;oBACC;;wBAAMA,KAAKI,GAAG,CAACf,KAAKgB,KAAK;;;oBAAjCJ,OAAQ;oBACd,+FAA+F;oBAC/F,yFAAyF;oBACzF,IAAIA,KAAKK,IAAI,CAAC,SAACC;+BAAMA,EAAEC,IAAI,KAAK;wBAAO;wBACrC,MAAM,IAAIN,oBAAU,CAAC,qBAAqB;oBAC5C;oBACA;;wBAAOD;;;;IACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n  const words: string[] = [];\n  const phrases: SearchToken[][] = [];\n  const unspaced: string[] = [];\n  let emptyPhrase = false;\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    const tokens = searchTokens(phrase);\n    if (tokens.length === 0) {\n      emptyPhrase = true;\n      return ' ';\n    }\n    unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n    phrases.push(tokens);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    for (const token of searchTokens(tok)) {\n      if (token.unspaced) unspaced.push(token.text);\n      else words.push(token.text);\n    }\n  }\n  return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n  return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n  const phraseParts = phrases.flatMap((phrase) => {\n    const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n    if (latin.length === 0) return [];\n    return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n  });\n  const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n  const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n  return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n  const colList = cols.map((c) => `content.${c}`).join(', ');\n  const lit = escapeFtsLiteral(query);\n  return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n    FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n  if (emptyPhrase) return [];\n  const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n  if (defaultQuery === '' && ngramQuery === '') return [];\n\n  const branches: string[] = [];\n  if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n  if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n  // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n  // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n  // candidates carry authored fields through the join below so the shared verifier can reject\n  // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n  const ranked = `SELECT path, SUM(score) AS score FROM (\n      ${branches.join(' UNION ALL ')}\n    ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n  const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n  const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n  const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n    ${ranked}\n  ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n  const stmt = await conn.prepare(sql);\n  const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n  // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n  // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n  if (rows.some((r) => r.path === null)) {\n    throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n  }\n  const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n  return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["queryLexical","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","searchTokens","length","push","filter","map","text","split","tok","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","stemFolded","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","conn","stmt","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","rows","matches","SenseError","prepare","all","limit","r","path","row","every","field","matchesSearchPhrase","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;;;;;+BAwFjFA;;;eAAAA;;;wBAvFK;yBACyC;6BAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE3B,IAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,IAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,IAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,UAA2B,EAAE;IACnC,IAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,IAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACU,IAAIC;YAOtDH;QANA,IAAMI,SAASD,MAAMP,IAAI;QACzB,IAAMS,SAASC,IAAAA,uBAAY,EAACF;QAC5B,IAAIC,OAAOE,MAAM,KAAK,GAAG;YACvBN,cAAc;YACd,OAAO;QACT;QACAD,CAAAA,YAAAA,UAASQ,IAAI,OAAbR,WAAc,qBAAGK,OAAOI,MAAM,CAAC,SAACd;mBAAUA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUA,MAAMgB,IAAI;;QACnFZ,QAAQS,IAAI,CAACH;QACb,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAad,eAAeqB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIN,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACO,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAMlB,QAAN;oBACH,IAAIA,MAAMK,QAAQ,EAAEA,SAASQ,IAAI,CAACb,MAAMgB,IAAI;yBACvCb,MAAMU,IAAI,CAACb,MAAMgB,IAAI;gBAC5B;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAIP;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,OAAO;QAAEb,OAAAA;QAAOC,SAAAA;QAASC,UAAAA;QAAUC,aAAAA;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASa,iBAAiBC,KAAa;IACrC,OAAOA,MAAMvB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASwB,aAAalB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,IAAMiB,cAAclB,QAAQmB,OAAO,CAAC,SAACd;QACnC,IAAMe,QAAQf,OAAOK,MAAM,CAAC,SAACd;mBAAU,CAACA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUyB,IAAAA,yBAAU,EAACzB,MAAMgB,IAAI;;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOH,OAAOiB,IAAI,CAAC,SAAC1B;mBAAUA,MAAMK,QAAQ;aAAImB,QAAQ;YAACA,MAAMG,IAAI,CAAC;SAAK;IAC3E;IACA,IAAMC,eAAe,AAAC,qBAAGzB,MAAMY,GAAG,CAACU,yBAAU,UAAG,qBAAGH,cAAaP,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IACjF,IAAMC,gBAAgBzB,SAASU,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAChD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEd,KAAa,EAAEe,IAAyB;IAClF,IAAMC,UAAUF,KAAKnB,GAAG,CAAC,SAACsB;eAAM,AAAC,WAAY,OAAFA;OAAKV,IAAI,CAAC;IACrD,IAAMW,MAAMnB,iBAAiBC;IAC7B,OAAO,AAAC,4CAAwDkB,OAAbF,SAAQ,OAC1CD,OAD+CG,KAAI,kCACjBF,OAAlCD,KAAKI,SAAS,EAAC,qBAAgCD,OAAbF,SAAQ,OAAcD,OAATG,KAAI,OAAuBH,OAAlBA,KAAKK,SAAS,EAAC,KAAkB,OAAfL,KAAKM,SAAS;AAC3G;AAIO,SAAetD,aAAauD,IAAgB,EAAE/C,KAAa,EAAEwC,IAAyB;;YA2BvEQ,OA1BdC,aAI4C1C,aAA1CC,OAAOC,SAASC,UAAUC,aAEGe,eAA7BU,cAAcC,YAGhBa,UAQAC,QAGAC,cACAC,YACAC,KAGAN,MACAO,MAMAC;;;;oBAhCAP,cAAclD,oBAAoBC;oBACxC,IAAIiD,gBAAgB,MAAM;wBACxB,MAAM,IAAIQ,oBAAU,CAAC,4BAA4B,AAAC,4CAAiER,OAAvBA,YAAYpD,KAAK,EAAC,OAAqDG,OAAhDiD,YAAY5C,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBAClL;oBACkDC,cAAAA,WAAWP,QAArDQ,QAA0CD,YAA1CC,OAAOC,UAAmCF,YAAnCE,SAASC,WAA0BH,YAA1BG,UAAUC,cAAgBJ,YAAhBI;oBAClC,IAAIA,aAAa;;;;oBACoBe,gBAAAA,aAAalB,OAAOC,SAASC,WAA1D0B,eAA6BV,cAA7BU,cAAcC,aAAeX,cAAfW;oBACtB,IAAID,iBAAiB,MAAMC,eAAe,IAAI;;;;oBAExCa;oBACN,IAAId,iBAAiB,IAAIc,SAAShC,IAAI,CAACoB,UAAU7C,QAAQ2C,cAAcI;oBACvE,IAAIH,eAAe,IAAIa,SAAShC,IAAI,CAACoB,UAAU5C,cAAc2C,YAAYG;oBAEzE,8FAA8F;oBAC9F,0FAA0F;oBAC1F,4FAA4F;oBAC5F,uFAAuF;oBACjFW,SAAS,AAAC,kDAE4BD,OADtCA,SAASlB,IAAI,CAAC,gBAAe,kDACyB,OAAhBkB,SAASjC,MAAM;oBACrDmC,eAAe3C,QAAQQ,MAAM,GAAG,IAAI,mDAAmD;oBACvFoC,aAAa5C,QAAQQ,MAAM,GAAG,IAAI,4CAA4C;oBAC9EqC,MAAM,AAAC,2BACTH,OADmCC,cAAa,iBAE9CC,OADFF,QAAO,YACwC1C,OAA7C4C,YAAW,oCAAuE,OAArC5C,QAAQQ,MAAM,GAAG,IAAI,KAAK;oBAChE;;wBAAM8B,KAAKW,OAAO,CAACJ;;;oBAA1BN,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKW,GAAG,OAARX,OAAS,qBAAIvC,QAAQQ,MAAM,GAAG;4BAAUuB,KAAKoB,KAAK;;;;oBAAhEL,OAAQ;oBACd,+FAA+F;oBAC/F,yFAAyF;oBACzF,IAAIA,KAAKxB,IAAI,CAAC,SAAC8B;+BAAMA,EAAEC,IAAI,KAAK;wBAAO;wBACrC,MAAM,IAAIL,oBAAU,CAAC,qBAAqB;oBAC5C;oBACMD,UAAU/C,QAAQQ,MAAM,KAAK,IAAIsC,OAAOA,KAAKpC,MAAM,CAAC,SAAC4C;+BAAQtD,QAAQuD,KAAK,CAAC,SAAClD;mCAAWnB,gBAAgBoC,IAAI,CAAC,SAACkC;oCAA8BF;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAIjD;;;;oBACnK;;wBAAO0C,QAAQW,KAAK,CAAC,GAAG3B,KAAKoB,KAAK,EAAExC,GAAG,CAAC;gCAAG0C,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAChE"}

package/dist/cjs/store/turso/open.d.cts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "5";
+export declare const SCHEMA_VERSION = "8";
 export type { OpenResult };
 export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "5";
+export declare const SCHEMA_VERSION = "8";
 export type { OpenResult };
 export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/cjs/store/turso/open.js

@@ -240,7 +240,7 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
 var DB_FILENAME = 'cache.turso.db';
-var SCHEMA_VERSION = '5';
+var SCHEMA_VERSION = '8';
 // This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
 // extra native state to thread, unlike duckdb's separate instance/connection pair.
 function ensureSchema(_handle, conn, cfg) {
@@ -257,7 +257,7 @@ function ensureSchema(_handle, conn, cfg) {
                     _state.sent();
                     return [
                         4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)')
+                        conn.exec('CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_stem TEXT, summary_stem TEXT, text_stem TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)')
                     ];
                 case 2:
                     _state.sent();

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { CONNECT_OPTS, TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // `timeout` is connect-time only in this client; the derived value is set via runtime PRAGMA\n    // below. Options in native.ts, shared with reclaimSpace's own connection.\n    db = await turso.connect(dbPath, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","checkpointWal","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","CONNECT_OPTS","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAwESC;eAAAA;;QAfTC;eAAAA;;;uBAzEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEe;wBACM;2BACR;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMC,IAAAA,2BAAa,EAACD;;;oBAApB;oBACA;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeG,sBAAsBlB,OAAiB,EAAEC,IAAgB,EAAEkB,EAAU;;;;;oBAClF;;wBAAMlB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHc;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ,wCAAKS,sBAAY;4BAAEC,cAAe,qBAAGD,sBAAY,CAACC,YAAY;;;;oBAF/F,6FAA6F;oBAC7F,0EAA0E;oBAC1EN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEb,QAAQS;4BAAIxB,MAAM+B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM3B,mBAA0C;IACrDmC,UAAUtC;IACVuC,eAAetC;IACfuC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAL,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGsB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG9B,cAAAA;IACAmB,uBAAAA;IACAqB,aAAa,SAAbA,YAAcvB,QAAQf;eAASsC,IAAAA,oBAAW,EAACvB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOsC,IAAAA,uBAAe,EAACtC,KAAKJ;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { CONNECT_OPTS, TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '8';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_stem TEXT, summary_stem TEXT, text_stem TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // `timeout` is connect-time only in this client; the derived value is set via runtime PRAGMA\n    // below. Options in native.ts, shared with reclaimSpace's own connection.\n    db = await turso.connect(dbPath, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","checkpointWal","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","CONNECT_OPTS","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAwESC;eAAAA;;QAfTC;eAAAA;;;uBAzEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEe;wBACM;2BACR;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMC,IAAAA,2BAAa,EAACD;;;oBAApB;oBACA;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeG,sBAAsBlB,OAAiB,EAAEC,IAAgB,EAAEkB,EAAU;;;;;oBAClF;;wBAAMlB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHc;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ,wCAAKS,sBAAY;4BAAEC,cAAe,qBAAGD,sBAAY,CAACC,YAAY;;;;oBAF/F,6FAA6F;oBAC7F,0EAA0E;oBAC1EN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEb,QAAQS;4BAAIxB,MAAM+B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM3B,mBAA0C;IACrDmC,UAAUtC;IACVuC,eAAetC;IACfuC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAL,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGsB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG9B,cAAAA;IACAmB,uBAAAA;IACAqB,aAAa,SAAbA,YAAcvB,QAAQf;eAASsC,IAAAA,oBAAW,EAACvB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOsC,IAAAA,uBAAe,EAACtC,KAAKJ;;;IAC9B"}

package/dist/cjs/store/turso/reconcile.d.cts

@@ -1,5 +1,5 @@
 import type { ReconcileDialect } from '../types.js';
-export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
 export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare const FTS_REBUILD_THRESHOLD = 250;
 export declare const tursoDialect: ReconcileDialect;

package/dist/cjs/store/turso/reconcile.d.ts

@@ -1,5 +1,5 @@
 import type { ReconcileDialect } from '../types.js';
-export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
 export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare const FTS_REBUILD_THRESHOLD = 250;
 export declare const tursoDialect: ReconcileDialect;

package/dist/cjs/store/turso/reconcile.js

@@ -26,6 +26,7 @@ var _errorsts = require("../../errors.js");
 var _segmentts = require("../../text/segment.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
+var _lexicaltextts = require("./lexical-text.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -152,7 +153,7 @@ function _ts_generator(thisArg, body) {
     }
 }
 var CONTENT_FTS_DDL = [
-    "CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')",
+    "CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')",
     "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"
 ];
 var CONTENT_FTS_NAMES = [
@@ -169,7 +170,7 @@ function ngramSidecar(text) {
 var MAX_FRONTMATTER_COLUMNS = 2000;
 var FTS_REBUILD_THRESHOLD = 250;
 // No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.
-var INSERT_CONTENT_SQL = 'INSERT INTO content ("path", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)';
+var INSERT_CONTENT_SQL = 'INSERT INTO content ("path", title, summary, text, title_stem, summary_stem, text_stem, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
 function contentRow(doc) {
     var _doc_search = doc.search, title = _doc_search.title, summary = _doc_search.summary, text = _doc_search.text;
     return [
@@ -177,6 +178,9 @@ function contentRow(doc) {
         title,
         summary,
         text,
+        (0, _lexicaltextts.stemFolded)(title),
+        (0, _lexicaltextts.stemFolded)(summary),
+        (0, _lexicaltextts.stemFolded)(text),
         ngramSidecar(title),
         ngramSidecar(summary),
         ngramSidecar(text)

package/dist/cjs/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","FTS_REBUILD_THRESHOLD","tursoDialect","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","reconcileContent","conn","touched","docs","delta","churn","bulk","name","ddl","reparsed","length","vanished","files","exec","runBatch","map","p","addColumns","names","quoteIdent","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIAC;eAAAA;;QAcAC;eAAAA;;QAkCAC;eAAAA;;;wBAnEc;yBAGI;wBACqB;6BACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAUrB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,6FAA6F;AAC7F,gGAAgG;AAChG,SAASG,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAIzB,IAAML,wBAAwB;AAErC,yFAAyF;AACzF,IAAMM,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEA,SAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OAIAC,MACS,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,WAUN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBAffH,QAAQD,MAAMK,QAAQ,CAACC,MAAM,GAAGN,MAAMO,QAAQ,CAACD,MAAM;oBAE3D,6FAA6F;oBAC7F,qDAAqD;oBAC/CJ,OAAOF,MAAMQ,KAAK,CAACF,MAAM,KAAK,KAAKL,QAAQlB;oBAClC,kCAAA,2BAAA;yBAAXmB,MAAAA;;;;;;;;;;;;oBAAW,YAAcpB;;;2BAAd,6BAAA,QAAA;;;;oBAAMqB,OAAN;oBAAiC;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,wBAA4B,OAALN;;;oBAAxC;;;oBAAjC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;yBAIXL,CAAAA,QAAQQ,MAAM,GAAG,CAAA,GAAjBR;;;;oBACF;;wBAAMD,KAAKa,QAAQ,CACjB,wCACAZ,QAAQa,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEb,CAAAA,KAAKO,MAAM,GAAG,CAAA,GAAdP;;;;oBAAiB;;wBAAMF,KAAKa,QAAQ,CAACrB,oBAAoBU,KAAKY,GAAG,CAACrB;;;oBAAjD;;;oBACN,mCAAA,4BAAA;yBAAXY,MAAAA;;;;;;;;;;;;oBAAW,aAAarB;;;2BAAb,8BAAA,SAAA;;;;oBAAMuB,MAAN;oBAA8B;;wBAAMP,KAAKY,IAAI,CAACL;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACjB;;AAEA,yFAAyF;AACzF,SAAeS,WAAWhB,IAAgB,EAAEiB,KAAe;;YACpD,2BAAA,mBAAA,gBAAA,WAAA,OAAMX;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcW;;;2BAAd,6BAAA,QAAA;;;;oBAAMX,OAAN;oBAAqB;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,sCAAsD,OAAjBM,IAAAA,oBAAU,EAACZ;;;oBAAjE;;;oBAArB;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACP;;AAEO,IAAMnB,eAAiC;IAC5CgC,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQ/B,yBAAyB;YACnC,MAAM,IAAIgC,oBAAU,CAClB,gBACA,AAAC,0BAA4FhC,OAAnE+B,OAAM,+DAAqF,OAAxB/B,yBAAwB;QAEzH;IACF;IACAyB,YAAAA;IACAjB,kBAAAA;IACAyB,gBAAgBC,iCAAuB;AACzC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_stem, summary_stem, text_stem, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, stemFolded(title), stemFolded(summary), stemFolded(text), ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","FTS_REBUILD_THRESHOLD","tursoDialect","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","stemFolded","reconcileContent","conn","touched","docs","delta","churn","bulk","name","ddl","reparsed","length","vanished","files","exec","runBatch","map","p","addColumns","names","quoteIdent","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAIAC;eAAAA;;QAcAC;eAAAA;;QAkCAC;eAAAA;;;wBApEc;yBAGI;wBACqB;6BACxB;6BAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AASpB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,6FAA6F;AAC7F,gGAAgG;AAChG,SAASG,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAIzB,IAAML,wBAAwB;AAErC,yFAAyF;AACzF,IAAMM,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMU,IAAAA,yBAAU,EAACH;QAAQG,IAAAA,yBAAU,EAACF;QAAUE,IAAAA,yBAAU,EAACV;QAAOD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AACtK;AAEA,SAAeW,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OAIAC,MACS,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,WAUN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBAffH,QAAQD,MAAMK,QAAQ,CAACC,MAAM,GAAGN,MAAMO,QAAQ,CAACD,MAAM;oBAE3D,6FAA6F;oBAC7F,qDAAqD;oBAC/CJ,OAAOF,MAAMQ,KAAK,CAACF,MAAM,KAAK,KAAKL,QAAQnB;oBAClC,kCAAA,2BAAA;yBAAXoB,MAAAA;;;;;;;;;;;;oBAAW,YAAcrB;;;2BAAd,6BAAA,QAAA;;;;oBAAMsB,OAAN;oBAAiC;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,wBAA4B,OAALN;;;oBAAxC;;;oBAAjC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;yBAIXL,CAAAA,QAAQQ,MAAM,GAAG,CAAA,GAAjBR;;;;oBACF;;wBAAMD,KAAKa,QAAQ,CACjB,wCACAZ,QAAQa,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEb,CAAAA,KAAKO,MAAM,GAAG,CAAA,GAAdP;;;;oBAAiB;;wBAAMF,KAAKa,QAAQ,CAACtB,oBAAoBW,KAAKY,GAAG,CAACtB;;;oBAAjD;;;oBACN,mCAAA,4BAAA;yBAAXa,MAAAA;;;;;;;;;;;;oBAAW,aAAatB;;;2BAAb,8BAAA,SAAA;;;;oBAAMwB,MAAN;oBAA8B;;wBAAMP,KAAKY,IAAI,CAACL;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACjB;;AAEA,yFAAyF;AACzF,SAAeS,WAAWhB,IAAgB,EAAEiB,KAAe;;YACpD,2BAAA,mBAAA,gBAAA,WAAA,OAAMX;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcW;;;2BAAd,6BAAA,QAAA;;;;oBAAMX,OAAN;oBAAqB;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,sCAAsD,OAAjBM,IAAAA,oBAAU,EAACZ;;;oBAAjE;;;oBAArB;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACP;;AAEO,IAAMpB,eAAiC;IAC5CiC,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQhC,yBAAyB;YACnC,MAAM,IAAIiC,oBAAU,CAClB,gBACA,AAAC,0BAA4FjC,OAAnEgC,OAAM,+DAAqF,OAAxBhC,yBAAwB;QAEzH;IACF;IACA0B,YAAAA;IACAjB,kBAAAA;IACAyB,gBAAgBC,iCAAuB;AACzC"}

package/dist/cjs/store/turso/sql-functions.d.cts

@@ -0,0 +1 @@
+export declare function rewriteFunctions(sql: string): string;

package/dist/cjs/store/turso/sql-functions.d.ts

@@ -0,0 +1 @@
+export declare function rewriteFunctions(sql: string): string;

package/dist/cjs/store/turso/sql-functions.js

@@ -0,0 +1,174 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "rewriteFunctions", {
+    enumerable: true,
+    get: function() {
+        return rewriteFunctions;
+    }
+});
+var _errorsts = require("../../errors.js");
+// Turso can't register UDFs: has()/basename() are rewritten into portable SQL below; segment()
+// (Intl.Segmenter grapheme clustering, types.ts's 'segment' capability) has none, so it's rejected like an unsupported FTS5 operator.
+// Non-global: recursion (readArgs' contents rewritten via rewriteFunctions) reenters this module's
+// regexes before the outer scan resumes, and a global regex's lastIndex does not nest across that.
+var CALL_ANYWHERE = /\b(has|basename|segment)\s*\(/i;
+var CALL_AT_START = /^(has|basename|segment)\s*\(/i;
+function isWordChar(ch) {
+    return ch !== undefined && /[A-Za-z0-9_]/.test(ch);
+}
+// The index just past the quoted literal or SQL comment starting at i, or null if i starts
+// neither. Shared by the scan and readArgs so both leave quoted/comment text opaque to a call match.
+function skipOpaque(sql, i) {
+    var ch = sql[i];
+    if (ch === "'" || ch === '"') {
+        var j = i + 1;
+        while(j < sql.length){
+            if (sql[j] === ch) {
+                if (sql[j + 1] === ch) {
+                    j += 2;
+                    continue;
+                }
+                return j + 1;
+            }
+            j++;
+        }
+        return j; // unterminated literal: consume to end rather than loop forever
+    }
+    if (ch === '-' && sql[i + 1] === '-') {
+        var nl = sql.indexOf('\n', i);
+        return nl === -1 ? sql.length : nl + 1;
+    }
+    if (ch === '/' && sql[i + 1] === '*') {
+        var close = sql.indexOf('*/', i + 2);
+        return close === -1 ? sql.length : close + 2;
+    }
+    return null;
+}
+// `?N`/named placeholders bind by number/name, so repeating one is safe; an anonymous `?` takes
+// the next slot per occurrence, so it's numbered here first -- sqlHas/sqlBasename duplicate their arguments.
+function numberPlaceholders(sql) {
+    var out = '';
+    var i = 0;
+    var n = 0;
+    while(i < sql.length){
+        var _sql_;
+        var skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            out += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        if (sql[i] === '?' && !/[0-9]/.test((_sql_ = sql[i + 1]) !== null && _sql_ !== void 0 ? _sql_ : '')) {
+            n++;
+            out += "?".concat(n);
+            i++;
+            continue;
+        }
+        out += sql[i];
+        i++;
+    }
+    return out;
+}
+// The un-rewritten argument texts of the call whose '(' sits at parenStart, and the index just
+// past its matching ')'. Paren depth is tracked so a nested call's commas/parens aren't mistaken for ours.
+function readArgs(sql, parenStart) {
+    var args = [];
+    var depth = 1;
+    var cur = '';
+    var i = parenStart + 1;
+    while(i < sql.length){
+        var skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            cur += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        var ch = sql[i];
+        if (ch === '(') {
+            depth++;
+            cur += ch;
+            i++;
+            continue;
+        }
+        if (ch === ')') {
+            depth--;
+            i++;
+            if (depth === 0) break;
+            cur += ch;
+            continue;
+        }
+        if (ch === ',' && depth === 1) {
+            args.push(cur.trim());
+            cur = '';
+            i++;
+            continue;
+        }
+        cur += ch;
+        i++;
+    }
+    if (depth !== 0) throw new Error('turso: unterminated argument list starting at "'.concat(sql.slice(parenStart, parenStart + 30), '..."'));
+    if (cur.trim() !== '') args.push(cur.trim());
+    return {
+        args: args,
+        end: i
+    };
+}
+// field.startsWith('[') && JSON.parse(field) is an array, matched by membership; every other
+// field, stringified, matched by substring (sql-functions.ts's hasImpl, mirrored branch for branch).
+function sqlHas(fieldExpr, valueExpr) {
+    var isArray = "substr(".concat(fieldExpr, ",1,1) = '[' AND json_valid(").concat(fieldExpr, ")");
+    // json_each's argument is re-guarded by its own CASE (falling back to '[]'): turso evaluates a
+    // CASE branch's table function even when untaken, and a non-array argument there raises rather than yielding 0 rows (spike-verified).
+    return "(CASE\n    WHEN ".concat(fieldExpr, " IS NULL THEN 0\n    WHEN ").concat(isArray, " THEN\n      (SELECT COUNT(*) FROM json_each(CASE WHEN ").concat(isArray, " THEN ").concat(fieldExpr, " ELSE '[]' END) je WHERE CAST(je.value AS TEXT) = CAST(").concat(valueExpr, " AS TEXT)) > 0\n    ELSE instr(CAST(").concat(fieldExpr, " AS TEXT), CAST(").concat(valueExpr, " AS TEXT)) > 0\n  END)");
+}
+// posix.basename(String(path)), suffix stripped Unix-style (sql-functions.ts's basenameImpl). No
+// last-'/' primitive exists, so reverse() finds the first '/' from the end, then reverse() restores order.
+function sqlBasename(pathExpr, suffixExpr) {
+    var trimmed = "rtrim(CAST(".concat(pathExpr, " AS TEXT), '/')");
+    var name = "CASE\n      WHEN ".concat(trimmed, " = '' THEN ''\n      WHEN instr(").concat(trimmed, ", '/') = 0 THEN ").concat(trimmed, "\n      ELSE reverse(substr(reverse(").concat(trimmed, "), 1, instr(reverse(").concat(trimmed, "), '/') - 1))\n    END");
+    return "(CASE WHEN ".concat(pathExpr, " IS NULL THEN NULL ELSE\n    (CASE WHEN ").concat(suffixExpr, " IS NOT NULL AND CAST(").concat(suffixExpr, " AS TEXT) != '' AND CAST(").concat(suffixExpr, " AS TEXT) != (").concat(name, ")\n          AND substr((").concat(name, "), -length(CAST(").concat(suffixExpr, " AS TEXT))) = CAST(").concat(suffixExpr, " AS TEXT)\n     THEN substr((").concat(name, "), 1, length((").concat(name, ")) - length(CAST(").concat(suffixExpr, " AS TEXT)))\n     ELSE (").concat(name, ") END)\n  END)");
+}
+function rewriteFunctions(sql) {
+    if (!CALL_ANYWHERE.test(sql)) return sql;
+    // Numbered once, over the whole statement, before any argument is duplicated below -- rewrite()
+    // recurses over already-numbered substrings, never renumbering them a second time.
+    return rewrite(numberPlaceholders(sql));
+}
+function rewrite(sql) {
+    var out = '';
+    var i = 0;
+    while(i < sql.length){
+        var skip = skipOpaque(sql, i);
+        if (skip !== null) {
+            out += sql.slice(i, skip);
+            i = skip;
+            continue;
+        }
+        var m = !isWordChar(sql[i - 1]) ? CALL_AT_START.exec(sql.slice(i)) : null;
+        if (m) {
+            var name = m[1].toLowerCase();
+            var parenStart = i + m[0].length - 1;
+            if (name === 'segment') {
+                throw new _errorsts.SenseError('STORE_CAPABILITY_MISSING', 'store "turso" does not implement segment() as a SQL function in this build (its client cannot register UDFs); rephrase the query without it, or set "store" to "sqlite" in this tree\'s config to run it as written');
+            }
+            var _readArgs = readArgs(sql, parenStart), args = _readArgs.args, end = _readArgs.end;
+            var rewrittenArgs = args.map(rewrite);
+            if (name === 'has') {
+                if (rewrittenArgs.length !== 2) throw new Error("turso: has() takes exactly 2 arguments (field, value), got ".concat(rewrittenArgs.length));
+                out += sqlHas(rewrittenArgs[0], rewrittenArgs[1]);
+            } else {
+                var _rewrittenArgs_;
+                if (rewrittenArgs.length < 1 || rewrittenArgs.length > 2) throw new Error("turso: basename() takes 1 or 2 arguments (path, suffix?), got ".concat(rewrittenArgs.length));
+                out += sqlBasename(rewrittenArgs[0], (_rewrittenArgs_ = rewrittenArgs[1]) !== null && _rewrittenArgs_ !== void 0 ? _rewrittenArgs_ : 'NULL');
+            }
+            i = end;
+            continue;
+        }
+        out += sql[i];
+        i++;
+    }
+    return out;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/turso/sql-functions.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/sql-functions.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\n\n// Turso can't register UDFs: has()/basename() are rewritten into portable SQL below; segment()\n// (Intl.Segmenter grapheme clustering, types.ts's 'segment' capability) has none, so it's rejected like an unsupported FTS5 operator.\n\n// Non-global: recursion (readArgs' contents rewritten via rewriteFunctions) reenters this module's\n// regexes before the outer scan resumes, and a global regex's lastIndex does not nest across that.\nconst CALL_ANYWHERE = /\\b(has|basename|segment)\\s*\\(/i;\nconst CALL_AT_START = /^(has|basename|segment)\\s*\\(/i;\n\nfunction isWordChar(ch: string | undefined): boolean {\n  return ch !== undefined && /[A-Za-z0-9_]/.test(ch);\n}\n\n// The index just past the quoted literal or SQL comment starting at i, or null if i starts\n// neither. Shared by the scan and readArgs so both leave quoted/comment text opaque to a call match.\nfunction skipOpaque(sql: string, i: number): number | null {\n  const ch = sql[i];\n  if (ch === \"'\" || ch === '\"') {\n    let j = i + 1;\n    while (j < sql.length) {\n      if (sql[j] === ch) {\n        if (sql[j + 1] === ch) {\n          j += 2;\n          continue;\n        }\n        return j + 1;\n      }\n      j++;\n    }\n    return j; // unterminated literal: consume to end rather than loop forever\n  }\n  if (ch === '-' && sql[i + 1] === '-') {\n    const nl = sql.indexOf('\\n', i);\n    return nl === -1 ? sql.length : nl + 1;\n  }\n  if (ch === '/' && sql[i + 1] === '*') {\n    const close = sql.indexOf('*/', i + 2);\n    return close === -1 ? sql.length : close + 2;\n  }\n  return null;\n}\n\n// `?N`/named placeholders bind by number/name, so repeating one is safe; an anonymous `?` takes\n// the next slot per occurrence, so it's numbered here first -- sqlHas/sqlBasename duplicate their arguments.\nfunction numberPlaceholders(sql: string): string {\n  let out = '';\n  let i = 0;\n  let n = 0;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      out += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    if (sql[i] === '?' && !/[0-9]/.test(sql[i + 1] ?? '')) {\n      n++;\n      out += `?${n}`;\n      i++;\n      continue;\n    }\n    out += sql[i];\n    i++;\n  }\n  return out;\n}\n\n// The un-rewritten argument texts of the call whose '(' sits at parenStart, and the index just\n// past its matching ')'. Paren depth is tracked so a nested call's commas/parens aren't mistaken for ours.\nfunction readArgs(sql: string, parenStart: number): { args: string[]; end: number } {\n  const args: string[] = [];\n  let depth = 1;\n  let cur = '';\n  let i = parenStart + 1;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      cur += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    const ch = sql[i];\n    if (ch === '(') {\n      depth++;\n      cur += ch;\n      i++;\n      continue;\n    }\n    if (ch === ')') {\n      depth--;\n      i++;\n      if (depth === 0) break;\n      cur += ch;\n      continue;\n    }\n    if (ch === ',' && depth === 1) {\n      args.push(cur.trim());\n      cur = '';\n      i++;\n      continue;\n    }\n    cur += ch;\n    i++;\n  }\n  if (depth !== 0) throw new Error(`turso: unterminated argument list starting at \"${sql.slice(parenStart, parenStart + 30)}...\"`);\n  if (cur.trim() !== '') args.push(cur.trim());\n  return { args, end: i };\n}\n\n// field.startsWith('[') && JSON.parse(field) is an array, matched by membership; every other\n// field, stringified, matched by substring (sql-functions.ts's hasImpl, mirrored branch for branch).\nfunction sqlHas(fieldExpr: string, valueExpr: string): string {\n  const isArray = `substr(${fieldExpr},1,1) = '[' AND json_valid(${fieldExpr})`;\n  // json_each's argument is re-guarded by its own CASE (falling back to '[]'): turso evaluates a\n  // CASE branch's table function even when untaken, and a non-array argument there raises rather than yielding 0 rows (spike-verified).\n  return `(CASE\n    WHEN ${fieldExpr} IS NULL THEN 0\n    WHEN ${isArray} THEN\n      (SELECT COUNT(*) FROM json_each(CASE WHEN ${isArray} THEN ${fieldExpr} ELSE '[]' END) je WHERE CAST(je.value AS TEXT) = CAST(${valueExpr} AS TEXT)) > 0\n    ELSE instr(CAST(${fieldExpr} AS TEXT), CAST(${valueExpr} AS TEXT)) > 0\n  END)`;\n}\n\n// posix.basename(String(path)), suffix stripped Unix-style (sql-functions.ts's basenameImpl). No\n// last-'/' primitive exists, so reverse() finds the first '/' from the end, then reverse() restores order.\nfunction sqlBasename(pathExpr: string, suffixExpr: string): string {\n  const trimmed = `rtrim(CAST(${pathExpr} AS TEXT), '/')`;\n  const name = `CASE\n      WHEN ${trimmed} = '' THEN ''\n      WHEN instr(${trimmed}, '/') = 0 THEN ${trimmed}\n      ELSE reverse(substr(reverse(${trimmed}), 1, instr(reverse(${trimmed}), '/') - 1))\n    END`;\n  return `(CASE WHEN ${pathExpr} IS NULL THEN NULL ELSE\n    (CASE WHEN ${suffixExpr} IS NOT NULL AND CAST(${suffixExpr} AS TEXT) != '' AND CAST(${suffixExpr} AS TEXT) != (${name})\n          AND substr((${name}), -length(CAST(${suffixExpr} AS TEXT))) = CAST(${suffixExpr} AS TEXT)\n     THEN substr((${name}), 1, length((${name})) - length(CAST(${suffixExpr} AS TEXT)))\n     ELSE (${name}) END)\n  END)`;\n}\n\n// Rewrites every has()/basename() call into the portable SQL above, recursing into each call's own\n// arguments first so a nested `instr(basename(f.path), 'x')` resolves inside out; segment() instead raises STORE_CAPABILITY_MISSING, named, like an unsupported FTS5 operator.\nexport function rewriteFunctions(sql: string): string {\n  if (!CALL_ANYWHERE.test(sql)) return sql;\n  // Numbered once, over the whole statement, before any argument is duplicated below -- rewrite()\n  // recurses over already-numbered substrings, never renumbering them a second time.\n  return rewrite(numberPlaceholders(sql));\n}\n\nfunction rewrite(sql: string): string {\n  let out = '';\n  let i = 0;\n  while (i < sql.length) {\n    const skip = skipOpaque(sql, i);\n    if (skip !== null) {\n      out += sql.slice(i, skip);\n      i = skip;\n      continue;\n    }\n    const m = !isWordChar(sql[i - 1]) ? CALL_AT_START.exec(sql.slice(i)) : null;\n    if (m) {\n      const name = m[1].toLowerCase();\n      const parenStart = i + m[0].length - 1;\n      if (name === 'segment') {\n        throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement segment() as a SQL function in this build (its client cannot register UDFs); rephrase the query without it, or set \"store\" to \"sqlite\" in this tree's config to run it as written`);\n      }\n      const { args, end } = readArgs(sql, parenStart);\n      const rewrittenArgs = args.map(rewrite);\n      if (name === 'has') {\n        if (rewrittenArgs.length !== 2) throw new Error(`turso: has() takes exactly 2 arguments (field, value), got ${rewrittenArgs.length}`);\n        out += sqlHas(rewrittenArgs[0], rewrittenArgs[1]);\n      } else {\n        if (rewrittenArgs.length < 1 || rewrittenArgs.length > 2) throw new Error(`turso: basename() takes 1 or 2 arguments (path, suffix?), got ${rewrittenArgs.length}`);\n        out += sqlBasename(rewrittenArgs[0], rewrittenArgs[1] ?? 'NULL');\n      }\n      i = end;\n      continue;\n    }\n    out += sql[i];\n    i++;\n  }\n  return out;\n}\n"],"names":["rewriteFunctions","CALL_ANYWHERE","CALL_AT_START","isWordChar","ch","undefined","test","skipOpaque","sql","i","j","length","nl","indexOf","close","numberPlaceholders","out","n","skip","slice","readArgs","parenStart","args","depth","cur","push","trim","Error","end","sqlHas","fieldExpr","valueExpr","isArray","sqlBasename","pathExpr","suffixExpr","trimmed","name","rewrite","m","exec","toLowerCase","SenseError","rewrittenArgs","map"],"mappings":";;;;+BA+IgBA;;;eAAAA;;;wBA/IW;AAE3B,+FAA+F;AAC/F,sIAAsI;AAEtI,mGAAmG;AACnG,mGAAmG;AACnG,IAAMC,gBAAgB;AACtB,IAAMC,gBAAgB;AAEtB,SAASC,WAAWC,EAAsB;IACxC,OAAOA,OAAOC,aAAa,eAAeC,IAAI,CAACF;AACjD;AAEA,2FAA2F;AAC3F,qGAAqG;AACrG,SAASG,WAAWC,GAAW,EAAEC,CAAS;IACxC,IAAML,KAAKI,GAAG,CAACC,EAAE;IACjB,IAAIL,OAAO,OAAOA,OAAO,KAAK;QAC5B,IAAIM,IAAID,IAAI;QACZ,MAAOC,IAAIF,IAAIG,MAAM,CAAE;YACrB,IAAIH,GAAG,CAACE,EAAE,KAAKN,IAAI;gBACjB,IAAII,GAAG,CAACE,IAAI,EAAE,KAAKN,IAAI;oBACrBM,KAAK;oBACL;gBACF;gBACA,OAAOA,IAAI;YACb;YACAA;QACF;QACA,OAAOA,GAAG,gEAAgE;IAC5E;IACA,IAAIN,OAAO,OAAOI,GAAG,CAACC,IAAI,EAAE,KAAK,KAAK;QACpC,IAAMG,KAAKJ,IAAIK,OAAO,CAAC,MAAMJ;QAC7B,OAAOG,OAAO,CAAC,IAAIJ,IAAIG,MAAM,GAAGC,KAAK;IACvC;IACA,IAAIR,OAAO,OAAOI,GAAG,CAACC,IAAI,EAAE,KAAK,KAAK;QACpC,IAAMK,QAAQN,IAAIK,OAAO,CAAC,MAAMJ,IAAI;QACpC,OAAOK,UAAU,CAAC,IAAIN,IAAIG,MAAM,GAAGG,QAAQ;IAC7C;IACA,OAAO;AACT;AAEA,gGAAgG;AAChG,6GAA6G;AAC7G,SAASC,mBAAmBP,GAAW;IACrC,IAAIQ,MAAM;IACV,IAAIP,IAAI;IACR,IAAIQ,IAAI;IACR,MAAOR,IAAID,IAAIG,MAAM,CAAE;YAOeH;QANpC,IAAMU,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBF,OAAOR,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,IAAIV,GAAG,CAACC,EAAE,KAAK,OAAO,CAAC,QAAQH,IAAI,EAACE,QAAAA,GAAG,CAACC,IAAI,EAAE,cAAVD,mBAAAA,QAAc,KAAK;YACrDS;YACAD,OAAO,AAAC,IAAK,OAAFC;YACXR;YACA;QACF;QACAO,OAAOR,GAAG,CAACC,EAAE;QACbA;IACF;IACA,OAAOO;AACT;AAEA,+FAA+F;AAC/F,2GAA2G;AAC3G,SAASI,SAASZ,GAAW,EAAEa,UAAkB;IAC/C,IAAMC,OAAiB,EAAE;IACzB,IAAIC,QAAQ;IACZ,IAAIC,MAAM;IACV,IAAIf,IAAIY,aAAa;IACrB,MAAOZ,IAAID,IAAIG,MAAM,CAAE;QACrB,IAAMO,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBM,OAAOhB,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,IAAMd,KAAKI,GAAG,CAACC,EAAE;QACjB,IAAIL,OAAO,KAAK;YACdmB;YACAC,OAAOpB;YACPK;YACA;QACF;QACA,IAAIL,OAAO,KAAK;YACdmB;YACAd;YACA,IAAIc,UAAU,GAAG;YACjBC,OAAOpB;YACP;QACF;QACA,IAAIA,OAAO,OAAOmB,UAAU,GAAG;YAC7BD,KAAKG,IAAI,CAACD,IAAIE,IAAI;YAClBF,MAAM;YACNf;YACA;QACF;QACAe,OAAOpB;QACPK;IACF;IACA,IAAIc,UAAU,GAAG,MAAM,IAAII,MAAM,AAAC,kDAAwF,OAAvCnB,IAAIW,KAAK,CAACE,YAAYA,aAAa,KAAI;IAC1H,IAAIG,IAAIE,IAAI,OAAO,IAAIJ,KAAKG,IAAI,CAACD,IAAIE,IAAI;IACzC,OAAO;QAAEJ,MAAAA;QAAMM,KAAKnB;IAAE;AACxB;AAEA,6FAA6F;AAC7F,qGAAqG;AACrG,SAASoB,OAAOC,SAAiB,EAAEC,SAAiB;IAClD,IAAMC,UAAU,AAAC,UAAgDF,OAAvCA,WAAU,+BAAuC,OAAVA,WAAU;IAC3E,+FAA+F;IAC/F,sIAAsI;IACtI,OAAO,AAAC,mBAECE,OADAF,WAAU,8BAE6BE,OADvCA,SAAQ,2DAC+CF,OAAhBE,SAAQ,UAA2ED,OAAnED,WAAU,2DACtDA,OAD+GC,WAAU,wCAC7FA,OAA5BD,WAAU,oBAA4B,OAAVC,WAAU;AAE5D;AAEA,iGAAiG;AACjG,2GAA2G;AAC3G,SAASE,YAAYC,QAAgB,EAAEC,UAAkB;IACvD,IAAMC,UAAU,AAAC,cAAsB,OAATF,UAAS;IACvC,IAAMG,OAAO,AAAC,oBAEGD,OADNA,SAAQ,oCACwBA,OAA1BA,SAAQ,oBACSA,OADSA,SAAQ,wCACaA,OAA9BA,SAAQ,wBAA8B,OAARA,SAAQ;IAExE,OAAO,AAAC,cACOD,OADMD,UAAS,4CACoBC,OAAnCA,YAAW,0BAA8DA,OAAtCA,YAAW,6BAAsDE,OAA3BF,YAAW,kBAC7EE,OAD6FA,MAAK,6BAC3EF,OAAvBE,MAAK,oBAAkDF,OAAhCA,YAAW,uBACtCE,OAD2DF,YAAW,iCACjDE,OAArBA,MAAK,kBAAwCF,OAAxBE,MAAK,qBACjCA,OADoDF,YAAW,4BAC1D,OAALE,MAAK;AAElB;AAIO,SAASrC,iBAAiBQ,GAAW;IAC1C,IAAI,CAACP,cAAcK,IAAI,CAACE,MAAM,OAAOA;IACrC,gGAAgG;IAChG,mFAAmF;IACnF,OAAO8B,QAAQvB,mBAAmBP;AACpC;AAEA,SAAS8B,QAAQ9B,GAAW;IAC1B,IAAIQ,MAAM;IACV,IAAIP,IAAI;IACR,MAAOA,IAAID,IAAIG,MAAM,CAAE;QACrB,IAAMO,OAAOX,WAAWC,KAAKC;QAC7B,IAAIS,SAAS,MAAM;YACjBF,OAAOR,IAAIW,KAAK,CAACV,GAAGS;YACpBT,IAAIS;YACJ;QACF;QACA,IAAMqB,IAAI,CAACpC,WAAWK,GAAG,CAACC,IAAI,EAAE,IAAIP,cAAcsC,IAAI,CAAChC,IAAIW,KAAK,CAACV,MAAM;QACvE,IAAI8B,GAAG;YACL,IAAMF,OAAOE,CAAC,CAAC,EAAE,CAACE,WAAW;YAC7B,IAAMpB,aAAaZ,IAAI8B,CAAC,CAAC,EAAE,CAAC5B,MAAM,GAAG;YACrC,IAAI0B,SAAS,WAAW;gBACtB,MAAM,IAAIK,oBAAU,CAAC,4BAA4B;YACnD;YACA,IAAsBtB,YAAAA,SAASZ,KAAKa,aAA5BC,OAAcF,UAAdE,MAAMM,MAAQR,UAARQ;YACd,IAAMe,gBAAgBrB,KAAKsB,GAAG,CAACN;YAC/B,IAAID,SAAS,OAAO;gBAClB,IAAIM,cAAchC,MAAM,KAAK,GAAG,MAAM,IAAIgB,MAAM,AAAC,8DAAkF,OAArBgB,cAAchC,MAAM;gBAClIK,OAAOa,OAAOc,aAAa,CAAC,EAAE,EAAEA,aAAa,CAAC,EAAE;YAClD,OAAO;oBAEgCA;gBADrC,IAAIA,cAAchC,MAAM,GAAG,KAAKgC,cAAchC,MAAM,GAAG,GAAG,MAAM,IAAIgB,MAAM,AAAC,iEAAqF,OAArBgB,cAAchC,MAAM;gBAC/JK,OAAOiB,YAAYU,aAAa,CAAC,EAAE,GAAEA,kBAAAA,aAAa,CAAC,EAAE,cAAhBA,6BAAAA,kBAAoB;YAC3D;YACAlC,IAAImB;YACJ;QACF;QACAZ,OAAOR,GAAG,CAACC,EAAE;QACbA;IACF;IACA,OAAOO;AACT"}

package/dist/cjs/store/turso/store.js

@@ -23,6 +23,7 @@ var _vectorsts = require("../vectors.js");
 var _connectionts = require("./connection.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
+var _sqlfunctionsts = require("./sql-functions.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -363,8 +364,6 @@ function _wrap_async_generator(fn) {
     };
 }
 var CAPABILITIES = new Set([
-    'lexical',
-    'phrases',
     'vectors'
 ]);
 // close() reclaims once the cache file outgrows its compact size by this factor: the disk
@@ -515,7 +514,7 @@ function createStore(db, conn) {
                             case 0:
                                 return [
                                     4,
-                                    db.prepare(sql)
+                                    db.prepare((0, _sqlfunctionsts.rewriteFunctions)(sql))
                                 ];
                             case 1:
                                 stmt = _state.sent();

package/dist/cjs/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns, getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// close() reclaims once the cache file outgrows its compact size by this factor: the disk\n// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.\nconst BLOAT_FACTOR = 1.5;\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.\n      await checkpointWal(db);\n      let reclaimPath: string | null = null;\n      try {\n        const path = await cacheFilePath(conn);\n        const base = Number((await getMeta(conn, 'compact_size')) ?? '0');\n        const size = path === null ? null : await fileSize(path);\n        if (size !== null) {\n          // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a\n          // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.\n          if (base === 0) {\n            await setMeta(conn, 'compact_size', String(size));\n            // The meta write landed past the checkpoint; leave no WAL for the next opener.\n            await checkpointWal(db);\n          } else if (size > BLOAT_FACTOR * base) {\n            reclaimPath = path;\n          }\n        }\n      } catch (err) {\n        console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${(err as Error).message}`);\n      }\n      await db.close();\n      // After this store's connection is closed, never before: turso hands the file to one\n      // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).\n      if (reclaimPath !== null) await reclaimSpace(reclaimPath);\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","BLOAT_FACTOR","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close","reclaimPath","base","size","err","checkpointWal","cacheFilePath","getMeta","Number","fileSize","setMeta","String","console","error","message","reclaimSpace"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAQGC;eAAAA;;;uBApBW;wBACkB;6BACb;yBAEU;4BAC2B;4BAC1C;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAE9F,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,eAAe;AAId,SAASF,YAAYG,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcP;QACRQ,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACD;;;gCAAxBwC,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;oBAGAC,aAGmB,MADfd,MACAe,MACAC,YAYCC;;;;4BAlBT,4FAA4F;4BAC5F;;gCAAMC,IAAAA,2BAAa,EAACtD;;;4BAApB;4BACIkD,cAA6B;;;;;;;;;4BAElB;;gCAAMK,IAAAA,2BAAa,EAACtD;;;4BAA3BmC,OAAO;4BACQ;;gCAAMoB,IAAAA,iBAAO,EAACvD,MAAM;;;4BAAnCkD,OAAOM;iCAAQ,OAAA,2BAAA,kBAAA,OAAwC;;iCAChDrB,CAAAA,SAAS,IAAG,GAAZA;;;;mCAAgB;;;;;;4BAAO;;gCAAMsB,IAAAA,sBAAQ,EAACtB;;;mCAAf;;;4BAA9BgB;iCACFA,CAAAA,SAAS,IAAG,GAAZA;;;;iCAGED,CAAAA,SAAS,CAAA,GAATA;;;;4BACF;;gCAAMQ,IAAAA,iBAAO,EAAC1D,MAAM,gBAAgB2D,OAAOR;;;4BAA3C;4BACA,+EAA+E;4BAC/E;;gCAAME,IAAAA,2BAAa,EAACtD;;;4BAApB;;;;;;4BACK,IAAIoD,OAAOrD,eAAeoD,MAAM;gCACrCD,cAAcd;4BAChB;;;;;;;;4BAEKiB;4BACPQ,QAAQC,KAAK,CAAC,AAAC,4FAAkH,OAAvB,AAACT,IAAcU,OAAO;;;;;;4BAElI;;gCAAM/D,GAAGiD,KAAK;;;4BAAd;iCAGIC,CAAAA,gBAAgB,IAAG,GAAnBA;;;;4BAAsB;;gCAAMc,IAAAA,0BAAY,EAACd;;;4BAAnB;;;;;;;;YAC5B;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns, getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'segment': has()/basename() resolve via SQL rewrite below (sql-functions.ts), but segment() has no SQL form and the client cannot register it as a UDF.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors']);\n\n// close() reclaims once the cache file outgrows its compact size by this factor: the disk\n// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.\nconst BLOAT_FACTOR = 1.5;\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(rewriteFunctions(sql));\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.\n      await checkpointWal(db);\n      let reclaimPath: string | null = null;\n      try {\n        const path = await cacheFilePath(conn);\n        const base = Number((await getMeta(conn, 'compact_size')) ?? '0');\n        const size = path === null ? null : await fileSize(path);\n        if (size !== null) {\n          // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a\n          // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.\n          if (base === 0) {\n            await setMeta(conn, 'compact_size', String(size));\n            // The meta write landed past the checkpoint; leave no WAL for the next opener.\n            await checkpointWal(db);\n          } else if (size > BLOAT_FACTOR * base) {\n            reclaimPath = path;\n          }\n        }\n      } catch (err) {\n        console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${(err as Error).message}`);\n      }\n      await db.close();\n      // After this store's connection is closed, never before: turso hands the file to one\n      // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).\n      if (reclaimPath !== null) await reclaimSpace(reclaimPath);\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","BLOAT_FACTOR","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","rewriteFunctions","safeIntegers","iterate","params","close","reclaimPath","base","size","err","checkpointWal","cacheFilePath","getMeta","Number","fileSize","setMeta","String","console","error","message","reclaimSpace"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAQGC;eAAAA;;;uBApBW;wBACkB;6BACb;yBAEU;4BAC2B;4BAC1C;yBACE;8BACI;0BAC6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;CAAU;AAExE,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,eAAe;AAId,SAASF,YAAYG,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcP;QACRQ,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACwC,IAAAA,gCAAgB,EAACzC;;;gCAAzCwC,OAAO;gCACbA,KAAKE,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLlC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DmC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCJ;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKG,OAAO,OAAZH,OAAa,qBAAGI;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;oBAGAC,aAGmB,MADff,MACAgB,MACAC,YAYCC;;;;4BAlBT,4FAA4F;4BAC5F;;gCAAMC,IAAAA,2BAAa,EAACvD;;;4BAApB;4BACImD,cAA6B;;;;;;;;;4BAElB;;gCAAMK,IAAAA,2BAAa,EAACvD;;;4BAA3BmC,OAAO;4BACQ;;gCAAMqB,IAAAA,iBAAO,EAACxD,MAAM;;;4BAAnCmD,OAAOM;iCAAQ,OAAA,2BAAA,kBAAA,OAAwC;;iCAChDtB,CAAAA,SAAS,IAAG,GAAZA;;;;mCAAgB;;;;;;4BAAO;;gCAAMuB,IAAAA,sBAAQ,EAACvB;;;mCAAf;;;4BAA9BiB;iCACFA,CAAAA,SAAS,IAAG,GAAZA;;;;iCAGED,CAAAA,SAAS,CAAA,GAATA;;;;4BACF;;gCAAMQ,IAAAA,iBAAO,EAAC3D,MAAM,gBAAgB4D,OAAOR;;;4BAA3C;4BACA,+EAA+E;4BAC/E;;gCAAME,IAAAA,2BAAa,EAACvD;;;4BAApB;;;;;;4BACK,IAAIqD,OAAOtD,eAAeqD,MAAM;gCACrCD,cAAcf;4BAChB;;;;;;;;4BAEKkB;4BACPQ,QAAQC,KAAK,CAAC,AAAC,4FAAkH,OAAvB,AAACT,IAAcU,OAAO;;;;;;4BAElI;;gCAAMhE,GAAGkD,KAAK;;;4BAAd;iCAGIC,CAAAA,gBAAgB,IAAG,GAAnBA;;;;4BAAsB;;gCAAMc,IAAAA,0BAAY,EAACd;;;4BAAnB;;;;;;;;YAC5B;;IACF;AACF"}