sensemaking 0.21.1 → 0.22.1

package/dist/cjs/store/stages.js

@@ -0,0 +1,372 @@
+// Per-stage wall time for one reconcile, all on this thread's clock. Stages name work a build has
+// to do, never the function or file doing it, so the vocabulary outlives a pipeline refactor.
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get FEATURE_HOOKS () {
+        return FEATURE_HOOKS;
+    },
+    get FIXED_STAGES () {
+        return FIXED_STAGES;
+    },
+    get STAGES_VERSION () {
+        return STAGES_VERSION;
+    },
+    get featureStage () {
+        return featureStage;
+    },
+    get stageRecorder () {
+        return stageRecorder;
+    },
+    get unaccountedMs () {
+        return unaccountedMs;
+    }
+});
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var STAGES_VERSION = 's1';
+var FIXED_STAGES = [
+    'list',
+    'existing',
+    'parse',
+    'alter',
+    'added-recheck',
+    'fm-upsert',
+    'text-index',
+    'presets',
+    'vanished',
+    'meta'
+];
+var FEATURE_HOOKS = [
+    'remove',
+    'store',
+    'after'
+];
+function featureStage(name, hook) {
+    return "feature:".concat(name, ":").concat(hook);
+}
+var round = function round(ms) {
+    return Math.round(ms * 10) / 10;
+};
+function stageRecorder() {
+    var featureNames = arguments.length > 0 && arguments[0] !== void 0 ? arguments[0] : [];
+    var spans = new Map();
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = FIXED_STAGES[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var stage = _step.value;
+            spans.set(stage, 0);
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined, _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
+    try {
+        for(var _iterator1 = featureNames[Symbol.iterator](), _step1; !(_iteratorNormalCompletion2 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion2 = true){
+            var name = _step1.value;
+            try {
+                for(var _iterator2 = FEATURE_HOOKS[Symbol.iterator](), _step2; !(_iteratorNormalCompletion1 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion1 = true){
+                    var hook = _step2.value;
+                    spans.set(featureStage(name, hook), 0);
+                }
+            } catch (err) {
+                _didIteratorError1 = true;
+                _iteratorError1 = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion1 && _iterator2.return != null) {
+                        _iterator2.return();
+                    }
+                } finally{
+                    if (_didIteratorError1) {
+                        throw _iteratorError1;
+                    }
+                }
+            }
+        }
+    } catch (err) {
+        _didIteratorError2 = true;
+        _iteratorError2 = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion2 && _iterator1.return != null) {
+                _iterator1.return();
+            }
+        } finally{
+            if (_didIteratorError2) {
+                throw _iteratorError2;
+            }
+        }
+    }
+    return {
+        time: // A throw records nothing: an aborted reconcile must not report a partial stage as a complete one.
+        function time(stage, fn) {
+            return _async_to_generator(function() {
+                var _spans_get, at, value;
+                return _ts_generator(this, function(_state) {
+                    switch(_state.label){
+                        case 0:
+                            at = process.hrtime.bigint();
+                            return [
+                                4,
+                                fn()
+                            ];
+                        case 1:
+                            value = _state.sent();
+                            spans.set(stage, ((_spans_get = spans.get(stage)) !== null && _spans_get !== void 0 ? _spans_get : 0) + Number(process.hrtime.bigint() - at) / 1e6);
+                            return [
+                                2,
+                                value
+                            ];
+                    }
+                });
+            })();
+        },
+        // Every declared stage reports, 0 included, so a store that skips some work and a store that
+        // does it cheaply read alike across a timeline. An absent `stages` is a build with no timers.
+        take: function take(totalMs, txMs) {
+            var parseWorkerMs = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 0;
+            var out = {};
+            var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+            try {
+                for(var _iterator = spans[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                    var _step_value = _sliced_to_array(_step.value, 2), stage = _step_value[0], ms = _step_value[1];
+                    out[stage] = round(ms);
+                }
+            } catch (err) {
+                _didIteratorError = true;
+                _iteratorError = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion && _iterator.return != null) {
+                        _iterator.return();
+                    }
+                } finally{
+                    if (_didIteratorError) {
+                        throw _iteratorError;
+                    }
+                }
+            }
+            return {
+                version: STAGES_VERSION,
+                totalMs: round(totalMs),
+                txMs: round(txMs),
+                parseWorkerMs: round(parseWorkerMs),
+                spans: out
+            };
+        }
+    };
+}
+function unaccountedMs(stages) {
+    var sum = 0;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = Object.values(stages.spans)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var ms = _step.value;
+            sum += ms;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return round(stages.totalMs - sum);
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/stages.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/stages.ts"],"sourcesContent":["// Per-stage wall time for one reconcile, all on this thread's clock. Stages name work a build has\n// to do, never the function or file doing it, so the vocabulary outlives a pipeline refactor.\nexport const STAGES_VERSION = 's1';\n\n// Every stage outside the feature hooks, in execution order. Feature hooks add\n// `feature:<name>:<hook>` keys, so a new feature becomes visible without touching this list.\nexport const FIXED_STAGES = ['list', 'existing', 'parse', 'alter', 'added-recheck', 'fm-upsert', 'text-index', 'presets', 'vanished', 'meta'] as const;\n\nexport const FEATURE_HOOKS = ['remove', 'store', 'after'] as const;\n\nexport type FeatureHook = (typeof FEATURE_HOOKS)[number];\n\nexport interface Stages {\n  version: string;\n  totalMs: number;\n  // The write transaction's own duration. It contains most of the spans below, so it is reported\n  // beside them rather than among them, and never counts toward the sum.\n  txMs: number;\n  // Summed worker-side parseFile time inside `parse`: CPU across workers, not a wall span, so it\n  // sits beside the spans like txMs and never counts toward the sum. 0 on the serial path.\n  parseWorkerMs: number;\n  // Disjoint spans, so `totalMs` less their sum is time no stage claims.\n  spans: Record<string, number>;\n}\n\nexport interface StageRecorder {\n  time<T>(stage: string, fn: () => T | Promise<T>): Promise<T>;\n  take(totalMs: number, txMs: number, parseWorkerMs?: number): Stages;\n}\n\nexport function featureStage(name: string, hook: FeatureHook): string {\n  return `feature:${name}:${hook}`;\n}\n\nconst round = (ms: number): number => Math.round(ms * 10) / 10;\n\n// `featureNames` seeds this build's feature-hook keys, so the key set is a function of config\n// alone: a cold build (nothing to remove) and an incremental one report the same stages.\nexport function stageRecorder(featureNames: readonly string[] = []): StageRecorder {\n  const spans = new Map<string, number>();\n  for (const stage of FIXED_STAGES) spans.set(stage, 0);\n  for (const name of featureNames) for (const hook of FEATURE_HOOKS) spans.set(featureStage(name, hook), 0);\n  return {\n    // A throw records nothing: an aborted reconcile must not report a partial stage as a complete one.\n    async time(stage, fn) {\n      const at = process.hrtime.bigint();\n      const value = await fn();\n      spans.set(stage, (spans.get(stage) ?? 0) + Number(process.hrtime.bigint() - at) / 1e6);\n      return value;\n    },\n    // Every declared stage reports, 0 included, so a store that skips some work and a store that\n    // does it cheaply read alike across a timeline. An absent `stages` is a build with no timers.\n    take(totalMs, txMs, parseWorkerMs = 0) {\n      const out: Record<string, number> = {};\n      for (const [stage, ms] of spans) out[stage] = round(ms);\n      return { version: STAGES_VERSION, totalMs: round(totalMs), txMs: round(txMs), parseWorkerMs: round(parseWorkerMs), spans: out };\n    },\n  };\n}\n\n// Time no stage claims: transaction commit, lock wait, and any cost added since without a timer.\n// A residual that grows across releases is the signal that this vocabulary stopped covering the build.\nexport function unaccountedMs(stages: Stages): number {\n  let sum = 0;\n  for (const ms of Object.values(stages.spans)) sum += ms;\n  return round(stages.totalMs - sum);\n}\n"],"names":["FEATURE_HOOKS","FIXED_STAGES","STAGES_VERSION","featureStage","stageRecorder","unaccountedMs","name","hook","round","ms","Math","featureNames","spans","Map","stage","set","time","fn","at","value","process","hrtime","bigint","get","Number","take","totalMs","txMs","parseWorkerMs","out","version","stages","sum","Object","values"],"mappings":"AAAA,kGAAkG;AAClG,8FAA8F;;;;;;;;;;;;QAOjFA;eAAAA;;QAFAC;eAAAA;;QAJAC;eAAAA;;QA4BGC;eAAAA;;QAQAC;eAAAA;;QAwBAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA5DT,IAAMH,iBAAiB;AAIvB,IAAMD,eAAe;IAAC;IAAQ;IAAY;IAAS;IAAS;IAAiB;IAAa;IAAc;IAAW;IAAY;CAAO;AAEtI,IAAMD,gBAAgB;IAAC;IAAU;IAAS;CAAQ;AAsBlD,SAASG,aAAaG,IAAY,EAAEC,IAAiB;IAC1D,OAAO,AAAC,WAAkBA,OAARD,MAAK,KAAQ,OAALC;AAC5B;AAEA,IAAMC,QAAQ,eAACC;WAAuBC,KAAKF,KAAK,CAACC,KAAK,MAAM;;AAIrD,SAASL;QAAcO,eAAAA,iEAAkC,EAAE;IAChE,IAAMC,QAAQ,IAAIC;QACb,kCAAA,2BAAA;;QAAL,QAAK,YAAeZ,iCAAf,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMa,QAAN;YAA6BF,MAAMG,GAAG,CAACD,OAAO;;;QAA9C;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;QACiC,mCAAA,4BAAA,6BAAjC,mCAAA,4BAAA;;QAAL,QAAK,aAAcH,iCAAd,UAAA,8BAAA,SAAA,0BAAA;YAAA,IAAML,OAAN;;gBAA4B,QAAK,aAAcN,kCAAd,UAAA,8BAAA,SAAA,0BAAA;oBAAA,IAAMO,OAAN;oBAA6BK,MAAMG,GAAG,CAACZ,aAAaG,MAAMC,OAAO;;;gBAAjE;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;;;QAAjC;QAAA;;;iBAAA,8BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAO;QAECS,MADN,mGAAmG;QACnG,SAAMA,KAAKF,KAAK,EAAEG,EAAE;;oBAGAL,YAFZM,IACAC;;;;4BADAD,KAAKE,QAAQC,MAAM,CAACC,MAAM;4BAClB;;gCAAML;;;4BAAdE,QAAQ;4BACdP,MAAMG,GAAG,CAACD,OAAO,EAACF,aAAAA,MAAMW,GAAG,CAACT,oBAAVF,wBAAAA,aAAoB,KAAKY,OAAOJ,QAAQC,MAAM,CAACC,MAAM,KAAKJ,MAAM;4BAClF;;gCAAOC;;;;YACT;;QACA,6FAA6F;QAC7F,8FAA8F;QAC9FM,MAAAA,SAAAA,KAAKC,OAAO,EAAEC,IAAI;gBAAEC,gBAAAA,iEAAgB;YAClC,IAAMC,MAA8B,CAAC;gBAChC,kCAAA,2BAAA;;gBAAL,QAAK,YAAqBjB,0BAArB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,mCAAA,iBAAOE,wBAAOL;oBAAcoB,GAAG,CAACf,MAAM,GAAGN,MAAMC;;;gBAA/C;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YACL,OAAO;gBAAEqB,SAAS5B;gBAAgBwB,SAASlB,MAAMkB;gBAAUC,MAAMnB,MAAMmB;gBAAOC,eAAepB,MAAMoB;gBAAgBhB,OAAOiB;YAAI;QAChI;IACF;AACF;AAIO,SAASxB,cAAc0B,MAAc;IAC1C,IAAIC,MAAM;QACL,kCAAA,2BAAA;;QAAL,QAAK,YAAYC,OAAOC,MAAM,CAACH,OAAOnB,KAAK,sBAAtC,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMH,KAAN;YAAyCuB,OAAOvB;;;QAAhD;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOD,MAAMuB,OAAOL,OAAO,GAAGM;AAChC"}

package/dist/cjs/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAIGC;eAAAA;;;uBAfW;wBACA;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACD;;;gCAAxBwC,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;;;4BACJ;;gCAAMjD,GAAGiD,KAAK;;;4BAAd;;;;;;YACF;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAIGC;eAAAA;;;uBAfW;wBACA;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAc,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACvB;;YAC3BwB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,MAAM2B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,MAAM6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACpC,MAAM2B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACtC,MAAMmC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAMzC,GAAGM,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCoC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;;;gCAAO;;oCAAM7C,GAAGM,OAAO,CAACD;;;gCAAxBwC,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;;;4BACJ;;gCAAMjD,GAAGiD,KAAK;;;4BAAd;;;;;;YACF;;IACF;AACF"}

package/dist/cjs/store/types.d.cts

@@ -1,7 +1,7 @@
 import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
-export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "watch-concurrency", "lexical", "vectors", "sql-functions"];
+export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "lexical", "vectors", "sql-functions"];
 export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;
@@ -21,6 +21,7 @@ export interface Connection {
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
+    appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface ReconcileDialect {
     beginMode(): string;
@@ -28,7 +29,6 @@ export interface ReconcileDialect {
     addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
-    insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface OpenDialect<Handle> {
     filename: string;

package/dist/cjs/store/types.d.ts

@@ -1,7 +1,7 @@
 import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
-export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "watch-concurrency", "lexical", "vectors", "sql-functions"];
+export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "lexical", "vectors", "sql-functions"];
 export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;
@@ -21,6 +21,7 @@ export interface Connection {
     exec(sql: string): Promise<void>;
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
+    appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface ReconcileDialect {
     beginMode(): string;
@@ -28,7 +29,6 @@ export interface ReconcileDialect {
     addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
-    insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;
 }
 export interface OpenDialect<Handle> {
     filename: string;

package/dist/cjs/store/types.js

@@ -13,7 +13,6 @@ Object.defineProperty(exports, "CAPABILITY_NAMES", {
 var CAPABILITY_NAMES = [
     'phrases',
     'snippets',
-    'watch-concurrency',
     'lexical',
     'vectors',
     'sql-functions'

package/dist/cjs/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n  // Inserts rows whose path cannot already exist in `table` (no ON CONFLICT needed), through a\n  // faster append-only path. Optional: sqlite/turso omit it and every row goes through the upsert.\n  insertNew?(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB"}

package/dist/cjs/text/segment.js

@@ -35,10 +35,9 @@ function hasUnspacedRun(text) {
     return HAS_RUN.test(text);
 }
 // Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.
-// Built once (construction cost amortizes) and used for both index and query splitting.
-var GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, {
-    granularity: 'grapheme'
-});
+// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run
+// never segments at all, so no command pays it at module load.
+var graphemeSegmenter;
 // unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions
 // into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.
 var PUNCTUATION = RegExp("[\\u0021-\\u0023\\u0025-\\u002A\\u002C-\\u002F\\u003A-\\u003B\\u003F-\\u0040\\u005B-\\u005D\\u005F\\u007B\\u007D\\u00A1\\u00A7\\u00AB\\u00B6-\\u00B7\\u00BB\\u00BF\\u037E\\u0387\\u055A-\\u055F\\u0589-\\u058A\\u05BE\\u05C0\\u05C3\\u05C6\\u05F3-\\u05F4\\u0609-\\u060A\\u060C-\\u060D\\u061B\\u061D-\\u061F\\u066A-\\u066D\\u06D4\\u0700-\\u070D\\u07F7-\\u07F9\\u0830-\\u083E\\u085E\\u0964-\\u0965\\u0970\\u09FD\\u0A76\\u0AF0\\u0C77\\u0C84\\u0DF4\\u0E4F\\u0E5A-\\u0E5B\\u0F04-\\u0F12\\u0F14\\u0F3A-\\u0F3D\\u0F85\\u0FD0-\\u0FD4\\u0FD9-\\u0FDA\\u104A-\\u104F\\u10FB\\u1360-\\u1368\\u1400\\u166E\\u169B-\\u169C\\u16EB-\\u16ED\\u1735-\\u1736\\u17D4-\\u17D6\\u17D8-\\u17DA\\u1800-\\u180A\\u1944-\\u1945\\u1A1E-\\u1A1F\\u1AA0-\\u1AA6\\u1AA8-\\u1AAD\\u1B4E-\\u1B4F\\u1B5A-\\u1B60\\u1B7D-\\u1B7F\\u1BFC-\\u1BFF\\u1C3B-\\u1C3F\\u1C7E-\\u1C7F\\u1CC0-\\u1CC7\\u1CD3\\u2010-\\u2027\\u2030-\\u2043\\u2045-\\u2051\\u2053-\\u205E\\u207D-\\u207E\\u208D-\\u208E\\u2308-\\u230B\\u2329-\\u232A\\u2768-\\u2775\\u27C5-\\u27C6\\u27E6-\\u27EF\\u2983-\\u2998\\u29D8-\\u29DB\\u29FC-\\u29FD\\u2CF9-\\u2CFC\\u2CFE-\\u2CFF\\u2D70\\u2E00-\\u2E2E\\u2E30-\\u2E4F\\u2E52-\\u2E5D\\u3001-\\u3003\\u3008-\\u3011\\u3014-\\u301F\\u3030\\u303D\\u30A0\\u30FB\\uA4FE-\\uA4FF\\uA60D-\\uA60F\\uA673\\uA67E\\uA6F2-\\uA6F7\\uA874-\\uA877\\uA8CE-\\uA8CF\\uA8F8-\\uA8FA\\uA8FC\\uA92E-\\uA92F\\uA95F\\uA9C1-\\uA9CD\\uA9DE-\\uA9DF\\uAA5C-\\uAA5F\\uAADE-\\uAADF\\uAAF0-\\uAAF1\\uABEB\\uFD3E-\\uFD3F\\uFE10-\\uFE19\\uFE30-\\uFE52\\uFE54-\\uFE61\\uFE63\\uFE68\\uFE6A-\\uFE6B\\uFF01-\\uFF03\\uFF05-\\uFF0A\\uFF0C-\\uFF0F\\uFF1A-\\uFF1B\\uFF1F-\\uFF20\\uFF3B-\\uFF3D\\uFF3F\\uFF5B\\uFF5D\\uFF5F-\\uFF65\\u{10100}-\\u{10102}\\u{1039F}\\u{103D0}\\u{1056F}\\u{10857}\\u{1091F}\\u{1093F}\\u{10A50}-\\u{10A58}\\u{10A7F}\\u{10AF0}-\\u{10AF6}\\u{10B39}-\\u{10B3F}\\u{10B99}-\\u{10B9C}\\u{10D6E}\\u{10EAD}\\u{10ED0}\\u{10F55}-\\u{10F59}\\u{10F86}-\\u{10F89}\\u{11047}-\\u{1104D}\\u{110BB}-\\u{110BC}\\u{110BE}-\\u{110C1}\\u{11140}-\\u{11143}\\u{11174}-\\u{11175}\\u{111C5}-\\u{111C8}\\u{111CD}\\u{111DB}\\u{111DD}-\\u{111DF}\\u{11238}-\\u{1123D}\\u{112A9}\\u{113D4}-\\u{113D5}\\u{113D7}-\\u{113D8}\\u{1144B}-\\u{1144F}\\u{1145A}-\\u{1145B}\\u{1145D}\\u{114C6}\\u{115C1}-\\u{115D7}\\u{11641}-\\u{11643}\\u{11660}-\\u{1166C}\\u{116B9}\\u{1173C}-\\u{1173E}\\u{1183B}\\u{11944}-\\u{11946}\\u{119E2}\\u{11A3F}-\\u{11A46}\\u{11A9A}-\\u{11A9C}\\u{11A9E}-\\u{11AA2}\\u{11B00}-\\u{11B09}\\u{11BE1}\\u{11C41}-\\u{11C45}\\u{11C70}-\\u{11C71}\\u{11EF7}-\\u{11EF8}\\u{11F43}-\\u{11F4F}\\u{11FFF}\\u{12470}-\\u{12474}\\u{12FF1}-\\u{12FF2}\\u{16A6E}-\\u{16A6F}\\u{16AF5}\\u{16B37}-\\u{16B3B}\\u{16B44}\\u{16D6D}-\\u{16D6F}\\u{16E97}-\\u{16E9A}\\u{16FE2}\\u{1BC9F}\\u{1DA87}-\\u{1DA8B}\\u{1E5FF}\\u{1E95E}-\\u{1E95F}]", "u");
@@ -46,7 +45,10 @@ var PUNCTUATION = RegExp("[\\u0021-\\u0023\\u0025-\\u002A\\u002C-\\u002F\\u003A-
 // are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.
 var BARRIER = 'ꟷ';
 function graphemes(run) {
-    return Array.from(GRAPHEME_SEGMENTER.segment(run), function(s) {
+    graphemeSegmenter !== null && graphemeSegmenter !== void 0 ? graphemeSegmenter : graphemeSegmenter = new Intl.Segmenter(undefined, {
+        granularity: 'grapheme'
+    });
+    return Array.from(graphemeSegmenter.segment(run), function(s) {
         return s.segment;
     });
 }

package/dist/cjs/text/segment.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built once (construction cost amortizes) and used for both index and query splitting.\nconst GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  return Array.from(GRAPHEME_SEGMENTER.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","hasUnspacedRun","segmentField","segmentMatch","RUN","RegExp","HAS_RUN","text","test","GRAPHEME_SEGMENTER","Intl","Segmenter","undefined","granularity","PUNCTUATION","BARRIER","graphemes","run","Array","from","segment","s","splitOnPunctuation","groups","g","push","length","filter","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;;;;;;;;;;;;QACrEA;eAAAA;;QAQGC;eAAAA;;QA8BAC;eAAAA;;QA6BAC;eAAAA;;;AAnET,IAAMH,mBAAmB;AAChC,2FAA2F;AAC3F,gDAAgD;AAChD,IAAMI,MAAM,IAAIC,OAAO,AAAC,QAAwB,OAAjBL,kBAAiB,gBAAc;AAC9D,IAAMM,UAAU,IAAID,OAAO,AAAC,IAAoB,OAAjBL,kBAAiB,MAAI;AAI7C,SAASC,eAAeM,IAAY;IACzC,OAAOD,QAAQE,IAAI,CAACD;AACtB;AACA,6FAA6F;AAC7F,wFAAwF;AACxF,IAAME,qBAAqB,IAAIC,KAAKC,SAAS,CAACC,WAAW;IAAEC,aAAa;AAAW;AACnF,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,IAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5B,OAAOC,MAAMC,IAAI,CAACV,mBAAmBW,OAAO,CAACH,MAAM,SAACI;eAAMA,EAAED,OAAO;;AACrE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBL,GAAW;IACrC,IAAMM,SAAqB;QAAC,EAAE;KAAC;QAC1B,kCAAA,2BAAA;;QAAL,QAAK,YAAWP,UAAUC,yBAArB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMO,IAAN;YACH,IAAIV,YAAYN,IAAI,CAACgB,IAAID,OAAOE,IAAI,CAAC,EAAE;iBAClCF,MAAM,CAACA,OAAOG,MAAM,GAAG,EAAE,CAACD,IAAI,CAACD;QACtC;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,OAAOI,MAAM,CAAC,SAACH;eAAMA,EAAEE,MAAM,GAAG;;AACzC;AAIO,SAASxB,aAAaK,IAAY;IACvC,IAAI,CAACD,QAAQE,IAAI,CAACD,OAAO,OAAO;IAChC,IAAMqB,MAAMrB,KAAKsB,OAAO,CAACzB,KAAK,SAACa;QAC7B,IAAMa,OAAOR,mBAAmBL,KAC7Bc,GAAG,CAAC,SAACP;mBAAMA,EAAEQ,IAAI,CAAC;WAClBA,IAAI,CAAC,AAAC,IAAW,OAARjB,SAAQ;QACpB,OAAO,AAAC,IAAce,OAAXf,SAAQ,KAAWA,OAARe,MAAK,KAAW,OAARf,SAAQ;IACxC;IACA,OAAOa,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,IAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAASnB,GAAW,EAAEoB,YAAoB;IACjD,OAAOf,mBAAmBL,KACvBc,GAAG,CAAC,SAACP;eAAM,AAAC,GAAiBA,OAAfa,cAAwD,OAAzCb,EAAEE,MAAM,GAAG,IAAI,AAAC,IAAe,OAAZF,EAAEQ,IAAI,CAAC,MAAK,OAAKR,CAAC,CAAC,EAAE;OACrEQ,IAAI,CAAC;AACV;AAIO,SAAS7B,aAAamC,KAAa;IACxC,IAAI,CAAChC,QAAQE,IAAI,CAAC8B,QAAQ,OAAOA;IACjC,IAAIV,MAAM;IACV,IAAIW,SAAS;IACb,IAAMC,SAASF,MAAMG,KAAK,CAACrC,MAAM,2CAA2C;IAC5E,IAAK,IAAIsC,IAAI,GAAGA,IAAIF,OAAOd,MAAM,EAAEgB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;gBACV,kCAAA,2BAAA;;gBAAL,QAAK,YAAYF,MAAM,CAACE,EAAE,qBAArB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,KAAN;oBAAuB,IAAIA,OAAO,KAAKJ,SAAS,CAACA;;;gBAAjD;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YACLX,OAAOY,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVX,OAAOY,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,IAAME,IAAIhB,IAAIiB,KAAK,CAACX;QACpB,IAAIU,GAAG;YACLhB,MAAM,AAAC,GAA0BgB,OAAxBhB,IAAIkB,KAAK,CAAC,GAAGF,EAAEG,KAAK,GAAWX,OAAPQ,CAAC,CAAC,EAAE,EAA8C,OAA3CR,SAASI,MAAM,CAACE,EAAE,EAAE,AAAC,GAASE,OAAPA,CAAC,CAAC,EAAE,EAAQ,OAALA,CAAC,CAAC,EAAE,EAAC;QAC7E,OAAO;YACLhB,OAAOQ,SAASI,MAAM,CAACE,EAAE,EAAEP;QAC7B;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","hasUnspacedRun","segmentField","segmentMatch","RUN","RegExp","HAS_RUN","text","test","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","run","Intl","Segmenter","undefined","granularity","Array","from","segment","s","splitOnPunctuation","groups","g","push","length","filter","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;;;;;;;;;;;;QACrEA;eAAAA;;QAQGC;eAAAA;;QAgCAC;eAAAA;;QA6BAC;eAAAA;;;AArET,IAAMH,mBAAmB;AAChC,2FAA2F;AAC3F,gDAAgD;AAChD,IAAMI,MAAM,IAAIC,OAAO,AAAC,QAAwB,OAAjBL,kBAAiB,gBAAc;AAC9D,IAAMM,UAAU,IAAID,OAAO,AAAC,IAAoB,OAAjBL,kBAAiB,MAAI;AAI7C,SAASC,eAAeM,IAAY;IACzC,OAAOD,QAAQE,IAAI,CAACD;AACtB;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAIE;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,IAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5BJ,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAIK,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOC,MAAMC,IAAI,CAACV,kBAAkBW,OAAO,CAACP,MAAM,SAACQ;eAAMA,EAAED,OAAO;;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBT,GAAW;IACrC,IAAMU,SAAqB;QAAC,EAAE;KAAC;QAC1B,kCAAA,2BAAA;;QAAL,QAAK,YAAWX,UAAUC,yBAArB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMW,IAAN;YACH,IAAId,YAAYF,IAAI,CAACgB,IAAID,OAAOE,IAAI,CAAC,EAAE;iBAClCF,MAAM,CAACA,OAAOG,MAAM,GAAG,EAAE,CAACD,IAAI,CAACD;QACtC;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,OAAOI,MAAM,CAAC,SAACH;eAAMA,EAAEE,MAAM,GAAG;;AACzC;AAIO,SAASxB,aAAaK,IAAY;IACvC,IAAI,CAACD,QAAQE,IAAI,CAACD,OAAO,OAAO;IAChC,IAAMqB,MAAMrB,KAAKsB,OAAO,CAACzB,KAAK,SAACS;QAC7B,IAAMiB,OAAOR,mBAAmBT,KAC7BkB,GAAG,CAAC,SAACP;mBAAMA,EAAEQ,IAAI,CAAC;WAClBA,IAAI,CAAC,AAAC,IAAW,OAARrB,SAAQ;QACpB,OAAO,AAAC,IAAcmB,OAAXnB,SAAQ,KAAWA,OAARmB,MAAK,KAAW,OAARnB,SAAQ;IACxC;IACA,OAAOiB,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,IAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAASvB,GAAW,EAAEwB,YAAoB;IACjD,OAAOf,mBAAmBT,KACvBkB,GAAG,CAAC,SAACP;eAAM,AAAC,GAAiBA,OAAfa,cAAwD,OAAzCb,EAAEE,MAAM,GAAG,IAAI,AAAC,IAAe,OAAZF,EAAEQ,IAAI,CAAC,MAAK,OAAKR,CAAC,CAAC,EAAE;OACrEQ,IAAI,CAAC;AACV;AAIO,SAAS7B,aAAamC,KAAa;IACxC,IAAI,CAAChC,QAAQE,IAAI,CAAC8B,QAAQ,OAAOA;IACjC,IAAIV,MAAM;IACV,IAAIW,SAAS;IACb,IAAMC,SAASF,MAAMG,KAAK,CAACrC,MAAM,2CAA2C;IAC5E,IAAK,IAAIsC,IAAI,GAAGA,IAAIF,OAAOd,MAAM,EAAEgB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;gBACV,kCAAA,2BAAA;;gBAAL,QAAK,YAAYF,MAAM,CAACE,EAAE,qBAArB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,KAAN;oBAAuB,IAAIA,OAAO,KAAKJ,SAAS,CAACA;;;gBAAjD;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YACLX,OAAOY,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVX,OAAOY,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,IAAME,IAAIhB,IAAIiB,KAAK,CAACX;QACpB,IAAIU,GAAG;YACLhB,MAAM,AAAC,GAA0BgB,OAAxBhB,IAAIkB,KAAK,CAAC,GAAGF,EAAEG,KAAK,GAAWX,OAAPQ,CAAC,CAAC,EAAE,EAA8C,OAA3CR,SAASI,MAAM,CAACE,EAAE,EAAE,AAAC,GAASE,OAAPA,CAAC,CAAC,EAAE,EAAQ,OAALA,CAAC,CAAC,EAAE,EAAC;QAC7E,OAAO;YACLhB,OAAOQ,SAASI,MAAM,CAACE,EAAE,EAAEP;QAC7B;IACF;IACA,OAAOP;AACT"}