sensemaking 0.17.1 → 0.18.0

package/dist/cjs/store/sqlite/lexical.js

@@ -0,0 +1,177 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "queryLexical", {
+    enumerable: true,
+    get: function() {
+        return queryLexical;
+    }
+});
+var _segmentts = require("../../text/segment.js");
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc
+// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, hit stays NULL for
+// the caller's JS excerpt fallback.
+var SNIPPET_BOUND = 16384;
+var WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';
+function queryLexical(conn, terms, opts, segmenting) {
+    return _async_to_generator(function() {
+        var whereJoin, whereCond, scopeCond, limit, query, sql, stmt;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    whereJoin = opts.whereJoin, whereCond = opts.whereCond, scopeCond = opts.scopeCond, limit = opts.limit;
+                    query = segmenting ? (0, _segmentts.segmentMatch)(terms) : terms;
+                    // A run of pure unspaced-script punctuation segments to nothing searchable (segmentField
+                    // drops empty groups the same way): an empty MATCH string is an FTS5 syntax error, not zero
+                    // rows, so this returns the zero rows directly rather than issuing it.
+                    if (query.trim() === '') return [
+                        2,
+                        []
+                    ];
+                    sql = "SELECT content.path AS path, CASE WHEN length(content.text) <= ".concat(SNIPPET_BOUND, " THEN snippet(content, 2, '\xab', '\xbb', '…', 10) ELSE NULL END AS hit FROM content ").concat(whereJoin, " WHERE content MATCH ? ").concat(whereCond, " ").concat(scopeCond, " ORDER BY ").concat(WEIGHTED_BM25, " LIMIT ").concat(limit);
+                    return [
+                        4,
+                        conn.prepare(sql)
+                    ];
+                case 1:
+                    stmt = _state.sent();
+                    return [
+                        4,
+                        stmt.all(query)
+                    ];
+                case 2:
+                    return [
+                        2,
+                        _state.sent()
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/sqlite/lexical.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\n// snippet() re-tokenizes each candidate doc, superlinearly: ~10s for one 1MB doc\n// (benchmark/reports/2026-08-23-hub-release-battery.md). Past this bound, hit stays NULL for\n// the caller's JS excerpt fallback.\nconst SNIPPET_BOUND = 16_384;\nconst WEIGHTED_BM25 = 'bm25(content, 10.0, 5.0, 1.0, 0, 10.0, 5.0, 1.0)';\n\n// Ranked BM25 word-match query with excerpt, scoped by the caller-built SQL fragments.\n// `segmenting` mirrors the predicate that decided whether the `_seg` sidecars were populated\n// (contentTokenize(cfg) === undefined, see sqlite/store.ts): an unspaced-script run in `terms`\n// becomes a quoted grapheme phrase against those sidecars only when they exist.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, segmenting: boolean): Promise<LexicalHit[]> {\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const query = segmenting ? segmentMatch(terms) : terms;\n  // A run of pure unspaced-script punctuation segments to nothing searchable (segmentField\n  // drops empty groups the same way): an empty MATCH string is an FTS5 syntax error, not zero\n  // rows, so this returns the zero rows directly rather than issuing it.\n  if (query.trim() === '') return [];\n  const sql = `SELECT content.path AS path, CASE WHEN length(content.text) <= ${SNIPPET_BOUND} THEN snippet(content, 2, '«', '»', '…', 10) ELSE NULL END AS hit FROM content ${whereJoin} WHERE content MATCH ? ${whereCond} ${scopeCond} ORDER BY ${WEIGHTED_BM25} LIMIT ${limit}`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["queryLexical","SNIPPET_BOUND","WEIGHTED_BM25","conn","terms","opts","segmenting","whereJoin","whereCond","scopeCond","limit","query","sql","stmt","segmentMatch","trim","prepare","all"],"mappings":";;;;+BAasBA;;;eAAAA;;;yBAbO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG7B,iFAAiF;AACjF,6FAA6F;AAC7F,oCAAoC;AACpC,IAAMC,gBAAgB;AACtB,IAAMC,gBAAgB;AAMf,SAAeF,aAAaG,IAAgB,EAAEC,KAAa,EAAEC,IAAyB,EAAEC,UAAmB;;YACxGC,WAAWC,WAAWC,WAAWC,OACnCC,OAKAC,KACAC;;;;oBAPEN,YAA2CF,KAA3CE,WAAWC,YAAgCH,KAAhCG,WAAWC,YAAqBJ,KAArBI,WAAWC,QAAUL,KAAVK;oBACnCC,QAAQL,aAAaQ,IAAAA,uBAAY,EAACV,SAASA;oBACjD,yFAAyF;oBACzF,4FAA4F;oBAC5F,uEAAuE;oBACvE,IAAIO,MAAMI,IAAI,OAAO,IAAI;;;;oBACnBH,MAAM,AAAC,kEAAgKL,OAA/FN,eAAc,yFAAoHO,OAAnCD,WAAU,2BAAsCE,OAAbD,WAAU,KAAyBN,OAAtBO,WAAU,cAAmCC,OAAvBR,eAAc,WAAe,OAANQ;oBAC7P;;wBAAMP,KAAKa,OAAO,CAACJ;;;oBAA1BC,OAAO;oBACL;;wBAAMA,KAAKI,GAAG,CAACN;;;oBAAvB;;wBAAQ;;;;IACV"}

package/dist/cjs/{db → store/sqlite}/open.d.cts

@@ -1,14 +1,14 @@
-import { DatabaseSync } from 'node:sqlite';
-import type { ResolvedConfig } from '../config/index.js';
+import type { ResolvedConfig } from '../../config/index.js';
+import type { Store } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
 export declare const SCHEMA_VERSION = "18";
 export interface OpenResult {
-    db: DatabaseSync;
+    store: Store;
     cfg: ResolvedConfig;
     dbPath: string;
     parsed: number;
     warnings: string[];
 }
-export declare function docCount(db: DatabaseSync): number;
-export declare function open(cfg: ResolvedConfig): OpenResult;
+export declare function openSqlite(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function docCount(store: Store): Promise<number>;
 export declare function clearCache(cfg: ResolvedConfig): void;

package/dist/cjs/{db → store/sqlite}/open.d.ts

@@ -1,14 +1,14 @@
-import { DatabaseSync } from 'node:sqlite';
-import type { ResolvedConfig } from '../config/index.js';
+import type { ResolvedConfig } from '../../config/index.js';
+import type { Store } from '../types.js';
 export declare const DB_FILENAME = "cache.db";
 export declare const SCHEMA_VERSION = "18";
 export interface OpenResult {
-    db: DatabaseSync;
+    store: Store;
     cfg: ResolvedConfig;
     dbPath: string;
     parsed: number;
     warnings: string[];
 }
-export declare function docCount(db: DatabaseSync): number;
-export declare function open(cfg: ResolvedConfig): OpenResult;
+export declare function openSqlite(cfg: ResolvedConfig): Promise<OpenResult>;
+export declare function docCount(store: Store): Promise<number>;
 export declare function clearCache(cfg: ResolvedConfig): void;