sensemaking 0.27.1 → 0.28.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/cli.js +16 -7
- package/dist/cjs/cli/constants.d.cts +2 -0
- package/dist/cjs/cli/constants.d.ts +2 -0
- package/dist/cjs/cli/constants.js +21 -0
- package/dist/cjs/cli/constants.js.map +1 -0
- package/dist/cjs/cli/named.js +2 -1
- package/dist/cjs/cli/named.js.map +1 -1
- package/dist/cjs/cli/shared.js +5 -4
- package/dist/cjs/cli/shared.js.map +1 -1
- package/dist/cjs/cli.js +10 -9
- package/dist/cjs/cli.js.map +1 -1
- package/dist/cjs/embed/query.js.map +1 -1
- package/dist/cjs/store/duckdb/engine.d.cts +11 -0
- package/dist/cjs/store/duckdb/engine.d.ts +11 -0
- package/dist/cjs/store/duckdb/engine.js +193 -0
- package/dist/cjs/store/duckdb/engine.js.map +1 -0
- package/dist/cjs/store/duckdb/lexical.js +34 -20
- package/dist/cjs/store/duckdb/lexical.js.map +1 -1
- package/dist/cjs/store/duckdb/native.d.cts +5 -2
- package/dist/cjs/store/duckdb/native.d.ts +5 -2
- package/dist/cjs/store/duckdb/native.js +157 -13
- package/dist/cjs/store/duckdb/native.js.map +1 -1
- package/dist/cjs/store/duckdb/open.js +151 -3
- package/dist/cjs/store/duckdb/open.js.map +1 -1
- package/dist/cjs/store/duckdb/ordered-bm25.d.cts +3 -3
- package/dist/cjs/store/duckdb/ordered-bm25.d.ts +3 -3
- package/dist/cjs/store/duckdb/ordered-bm25.js +4 -8
- package/dist/cjs/store/duckdb/ordered-bm25.js.map +1 -1
- package/dist/cjs/store/engine.d.cts +28 -0
- package/dist/cjs/store/engine.d.ts +28 -0
- package/dist/cjs/store/engine.js +362 -0
- package/dist/cjs/store/engine.js.map +1 -0
- package/dist/cjs/store/native.d.cts +8 -5
- package/dist/cjs/store/native.d.ts +8 -5
- package/dist/cjs/store/native.js +78 -73
- package/dist/cjs/store/native.js.map +1 -1
- package/dist/cjs/store/open.js +100 -40
- package/dist/cjs/store/open.js.map +1 -1
- package/dist/cjs/store/sqlite/engine.d.cts +12 -0
- package/dist/cjs/store/sqlite/engine.d.ts +12 -0
- package/dist/cjs/store/sqlite/engine.js +53 -0
- package/dist/cjs/store/sqlite/engine.js.map +1 -0
- package/dist/cjs/store/sqlite/lexical.js +3 -4
- package/dist/cjs/store/sqlite/lexical.js.map +1 -1
- package/dist/cjs/store/sqlite/open.js +119 -6
- package/dist/cjs/store/sqlite/open.js.map +1 -1
- package/dist/cjs/store/turso/engine.d.cts +9 -0
- package/dist/cjs/store/turso/engine.d.ts +9 -0
- package/dist/cjs/store/turso/engine.js +189 -0
- package/dist/cjs/store/turso/engine.js.map +1 -0
- package/dist/cjs/store/turso/lexical.js +16 -11
- package/dist/cjs/store/turso/lexical.js.map +1 -1
- package/dist/cjs/store/turso/native.d.cts +5 -1
- package/dist/cjs/store/turso/native.d.ts +5 -1
- package/dist/cjs/store/turso/native.js +157 -8
- package/dist/cjs/store/turso/native.js.map +1 -1
- package/dist/cjs/store/turso/open.js +172 -27
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/reconcile.d.cts +0 -1
- package/dist/cjs/store/turso/reconcile.d.ts +0 -1
- package/dist/cjs/store/turso/reconcile.js +22 -23
- package/dist/cjs/store/turso/reconcile.js.map +1 -1
- package/dist/cjs/store/types.d.cts +2 -0
- package/dist/cjs/store/types.d.ts +2 -0
- package/dist/cjs/store/types.js.map +1 -1
- package/dist/esm/cli/constants.d.ts +2 -0
- package/dist/esm/cli/constants.js +2 -0
- package/dist/esm/cli/constants.js.map +1 -0
- package/dist/esm/cli/named.js +2 -1
- package/dist/esm/cli/named.js.map +1 -1
- package/dist/esm/cli/shared.js +6 -5
- package/dist/esm/cli/shared.js.map +1 -1
- package/dist/esm/cli.js +10 -9
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/embed/query.js +1 -1
- package/dist/esm/embed/query.js.map +1 -1
- package/dist/esm/store/duckdb/engine.d.ts +11 -0
- package/dist/esm/store/duckdb/engine.js +18 -0
- package/dist/esm/store/duckdb/engine.js.map +1 -0
- package/dist/esm/store/duckdb/lexical.js +11 -11
- package/dist/esm/store/duckdb/lexical.js.map +1 -1
- package/dist/esm/store/duckdb/native.d.ts +5 -2
- package/dist/esm/store/duckdb/native.js +16 -12
- package/dist/esm/store/duckdb/native.js.map +1 -1
- package/dist/esm/store/duckdb/open.js +40 -2
- package/dist/esm/store/duckdb/open.js.map +1 -1
- package/dist/esm/store/duckdb/ordered-bm25.d.ts +3 -3
- package/dist/esm/store/duckdb/ordered-bm25.js +4 -9
- package/dist/esm/store/duckdb/ordered-bm25.js.map +1 -1
- package/dist/esm/store/engine.d.ts +28 -0
- package/dist/esm/store/engine.js +85 -0
- package/dist/esm/store/engine.js.map +1 -0
- package/dist/esm/store/native.d.ts +8 -5
- package/dist/esm/store/native.js +59 -57
- package/dist/esm/store/native.js.map +1 -1
- package/dist/esm/store/open.js +24 -7
- package/dist/esm/store/open.js.map +1 -1
- package/dist/esm/store/sqlite/engine.d.ts +12 -0
- package/dist/esm/store/sqlite/engine.js +16 -0
- package/dist/esm/store/sqlite/engine.js.map +1 -0
- package/dist/esm/store/sqlite/lexical.js +2 -3
- package/dist/esm/store/sqlite/lexical.js.map +1 -1
- package/dist/esm/store/sqlite/open.js +38 -7
- package/dist/esm/store/sqlite/open.js.map +1 -1
- package/dist/esm/store/turso/engine.d.ts +9 -0
- package/dist/esm/store/turso/engine.js +14 -0
- package/dist/esm/store/turso/engine.js.map +1 -0
- package/dist/esm/store/turso/lexical.js +9 -9
- package/dist/esm/store/turso/lexical.js.map +1 -1
- package/dist/esm/store/turso/native.d.ts +5 -1
- package/dist/esm/store/turso/native.js +16 -10
- package/dist/esm/store/turso/native.js.map +1 -1
- package/dist/esm/store/turso/open.js +44 -3
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/reconcile.d.ts +0 -1
- package/dist/esm/store/turso/reconcile.js +3 -8
- package/dist/esm/store/turso/reconcile.js.map +1 -1
- package/dist/esm/store/types.d.ts +2 -0
- package/dist/esm/store/types.js.map +1 -1
- package/package.json +21 -7
- package/skills/sense/EXAMPLES.md +0 -99
- package/skills/sense/SKILL.md +0 -145
- package/skills/sense/references/search.md +0 -63
- package/skills/sense/references/sql.md +0 -120
- package/skills/sense/references/stores/duckdb.md +0 -35
- package/skills/sense/references/stores/sqlite.md +0 -66
- package/skills/sense/references/stores/turso.md +0 -36
- package/skills/sense/references/terminology.md +0 -32
- package/skills/sense-bases/EXAMPLES.md +0 -103
- package/skills/sense-bases/SKILL.md +0 -147
- package/skills/sense-setup/EXAMPLES.md +0 -129
- package/skills/sense-setup/SKILL.md +0 -112
- package/skills/sense-setup/references/embeddings.md +0 -51
- package/skills/sense-setup/references/store-benchmarks.md +0 -16
- package/skills/sense-setup/references/stores/duckdb.md +0 -28
- package/skills/sense-setup/references/stores/sqlite.md +0 -21
- package/skills/sense-setup/references/stores/turso.md +0 -25
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/sqlite/lexical.ts"],"sourcesContent":["import { segmentMatch } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { sqliteFts } from './engine.ts';\n\n// Ranked BM25 word-match query, scoped by the caller-built SQL fragments. An\n// unspaced-script run in `terms` becomes a quoted grapheme phrase against the `_seg` sidecars.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n const { whereJoin, whereCond, scopeCond, limit } = opts;\n const query = segmentMatch(terms);\n // A run of pure unspaced-script punctuation segments to nothing searchable: an empty MATCH\n // string is an FTS5 syntax error, not zero rows, so return zero rows directly rather than issuing it.\n if (query.trim() === '') return [];\n const stmt = await conn.prepare(sqliteFts().rankedMatch(whereJoin, whereCond, scopeCond, limit));\n return (await stmt.all(query)) as unknown as LexicalHit[];\n}\n"],"names":["queryLexical","conn","terms","opts","whereJoin","whereCond","scopeCond","limit","query","stmt","segmentMatch","trim","prepare","sqliteFts","rankedMatch","all"],"mappings":";;;;+BAMsBA;;;eAAAA;;;yBANO;wBAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAInB,SAAeA,aAAaC,IAAgB,EAAEC,KAAa,EAAEC,IAAyB;;YACnFC,WAAWC,WAAWC,WAAWC,OACnCC,OAIAC;;;;oBALEL,YAA2CD,KAA3CC,WAAWC,YAAgCF,KAAhCE,WAAWC,YAAqBH,KAArBG,WAAWC,QAAUJ,KAAVI;oBACnCC,QAAQE,IAAAA,uBAAY,EAACR;oBAC3B,2FAA2F;oBAC3F,sGAAsG;oBACtG,IAAIM,MAAMG,IAAI,OAAO,IAAI;;;;oBACZ;;wBAAMV,KAAKW,OAAO,CAACC,IAAAA,mBAAS,IAAGC,WAAW,CAACV,WAAWC,WAAWC,WAAWC;;;oBAAnFE,OAAO;oBACL;;wBAAMA,KAAKM,GAAG,CAACP;;;oBAAvB;;wBAAQ;;;;IACV"}
|
|
@@ -26,10 +26,13 @@ _export(exports, {
|
|
|
26
26
|
});
|
|
27
27
|
var _nodesqlite = require("node:sqlite");
|
|
28
28
|
var _indexts = require("../../features/index.js");
|
|
29
|
+
var _enginets = require("../engine.js");
|
|
29
30
|
var _opents = require("../open.js");
|
|
30
31
|
var _sharedts = require("../shared.js");
|
|
31
32
|
var _transactionts = require("../transaction.js");
|
|
32
33
|
var _connectionts = require("./connection.js");
|
|
34
|
+
var _enginets1 = require("./engine.js");
|
|
35
|
+
var _lexicalts = require("./lexical.js");
|
|
33
36
|
var _reconcilets = require("./reconcile.js");
|
|
34
37
|
var _sqlfunctionsts = require("./sql-functions.js");
|
|
35
38
|
var _storets = require("./store.js");
|
|
@@ -160,11 +163,6 @@ function _ts_generator(thisArg, body) {
|
|
|
160
163
|
}
|
|
161
164
|
var DB_FILENAME = 'cache.db';
|
|
162
165
|
var SCHEMA_VERSION = '22';
|
|
163
|
-
// unicode61 splits on spaces, so a language written without them indexes a whole run as one
|
|
164
|
-
// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.
|
|
165
|
-
var TOKENIZE = 'porter unicode61';
|
|
166
|
-
// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)
|
|
167
|
-
// are documented against the first three columns. Each holds its field's exploded unspaced runs.
|
|
168
166
|
function createContentTable(conn) {
|
|
169
167
|
return _async_to_generator(function() {
|
|
170
168
|
return _ts_generator(this, function(_state) {
|
|
@@ -172,7 +170,7 @@ function createContentTable(conn) {
|
|
|
172
170
|
case 0:
|
|
173
171
|
return [
|
|
174
172
|
4,
|
|
175
|
-
conn.exec(
|
|
173
|
+
conn.exec((0, _enginets1.sqliteFts)().createContent())
|
|
176
174
|
];
|
|
177
175
|
case 1:
|
|
178
176
|
_state.sent();
|
|
@@ -408,12 +406,127 @@ function setDerivedBusyTimeout(handle, _conn, ms) {
|
|
|
408
406
|
});
|
|
409
407
|
})();
|
|
410
408
|
}
|
|
409
|
+
// The real schema, content write and lexical query over store/engine.ts's fixture, in memory.
|
|
410
|
+
function checkSqliteEngine() {
|
|
411
|
+
return _async_to_generator(function() {
|
|
412
|
+
var db, conn, docs, remedy;
|
|
413
|
+
return _ts_generator(this, function(_state) {
|
|
414
|
+
switch(_state.label){
|
|
415
|
+
case 0:
|
|
416
|
+
db = new _nodesqlite.DatabaseSync(':memory:');
|
|
417
|
+
_state.label = 1;
|
|
418
|
+
case 1:
|
|
419
|
+
_state.trys.push([
|
|
420
|
+
1,
|
|
421
|
+
,
|
|
422
|
+
7,
|
|
423
|
+
8
|
|
424
|
+
]);
|
|
425
|
+
(0, _sqlfunctionsts.registerFunctions)(db);
|
|
426
|
+
conn = (0, _connectionts.createConnection)(db);
|
|
427
|
+
return [
|
|
428
|
+
4,
|
|
429
|
+
conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)')
|
|
430
|
+
];
|
|
431
|
+
case 2:
|
|
432
|
+
_state.sent();
|
|
433
|
+
return [
|
|
434
|
+
4,
|
|
435
|
+
ensureCoreSchemaTables(conn)
|
|
436
|
+
];
|
|
437
|
+
case 3:
|
|
438
|
+
_state.sent();
|
|
439
|
+
return [
|
|
440
|
+
4,
|
|
441
|
+
conn.runBatch('INSERT INTO frontmatter ("path") VALUES (?)', _enginets.ENGINE_CHECK_DOCS.map(function(doc) {
|
|
442
|
+
return [
|
|
443
|
+
doc.path
|
|
444
|
+
];
|
|
445
|
+
}))
|
|
446
|
+
];
|
|
447
|
+
case 4:
|
|
448
|
+
_state.sent();
|
|
449
|
+
docs = _enginets.ENGINE_CHECK_DOCS.map(function(doc) {
|
|
450
|
+
return {
|
|
451
|
+
relPath: doc.path,
|
|
452
|
+
search: {
|
|
453
|
+
title: doc.title,
|
|
454
|
+
summary: doc.summary,
|
|
455
|
+
text: doc.text
|
|
456
|
+
}
|
|
457
|
+
};
|
|
458
|
+
});
|
|
459
|
+
return [
|
|
460
|
+
4,
|
|
461
|
+
conn.runBatch(_reconcilets.INSERT_CONTENT_SQL, docs.map(_reconcilets.contentRow))
|
|
462
|
+
];
|
|
463
|
+
case 5:
|
|
464
|
+
_state.sent();
|
|
465
|
+
remedy = "this SQLite is bundled with Node ".concat(process.version, " (SQLite ").concat(_enginets1.SQLITE_VERSION !== null && _enginets1.SQLITE_VERSION !== void 0 ? _enginets1.SQLITE_VERSION : 'unknown', "); run sense on a Node release whose bundled SQLite passes");
|
|
466
|
+
return [
|
|
467
|
+
4,
|
|
468
|
+
(0, _enginets.checkEngine)('sqlite', (0, _enginets1.sqliteIdentity)(), remedy, function(terms) {
|
|
469
|
+
return _async_to_generator(function() {
|
|
470
|
+
return _ts_generator(this, function(_state) {
|
|
471
|
+
switch(_state.label){
|
|
472
|
+
case 0:
|
|
473
|
+
return [
|
|
474
|
+
4,
|
|
475
|
+
(0, _lexicalts.queryLexical)(conn, terms, {
|
|
476
|
+
whereJoin: '',
|
|
477
|
+
whereCond: '',
|
|
478
|
+
scopeCond: '',
|
|
479
|
+
limit: 10
|
|
480
|
+
})
|
|
481
|
+
];
|
|
482
|
+
case 1:
|
|
483
|
+
return [
|
|
484
|
+
2,
|
|
485
|
+
_state.sent().map(function(hit) {
|
|
486
|
+
return hit.path;
|
|
487
|
+
})
|
|
488
|
+
];
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
})();
|
|
492
|
+
})
|
|
493
|
+
];
|
|
494
|
+
case 6:
|
|
495
|
+
_state.sent();
|
|
496
|
+
return [
|
|
497
|
+
3,
|
|
498
|
+
8
|
|
499
|
+
];
|
|
500
|
+
case 7:
|
|
501
|
+
db.close();
|
|
502
|
+
return [
|
|
503
|
+
7
|
|
504
|
+
];
|
|
505
|
+
case 8:
|
|
506
|
+
return [
|
|
507
|
+
2
|
|
508
|
+
];
|
|
509
|
+
}
|
|
510
|
+
});
|
|
511
|
+
})();
|
|
512
|
+
}
|
|
411
513
|
var sqliteOpenDialect = {
|
|
412
514
|
filename: DB_FILENAME,
|
|
413
515
|
schemaVersion: SCHEMA_VERSION,
|
|
414
516
|
reconcileDialect: _reconcilets.sqliteDialect,
|
|
415
517
|
connect: connect,
|
|
416
518
|
close: close,
|
|
519
|
+
engineIdentity: function engineIdentity() {
|
|
520
|
+
return _async_to_generator(function() {
|
|
521
|
+
return _ts_generator(this, function(_state) {
|
|
522
|
+
return [
|
|
523
|
+
2,
|
|
524
|
+
(0, _enginets1.sqliteIdentity)()
|
|
525
|
+
];
|
|
526
|
+
});
|
|
527
|
+
})();
|
|
528
|
+
},
|
|
529
|
+
checkEngine: checkSqliteEngine,
|
|
417
530
|
ensureCoreSchema: ensureCoreSchema,
|
|
418
531
|
ensureFeatureSchema: ensureFeatureSchema,
|
|
419
532
|
setDerivedBusyTimeout: setDerivedBusyTimeout,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '22';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Feature schema is published later with its generation.\nasync function ensureCoreSchemaTables(conn: Connection): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await createContentTable(conn);\n await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n const deadline = Date.now() + 5000;\n for (;;) {\n try {\n db.exec('PRAGMA journal_mode = WAL');\n return;\n } catch (err) {\n if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n }\n }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: SqliteHandle; conn: Connection }> {\n const db = new DatabaseSync(dbPath, options.existingOnly ? { readOnly: true } : {});\n try {\n // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n db.exec('PRAGMA busy_timeout = 30000');\n // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n // handler for it, so a concurrent cold open needs its own bounded wait.\n if (!options.observational) setJournalWal(db);\n registerFunctions(db);\n const conn = createConnection(db);\n return { handle: { db }, conn };\n } catch (err) {\n // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n // scratch cleanup fails with EPERM/EBUSY.\n db.close();\n throw err;\n }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n handle.db.close();\n}\n\nasync function ensureCoreSchema(_handle: SqliteHandle, conn: Connection): Promise<void> {\n // One writer at a time: two cold opens can bootstrap the same database concurrently.\n await withTransaction(conn, () => ensureCoreSchemaTables(conn), BEGIN_WRITE);\n}\n\nasync function ensureFeatureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n filename: DB_FILENAME,\n schemaVersion: SCHEMA_VERSION,\n reconcileDialect: sqliteDialect,\n connect,\n close,\n ensureCoreSchema,\n ensureFeatureSchema,\n setDerivedBusyTimeout,\n createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n return openWithDialect(cfg, sqliteOpenDialect, options);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openSqlite","sqliteOpenDialect","TOKENIZE","createContentTable","conn","exec","ensureCoreSchemaTables","getMeta","setMeta","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","options","DatabaseSync","existingOnly","readOnly","observational","registerFunctions","createConnection","handle","close","ensureCoreSchema","_handle","withTransaction","BEGIN_WRITE","ensureFeatureSchema","cfg","feature","activeFeatures","schema","setDerivedBusyTimeout","_conn","ms","filename","schemaVersion","reconcileDialect","sqliteDialect","createStore","openWithDialect"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAcpFA;eAAAA;;QAGAC;eAAAA;;QAgGSC;eAAAA;;QAZTC;eAAAA;;;0BApGgB;uBAEE;sBAEC;wBACC;6BACY;4BAEZ;2BACH;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,+FAA+F;AAC/F,IAAMG,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,SAAeC,mBAAmBC,IAAgB;;;;;oBAChD;;wBAAMA,KAAKC,IAAI,CAAC,AAAC,6IAAqJ,OAATH,UAAS;;;oBAAtK;;;;;;IACF;;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,SAAeI,uBAAuBF,IAAgB;;;;;oBACpD;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMF,mBAAmBC;;;oBAAzB;oBACA;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACK;;wBAAME,IAAAA,iBAAO,EAACH,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMI,IAAAA,iBAAO,EAACJ,MAAM,kBAAkBL;;;oBAAtC;;;;;;;;IACxD;;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASU,cAAcC,EAAgB;IACrC,IAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGL,IAAI,CAAC;YACR;QACF,EAAE,OAAOS,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB,EAAEC,OAA0D;;YAC/Gd,IASEN;;YATFM,KAAK,IAAIe,wBAAY,CAACH,QAAQE,QAAQE,YAAY,GAAG;gBAAEC,UAAU;YAAK,IAAI,CAAC;YACjF,IAAI;gBACF,6FAA6F;gBAC7F,yFAAyF;gBACzFjB,GAAGL,IAAI,CAAC;gBACR,yFAAyF;gBACzF,wEAAwE;gBACxE,IAAI,CAACmB,QAAQI,aAAa,EAAEnB,cAAcC;gBAC1CmB,IAAAA,iCAAiB,EAACnB;gBACZN,OAAO0B,IAAAA,8BAAgB,EAACpB;gBAC9B;;oBAAO;wBAAEqB,QAAQ;4BAAErB,IAAAA;wBAAG;wBAAGN,MAAAA;oBAAK;;YAChC,EAAE,OAAOU,KAAK;gBACZ,4FAA4F;gBAC5F,0CAA0C;gBAC1CJ,GAAGsB,KAAK;gBACR,MAAMlB;YACR;;;;;IACF;;AAEA,SAAekB,MAAMD,MAAoB;;;YACvCA,OAAOrB,EAAE,CAACsB,KAAK;;;;;IACjB;;AAEA,SAAeC,iBAAiBC,OAAqB,EAAE9B,IAAgB;;;;;oBACrE,qFAAqF;oBACrF;;wBAAM+B,IAAAA,8BAAe,EAAC/B,MAAM;mCAAME,uBAAuBF;2BAAOgC,0BAAW;;;oBAA3E;;;;;;IACF;;AAEA,SAAeC,oBAAoBH,OAAqB,EAAE9B,IAAgB,EAAEkC,GAAW;;YAChF,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,uBAAc,EAACF;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACrC;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACP;;AAEA,SAAesC,sBAAsBX,MAAoB,EAAEY,KAAiB,EAAEC,EAAU;;;YACtFb,OAAOrB,EAAE,CAACL,IAAI,CAAC,AAAC,yBAA2B,OAAHuC;;;;;IAC1C;;AAGO,IAAM3C,oBAA+C;IAC1D4C,UAAU/C;IACVgD,eAAe/C;IACfgD,kBAAkBC,0BAAa;IAC/B3B,SAAAA;IACAW,OAAAA;IACAC,kBAAAA;IACAI,qBAAAA;IACAK,uBAAAA;IACAO,aAAa,SAAbA,YAAclB,QAAQ3B;eAAS6C,IAAAA,oBAAW,EAAClB,OAAOrB,EAAE,EAAEN;;AACxD;AAEO,SAAeJ,WAAWsC,GAAmB,EAAEd,OAA6B;;;YACjF;;gBAAO0B,IAAAA,uBAAe,EAACZ,KAAKrC,mBAAmBuB;;;IACjD"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { ENGINE_CHECK_DOCS, checkEngine as runEngineCheck } from '../engine.ts';\nimport type { InternalOpenOptions, OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { SQLITE_VERSION, sqliteFts, sqliteIdentity } from './engine.ts';\nimport { queryLexical } from './lexical.ts';\nimport { contentRow, INSERT_CONTENT_SQL, sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '22';\n\nexport type { OpenResult };\n\ninterface SqliteHandle {\n db: DatabaseSync;\n}\n\nasync function createContentTable(conn: Connection): Promise<void> {\n await conn.exec(sqliteFts().createContent());\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Feature schema is published later with its generation.\nasync function ensureCoreSchemaTables(conn: Connection): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await createContentTable(conn);\n await conn.exec(`CREATE TABLE IF NOT EXISTS indexed_sources (\"path\" TEXT PRIMARY KEY, text TEXT NOT NULL)`);\n // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n const deadline = Date.now() + 5000;\n for (;;) {\n try {\n db.exec('PRAGMA journal_mode = WAL');\n return;\n } catch (err) {\n if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n }\n }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig, options: { existingOnly: boolean; observational: boolean }): Promise<{ handle: SqliteHandle; conn: Connection }> {\n const db = new DatabaseSync(dbPath, options.existingOnly ? { readOnly: true } : {});\n try {\n // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n db.exec('PRAGMA busy_timeout = 30000');\n // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n // handler for it, so a concurrent cold open needs its own bounded wait.\n if (!options.observational) setJournalWal(db);\n registerFunctions(db);\n const conn = createConnection(db);\n return { handle: { db }, conn };\n } catch (err) {\n // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n // scratch cleanup fails with EPERM/EBUSY.\n db.close();\n throw err;\n }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n handle.db.close();\n}\n\nasync function ensureCoreSchema(_handle: SqliteHandle, conn: Connection): Promise<void> {\n // One writer at a time: two cold opens can bootstrap the same database concurrently.\n await withTransaction(conn, () => ensureCoreSchemaTables(conn), BEGIN_WRITE);\n}\n\nasync function ensureFeatureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// The real schema, content write and lexical query over store/engine.ts's fixture, in memory.\nasync function checkSqliteEngine(): Promise<void> {\n const db = new DatabaseSync(':memory:');\n try {\n registerFunctions(db);\n const conn = createConnection(db);\n await conn.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)');\n await ensureCoreSchemaTables(conn);\n await conn.runBatch(\n 'INSERT INTO frontmatter (\"path\") VALUES (?)',\n ENGINE_CHECK_DOCS.map((doc) => [doc.path])\n );\n const docs = ENGINE_CHECK_DOCS.map((doc) => ({ relPath: doc.path, search: { title: doc.title, summary: doc.summary, text: doc.text } }) as ParsedDoc);\n await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n const remedy = `this SQLite is bundled with Node ${process.version} (SQLite ${SQLITE_VERSION ?? 'unknown'}); run sense on a Node release whose bundled SQLite passes`;\n await runEngineCheck('sqlite', sqliteIdentity(), remedy, async (terms) => (await queryLexical(conn, terms, { whereJoin: '', whereCond: '', scopeCond: '', limit: 10 })).map((hit) => hit.path));\n } finally {\n db.close();\n }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n filename: DB_FILENAME,\n schemaVersion: SCHEMA_VERSION,\n reconcileDialect: sqliteDialect,\n connect,\n close,\n engineIdentity: async () => sqliteIdentity(),\n checkEngine: checkSqliteEngine,\n ensureCoreSchema,\n ensureFeatureSchema,\n setDerivedBusyTimeout,\n createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig, options?: InternalOpenOptions): Promise<OpenResult> {\n return openWithDialect(cfg, sqliteOpenDialect, options);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openSqlite","sqliteOpenDialect","createContentTable","conn","exec","sqliteFts","createContent","ensureCoreSchemaTables","getMeta","setMeta","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","options","DatabaseSync","existingOnly","readOnly","observational","registerFunctions","createConnection","handle","close","ensureCoreSchema","_handle","withTransaction","BEGIN_WRITE","ensureFeatureSchema","cfg","feature","activeFeatures","schema","setDerivedBusyTimeout","_conn","ms","checkSqliteEngine","docs","remedy","runBatch","ENGINE_CHECK_DOCS","map","doc","path","relPath","search","title","summary","text","INSERT_CONTENT_SQL","contentRow","process","version","SQLITE_VERSION","runEngineCheck","sqliteIdentity","terms","queryLexical","whereJoin","whereCond","scopeCond","limit","hit","filename","schemaVersion","reconcileDialect","sqliteDialect","engineIdentity","checkEngine","createStore","openWithDialect"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAkBpFA;eAAAA;;QAGAC;eAAAA;;QAiHSC;eAAAA;;QAdTC;eAAAA;;;0BAvHgB;uBAEE;wBAEkC;sBAEjC;wBACC;6BACY;4BAEZ;yBACyB;yBAC7B;2BACiC;8BAC5B;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAQ9B,SAAeG,mBAAmBC,IAAgB;;;;;oBAChD;;wBAAMA,KAAKC,IAAI,CAACC,IAAAA,oBAAS,IAAGC,aAAa;;;oBAAzC;;;;;;IACF;;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,SAAeC,uBAAuBJ,IAAgB;;;;;oBACpD;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMF,mBAAmBC;;;oBAAzB;oBACA;;wBAAMA,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMD,KAAKC,IAAI,CAAC;;;oBAAhB;oBACK;;wBAAMI,IAAAA,iBAAO,EAACL,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM,kBAAkBJ;;;oBAAtC;;;;;;;;IACxD;;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASW,cAAcC,EAAgB;IACrC,IAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGP,IAAI,CAAC;YACR;QACF,EAAE,OAAOW,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB,EAAEC,OAA0D;;YAC/Gd,IASER;;YATFQ,KAAK,IAAIe,wBAAY,CAACH,QAAQE,QAAQE,YAAY,GAAG;gBAAEC,UAAU;YAAK,IAAI,CAAC;YACjF,IAAI;gBACF,6FAA6F;gBAC7F,yFAAyF;gBACzFjB,GAAGP,IAAI,CAAC;gBACR,yFAAyF;gBACzF,wEAAwE;gBACxE,IAAI,CAACqB,QAAQI,aAAa,EAAEnB,cAAcC;gBAC1CmB,IAAAA,iCAAiB,EAACnB;gBACZR,OAAO4B,IAAAA,8BAAgB,EAACpB;gBAC9B;;oBAAO;wBAAEqB,QAAQ;4BAAErB,IAAAA;wBAAG;wBAAGR,MAAAA;oBAAK;;YAChC,EAAE,OAAOY,KAAK;gBACZ,4FAA4F;gBAC5F,0CAA0C;gBAC1CJ,GAAGsB,KAAK;gBACR,MAAMlB;YACR;;;;;IACF;;AAEA,SAAekB,MAAMD,MAAoB;;;YACvCA,OAAOrB,EAAE,CAACsB,KAAK;;;;;IACjB;;AAEA,SAAeC,iBAAiBC,OAAqB,EAAEhC,IAAgB;;;;;oBACrE,qFAAqF;oBACrF;;wBAAMiC,IAAAA,8BAAe,EAACjC,MAAM;mCAAMI,uBAAuBJ;2BAAOkC,0BAAW;;;oBAA3E;;;;;;IACF;;AAEA,SAAeC,oBAAoBH,OAAqB,EAAEhC,IAAgB,EAAEoC,GAAW;;YAChF,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,uBAAc,EAACF;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACvC;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACP;;AAEA,SAAewC,sBAAsBX,MAAoB,EAAEY,KAAiB,EAAEC,EAAU;;;YACtFb,OAAOrB,EAAE,CAACP,IAAI,CAAC,AAAC,yBAA2B,OAAHyC;;;;;IAC1C;;AAEA,8FAA8F;AAC9F,SAAeC;;YACPnC,IAGER,MAOA4C,MAEAC;;;;oBAZFrC,KAAK,IAAIe,wBAAY,CAAC;;;;;;;;;oBAE1BI,IAAAA,iCAAiB,EAACnB;oBACZR,OAAO4B,IAAAA,8BAAgB,EAACpB;oBAC9B;;wBAAMR,KAAKC,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMG,uBAAuBJ;;;oBAA7B;oBACA;;wBAAMA,KAAK8C,QAAQ,CACjB,+CACAC,2BAAiB,CAACC,GAAG,CAAC,SAACC;mCAAQ;gCAACA,IAAIC,IAAI;6BAAC;;;;oBAF3C;oBAIMN,OAAOG,2BAAiB,CAACC,GAAG,CAAC,SAACC;+BAAS;4BAAEE,SAASF,IAAIC,IAAI;4BAAEE,QAAQ;gCAAEC,OAAOJ,IAAII,KAAK;gCAAEC,SAASL,IAAIK,OAAO;gCAAEC,MAAMN,IAAIM,IAAI;4BAAC;wBAAE;;oBACrI;;wBAAMvD,KAAK8C,QAAQ,CAACU,+BAAkB,EAAEZ,KAAKI,GAAG,CAACS,uBAAU;;;oBAA3D;oBACMZ,SAAS,AAjHnB,AAiHoB,2CAAmCa,QAAQC,OAAO,EAAC,aAAuC,OAA5BC,yBAAc,aAAdA,yBAAc,cAAdA,yBAAc,GAAI,WAAU;oBAC1G;;wBAAMC,IAAAA,qBAAc,EAAC,UAAUC,IAAAA,yBAAc,KAAIjB,QAAQ,SAAOkB;;;;;4CAAW;;gDAAMC,IAAAA,uBAAY,EAAChE,MAAM+D,OAAO;oDAAEE,WAAW;oDAAIC,WAAW;oDAAIC,WAAW;oDAAIC,OAAO;gDAAG;;;;;gDAAzF,cAA6FpB,GAAG,CAAC,SAACqB;2DAAQA,IAAInB,IAAI;;;;;;;;;oBAA7L;;;;;;oBAEA1C,GAAGsB,KAAK;;;;;;;;;;IAEZ;;AAGO,IAAMhC,oBAA+C;IAC1DwE,UAAU3E;IACV4E,eAAe3E;IACf4E,kBAAkBC,0BAAa;IAC/BtD,SAAAA;IACAW,OAAAA;IACA4C,gBAAgB,SAAhBA;;;;;oBAA4BZ,IAAAA,yBAAc;;;;;IAC1Ca,aAAahC;IACbZ,kBAAAA;IACAI,qBAAAA;IACAK,uBAAAA;IACAoC,aAAa,SAAbA,YAAc/C,QAAQ7B;eAAS4E,IAAAA,oBAAW,EAAC/C,OAAOrB,EAAE,EAAER;;AACxD;AAEO,SAAeH,WAAWuC,GAAmB,EAAEd,OAA6B;;;YACjF;;gBAAOuD,IAAAA,uBAAe,EAACzC,KAAKtC,mBAAmBwB;;;IACjD"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type EnginePatch } from '../engine.js';
|
|
2
|
+
export interface TursoFtsBuilders {
|
|
3
|
+
indexDdl(): readonly string[];
|
|
4
|
+
score(columns: string, literal: string): string;
|
|
5
|
+
match(columns: string, literal: string): string;
|
|
6
|
+
}
|
|
7
|
+
export declare const TURSO_FTS_BUILDERS: TursoFtsBuilders;
|
|
8
|
+
export declare const TURSO_PATCHES: readonly EnginePatch<TursoFtsBuilders>[];
|
|
9
|
+
export declare function tursoFts(): Promise<TursoFtsBuilders>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type EnginePatch } from '../engine.js';
|
|
2
|
+
export interface TursoFtsBuilders {
|
|
3
|
+
indexDdl(): readonly string[];
|
|
4
|
+
score(columns: string, literal: string): string;
|
|
5
|
+
match(columns: string, literal: string): string;
|
|
6
|
+
}
|
|
7
|
+
export declare const TURSO_FTS_BUILDERS: TursoFtsBuilders;
|
|
8
|
+
export declare const TURSO_PATCHES: readonly EnginePatch<TursoFtsBuilders>[];
|
|
9
|
+
export declare function tursoFts(): Promise<TursoFtsBuilders>;
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", {
|
|
3
|
+
value: true
|
|
4
|
+
});
|
|
5
|
+
function _export(target, all) {
|
|
6
|
+
for(var name in all)Object.defineProperty(target, name, {
|
|
7
|
+
enumerable: true,
|
|
8
|
+
get: Object.getOwnPropertyDescriptor(all, name).get
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
_export(exports, {
|
|
12
|
+
get TURSO_FTS_BUILDERS () {
|
|
13
|
+
return TURSO_FTS_BUILDERS;
|
|
14
|
+
},
|
|
15
|
+
get TURSO_PATCHES () {
|
|
16
|
+
return TURSO_PATCHES;
|
|
17
|
+
},
|
|
18
|
+
get tursoFts () {
|
|
19
|
+
return tursoFts;
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
var _enginets = require("../engine.js");
|
|
23
|
+
var _nativets = require("./native.js");
|
|
24
|
+
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
|
|
25
|
+
try {
|
|
26
|
+
var info = gen[key](arg);
|
|
27
|
+
var value = info.value;
|
|
28
|
+
} catch (error) {
|
|
29
|
+
reject(error);
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
if (info.done) resolve(value);
|
|
33
|
+
else Promise.resolve(value).then(_next, _throw);
|
|
34
|
+
}
|
|
35
|
+
function _async_to_generator(fn) {
|
|
36
|
+
return function() {
|
|
37
|
+
var self = this, args = arguments;
|
|
38
|
+
return new Promise(function(resolve, reject) {
|
|
39
|
+
var gen = fn.apply(self, args);
|
|
40
|
+
function _next(value) {
|
|
41
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
|
|
42
|
+
}
|
|
43
|
+
function _throw(err) {
|
|
44
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
|
|
45
|
+
}
|
|
46
|
+
_next(undefined);
|
|
47
|
+
});
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function _ts_generator(thisArg, body) {
|
|
51
|
+
var f, y, t, _ = {
|
|
52
|
+
label: 0,
|
|
53
|
+
sent: function() {
|
|
54
|
+
if (t[0] & 1) throw t[1];
|
|
55
|
+
return t[1];
|
|
56
|
+
},
|
|
57
|
+
trys: [],
|
|
58
|
+
ops: []
|
|
59
|
+
}, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
|
|
60
|
+
return d(g, "next", {
|
|
61
|
+
value: verb(0)
|
|
62
|
+
}), d(g, "throw", {
|
|
63
|
+
value: verb(1)
|
|
64
|
+
}), d(g, "return", {
|
|
65
|
+
value: verb(2)
|
|
66
|
+
}), typeof Symbol === "function" && d(g, Symbol.iterator, {
|
|
67
|
+
value: function() {
|
|
68
|
+
return this;
|
|
69
|
+
}
|
|
70
|
+
}), g;
|
|
71
|
+
function verb(n) {
|
|
72
|
+
return function(v) {
|
|
73
|
+
return step([
|
|
74
|
+
n,
|
|
75
|
+
v
|
|
76
|
+
]);
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function step(op) {
|
|
80
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
81
|
+
while(g && (g = 0, op[0] && (_ = 0)), _)try {
|
|
82
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
83
|
+
if (y = 0, t) op = [
|
|
84
|
+
op[0] & 2,
|
|
85
|
+
t.value
|
|
86
|
+
];
|
|
87
|
+
switch(op[0]){
|
|
88
|
+
case 0:
|
|
89
|
+
case 1:
|
|
90
|
+
t = op;
|
|
91
|
+
break;
|
|
92
|
+
case 4:
|
|
93
|
+
_.label++;
|
|
94
|
+
return {
|
|
95
|
+
value: op[1],
|
|
96
|
+
done: false
|
|
97
|
+
};
|
|
98
|
+
case 5:
|
|
99
|
+
_.label++;
|
|
100
|
+
y = op[1];
|
|
101
|
+
op = [
|
|
102
|
+
0
|
|
103
|
+
];
|
|
104
|
+
continue;
|
|
105
|
+
case 7:
|
|
106
|
+
op = _.ops.pop();
|
|
107
|
+
_.trys.pop();
|
|
108
|
+
continue;
|
|
109
|
+
default:
|
|
110
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
|
|
111
|
+
_ = 0;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
|
|
115
|
+
_.label = op[1];
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
if (op[0] === 6 && _.label < t[1]) {
|
|
119
|
+
_.label = t[1];
|
|
120
|
+
t = op;
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
if (t && _.label < t[2]) {
|
|
124
|
+
_.label = t[2];
|
|
125
|
+
_.ops.push(op);
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
if (t[2]) _.ops.pop();
|
|
129
|
+
_.trys.pop();
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
op = body.call(thisArg, _);
|
|
133
|
+
} catch (e) {
|
|
134
|
+
op = [
|
|
135
|
+
6,
|
|
136
|
+
e
|
|
137
|
+
];
|
|
138
|
+
y = 0;
|
|
139
|
+
} finally{
|
|
140
|
+
f = t = 0;
|
|
141
|
+
}
|
|
142
|
+
if (op[0] & 5) throw op[1];
|
|
143
|
+
return {
|
|
144
|
+
value: op[0] ? op[1] : void 0,
|
|
145
|
+
done: true
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
var TURSO_FTS_BUILDERS = {
|
|
150
|
+
indexDdl: function indexDdl() {
|
|
151
|
+
return [
|
|
152
|
+
"CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')",
|
|
153
|
+
"CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"
|
|
154
|
+
];
|
|
155
|
+
},
|
|
156
|
+
score: function score(columns, literal) {
|
|
157
|
+
return "fts_score(".concat(columns, ", '").concat(literal, "')");
|
|
158
|
+
},
|
|
159
|
+
match: function match(columns, literal) {
|
|
160
|
+
return "fts_match(".concat(columns, ", '").concat(literal, "')");
|
|
161
|
+
}
|
|
162
|
+
};
|
|
163
|
+
var TURSO_PATCHES = [];
|
|
164
|
+
function tursoFts() {
|
|
165
|
+
return _async_to_generator(function() {
|
|
166
|
+
var _tmp;
|
|
167
|
+
return _ts_generator(this, function(_state) {
|
|
168
|
+
switch(_state.label){
|
|
169
|
+
case 0:
|
|
170
|
+
_tmp = [
|
|
171
|
+
TURSO_FTS_BUILDERS,
|
|
172
|
+
TURSO_PATCHES
|
|
173
|
+
];
|
|
174
|
+
return [
|
|
175
|
+
4,
|
|
176
|
+
(0, _nativets.tursoNative)()
|
|
177
|
+
];
|
|
178
|
+
case 1:
|
|
179
|
+
return [
|
|
180
|
+
2,
|
|
181
|
+
_enginets.selectBuilders.apply(void 0, _tmp.concat([
|
|
182
|
+
_state.sent().version
|
|
183
|
+
]))
|
|
184
|
+
];
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
})();
|
|
188
|
+
}
|
|
189
|
+
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/turso/engine.ts"],"sourcesContent":["import { type EnginePatch, selectBuilders } from '../engine.ts';\nimport { tursoNative } from './native.ts';\n\n// Every statement this store sends to Turso's native FTS; a version that behaves differently gets a TURSO_PATCHES entry.\nexport interface TursoFtsBuilders {\n // The two FTS indexes. The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n // columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n indexDdl(): readonly string[];\n // `literal` is already escaped: the query argument must be a SQL literal, since bound as `?` the row set stays correct\n // but fts_score silently returns 0. fts_score is real only as a SELECT's bare projected expression for the index\n // that SELECT's fts_match chose; every other form silently returns 0.\n score(columns: string, literal: string): string;\n match(columns: string, literal: string): string;\n}\n\nexport const TURSO_FTS_BUILDERS: TursoFtsBuilders = {\n indexDdl: () => [\n `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title_stem, summary_stem, text_stem) WITH (weights = 'title_stem=10.0,summary_stem=5.0,text_stem=1.0')`,\n `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n ],\n score: (columns, literal) => `fts_score(${columns}, '${literal}')`,\n match: (columns, literal) => `fts_match(${columns}, '${literal}')`,\n};\n\nexport const TURSO_PATCHES: readonly EnginePatch<TursoFtsBuilders>[] = [];\n\nexport async function tursoFts(): Promise<TursoFtsBuilders> {\n return selectBuilders(TURSO_FTS_BUILDERS, TURSO_PATCHES, (await tursoNative()).version);\n}\n"],"names":["TURSO_FTS_BUILDERS","TURSO_PATCHES","tursoFts","indexDdl","score","columns","literal","match","tursoNative","selectBuilders","version"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QASAC;eAAAA;;QAESC;eAAAA;;;wBA1B2B;wBACrB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcrB,IAAMF,qBAAuC;IAClDG,UAAU,SAAVA;eAAgB;YACd;YACA;SACD;;IACDC,OAAO,SAAPA,MAAQC,SAASC;eAAY,AAAC,aAAyBA,OAAbD,SAAQ,OAAa,OAARC,SAAQ;;IAC/DC,OAAO,SAAPA,MAAQF,SAASC;eAAY,AAAC,aAAyBA,OAAbD,SAAQ,OAAa,OAARC,SAAQ;;AACjE;AAEO,IAAML,gBAA0D,EAAE;AAElE,SAAeC;;;;;;;wBACEF;wBAAoBC;;oBAAgB;;wBAAMO,IAAAA,qBAAW;;;oBAA3E;;wBAAOC,wBAAc;4BAAqC,cAAqBC,OAAO;;;;;IACxF"}
|
|
@@ -12,6 +12,7 @@ Object.defineProperty(exports, "queryLexical", {
|
|
|
12
12
|
});
|
|
13
13
|
var _errorsts = require("../../errors.js");
|
|
14
14
|
var _segmentts = require("../../text/segment.js");
|
|
15
|
+
var _enginets = require("./engine.js");
|
|
15
16
|
var _lexicaltextts = require("./lexical-text.js");
|
|
16
17
|
function _array_like_to_array(arr, len) {
|
|
17
18
|
if (len == null || len > arr.length) len = arr.length;
|
|
@@ -307,8 +308,7 @@ function splitTerms(terms) {
|
|
|
307
308
|
emptyPhrase: emptyPhrase
|
|
308
309
|
};
|
|
309
310
|
}
|
|
310
|
-
//
|
|
311
|
-
// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).
|
|
311
|
+
// engine.ts's score/match take the query as a SQL literal (PRINCIPLES: no-silent-modes).
|
|
312
312
|
function escapeFtsLiteral(query) {
|
|
313
313
|
return query.replace(/'/g, "''");
|
|
314
314
|
}
|
|
@@ -339,18 +339,17 @@ function buildQueries(words, phrases, unspaced) {
|
|
|
339
339
|
ngramQuery: unspacedParts.join(' AND ')
|
|
340
340
|
};
|
|
341
341
|
}
|
|
342
|
-
// fts_score
|
|
343
|
-
|
|
344
|
-
function ftsBranch(cols, query, opts) {
|
|
342
|
+
// One SELECT per index: fts_score is real only for the index its own SELECT's fts_match chose (engine.ts).
|
|
343
|
+
function ftsBranch(fts, cols, query, opts) {
|
|
345
344
|
var colList = cols.map(function(c) {
|
|
346
345
|
return "content.".concat(c);
|
|
347
346
|
}).join(', ');
|
|
348
347
|
var lit = escapeFtsLiteral(query);
|
|
349
|
-
return 'SELECT content."path" AS path,
|
|
348
|
+
return 'SELECT content."path" AS path, '.concat(fts.score(colList, lit), " AS score\n FROM content ").concat(opts.whereJoin, " WHERE ").concat(fts.match(colList, lit), " ").concat(opts.whereCond, " ").concat(opts.scopeCond);
|
|
350
349
|
}
|
|
351
350
|
function queryLexical(conn, terms, opts) {
|
|
352
351
|
return _async_to_generator(function() {
|
|
353
|
-
var _stmt, unsupported, _splitTerms, words, phrases, unspaced, emptyPhrase, _buildQueries, defaultQuery, ngramQuery, branches, ranked, phraseSelect, phraseJoin, sql, stmt, rows, matches;
|
|
352
|
+
var _stmt, unsupported, _splitTerms, words, phrases, unspaced, emptyPhrase, _buildQueries, defaultQuery, ngramQuery, fts, branches, ranked, phraseSelect, phraseJoin, sql, stmt, rows, matches;
|
|
354
353
|
return _ts_generator(this, function(_state) {
|
|
355
354
|
switch(_state.label){
|
|
356
355
|
case 0:
|
|
@@ -368,9 +367,15 @@ function queryLexical(conn, terms, opts) {
|
|
|
368
367
|
2,
|
|
369
368
|
[]
|
|
370
369
|
];
|
|
370
|
+
return [
|
|
371
|
+
4,
|
|
372
|
+
(0, _enginets.tursoFts)()
|
|
373
|
+
];
|
|
374
|
+
case 1:
|
|
375
|
+
fts = _state.sent();
|
|
371
376
|
branches = [];
|
|
372
|
-
if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));
|
|
373
|
-
if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));
|
|
377
|
+
if (defaultQuery !== '') branches.push(ftsBranch(fts, FIELDS, defaultQuery, opts));
|
|
378
|
+
if (ngramQuery !== '') branches.push(ftsBranch(fts, NGRAM_FIELDS, ngramQuery, opts));
|
|
374
379
|
// HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,
|
|
375
380
|
// `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase
|
|
376
381
|
// candidates carry authored fields through the join below so the shared verifier can reject
|
|
@@ -383,7 +388,7 @@ function queryLexical(conn, terms, opts) {
|
|
|
383
388
|
4,
|
|
384
389
|
conn.prepare(sql)
|
|
385
390
|
];
|
|
386
|
-
case
|
|
391
|
+
case 2:
|
|
387
392
|
stmt = _state.sent();
|
|
388
393
|
return [
|
|
389
394
|
4,
|
|
@@ -391,7 +396,7 @@ function queryLexical(conn, terms, opts) {
|
|
|
391
396
|
opts.limit
|
|
392
397
|
]))
|
|
393
398
|
];
|
|
394
|
-
case
|
|
399
|
+
case 3:
|
|
395
400
|
rows = _state.sent();
|
|
396
401
|
// A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,
|
|
397
402
|
// not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n for (const { label, re } of FTS5_OPERATORS) {\n const m = withoutPhrases.match(re);\n if (m) return { label, token: m[0].trim() };\n }\n return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n const words: string[] = [];\n const phrases: SearchToken[][] = [];\n const unspaced: string[] = [];\n let emptyPhrase = false;\n const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n const phrase = inner.trim();\n const tokens = searchTokens(phrase);\n if (tokens.length === 0) {\n emptyPhrase = true;\n return ' ';\n }\n unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n phrases.push(tokens);\n return ' ';\n });\n for (const tok of withoutPhrases.split(/\\s+/)) {\n if (tok.length === 0) continue;\n for (const token of searchTokens(tok)) {\n if (token.unspaced) unspaced.push(token.text);\n else words.push(token.text);\n }\n }\n return { words, phrases, unspaced, emptyPhrase };\n}\n\n// fts_match/fts_score's query argument must be a SQL literal: bound as `?`, the row set stays\n// correct but fts_score silently returns 0 for every row (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n const phraseParts = phrases.flatMap((phrase) => {\n const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n if (latin.length === 0) return [];\n return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n });\n const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// fts_score() is real only as a SELECT's bare projected expression, and only for the index that\n// SELECT's fts_match() chose; every other form silently returns 0. Hence one SELECT per index.\nfunction ftsBranch(cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n const colList = cols.map((c) => `content.${c}`).join(', ');\n const lit = escapeFtsLiteral(query);\n return `SELECT content.\"path\" AS path, fts_score(${colList}, '${lit}') AS score\n FROM content ${opts.whereJoin} WHERE fts_match(${colList}, '${lit}') ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n const unsupported = unsupportedOperator(terms);\n if (unsupported !== null) {\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n }\n const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n if (emptyPhrase) return [];\n const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n if (defaultQuery === '' && ngramQuery === '') return [];\n\n const branches: string[] = [];\n if (defaultQuery !== '') branches.push(ftsBranch(FIELDS, defaultQuery, opts));\n if (ngramQuery !== '') branches.push(ftsBranch(NGRAM_FIELDS, ngramQuery, opts));\n\n // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n // candidates carry authored fields through the join below so the shared verifier can reject\n // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n const ranked = `SELECT path, SUM(score) AS score FROM (\n ${branches.join(' UNION ALL ')}\n ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n ${ranked}\n ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n const stmt = await conn.prepare(sql);\n const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n if (rows.some((r) => r.path === null)) {\n throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n }\n const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["queryLexical","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","searchTokens","length","push","filter","map","text","split","tok","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","stemFolded","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","cols","opts","colList","c","lit","whereJoin","whereCond","scopeCond","conn","stmt","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","rows","matches","SenseError","prepare","all","limit","r","path","row","every","field","matchesSearchPhrase","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;;;;;+BAyFjFA;;;eAAAA;;;wBAxFK;yBACyC;6BAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE3B,IAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,IAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,IAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,UAA2B,EAAE;IACnC,IAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,IAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACU,IAAIC;YAOtDH;QANA,IAAMI,SAASD,MAAMP,IAAI;QACzB,IAAMS,SAASC,IAAAA,uBAAY,EAACF;QAC5B,IAAIC,OAAOE,MAAM,KAAK,GAAG;YACvBN,cAAc;YACd,OAAO;QACT;QACAD,CAAAA,YAAAA,UAASQ,IAAI,OAAbR,WAAc,qBAAGK,OAAOI,MAAM,CAAC,SAACd;mBAAUA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUA,MAAMgB,IAAI;;QACnFZ,QAAQS,IAAI,CAACH;QACb,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAad,eAAeqB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIN,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACO,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAMlB,QAAN;oBACH,IAAIA,MAAMK,QAAQ,EAAEA,SAASQ,IAAI,CAACb,MAAMgB,IAAI;yBACvCb,MAAMU,IAAI,CAACb,MAAMgB,IAAI;gBAC5B;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAIP;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,OAAO;QAAEb,OAAAA;QAAOC,SAAAA;QAASC,UAAAA;QAAUC,aAAAA;IAAY;AACjD;AAEA,8FAA8F;AAC9F,wFAAwF;AACxF,SAASa,iBAAiBC,KAAa;IACrC,OAAOA,MAAMvB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASwB,aAAalB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,IAAMiB,cAAclB,QAAQmB,OAAO,CAAC,SAACd;QACnC,IAAMe,QAAQf,OAAOK,MAAM,CAAC,SAACd;mBAAU,CAACA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUyB,IAAAA,yBAAU,EAACzB,MAAMgB,IAAI;;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOH,OAAOiB,IAAI,CAAC,SAAC1B;mBAAUA,MAAMK,QAAQ;aAAImB,QAAQ;YAACA,MAAMG,IAAI,CAAC;SAAK;IAC3E;IACA,IAAMC,eAAe,AAAC,qBAAGzB,MAAMY,GAAG,CAACU,yBAAU,UAAG,qBAAGH,cAAaP,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IACjF,IAAMC,gBAAgBzB,SAASU,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAChD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,SAASM,UAAUC,IAAuB,EAAEd,KAAa,EAAEe,IAAyB;IAClF,IAAMC,UAAUF,KAAKnB,GAAG,CAAC,SAACsB;eAAM,AAAC,WAAY,OAAFA;OAAKV,IAAI,CAAC;IACrD,IAAMW,MAAMnB,iBAAiBC;IAC7B,OAAO,AAAC,4CAAwDkB,OAAbF,SAAQ,OAC1CD,OAD+CG,KAAI,kCACjBF,OAAlCD,KAAKI,SAAS,EAAC,qBAAgCD,OAAbF,SAAQ,OAAcD,OAATG,KAAI,OAAuBH,OAAlBA,KAAKK,SAAS,EAAC,KAAkB,OAAfL,KAAKM,SAAS;AAC3G;AAIO,SAAetD,aAAauD,IAAgB,EAAE/C,KAAa,EAAEwC,IAAyB;;YA2BvEQ,OA1BdC,aAI4C1C,aAA1CC,OAAOC,SAASC,UAAUC,aAEGe,eAA7BU,cAAcC,YAGhBa,UAQAC,QAGAC,cACAC,YACAC,KAGAN,MACAO,MAMAC;;;;oBAhCAP,cAAclD,oBAAoBC;oBACxC,IAAIiD,gBAAgB,MAAM;wBACxB,MAAM,IAAIQ,oBAAU,CAAC,4BAA4B,AAAC,4CAAiER,OAAvBA,YAAYpD,KAAK,EAAC,OAAqDG,OAAhDiD,YAAY5C,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBAClL;oBACkDC,cAAAA,WAAWP,QAArDQ,QAA0CD,YAA1CC,OAAOC,UAAmCF,YAAnCE,SAASC,WAA0BH,YAA1BG,UAAUC,cAAgBJ,YAAhBI;oBAClC,IAAIA,aAAa;;;;oBACoBe,gBAAAA,aAAalB,OAAOC,SAASC,WAA1D0B,eAA6BV,cAA7BU,cAAcC,aAAeX,cAAfW;oBACtB,IAAID,iBAAiB,MAAMC,eAAe,IAAI;;;;oBAExCa;oBACN,IAAId,iBAAiB,IAAIc,SAAShC,IAAI,CAACoB,UAAU7C,QAAQ2C,cAAcI;oBACvE,IAAIH,eAAe,IAAIa,SAAShC,IAAI,CAACoB,UAAU5C,cAAc2C,YAAYG;oBAEzE,8FAA8F;oBAC9F,0FAA0F;oBAC1F,4FAA4F;oBAC5F,uFAAuF;oBACjFW,SAAS,AAAC,kDAE4BD,OADtCA,SAASlB,IAAI,CAAC,gBAAe,kDACyB,OAAhBkB,SAASjC,MAAM;oBACrDmC,eAAe3C,QAAQQ,MAAM,GAAG,IAAI,mDAAmD;oBACvFoC,aAAa5C,QAAQQ,MAAM,GAAG,IAAI,4CAA4C;oBAC9EqC,MAAM,AAAC,2BACTH,OADmCC,cAAa,iBAE9CC,OADFF,QAAO,YACwC1C,OAA7C4C,YAAW,oCAAuE,OAArC5C,QAAQQ,MAAM,GAAG,IAAI,KAAK;oBAChE;;wBAAM8B,KAAKW,OAAO,CAACJ;;;oBAA1BN,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKW,GAAG,OAARX,OAAS,qBAAIvC,QAAQQ,MAAM,GAAG;4BAAUuB,KAAKoB,KAAK;;;;oBAAhEL,OAAQ;oBACd,+FAA+F;oBAC/F,yFAAyF;oBACzF,IAAIA,KAAKxB,IAAI,CAAC,SAAC8B;+BAAMA,EAAEC,IAAI,KAAK;wBAAO;wBACrC,MAAM,IAAIL,oBAAU,CAAC,qBAAqB;oBAC5C;oBACMD,UAAU/C,QAAQQ,MAAM,KAAK,IAAIsC,OAAOA,KAAKpC,MAAM,CAAC,SAAC4C;+BAAQtD,QAAQuD,KAAK,CAAC,SAAClD;mCAAWnB,gBAAgBoC,IAAI,CAAC,SAACkC;oCAA8BF;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAIjD;;;;oBACnK;;wBAAO0C,QAAQW,KAAK,CAAC,GAAG3B,KAAKoB,KAAK,EAAExC,GAAG,CAAC;gCAAG0C,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAChE"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/turso/lexical.ts"],"sourcesContent":["// T1: Tantivy FTS via fts_match/fts_score on the default-tokenizer index for bare words and\n// quoted phrases, plus a second ngram-tokenized index over \"_ngram\" sidecars for unspaced-script runs.\nimport { SenseError } from '../../errors.ts';\nimport { matchesSearchPhrase, type SearchToken, searchTokens } from '../../text/segment.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\nimport { type TursoFtsBuilders, tursoFts } from './engine.ts';\nimport { stemFolded } from './lexical-text.ts';\n\nconst FIELDS = ['title_stem', 'summary_stem', 'text_stem'] as const;\nconst NGRAM_FIELDS = ['title_ngram', 'summary_ngram', 'text_ngram'] as const;\nconst AUTHORED_FIELDS = ['title', 'summary', 'text'] as const;\ntype AuthoredField = (typeof AUTHORED_FIELDS)[number];\n\n// FTS5 operator syntax a bare query would otherwise treat as literal text (PRINCIPLES:\n// no-silent-modes). `^` is rejected either way: boost to Tantivy, initial-token match to FTS5.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n { label: 'boost/initial-token operator', re: /\\S*\\^\\S*/ },\n { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n { label: 'grouping operator', re: /[()]/ },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n const withoutPhrases = terms.replace(/\"[^\"]*\"/g, 'phrase');\n for (const { label, re } of FTS5_OPERATORS) {\n const m = withoutPhrases.match(re);\n if (m) return { label, token: m[0].trim() };\n }\n return null;\n}\n\n// A run whose script marks no word boundaries indexes as one opaque token under every tokenizer\n// but ngram, so such runs -- bare or quoted -- go to the ngram sidecar index instead.\nfunction splitTerms(terms: string): { words: string[]; phrases: SearchToken[][]; unspaced: string[]; emptyPhrase: boolean } {\n const words: string[] = [];\n const phrases: SearchToken[][] = [];\n const unspaced: string[] = [];\n let emptyPhrase = false;\n const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n const phrase = inner.trim();\n const tokens = searchTokens(phrase);\n if (tokens.length === 0) {\n emptyPhrase = true;\n return ' ';\n }\n unspaced.push(...tokens.filter((token) => token.unspaced).map((token) => token.text));\n phrases.push(tokens);\n return ' ';\n });\n for (const tok of withoutPhrases.split(/\\s+/)) {\n if (tok.length === 0) continue;\n for (const token of searchTokens(tok)) {\n if (token.unspaced) unspaced.push(token.text);\n else words.push(token.text);\n }\n }\n return { words, phrases, unspaced, emptyPhrase };\n}\n\n// engine.ts's score/match take the query as a SQL literal (PRINCIPLES: no-silent-modes).\nfunction escapeFtsLiteral(query: string): string {\n return query.replace(/'/g, \"''\");\n}\n\n// Terms are quoted (\"O'Brien's\" is a parse error bare) and AND-joined: Tantivy's default for\n// bare terms is disjunctive, so space-joining would loosen \"apple banana\" into \"apple OR banana\".\nfunction buildQueries(words: string[], phrases: SearchToken[][], unspaced: string[]): { defaultQuery: string; ngramQuery: string } {\n const phraseParts = phrases.flatMap((phrase) => {\n const latin = phrase.filter((token) => !token.unspaced).map((token) => stemFolded(token.text));\n if (latin.length === 0) return [];\n return phrase.some((token) => token.unspaced) ? latin : [latin.join(' ')];\n });\n const defaultParts = [...words.map(stemFolded), ...phraseParts].map((t) => `\"${t}\"`);\n const unspacedParts = unspaced.map((t) => `\"${t}\"`);\n return { defaultQuery: defaultParts.join(' AND '), ngramQuery: unspacedParts.join(' AND ') };\n}\n\n// One SELECT per index: fts_score is real only for the index its own SELECT's fts_match chose (engine.ts).\nfunction ftsBranch(fts: TursoFtsBuilders, cols: readonly string[], query: string, opts: LexicalQueryOptions): string {\n const colList = cols.map((c) => `content.${c}`).join(', ');\n const lit = escapeFtsLiteral(query);\n return `SELECT content.\"path\" AS path, ${fts.score(colList, lit)} AS score\n FROM content ${opts.whereJoin} WHERE ${fts.match(colList, lit)} ${opts.whereCond} ${opts.scopeCond}`;\n}\n\n// Ranked query, scoped by the caller-built SQL fragments (same shape sqlite's/duckdb's\n// queryLexical take). `snippets` is always NULL: fts_highlight returns the full column text, not a bounded snippet, so every row goes through the caller's JS snippet fallback (commands/search.ts).\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]> {\n const unsupported = unsupportedOperator(terms);\n if (unsupported !== null) {\n throw new SenseError('STORE_CAPABILITY_MISSING', `store \"turso\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n }\n const { words, phrases, unspaced, emptyPhrase } = splitTerms(terms);\n if (emptyPhrase) return [];\n const { defaultQuery, ngramQuery } = buildQueries(words, phrases, unspaced);\n if (defaultQuery === '' && ngramQuery === '') return [];\n\n const fts = await tursoFts();\n const branches: string[] = [];\n if (defaultQuery !== '') branches.push(ftsBranch(fts, FIELDS, defaultQuery, opts));\n if (ngramQuery !== '') branches.push(ftsBranch(fts, NGRAM_FIELDS, ngramQuery, opts));\n\n // HAVING COUNT(*) = branches.length is the AND across indexes; with one branch it is a no-op,\n // `path` being content's primary key. fts_score is higher-is-better BM25, so DESC. Phrase\n // candidates carry authored fields through the join below so the shared verifier can reject\n // cross-field, reordered, and non-adjacent matches before applying the caller's limit.\n const ranked = `SELECT path, SUM(score) AS score FROM (\n ${branches.join(' UNION ALL ')}\n ) parts GROUP BY path HAVING COUNT(*) = ${branches.length}`;\n const phraseSelect = phrases.length > 0 ? ', content.title, content.summary, content.text' : '';\n const phraseJoin = phrases.length > 0 ? ' JOIN content ON content.path = sq.path' : '';\n const sql = `SELECT sq.path, sq.score${phraseSelect} FROM (\n ${ranked}\n ) sq${phraseJoin} ORDER BY sq.score DESC, sq.path${phrases.length > 0 ? '' : ' LIMIT ?'}`;\n const stmt = await conn.prepare(sql);\n const rows = (await stmt.all(...(phrases.length > 0 ? [] : [opts.limit]))) as Array<LexicalHit & Partial<Record<AuthoredField, string>>>;\n // A concurrent FTS read during an open write transaction can yield a {path: null} row. Raised,\n // not filtered: filtering returns a silently short result (PRINCIPLES: no-silent-modes).\n if (rows.some((r) => r.path === null)) {\n throw new SenseError('LEXICAL_NULL_PATH', 'store \"turso\" returned a search hit with no path (a known engine anomaly under concurrent FTS reads); retry the query');\n }\n const matches = phrases.length === 0 ? rows : rows.filter((row) => phrases.every((phrase) => AUTHORED_FIELDS.some((field) => matchesSearchPhrase(row[field] ?? '', phrase))));\n return matches.slice(0, opts.limit).map(({ path }) => ({ path }));\n}\n"],"names":["queryLexical","FIELDS","NGRAM_FIELDS","AUTHORED_FIELDS","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","phrases","unspaced","emptyPhrase","_m","inner","phrase","tokens","searchTokens","length","push","filter","map","text","split","tok","escapeFtsLiteral","query","buildQueries","phraseParts","flatMap","latin","stemFolded","some","join","defaultParts","t","unspacedParts","defaultQuery","ngramQuery","ftsBranch","fts","cols","opts","colList","c","lit","score","whereJoin","whereCond","scopeCond","conn","stmt","unsupported","branches","ranked","phraseSelect","phraseJoin","sql","rows","matches","SenseError","tursoFts","prepare","all","limit","r","path","row","every","field","matchesSearchPhrase","slice"],"mappings":"AAAA,4FAA4F;AAC5F,uGAAuG;;;;;+BAwFjFA;;;eAAAA;;;wBAvFK;yBACyC;wBAEpB;6BACrB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE3B,IAAMC,SAAS;IAAC;IAAc;IAAgB;CAAY;AAC1D,IAAMC,eAAe;IAAC;IAAe;IAAiB;CAAa;AACnE,IAAMC,kBAAkB;IAAC;IAAS;IAAW;CAAO;AAGpD,uFAAuF;AACvF,+FAA+F;AAC/F,IAAMC,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAAgCC,IAAI;IAAW;IACxD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;IACzD;QAAED,OAAO;QAAqBC,IAAI;IAAO;CAC1C;AAED,SAASC,oBAAoBC,KAAa;IACxC,IAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;QAC5C,kCAAA,2BAAA;;QAAL,QAAK,YAAuBN,mCAAvB,SAAA,6BAAA,QAAA,yBAAA,iCAAuC;YAAvC,kBAAA,aAAQC,oBAAAA,OAAOC,iBAAAA;YAClB,IAAMK,IAAIF,eAAeG,KAAK,CAACN;YAC/B,IAAIK,GAAG,OAAO;gBAAEN,OAAAA;gBAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;YAAG;QAC5C;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAO;AACT;AAEA,gGAAgG;AAChG,sFAAsF;AACtF,SAASC,WAAWP,KAAa;IAC/B,IAAMQ,QAAkB,EAAE;IAC1B,IAAMC,UAA2B,EAAE;IACnC,IAAMC,WAAqB,EAAE;IAC7B,IAAIC,cAAc;IAClB,IAAMV,iBAAiBD,MAAME,OAAO,CAAC,cAAc,SAACU,IAAIC;YAOtDH;QANA,IAAMI,SAASD,MAAMP,IAAI;QACzB,IAAMS,SAASC,IAAAA,uBAAY,EAACF;QAC5B,IAAIC,OAAOE,MAAM,KAAK,GAAG;YACvBN,cAAc;YACd,OAAO;QACT;QACAD,CAAAA,YAAAA,UAASQ,IAAI,OAAbR,WAAc,qBAAGK,OAAOI,MAAM,CAAC,SAACd;mBAAUA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUA,MAAMgB,IAAI;;QACnFZ,QAAQS,IAAI,CAACH;QACb,OAAO;IACT;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAad,eAAeqB,KAAK,CAAC,2BAAlC,SAAA,6BAAA,QAAA,yBAAA,iCAA0C;YAA1C,IAAMC,MAAN;YACH,IAAIA,IAAIN,MAAM,KAAK,GAAG;gBACjB,mCAAA,4BAAA;;gBAAL,QAAK,aAAeD,IAAAA,uBAAY,EAACO,yBAA5B,UAAA,8BAAA,SAAA,0BAAA,kCAAkC;oBAAlC,IAAMlB,QAAN;oBACH,IAAIA,MAAMK,QAAQ,EAAEA,SAASQ,IAAI,CAACb,MAAMgB,IAAI;yBACvCb,MAAMU,IAAI,CAACb,MAAMgB,IAAI;gBAC5B;;gBAHK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QAIP;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,OAAO;QAAEb,OAAAA;QAAOC,SAAAA;QAASC,UAAAA;QAAUC,aAAAA;IAAY;AACjD;AAEA,yFAAyF;AACzF,SAASa,iBAAiBC,KAAa;IACrC,OAAOA,MAAMvB,OAAO,CAAC,MAAM;AAC7B;AAEA,6FAA6F;AAC7F,kGAAkG;AAClG,SAASwB,aAAalB,KAAe,EAAEC,OAAwB,EAAEC,QAAkB;IACjF,IAAMiB,cAAclB,QAAQmB,OAAO,CAAC,SAACd;QACnC,IAAMe,QAAQf,OAAOK,MAAM,CAAC,SAACd;mBAAU,CAACA,MAAMK,QAAQ;WAAEU,GAAG,CAAC,SAACf;mBAAUyB,IAAAA,yBAAU,EAACzB,MAAMgB,IAAI;;QAC5F,IAAIQ,MAAMZ,MAAM,KAAK,GAAG,OAAO,EAAE;QACjC,OAAOH,OAAOiB,IAAI,CAAC,SAAC1B;mBAAUA,MAAMK,QAAQ;aAAImB,QAAQ;YAACA,MAAMG,IAAI,CAAC;SAAK;IAC3E;IACA,IAAMC,eAAe,AAAC,qBAAGzB,MAAMY,GAAG,CAACU,yBAAU,UAAG,qBAAGH,cAAaP,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IACjF,IAAMC,gBAAgBzB,SAASU,GAAG,CAAC,SAACc;eAAM,AAAC,IAAK,OAAFA,GAAE;;IAChD,OAAO;QAAEE,cAAcH,aAAaD,IAAI,CAAC;QAAUK,YAAYF,cAAcH,IAAI,CAAC;IAAS;AAC7F;AAEA,2GAA2G;AAC3G,SAASM,UAAUC,GAAqB,EAAEC,IAAuB,EAAEf,KAAa,EAAEgB,IAAyB;IACzG,IAAMC,UAAUF,KAAKpB,GAAG,CAAC,SAACuB;eAAM,AAAC,WAAY,OAAFA;OAAKX,IAAI,CAAC;IACrD,IAAMY,MAAMpB,iBAAiBC;IAC7B,OAAO,AAAC,kCACSgB,OADwBF,IAAIM,KAAK,CAACH,SAASE,MAAK,gCACxBL,OAAxBE,KAAKK,SAAS,EAAC,WAAoCL,OAA3BF,IAAInC,KAAK,CAACsC,SAASE,MAAK,KAAqBH,OAAlBA,KAAKM,SAAS,EAAC,KAAkB,OAAfN,KAAKO,SAAS;AACtG;AAIO,SAAexD,aAAayD,IAAgB,EAAEjD,KAAa,EAAEyC,IAAyB;;YA4BvES,OA3BdC,aAI4C5C,aAA1CC,OAAOC,SAASC,UAAUC,aAEGe,eAA7BU,cAAcC,YAGhBE,KACAa,UAQAC,QAGAC,cACAC,YACAC,KAGAN,MACAO,MAMAC;;;;oBAjCAP,cAAcpD,oBAAoBC;oBACxC,IAAImD,gBAAgB,MAAM;wBACxB,MAAM,IAAIQ,oBAAU,CAAC,4BAA4B,AAAC,4CAAiER,OAAvBA,YAAYtD,KAAK,EAAC,OAAqDG,OAAhDmD,YAAY9C,KAAK,EAAC,gCAA2C,OAAbL,MAAMM,IAAI,IAAG;oBAClL;oBACkDC,cAAAA,WAAWP,QAArDQ,QAA0CD,YAA1CC,OAAOC,UAAmCF,YAAnCE,SAASC,WAA0BH,YAA1BG,UAAUC,cAAgBJ,YAAhBI;oBAClC,IAAIA,aAAa;;;;oBACoBe,gBAAAA,aAAalB,OAAOC,SAASC,WAA1D0B,eAA6BV,cAA7BU,cAAcC,aAAeX,cAAfW;oBACtB,IAAID,iBAAiB,MAAMC,eAAe,IAAI;;;;oBAElC;;wBAAMuB,IAAAA,kBAAQ;;;oBAApBrB,MAAM;oBACNa;oBACN,IAAIhB,iBAAiB,IAAIgB,SAASlC,IAAI,CAACoB,UAAUC,KAAK9C,QAAQ2C,cAAcK;oBAC5E,IAAIJ,eAAe,IAAIe,SAASlC,IAAI,CAACoB,UAAUC,KAAK7C,cAAc2C,YAAYI;oBAE9E,8FAA8F;oBAC9F,0FAA0F;oBAC1F,4FAA4F;oBAC5F,uFAAuF;oBACjFY,SAAS,AAAC,kDAE4BD,OADtCA,SAASpB,IAAI,CAAC,gBAAe,kDACyB,OAAhBoB,SAASnC,MAAM;oBACrDqC,eAAe7C,QAAQQ,MAAM,GAAG,IAAI,mDAAmD;oBACvFsC,aAAa9C,QAAQQ,MAAM,GAAG,IAAI,4CAA4C;oBAC9EuC,MAAM,AAAC,2BACTH,OADmCC,cAAa,iBAE9CC,OADFF,QAAO,YACwC5C,OAA7C8C,YAAW,oCAAuE,OAArC9C,QAAQQ,MAAM,GAAG,IAAI,KAAK;oBAChE;;wBAAMgC,KAAKY,OAAO,CAACL;;;oBAA1BN,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKY,GAAG,OAARZ,OAAS,qBAAIzC,QAAQQ,MAAM,GAAG;4BAAUwB,KAAKsB,KAAK;;;;oBAAhEN,OAAQ;oBACd,+FAA+F;oBAC/F,yFAAyF;oBACzF,IAAIA,KAAK1B,IAAI,CAAC,SAACiC;+BAAMA,EAAEC,IAAI,KAAK;wBAAO;wBACrC,MAAM,IAAIN,oBAAU,CAAC,qBAAqB;oBAC5C;oBACMD,UAAUjD,QAAQQ,MAAM,KAAK,IAAIwC,OAAOA,KAAKtC,MAAM,CAAC,SAAC+C;+BAAQzD,QAAQ0D,KAAK,CAAC,SAACrD;mCAAWnB,gBAAgBoC,IAAI,CAAC,SAACqC;oCAA8BF;uCAApBG,IAAAA,8BAAmB,GAACH,aAAAA,GAAG,CAACE,MAAM,cAAVF,wBAAAA,aAAc,IAAIpD;;;;oBACnK;;wBAAO4C,QAAQY,KAAK,CAAC,GAAG7B,KAAKsB,KAAK,EAAE3C,GAAG,CAAC;gCAAG6C,aAAAA;mCAAY;gCAAEA,MAAAA;4BAAK;;;;;IAChE"}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
import { type NativeModule } from '../native.js';
|
|
1
2
|
export declare const TURSO_PACKAGE = "@tursodatabase/database";
|
|
2
3
|
export declare const CONNECT_OPTS: {
|
|
3
4
|
readonly timeout: 30000;
|
|
4
5
|
readonly experimental: readonly ["index_method"];
|
|
5
6
|
};
|
|
6
|
-
|
|
7
|
+
type TursoApi = typeof import('@tursodatabase/database');
|
|
8
|
+
export declare function tursoNative(): Promise<NativeModule<TursoApi>>;
|
|
9
|
+
export declare function tursoApi(): Promise<TursoApi>;
|
|
10
|
+
export {};
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
import { type NativeModule } from '../native.js';
|
|
1
2
|
export declare const TURSO_PACKAGE = "@tursodatabase/database";
|
|
2
3
|
export declare const CONNECT_OPTS: {
|
|
3
4
|
readonly timeout: 30000;
|
|
4
5
|
readonly experimental: readonly ["index_method"];
|
|
5
6
|
};
|
|
6
|
-
|
|
7
|
+
type TursoApi = typeof import('@tursodatabase/database');
|
|
8
|
+
export declare function tursoNative(): Promise<NativeModule<TursoApi>>;
|
|
9
|
+
export declare function tursoApi(): Promise<TursoApi>;
|
|
10
|
+
export {};
|