sensemaking 0.19.0 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/store/sqlite/connection.js +1 -1
- package/dist/cjs/store/sqlite/connection.js.map +1 -1
- package/dist/cjs/store/sqlite/open.js +25 -5
- package/dist/cjs/store/sqlite/open.js.map +1 -1
- package/dist/cjs/store/sqlite/reconcile.js +111 -101
- package/dist/cjs/store/sqlite/reconcile.js.map +1 -1
- package/dist/cjs/store/transaction.d.cts +2 -1
- package/dist/cjs/store/transaction.d.ts +2 -1
- package/dist/cjs/store/transaction.js +16 -6
- package/dist/cjs/store/transaction.js.map +1 -1
- package/dist/cjs/store/turso/connection.js +1 -1
- package/dist/cjs/store/turso/connection.js.map +1 -1
- package/dist/cjs/store/turso/open.d.cts +2 -0
- package/dist/cjs/store/turso/open.d.ts +2 -0
- package/dist/cjs/store/turso/open.js +112 -55
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/reconcile.js +264 -130
- package/dist/cjs/store/turso/reconcile.js.map +1 -1
- package/dist/esm/store/sqlite/connection.js +2 -2
- package/dist/esm/store/sqlite/connection.js.map +1 -1
- package/dist/esm/store/sqlite/open.js +24 -6
- package/dist/esm/store/sqlite/open.js.map +1 -1
- package/dist/esm/store/sqlite/reconcile.js +7 -4
- package/dist/esm/store/sqlite/reconcile.js.map +1 -1
- package/dist/esm/store/transaction.d.ts +2 -1
- package/dist/esm/store/transaction.js +11 -6
- package/dist/esm/store/transaction.js.map +1 -1
- package/dist/esm/store/turso/connection.js +2 -2
- package/dist/esm/store/turso/connection.js.map +1 -1
- package/dist/esm/store/turso/open.d.ts +2 -0
- package/dist/esm/store/turso/open.js +11 -2
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/reconcile.js +16 -2
- package/dist/esm/store/turso/reconcile.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,9 +4,17 @@
|
|
|
4
4
|
Object.defineProperty(exports, "__esModule", {
|
|
5
5
|
value: true
|
|
6
6
|
});
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
function _export(target, all) {
|
|
8
|
+
for(var name in all)Object.defineProperty(target, name, {
|
|
9
|
+
enumerable: true,
|
|
10
|
+
get: Object.getOwnPropertyDescriptor(all, name).get
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
_export(exports, {
|
|
14
|
+
get BEGIN_WRITE () {
|
|
15
|
+
return BEGIN_WRITE;
|
|
16
|
+
},
|
|
17
|
+
get withTransaction () {
|
|
10
18
|
return withTransaction;
|
|
11
19
|
}
|
|
12
20
|
});
|
|
@@ -149,7 +157,7 @@ function stateFor(conn) {
|
|
|
149
157
|
}
|
|
150
158
|
return state;
|
|
151
159
|
}
|
|
152
|
-
function enter(conn, state) {
|
|
160
|
+
function enter(conn, state, begin) {
|
|
153
161
|
return _async_to_generator(function() {
|
|
154
162
|
return _ts_generator(this, function(_state) {
|
|
155
163
|
switch(_state.label){
|
|
@@ -160,7 +168,7 @@ function enter(conn, state) {
|
|
|
160
168
|
];
|
|
161
169
|
return [
|
|
162
170
|
4,
|
|
163
|
-
conn.exec(
|
|
171
|
+
conn.exec(begin)
|
|
164
172
|
];
|
|
165
173
|
case 1:
|
|
166
174
|
_state.sent();
|
|
@@ -237,7 +245,9 @@ function leaveErr(conn, state) {
|
|
|
237
245
|
});
|
|
238
246
|
})();
|
|
239
247
|
}
|
|
248
|
+
var BEGIN_WRITE = 'BEGIN IMMEDIATE';
|
|
240
249
|
function withTransaction(conn, fn) {
|
|
250
|
+
var begin = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 'BEGIN';
|
|
241
251
|
return _async_to_generator(function() {
|
|
242
252
|
var state, result, err;
|
|
243
253
|
return _ts_generator(this, function(_state) {
|
|
@@ -246,7 +256,7 @@ function withTransaction(conn, fn) {
|
|
|
246
256
|
state = stateFor(conn);
|
|
247
257
|
return [
|
|
248
258
|
4,
|
|
249
|
-
enter(conn, state)
|
|
259
|
+
enter(conn, state, begin)
|
|
250
260
|
];
|
|
251
261
|
case 1:
|
|
252
262
|
_state.sent();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/transaction.ts"],"sourcesContent":["// Store-agnostic joining transaction helper: node:sqlite and DuckDB both support only\n// BEGIN/COMMIT/ROLLBACK, no SAVEPOINT -- nesting joins the outer transaction, sharing one semantics through exec(sql).\ninterface ExecConnection {\n exec(sql: string): Promise<void>;\n}\n\ninterface TxState {\n depth: number;\n rollbackOnly: boolean;\n}\n\n// Keyed by connection, not a module singleton: two connections can coexist in one process\n// (e.g. watch.ts alongside a CLI query) and must not share depth.\nconst states = new WeakMap<ExecConnection, TxState>();\n\nfunction stateFor(conn: ExecConnection): TxState {\n let state = states.get(conn);\n if (!state) {\n state = { depth: 0, rollbackOnly: false };\n states.set(conn, state);\n }\n return state;\n}\n\nasync function enter(conn: ExecConnection, state: TxState): Promise<void> {\n if (state.depth === 0) {\n await conn.exec(
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/transaction.ts"],"sourcesContent":["// Store-agnostic joining transaction helper: node:sqlite and DuckDB both support only\n// BEGIN/COMMIT/ROLLBACK, no SAVEPOINT -- nesting joins the outer transaction, sharing one semantics through exec(sql).\ninterface ExecConnection {\n exec(sql: string): Promise<void>;\n}\n\ninterface TxState {\n depth: number;\n rollbackOnly: boolean;\n}\n\n// Keyed by connection, not a module singleton: two connections can coexist in one process\n// (e.g. watch.ts alongside a CLI query) and must not share depth.\nconst states = new WeakMap<ExecConnection, TxState>();\n\nfunction stateFor(conn: ExecConnection): TxState {\n let state = states.get(conn);\n if (!state) {\n state = { depth: 0, rollbackOnly: false };\n states.set(conn, state);\n }\n return state;\n}\n\nasync function enter(conn: ExecConnection, state: TxState, begin: string): Promise<void> {\n if (state.depth === 0) {\n await conn.exec(begin);\n state.rollbackOnly = false;\n }\n state.depth++;\n}\n\n// Reaching depth 0 still rollback-only means an inner scope failed and an enclosing catch\n// swallowed it; commit here would persist a partial write, so this rolls back and throws instead.\nasync function leaveOk(conn: ExecConnection, state: TxState): Promise<void> {\n state.depth--;\n if (state.depth > 0) return;\n if (state.rollbackOnly) {\n await conn.exec('ROLLBACK');\n throw new Error('transaction rolled back: a nested scope failed');\n }\n await conn.exec('COMMIT');\n}\n\nasync function leaveErr(conn: ExecConnection, state: TxState): Promise<void> {\n state.rollbackOnly = true;\n state.depth--;\n if (state.depth === 0) await conn.exec('ROLLBACK');\n}\n\n// A writer takes its lock at BEGIN rather than on first write: SQLite will not retry a deferred\n// transaction that read first and then upgrades, so busy_timeout never applies and the second\n// concurrent command fails outright instead of queueing. Reads keep plain BEGIN, or every snapshot\n// read (map, peek) would serialize against writers. DuckDB has no IMMEDIATE and passes nothing.\nexport const BEGIN_WRITE = 'BEGIN IMMEDIATE';\n\n// Depth-0 opens the transaction; a nested call joins it and issues no SQL of its own. The outermost\n// call owns COMMIT/ROLLBACK, so join semantics never differ by call site.\nexport async function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>, begin = 'BEGIN'): Promise<T> {\n const state = stateFor(conn);\n await enter(conn, state, begin);\n let result: T;\n try {\n result = await fn();\n } catch (err) {\n await leaveErr(conn, state);\n throw err;\n }\n await leaveOk(conn, state);\n return result;\n}\n"],"names":["BEGIN_WRITE","withTransaction","states","WeakMap","stateFor","conn","state","get","depth","rollbackOnly","set","enter","begin","exec","leaveOk","Error","leaveErr","fn","result","err"],"mappings":"AAAA,sFAAsF;AACtF,uHAAuH;;;;;;;;;;;;QAqD1GA;eAAAA;;QAISC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA/CtB,0FAA0F;AAC1F,kEAAkE;AAClE,IAAMC,SAAS,IAAIC;AAEnB,SAASC,SAASC,IAAoB;IACpC,IAAIC,QAAQJ,OAAOK,GAAG,CAACF;IACvB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;YAAGC,cAAc;QAAM;QACxCP,OAAOQ,GAAG,CAACL,MAAMC;IACnB;IACA,OAAOA;AACT;AAEA,SAAeK,MAAMN,IAAoB,EAAEC,KAAc,EAAEM,KAAa;;;;;yBAClEN,CAAAA,MAAME,KAAK,KAAK,CAAA,GAAhBF;;;;oBACF;;wBAAMD,KAAKQ,IAAI,CAACD;;;oBAAhB;oBACAN,MAAMG,YAAY,GAAG;;;oBAEvBH,MAAME,KAAK;;;;;;IACb;;AAEA,0FAA0F;AAC1F,kGAAkG;AAClG,SAAeM,QAAQT,IAAoB,EAAEC,KAAc;;;;;oBACzDA,MAAME,KAAK;oBACX,IAAIF,MAAME,KAAK,GAAG,GAAG;;;yBACjBF,MAAMG,YAAY,EAAlBH;;;;oBACF;;wBAAMD,KAAKQ,IAAI,CAAC;;;oBAAhB;oBACA,MAAM,IAAIE,MAAM;;oBAElB;;wBAAMV,KAAKQ,IAAI,CAAC;;;oBAAhB;;;;;;IACF;;AAEA,SAAeG,SAASX,IAAoB,EAAEC,KAAc;;;;;oBAC1DA,MAAMG,YAAY,GAAG;oBACrBH,MAAME,KAAK;yBACPF,CAAAA,MAAME,KAAK,KAAK,CAAA,GAAhBF;;;;oBAAmB;;wBAAMD,KAAKQ,IAAI,CAAC;;;oBAAhB;;;;;;;;IACzB;;AAMO,IAAMb,cAAc;AAIpB,SAAeC,gBAAmBI,IAAoB,EAAEY,EAAoB;QAAEL,QAAAA,iEAAQ;;YACrFN,OAEFY,QAGKC;;;;oBALHb,QAAQF,SAASC;oBACvB;;wBAAMM,MAAMN,MAAMC,OAAOM;;;oBAAzB;;;;;;;;;oBAGW;;wBAAMK;;;oBAAfC,SAAS;;;;;;oBACFC;oBACP;;wBAAMH,SAASX,MAAMC;;;oBAArB;oBACA,MAAMa;;oBAER;;wBAAML,QAAQT,MAAMC;;;oBAApB;oBACA;;wBAAOY;;;;IACT"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n private stmt: TursoStatement;\n\n constructor(stmt: TursoStatement) {\n this.stmt = stmt;\n }\n\n async run(...params: unknown[]): Promise<RunResult> {\n return this.stmt.run(...params);\n }\n\n async get(...params: unknown[]): Promise<unknown> {\n return this.stmt.get(...params);\n }\n\n async all(...params: unknown[]): Promise<unknown[]> {\n return this.stmt.all(...params);\n }\n\n async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n yield* this.stmt.iterate(...params);\n }\n\n columns(): Array<{ name: string }> {\n return this.stmt.columns();\n }\n\n setReadBigInts(enabled: boolean): void {\n this.stmt.safeIntegers(enabled);\n }\n}\n\nexport function createConnection(db: Database): Connection {\n const conn: Connection = {\n async exec(sql: string): Promise<void> {\n await db.exec(sql);\n },\n async prepare(sql: string): Promise<Statement> {\n return new TursoStatementWrapper(await db.prepare(sql));\n },\n // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n if (paramRows.length === 0) return;\n await withTransaction(conn
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n private stmt: TursoStatement;\n\n constructor(stmt: TursoStatement) {\n this.stmt = stmt;\n }\n\n async run(...params: unknown[]): Promise<RunResult> {\n return this.stmt.run(...params);\n }\n\n async get(...params: unknown[]): Promise<unknown> {\n return this.stmt.get(...params);\n }\n\n async all(...params: unknown[]): Promise<unknown[]> {\n return this.stmt.all(...params);\n }\n\n async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n yield* this.stmt.iterate(...params);\n }\n\n columns(): Array<{ name: string }> {\n return this.stmt.columns();\n }\n\n setReadBigInts(enabled: boolean): void {\n this.stmt.safeIntegers(enabled);\n }\n}\n\nexport function createConnection(db: Database): Connection {\n const conn: Connection = {\n async exec(sql: string): Promise<void> {\n await db.exec(sql);\n },\n async prepare(sql: string): Promise<Statement> {\n return new TursoStatementWrapper(await db.prepare(sql));\n },\n // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n if (paramRows.length === 0) return;\n await withTransaction(\n conn,\n async () => {\n await db.batch(paramRows.map((args) => ({ sql, args })));\n },\n BEGIN_WRITE\n );\n },\n };\n return conn;\n}\n"],"names":["createConnection","TursoStatementWrapper","stmt","run","params","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","db","conn","exec","sql","prepare","runBatch","paramRows","length","withTransaction","batch","map","args","BEGIN_WRITE"],"mappings":";;;;+BA0CgBA;;;eAAAA;;;6BAzC6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO7C,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,sCAAN;;aAAMA,sBAGQC,IAAoB;gCAH5BD;QAIF,IAAI,CAACC,IAAI,GAAGA;;iBAJVD;IAOJ,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACC,GAAG,OAAb,YAAc,qBAAGC;;;QAC1B;;IAEA,OAAMC,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGD,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACG,GAAG,OAAb,YAAc,qBAAGD;;;QAC1B;;IAEA,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGF,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACI,GAAG,OAAb,YAAc,qBAAGF;;;QAC1B;;IAEA,OAAOG,OAEN,GAFD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGH,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAC1B;;;;wBAAP;;uCAAA,0CAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACK,OAAO,OAAjB,YAAkB,qBAAGH;;;wBAA5B;;;;;;QACF;;IAEAI,OAAAA,OAEC,GAFDA,SAAAA;QACE,OAAO,IAAI,CAACN,IAAI,CAACM,OAAO;IAC1B;IAEAC,OAAAA,cAEC,GAFDA,SAAAA,eAAeC,OAAgB;QAC7B,IAAI,CAACR,IAAI,CAACS,YAAY,CAACD;IACzB;WA7BIT;;AAgCC,SAASD,iBAAiBY,EAAY;IAC3C,IAAMC,OAAmB;QACjBC,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMH,GAAGE,IAAI,CAACC;;;4BAAd;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZd;4BAAsB;;gCAAMW,GAAGI,OAAO,CAACD;;;4BAAlD;;gCAAO,IAAA,CAAA,EAAA,MAAId;;oCAAsB;kCAAqB;;;;YACxD;;QAGMgB,UAFN,2FAA2F;QAC3F,uLAAuL;QACvL,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMC,IAAAA,8BAAe,EACnBP,MACA;;;;;oDACE;;wDAAMD,GAAGS,KAAK,CAACH,UAAUI,GAAG,CAAC,SAACC;mEAAU;gEAAER,KAAAA;gEAAKQ,MAAAA;4DAAK;;;;oDAApD;;;;;;oCACF;mCACAC,0BAAW;;;4BALb;;;;;;YAOF;;IACF;IACA,OAAOX;AACT"}
|
|
@@ -9,4 +9,6 @@ export interface OpenResult {
|
|
|
9
9
|
parsed: number;
|
|
10
10
|
warnings: string[];
|
|
11
11
|
}
|
|
12
|
+
export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
|
|
13
|
+
export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
|
|
12
14
|
export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;
|
|
@@ -9,4 +9,6 @@ export interface OpenResult {
|
|
|
9
9
|
parsed: number;
|
|
10
10
|
warnings: string[];
|
|
11
11
|
}
|
|
12
|
+
export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
|
|
13
|
+
export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
|
|
12
14
|
export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;
|
|
@@ -9,6 +9,12 @@ function _export(target, all) {
|
|
|
9
9
|
});
|
|
10
10
|
}
|
|
11
11
|
_export(exports, {
|
|
12
|
+
get CONTENT_FTS_DDL () {
|
|
13
|
+
return CONTENT_FTS_DDL;
|
|
14
|
+
},
|
|
15
|
+
get CONTENT_FTS_NAMES () {
|
|
16
|
+
return CONTENT_FTS_NAMES;
|
|
17
|
+
},
|
|
12
18
|
get DB_FILENAME () {
|
|
13
19
|
return DB_FILENAME;
|
|
14
20
|
},
|
|
@@ -165,11 +171,17 @@ function _ts_generator(thisArg, body) {
|
|
|
165
171
|
}
|
|
166
172
|
var DB_FILENAME = 'cache.turso.db';
|
|
167
173
|
var SCHEMA_VERSION = '3';
|
|
168
|
-
|
|
169
|
-
|
|
174
|
+
var CONTENT_FTS_DDL = [
|
|
175
|
+
"CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')",
|
|
176
|
+
"CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"
|
|
177
|
+
];
|
|
178
|
+
var CONTENT_FTS_NAMES = [
|
|
179
|
+
'content_fts',
|
|
180
|
+
'content_fts_ngram'
|
|
181
|
+
];
|
|
170
182
|
function ensureSchema(conn, cfg) {
|
|
171
183
|
return _async_to_generator(function() {
|
|
172
|
-
var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
|
|
184
|
+
var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, ddl, err, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, feature, err;
|
|
173
185
|
return _ts_generator(this, function(_state) {
|
|
174
186
|
switch(_state.label){
|
|
175
187
|
case 0:
|
|
@@ -185,136 +197,181 @@ function ensureSchema(conn, cfg) {
|
|
|
185
197
|
];
|
|
186
198
|
case 2:
|
|
187
199
|
_state.sent();
|
|
200
|
+
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
201
|
+
_state.label = 3;
|
|
202
|
+
case 3:
|
|
203
|
+
_state.trys.push([
|
|
204
|
+
3,
|
|
205
|
+
8,
|
|
206
|
+
9,
|
|
207
|
+
10
|
|
208
|
+
]);
|
|
209
|
+
_iterator = CONTENT_FTS_DDL[Symbol.iterator]();
|
|
210
|
+
_state.label = 4;
|
|
211
|
+
case 4:
|
|
212
|
+
if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
|
|
213
|
+
3,
|
|
214
|
+
7
|
|
215
|
+
];
|
|
216
|
+
ddl = _step.value;
|
|
188
217
|
return [
|
|
189
218
|
4,
|
|
190
|
-
conn.exec(
|
|
219
|
+
conn.exec(ddl)
|
|
191
220
|
];
|
|
192
|
-
case
|
|
221
|
+
case 5:
|
|
193
222
|
_state.sent();
|
|
223
|
+
_state.label = 6;
|
|
224
|
+
case 6:
|
|
225
|
+
_iteratorNormalCompletion = true;
|
|
194
226
|
return [
|
|
195
|
-
|
|
196
|
-
|
|
227
|
+
3,
|
|
228
|
+
4
|
|
197
229
|
];
|
|
198
|
-
case
|
|
199
|
-
|
|
230
|
+
case 7:
|
|
231
|
+
return [
|
|
232
|
+
3,
|
|
233
|
+
10
|
|
234
|
+
];
|
|
235
|
+
case 8:
|
|
236
|
+
err = _state.sent();
|
|
237
|
+
_didIteratorError = true;
|
|
238
|
+
_iteratorError = err;
|
|
239
|
+
return [
|
|
240
|
+
3,
|
|
241
|
+
10
|
|
242
|
+
];
|
|
243
|
+
case 9:
|
|
244
|
+
try {
|
|
245
|
+
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
246
|
+
_iterator.return();
|
|
247
|
+
}
|
|
248
|
+
} finally{
|
|
249
|
+
if (_didIteratorError) {
|
|
250
|
+
throw _iteratorError;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
return [
|
|
254
|
+
7
|
|
255
|
+
];
|
|
256
|
+
case 10:
|
|
200
257
|
return [
|
|
201
258
|
4,
|
|
202
259
|
conn.exec('CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))')
|
|
203
260
|
];
|
|
204
|
-
case
|
|
261
|
+
case 11:
|
|
205
262
|
_state.sent();
|
|
206
263
|
return [
|
|
207
264
|
4,
|
|
208
265
|
conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)')
|
|
209
266
|
];
|
|
210
|
-
case
|
|
267
|
+
case 12:
|
|
211
268
|
_state.sent();
|
|
212
|
-
|
|
213
|
-
_state.label =
|
|
214
|
-
case
|
|
269
|
+
_iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
|
|
270
|
+
_state.label = 13;
|
|
271
|
+
case 13:
|
|
215
272
|
_state.trys.push([
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
273
|
+
13,
|
|
274
|
+
20,
|
|
275
|
+
21,
|
|
276
|
+
22
|
|
220
277
|
]);
|
|
221
|
-
|
|
222
|
-
_state.label =
|
|
223
|
-
case
|
|
224
|
-
if (!!(
|
|
278
|
+
_iterator1 = (0, _indexts1.activeFeatures)(cfg)[Symbol.iterator]();
|
|
279
|
+
_state.label = 14;
|
|
280
|
+
case 14:
|
|
281
|
+
if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
|
|
225
282
|
3,
|
|
226
|
-
|
|
283
|
+
19
|
|
227
284
|
];
|
|
228
|
-
feature =
|
|
285
|
+
feature = _step1.value;
|
|
229
286
|
if (!(feature.name === 'embed')) return [
|
|
230
287
|
3,
|
|
231
|
-
|
|
288
|
+
16
|
|
232
289
|
];
|
|
233
290
|
return [
|
|
234
291
|
4,
|
|
235
292
|
conn.exec('CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB('.concat(_typests.STORE_DIMS, '), PRIMARY KEY ("path", chunk))'))
|
|
236
293
|
];
|
|
237
|
-
case
|
|
294
|
+
case 15:
|
|
238
295
|
_state.sent();
|
|
239
296
|
return [
|
|
240
297
|
3,
|
|
241
|
-
|
|
298
|
+
18
|
|
242
299
|
];
|
|
243
|
-
case
|
|
300
|
+
case 16:
|
|
244
301
|
return [
|
|
245
302
|
4,
|
|
246
303
|
feature.schema(conn)
|
|
247
304
|
];
|
|
248
|
-
case
|
|
305
|
+
case 17:
|
|
249
306
|
_state.sent();
|
|
250
|
-
_state.label =
|
|
251
|
-
case
|
|
252
|
-
|
|
307
|
+
_state.label = 18;
|
|
308
|
+
case 18:
|
|
309
|
+
_iteratorNormalCompletion1 = true;
|
|
253
310
|
return [
|
|
254
311
|
3,
|
|
255
|
-
|
|
312
|
+
14
|
|
256
313
|
];
|
|
257
|
-
case
|
|
314
|
+
case 19:
|
|
258
315
|
return [
|
|
259
316
|
3,
|
|
260
|
-
|
|
317
|
+
22
|
|
261
318
|
];
|
|
262
|
-
case
|
|
319
|
+
case 20:
|
|
263
320
|
err = _state.sent();
|
|
264
|
-
|
|
265
|
-
|
|
321
|
+
_didIteratorError1 = true;
|
|
322
|
+
_iteratorError1 = err;
|
|
266
323
|
return [
|
|
267
324
|
3,
|
|
268
|
-
|
|
325
|
+
22
|
|
269
326
|
];
|
|
270
|
-
case
|
|
327
|
+
case 21:
|
|
271
328
|
try {
|
|
272
|
-
if (!
|
|
273
|
-
|
|
329
|
+
if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
|
|
330
|
+
_iterator1.return();
|
|
274
331
|
}
|
|
275
332
|
} finally{
|
|
276
|
-
if (
|
|
277
|
-
throw
|
|
333
|
+
if (_didIteratorError1) {
|
|
334
|
+
throw _iteratorError1;
|
|
278
335
|
}
|
|
279
336
|
}
|
|
280
337
|
return [
|
|
281
338
|
7
|
|
282
339
|
];
|
|
283
|
-
case
|
|
340
|
+
case 22:
|
|
284
341
|
return [
|
|
285
342
|
4,
|
|
286
343
|
(0, _sharedts.getMeta)(conn, 'schema_version')
|
|
287
344
|
];
|
|
288
|
-
case
|
|
345
|
+
case 23:
|
|
289
346
|
if (!(_state.sent() === null)) return [
|
|
290
347
|
3,
|
|
291
|
-
|
|
348
|
+
25
|
|
292
349
|
];
|
|
293
350
|
return [
|
|
294
351
|
4,
|
|
295
352
|
(0, _sharedts.setMeta)(conn, 'schema_version', SCHEMA_VERSION)
|
|
296
353
|
];
|
|
297
|
-
case
|
|
354
|
+
case 24:
|
|
298
355
|
_state.sent();
|
|
299
|
-
_state.label =
|
|
300
|
-
case
|
|
356
|
+
_state.label = 25;
|
|
357
|
+
case 25:
|
|
301
358
|
return [
|
|
302
359
|
4,
|
|
303
360
|
(0, _sharedts.getMeta)(conn, 'features')
|
|
304
361
|
];
|
|
305
|
-
case
|
|
362
|
+
case 26:
|
|
306
363
|
if (!(_state.sent() === null)) return [
|
|
307
364
|
3,
|
|
308
|
-
|
|
365
|
+
28
|
|
309
366
|
];
|
|
310
367
|
return [
|
|
311
368
|
4,
|
|
312
369
|
(0, _sharedts.setMeta)(conn, 'features', (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES))
|
|
313
370
|
];
|
|
314
|
-
case
|
|
371
|
+
case 27:
|
|
315
372
|
_state.sent();
|
|
316
|
-
_state.label =
|
|
317
|
-
case
|
|
373
|
+
_state.label = 28;
|
|
374
|
+
case 28:
|
|
318
375
|
return [
|
|
319
376
|
2
|
|
320
377
|
];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n store: Store;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n await conn.exec(`CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`);\n await conn.exec(`CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n for (const feature of activeFeatures(cfg)) {\n // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n if (feature.name === 'embed') {\n await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n continue;\n }\n await feature.schema(conn);\n }\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n db: Database;\n conn: Connection;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n const stateDir = join(cfg.baseDir, STATE_DIR);\n mkdirSync(stateDir, { recursive: true });\n const dbPath = join(stateDir, DB_FILENAME);\n\n // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n let turso: Awaited<ReturnType<typeof tursoApi>>;\n try {\n turso = await tursoApi();\n } catch (err) {\n if (err instanceof SenseError) throw err;\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n }\n\n let db: Database;\n try {\n // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n } catch (err) {\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n }\n // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n // Windows. closed guards the catch against double-closing.\n let closed = false;\n try {\n const conn = createConnection(db);\n\n await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n const version = await getMeta(conn, 'schema_version');\n const features = await getMeta(conn, 'features');\n const wantFeatures = featureSignature(cfg, FEATURES);\n if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n console.error(`sense: ${reason}; rebuilding the index`);\n closed = true;\n await db.close();\n clearCache(cfg);\n return connect(cfg);\n }\n\n await ensureSchema(conn, cfg);\n\n // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n return { db, conn, cfg, dbPath, parsed, warnings };\n } catch (err) {\n if (!closed) await db.close();\n throw err;\n }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n rekeyChunkText(conn, store);\n return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAiBaA;eAAAA;;QAGAC;eAAAA;;QAwGSC;eAAAA;;;sBA5HI;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;wBACM;4BAEA;wBACO;2BACd;uBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAU9B,+FAA+F;AAC/F,2FAA2F;AAC3F,SAAeE,aAAaC,IAAgB,EAAEC,GAAW;;YAOlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBANX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,wIAAkJ,OAAXG,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMW,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQX,GAAmB;;YAClCY,UAEAC,QAIFC,OAGKC,KAKLC,IAKKD,MAKLE,QAwB4B,MAtBxBlB,MAIAmB,SACAC,UACAC,cAIEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBAtDHH,WAAWa,IAAAA,cAAI,EAACzB,IAAI0B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUjB;;;;;;;;;oBAMpB;;wBAAMmC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHnB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,6FAA6F;oBAC7F,2DAA2D;oBACvDhB,SAAS;;;;;;;;;oBAELlB,OAAOqC,IAAAA,8BAAgB,EAACpB;oBAE9B;;wBAAMjB,KAAKG,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACR,MAAM;;;oBAA9BmB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACR,MAAM;;;oBAA/BoB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,yFAAyF;oBACzF,iGAAiG;oBAC3FC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;oBACnHyC,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPjB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGuB,KAAK;;;oBAAd;oBACAC,IAAAA,mBAAU,EAACxC;oBACX;;wBAAOW,QAAQX;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMO,IAAAA,iBAAO,EAACR,MAAM;;;oBAA5CuB,gBAAgBmB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAM1C,KAAKG,IAAI,CAAC,AAAC,yBAA8E,OAAtDwC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAItB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMuB,IAAAA,sBAAS,EAAC9C,MAAMC,KAAKA,IAAI0B,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIjB,MAAAA;4BAAMC,KAAAA;4BAAKa,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGuB,KAAK;;;oBAAd;;;oBACb,MAAMxB;;;;;;;IAEV;;AAIO,SAAelB,UAAUG,GAAmB;;YACgB,MAAzDgB,IAAIjB,MAAW+C,aAAajC,QAAQU,QAAQC,UAC9CuB;;;;oBAD2D;;wBAAMpC,QAAQX;;;oBAAd,OAAA,eAAzDgB,KAAyD,KAAzDA,IAAIjB,OAAqD,KAArDA,MAAW+C,cAA0C,KAA/C9C,KAAkBa,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC9CuB,QAAQC,IAAAA,oBAAW,EAAChC,IAAIjB,MAAM+C,aAAaA,YAAYpB,OAAO;oBACpE,0FAA0F;oBAC1FuB,IAAAA,yBAAc,EAAClD,MAAMgD;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAO/C,KAAK8C;4BAAajC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n store: Store;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n for (const feature of activeFeatures(cfg)) {\n // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n if (feature.name === 'embed') {\n await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n continue;\n }\n await feature.schema(conn);\n }\n if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n db: Database;\n conn: Connection;\n cfg: ResolvedConfig;\n dbPath: string;\n parsed: number;\n warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n const stateDir = join(cfg.baseDir, STATE_DIR);\n mkdirSync(stateDir, { recursive: true });\n const dbPath = join(stateDir, DB_FILENAME);\n\n // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n let turso: Awaited<ReturnType<typeof tursoApi>>;\n try {\n turso = await tursoApi();\n } catch (err) {\n if (err instanceof SenseError) throw err;\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n }\n\n let db: Database;\n try {\n // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n } catch (err) {\n throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n }\n // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n // Windows. closed guards the catch against double-closing.\n let closed = false;\n try {\n const conn = createConnection(db);\n\n await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n const version = await getMeta(conn, 'schema_version');\n const features = await getMeta(conn, 'features');\n const wantFeatures = featureSignature(cfg, FEATURES);\n if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n console.error(`sense: ${reason}; rebuilding the index`);\n closed = true;\n await db.close();\n clearCache(cfg);\n return connect(cfg);\n }\n\n await ensureSchema(conn, cfg);\n\n // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n return { db, conn, cfg, dbPath, parsed, warnings };\n } catch (err) {\n if (!closed) await db.close();\n throw err;\n }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n rekeyChunkText(conn, store);\n return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","ddl","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAkCaA;eAAAA;;QAIAC;eAAAA;;QArBAC;eAAAA;;QAGAC;eAAAA;;QA+GSC;eAAAA;;;sBAnII;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;wBACM;4BAEA;wBACO;2BACd;uBACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAcvB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,SAAeI,aAAaC,IAAgB,EAAEC,GAAW;;YAGlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaV;;;2BAAb,6BAAA,QAAA;;;;oBAAMQ,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBC,IAAAA,wBAAc,EAACJ;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXG,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACR;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMY,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,YAAYW,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQZ,GAAmB;;YAClCa,UAEAC,QAIFC,OAGKC,KAKLC,IAKKD,MAKLE,QAwB4B,MAtBxBnB,MAIAoB,SACAC,UACAC,cAIEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBAtDHH,WAAWa,IAAAA,cAAI,EAAC1B,IAAI2B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUlB;;;;;;;;;oBAMpB;;wBAAMoC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHnB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,6FAA6F;oBAC7F,2DAA2D;oBACvDhB,SAAS;;;;;;;;;oBAELnB,OAAOsC,IAAAA,8BAAgB,EAACpB;oBAE9B;;wBAAMlB,KAAKI,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACT,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACT,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,yFAAyF;oBACzF,iGAAiG;oBAC3FC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;oBACnH0C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPjB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGuB,KAAK;;;oBAAd;oBACAC,IAAAA,mBAAU,EAACzC;oBACX;;wBAAOY,QAAQZ;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMQ,IAAAA,iBAAO,EAACT,MAAM;;;oBAA5CwB,gBAAgBmB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAM3C,KAAKI,IAAI,CAAC,AAAC,yBAA8E,OAAtDwC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAItB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMuB,IAAAA,sBAAS,EAAC/C,MAAMC,KAAKA,IAAI2B,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIlB,MAAAA;4BAAMC,KAAAA;4BAAKc,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGuB,KAAK;;;oBAAd;;;oBACb,MAAMxB;;;;;;;IAEV;;AAIO,SAAenB,UAAUG,GAAmB;;YACgB,MAAzDiB,IAAIlB,MAAWgD,aAAajC,QAAQU,QAAQC,UAC9CuB;;;;oBAD2D;;wBAAMpC,QAAQZ;;;oBAAd,OAAA,eAAzDiB,KAAyD,KAAzDA,IAAIlB,OAAqD,KAArDA,MAAWgD,cAA0C,KAA/C/C,KAAkBc,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC9CuB,QAAQC,IAAAA,oBAAW,EAAChC,IAAIlB,MAAMgD,aAAaA,YAAYpB,OAAO;oBACpE,0FAA0F;oBAC1FuB,IAAAA,yBAAc,EAACnD,MAAMiD;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAOhD,KAAK+C;4BAAajC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
|