pi-duckdb-search 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,576 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * duckdb-search-test.cjs — Test suite for the DuckDB session search extension.
4
+ *
5
+ * Tests the core database operations directly (without the pi extension runtime):
6
+ * 1. Database creation and schema
7
+ * 2. Content extraction (text + thinking + tool names)
8
+ * 3. BM25 keyword search with prepared statements
9
+ * 4. Cosine semantic search with prepared statements
10
+ * 5. Hybrid RRF search
11
+ * 6. session_read by path + line
12
+ * 7. Metadata persistence
13
+ * 8. SQL injection resistance
14
+ * 9. Edge cases (empty query, missing file, limit clamping)
15
+ * 10. Incremental build (new file + delete file)
16
+ *
17
+ * Run: node duckdb-search-test.cjs
18
+ * Exits 0 on success, 1 on any failure.
19
+ */
20
+
21
+ const { DuckDBInstance } = require("@duckdb/node-api");
22
+ const path = require("node:path");
23
+ const fs = require("node:fs");
24
+ const os = require("node:os");
25
+
26
+ const EMBED_DIM = 384;
27
+ const MAX_CONTENT_CHARS = 2000;
28
+
29
+ let passed = 0;
30
+ let failed = 0;
31
+ const failures = [];
32
+
33
+ function assert(cond, msg) {
34
+ if (cond) {
35
+ passed++;
36
+ } else {
37
+ failed++;
38
+ failures.push(msg);
39
+ console.error(` ✗ FAIL: ${msg}`);
40
+ }
41
+ }
42
+
43
+ function assertEq(actual, expected, msg) {
44
+ if (actual === expected) {
45
+ passed++;
46
+ } else {
47
+ failed++;
48
+ failures.push(`${msg} (expected ${expected}, got ${actual})`);
49
+ console.error(` ✗ FAIL: ${msg} (expected ${expected}, got ${actual})`);
50
+ }
51
+ }
52
+
53
+ const TEST_DIR = path.join(os.tmpdir(), "duckdb-search-test-" + Date.now());
54
+ const TEST_DB = path.join(TEST_DIR, "test.duckdb");
55
+ const TEST_SESSIONS = path.join(TEST_DIR, "sessions", "test-project");
56
+ const TEST_GLOB = path.join(TEST_SESSIONS, "**", "*.jsonl");
57
+
58
+ /** Create a fake session JSONL file for testing. */
59
+ function createSessionFile(name, entries) {
60
+ const filePath = path.join(TEST_SESSIONS, name);
61
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
62
+ const lines = entries.map((e) => JSON.stringify(e));
63
+ fs.writeFileSync(filePath, lines.join("\n") + "\n");
64
+ return filePath;
65
+ }
66
+
67
+ /** Content extraction SQL matching the extension. */
68
+ const CONTENT_EXTRACTION_SQL = `
69
+ CASE
70
+ WHEN json_type(json->'message'->'content') = 'ARRAY'
71
+ THEN substring(
72
+ concat(
73
+ COALESCE(array_to_string(json_extract_string(json->'message'->'content', '$[*].text'), ' '), ''),
74
+ COALESCE(array_to_string(json_extract_string(json->'message'->'content', '$[*].thinking'), ' '), ''),
75
+ COALESCE(array_to_string(json_extract_string(json->'message'->'content', '$[*].name'), ' '), '')
76
+ ),
77
+ 1, ${MAX_CONTENT_CHARS}
78
+ )
79
+ ELSE substring(COALESCE(json->'message'->>'content', json->>'content', ''), 1, ${MAX_CONTENT_CHARS})
80
+ END
81
+ `;
82
+
83
+ async function buildTestDB(conn) {
84
+ await conn.run("INSTALL fts; LOAD fts;");
85
+ await conn.run("DROP TABLE IF EXISTS session_entries;");
86
+
87
+ const escapedGlob = TEST_GLOB.replace(/'/g, "''");
88
+
89
+ await conn.run(`
90
+ CREATE TABLE session_entries AS
91
+ SELECT
92
+ filename,
93
+ line_number,
94
+ json->>'type' AS entry_type,
95
+ json->>'id' AS entry_id,
96
+ json->>'parentId' AS parent_id,
97
+ json->>'timestamp' AS timestamp,
98
+ json->'message'->>'role' AS role,
99
+ json->>'customType' AS custom_type,
100
+ ${CONTENT_EXTRACTION_SQL} AS content_text
101
+ FROM (
102
+ SELECT
103
+ filename,
104
+ row_number() OVER (PARTITION BY filename ORDER BY filename) AS line_number,
105
+ json
106
+ FROM read_json_objects('${escapedGlob}', format='newline_delimited', filename=true, ignore_errors=true)
107
+ )
108
+ WHERE json->>'type' IN ('message', 'custom_message')
109
+ `);
110
+
111
+ await conn.run("PRAGMA create_fts_index('session_entries', 'entry_id', 'content_text', 'role', stemmer='porter', overwrite=true);");
112
+ await conn.run(`ALTER TABLE session_entries ADD COLUMN IF NOT EXISTS embedding FLOAT[${EMBED_DIM}];`);
113
+
114
+ // Add fake embeddings for testing (deterministic, not random)
115
+ const result = await conn.run("SELECT entry_id, content_text FROM session_entries WHERE embedding IS NULL AND length(content_text) > 0 ORDER BY rowid;");
116
+ const rows = await result.getRows();
117
+
118
+ await conn.run(`CREATE TABLE IF NOT EXISTS _emb_temp (entry_id VARCHAR, embedding FLOAT[${EMBED_DIM}]);`);
119
+ await conn.run("DELETE FROM _emb_temp;");
120
+
121
+ for (const row of rows) {
122
+ const entryId = row[0];
123
+ const text = String(row[1] || "");
124
+ // Deterministic fake embedding: hash of text -> 384 dims
125
+ const embedding = new Float32Array(EMBED_DIM);
126
+ for (let i = 0; i < EMBED_DIM; i++) {
127
+ embedding[i] = Math.sin(text.charCodeAt(i % text.length) + i) * 0.1;
128
+ }
129
+ // Normalize
130
+ let norm = 0;
131
+ for (const v of embedding) norm += v * v;
132
+ norm = Math.sqrt(norm);
133
+ if (norm > 0) for (let i = 0; i < EMBED_DIM; i++) embedding[i] /= norm;
134
+
135
+ const arrStr = `[${Array.from(embedding).map((v) => v.toFixed(6)).join(",")}]`;
136
+ await conn.run(
137
+ `INSERT INTO _emb_temp VALUES (?, ?::FLOAT[${EMBED_DIM}])`,
138
+ [entryId, arrStr]
139
+ );
140
+ }
141
+
142
+ await conn.run("UPDATE session_entries SET embedding = (SELECT embedding FROM _emb_temp WHERE _emb_temp.entry_id = session_entries.entry_id) WHERE entry_id IN (SELECT entry_id FROM _emb_temp);");
143
+ await conn.run("DROP TABLE _emb_temp;");
144
+
145
+ // Metadata
146
+ await conn.run("CREATE TABLE IF NOT EXISTS _metadata (key VARCHAR PRIMARY KEY, value VARCHAR);");
147
+ const entryCount = Number((await (await conn.run("SELECT COUNT(*) FROM session_entries")).getRows())[0][0]);
148
+ const sessionCount = Number((await (await conn.run("SELECT COUNT(DISTINCT filename) FROM session_entries")).getRows())[0][0]);
149
+ await conn.run("INSERT OR REPLACE INTO _metadata VALUES (?, ?)", ["entry_count", String(entryCount)]);
150
+ await conn.run("INSERT OR REPLACE INTO _metadata VALUES (?, ?)", ["session_count", String(sessionCount)]);
151
+ await conn.run("INSERT OR REPLACE INTO _metadata VALUES (?, ?)", ["has_embeddings", "true"]);
152
+
153
+ await conn.run("CHECKPOINT;");
154
+ return { entryCount, sessionCount };
155
+ }
156
+
157
+ async function main() {
158
+ console.log("=== DuckDB Session Search Test Suite ===\n");
159
+
160
+ // Setup test directory
161
+ fs.mkdirSync(TEST_DIR, { recursive: true });
162
+
163
+ // Create test session files
164
+ const file1 = createSessionFile("session1.jsonl", [
165
+ { type: "session", version: 3, id: "s1", timestamp: "2026-01-01T00:00:00Z", cwd: "/test" },
166
+ { type: "message", id: "m1", parentId: null, timestamp: "2026-01-01T00:00:01Z", message: { role: "user", content: "How do I configure BGP unnumbered on Juniper?" } },
167
+ { type: "message", id: "m2", parentId: "m1", timestamp: "2026-01-01T00:00:02Z", message: { role: "assistant", content: [
168
+ { type: "thinking", thinking: "The user wants BGP unnumbered config. I should check RFC 5549." },
169
+ { type: "text", text: "BGP unnumbered uses RFC 5549 extended nexthop. Configure it with family inet6 ipv6-nd." }
170
+ ]}},
171
+ { type: "message", id: "m3", parentId: "m2", timestamp: "2026-01-01T00:00:03Z", message: { role: "assistant", content: [
172
+ { type: "toolCall", name: "junos_cli", input: { command: "show bgp summary" } }
173
+ ]}},
174
+ { type: "message", id: "m4", parentId: "m3", timestamp: "2026-01-01T00:00:04Z", message: { role: "toolResult", content: "BGP summary: 0 peers configured" } },
175
+ ]);
176
+
177
+ const file2 = createSessionFile("session2.jsonl", [
178
+ { type: "session", version: 3, id: "s2", timestamp: "2026-01-02T00:00:00Z", cwd: "/test" },
179
+ { type: "message", id: "m5", parentId: null, timestamp: "2026-01-02T00:00:01Z", message: { role: "user", content: "DuckDB FTS index optimization for large datasets" } },
180
+ { type: "message", id: "m6", parentId: "m5", timestamp: "2026-01-02T00:00:02Z", message: { role: "assistant", content: "Use PRAGMA create_fts_index with porter stemmer for better matching." } },
181
+ { type: "message", id: "m7", parentId: "m6", timestamp: "2026-01-02T00:00:03Z", message: { role: "user", content: "What about cosine similarity for embeddings?" } },
182
+ ]);
183
+
184
+ const file3 = createSessionFile("session3.jsonl", [
185
+ { type: "session", version: 3, id: "s3", timestamp: "2026-01-03T00:00:00Z", cwd: "/test" },
186
+ // Entry with only thinking (no text) — should still get content extracted
187
+ { type: "message", id: "m8", parentId: null, timestamp: "2026-01-03T00:00:01Z", message: { role: "assistant", content: [
188
+ { type: "thinking", thinking: "I need to check the EVPN configuration on the spine switch." }
189
+ ]}},
190
+ // Error response with empty content array — should have empty content_text
191
+ { type: "message", id: "m9", parentId: "m8", timestamp: "2026-01-03T00:00:02Z", message: { role: "assistant", content: [], stopReason: "error", errorMessage: "Internal Error" } },
192
+ ]);
193
+
194
+ // Create DB
195
+ console.log("Building test database...");
196
+ const inst = await DuckDBInstance.create(TEST_DB);
197
+ const conn = await inst.connect();
198
+ const { entryCount, sessionCount } = await buildTestDB(conn);
199
+ console.log(`Built: ${entryCount} entries, ${sessionCount} sessions\n`);
200
+
201
+ // === Test 1: Schema and content extraction ===
202
+ console.log("Test 1: Schema and content extraction");
203
+ {
204
+ const r = await conn.run("SELECT COUNT(*) FROM session_entries");
205
+ assertEq(Number((await r.getRows())[0][0]), 9, "9 entries (3 sessions + 6 messages)");
206
+
207
+ // Content extraction: thinking-only entry should have thinking text
208
+ const r2 = await conn.run("SELECT content_text FROM session_entries WHERE entry_id = 'm8'");
209
+ const m8Content = (await r2.getRows())[0][0];
210
+ assert(String(m8Content).includes("EVPN configuration"), "Thinking content extracted for m8");
211
+
212
+ // Error response should have empty content
213
+ const r3 = await conn.run("SELECT content_text FROM session_entries WHERE entry_id = 'm9'");
214
+ const m9Content = (await r3.getRows())[0][0];
215
+ assertEq(String(m9Content), "", "Error response m9 has empty content_text");
216
+
217
+ // Tool call name should be in content
218
+ const r4 = await conn.run("SELECT content_text FROM session_entries WHERE entry_id = 'm3'");
219
+ const m3Content = (await r4.getRows())[0][0];
220
+ assert(String(m3Content).includes("junos_cli"), "Tool call name extracted for m3");
221
+ }
222
+ console.log(" ✓ Schema and content extraction passed\n");
223
+
224
+ // === Test 2: BM25 search with prepared statements ===
225
+ console.log("Test 2: BM25 keyword search");
226
+ {
227
+ const r = await conn.run(
228
+ `SELECT entry_id, fts_main_session_entries.match_bm25(entry_id, ?) AS score
229
+ FROM session_entries
230
+ WHERE fts_main_session_entries.match_bm25(entry_id, ?) IS NOT NULL
231
+ ORDER BY score DESC LIMIT ?`,
232
+ ["BGP unnumbered", "BGP unnumbered", 10]
233
+ );
234
+ const rows = await r.getRows();
235
+ assert(rows.length > 0, "BM25 finds results for 'BGP unnumbered'");
236
+ // Top result should be m1 or m2 (both contain "BGP unnumbered")
237
+ assert(["m1", "m2"].includes(String(rows[0][0])), `Top BM25 result is m1 or m2 (got ${rows[0][0]})`);
238
+ }
239
+ console.log(" ✓ BM25 search passed\n");
240
+
241
+ // === Test 3: Cosine search with prepared statements ===
242
+ console.log("Test 3: Semantic cosine search");
243
+ {
244
+ // Create a query vector similar to m2's embedding
245
+ const queryText = "BGP unnumbered RFC 5549 extended nexthop";
246
+ const embedding = new Float32Array(EMBED_DIM);
247
+ for (let i = 0; i < EMBED_DIM; i++) {
248
+ embedding[i] = Math.sin(queryText.charCodeAt(i % queryText.length) + i) * 0.1;
249
+ }
250
+ let norm = 0;
251
+ for (const v of embedding) norm += v * v;
252
+ norm = Math.sqrt(norm);
253
+ if (norm > 0) for (let i = 0; i < EMBED_DIM; i++) embedding[i] /= norm;
254
+ const arrStr = `[${Array.from(embedding).map((v) => v.toFixed(6)).join(",")}]`;
255
+
256
+ const r = await conn.run(
257
+ `SELECT entry_id, array_cosine_similarity(embedding, ?::FLOAT[${EMBED_DIM}]) AS sim
258
+ FROM session_entries
259
+ WHERE embedding IS NOT NULL
260
+ ORDER BY sim DESC LIMIT ?`,
261
+ [arrStr, 5]
262
+ );
263
+ const rows = await r.getRows();
264
+ assert(rows.length > 0, "Cosine search returns results");
265
+ assert(Number(rows[0][1]) > 0, "Top cosine similarity is positive");
266
+ }
267
+ console.log(" ✓ Semantic search passed\n");
268
+
269
+ // === Test 4: Hybrid RRF ===
270
+ console.log("Test 4: Hybrid RRF search");
271
+ {
272
+ const bm25R = await conn.run(
273
+ `SELECT entry_id FROM session_entries
274
+ WHERE fts_main_session_entries.match_bm25(entry_id, ?) IS NOT NULL
275
+ ORDER BY fts_main_session_entries.match_bm25(entry_id, ?) DESC LIMIT 50`,
276
+ ["BGP", "BGP"]
277
+ );
278
+ const bm25Rows = await bm25R.getRows();
279
+ assert(bm25Rows.length > 0, "Hybrid: BM25 subquery returns results");
280
+
281
+ const cosineR = await conn.run(
282
+ `SELECT entry_id FROM session_entries
283
+ WHERE embedding IS NOT NULL
284
+ ORDER BY array_cosine_similarity(embedding, (SELECT embedding FROM session_entries WHERE embedding IS NOT NULL LIMIT 1)) DESC
285
+ LIMIT 50`
286
+ );
287
+ const cosineRows = await cosineR.getRows();
288
+ assert(cosineRows.length > 0, "Hybrid: cosine subquery returns results");
289
+
290
+ // RRF fusion
291
+ const rrfScores = new Map();
292
+ const RRF_K = 60;
293
+ for (let i = 0; i < bm25Rows.length; i++) {
294
+ const id = String(bm25Rows[i][0]);
295
+ rrfScores.set(id, { score: 1 / (RRF_K + i + 1), bm25_rank: i + 1, cosine_rank: null });
296
+ }
297
+ for (let i = 0; i < cosineRows.length; i++) {
298
+ const id = String(cosineRows[i][0]);
299
+ const rrf = 1 / (RRF_K + i + 1);
300
+ const existing = rrfScores.get(id);
301
+ if (existing) {
302
+ existing.score += rrf;
303
+ existing.cosine_rank = i + 1;
304
+ } else {
305
+ rrfScores.set(id, { score: rrf, bm25_rank: null, cosine_rank: i + 1 });
306
+ }
307
+ }
308
+ const fused = Array.from(rrfScores.values()).sort((a, b) => b.score - a.score);
309
+ assert(fused.length > 0, "Hybrid: RRF fusion produces ranked results");
310
+ assert(fused[0].score > 0, "Hybrid: top RRF score is positive");
311
+ }
312
+ console.log(" ✓ Hybrid search passed\n");
313
+
314
+ // === Test 5: session_read ===
315
+ console.log("Test 5: session_read by path + line");
316
+ {
317
+ const r = await conn.run(
318
+ `WITH numbered AS (
319
+ SELECT row_number() OVER () AS rn, json
320
+ FROM read_json_objects(?, format='newline_delimited', ignore_errors=true)
321
+ )
322
+ SELECT json->>'type' AS entry_type, json->>'id' AS entry_id
323
+ FROM numbered WHERE rn = ? LIMIT 1`,
324
+ [file1, 1]
325
+ );
326
+ const rows = await r.getRows();
327
+ assert(rows.length > 0, "session_read returns entry at line 1");
328
+ assertEq(String(rows[0][0]), "session", "Line 1 is a session entry");
329
+ assertEq(String(rows[0][1]), "s1", "Session ID is s1");
330
+
331
+ // Line 2 should be the first message
332
+ const r2 = await conn.run(
333
+ `WITH numbered AS (
334
+ SELECT row_number() OVER () AS rn, json
335
+ FROM read_json_objects(?, format='newline_delimited', ignore_errors=true)
336
+ )
337
+ SELECT json->>'type' AS entry_type, json->'message'->>'role' AS role
338
+ FROM numbered WHERE rn = ? LIMIT 1`,
339
+ [file1, 2]
340
+ );
341
+ const rows2 = await r2.getRows();
342
+ assertEq(String(rows2[0][0]), "message", "Line 2 is a message");
343
+ assertEq(String(rows2[0][1]), "user", "Line 2 role is user");
344
+ }
345
+ console.log(" ✓ session_read passed\n");
346
+
347
+ // === Test 6: Metadata persistence ===
348
+ console.log("Test 6: Metadata persistence");
349
+ {
350
+ const r = await conn.run("SELECT value FROM _metadata WHERE key = ?", ["entry_count"]);
351
+ const count = Number((await r.getRows())[0][0]);
352
+ assertEq(count, entryCount, "Metadata entry_count matches");
353
+
354
+ const r2 = await conn.run("SELECT value FROM _metadata WHERE key = ?", ["session_count"]);
355
+ const sessCount = Number((await r2.getRows())[0][0]);
356
+ assertEq(sessCount, sessionCount, "Metadata session_count matches");
357
+
358
+ const r3 = await conn.run("SELECT value FROM _metadata WHERE key = ?", ["has_embeddings"]);
359
+ assertEq(String((await r3.getRows())[0][0]), "true", "Metadata has_embeddings is true");
360
+ }
361
+ console.log(" ✓ Metadata persistence passed\n");
362
+
363
+ // === Test 7: SQL injection resistance ===
364
+ console.log("Test 7: SQL injection resistance");
365
+ {
366
+ // Attempt SQL injection via query parameter
367
+ const maliciousQuery = "'; DROP TABLE session_entries; --";
368
+ const r = await conn.run(
369
+ `SELECT entry_id FROM session_entries
370
+ WHERE fts_main_session_entries.match_bm25(entry_id, ?) IS NOT NULL
371
+ LIMIT 1`,
372
+ [maliciousQuery]
373
+ );
374
+ const rows = await r.getRows();
375
+ // Table should still exist
376
+ const checkTable = await conn.run("SELECT COUNT(*) FROM session_entries");
377
+ assertEq(Number((await checkTable.getRows())[0][0]), entryCount, "session_entries table survives injection attempt");
378
+
379
+ // Attempt injection via path parameter
380
+ const maliciousPath = file1 + "'; DROP TABLE session_entries; --";
381
+ try {
382
+ const r2 = await conn.run(
383
+ `SELECT COUNT(*) FROM read_json_objects(?, format='newline_delimited', ignore_errors=true)`,
384
+ [maliciousPath]
385
+ );
386
+ // If it doesn't throw, table should still exist
387
+ const checkTable2 = await conn.run("SELECT COUNT(*) FROM session_entries");
388
+ assertEq(Number((await checkTable2.getRows())[0][0]), entryCount, "Table survives path injection");
389
+ } catch (e) {
390
+ // Expected: DuckDB should reject the malicious path
391
+ assert(true, "Path injection rejected by DuckDB");
392
+ }
393
+ }
394
+ console.log(" ✓ SQL injection resistance passed\n");
395
+
396
+ // === Test 8: Edge cases ===
397
+ console.log("Test 8: Edge cases");
398
+ {
399
+ // Empty query should return no results (not crash)
400
+ const r = await conn.run(
401
+ `SELECT entry_id FROM session_entries
402
+ WHERE fts_main_session_entries.match_bm25(entry_id, ?) IS NOT NULL
403
+ LIMIT ?`,
404
+ ["", 10]
405
+ );
406
+ const rows = await r.getRows();
407
+ // Empty query may return 0 results or all results depending on FTS behavior
408
+ assert(true, "Empty query doesn't crash");
409
+
410
+ // Limit clamping
411
+ const r2 = await conn.run(
412
+ `SELECT entry_id FROM session_entries
413
+ WHERE fts_main_session_entries.match_bm25(entry_id, ?) IS NOT NULL
414
+ LIMIT ?`,
415
+ ["BGP", 1000]
416
+ );
417
+ const rows2 = await r2.getRows();
418
+ assert(rows2.length <= 1000, "Limit clamped to reasonable value");
419
+
420
+ // Non-existent file
421
+ try {
422
+ await conn.run(
423
+ `SELECT COUNT(*) FROM read_json_objects(?, format='newline_delimited', ignore_errors=true)`,
424
+ ["/nonexistent/path/file.jsonl"]
425
+ );
426
+ // With ignore_errors, might return 0 or throw
427
+ assert(true, "Non-existent file handled gracefully");
428
+ } catch (e) {
429
+ assert(true, "Non-existent file throws error (expected)");
430
+ }
431
+
432
+ // Line beyond file
433
+ const r3 = await conn.run(
434
+ `WITH numbered AS (
435
+ SELECT row_number() OVER () AS rn, json
436
+ FROM read_json_objects(?, format='newline_delimited', ignore_errors=true)
437
+ )
438
+ SELECT COUNT(*) FROM numbered WHERE rn = ?`,
439
+ [file1, 999]
440
+ );
441
+ assertEq(Number((await r3.getRows())[0][0]), 0, "Line 999 returns 0 results");
442
+ }
443
+ console.log(" ✓ Edge cases passed\n");
444
+
445
+ // === Test 9: FTS overwrite ===
446
+ console.log("Test 9: FTS overwrite without drop");
447
+ {
448
+ // Create FTS index again with overwrite=true (no drop needed first)
449
+ const start = Date.now();
450
+ await conn.run("PRAGMA create_fts_index('session_entries', 'entry_id', 'content_text', 'role', stemmer='porter', overwrite=true);");
451
+ const elapsed = Date.now() - start;
452
+ assert(elapsed < 5000, `FTS overwrite in <5s (got ${elapsed}ms)`);
453
+
454
+ // Verify FTS still works
455
+ const r = await conn.run(
456
+ `SELECT entry_id FROM session_entries
457
+ WHERE fts_main_session_entries.match_bm25(entry_id, ?) IS NOT NULL
458
+ LIMIT ?`,
459
+ ["BGP", 5]
460
+ );
461
+ const rows = await r.getRows();
462
+ assert(rows.length > 0, "FTS works after overwrite");
463
+ }
464
+ console.log(" ✓ FTS overwrite passed\n");
465
+
466
+ // === Test 10: ignore_errors with malformed JSON ===
467
+ console.log("Test 10: ignore_errors with malformed JSON");
468
+ {
469
+ const malformedFile = path.join(TEST_SESSIONS, "malformed.jsonl");
470
+ fs.writeFileSync(malformedFile, [
471
+ JSON.stringify({ type: "message", id: "good1", message: { role: "user", content: "valid entry" } }),
472
+ "{invalid json line}",
473
+ JSON.stringify({ type: "message", id: "good2", message: { role: "assistant", content: "also valid" } }),
474
+ ].join("\n") + "\n");
475
+
476
+ const r = await conn.run(
477
+ `SELECT COUNT(*) FROM read_json_objects(?, format='newline_delimited', ignore_errors=true) WHERE json IS NOT NULL`,
478
+ [malformedFile]
479
+ );
480
+ const count = Number((await r.getRows())[0][0]);
481
+ assertEq(count, 2, "ignore_errors: 2 valid (non-null) entries read, malformed line is null");
482
+
483
+ // Without ignore_errors, should fail
484
+ try {
485
+ await conn.run(
486
+ `SELECT COUNT(*) FROM read_json_objects(?, format='newline_delimited')`,
487
+ [malformedFile]
488
+ );
489
+ assert(false, "Should have thrown on malformed JSON without ignore_errors");
490
+ } catch (e) {
491
+ assert(true, "Without ignore_errors, malformed JSON throws");
492
+ }
493
+ }
494
+ console.log(" ✓ ignore_errors passed\n");
495
+
496
+ // === Test 11: Content truncation ===
497
+ console.log("Test 11: Content truncation");
498
+ {
499
+ // Create entry with very long content
500
+ const longContent = "A".repeat(5000);
501
+ const longFile = path.join(TEST_SESSIONS, "long.jsonl");
502
+ fs.writeFileSync(longFile, JSON.stringify({
503
+ type: "message", id: "long1",
504
+ message: { role: "user", content: longContent }
505
+ }) + "\n");
506
+
507
+ const escapedPath = longFile.replace(/'/g, "''");
508
+ const r = await conn.run(`
509
+ SELECT length(${CONTENT_EXTRACTION_SQL}) AS content_len
510
+ FROM read_json_objects('${escapedPath}', format='newline_delimited')
511
+ WHERE json->>'type' = 'message'
512
+ `);
513
+ const len = Number((await r.getRows())[0][0]);
514
+ assertEq(len, MAX_CONTENT_CHARS, `Content truncated to ${MAX_CONTENT_CHARS} chars (got ${len})`);
515
+ }
516
+ console.log(" ✓ Content truncation passed\n");
517
+
518
+ // === Test 12: Read-only mode ===
519
+ console.log("Test 12: Read-only mode");
520
+ {
521
+ await conn.disconnectSync();
522
+ inst.closeSync();
523
+
524
+ const roInst = await DuckDBInstance.create(TEST_DB, { access_mode: "READ_ONLY" });
525
+ const roConn = await roInst.connect();
526
+ await roConn.run("LOAD fts;");
527
+
528
+ // Read should work
529
+ const r = await roConn.run("SELECT COUNT(*) FROM session_entries");
530
+ assertEq(Number((await r.getRows())[0][0]), entryCount, "Read-only: SELECT works");
531
+
532
+ // Write should fail
533
+ try {
534
+ await roConn.run("INSERT INTO session_entries VALUES ('test', 1, 'test', 'test', null, null, 'user', null, 'test')");
535
+ assert(false, "Read-only: INSERT should fail");
536
+ } catch (e) {
537
+ assert(true, "Read-only: INSERT correctly rejected");
538
+ }
539
+
540
+ // FTS search should work in read-only
541
+ const r2 = await roConn.run(
542
+ `SELECT entry_id FROM session_entries
543
+ WHERE fts_main_session_entries.match_bm25(entry_id, ?) IS NOT NULL
544
+ LIMIT ?`,
545
+ ["BGP", 5]
546
+ );
547
+ assert((await r2.getRows()).length > 0, "Read-only: FTS search works");
548
+
549
+ roConn.disconnectSync();
550
+ roInst.closeSync();
551
+ }
552
+ console.log(" ✓ Read-only mode passed\n");
553
+
554
+ // === Results ===
555
+ console.log("=== Results ===");
556
+ console.log(` Passed: ${passed}`);
557
+ console.log(` Failed: ${failed}`);
558
+ if (failures.length > 0) {
559
+ console.log("\nFailures:");
560
+ for (const f of failures) console.log(` - ${f}`);
561
+ }
562
+
563
+ // Cleanup
564
+ try {
565
+ fs.rmSync(TEST_DIR, { recursive: true, force: true });
566
+ } catch {}
567
+
568
+ process.exit(failed > 0 ? 1 : 0);
569
+ }
570
+
571
+ main().catch((e) => {
572
+ console.error("FATAL:", e.message);
573
+ console.error(e.stack);
574
+ try { fs.rmSync(TEST_DIR, { recursive: true, force: true }); } catch {}
575
+ process.exit(1);
576
+ });