tinywebui 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +20 -13
  2. package/bin/tinywebui.js +69 -63
  3. package/package.json +25 -8
  4. package/public/app.js +40 -14
  5. package/public/chat/composer.js +1 -1
  6. package/public/chat/sidebar.js +8 -5
  7. package/public/index.html +88 -49
  8. package/public/panels/admin.js +40 -1
  9. package/public/panels/model-editor.js +336 -0
  10. package/public/panels/panels.js +37 -12
  11. package/public/panels/settings-home.js +125 -0
  12. package/public/panels/settings.js +124 -36
  13. package/public/shared/attachment_note.js +7 -1
  14. package/public/shared/events.js +1 -0
  15. package/public/shared/features.js +1 -0
  16. package/public/shared/origins.js +1 -0
  17. package/public/shared/usage.js +16 -1
  18. package/public/styles.css +83 -20
  19. package/src/access/auth.js +155 -155
  20. package/src/access/auth_gate.js +39 -38
  21. package/src/access/email.js +17 -0
  22. package/src/access/policy.js +123 -138
  23. package/src/audit.js +1 -1
  24. package/src/automations/automation.js +135 -123
  25. package/src/automations/scheduler.js +138 -127
  26. package/src/chat/compact.js +149 -156
  27. package/src/chat/llm.js +380 -382
  28. package/src/chat/runs.js +162 -151
  29. package/src/chat/tool_executor.js +229 -218
  30. package/src/cli.js +535 -433
  31. package/src/config/approval.js +35 -29
  32. package/src/config/config.js +546 -451
  33. package/src/config/data_dir.js +24 -0
  34. package/src/config/models.js +191 -115
  35. package/src/config/refs.js +32 -0
  36. package/src/config/secrets.js +153 -0
  37. package/src/files/documents.js +57 -68
  38. package/src/files/images.js +30 -31
  39. package/src/harness/agent.js +60 -60
  40. package/src/harness/attribution.js +66 -58
  41. package/src/harness/checkpoint.js +67 -62
  42. package/src/harness/history.js +28 -23
  43. package/src/harness/prompt.js +40 -57
  44. package/src/harness/provider.js +284 -283
  45. package/src/harness/types.js +1 -0
  46. package/src/http.js +98 -66
  47. package/src/log.js +11 -12
  48. package/src/mcp.js +398 -374
  49. package/src/retrieval/embedding.js +228 -240
  50. package/src/retrieval/retrieval.js +182 -173
  51. package/src/routes/admin.js +178 -138
  52. package/src/routes/auth.js +184 -100
  53. package/src/routes/automations.js +99 -82
  54. package/src/routes/chats.js +240 -218
  55. package/src/routes/documents.js +130 -120
  56. package/src/routes/library.js +43 -46
  57. package/src/routes/mcp.js +49 -49
  58. package/src/routes/settings.js +127 -122
  59. package/src/routes/types.js +1 -0
  60. package/src/server.js +428 -392
  61. package/src/store/automations.js +106 -106
  62. package/src/store/chats.js +217 -242
  63. package/src/store/documents.js +181 -185
  64. package/src/store/embeddings.js +52 -54
  65. package/src/store/index.js +95 -49
  66. package/src/store/messages.js +55 -80
  67. package/src/store/schema.js +128 -107
  68. package/src/store/scope.js +18 -17
  69. package/src/store/search.js +25 -27
  70. package/src/store/turns.js +53 -53
  71. package/src/store/usage.js +343 -269
  72. package/src/store/users.js +162 -87
  73. package/src/store/wire.js +50 -41
  74. package/src/tools/ask_tool.js +68 -68
  75. package/src/tools/chat_search_tool.js +38 -44
  76. package/src/tools/context_tool.js +131 -138
  77. package/src/tools/document_tool.js +73 -78
  78. package/src/tools/mcp_tool.js +274 -232
  79. package/src/tools/task_tool.js +53 -50
  80. package/src/tools/types.js +1 -0
  81. package/public/panels/connectors.js +0 -184
  82. package/src/set-password.js +0 -43
@@ -1,5 +1,5 @@
1
1
  import { DatabaseSync } from 'node:sqlite';
2
- import { mkdirSync } from 'node:fs';
2
+ import { chmodSync, closeSync, existsSync, mkdirSync, openSync } from 'node:fs';
3
3
  import { dirname } from 'node:path';
4
4
  import { SCHEMA_VERSION, migrate as migrateSchema } from './schema.js';
5
5
  import { ChatStore } from './chats.js';
@@ -11,14 +11,12 @@ import { EmbeddingStore } from './embeddings.js';
11
11
  import { TurnStore } from './turns.js';
12
12
  import { SearchStore } from './search.js';
13
13
  import { UserStore } from './users.js';
14
-
15
14
  export { SCHEMA_VERSION } from './schema.js';
16
15
  export { ALL_USERS } from './scope.js';
17
16
  export { PASSAGE_SIZE, PASSAGE_OVERLAP } from './documents.js';
18
17
  export { toWire, toView } from './wire.js';
19
18
  export { CORPUS } from './embeddings.js';
20
19
  export { AUX_KIND } from './usage.js';
21
-
22
20
  /**
23
21
  * Conversation storage.
24
22
  *
@@ -37,54 +35,102 @@ export { AUX_KIND } from './usage.js';
37
35
  * file of the same name next to this one. An area that needs another is
38
36
  * handed it here, so every dependency between areas is listed in one place.
39
37
  */
40
-
38
+ /**
39
+ * Conversations, password hashes and session hashes: readable by the owner
40
+ * only, like the config. A new file is created that way before SQLite opens
41
+ * it, since SQLite gives its -wal and -shm files the database's own mode; an
42
+ * existing one is tightened. A file someone else owns keeps the mode they set.
43
+ */
44
+ function ownerOnly(path) {
45
+ if (!existsSync(path))
46
+ closeSync(openSync(path, 'a', 0o600));
47
+ if (process.platform === 'win32')
48
+ return;
49
+ for (const p of [path, `${path}-wal`, `${path}-shm`]) {
50
+ try {
51
+ if (existsSync(p))
52
+ chmodSync(p, 0o600);
53
+ }
54
+ catch { /* not ours to change */ }
55
+ }
56
+ }
41
57
  /** Thrown when a database is behind and migrating was not allowed. */
42
- export class MigrationRequiredError extends Error {}
43
-
58
+ export class MigrationRequiredError extends Error {
59
+ }
44
60
  export class Store {
45
- /**
46
- * `migrate: false` refuses to touch an existing database that is behind
47
- * SCHEMA_VERSION instead of upgrading it on open; a brand-new file is
48
- * always created at the current version.
49
- */
50
- constructor(path, { migrate = true } = {}) {
51
- if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true });
52
- this.db = new DatabaseSync(path);
53
- const found = this.db.prepare('PRAGMA user_version').get().user_version;
54
- const fresh = !this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chats'").get();
55
- if (!fresh && found > SCHEMA_VERSION) {
56
- this.db.close();
57
- throw new MigrationRequiredError(`database is at schema ${found}, newer than this TinyWebUI (${SCHEMA_VERSION}); upgrade TinyWebUI`);
61
+ db;
62
+ migratedFrom;
63
+ embeddings;
64
+ messages;
65
+ automations;
66
+ documents;
67
+ chats;
68
+ turns;
69
+ usage;
70
+ search;
71
+ users;
72
+ /**
73
+ * `migrate: false` refuses to touch an existing database that is behind
74
+ * SCHEMA_VERSION instead of upgrading it on open; a brand-new file is
75
+ * always created at the current version.
76
+ */
77
+ constructor(path, { migrate = true } = {}) {
78
+ if (path !== ':memory:') {
79
+ mkdirSync(dirname(path), { recursive: true });
80
+ ownerOnly(path);
81
+ }
82
+ this.db = new DatabaseSync(path);
83
+ const found = this.db.prepare('PRAGMA user_version').get().user_version;
84
+ const fresh = !this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chats'").get();
85
+ if (!fresh && found > SCHEMA_VERSION) {
86
+ this.db.close();
87
+ throw new MigrationRequiredError(`database is at schema ${found}, newer than this TinyWebUI (${SCHEMA_VERSION}); upgrade TinyWebUI`);
88
+ }
89
+ if (!fresh && found < SCHEMA_VERSION && !migrate) {
90
+ this.db.close();
91
+ throw new MigrationRequiredError(`database is at schema ${found}, needs ${SCHEMA_VERSION}: run \`tinywebui migrate\``);
92
+ }
93
+ this.migratedFrom = fresh ? null : found;
94
+ migrateSchema(this.db, { fresh });
95
+ const { db } = this;
96
+ this.embeddings = new EmbeddingStore(db);
97
+ this.messages = new MessageStore(db);
98
+ this.automations = new AutomationStore(db);
99
+ this.documents = new DocumentStore(db, { embeddings: this.embeddings });
100
+ this.chats = new ChatStore(db, { messages: this.messages, automations: this.automations, documents: this.documents, embeddings: this.embeddings });
101
+ this.turns = new TurnStore(db, { chats: this.chats });
102
+ this.usage = new UsageStore(db);
103
+ this.search = new SearchStore(db);
104
+ this.users = new UserStore(db);
105
+ }
106
+ schemaVersion() {
107
+ return this.db.prepare('PRAGMA user_version').get().user_version;
108
+ }
109
+ /** Gives pre-account, unowned data to the first local administrator. */
110
+ claimUnowned(userId, { removeUserId = null } = {}) {
111
+ this.db.exec('BEGIN');
112
+ try {
113
+ this.chats.claimUnowned(userId);
114
+ this.documents.claimUnowned(userId);
115
+ this.messages.claimUnownedArtifacts(userId);
116
+ this.automations.claimUnowned(userId);
117
+ if (removeUserId)
118
+ this.users.removePasswordless(removeUserId);
119
+ this.db.exec('COMMIT');
120
+ }
121
+ catch (err) {
122
+ this.db.exec('ROLLBACK');
123
+ throw err;
124
+ }
125
+ }
126
+ /** Cheap liveness probe of the database, for /readyz. */
127
+ ping() {
128
+ return this.db.prepare('SELECT 1 AS ok').get().ok === 1;
58
129
  }
59
- if (!fresh && found < SCHEMA_VERSION && !migrate) {
60
- this.db.close();
61
- throw new MigrationRequiredError(`database is at schema ${found}, needs ${SCHEMA_VERSION}: run \`tinywebui migrate\``);
130
+ close() {
131
+ try {
132
+ this.db.close();
133
+ }
134
+ catch { /* already gone */ }
62
135
  }
63
- this.migratedFrom = fresh ? null : found;
64
- migrateSchema(this.db, { fresh });
65
-
66
- const { db } = this;
67
- this.embeddings = new EmbeddingStore(db);
68
- this.messages = new MessageStore(db);
69
- this.automations = new AutomationStore(db);
70
- this.documents = new DocumentStore(db, { embeddings: this.embeddings });
71
- this.chats = new ChatStore(db, { messages: this.messages, automations: this.automations, documents: this.documents, embeddings: this.embeddings });
72
- this.turns = new TurnStore(db, { chats: this.chats });
73
- this.usage = new UsageStore(db);
74
- this.search = new SearchStore(db);
75
- this.users = new UserStore(db);
76
- }
77
-
78
- schemaVersion() {
79
- return this.db.prepare('PRAGMA user_version').get().user_version;
80
- }
81
-
82
- /** Cheap liveness probe of the database, for /readyz. */
83
- ping() {
84
- return this.db.prepare('SELECT 1 AS ok').get().ok === 1;
85
- }
86
-
87
- close() {
88
- try { this.db.close(); } catch { /* already gone */ }
89
- }
90
136
  }
@@ -1,93 +1,68 @@
1
1
  import { randomBytes } from 'node:crypto';
2
-
3
2
  // The transcript: messages in order, and the tool-result artifacts behind them.
4
3
  // One area of the Store; see index.js.
5
-
6
4
  export class MessageStore {
7
- constructor(db, deps = {}) {
8
- this.db = db;
9
- Object.assign(this, deps);
10
- }
11
-
12
- nextSeq(chatId) {
13
- const row = this.db.prepare('SELECT MAX(seq) AS m FROM messages WHERE chat_id = ?').get(chatId);
14
- return (row?.m ?? -1) + 1;
15
- }
16
-
17
- /** Drops a chat's messages from `seq` on (a rewind). Returns how many went. */
18
- deleteFrom(chatId, seq) {
19
- return this.db.prepare('DELETE FROM messages WHERE chat_id = ? AND seq >= ?').run(chatId, seq).changes;
20
- }
21
-
22
- /** Drops a chat's whole transcript and the artifacts behind it (the chat is going). */
23
- deleteForChat(chatId) {
24
- this.db.prepare('DELETE FROM messages WHERE chat_id = ?').run(chatId);
25
- this.db.prepare('DELETE FROM artifacts WHERE chat_id = ?').run(chatId);
26
- }
27
-
28
- /** Appends one message. Append-only by design: rows are never reordered. */
29
- add(chatId, msg) {
30
- const seq = this.nextSeq(chatId);
31
- this.db.prepare(`
5
+ db;
6
+ constructor(db) {
7
+ this.db = db;
8
+ }
9
+ claimUnownedArtifacts(userId) {
10
+ this.db.prepare('UPDATE artifacts SET user_id = ? WHERE user_id IS NULL').run(userId);
11
+ }
12
+ nextSeq(chatId) {
13
+ const row = this.db.prepare('SELECT MAX(seq) AS m FROM messages WHERE chat_id = ?').get(chatId);
14
+ return (row?.m ?? -1) + 1;
15
+ }
16
+ /** Drops a chat's messages from `seq` on (a rewind). Returns how many went. */
17
+ deleteFrom(chatId, seq) {
18
+ return this.db.prepare('DELETE FROM messages WHERE chat_id = ? AND seq >= ?').run(chatId, seq).changes;
19
+ }
20
+ /** Drops a chat's whole transcript and the artifacts behind it (the chat is going). */
21
+ deleteForChat(chatId) {
22
+ this.db.prepare('DELETE FROM messages WHERE chat_id = ?').run(chatId);
23
+ this.db.prepare('DELETE FROM artifacts WHERE chat_id = ?').run(chatId);
24
+ }
25
+ /** Appends one message. Append-only by design: rows are never reordered. */
26
+ add(chatId, msg) {
27
+ const seq = this.nextSeq(chatId);
28
+ this.db.prepare(`
32
29
  INSERT INTO messages
33
30
  (chat_id, seq, role, content, tool_call_id, tool_calls_json,
34
31
  reasoning, reasoning_details_json, artifact_id, stub_text, usage_json,
35
32
  model, created_at, images_json, origin_json)
36
33
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
37
- `).run(
38
- chatId, seq, msg.role,
39
- msg.content ?? null,
40
- msg.tool_call_id ?? null,
41
- msg.tool_calls ? JSON.stringify(msg.tool_calls) : null,
42
- msg.reasoning ?? null,
43
- msg.reasoning_details ? JSON.stringify(msg.reasoning_details) : null,
44
- msg.artifact_id ?? null,
45
- msg.stub_text ?? null,
46
- msg.usage ? JSON.stringify(msg.usage) : null,
47
- msg.model ?? null,
48
- Date.now(),
49
- msg.images ? JSON.stringify(msg.images) : null,
50
- msg.origin ? JSON.stringify(msg.origin) : null
51
- );
52
- return seq;
53
- }
54
-
55
- updateUsage(chatId, seq, usage) {
56
- this.db.prepare('UPDATE messages SET usage_json = ? WHERE chat_id = ? AND seq = ?')
57
- .run(JSON.stringify(usage), chatId, seq);
58
- }
59
-
60
- list(chatId) {
61
- return this.db.prepare(
62
- 'SELECT * FROM messages WHERE chat_id = ? ORDER BY seq ASC'
63
- ).all(chatId);
64
- }
65
-
66
- /**
67
- * Writes a stub onto an already-stored tool message. This is the ONE place
68
- * history is rewritten, and it happens only at an epoch boundary. `content`
69
- * is left untouched so the human transcript keeps the full output -- only
70
- * what goes to the model shrinks.
71
- */
72
- setStub(messageId, stubText) {
73
- this.db.prepare('UPDATE messages SET stub_text = ? WHERE id = ?').run(stubText, messageId);
74
- }
75
-
76
- /** Takes a message's images off the wire at an epoch; the transcript keeps them. */
77
- dropImages(messageId) {
78
- this.db.prepare('UPDATE messages SET images_dropped = 1 WHERE id = ?').run(messageId);
79
- }
80
-
81
- addArtifact(chatId, { toolName, args, content }) {
82
- const id = randomBytes(12).toString('hex');
83
- this.db.prepare(`
34
+ `).run(chatId, seq, msg.role, msg.content ?? null, msg.tool_call_id ?? null, msg.tool_calls ? JSON.stringify(msg.tool_calls) : null, msg.reasoning ?? null, msg.reasoning_details ? JSON.stringify(msg.reasoning_details) : null, msg.artifact_id ?? null, msg.stub_text ?? null, msg.usage ? JSON.stringify(msg.usage) : null, msg.model ?? null, Date.now(), msg.images ? JSON.stringify(msg.images) : null, msg.origin ? JSON.stringify(msg.origin) : null);
35
+ return seq;
36
+ }
37
+ updateUsage(chatId, seq, usage) {
38
+ this.db.prepare('UPDATE messages SET usage_json = ? WHERE chat_id = ? AND seq = ?')
39
+ .run(JSON.stringify(usage), chatId, seq);
40
+ }
41
+ list(chatId) {
42
+ return this.db.prepare('SELECT * FROM messages WHERE chat_id = ? ORDER BY seq ASC').all(chatId);
43
+ }
44
+ /**
45
+ * Writes a stub onto an already-stored tool message. This is the ONE place
46
+ * history is rewritten, and it happens only at an epoch boundary. `content`
47
+ * is left untouched so the human transcript keeps the full output -- only
48
+ * what goes to the model shrinks.
49
+ */
50
+ setStub(messageId, stubText) {
51
+ this.db.prepare('UPDATE messages SET stub_text = ? WHERE id = ?').run(stubText, messageId);
52
+ }
53
+ /** Takes a message's images off the wire at an epoch; the transcript keeps them. */
54
+ dropImages(messageId) {
55
+ this.db.prepare('UPDATE messages SET images_dropped = 1 WHERE id = ?').run(messageId);
56
+ }
57
+ addArtifact(chatId, { toolName, args, content }) {
58
+ const id = randomBytes(12).toString('hex');
59
+ this.db.prepare(`
84
60
  INSERT INTO artifacts (id, chat_id, tool_name, args_json, content, char_len, created_at)
85
61
  VALUES (?, ?, ?, ?, ?, ?, ?)
86
62
  `).run(id, chatId, toolName, JSON.stringify(args ?? {}), content, content.length, Date.now());
87
- return id;
88
- }
89
-
90
- getArtifact(id, chatId) {
91
- return this.db.prepare('SELECT * FROM artifacts WHERE id = ? AND chat_id = ?').get(id, chatId) || null;
92
- }
63
+ return id;
64
+ }
65
+ getArtifact(id, chatId) {
66
+ return this.db.prepare('SELECT * FROM artifacts WHERE id = ? AND chat_id = ?').get(id, chatId) || null;
67
+ }
93
68
  }
@@ -1,8 +1,6 @@
1
- /* The database shape: tables, triggers, the schema version, and the migrations that reach it. */
2
-
3
1
  export const SCHEMA = `
4
- -- Optional auth/RBAC (config authMode 'single'/'multiuser'). Unused and
5
- -- empty when authMode is explicitly 'none' -- these tables cost nothing
2
+ -- Optional auth/RBAC (config authMode 'multiuser'). Unused and
3
+ -- empty when authMode is explicitly 'local' -- these tables cost nothing
6
4
  -- to have around.
7
5
  CREATE TABLE IF NOT EXISTS users (
8
6
  id TEXT PRIMARY KEY,
@@ -25,6 +23,20 @@ CREATE TABLE IF NOT EXISTS sessions (
25
23
  expires_at INTEGER NOT NULL
26
24
  );
27
25
 
26
+ -- Local multi-user enrollment. The browser receives the raw invite once;
27
+ -- only its sha256 digest is stored, just like a session token.
28
+ CREATE TABLE IF NOT EXISTS invitations (
29
+ id TEXT PRIMARY KEY,
30
+ token_hash TEXT NOT NULL UNIQUE,
31
+ email TEXT,
32
+ created_by TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
33
+ created_at INTEGER NOT NULL,
34
+ expires_at INTEGER NOT NULL,
35
+ used_at INTEGER,
36
+ used_by TEXT REFERENCES users(id) ON DELETE SET NULL
37
+ );
38
+ CREATE INDEX IF NOT EXISTS invitations_active ON invitations(expires_at) WHERE used_at IS NULL;
39
+
28
40
  -- One-time migration flags (e.g. backfilling user_id onto pre-auth rows),
29
41
  -- so a backfill never accidentally reruns and clobbers real ownership later.
30
42
  CREATE TABLE IF NOT EXISTS meta (
@@ -315,7 +327,6 @@ CREATE TRIGGER messages_fts_au AFTER UPDATE ON messages BEGIN
315
327
  SELECT new.id, coalesce(new.content, '') WHERE new.role IN ('user', 'assistant', 'system');
316
328
  END;
317
329
  `;
318
-
319
330
  /**
320
331
  * The bulk form of the supersede trigger above, for correcting whatever an
321
332
  * earlier shape of these triggers already left indexed: any assistant row
@@ -328,14 +339,12 @@ const SUPERSEDE_CLEANUP = `
328
339
  WHERE m.role = 'assistant' AND nxt.role IN ('assistant', 'tool')
329
340
  )
330
341
  `;
331
-
332
342
  /**
333
343
  * Bumped whenever the constructor's migrations change the shape of the
334
344
  * database. Stored in SQLite's user_version, so deployments can see where a
335
345
  * file stands and run migrations deliberately (`tinywebui migrate`).
336
346
  */
337
- export const SCHEMA_VERSION = 12;
338
-
347
+ export const SCHEMA_VERSION = 13;
339
348
  /**
340
349
  * Adds a column if it isn't already there. `CREATE TABLE IF NOT EXISTS` is a
341
350
  * no-op against a live table, so a column added to a table's shape after
@@ -344,10 +353,10 @@ export const SCHEMA_VERSION = 12;
344
353
  * `messages`. `ALTER TABLE ADD COLUMN` itself has no IF NOT EXISTS form.
345
354
  */
346
355
  function ensureColumn(db, table, col, decl) {
347
- const cols = db.prepare(`PRAGMA table_info(${table})`).all().map((c) => c.name);
348
- if (!cols.includes(col)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${col} ${decl}`);
356
+ const cols = db.prepare(`PRAGMA table_info(${table})`).all().map((c) => c.name);
357
+ if (!cols.includes(col))
358
+ db.exec(`ALTER TABLE ${table} ADD COLUMN ${col} ${decl}`);
349
359
  }
350
-
351
360
  /**
352
361
  * Schema 4 renamed "chunks" to what they are: document_chunks (FTS) and
353
362
  * its map became document_passages / document_passage_map, and the name
@@ -355,126 +364,138 @@ function ensureColumn(db, table, col, decl) {
355
364
  * would otherwise see the old FTS table under the new chunk table's name.
356
365
  */
357
366
  function renameToPassages(db) {
358
- const has = (name) => Boolean(db.prepare('SELECT 1 FROM sqlite_master WHERE name = ?').get(name));
359
- if (has('document_chunk_map')) {
360
- db.exec('BEGIN');
361
- try {
362
- db.exec('ALTER TABLE document_chunks RENAME TO document_passages');
363
- db.exec('ALTER TABLE document_chunk_map RENAME TO document_passage_map');
364
- db.exec('ALTER TABLE document_passage_map RENAME COLUMN chunk_rowid TO passage_rowid');
365
- db.exec('ALTER TABLE document_passage_map RENAME COLUMN chunk_idx TO passage_idx');
366
- db.exec('DROP INDEX IF EXISTS document_chunk_map_doc');
367
- db.exec('COMMIT');
368
- } catch (err) { db.exec('ROLLBACK'); throw err; }
369
- }
370
- // Vectors from the unreleased schema 3 layout: recomputed by the backfill.
371
- db.exec('DROP TABLE IF EXISTS document_chunk_embeddings');
367
+ const has = (name) => Boolean(db.prepare('SELECT 1 FROM sqlite_master WHERE name = ?').get(name));
368
+ if (has('document_chunk_map')) {
369
+ db.exec('BEGIN');
370
+ try {
371
+ db.exec('ALTER TABLE document_chunks RENAME TO document_passages');
372
+ db.exec('ALTER TABLE document_chunk_map RENAME TO document_passage_map');
373
+ db.exec('ALTER TABLE document_passage_map RENAME COLUMN chunk_rowid TO passage_rowid');
374
+ db.exec('ALTER TABLE document_passage_map RENAME COLUMN chunk_idx TO passage_idx');
375
+ db.exec('DROP INDEX IF EXISTS document_chunk_map_doc');
376
+ db.exec('COMMIT');
377
+ }
378
+ catch (err) {
379
+ db.exec('ROLLBACK');
380
+ throw err;
381
+ }
382
+ }
383
+ // Vectors from the unreleased schema 3 layout: recomputed by the backfill.
384
+ db.exec('DROP TABLE IF EXISTS document_chunk_embeddings');
372
385
  }
373
-
374
386
  /** Schema 7 and earlier kept document vectors in their own table; they move into `embeddings` as-is. */
375
387
  function moveDocumentVectors(db) {
376
- const cols = db.prepare('PRAGMA table_info(document_chunks)').all().map((c) => c.name);
377
- if (!cols.includes('vec')) return;
378
- db.exec('BEGIN');
379
- try {
380
- db.exec(`INSERT OR IGNORE INTO embeddings (corpus, unit_id, owner_id, model_key, digest, chunk, vec)
388
+ const cols = db.prepare('PRAGMA table_info(document_chunks)').all().map((c) => c.name);
389
+ if (!cols.includes('vec'))
390
+ return;
391
+ db.exec('BEGIN');
392
+ try {
393
+ db.exec(`INSERT OR IGNORE INTO embeddings (corpus, unit_id, owner_id, model_key, digest, chunk, vec)
381
394
  SELECT 'documents', passage_rowid, doc_id, model_key, '', chunk, vec FROM document_chunks`);
382
- db.exec('DROP TABLE document_chunks');
383
- db.exec('COMMIT');
384
- } catch (err) { db.exec('ROLLBACK'); throw err; }
395
+ db.exec('DROP TABLE document_chunks');
396
+ db.exec('COMMIT');
397
+ }
398
+ catch (err) {
399
+ db.exec('ROLLBACK');
400
+ throw err;
401
+ }
385
402
  }
386
-
387
403
  /** Brings an open database to SCHEMA_VERSION. `fresh`: it has no tables yet. */
388
404
  export function migrate(db, { fresh }) {
389
- db.exec('PRAGMA journal_mode = WAL');
390
- db.exec('PRAGMA foreign_keys = ON');
391
- if (!fresh) renameToPassages(db);
392
- db.exec(SCHEMA);
393
- if (!fresh) moveDocumentVectors(db);
394
- // Nullable everywhere: unused (null) when auth is off, the exact behavior
395
- // installs already have; set only once per-user scoping is opted into.
396
- ensureColumn(db, 'chats', 'user_id', 'TEXT');
397
- ensureColumn(db, 'chats', 'folder', 'TEXT');
398
- ensureColumn(db, 'chats', 'tags_json', "TEXT NOT NULL DEFAULT '[]'");
399
- // First message the model still sees once the hard window has moved; -1
400
- // until it ever has. See planWindow in compact.js.
401
- ensureColumn(db, 'chats', 'window_seq', 'INTEGER NOT NULL DEFAULT -1');
402
- // LLM summary of what the window cut off, as JSON { summary, throughSeq,
403
- // scopeStart, model }; NULL when there is none. Only valid together with
404
- // window_seq, so the two are always written in one statement.
405
- ensureColumn(db, 'chats', 'checkpoint_json', 'TEXT');
406
- // The message seq a task was added at, so a rewind past it removes it. -1
407
- // for tasks from before this was tracked: no rewind reaches them.
408
- ensureColumn(db, 'tasks', 'created_seq', 'INTEGER NOT NULL DEFAULT -1');
409
- // Set once an epoch has taken a message's images off the wire; the images
410
- // themselves stay in images_json for the transcript.
411
- ensureColumn(db, 'messages', 'images_dropped', 'INTEGER NOT NULL DEFAULT 0');
412
- // Where a message came from when it was not typed by the user, e.g. an
413
- // automation run: the page draws it as that, not as a user bubble. The
414
- // model still gets `content` as an ordinary user turn.
415
- ensureColumn(db, 'messages', 'origin_json', 'TEXT');
416
- ensureColumn(db, 'documents', 'user_id', 'TEXT');
417
- // The passage split a document was stored with; NULL means the original
418
- // 1800/200, from before it was configurable.
419
- ensureColumn(db, 'documents', 'passage_size', 'INTEGER');
420
- ensureColumn(db, 'documents', 'passage_overlap', 'INTEGER');
421
- // Fingerprint of the uploaded bytes (see document_blobs); NULL for documents
422
- // stored before originals were kept.
423
- ensureColumn(db, 'documents', 'md5', 'TEXT');
424
- db.exec('CREATE INDEX IF NOT EXISTS documents_md5 ON documents(md5)');
425
- ensureColumn(db, 'artifacts', 'user_id', 'TEXT');
426
- ensureColumn(db, 'automation_runs', 'trigger_type', "TEXT NOT NULL DEFAULT 'schedule'");
427
- // Trusted-header auth: the gateway's immutable subject. Email and name are
428
- // metadata reconciled from the gateway; external_id decides who owns what.
429
- ensureColumn(db, 'users', 'external_id', 'TEXT');
430
- ensureColumn(db, 'users', 'name', 'TEXT');
431
- db.exec('CREATE UNIQUE INDEX IF NOT EXISTS users_external_id ON users(external_id)');
432
- // Per-user preferences (tier 3): harmless choices like which allowed
433
- // model to use. Deployment settings never live here.
434
- db.exec(`CREATE TABLE IF NOT EXISTS user_prefs (
405
+ db.exec('PRAGMA journal_mode = WAL');
406
+ db.exec('PRAGMA foreign_keys = ON');
407
+ if (!fresh)
408
+ renameToPassages(db);
409
+ db.exec(SCHEMA);
410
+ if (!fresh)
411
+ moveDocumentVectors(db);
412
+ // Nullable everywhere: unused (null) when auth is off, the exact behavior
413
+ // installs already have; set only once per-user scoping is opted into.
414
+ ensureColumn(db, 'chats', 'user_id', 'TEXT');
415
+ ensureColumn(db, 'chats', 'folder', 'TEXT');
416
+ ensureColumn(db, 'chats', 'tags_json', "TEXT NOT NULL DEFAULT '[]'");
417
+ // First message the model still sees once the hard window has moved; -1
418
+ // until it ever has. See planWindow in compact.js.
419
+ ensureColumn(db, 'chats', 'window_seq', 'INTEGER NOT NULL DEFAULT -1');
420
+ // LLM summary of what the window cut off, as JSON { summary, throughSeq,
421
+ // scopeStart, model }; NULL when there is none. Only valid together with
422
+ // window_seq, so the two are always written in one statement.
423
+ ensureColumn(db, 'chats', 'checkpoint_json', 'TEXT');
424
+ // The message seq a task was added at, so a rewind past it removes it. -1
425
+ // for tasks from before this was tracked: no rewind reaches them.
426
+ ensureColumn(db, 'tasks', 'created_seq', 'INTEGER NOT NULL DEFAULT -1');
427
+ // Set once an epoch has taken a message's images off the wire; the images
428
+ // themselves stay in images_json for the transcript.
429
+ ensureColumn(db, 'messages', 'images_dropped', 'INTEGER NOT NULL DEFAULT 0');
430
+ // Where a message came from when it was not typed by the user, e.g. an
431
+ // automation run: the page draws it as that, not as a user bubble. The
432
+ // model still gets `content` as an ordinary user turn.
433
+ ensureColumn(db, 'messages', 'origin_json', 'TEXT');
434
+ ensureColumn(db, 'documents', 'user_id', 'TEXT');
435
+ // The passage split a document was stored with; NULL means the original
436
+ // 1800/200, from before it was configurable.
437
+ ensureColumn(db, 'documents', 'passage_size', 'INTEGER');
438
+ ensureColumn(db, 'documents', 'passage_overlap', 'INTEGER');
439
+ // Fingerprint of the uploaded bytes (see document_blobs); NULL for documents
440
+ // stored before originals were kept.
441
+ ensureColumn(db, 'documents', 'md5', 'TEXT');
442
+ db.exec('CREATE INDEX IF NOT EXISTS documents_md5 ON documents(md5)');
443
+ ensureColumn(db, 'artifacts', 'user_id', 'TEXT');
444
+ ensureColumn(db, 'automation_runs', 'trigger_type', "TEXT NOT NULL DEFAULT 'schedule'");
445
+ // Trusted-header auth: the gateway's immutable subject. Email and name are
446
+ // metadata reconciled from the gateway; external_id decides who owns what.
447
+ ensureColumn(db, 'users', 'external_id', 'TEXT');
448
+ ensureColumn(db, 'users', 'name', 'TEXT');
449
+ db.exec('CREATE UNIQUE INDEX IF NOT EXISTS users_external_id ON users(external_id)');
450
+ // Per-user preferences (tier 3): harmless choices like which allowed
451
+ // model to use. Deployment settings never live here.
452
+ db.exec(`CREATE TABLE IF NOT EXISTS user_prefs (
435
453
  user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
436
454
  key TEXT NOT NULL,
437
455
  value TEXT,
438
456
  PRIMARY KEY (user_id, key)
439
457
  )`);
440
- // `messages` may already exist from before these columns did --
441
- // CREATE TABLE IF NOT EXISTS above is a no-op against a live table, so
442
- // they're added here instead, self-repairing like the FTS backfill below.
443
- const cols = db.prepare('PRAGMA table_info(messages)').all().map((c) => c.name);
444
- if (!cols.includes('model')) db.exec('ALTER TABLE messages ADD COLUMN model TEXT');
445
- if (!cols.includes('created_at')) db.exec('ALTER TABLE messages ADD COLUMN created_at INTEGER');
446
- if (!cols.includes('images_json')) db.exec('ALTER TABLE messages ADD COLUMN images_json TEXT');
447
- db.exec('CREATE INDEX IF NOT EXISTS messages_usage_ts ON messages(created_at) WHERE usage_json IS NOT NULL');
448
- // A tool row, or an assistant note later superseded within its own turn,
449
- // indexed under an earlier shape of the triggers above is corrected on
450
- // open rather than left to linger -- the triggers alone only ever
451
- // prevent NEW drift, not clean up old drift.
452
- db.exec(`
458
+ // `messages` may already exist from before these columns did --
459
+ // CREATE TABLE IF NOT EXISTS above is a no-op against a live table, so
460
+ // they're added here instead, self-repairing like the FTS backfill below.
461
+ const cols = db.prepare('PRAGMA table_info(messages)').all().map((c) => c.name);
462
+ if (!cols.includes('model'))
463
+ db.exec('ALTER TABLE messages ADD COLUMN model TEXT');
464
+ if (!cols.includes('created_at'))
465
+ db.exec('ALTER TABLE messages ADD COLUMN created_at INTEGER');
466
+ if (!cols.includes('images_json'))
467
+ db.exec('ALTER TABLE messages ADD COLUMN images_json TEXT');
468
+ db.exec('CREATE INDEX IF NOT EXISTS messages_usage_ts ON messages(created_at) WHERE usage_json IS NOT NULL');
469
+ // A tool row, or an assistant note later superseded within its own turn,
470
+ // indexed under an earlier shape of the triggers above is corrected on
471
+ // open rather than left to linger -- the triggers alone only ever
472
+ // prevent NEW drift, not clean up old drift.
473
+ db.exec(`
453
474
  DELETE FROM messages_fts WHERE rowid IN (
454
475
  SELECT id FROM messages WHERE role NOT IN ('user', 'assistant', 'system')
455
476
  )
456
477
  `);
457
- db.exec(SUPERSEDE_CLEANUP);
458
- // The triggers keep the index in step with every write from here on, but
459
- // they cannot backfill history that predates them -- a database from
460
- // before search existed opens with a `messages_fts` that is real but
461
- // empty for rows that should be in it. A count mismatch, restricted to
462
- // the roles that belong in the index, is also what any other divergence
463
- // would look like, so this doubles as self-repair: whatever is missing
464
- // is added, nothing already correct is touched.
465
- const { m, f } = db.prepare(`
478
+ db.exec(SUPERSEDE_CLEANUP);
479
+ // The triggers keep the index in step with every write from here on, but
480
+ // they cannot backfill history that predates them -- a database from
481
+ // before search existed opens with a `messages_fts` that is real but
482
+ // empty for rows that should be in it. A count mismatch, restricted to
483
+ // the roles that belong in the index, is also what any other divergence
484
+ // would look like, so this doubles as self-repair: whatever is missing
485
+ // is added, nothing already correct is touched.
486
+ const { m, f } = db.prepare(`
466
487
  SELECT
467
488
  (SELECT COUNT(*) FROM messages WHERE role IN ('user', 'assistant', 'system')) AS m,
468
489
  (SELECT COUNT(*) FROM messages_fts) AS f
469
490
  `).get();
470
- if (m !== f) {
471
- db.exec(`
491
+ if (m !== f) {
492
+ db.exec(`
472
493
  INSERT INTO messages_fts(rowid, body)
473
494
  SELECT id, coalesce(content, '')
474
495
  FROM messages
475
496
  WHERE role IN ('user', 'assistant', 'system')
476
497
  AND id NOT IN (SELECT rowid FROM messages_fts)
477
498
  `);
478
- }
479
- db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
499
+ }
500
+ db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
480
501
  }