tinywebui 0.1.0

package/src/store/embeddings.js

@@ -0,0 +1,67 @@
+// Chunk vectors for every retrieval corpus (see retrieval/retrieval.js).
+// One area of the Store; see index.js.
+
+/** The retrieval corpora whose vectors live here (see retrieval/retrieval.js). */
+export const CORPUS = Object.freeze({ DOCUMENTS: 'documents', CHATS: 'chats' });
+
+export class EmbeddingStore {
+  constructor(db, deps = {}) {
+    this.db = db;
+    Object.assign(this, deps);
+  }
+
+  /** Chunk vectors of the given units under one model: Map(unit id -> Float32Array[], one per chunk). */
+  vectors(corpus, modelKey, unitIds) {
+    const out = new Map();
+    if (!unitIds.length) return out;
+    const rows = this.db.prepare(`
+      SELECT unit_id, vec FROM embeddings
+      WHERE corpus = ? AND model_key = ? AND unit_id IN (SELECT value FROM json_each(?))
+      ORDER BY unit_id, chunk
+    `).all(corpus, modelKey, JSON.stringify(unitIds));
+    for (const r of rows) {
+      const buf = r.vec;
+      if (!out.has(r.unit_id)) out.set(r.unit_id, []);
+      out.get(r.unit_id).push(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4).slice());
+    }
+    return out;
+  }
+
+  /** What each of an owner's units was embedded from: Map(unit id -> digest). */
+  digests(corpus, ownerId, modelKey) {
+    const rows = this.db.prepare('SELECT unit_id, digest FROM embeddings WHERE corpus = ? AND owner_id = ? AND model_key = ? AND chunk = 0')
+      .all(corpus, ownerId, modelKey);
+    return new Map(rows.map((r) => [r.unit_id, r.digest]));
+  }
+
+  /** rows: [{ unitId, digest, vecs: Float32Array[] (its chunks, in order) }], replacing what those units had. */
+  put(corpus, ownerId, modelKey, rows) {
+    const clear = this.db.prepare('DELETE FROM embeddings WHERE corpus = ? AND unit_id = ? AND model_key = ?');
+    const put = this.db.prepare('INSERT INTO embeddings (corpus, unit_id, owner_id, model_key, digest, chunk, vec) VALUES (?, ?, ?, ?, ?, ?, ?)');
+    this.db.exec('BEGIN');
+    try {
+      for (const { unitId, digest, vecs } of rows) {
+        clear.run(corpus, unitId, modelKey);
+        vecs.forEach((vec, chunk) => put.run(corpus, unitId, ownerId, modelKey, digest, chunk, Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength)));
+      }
+      this.db.exec('COMMIT');
+    } catch (err) { this.db.exec('ROLLBACK'); throw err; }
+  }
+
+  /** Drops an owner's vectors for units it no longer has. */
+  dropStale(corpus, ownerId, keepUnitIds) {
+    return this.db.prepare(`
+      DELETE FROM embeddings WHERE corpus = ? AND owner_id = ? AND unit_id NOT IN (SELECT value FROM json_each(?))
+    `).run(corpus, ownerId, JSON.stringify(keepUnitIds)).changes;
+  }
+
+  /** Drops vectors from models (or chunkings) no longer configured. */
+  prune(keepModelKey) {
+    return this.db.prepare('DELETE FROM embeddings WHERE model_key != ?').run(keepModelKey).changes;
+  }
+
+  /** Drops every vector an owner (a document, a chat) has in a corpus. */
+  deleteOwner(corpus, ownerId) {
+    this.db.prepare('DELETE FROM embeddings WHERE corpus = ? AND owner_id = ?').run(corpus, ownerId);
+  }
+}

package/src/store/index.js

@@ -0,0 +1,90 @@
+import { DatabaseSync } from 'node:sqlite';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { SCHEMA_VERSION, migrate as migrateSchema } from './schema.js';
+import { ChatStore } from './chats.js';
+import { MessageStore } from './messages.js';
+import { AutomationStore } from './automations.js';
+import { UsageStore } from './usage.js';
+import { DocumentStore } from './documents.js';
+import { EmbeddingStore } from './embeddings.js';
+import { TurnStore } from './turns.js';
+import { SearchStore } from './search.js';
+import { UserStore } from './users.js';
+
+export { SCHEMA_VERSION } from './schema.js';
+export { ALL_USERS } from './scope.js';
+export { PASSAGE_SIZE, PASSAGE_OVERLAP } from './documents.js';
+export { toWire, toView } from './wire.js';
+export { CORPUS } from './embeddings.js';
+export { AUX_KIND } from './usage.js';
+
+/**
+ * Conversation storage.
+ *
+ * The client used to hold the whole transcript and POST it back every turn,
+ * which meant every tool result -- including page-sized ones -- crossed the wire
+ * again on each message and eventually blew the localStorage quota. State lives
+ * here now, and the client sends only the new user message.
+ *
+ * Two things this file deliberately does NOT do: it never parses tool output,
+ * and it never special-cases a tool name. A tool result is an opaque blob from
+ * an arbitrary MCP server. Everything downstream (compaction, expansion) holds
+ * to the same rule.
+ *
+ * One connection, one object per area. Every call names its area --
+ * `store.chats.list()`, `store.documents.add()` -- and each area lives in the
+ * file of the same name next to this one. An area that needs another is
+ * handed it here, so every dependency between areas is listed in one place.
+ */
+
+/** Thrown when a database is behind and migrating was not allowed. */
+export class MigrationRequiredError extends Error {}
+
+export class Store {
+  /**
+   * `migrate: false` refuses to touch an existing database that is behind
+   * SCHEMA_VERSION instead of upgrading it on open; a brand-new file is
+   * always created at the current version.
+   */
+  constructor(path, { migrate = true } = {}) {
+    if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true });
+    this.db = new DatabaseSync(path);
+    const found = this.db.prepare('PRAGMA user_version').get().user_version;
+    const fresh = !this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='chats'").get();
+    if (!fresh && found > SCHEMA_VERSION) {
+      this.db.close();
+      throw new MigrationRequiredError(`database is at schema ${found}, newer than this TinyWebUI (${SCHEMA_VERSION}); upgrade TinyWebUI`);
+    }
+    if (!fresh && found < SCHEMA_VERSION && !migrate) {
+      this.db.close();
+      throw new MigrationRequiredError(`database is at schema ${found}, needs ${SCHEMA_VERSION}: run \`tinywebui migrate\``);
+    }
+    this.migratedFrom = fresh ? null : found;
+    migrateSchema(this.db, { fresh });
+
+    const { db } = this;
+    this.embeddings = new EmbeddingStore(db);
+    this.messages = new MessageStore(db);
+    this.automations = new AutomationStore(db);
+    this.documents = new DocumentStore(db, { embeddings: this.embeddings });
+    this.chats = new ChatStore(db, { messages: this.messages, automations: this.automations, documents: this.documents, embeddings: this.embeddings });
+    this.turns = new TurnStore(db, { chats: this.chats });
+    this.usage = new UsageStore(db);
+    this.search = new SearchStore(db);
+    this.users = new UserStore(db);
+  }
+
+  schemaVersion() {
+    return this.db.prepare('PRAGMA user_version').get().user_version;
+  }
+
+  /** Cheap liveness probe of the database, for /readyz. */
+  ping() {
+    return this.db.prepare('SELECT 1 AS ok').get().ok === 1;
+  }
+
+  close() {
+    try { this.db.close(); } catch { /* already gone */ }
+  }
+}

package/src/store/messages.js

@@ -0,0 +1,93 @@
+import { randomBytes } from 'node:crypto';
+
+// The transcript: messages in order, and the tool-result artifacts behind them.
+// One area of the Store; see index.js.
+
+export class MessageStore {
+  constructor(db, deps = {}) {
+    this.db = db;
+    Object.assign(this, deps);
+  }
+
+  nextSeq(chatId) {
+    const row = this.db.prepare('SELECT MAX(seq) AS m FROM messages WHERE chat_id = ?').get(chatId);
+    return (row?.m ?? -1) + 1;
+  }
+
+  /** Drops a chat's messages from `seq` on (a rewind). Returns how many went. */
+  deleteFrom(chatId, seq) {
+    return this.db.prepare('DELETE FROM messages WHERE chat_id = ? AND seq >= ?').run(chatId, seq).changes;
+  }
+
+  /** Drops a chat's whole transcript and the artifacts behind it (the chat is going). */
+  deleteForChat(chatId) {
+    this.db.prepare('DELETE FROM messages WHERE chat_id = ?').run(chatId);
+    this.db.prepare('DELETE FROM artifacts WHERE chat_id = ?').run(chatId);
+  }
+
+  /** Appends one message. Append-only by design: rows are never reordered. */
+  add(chatId, msg) {
+    const seq = this.nextSeq(chatId);
+    this.db.prepare(`
+      INSERT INTO messages
+        (chat_id, seq, role, content, tool_call_id, tool_calls_json,
+         reasoning, reasoning_details_json, artifact_id, stub_text, usage_json,
+         model, created_at, images_json, origin_json)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      chatId, seq, msg.role,
+      msg.content ?? null,
+      msg.tool_call_id ?? null,
+      msg.tool_calls ? JSON.stringify(msg.tool_calls) : null,
+      msg.reasoning ?? null,
+      msg.reasoning_details ? JSON.stringify(msg.reasoning_details) : null,
+      msg.artifact_id ?? null,
+      msg.stub_text ?? null,
+      msg.usage ? JSON.stringify(msg.usage) : null,
+      msg.model ?? null,
+      Date.now(),
+      msg.images ? JSON.stringify(msg.images) : null,
+      msg.origin ? JSON.stringify(msg.origin) : null
+    );
+    return seq;
+  }
+
+  updateUsage(chatId, seq, usage) {
+    this.db.prepare('UPDATE messages SET usage_json = ? WHERE chat_id = ? AND seq = ?')
+      .run(JSON.stringify(usage), chatId, seq);
+  }
+
+  list(chatId) {
+    return this.db.prepare(
+      'SELECT * FROM messages WHERE chat_id = ? ORDER BY seq ASC'
+    ).all(chatId);
+  }
+
+  /**
+   * Writes a stub onto an already-stored tool message. This is the ONE place
+   * history is rewritten, and it happens only at an epoch boundary. `content`
+   * is left untouched so the human transcript keeps the full output -- only
+   * what goes to the model shrinks.
+   */
+  setStub(messageId, stubText) {
+    this.db.prepare('UPDATE messages SET stub_text = ? WHERE id = ?').run(stubText, messageId);
+  }
+
+  /** Takes a message's images off the wire at an epoch; the transcript keeps them. */
+  dropImages(messageId) {
+    this.db.prepare('UPDATE messages SET images_dropped = 1 WHERE id = ?').run(messageId);
+  }
+
+  addArtifact(chatId, { toolName, args, content }) {
+    const id = randomBytes(12).toString('hex');
+    this.db.prepare(`
+      INSERT INTO artifacts (id, chat_id, tool_name, args_json, content, char_len, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(id, chatId, toolName, JSON.stringify(args ?? {}), content, content.length, Date.now());
+    return id;
+  }
+
+  getArtifact(id, chatId) {
+    return this.db.prepare('SELECT * FROM artifacts WHERE id = ? AND chat_id = ?').get(id, chatId) || null;
+  }
+}