sqlite-hub 0.12.0 → 0.17.0

package/server/services/storage/appStateStore.js

@@ -1,4 +1,5 @@
 const fs = require("node:fs");
+const crypto = require("node:crypto");
 const path = require("node:path");
 const { pathToFileURL } = require("node:url");
 const Database = require("better-sqlite3");
@@ -41,11 +42,110 @@ const CONNECTION_LOGO_DIRECTORY = "db_logos";
 const LEGACY_STATE_FILENAME = "app-state.json";
 const STATE_DATABASE_FILENAME = "sqlite-hub-state.db";
 const MAX_CONNECTION_LOGO_SIZE_BYTES = 5 * 1024 * 1024;
+const MAX_DOCUMENT_CONTENT_BYTES = 5 * 1024 * 1024;
+const MAX_DOCUMENT_FILENAME_LENGTH = 160;
 const CONNECTION_LOGO_EXTENSION_BY_MIME_TYPE = {
   "image/jpeg": "jpg",
   "image/png": "png",
   "image/webp": "webp",
 };
+
+function normalizeDocumentDatabaseKey(databaseKey) {
+  const normalizedDatabaseKey = String(databaseKey ?? "").trim();
+
+  if (!normalizedDatabaseKey) {
+    throw new ValidationError("Database key is required.");
+  }
+
+  return normalizedDatabaseKey;
+}
+
+function normalizeDocumentId(documentId) {
+  const normalizedDocumentId = String(documentId ?? "").trim();
+
+  if (!normalizedDocumentId) {
+    throw new ValidationError("Document id is required.");
+  }
+
+  return normalizedDocumentId;
+}
+
+function splitMarkdownFilename(filename) {
+  const normalizedFilename = String(filename ?? "");
+  const extensionMatch = normalizedFilename.match(/\.md$/i);
+
+  if (!extensionMatch) {
+    return {
+      baseName: normalizedFilename,
+      extension: ".md",
+    };
+  }
+
+  return {
+    baseName: normalizedFilename.slice(0, -extensionMatch[0].length),
+    extension: normalizedFilename.slice(-extensionMatch[0].length),
+  };
+}
+
+function normalizeDocumentFilename(value, fallback = "Untitled.md") {
+  let filename = String(value ?? "").trim();
+
+  if (!filename) {
+    filename = fallback;
+  }
+
+  filename = filename
+    .replace(/[\u0000-\u001f\u007f]/g, " ")
+    .replace(/[\\/]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim()
+    .replace(/^\.+/, "")
+    .trim();
+
+  if (!filename) {
+    filename = fallback;
+  }
+
+  if (!/\.md$/i.test(filename)) {
+    filename = `${filename}.md`;
+  }
+
+  if (filename.length > MAX_DOCUMENT_FILENAME_LENGTH) {
+    const { baseName, extension } = splitMarkdownFilename(filename);
+    filename = `${baseName.slice(0, MAX_DOCUMENT_FILENAME_LENGTH - extension.length)}${extension}`;
+  }
+
+  return filename;
+}
+
+function buildDocumentTitleFromFilename(filename) {
+  const { baseName } = splitMarkdownFilename(filename);
+  const title = baseName.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim();
+
+  return title || "Untitled";
+}
+
+function normalizeDocumentTitle(value, filename) {
+  const title = String(value ?? "").trim();
+
+  return title || buildDocumentTitleFromFilename(filename);
+}
+
+function normalizeDocumentContent(value) {
+  const content = String(value ?? "");
+  const byteLength = Buffer.byteLength(content, "utf8");
+
+  if (byteLength > MAX_DOCUMENT_CONTENT_BYTES) {
+    throw new ValidationError("Document content is too large.", {
+      details: {
+        maxBytes: MAX_DOCUMENT_CONTENT_BYTES,
+        actualBytes: byteLength,
+      },
+    });
+  }
+
+  return content;
+}
 const CONNECTION_LOGO_EXTENSION_BY_FILE_EXTENSION = {
   ".jpeg": "jpg",
   ".jpg": "jpg",
@@ -343,6 +443,20 @@ class AppStateStore {
         mapping_table TEXT NOT NULL DEFAULT '',
         updated_at TEXT NOT NULL
       );
+
+      CREATE TABLE IF NOT EXISTS database_documents (
+        id TEXT PRIMARY KEY,
+        database_key TEXT NOT NULL,
+        title TEXT NOT NULL,
+        filename TEXT NOT NULL,
+        content TEXT NOT NULL DEFAULT '',
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL,
+        UNIQUE(database_key, filename)
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_database_documents_database_updated
+      ON database_documents(database_key, updated_at DESC, id ASC);
     `);
 
     const recentConnectionColumns = new Set(
@@ -2221,6 +2335,205 @@ class AppStateStore {
     return this.getSettings();
   }
 
+  decorateDatabaseDocumentRow(row = {}) {
+    return {
+      id: String(row.id ?? ""),
+      databaseKey: row.database_key ?? row.databaseKey ?? "",
+      title: String(row.title ?? ""),
+      filename: String(row.filename ?? ""),
+      content: row.content === undefined ? undefined : String(row.content ?? ""),
+      contentLength: Number(row.content_length ?? row.contentLength ?? 0),
+      createdAt: row.created_at ?? row.createdAt ?? null,
+      updatedAt: row.updated_at ?? row.updatedAt ?? null,
+    };
+  }
+
+  documentFilenameExists(databaseKey, filename, ignoredDocumentId = null) {
+    const row = this.db
+      .prepare(
+        `
+          SELECT id
+          FROM database_documents
+          WHERE database_key = ?
+            AND filename = ?
+            AND (? IS NULL OR id != ?)
+          LIMIT 1
+        `
+      )
+      .get(databaseKey, filename, ignoredDocumentId, ignoredDocumentId);
+
+    return Boolean(row);
+  }
+
+  resolveUniqueDocumentFilename(databaseKey, desiredFilename, ignoredDocumentId = null) {
+    const normalizedFilename = normalizeDocumentFilename(desiredFilename);
+
+    if (!this.documentFilenameExists(databaseKey, normalizedFilename, ignoredDocumentId)) {
+      return normalizedFilename;
+    }
+
+    const { baseName, extension } = splitMarkdownFilename(normalizedFilename);
+
+    for (let index = 2; index < 1000; index += 1) {
+      const suffix = ` ${index}`;
+      const maxBaseLength = MAX_DOCUMENT_FILENAME_LENGTH - extension.length - suffix.length;
+      const candidate = `${baseName.slice(0, Math.max(1, maxBaseLength))}${suffix}${extension}`;
+
+      if (!this.documentFilenameExists(databaseKey, candidate, ignoredDocumentId)) {
+        return candidate;
+      }
+    }
+
+    throw new ConflictError("Could not create a unique document filename.");
+  }
+
+  listDatabaseDocuments(databaseKey) {
+    const normalizedDatabaseKey = normalizeDocumentDatabaseKey(databaseKey);
+
+    return this.db
+      .prepare(
+        `
+          SELECT
+            id,
+            database_key,
+            title,
+            filename,
+            LENGTH(content) AS content_length,
+            created_at,
+            updated_at
+          FROM database_documents
+          WHERE database_key = ?
+          ORDER BY updated_at DESC, id ASC
+        `
+      )
+      .all(normalizedDatabaseKey)
+      .map((row) => this.decorateDatabaseDocumentRow(row));
+  }
+
+  getDatabaseDocument(databaseKey, documentId) {
+    const normalizedDatabaseKey = normalizeDocumentDatabaseKey(databaseKey);
+    const normalizedDocumentId = normalizeDocumentId(documentId);
+    const row = this.db
+      .prepare(
+        `
+          SELECT
+            id,
+            database_key,
+            title,
+            filename,
+            content,
+            LENGTH(content) AS content_length,
+            created_at,
+            updated_at
+          FROM database_documents
+          WHERE database_key = ?
+            AND id = ?
+        `
+      )
+      .get(normalizedDatabaseKey, normalizedDocumentId);
+
+    if (!row) {
+      throw new NotFoundError("Document was not found.");
+    }
+
+    return this.decorateDatabaseDocumentRow(row);
+  }
+
+  createDatabaseDocument(databaseKey, document = {}) {
+    const normalizedDatabaseKey = normalizeDocumentDatabaseKey(databaseKey);
+    const filename = this.resolveUniqueDocumentFilename(
+      normalizedDatabaseKey,
+      normalizeDocumentFilename(document.filename)
+    );
+    const title = normalizeDocumentTitle(document.title, filename);
+    const content = normalizeDocumentContent(document.content);
+    const now = new Date().toISOString();
+    const id = crypto.randomUUID();
+
+    this.db
+      .prepare(
+        `
+          INSERT INTO database_documents (
+            id,
+            database_key,
+            title,
+            filename,
+            content,
+            created_at,
+            updated_at
+          )
+          VALUES (?, ?, ?, ?, ?, ?, ?)
+        `
+      )
+      .run(id, normalizedDatabaseKey, title, filename, content, now, now);
+
+    return this.getDatabaseDocument(normalizedDatabaseKey, id);
+  }
+
+  updateDatabaseDocument(databaseKey, documentId, patch = {}) {
+    const normalizedDatabaseKey = normalizeDocumentDatabaseKey(databaseKey);
+    const existingDocument = this.getDatabaseDocument(normalizedDatabaseKey, documentId);
+    const hasFilename = Object.prototype.hasOwnProperty.call(patch, "filename");
+    const hasTitle = Object.prototype.hasOwnProperty.call(patch, "title");
+    const hasContent = Object.prototype.hasOwnProperty.call(patch, "content");
+    const filename = hasFilename
+      ? this.resolveUniqueDocumentFilename(
+          normalizedDatabaseKey,
+          normalizeDocumentFilename(patch.filename, existingDocument.filename),
+          existingDocument.id
+        )
+      : existingDocument.filename;
+    const title = hasTitle
+      ? normalizeDocumentTitle(patch.title, filename)
+      : hasFilename
+        ? buildDocumentTitleFromFilename(filename)
+        : existingDocument.title;
+    const content = hasContent
+      ? normalizeDocumentContent(patch.content)
+      : existingDocument.content;
+    const updatedAt = new Date().toISOString();
+
+    this.db
+      .prepare(
+        `
+          UPDATE database_documents
+          SET
+            title = ?,
+            filename = ?,
+            content = ?,
+            updated_at = ?
+          WHERE database_key = ?
+            AND id = ?
+        `
+      )
+      .run(title, filename, content, updatedAt, normalizedDatabaseKey, existingDocument.id);
+
+    return this.getDatabaseDocument(normalizedDatabaseKey, existingDocument.id);
+  }
+
+  deleteDatabaseDocument(databaseKey, documentId) {
+    const normalizedDatabaseKey = normalizeDocumentDatabaseKey(databaseKey);
+    const normalizedDocumentId = normalizeDocumentId(documentId);
+    const result = this.db
+      .prepare(
+        `
+          DELETE FROM database_documents
+          WHERE database_key = ?
+            AND id = ?
+        `
+      )
+      .run(normalizedDatabaseKey, normalizedDocumentId);
+
+    if (result.changes < 1) {
+      throw new NotFoundError("Document was not found.");
+    }
+
+    return {
+      id: normalizedDocumentId,
+      deleted: true,
+    };
+  }
+
   getMediaTaggingConfig(databaseKey) {
     const normalizedDatabaseKey = String(databaseKey ?? "").trim();
 

package/tests/check-constraint-options.test.js

@@ -3,6 +3,7 @@ const assert = require("node:assert/strict");
 const test = require("node:test");
 const { getTableDetail } = require("../server/services/sqlite/introspection");
 const { SqlExecutor } = require("../server/services/sqlite/sqlExecutor");
+const { buildTableDesignerDraft } = require("../server/services/sqlite/tableDesigner/schemaMapping");
 
 test("table detail exposes string options from simple CHECK IN constraints", () => {
   const db = new Database(":memory:");
@@ -34,6 +35,7 @@ test("table detail exposes string options from simple CHECK IN constraints", ()
     `);
 
     const tableDetail = getTableDetail(db, "stream_company_mentions");
+    const designerDraft = buildTableDesignerDraft(tableDetail);
     const mentionTypeColumn = tableDetail.columns.find(
       (column) => column.name === "mention_type"
     );
@@ -70,6 +72,18 @@ test("table detail exposes string options from simple CHECK IN constraints", ()
     );
 
     assert.deepEqual(editableColumn.allowedValues, mentionTypeColumn.allowedValues);
+
+    assert.equal(designerDraft.designerVersion, 2);
+    assert.equal(designerDraft.checkConstraints.length, 1);
+    assert.match(designerDraft.checkConstraints[0].expression, /CHECK/i);
+    assert.equal(designerDraft.checkConstraints[0].originalExpression, designerDraft.checkConstraints[0].expression);
+    assert.equal(designerDraft.checkConstraints[0].editable, true);
+    assert.deepEqual(designerDraft.checkConstraints[0].columns, [
+      {
+        name: "mention_type",
+        allowedValues: mentionTypeColumn.allowedValues,
+      },
+    ]);
   } finally {
     db.close();
   }

package/tests/cli-args.test.js

@@ -0,0 +1,100 @@
+const assert = require("node:assert/strict");
+const test = require("node:test");
+
+const {
+  normalizeExportFormat,
+  parseCliArguments,
+} = require("../bin/sqlite-hub");
+
+test("parses database execute command", () => {
+  const options = parseCliArguments([
+    "--database:trump live interviews",
+    "--execute:Stock Winners",
+  ]);
+
+  assert.equal(options.databaseName, "trump live interviews");
+  assert.equal(options.executeQuery, "Stock Winners");
+  assert.equal(options.queries, false);
+});
+
+test("parses database table list command", () => {
+  const options = parseCliArguments(["--database:trump live interviews", "--tables"]);
+
+  assert.equal(options.databaseName, "trump live interviews");
+  assert.equal(options.tables, true);
+});
+
+test("parses database info commands with the new schema", () => {
+  assert.equal(parseCliArguments(["--database:db", "--path"]).pathInfo, true);
+  assert.equal(parseCliArguments(["--database:db", "---path"]).pathInfo, true);
+  assert.equal(parseCliArguments(["--database:db", "--size"]).sizeInfo, true);
+  assert.equal(parseCliArguments(["--database:db", "--lastopened"]).lastOpenedInfo, true);
+});
+
+test("keeps old sqleditor aliases working", () => {
+  const listOptions = parseCliArguments(["--database:Unit-00", "--sqleditor"]);
+  const executeOptions = parseCliArguments(["--database:Unit-00", "--sqleditor:Saved Query"]);
+
+  assert.equal(listOptions.queries, true);
+  assert.equal(executeOptions.executeQuery, "Saved Query");
+});
+
+test("keeps old database detail aliases working", () => {
+  const pathOptions = parseCliArguments(["--database-path:Billly", "--queries"]);
+  const tableOptions = parseCliArguments(["--database-tables:Billly"]);
+
+  assert.equal(pathOptions.databaseName, "Billly");
+  assert.equal(pathOptions.pathInfo, true);
+  assert.equal(pathOptions.queries, true);
+  assert.equal(tableOptions.databaseName, "Billly");
+  assert.equal(tableOptions.tables, true);
+});
+
+test("parses query display and export commands", () => {
+  const showOptions = parseCliArguments(["--database:db", "--query:Stock Winners"]);
+  const notesOptions = parseCliArguments(["--database:db", "--notes:Stock Winners"]);
+  const exportOptions = parseCliArguments([
+    "--database:db",
+    "--export:Stock Winners",
+    "--format:md",
+  ]);
+
+  assert.equal(showOptions.showQuery, "Stock Winners");
+  assert.equal(notesOptions.showNotes, "Stock Winners");
+  assert.equal(exportOptions.exportTarget, "Stock Winners");
+  assert.equal(exportOptions.exportFormat, "md");
+});
+
+test("parses row json export command", () => {
+  const options = parseCliArguments([
+    "--database:db",
+    "--table:companies",
+    "--export:0a754aba373d34972998792a0be4333c",
+  ]);
+
+  assert.equal(options.tableName, "companies");
+  assert.equal(options.exportTarget, "0a754aba373d34972998792a0be4333c");
+});
+
+test("parses document commands", () => {
+  const listOptions = parseCliArguments(["--database:db", "--documents"]);
+  const showOptions = parseCliArguments(["--database:db", "--documents:Research Note"]);
+  const exportOptions = parseCliArguments(["--database:db", "--documents:Research Note", "--export"]);
+  const compactExportOptions = parseCliArguments(["--database:db", "--documents:Research Note--export"]);
+
+  assert.equal(listOptions.documents, true);
+  assert.equal(listOptions.documentName, null);
+  assert.equal(showOptions.documents, true);
+  assert.equal(showOptions.documentName, "Research Note");
+  assert.equal(showOptions.documentExport, false);
+  assert.equal(exportOptions.documentName, "Research Note");
+  assert.equal(exportOptions.documentExport, true);
+  assert.equal(compactExportOptions.documentName, "Research Note");
+  assert.equal(compactExportOptions.documentExport, true);
+});
+
+test("validates export formats", () => {
+  assert.equal(normalizeExportFormat("csv"), "csv");
+  assert.equal(normalizeExportFormat("TSV"), "tsv");
+  assert.throws(() => normalizeExportFormat("json"), /Unsupported export format/);
+});

package/tests/copy-column-modal.test.js

@@ -0,0 +1,83 @@
+const assert = require("node:assert/strict");
+const { readFileSync } = require("node:fs");
+const path = require("node:path");
+const { pathToFileURL } = require("node:url");
+const test = require("node:test");
+
+let modalModulePromise = null;
+
+function loadModalModule() {
+  if (!modalModulePromise) {
+    modalModulePromise = import(
+      pathToFileURL(path.resolve(__dirname, "../frontend/js/components/modal.js")).href
+    );
+  }
+
+  return modalModulePromise;
+}
+
+function buildState(copyMode) {
+  return {
+    modal: {
+      kind: "copy-column",
+      scope: "editor",
+      columnName: "task",
+      copyMode,
+      separator: ",",
+      wrapper: '"',
+      lineBreaks: false,
+      error: null,
+      submitting: false,
+    },
+    editor: {
+      result: {
+        columns: ["task"],
+        rows: [{ task: "item1" }, { task: "item2" }],
+      },
+    },
+    charts: {
+      result: null,
+    },
+    connections: {
+      recent: [],
+      active: null,
+    },
+  };
+}
+
+test("markdown todo column export renders an editable preview textarea", async () => {
+  const { renderCopyColumnModal } = await loadModalModule();
+  const state = buildState("markdown-todo");
+  const html = renderCopyColumnModal(state.modal, state);
+
+  assert.match(html, /Editable Preview/);
+  assert.match(html, /<textarea[^>]+name="editedText"/);
+  assert.match(html, /- \[ \] item1/);
+  assert.match(html, /- \[ \] item2/);
+  assert.match(html, /Export to document folder/);
+});
+
+test("regular copy column preview stays read-only", async () => {
+  const { renderCopyColumnModal } = await loadModalModule();
+  const state = buildState("column");
+  const html = renderCopyColumnModal(state.modal, state);
+
+  assert.match(html, /<pre class="copy-column-preview custom-scrollbar">/);
+  assert.doesNotMatch(html, /name="editedText"/);
+});
+
+test("modal footer close buttons are not right-aligned", () => {
+  const source = readFileSync(
+    path.resolve(__dirname, "../frontend/js/components/modal.js"),
+    "utf8"
+  );
+
+  assert.doesNotMatch(
+    source,
+    /<div class="[^"]*justify-end[^"]*pt-2[^"]*"[\s\S]{0,500}?data-action="close-modal"/
+  );
+  assert.doesNotMatch(
+    source,
+    /'<div class="[^']*justify-end[^']*pt-2[^']*>'[\s\S]{0,500}?data-action="close-modal"/
+  );
+});

package/tests/database-documents.test.js

@@ -0,0 +1,85 @@
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const test = require("node:test");
+
+const { AppStateStore } = require("../server/services/storage/appStateStore");
+const { ensureDatabaseDocuments } = require("../server/routes/documents");
+
+function createStore(t) {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), "sqlite-hub-documents-"));
+  const store = new AppStateStore(path.join(directory, "state.db"));
+
+  t.after(() => {
+    store.db.close();
+    fs.rmSync(directory, { recursive: true, force: true });
+  });
+
+  return store;
+}
+
+test("database documents are scoped to one database and normalize filenames", (t) => {
+  const store = createStore(t);
+  const first = store.createDatabaseDocument("db-one", {
+    filename: "../Daily Notes",
+    content: "# Tasks\n\n- [ ] item1",
+  });
+  const second = store.createDatabaseDocument("db-one", {
+    filename: "Daily Notes.md",
+    content: "second",
+  });
+
+  assert.equal(first.filename, "Daily Notes.md");
+  assert.equal(second.filename, "Daily Notes 2.md");
+  assert.equal(store.listDatabaseDocuments("db-one").length, 2);
+  assert.equal(store.listDatabaseDocuments("db-two").length, 0);
+  assert.throws(() => store.getDatabaseDocument("db-two", first.id), /Document was not found/);
+});
+
+test("database document updates preserve raw markdown content", (t) => {
+  const store = createStore(t);
+  const document = store.createDatabaseDocument("db-one", {
+    filename: "todo",
+    content: "- [ ] item1",
+  });
+  const updated = store.updateDatabaseDocument("db-one", document.id, {
+    filename: "todo-renamed",
+    content: "- [x] item1\n1717682400",
+  });
+
+  assert.equal(updated.filename, "todo-renamed.md");
+  assert.equal(updated.content, "- [x] item1\n1717682400");
+
+  const deleted = store.deleteDatabaseDocument("db-one", document.id);
+  assert.deepEqual(deleted, { id: document.id, deleted: true });
+  assert.equal(store.listDatabaseDocuments("db-one").length, 0);
+});
+
+test("document folder creates one initial document for an empty database namespace", (t) => {
+  const store = createStore(t);
+  const connectionManager = {
+    getActiveConnection: () => ({
+      id: "db-one",
+      label: "trump.sqlite",
+      path: "/tmp/trump.sqlite",
+    }),
+  };
+
+  const firstList = ensureDatabaseDocuments({
+    appStateStore: store,
+    connectionManager,
+    databaseKey: "db-one",
+  });
+  const secondList = ensureDatabaseDocuments({
+    appStateStore: store,
+    connectionManager,
+    databaseKey: "db-one",
+  });
+  const document = store.getDatabaseDocument("db-one", firstList[0].id);
+
+  assert.equal(firstList.length, 1);
+  assert.equal(secondList.length, 1);
+  assert.equal(document.filename, "trump.sqlite.md");
+  assert.equal(document.content, "# trump.sqlite\n");
+});

package/tests/export-filenames.test.js

@@ -0,0 +1,34 @@
+const assert = require("node:assert/strict");
+const { readFileSync } = require("node:fs");
+const path = require("node:path");
+const test = require("node:test");
+
+let exportFilenamesModulePromise = null;
+
+function loadExportFilenamesModule() {
+  if (!exportFilenamesModulePromise) {
+    const source = readFileSync(
+      path.resolve(__dirname, "../frontend/js/utils/exportFilenames.js"),
+      "utf8"
+    );
+    const url = `data:text/javascript;base64,${Buffer.from(source).toString("base64")}`;
+
+    exportFilenamesModulePromise = import(url);
+  }
+
+  return exportFilenamesModulePromise;
+}
+
+test("text export filenames use the selected format extension", async () => {
+  const { buildTextExportFilename } = await loadExportFilenamesModule();
+
+  assert.equal(buildTextExportFilename("query-results.csv", { format: "tsv" }), "query-results.tsv");
+  assert.equal(buildTextExportFilename("white_house_live_streams", { format: "md" }), "white_house_live_streams.md");
+});
+
+test("text export filenames sanitize unsafe names and fallback when empty", async () => {
+  const { buildTextExportFilename } = await loadExportFilenamesModule();
+
+  assert.equal(buildTextExportFilename("", { format: "csv", fallback: "table" }), "table.csv");
+  assert.equal(buildTextExportFilename("../bad:name?.csv", { format: "csv", fallback: "table" }), "bad name.csv");
+});