open-webui-sqlite-migration 0.1.0__tar.gz

open_webui_sqlite_migration-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Digitalist Open Cloud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

open_webui_sqlite_migration-0.1.0/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.3
+Name: open-webui-sqlite-migration
+Version: 0.1.0
+Summary: Migrate Open WebUI from SQLite to PostgreSQL
+License: MIT
+Author: Mikke Schirén
+Author-email: mikke.schiren@digitalist.com
+Requires-Python: >=3.11
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: psycopg2-binary (>=2.9.11)
+Requires-Dist: rich (>=13.9.4)
+Description-Content-Type: text/markdown
+
+# Migrate tool for OpenWebUI SQLite
+
+Migrate from using SQLite database in Open WebUI to use Postgres.
+
+Inspiration from: <https://github.com/taylorwilsdon/open-webui-postgres-migration>,
+one of the big differences is that the migration require no input, with environment
+variables set, you just run the script, no input.
+
+This so you can automate the process, instead of manual input.
+
+## Configuration
+
+Before anything else, backup you SQLite database and keep it in a safe place.
+
+Needed environment variables:
+
+- `SQLITE_DB_PATH` - exact path to your open webui db, like: `/app/backend/data/webui.db`.
+- `MIGRATE_DATABASE_URL` - normally the same you should use for `DATABASE_URL`, like `postgresql://user:pass@postgres:5432/openwebui`
+
+Also you need to start Open WebUI with `DATABASE_URL`, so needed tables are created. After that,
+you remove that variable so you go back to use SQLite. When using SQLite, you run the migration script,
+then you stop Open WebUI, and then again set `DATABASE_URL`. If everything now runs smoothly, you can remove the
+SQLite database. Keep a backup of the database until you are really sure that all things are working as they
+should.
+
+## License
+
+MIT
+Copyright (c) Digitalist Open Cloud.
+

open_webui_sqlite_migration-0.1.0/README.md

@@ -0,0 +1,29 @@
+# Migrate tool for OpenWebUI SQLite
+
+Migrate from using SQLite database in Open WebUI to use Postgres.
+
+Inspiration from: <https://github.com/taylorwilsdon/open-webui-postgres-migration>,
+one of the big differences is that the migration require no input, with environment
+variables set, you just run the script, no input.
+
+This so you can automate the process, instead of manual input.
+
+## Configuration
+
+Before anything else, backup you SQLite database and keep it in a safe place.
+
+Needed environment variables:
+
+- `SQLITE_DB_PATH` - exact path to your open webui db, like: `/app/backend/data/webui.db`.
+- `MIGRATE_DATABASE_URL` - normally the same you should use for `DATABASE_URL`, like `postgresql://user:pass@postgres:5432/openwebui`
+
+Also you need to start Open WebUI with `DATABASE_URL`, so needed tables are created. After that,
+you remove that variable so you go back to use SQLite. When using SQLite, you run the migration script,
+then you stop Open WebUI, and then again set `DATABASE_URL`. If everything now runs smoothly, you can remove the
+SQLite database. Keep a backup of the database until you are really sure that all things are working as they
+should.
+
+## License
+
+MIT
+Copyright (c) Digitalist Open Cloud.

open_webui_sqlite_migration-0.1.0/open_webui_sqlite_migration/__init__.py

@@ -0,0 +1 @@
+__all__ = ["migrate"]

open_webui_sqlite_migration-0.1.0/open_webui_sqlite_migration/migrate.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+"""
+SQLite to PostgreSQL migration for Open WebUI
+"""
+
+import os
+import json
+import sqlite3
+import csv
+import argparse
+from pathlib import Path
+from typing import Dict, Iterable, List
+from io import StringIO
+
+import psycopg2
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn
+from rich.panel import Panel
+
+__version__ = "0.1.0"
+console = Console()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="SQLite → PostgreSQL migration for Open WebUI"
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Validate and preview migration without writing to PostgreSQL",
+    )
+    return parser.parse_args()
+
+ARGS = parse_args()
+DRY_RUN = ARGS.dry_run
+
+def env(key: str, default=None, *, required=False, cast=str):
+    value = os.getenv(key, default)
+    if required and value is None:
+        raise RuntimeError(f"Missing environment variable: {key}")
+    try:
+        return cast(value) if value is not None else value
+    except Exception:
+        raise RuntimeError(f"Invalid value for {key}: {value}")
+
+SQLITE_PATH = Path(env("SQLITE_DB_PATH", required=True))
+MIGRATE_DATABASE_URL = env("MIGRATE_DATABASE_URL", required=True)
+BATCH_SIZE = env("BATCH_SIZE", 5000, cast=int)
+
+def validate_sqlite(path: Path) -> None:
+    with sqlite3.connect(path) as conn:
+        conn.execute("PRAGMA integrity_check")
+
+def validate_postgres(db_url: str) -> None:
+    conn = psycopg2.connect(db_url)
+    conn.close()
+
+def pg_ident(name: str) -> str:
+    if name.lower() in {"user", "group", "order", "table", "select"}:
+        return f'"{name}"'
+    return name
+
+def sqlite_tables(conn: sqlite3.Connection) -> List[str]:
+    cur = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table'"
+    )
+    return [
+        r[0] for r in cur.fetchall()
+        if r[0] not in {"alembic_version", "migratehistory"}
+    ]
+
+def sqlite_schema(conn: sqlite3.Connection, table: str):
+    return conn.execute(f'PRAGMA table_info("{table}")').fetchall()
+
+def pg_column_types(conn, table: str) -> Dict[str, str]:
+    with conn.cursor() as cur:
+        cur.execute("""
+            SELECT column_name, data_type
+            FROM information_schema.columns
+            WHERE table_schema = 'public'
+              AND table_name = %s
+        """, (table,))
+        return dict(cur.fetchall())
+
+def stream_sqlite_rows(
+    conn: sqlite3.Connection,
+    table: str,
+    columns: List[str],
+) -> Iterable[tuple]:
+    col_sql = ", ".join(f'"{c}"' for c in columns)
+    cur = conn.execute(f'SELECT {col_sql} FROM "{table}"')
+    while True:
+        rows = cur.fetchmany(BATCH_SIZE)
+        if not rows:
+            break
+        for row in rows:
+            yield row
+
+def normalize_row(row, columns, pg_types):
+    out = []
+    for value, col in zip(row, columns):
+        col_type = pg_types.get(col)
+        if value is None:
+            out.append(None)
+        elif col_type == "jsonb":
+            if isinstance(value, (dict, list)):
+                out.append(json.dumps(value))
+            else:
+                try:
+                    json.loads(value)
+                    out.append(value)
+                except Exception:
+                    out.append("{}")
+        else:
+            out.append(value)
+    return tuple(out)
+
+def migrate_table(sqlite_conn: sqlite3.Connection, pg_conn, table: str):
+    sqlite_count = sqlite_conn.execute(
+        f'SELECT COUNT(*) FROM "{table}"'
+    ).fetchone()[0]
+
+    console.print(
+        f"[cyan]Table:[/] {table} "
+        f"[dim](rows: {sqlite_count})[/]"
+    )
+
+    if DRY_RUN:
+        console.print(f"[yellow]DRY‑RUN: for {table}[/]")
+        return
+
+    schema = sqlite_schema(sqlite_conn, table)
+    columns = [c[1] for c in schema]
+    pg_types = pg_column_types(pg_conn, table)
+
+    rows = (
+        normalize_row(row, columns, pg_types)
+        for row in stream_sqlite_rows(sqlite_conn, table, columns)
+    )
+
+    buffer = StringIO()
+    writer = csv.writer(
+        buffer,
+        lineterminator="\n",
+        quoting=csv.QUOTE_MINIMAL,
+    )
+
+    for row in rows:
+        writer.writerow("" if v is None else v for v in row)
+
+    buffer.seek(0)
+
+    with pg_conn.cursor() as cur:
+        cur.execute(f"TRUNCATE TABLE {pg_ident(table)} CASCADE")
+        cur.copy_expert(
+            f"COPY {pg_ident(table)} ({', '.join(columns)}) FROM STDIN WITH CSV",
+            buffer,
+        )
+
+    pg_conn.commit()
+
+def main():
+    console.print(
+        Panel(
+            f"SQLite → PostgreSQL Migration "
+            f"{'(DRY‑RUN)' if DRY_RUN else ''}",
+            style="cyan",
+        )
+    )
+
+    validate_sqlite(SQLITE_PATH)
+    validate_postgres(MIGRATE_DATABASE_URL)
+
+    sqlite_conn = sqlite3.connect(SQLITE_PATH)
+    sqlite_conn.text_factory = lambda b: b.decode("utf-8", errors="replace")
+
+    pg_conn = psycopg2.connect(MIGRATE_DATABASE_URL)
+
+    if DRY_RUN:
+        with pg_conn.cursor() as cur:
+            cur.execute("SET default_transaction_read_only = on")
+        console.print("[yellow]DRY‑RUN: PostgreSQL session is read‑only[/]")
+    else:
+        with pg_conn.cursor() as cur:
+            cur.execute("SET session_replication_role = replica")
+        pg_conn.commit()
+
+    tables = sqlite_tables(sqlite_conn)
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+    ) as progress:
+        task = progress.add_task("Processing tables...", total=len(tables))
+        for table in tables:
+            migrate_table(sqlite_conn, pg_conn, table)
+            progress.advance(task)
+
+    sqlite_conn.close()
+
+    if not DRY_RUN:
+        with pg_conn.cursor() as cur:
+            cur.execute("SET session_replication_role = origin")
+        pg_conn.commit()
+
+    pg_conn.close()
+
+    console.print(Panel("✅ Done", style="green"))
+
+if __name__ == "__main__":
+    main()

open_webui_sqlite_migration-0.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "open-webui-sqlite-migration"
+version = "0.1.0"
+description = "Migrate Open WebUI from SQLite to PostgreSQL"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [
+    { name = "Mikke Schirén", email = "mikke.schiren@digitalist.com" }
+]
+requires-python = ">=3.11"
+
+dependencies = [
+    "psycopg2-binary>=2.9.11",
+    "rich>=13.9.4"
+]
+
+[project.scripts]
+open-webui-migrate-sqlite = "open_webui_sqlite_migration.migrate:main"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+package-mode = true