sqlite2duckdb 0.4.0__py3-none-any.whl

sqlite2duckdb/__init__.py

@@ -0,0 +1,14 @@
+import importlib.metadata
+
+from sqlite2duckdb.sqlite_to_duckdb import ConversionResult, sqlite_to_duckdb
+
+try:
+    __version__ = importlib.metadata.version("sqlite2duckdb")
+except importlib.metadata.PackageNotFoundError:
+    # Running from a source checkout that was never installed.
+    __version__ = "0.0.0.dev0"
+
+# Deprecated alias, kept so that existing imports keep working.
+__VERSION__ = __version__
+
+__all__ = ["ConversionResult", "__version__", "sqlite_to_duckdb"]

sqlite2duckdb/__main__.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+
+from sqlite2duckdb import __version__, sqlite_to_duckdb
+
+
+def main_cli() -> int:
+    parser = argparse.ArgumentParser(
+        prog="sqlite2duckdb",
+        description="Convert Sqlite database to Duckdb database",
+        usage="sqlite2duckdb [-f] <sqlite_path> <duckdb_path>",
+    )
+
+    parser.add_argument("sqlite_path", type=str, help="sqlite file path")
+    parser.add_argument("duckdb_path", type=str, help="duckdb file path")
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="overwrite the duckdb file if it already exists",
+    )
+    parser.add_argument("-q", "--quiet", action="store_true", help="only report errors")
+    parser.add_argument("--verbose", action="store_true", help="report every step")
+    parser.add_argument(
+        "-v", "--version", action="version", version=f"sqlite2duckdb {__version__}"
+    )
+
+    args = parser.parse_args()
+
+    if args.quiet:
+        level = logging.WARNING
+    elif args.verbose:
+        level = logging.DEBUG
+    else:
+        level = logging.INFO
+    # Progress goes to stderr so that stdout stays free for pipelines.
+    logging.basicConfig(level=level, format="%(message)s", stream=sys.stderr)
+
+    overwrite = args.force
+    if not overwrite and os.path.exists(args.duckdb_path):
+        if not sys.stdin.isatty():
+            print(
+                f"{args.duckdb_path} already exists. Use --force to overwrite it.",
+                file=sys.stderr,
+            )
+            return 1
+        try:
+            answer = (
+                input(
+                    f"{args.duckdb_path} already exists. do you want to delete this file ? (yes/no): "
+                )
+                .strip()
+                .lower()
+            )
+        except EOFError:
+            print(f"{args.duckdb_path} already exists.", file=sys.stderr)
+            return 1
+        if answer not in ("yes", "y"):
+            return 1
+        overwrite = True
+
+    try:
+        sqlite_to_duckdb(args.sqlite_path, args.duckdb_path, overwrite=overwrite)
+    except (FileNotFoundError, FileExistsError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main_cli())

sqlite2duckdb/sqlite_to_duckdb.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import contextlib
+import logging
+import os
+import sqlite3
+import time
+from dataclasses import dataclass
+
+import duckdb
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ConversionResult:
+    """What a successful conversion produced."""
+
+    target: str
+    tables: int
+    elapsed: float
+    """Wall clock duration of the conversion, in seconds."""
+
+
+def _quote_identifier(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'
+
+
+def _format_duration(seconds: float) -> str:
+    if seconds < 1:
+        return f"{seconds * 1000:.2f} ms"
+    return f"{seconds:.2f} s"
+
+
+def _remove_quietly(path: str) -> None:
+    for candidate in (path, path + ".wal"):
+        try:
+            os.remove(candidate)
+        except OSError:
+            pass
+
+
+def _brackets_to_quotes(sql: str) -> str:
+    """Rewrite sqlite's [bracket] identifiers into standard "double quoted" ones.
+
+    Quoted runs are copied verbatim so that a `[` inside a string literal survives.
+    """
+
+    out = []
+    index = 0
+    length = len(sql)
+
+    while index < length:
+        char = sql[index]
+
+        if char in "'\"`":
+            end = index + 1
+            while end < length:
+                if sql[end] == char:
+                    if end + 1 < length and sql[end + 1] == char:
+                        end += 2
+                        continue
+                    break
+                end += 1
+            out.append(sql[index : end + 1])
+            index = end + 1
+
+        elif char == "[":
+            end = sql.find("]", index)
+            if end == -1:
+                out.append(char)
+                index += 1
+            else:
+                out.append(_quote_identifier(sql[index + 1 : end]))
+                index = end + 1
+
+        else:
+            out.append(char)
+            index += 1
+
+    return "".join(out)
+
+
+def _copy_indexes(conn: duckdb.DuckDBPyConnection, sqlite_path: str) -> None:
+    """Recreate the source indexes, translating sqlite quoting on the way."""
+
+    with contextlib.closing(sqlite3.connect(sqlite_path)) as source:
+        indexes = source.execute(
+            "SELECT name, sql FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL"
+        ).fetchall()
+
+    for name, sql in indexes:
+        try:
+            conn.sql(_brackets_to_quotes(sql))
+        except duckdb.Error as error:
+            logger.warning("Could not recreate index %s: %s", name, error)
+
+
+def _copy_tables(
+    conn: duckdb.DuckDBPyConnection, tables: list[tuple[str, str]]
+) -> None:
+    """Recreate each table then fill it from the attached database."""
+
+    for table_name, ddl in tables:
+        # This DDL is regenerated by duckdb from the attached catalog, so unlike
+        # sqlite's own it always parses, and it still carries the primary keys
+        # and the NOT NULL constraints.
+        conn.sql(ddl)
+        quoted = _quote_identifier(table_name)
+        conn.sql(f"INSERT INTO {quoted} SELECT * FROM __other.{quoted}")
+
+
+def sqlite_to_duckdb(
+    sqlite_db: str | os.PathLike[str],
+    duck_db: str | os.PathLike[str],
+    *,
+    overwrite: bool = False,
+) -> ConversionResult:
+    """Copy a sqlite database into a new duckdb database.
+
+    Tables, data, primary keys, NOT NULL constraints and indexes are copied.
+    Views and UNIQUE / FOREIGN KEY / CHECK constraints are not: duckdb's sqlite
+    extension does not expose them on the attached database.
+
+    Raises FileNotFoundError if `sqlite_db` does not exist, and FileExistsError
+    if `duck_db` already exists and `overwrite` is False.
+    """
+
+    sqlite_path = os.fspath(sqlite_db)
+    duck_path = os.fspath(duck_db)
+
+    if not os.path.exists(sqlite_path):
+        raise FileNotFoundError(f"File {sqlite_path} doesn't exist")
+
+    if os.path.exists(duck_path):
+        if not overwrite:
+            raise FileExistsError(f"Database {duck_path} already exists")
+        _remove_quietly(duck_path)
+
+    logger.info("Create %s database", duck_path)
+    start_time = time.perf_counter()
+
+    conn = duckdb.connect(duck_path)
+    try:
+        ## Install sqlite
+        conn.sql("INSTALL sqlite; LOAD sqlite;")
+
+        # Bound parameters are not allowed in ATTACH, so escape the quotes ourselves
+        source_path = sqlite_path.replace("'", "''")
+        conn.sql(f"ATTACH '{source_path}' AS __other (TYPE SQLITE, READ_ONLY)")
+
+        ## Get sqlite Names, along with the CREATE statement duckdb derives for them
+        tables = conn.sql(
+            "SELECT table_name, sql FROM duckdb_tables WHERE database_name = '__other'"
+        ).fetchall()
+        logger.info("%d table(s) found", len(tables))
+
+        _copy_tables(conn, tables)
+        _copy_indexes(conn, sqlite_path)
+
+        conn.sql("DETACH __other")
+    except BaseException:
+        # Never leave a half-written database behind: it would trip the
+        # "already exists" guard on the next run.
+        conn.close()
+        _remove_quietly(duck_path)
+        raise
+
+    conn.close()
+
+    elapsed = time.perf_counter() - start_time
+    logger.info("Done in %s !", _format_duration(elapsed))
+
+    return ConversionResult(target=duck_path, tables=len(tables), elapsed=elapsed)

sqlite2duckdb-0.4.0.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.5
+Name: sqlite2duckdb
+Version: 0.4.0
+Summary: A tool to convert sqlite database to duckdb database
+Project-URL: Homepage, https://github.com/dridk/sqlite2duckdb
+Project-URL: Issues, https://github.com/dridk/sqlite2duckdb/issues
+Author-email: Sacha Schutz <sacha.schutz@pm.me>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: database,duckdb,olap,oltp,sqlite
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.9
+Requires-Dist: duckdb>=1.1.0
+Description-Content-Type: text/markdown
+
+# sqlite2duckdb
+
+![CI](https://github.com/dridk/sqlite2duckdb/actions/workflows/ci.yml/badge.svg)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlite2duckdb)
+![PyPI - Downloads](https://img.shields.io/pypi/dm/sqlite2duckdb)
+
+A tool for converting a [sqlite](https://www.sqlite.org/) database into a [duckdb](https://duckdb.org/) database
+
+## Description
+
+Sqlite is an embedded online database designed for transactional reading and writing.
+Duckdb is also an embedded database, but column-oriented, designed for analytical process with a very high reading efficiency.
+
+For more details [https://towardsdatascience.com/forget-about-sqlite-use-duckdb-instead-and-thank-me-later-df76ee9bb777](https://towardsdatascience.com/forget-about-sqlite-use-duckdb-instead-and-thank-me-later-df76ee9bb777)
+
+Requires Python >= 3.9 and duckdb >= 1.1.0 (indexes are only copied from that version on).
+
+## Installation
+
+With [uv](https://docs.astral.sh/uv/), no installation is required. `uvx` downloads and runs the tool in one go:
+
+```bash
+uvx sqlite2duckdb source.db target.db
+```
+
+To keep it around:
+
+```bash
+uv tool install sqlite2duckdb
+```
+
+Or with pip:
+
+```bash
+pip install sqlite2duckdb
+```
+
+## Usage
+
+### As a command line
+
+```
+usage: sqlite2duckdb [-f] <sqlite_path> <duckdb_path>
+
+Convert Sqlite database to Duckdb database
+
+positional arguments:
+  sqlite_path    sqlite file path
+  duckdb_path    duckdb file path
+
+options:
+  -h, --help     show this help message and exit
+  -f, --force    overwrite the duckdb file if it already exists
+  -q, --quiet    only report errors
+  --verbose      report every step
+  -v, --version  show program's version number and exit
+```
+
+The tool never overwrites an existing target silently. On a terminal it asks for
+confirmation; anywhere else (a script, a CI job, a pipe) it exits with code 1 and tells you
+to pass `--force`. Progress is written to stderr, so stdout stays free for pipelines.
+
+### Examples
+
+```bash
+uvx sqlite2duckdb source.db target.db
+uvx sqlite2duckdb --force source.db target.db   # overwrite target.db without asking
+```
+
+### From python
+
+```python
+from sqlite2duckdb import sqlite_to_duckdb
+
+result = sqlite_to_duckdb("source.sqlite", "target.duckdb")
+print(result.tables, result.elapsed)
+```
+
+`sqlite_to_duckdb(sqlite_db, duck_db, *, overwrite=False)` accepts `str` or `pathlib.Path`
+and returns a `ConversionResult` (`target`, `tables`, `elapsed`). It raises
+`FileNotFoundError` if the source is missing and `FileExistsError` if the target already
+exists and `overwrite` is False. If the conversion fails halfway, the partially written
+target file is removed rather than left behind. Progress is reported through the standard
+`logging` module (logger `sqlite2duckdb.sqlite_to_duckdb`), never printed.
+
+## What is converted
+
+| | |
+|---|---|
+| Tables and data | ✅ |
+| Primary keys, NOT NULL constraints, indexes | ✅ |
+| UNIQUE, FOREIGN KEY and CHECK constraints | ❌ |
+| Views | ❌ (silently dropped) |
+
+Duckdb's sqlite extension does not expose the last two on the attached database, so they
+cannot be copied. Reading them back from `sqlite_master` would be needed.
+
+Tables are recreated from the DDL duckdb derives for the attached database, then filled
+from it, and the indexes are read back from `sqlite_master`. This is what makes sqlite
+files that quote their DDL with `[brackets]` (chinook.db, MS Access exports) convert
+correctly: duckdb's own parser rejects that syntax, so the quoting is translated first.
+
+## Todo
+
+- [ ] Custom type mapping
+- [x] Primary keys, NOT NULL constraints and indexes
+- [ ] Views, and UNIQUE / FOREIGN KEY / CHECK constraints
+
+## Contributing
+
+The project uses [uv](https://docs.astral.sh/uv/) for everything:
+
+```bash
+make dev     # uv sync — installs duckdb plus the test deps (pytest, faker, ruff)
+make test    # uv run pytest
+make lint    # uv run ruff check . && uv run ruff format --check .
+make build   # uv build
+make publish # uv publish (PyPI trusted publishing, also run on tags by CI)
+```
+
+### See also
+
+- [Harlequin](https://github.com/tconbeer/harlequin): A nice duckdb IDE for your terminal

sqlite2duckdb-0.4.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+sqlite2duckdb/__init__.py,sha256=GJEB0tNxkIzTIHMFTpUyywaGqLRMp6zWYCiniVMc74I,473
+sqlite2duckdb/__main__.py,sha256=o4BxA4togX5XVZGtqwxebtLK0Sk3K8W9sqEccflKVAo,2319
+sqlite2duckdb/sqlite_to_duckdb.py,sha256=E_8RumKshUB_XqePhNQ2okPKUpJmUnKQB7znb5Sn6Fk,5166
+sqlite2duckdb-0.4.0.dist-info/METADATA,sha256=MQjtJBMAoU1FtiQNotPpW0FiR4QSg-ggRKilmdYzMs4,4951
+sqlite2duckdb-0.4.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+sqlite2duckdb-0.4.0.dist-info/entry_points.txt,sha256=HXDOw1q3RRjrTvb8pZ9cfwp41b0AQXaSwCF7CQPlgQA,66
+sqlite2duckdb-0.4.0.dist-info/licenses/LICENSE,sha256=m02ByVVkLjRHEV9sN02uVqtLRTfaO74CBS_s8eA_UII,1069
+sqlite2duckdb-0.4.0.dist-info/RECORD,,

sqlite2duckdb-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

sqlite2duckdb-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sqlite2duckdb = sqlite2duckdb.__main__:main_cli

sqlite2duckdb-0.4.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 sacha schutz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.