ducklake-client 0.1.0__tar.gz

ducklake_client-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,74 @@
+name: release
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: "Source ref to release from"
+        required: true
+        default: "main"
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.source }}
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Resolve release version
+        id: release
+        shell: bash
+        run: |
+          version="$(python - <<'PY'
+          from pathlib import Path
+          import tomllib
+
+          pyproject = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))
+          print(pyproject["project"]["version"])
+          PY
+          )"
+          branch="release/v${version}"
+          echo "version=${version}" >> "${GITHUB_OUTPUT}"
+          echo "branch=${branch}" >> "${GITHUB_OUTPUT}"
+          echo "Release version: ${version}"
+          echo "Release branch: ${branch}"
+
+      - name: Ensure release branch does not exist
+        shell: bash
+        run: |
+          branch="${{ steps.release.outputs.branch }}"
+          if git ls-remote --exit-code --heads origin "${branch}" >/dev/null 2>&1; then
+            echo "::error::Release branch ${branch} already exists"
+            exit 1
+          fi
+
+      - name: Create release branch
+        shell: bash
+        run: |
+          branch="${{ steps.release.outputs.branch }}"
+          git switch -c "${branch}"
+          git push origin "HEAD:refs/heads/${branch}"
+
+      - name: Build package
+        shell: bash
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

ducklake_client-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+demo/
+demo.ducklake/
+
+# Virtual environments
+.venv

ducklake_client-0.1.0/.python-version

@@ -0,0 +1 @@
+3.14

ducklake_client-0.1.0/PKG-INFO

@@ -0,0 +1,135 @@
+Metadata-Version: 2.4
+Name: ducklake-client
+Version: 0.1.0
+Summary: Lightweight wrapper over DuckDB with convenience helpers for DuckLake.
+Author-email: Ekku Leivonen <ekku.leivonen@gmail.com>
+Requires-Python: >=3.14
+Requires-Dist: duckdb
+Description-Content-Type: text/markdown
+
+# ducklake-client
+
+Lightweight Python helpers for opening DuckLake connections through DuckDB.
+
+## Install
+
+```bash
+pip install ducklake-client
+```
+
+## Open a DuckLake connection
+
+```python
+from ducklake_client import ColumnDef, DiskStorage, DuckDBCatalog, DuckLake
+
+lake = DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+)
+
+try:
+    lake.schema.create("main")
+    lake.table.create(
+        "items",
+        id=ColumnDef("INTEGER", nullable=False),
+        name=ColumnDef("VARCHAR"),
+    )
+    rows = lake.connection.sql("SELECT * FROM lake.main.items").fetchall()
+finally:
+    lake.close()
+```
+
+`DuckLake` opens the underlying DuckDB connection lazily on first use. The client installs and loads the DuckDB `ducklake` and `parquet` extensions, attaches the catalog as `lake`, and exposes the native DuckDB connection through `connection`.
+
+## Context manager usage
+
+```python
+from ducklake_client import DiskStorage, DuckDBCatalog, DuckLake
+
+with DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+) as lake:
+    lake.connection.execute("CREATE TABLE IF NOT EXISTS lake.main.events (id INTEGER)")
+    lake.connection.execute("INSERT INTO lake.main.events VALUES (?)", [1])
+    print(lake.connection.sql("SELECT count(*) FROM lake.main.events").fetchone())
+```
+
+## Modules
+
+DuckLake-specific helpers are grouped into modules. Native DuckDB behavior stays on `lake.connection`.
+
+```python
+from ducklake_client import ColumnDef, DiskStorage, DuckDBCatalog, DuckLake
+
+with DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+) as lake:
+    lake.schema.create("main")
+    lake.table.create_from_csv(
+        "nl_train_stations",
+        "https://blobs.duckdb.org/nl_stations.csv",
+    )
+    lake.table.comment("nl_train_stations", "Dutch railway stations")
+    lake.table.comment(
+        "nl_train_stations",
+        "Full station name",
+        column_name="name_long",
+    )
+    tables = lake.table.list()
+    views = lake.view.list()
+    info = lake.table.info("nl_train_stations")
+```
+
+## Transactions
+
+Use `transaction()` to automatically begin, commit, or roll back a block on the native DuckDB connection.
+
+```python
+from ducklake_client import ColumnDef, DiskStorage, DuckDBCatalog, DuckLake
+
+with DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+) as lake:
+    with lake.transaction():
+        lake.schema.create("main")
+        lake.table.create(
+            "items",
+            id=ColumnDef("INTEGER", nullable=False),
+            name=ColumnDef("VARCHAR"),
+        )
+        lake.connection.execute("INSERT INTO lake.main.items VALUES (?, ?)", [1, "example"])
+```
+
+## Configuration
+
+`DuckLake` requires explicit catalog and storage config objects:
+
+```python
+from ducklake_client import DiskStorage, DuckDBCatalog, DuckLake
+
+lake = DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+)
+```
+
+You can pass DuckDB runtime settings with `DuckDBConfig`:
+
+```python
+from ducklake_client import DiskStorage, DuckDBConfig, DuckDBCatalog, DuckLake
+
+lake = DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+    duckdb=DuckDBConfig(
+        database=":memory:",
+        threads=4,
+        memory_limit="2GB",
+    ),
+)
+```
+
+Catalogs can be `DuckDBCatalog`, `SqliteCatalog`, or `PostgresCatalog`. Storage can be `DiskStorage` or `S3Storage`.

ducklake_client-0.1.0/README.md

@@ -0,0 +1,126 @@
+# ducklake-client
+
+Lightweight Python helpers for opening DuckLake connections through DuckDB.
+
+## Install
+
+```bash
+pip install ducklake-client
+```
+
+## Open a DuckLake connection
+
+```python
+from ducklake_client import ColumnDef, DiskStorage, DuckDBCatalog, DuckLake
+
+lake = DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+)
+
+try:
+    lake.schema.create("main")
+    lake.table.create(
+        "items",
+        id=ColumnDef("INTEGER", nullable=False),
+        name=ColumnDef("VARCHAR"),
+    )
+    rows = lake.connection.sql("SELECT * FROM lake.main.items").fetchall()
+finally:
+    lake.close()
+```
+
+`DuckLake` opens the underlying DuckDB connection lazily on first use. The client installs and loads the DuckDB `ducklake` and `parquet` extensions, attaches the catalog as `lake`, and exposes the native DuckDB connection through `connection`.
+
+## Context manager usage
+
+```python
+from ducklake_client import DiskStorage, DuckDBCatalog, DuckLake
+
+with DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+) as lake:
+    lake.connection.execute("CREATE TABLE IF NOT EXISTS lake.main.events (id INTEGER)")
+    lake.connection.execute("INSERT INTO lake.main.events VALUES (?)", [1])
+    print(lake.connection.sql("SELECT count(*) FROM lake.main.events").fetchone())
+```
+
+## Modules
+
+DuckLake-specific helpers are grouped into modules. Native DuckDB behavior stays on `lake.connection`.
+
+```python
+from ducklake_client import ColumnDef, DiskStorage, DuckDBCatalog, DuckLake
+
+with DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+) as lake:
+    lake.schema.create("main")
+    lake.table.create_from_csv(
+        "nl_train_stations",
+        "https://blobs.duckdb.org/nl_stations.csv",
+    )
+    lake.table.comment("nl_train_stations", "Dutch railway stations")
+    lake.table.comment(
+        "nl_train_stations",
+        "Full station name",
+        column_name="name_long",
+    )
+    tables = lake.table.list()
+    views = lake.view.list()
+    info = lake.table.info("nl_train_stations")
+```
+
+## Transactions
+
+Use `transaction()` to automatically begin, commit, or roll back a block on the native DuckDB connection.
+
+```python
+from ducklake_client import ColumnDef, DiskStorage, DuckDBCatalog, DuckLake
+
+with DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+) as lake:
+    with lake.transaction():
+        lake.schema.create("main")
+        lake.table.create(
+            "items",
+            id=ColumnDef("INTEGER", nullable=False),
+            name=ColumnDef("VARCHAR"),
+        )
+        lake.connection.execute("INSERT INTO lake.main.items VALUES (?, ?)", [1, "example"])
+```
+
+## Configuration
+
+`DuckLake` requires explicit catalog and storage config objects:
+
+```python
+from ducklake_client import DiskStorage, DuckDBCatalog, DuckLake
+
+lake = DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+)
+```
+
+You can pass DuckDB runtime settings with `DuckDBConfig`:
+
+```python
+from ducklake_client import DiskStorage, DuckDBConfig, DuckDBCatalog, DuckLake
+
+lake = DuckLake(
+    catalog=DuckDBCatalog("metadata.ducklake"),
+    storage=DiskStorage("data"),
+    duckdb=DuckDBConfig(
+        database=":memory:",
+        threads=4,
+        memory_limit="2GB",
+    ),
+)
+```
+
+Catalogs can be `DuckDBCatalog`, `SqliteCatalog`, or `PostgresCatalog`. Storage can be `DiskStorage` or `S3Storage`.

ducklake_client-0.1.0/demo.py

@@ -0,0 +1,31 @@
+"""Examples for ducklake-client."""
+
+import pprint
+from ducklake_client import DiskStorage, DuckDBCatalog, DuckLake
+
+
+def main() -> None:
+    with DuckLake(
+        catalog=DuckDBCatalog("demo.ducklake"),
+        storage=DiskStorage("demo"),
+    ) as lake:
+        with lake.transaction():
+            lake.schema.create("main")
+            lake.table.create_from_csv(
+                "nl_train_stations",
+                "https://blobs.duckdb.org/nl_stations.csv",
+            )
+            lake.table.comment("nl_train_stations", "Dutch railway stations")
+            lake.table.comment(
+                "nl_train_stations",
+                "Station ID",
+                column_name="id",
+            )
+
+        # rows = lake.connection.sql("SELECT * FROM lake.main.nl_train_stations LIMIT 5").fetchall()
+        info = lake.table.info("nl_train_stations")
+        pprint.pprint(info.columns[0], indent=2)
+
+
+if __name__ == "__main__":
+    main()

ducklake_client-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "ducklake-client"
+version = "0.1.0"
+description = "Lightweight wrapper over DuckDB with convenience helpers for DuckLake."
+readme = "README.md"
+requires-python = ">=3.14"
+dependencies = [
+  "duckdb",
+]
+
+
+authors = [
+  { name = "Ekku Leivonen", email = "ekku.leivonen@gmail.com" }
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ducklake_client"]

ducklake_client-0.1.0/src/ducklake_client/__init__.py

@@ -0,0 +1,63 @@
+"""Lightweight Python helpers for DuckLake connections."""
+
+from ducklake_client.config import (
+    CatalogConfig,
+    DiskStorage,
+    DuckDBCatalog,
+    DuckDBConfig,
+    PostgresCatalog,
+    S3Storage,
+    SqliteCatalog,
+    StorageConfig,
+)
+from ducklake_client.exceptions import (
+    DuckLakeConfigError,
+    DuckLakeConnectionError,
+    DuckLakeError,
+    DuckLakeQueryError,
+)
+from ducklake_client.client import DuckLake
+from ducklake_client.modules import SchemaModule, TableModule, ViewModule
+from ducklake_client.schema import (
+    ColumnDataType,
+    ColumnDef,
+    DuckLakeTableMetadata,
+    TableColumnSummary,
+    TableInfo,
+    TableInfoColumn,
+    TableListing,
+    TablePartitionSpec,
+    TableSnapshotInfo,
+    TableSortSpec,
+    ViewListing,
+)
+
+__all__ = [
+    "CatalogConfig",
+    "ColumnDataType",
+    "ColumnDef",
+    "DuckDBCatalog",
+    "DuckDBConfig",
+    "DuckLake",
+    "DuckLakeConfigError",
+    "DuckLakeConnectionError",
+    "DuckLakeError",
+    "DuckLakeQueryError",
+    "DuckLakeTableMetadata",
+    "DiskStorage",
+    "PostgresCatalog",
+    "S3Storage",
+    "SchemaModule",
+    "SqliteCatalog",
+    "StorageConfig",
+    "TableColumnSummary",
+    "TableInfo",
+    "TableInfoColumn",
+    "TableListing",
+    "TableModule",
+    "TablePartitionSpec",
+    "TableSnapshotInfo",
+    "TableSortSpec",
+    "ViewModule",
+    "ViewListing",
+]

ducklake_client-0.1.0/src/ducklake_client/_attach.py

@@ -0,0 +1,34 @@
+"""Internal helpers for DuckLake ATTACH SQL generation."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from ducklake_client.config import CatalogConfig, StorageConfig, quote_identifier, quote_literal
+
+
+def build_attach_sql(
+    *,
+    catalog: CatalogConfig,
+    storage: StorageConfig,
+    alias: str,
+    attach_options: Mapping[str, object] | None = None,
+) -> str:
+    options: dict[str, object] = {"DATA_PATH": storage.data_path()}
+    if attach_options:
+        options.update(attach_options)
+    rendered_options = ", ".join(
+        f"{key.upper()} {_format_attach_value(value)}" for key, value in options.items()
+    )
+    return (
+        f"ATTACH {quote_literal(catalog.attach_uri())} "
+        f"AS {quote_identifier(alias)} ({rendered_options})"
+    )
+
+
+def _format_attach_value(value: object) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int | float):
+        return str(value)
+    return quote_literal(value)

ducklake_client-0.1.0/src/ducklake_client/_connection.py

@@ -0,0 +1,95 @@
+"""Lazy DuckDB connection management for DuckLake."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, cast
+
+from ducklake_client._attach import build_attach_sql
+from ducklake_client.config import (
+    CatalogConfig,
+    DuckDBConfig,
+    DuckDBSettingValue,
+    StorageConfig,
+    quote_literal,
+)
+from ducklake_client.exceptions import DuckLakeConnectionError
+
+if TYPE_CHECKING:
+    import duckdb
+
+_SETTING_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+@dataclass
+class ConnectionManager:
+    catalog: CatalogConfig
+    storage: StorageConfig
+    alias: str
+    duckdb: DuckDBConfig = field(default_factory=DuckDBConfig)
+    attach_options: Mapping[str, object] | None = None
+    _connection: duckdb.DuckDBPyConnection | None = field(default=None, init=False, repr=False)
+
+    def get(self) -> duckdb.DuckDBPyConnection:
+        if self._connection is None:
+            self._connection = self._connect()
+        return self._connection
+
+    def close(self) -> None:
+        if self._connection is not None:
+            self._connection.close()
+            self._connection = None
+
+    def _connect(self) -> duckdb.DuckDBPyConnection:
+        try:
+            import duckdb
+
+            config = cast(dict[str, str | bool | int | float | list[str]], dict(self.duckdb.config))
+            conn = (
+                duckdb.connect(str(self.duckdb.database), config=config)
+                if config
+                else duckdb.connect(str(self.duckdb.database))
+            )
+            for name, value in self.duckdb.runtime_settings().items():
+                conn.execute(_setting_sql(name, value))
+            for extension in self._required_extensions():
+                if self.duckdb.install_extensions:
+                    conn.install_extension(extension)
+                conn.load_extension(extension)
+            for statement in self.storage.setup_statements(secret_name=f"{self.alias}_storage"):
+                conn.execute(statement)
+            conn.execute(
+                build_attach_sql(
+                    catalog=self.catalog,
+                    storage=self.storage,
+                    alias=self.alias,
+                    attach_options=self.attach_options,
+                )
+            )
+            return conn
+        except Exception as exc:
+            raise DuckLakeConnectionError("failed to initialize DuckLake connection") from exc
+
+    def _required_extensions(self) -> tuple[str, ...]:
+        names = [
+            "ducklake",
+            "parquet",
+            *self.catalog.required_extensions(),
+            *self.storage.required_extensions(),
+            *self.duckdb.extensions,
+        ]
+        return tuple(dict.fromkeys(names))
+
+
+def _setting_sql(name: str, value: DuckDBSettingValue) -> str:
+    if not _SETTING_NAME.fullmatch(name):
+        raise DuckLakeConnectionError(f"invalid DuckDB setting name: {name!r}")
+    if isinstance(value, bool):
+        rendered_value = "true" if value else "false"
+    elif isinstance(value, int | float):
+        rendered_value = str(value)
+    else:
+        rendered_value = quote_literal(value)
+    return f"SET {name} = {rendered_value}"

ducklake_client-0.1.0/src/ducklake_client/client.py

@@ -0,0 +1,100 @@
+"""Public DuckLake client entry point."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from functools import cached_property
+from typing import TYPE_CHECKING
+
+from ducklake_client._connection import ConnectionManager
+from ducklake_client.config import (
+    CatalogConfig,
+    CatalogInput,
+    DuckDBConfig,
+    StorageConfig,
+    StorageInput,
+)
+from ducklake_client.modules.schema import SchemaModule
+from ducklake_client.modules.table import TableModule
+from ducklake_client.modules.view import ViewModule
+
+if TYPE_CHECKING:
+    import duckdb
+
+
+class DuckLake:
+    """A lazy DuckLake connection wrapper."""
+
+    def __init__(
+        self,
+        *,
+        catalog: CatalogInput,
+        storage: StorageInput,
+        alias: str = "lake",
+        duckdb: DuckDBConfig | None = None,
+        attach_options: Mapping[str, object] | None = None,
+    ) -> None:
+        if not isinstance(catalog, CatalogConfig):
+            raise TypeError("catalog must be a DuckDBCatalog, PostgresCatalog, or SqliteCatalog")
+        if not isinstance(storage, StorageConfig):
+            raise TypeError("storage must be a DiskStorage or S3Storage")
+
+        self.alias = alias
+        self._manager = ConnectionManager(
+            catalog=catalog,
+            storage=storage,
+            alias=alias,
+            duckdb=duckdb or DuckDBConfig(),
+            attach_options=attach_options,
+        )
+
+    @property
+    def connection(self) -> duckdb.DuckDBPyConnection:
+        return self._manager.get()
+
+    @cached_property
+    def schema(self) -> SchemaModule:
+        return SchemaModule(self)
+
+    @cached_property
+    def table(self) -> TableModule:
+        return TableModule(self)
+
+    @cached_property
+    def view(self) -> ViewModule:
+        return ViewModule(self)
+
+    @contextmanager
+    def transaction(self) -> Iterator[DuckLake]:
+        """Run a block inside a DuckDB transaction on this lake's connection."""
+
+        connection = self.connection
+        connection.begin()
+        try:
+            yield self
+        except BaseException:
+            try:
+                connection.rollback()
+            except Exception:
+                pass
+            raise
+        else:
+            try:
+                connection.commit()
+            except Exception:
+                try:
+                    connection.rollback()
+                except Exception:
+                    pass
+                raise
+
+    def close(self) -> None:
+        self._manager.close()
+
+    def __enter__(self) -> DuckLake:
+        self.connection
+        return self
+
+    def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
+        self.close()