duckbricks-utils 0.1.0__tar.gz

duckbricks_utils-0.1.0/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.3
+Name: duckbricks-utils
+Version: 0.1.0
+Summary: DuckLake connection utilities for DuckBricks notebooks and pipelines
+Author: DuckBricks Team
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: duckdb (>=1.3.0)
+Requires-Dist: python-dotenv (>=1.0)
+Description-Content-Type: text/markdown
+
+# duckbricks-utils
+
+DuckLake connection utilities for DuckBricks notebooks and pipelines.
+
+## Installation
+
+```bash
+# From local path (development)
+pip install /path/to/duckbricks-utils
+
+# Once published to PyPI
+pip install duckbricks-utils
+```
+
+## Usage
+
+```python
+from duckbricks_utils import connect
+
+conn = connect()
+result = conn.execute("SELECT * FROM my_table LIMIT 10").df()
+```
+
+## Configuration
+
+Connection settings are read from environment variables:
+
+| Variable | Default | Description |
+|---|---|---|
+| `DUCKLAKE_PG_HOST` | `localhost` | PostgreSQL host |
+| `DUCKLAKE_PG_PORT` | `5432` | PostgreSQL port |
+| `DUCKLAKE_PG_DATABASE` | `duckbricks` | PostgreSQL database name |
+| `DUCKLAKE_PG_USER` | `duckbricks` | PostgreSQL user |
+| `DUCKLAKE_PG_PASSWORD` | `duckbricks` | PostgreSQL password |
+| `DUCKBRICKS_DUCKLAKE_NAME` | `duckbricks` | DuckLake catalog name |
+| `DUCKBRICKS_DATA_PATH` | `/data/parquet/` | Parquet storage path |
+

duckbricks_utils-0.1.0/README.md

@@ -0,0 +1,36 @@
+# duckbricks-utils
+
+DuckLake connection utilities for DuckBricks notebooks and pipelines.
+
+## Installation
+
+```bash
+# From local path (development)
+pip install /path/to/duckbricks-utils
+
+# Once published to PyPI
+pip install duckbricks-utils
+```
+
+## Usage
+
+```python
+from duckbricks_utils import connect
+
+conn = connect()
+result = conn.execute("SELECT * FROM my_table LIMIT 10").df()
+```
+
+## Configuration
+
+Connection settings are read from environment variables:
+
+| Variable | Default | Description |
+|---|---|---|
+| `DUCKLAKE_PG_HOST` | `localhost` | PostgreSQL host |
+| `DUCKLAKE_PG_PORT` | `5432` | PostgreSQL port |
+| `DUCKLAKE_PG_DATABASE` | `duckbricks` | PostgreSQL database name |
+| `DUCKLAKE_PG_USER` | `duckbricks` | PostgreSQL user |
+| `DUCKLAKE_PG_PASSWORD` | `duckbricks` | PostgreSQL password |
+| `DUCKBRICKS_DUCKLAKE_NAME` | `duckbricks` | DuckLake catalog name |
+| `DUCKBRICKS_DATA_PATH` | `/data/parquet/` | Parquet storage path |

duckbricks_utils-0.1.0/duckbricks_utils/__init__.py

@@ -0,0 +1,16 @@
+"""DuckBricks workspace utilities.
+
+Provides a consistent interface for connecting to the DuckLake catalog from
+any environment: local IDE, Marimo notebooks, or job executors.
+
+Example usage:
+    from duckbricks_utils import connect
+
+    conn = connect()
+    result = conn.execute("SELECT * FROM my_table LIMIT 10").df()
+"""
+
+from duckbricks_utils._connection import catalog_name, connect, data_path
+from duckbricks_utils.storage import StorageBackend, StorageBackendFactory
+
+__all__ = ["StorageBackend", "StorageBackendFactory", "catalog_name", "connect", "data_path"]

duckbricks_utils-0.1.0/duckbricks_utils/_connection.py

@@ -0,0 +1,63 @@
+"""DuckLake connection helpers."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+
+from duckbricks_utils.storage.factory import StorageBackendFactory
+
+
+def _pg_dsn() -> str:
+    host = os.getenv("DUCKLAKE_PG_HOST", "localhost")
+    port = os.getenv("DUCKLAKE_PG_PORT", "5432")
+    database = os.getenv("DUCKLAKE_PG_DATABASE", "duckbricks")
+    user = os.getenv("DUCKLAKE_PG_USER", "duckbricks")
+    password = os.getenv("DUCKLAKE_PG_PASSWORD", "duckbricks")
+    return f"host={host} port={port} dbname={database} user={user} password={password}"
+
+
+def catalog_name() -> str:
+    """Return the configured DuckLake catalog name."""
+    return os.getenv("DUCKBRICKS_DUCKLAKE_NAME", "duckbricks")
+
+
+def data_path() -> str:
+    """Return the active storage backend's data path.
+
+    Delegates to the configured ``StorageBackend`` so that cloud paths
+    (s3://, gs://, az://) are returned correctly for non-local backends.
+    """
+    return StorageBackendFactory.from_env().data_path()
+
+
+def connect(override_data_path: str | None = None) -> duckdb.DuckDBPyConnection:
+    """Return a DuckDB connection with the DuckLake catalog attached.
+
+    The storage backend is selected by ``DUCKBRICKS_STORAGE_BACKEND``
+    (default: ``local``). Each backend installs its required DuckDB
+    extension and creates a named secret before the catalog is attached.
+
+    Args:
+        override_data_path: Optional path override for the DATA_PATH used
+            in the ATTACH statement. Bypasses the backend's own data_path().
+
+    Returns:
+        An open DuckDB connection with the DuckLake catalog set as default.
+    """
+    backend = StorageBackendFactory.from_env()
+    storage_path = override_data_path or backend.data_path()
+    name = catalog_name()
+    dsn = _pg_dsn()
+
+    conn = duckdb.connect()
+    backend.configure(conn)
+    conn.execute("INSTALL ducklake; LOAD ducklake;")
+    conn.execute("INSTALL postgres; LOAD postgres;")
+    conn.execute(
+        f"ATTACH 'ducklake:postgres:{dsn}' AS {name} "
+        f"(DATA_PATH '{storage_path}', AUTOMATIC_MIGRATION TRUE)"
+    )
+    conn.execute(f"USE {name}")
+    return conn

duckbricks_utils-0.1.0/duckbricks_utils/storage/__init__.py

@@ -0,0 +1,6 @@
+"""Storage backend abstractions for DuckLake."""
+
+from duckbricks_utils.storage.base import StorageBackend
+from duckbricks_utils.storage.factory import StorageBackendFactory
+
+__all__ = ["StorageBackend", "StorageBackendFactory"]

duckbricks_utils-0.1.0/duckbricks_utils/storage/azure.py

@@ -0,0 +1,45 @@
+"""Azure Blob Storage backend."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+
+from duckbricks_utils.storage.base import StorageBackend
+
+
+class AzureStorage(StorageBackend):
+    """Stores DuckLake Parquet files on Azure Blob Storage.
+
+    Supports two authentication methods (in priority order):
+      1. Connection string: ``AZURE_CONNECTION_STRING``
+      2. Account + key:     ``AZURE_ACCOUNT`` + ``AZURE_KEY``
+
+    Required environment variables:
+        AZURE_CONNECTION_STRING  — full Azure connection string (option 1)
+        AZURE_ACCOUNT            — storage account name (option 2)
+        AZURE_KEY                — storage account key (option 2)
+        DUCKBRICKS_DATA_PATH     — az://container/prefix/
+    """
+
+    def configure(self, conn: duckdb.DuckDBPyConnection) -> None:
+        conn.execute("INSTALL azure; LOAD azure;")
+        connection_string = os.getenv("AZURE_CONNECTION_STRING")
+        if connection_string:
+            conn.execute(
+                f"CREATE OR REPLACE SECRET duckbricks_storage ("
+                f"TYPE AZURE, CONNECTION_STRING '{connection_string}'"
+                f")"
+            )
+        else:
+            account = os.environ["AZURE_ACCOUNT"]
+            key = os.environ["AZURE_KEY"]
+            conn.execute(
+                f"CREATE OR REPLACE SECRET duckbricks_storage ("
+                f"TYPE AZURE, ACCOUNT_NAME '{account}', ACCOUNT_KEY '{key}'"
+                f")"
+            )
+
+    def data_path(self) -> str:
+        return os.environ["DUCKBRICKS_DATA_PATH"]

duckbricks_utils-0.1.0/duckbricks_utils/storage/base.py

@@ -0,0 +1,25 @@
+"""Abstract base class for DuckLake storage backends."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+import duckdb
+
+
+class StorageBackend(ABC):
+    """Encapsulates the DuckDB extension setup and secret configuration
+    required for a specific DuckLake storage provider.
+
+    Subclasses implement ``configure`` to install the necessary DuckDB
+    extension and create a named secret, and ``data_path`` to return the
+    provider-specific path prefix used in the ATTACH statement.
+    """
+
+    @abstractmethod
+    def configure(self, conn: duckdb.DuckDBPyConnection) -> None:
+        """Install required extensions and create DuckDB secrets on ``conn``."""
+
+    @abstractmethod
+    def data_path(self) -> str:
+        """Return the DATA_PATH value to use in the DuckLake ATTACH statement."""

duckbricks_utils-0.1.0/duckbricks_utils/storage/factory.py

@@ -0,0 +1,45 @@
+"""Factory for resolving the active storage backend from environment."""
+
+from __future__ import annotations
+
+import os
+
+from duckbricks_utils.storage.base import StorageBackend
+
+_BACKEND_REGISTRY: dict[str, str] = {
+    "local": "duckbricks_utils.storage.local.LocalStorage",
+    "s3": "duckbricks_utils.storage.s3.S3Storage",
+    "minio": "duckbricks_utils.storage.minio.MinIOStorage",
+    "r2": "duckbricks_utils.storage.r2.R2Storage",
+    "gcs": "duckbricks_utils.storage.gcs.GCSStorage",
+    "azure": "duckbricks_utils.storage.azure.AzureStorage",
+}
+
+
+class StorageBackendFactory:
+    """Resolves and instantiates the correct ``StorageBackend`` from environment.
+
+    The active backend is selected by the ``DUCKBRICKS_STORAGE_BACKEND``
+    environment variable (default: ``local``).
+    """
+
+    @staticmethod
+    def from_env() -> StorageBackend:
+        """Return a ``StorageBackend`` instance for the configured provider."""
+        backend_name = os.getenv("DUCKBRICKS_STORAGE_BACKEND", "local").lower()
+        if backend_name not in _BACKEND_REGISTRY:
+            supported = ", ".join(_BACKEND_REGISTRY)
+            raise ValueError(
+                f"Unknown storage backend '{backend_name}'. "
+                f"Supported values: {supported}"
+            )
+        import importlib
+
+        module_path, class_name = _BACKEND_REGISTRY[backend_name].rsplit(".", 1)
+        module = importlib.import_module(module_path)
+        return getattr(module, class_name)()
+
+    @staticmethod
+    def supported_backends() -> list[str]:
+        """Return the list of supported backend names."""
+        return list(_BACKEND_REGISTRY)

duckbricks_utils-0.1.0/duckbricks_utils/storage/gcs.py

@@ -0,0 +1,32 @@
+"""Google Cloud Storage backend."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+
+from duckbricks_utils.storage.base import StorageBackend
+
+
+class GCSStorage(StorageBackend):
+    """Stores DuckLake Parquet files on Google Cloud Storage.
+
+    Required environment variables:
+        GCS_KEY_ID              — GCS HMAC access key ID
+        GCS_SECRET              — GCS HMAC secret
+        DUCKBRICKS_DATA_PATH    — gs://bucket/prefix/
+    """
+
+    def configure(self, conn: duckdb.DuckDBPyConnection) -> None:
+        conn.execute("INSTALL httpfs; LOAD httpfs;")
+        key_id = os.environ["GCS_KEY_ID"]
+        secret = os.environ["GCS_SECRET"]
+        conn.execute(
+            f"CREATE OR REPLACE SECRET duckbricks_storage ("
+            f"TYPE GCS, KEY_ID '{key_id}', SECRET '{secret}'"
+            f")"
+        )
+
+    def data_path(self) -> str:
+        return os.environ["DUCKBRICKS_DATA_PATH"]

duckbricks_utils-0.1.0/duckbricks_utils/storage/local.py

@@ -0,0 +1,23 @@
+"""Local filesystem storage backend (default)."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+
+from duckbricks_utils.storage.base import StorageBackend
+
+
+class LocalStorage(StorageBackend):
+    """Stores DuckLake Parquet files on the local filesystem.
+
+    No DuckDB extensions or secrets are required beyond ducklake itself.
+    The data path is read from ``DUCKBRICKS_DATA_PATH``.
+    """
+
+    def configure(self, conn: duckdb.DuckDBPyConnection) -> None:
+        pass
+
+    def data_path(self) -> str:
+        return os.getenv("DUCKBRICKS_DATA_PATH", "/data/parquet/")

duckbricks_utils-0.1.0/duckbricks_utils/storage/minio.py

@@ -0,0 +1,35 @@
+"""MinIO storage backend."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+
+from duckbricks_utils.storage.base import StorageBackend
+
+
+class MinIOStorage(StorageBackend):
+    """Stores DuckLake Parquet files on a MinIO-compatible S3 endpoint.
+
+    Required environment variables:
+        AWS_ACCESS_KEY_ID       — MinIO access key
+        AWS_SECRET_ACCESS_KEY   — MinIO secret key
+        MINIO_ENDPOINT          — MinIO endpoint URL (e.g. http://minio:9000)
+        DUCKBRICKS_DATA_PATH    — s3://bucket/prefix/
+    """
+
+    def configure(self, conn: duckdb.DuckDBPyConnection) -> None:
+        conn.execute("INSTALL httpfs; LOAD httpfs;")
+        key_id = os.environ["AWS_ACCESS_KEY_ID"]
+        secret = os.environ["AWS_SECRET_ACCESS_KEY"]
+        endpoint = os.environ["MINIO_ENDPOINT"].rstrip("/")
+        conn.execute(
+            f"CREATE OR REPLACE SECRET duckbricks_storage ("
+            f"TYPE S3, KEY_ID '{key_id}', SECRET '{secret}', "
+            f"ENDPOINT '{endpoint}', USE_SSL false, URL_STYLE path"
+            f")"
+        )
+
+    def data_path(self) -> str:
+        return os.environ["DUCKBRICKS_DATA_PATH"]

duckbricks_utils-0.1.0/duckbricks_utils/storage/r2.py

@@ -0,0 +1,36 @@
+"""Cloudflare R2 storage backend."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+
+from duckbricks_utils.storage.base import StorageBackend
+
+
+class R2Storage(StorageBackend):
+    """Stores DuckLake Parquet files on Cloudflare R2.
+
+    Required environment variables:
+        AWS_ACCESS_KEY_ID       — R2 access key ID
+        AWS_SECRET_ACCESS_KEY   — R2 secret access key
+        R2_ACCOUNT_ID           — Cloudflare account ID
+        DUCKBRICKS_DATA_PATH    — s3://bucket/prefix/
+    """
+
+    def configure(self, conn: duckdb.DuckDBPyConnection) -> None:
+        conn.execute("INSTALL httpfs; LOAD httpfs;")
+        key_id = os.environ["AWS_ACCESS_KEY_ID"]
+        secret = os.environ["AWS_SECRET_ACCESS_KEY"]
+        account_id = os.environ["R2_ACCOUNT_ID"]
+        endpoint = f"https://{account_id}.r2.cloudflarestorage.com"
+        conn.execute(
+            f"CREATE OR REPLACE SECRET duckbricks_storage ("
+            f"TYPE S3, KEY_ID '{key_id}', SECRET '{secret}', "
+            f"ENDPOINT '{endpoint}', REGION 'auto'"
+            f")"
+        )
+
+    def data_path(self) -> str:
+        return os.environ["DUCKBRICKS_DATA_PATH"]

duckbricks_utils-0.1.0/duckbricks_utils/storage/s3.py

@@ -0,0 +1,34 @@
+"""AWS S3 storage backend."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+
+from duckbricks_utils.storage.base import StorageBackend
+
+
+class S3Storage(StorageBackend):
+    """Stores DuckLake Parquet files on AWS S3.
+
+    Required environment variables:
+        AWS_ACCESS_KEY_ID       — AWS access key
+        AWS_SECRET_ACCESS_KEY   — AWS secret key
+        AWS_REGION              — AWS region (default: us-east-1)
+        DUCKBRICKS_DATA_PATH    — s3://bucket/prefix/
+    """
+
+    def configure(self, conn: duckdb.DuckDBPyConnection) -> None:
+        conn.execute("INSTALL httpfs; LOAD httpfs;")
+        key_id = os.environ["AWS_ACCESS_KEY_ID"]
+        secret = os.environ["AWS_SECRET_ACCESS_KEY"]
+        region = os.getenv("AWS_REGION", "us-east-1")
+        conn.execute(
+            f"CREATE OR REPLACE SECRET duckbricks_storage ("
+            f"TYPE S3, KEY_ID '{key_id}', SECRET '{secret}', REGION '{region}'"
+            f")"
+        )
+
+    def data_path(self) -> str:
+        return os.environ["DUCKBRICKS_DATA_PATH"]

duckbricks_utils-0.1.0/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "duckbricks-utils"
+version = "0.1.0"
+description = "DuckLake connection utilities for DuckBricks notebooks and pipelines"
+authors = ["DuckBricks Team"]
+readme = "README.md"
+packages = [{include = "duckbricks_utils"}]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+duckdb = ">=1.3.0"
+python-dotenv = ">=1.0"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "*"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"