dbt-polars-catalog-databricks 0.3.0__tar.gz

dbt_polars_catalog_databricks-0.3.0/.gitignore

@@ -0,0 +1,228 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer,
+#   you could uncomment the following to ignore the entire vscode folder
+.vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# uv.lock
+uv.lock
+
+# dbt-polars
+logs/
+test.env
+.claude/
+decisions.md
+test_project*/

dbt_polars_catalog_databricks-0.3.0/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.5
+Name: dbt-polars-catalog-databricks
+Version: 0.3.0
+Summary: Databricks Unity Catalog catalog for dbt-polars
+Project-URL: Homepage, https://github.com/datamindedbe/incubator-dbt-polars
+Project-URL: Repository, https://github.com/datamindedbe/incubator-dbt-polars
+Project-URL: Changelog, https://github.com/datamindedbe/incubator-dbt-polars/releases
+Author-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
+Maintainer-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
+License-Expression: Apache-2.0
+Keywords: catalog,databricks,dbt,polars,unity-catalog
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Requires-Python: <3.14,>=3.10
+Requires-Dist: databricks-sdk>=0.30.0
+Requires-Dist: dbt-polars==0.3.0
+Description-Content-Type: text/markdown
+
+# dbt-polars
+
+dbt-polars integrates the flexibility and performance of Polars with the data governance and lineage capabilities of dbt.
+
+Key features of **dbt-polars**:
+
+* **Run anywhere**: Polars runs on any compute platform, so you’re not locked into an expensive data warehouse.
+* **Flexible**: Available catalogs let you connect to local storage, Iceberg catalogs, Databricks, and more.
+* **Extensible**: Create a custom catalog to fit your unique storage and warehouse needs.
+* **First-class Python models**: Run Python models with the same performance and capabilities as SQL models.
+* **Mix Python and SQL**: Choose the language that best fits the transformation.
+* **dbt-native**: Bring lineage, documentation, and data testing into your workflow.
+* **Ingestion**: Build efficient, incremental API ingestion with Python models and keep the entire pipeline visible in the lineage graph.
+
+## Installation
+
+```bash
+pip install dbt-polars
+```
+
+Extra dependencies are required for cloud and Iceberg backends:
+
+```bash
+pip install 'dbt-polars[azure]'       # Azure Blob Storage
+pip install 'dbt-polars[databricks]'  # Azure Blob Storage
+pip install 'dbt-polars[iceberg]'     # Apache Iceberg (SQLite, REST, Databricks, …)
+```
+
+## Catalogs
+
+dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
+
+| Catalog | Backend | Status |
+|---|---|---|
+| `local` | Local filesystem | Usable |
+| `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
+| `iceberg` with REST + Databricks | Iceberg REST API (Unity Catalog) | Experimental |
+| `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
+| `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
+| `databricks` | Unity Catalog REST API + credential vending | Experimental |
+
+## Quick start
+
+See the [Getting started](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/demos/getting_started.md) guide.
+
+A minimal `profiles.yml` for local development is:
+
+```yaml
+my_project:
+  target: dev
+  outputs:
+    dev:
+      type: polars
+      schema: dev
+
+      catalog_type: local
+      root: ./data
+```
+
+## Documentation
+
+- [Getting started](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/demos/getting_started.md)
+- [Catalog overview and profile reference](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalog-overview.md)
+- [Local catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/local.md)
+- [Azure catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/azure.md)
+- [Iceberg catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/iceberg.md)
+- [Databricks catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/databricks.md)
+- [Custom catalogs](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/custom.md)
+- [Python models and tests](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/python-models.md)

dbt_polars_catalog_databricks-0.3.0/README.md

@@ -0,0 +1,69 @@
+# dbt-polars
+
+dbt-polars integrates the flexibility and performance of Polars with the data governance and lineage capabilities of dbt.
+
+Key features of **dbt-polars**:
+
+* **Run anywhere**: Polars runs on any compute platform, so you’re not locked into an expensive data warehouse.
+* **Flexible**: Available catalogs let you connect to local storage, Iceberg catalogs, Databricks, and more.
+* **Extensible**: Create a custom catalog to fit your unique storage and warehouse needs.
+* **First-class Python models**: Run Python models with the same performance and capabilities as SQL models.
+* **Mix Python and SQL**: Choose the language that best fits the transformation.
+* **dbt-native**: Bring lineage, documentation, and data testing into your workflow.
+* **Ingestion**: Build efficient, incremental API ingestion with Python models and keep the entire pipeline visible in the lineage graph.
+
+## Installation
+
+```bash
+pip install dbt-polars
+```
+
+Extra dependencies are required for cloud and Iceberg backends:
+
+```bash
+pip install 'dbt-polars[azure]'       # Azure Blob Storage
+pip install 'dbt-polars[databricks]'  # Azure Blob Storage
+pip install 'dbt-polars[iceberg]'     # Apache Iceberg (SQLite, REST, Databricks, …)
+```
+
+## Catalogs
+
+dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
+
+| Catalog | Backend | Status |
+|---|---|---|
+| `local` | Local filesystem | Usable |
+| `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
+| `iceberg` with REST + Databricks | Iceberg REST API (Unity Catalog) | Experimental |
+| `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
+| `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
+| `databricks` | Unity Catalog REST API + credential vending | Experimental |
+
+## Quick start
+
+See the [Getting started](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/demos/getting_started.md) guide.
+
+A minimal `profiles.yml` for local development is:
+
+```yaml
+my_project:
+  target: dev
+  outputs:
+    dev:
+      type: polars
+      schema: dev
+
+      catalog_type: local
+      root: ./data
+```
+
+## Documentation
+
+- [Getting started](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/demos/getting_started.md)
+- [Catalog overview and profile reference](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalog-overview.md)
+- [Local catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/local.md)
+- [Azure catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/azure.md)
+- [Iceberg catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/iceberg.md)
+- [Databricks catalog](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/databricks.md)
+- [Custom catalogs](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/catalogs/custom.md)
+- [Python models and tests](https://github.com/datamindedbe/incubator-dbt-polars/blob/main/docs/python-models.md)

dbt_polars_catalog_databricks-0.3.0/dbt_polars_catalog_databricks/__init__.py

@@ -0,0 +1,23 @@
+from dbt.adapters.polars.catalogs import BaseCatalog
+
+from dbt_polars_catalog_databricks.config import DatabricksCatalogConfig
+from dbt_polars_catalog_databricks.delta import DatabricksDeltaCatalog
+
+CATALOG_BY_TABLE_FORMAT: dict[str, type[BaseCatalog]] = {
+    "delta": DatabricksDeltaCatalog,
+}
+
+config_class = DatabricksCatalogConfig
+
+
+def create_catalog(config: DatabricksCatalogConfig, project_root: str) -> BaseCatalog:
+    return CATALOG_BY_TABLE_FORMAT[config.table_format](config, project_root)
+
+
+__all__ = [
+    "CATALOG_BY_TABLE_FORMAT",
+    "DatabricksCatalogConfig",
+    "DatabricksDeltaCatalog",
+    "config_class",
+    "create_catalog",
+]

dbt_polars_catalog_databricks-0.3.0/dbt_polars_catalog_databricks/config.py

@@ -0,0 +1,59 @@
+from dbt.adapters.polars.catalogs import CatalogConfig
+from dbt_common.exceptions import DbtRuntimeError
+
+SUPPORTED_TABLE_FORMATS = ("delta",)
+
+
+class DatabricksCatalogConfig(CatalogConfig):
+    """Config for the Databricks/Unity Catalog backend.
+
+    `catalog_name` is the only required dbt-polars-specific field (which Unity
+    Catalog catalog to register tables in - schema storage locations are read
+    from Unity Catalog itself, not configured here). Every other keyword is
+    passed straight through to `databricks.sdk.Config`, so any auth method it
+    supports (PAT, OAuth M2M, Azure client-secret, external-browser, azure-cli,
+    databricks-cli profile, ...) works by supplying the matching kwargs.
+
+    `persist_docs_http_path`: docs (table/column comments from persist_docs)
+    are always written to the Delta table's own metadata. If a SQL warehouse's
+    HTTP path (e.g. `/sql/1.0/warehouses/<warehouse_id>`, as shown on the
+    warehouse's Connection Details page) is also set here, they're additionally
+    written into Unity Catalog's native comment fields via `COMMENT ON TABLE`/
+    `ALTER TABLE ... ALTER COLUMN ... COMMENT` - the only way to set those,
+    since Unity Catalog's REST API has no endpoint for it at all. Left unset,
+    docs never reach Unity Catalog itself.
+    """
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        type: str,
+        schema: str,
+        catalog_name: str,
+        host: str | None = None,
+        persist_docs_http_path: str | None = None,
+        table_format: str = "delta",
+        **kwargs: object,
+    ) -> None:
+        if table_format not in SUPPORTED_TABLE_FORMATS:
+            raise DbtRuntimeError(
+                f"Databricks catalog '{name}': table_format '{table_format}' is not "
+                f"supported, only {', '.join(SUPPORTED_TABLE_FORMATS)} for now."
+            )
+        self.name = name
+        self.type = type
+        self.schema = schema
+        self.catalog_name = catalog_name
+        self.persist_docs_http_path = persist_docs_http_path
+        self.table_format = table_format
+        self.sdk_kwargs: dict[str, object] = {
+            **({"host": host} if host else {}),
+            **kwargs,
+        }
+
+    def unique_field(self) -> str:
+        return f"{self.catalog_name}/{self.schema}"
+
+    def connection_keys(self) -> tuple[str, ...]:
+        return ("name", "catalog_name", "host", "persist_docs_http_path")

dbt_polars_catalog_databricks-0.3.0/dbt_polars_catalog_databricks/databricks_types.py

@@ -0,0 +1,119 @@
+"""Maps Polars dtypes to Unity Catalog column type descriptors.
+
+Used only to populate column metadata when registering a table via the Unity
+Catalog Tables API (`POST /api/2.1/unity-catalog/tables`). Unity Catalog does
+not validate this against the table's actual Parquet/Delta physical types, so
+this is a best-effort, display/documentation-oriented mapping, not a source of
+truth for how data is physically stored.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import polars as pl
+from dbt.adapters.events.logging import AdapterLogger
+
+logger = AdapterLogger("polars")
+
+# Delta/Unity Catalog has no unsigned integer types; describe them as the
+# next signed type wide enough to hold every value (mirrors IcebergCatalog's
+# _cast_unsigned_to_signed rationale in icebergCatalog.py).
+UNSIGNED_TO_SIGNED_NAME_TEXT: dict[type, tuple[str, str]] = {
+    pl.UInt8: ("SHORT", "smallint"),
+    pl.UInt16: ("INT", "int"),
+    pl.UInt32: ("LONG", "bigint"),
+    pl.UInt64: ("DECIMAL", "decimal(20,0)"),
+}
+
+SIMPLE_NAME_TEXT: dict[type, tuple[str, str]] = {
+    pl.Boolean: ("BOOLEAN", "boolean"),
+    pl.Int8: ("BYTE", "tinyint"),
+    pl.Int16: ("SHORT", "smallint"),
+    pl.Int32: ("INT", "int"),
+    pl.Int64: ("LONG", "bigint"),
+    pl.Float32: ("FLOAT", "float"),
+    pl.Float64: ("DOUBLE", "double"),
+    pl.Utf8: ("STRING", "string"),
+    pl.Categorical: ("STRING", "string"),
+    pl.Date: ("DATE", "date"),
+    pl.Binary: ("BINARY", "binary"),
+}
+
+
+def decimal_name_text(dtype: pl.Decimal) -> tuple[str, str]:
+    precision = dtype.precision if dtype.precision is not None else 38
+    scale = dtype.scale or 0
+    return "DECIMAL", f"decimal({precision},{scale})"
+
+
+def datetime_name_text(dtype: pl.Datetime) -> tuple[str, str]:
+    if dtype.time_zone:
+        return "TIMESTAMP", "timestamp"
+    return "TIMESTAMP_NTZ", "timestamp_ntz"
+
+
+def json_value(dtype: Any) -> object:
+    """Return the JSON-able value (str or dict) describing dtype, Spark-schema style."""
+    base = type(dtype)
+
+    if base in UNSIGNED_TO_SIGNED_NAME_TEXT:
+        return UNSIGNED_TO_SIGNED_NAME_TEXT[base][1]
+    if base in SIMPLE_NAME_TEXT:
+        return SIMPLE_NAME_TEXT[base][1]
+    if isinstance(dtype, pl.Decimal):
+        return decimal_name_text(dtype)[1]
+    if isinstance(dtype, pl.Datetime):
+        return datetime_name_text(dtype)[1]
+    if isinstance(dtype, (pl.List, pl.Array)):
+        return {
+            "type": "array",
+            "elementType": json_value(dtype.inner),
+            "containsNull": True,
+        }
+    if isinstance(dtype, pl.Struct):
+        return {
+            "type": "struct",
+            "fields": [
+                {
+                    "name": f.name,
+                    "type": json_value(f.dtype),
+                    "nullable": True,
+                    "metadata": {},
+                }
+                for f in dtype.fields
+            ],
+        }
+    logger.debug(
+        f"No Unity Catalog type mapping for Polars dtype {dtype!r}; "
+        "describing this column as STRING in the registered table metadata."
+    )
+    return "string"
+
+
+def polars_dtype_to_uc_type(dtype: Any) -> tuple[str, str, str]:
+    """Return (type_name, type_text, type_json) for a Unity Catalog table column."""
+    base = type(dtype)
+
+    if base in UNSIGNED_TO_SIGNED_NAME_TEXT:
+        name, text = UNSIGNED_TO_SIGNED_NAME_TEXT[base]
+    elif base in SIMPLE_NAME_TEXT:
+        name, text = SIMPLE_NAME_TEXT[base]
+    elif isinstance(dtype, pl.Decimal):
+        name, text = decimal_name_text(dtype)
+    elif isinstance(dtype, pl.Datetime):
+        name, text = datetime_name_text(dtype)
+    elif isinstance(dtype, (pl.List, pl.Array)):
+        _, inner_text, _ = polars_dtype_to_uc_type(dtype.inner)
+        name, text = "ARRAY", f"array<{inner_text}>"
+    elif isinstance(dtype, pl.Struct):
+        field_texts = []
+        for f in dtype.fields:
+            _, f_text, _ = polars_dtype_to_uc_type(f.dtype)
+            field_texts.append(f"{f.name}:{f_text}")
+        name, text = "STRUCT", f"struct<{','.join(field_texts)}>"
+    else:
+        name, text = "STRING", "string"
+
+    return name, text, json.dumps(json_value(dtype))