dbt-polars 0.0.1__tar.gz

dbt_polars-0.0.1/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: dbt-polars
+Version: 0.0.1
+Summary: A dbt adapter for running models with Polars
+Author-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
+Maintainer-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
+License: Apache-2.0
+Keywords: dbt,polars,adapter,data
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Requires-Python: <3.14,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: dbt-adapters>=1.0.0
+Requires-Dist: dbt-common>=1.0.0
+Requires-Dist: dbt-core>=1.11.2
+Requires-Dist: polars[deltalake,pyarrow]!=1.43.0,>=1.0.0
+Requires-Dist: sqlglot>=1.0.0
+Provides-Extra: iceberg
+Requires-Dist: pyiceberg[adlfs,sql-sqlite]<=0.11.1,>=0.11.0; extra == "iceberg"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.20.0; extra == "s3"
+Provides-Extra: azure
+Requires-Dist: azure-storage-file-datalake>=12.0.0; extra == "azure"
+Requires-Dist: azure-identity>=1.0.0; extra == "azure"
+
+# dbt-polars
+
+A [dbt](https://www.getdbt.com/) adapter that runs models locally using [Polars](https://pola.rs/). Instead of connecting to a database, dbt-polars reads and writes files — on your local filesystem, Azure Blob Storage, AWS S3, or an Iceberg catalog.
+
+## Installation
+
+```bash
+pip install dbt-polars
+```
+
+Extra dependencies are required for cloud and Iceberg backends:
+
+```bash
+pip install 'dbt-polars[azure]'    # Azure Blob Storage
+pip install 'dbt-polars[s3]'       # AWS S3
+pip install 'dbt-polars[iceberg]'  # Apache Iceberg (SQLite, REST, Databricks, …)
+```
+
+## Catalogs
+
+dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
+
+| Catalog | Backend | Status |
+|---|---|---|
+| `local` | Local filesystem | Usable |
+| `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
+| `iceberg` with REST + Databricks | Iceberg REST API (Unity Catalog) | Experimental |
+| `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
+| `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
+
+## Quick start
+
+A minimal `profiles.yml` for local development:
+
+```yaml
+my_project:
+  target: dev
+  outputs:
+    dev:
+      type: polars
+      schema: dev
+      catalogs:
+        - name: my_catalog
+          type: local
+          root: ./data
+```
+
+## Documentation
+
+- [Catalog overview and profile reference](docs/index.md)
+- [Local catalog](docs/catalogs/local.md)
+- [Azure catalog](docs/catalogs/azure.md)
+- [Iceberg catalog](docs/catalogs/iceberg.md)
+- [Python models and tests](docs/python-models.md)
+- [Roadmap and known limitations](docs/roadmap.md)

dbt_polars-0.0.1/README.md

@@ -0,0 +1,55 @@
+# dbt-polars
+
+A [dbt](https://www.getdbt.com/) adapter that runs models locally using [Polars](https://pola.rs/). Instead of connecting to a database, dbt-polars reads and writes files — on your local filesystem, Azure Blob Storage, AWS S3, or an Iceberg catalog.
+
+## Installation
+
+```bash
+pip install dbt-polars
+```
+
+Extra dependencies are required for cloud and Iceberg backends:
+
+```bash
+pip install 'dbt-polars[azure]'    # Azure Blob Storage
+pip install 'dbt-polars[s3]'       # AWS S3
+pip install 'dbt-polars[iceberg]'  # Apache Iceberg (SQLite, REST, Databricks, …)
+```
+
+## Catalogs
+
+dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
+
+| Catalog | Backend | Status |
+|---|---|---|
+| `local` | Local filesystem | Usable |
+| `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
+| `iceberg` with REST + Databricks | Iceberg REST API (Unity Catalog) | Experimental |
+| `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
+| `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
+
+## Quick start
+
+A minimal `profiles.yml` for local development:
+
+```yaml
+my_project:
+  target: dev
+  outputs:
+    dev:
+      type: polars
+      schema: dev
+      catalogs:
+        - name: my_catalog
+          type: local
+          root: ./data
+```
+
+## Documentation
+
+- [Catalog overview and profile reference](docs/index.md)
+- [Local catalog](docs/catalogs/local.md)
+- [Azure catalog](docs/catalogs/azure.md)
+- [Iceberg catalog](docs/catalogs/iceberg.md)
+- [Python models and tests](docs/python-models.md)
+- [Roadmap and known limitations](docs/roadmap.md)

dbt_polars-0.0.1/dbt/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

dbt_polars-0.0.1/dbt/adapters/polars/__init__.py

@@ -0,0 +1,84 @@
+from dbt.adapters.base.plugin import AdapterPlugin
+from dbt.exceptions import ModelConfigError
+from dbt_common.dataclass_schema import ValidationError
+from dbt_common.exceptions import DbtRuntimeError
+
+from dbt.adapters.polars.connections import PolarsCredentials
+from dbt.adapters.polars.impl import PolarsAdapter
+from dbt.include.polars import PACKAGE_PATH
+
+# Two dbt-core monkeypatches to support Python singular tests (dbt-core 1.11.x):
+#
+# 1. get_file_types_for_project: adds ".py" to ParseFileType.SingularTest extensions
+#    so Python test files are discovered by the parser.
+#
+# 2. SingularTestParser.render_update: runs parse_python_model for .py test nodes so
+#    dbt.ref() calls are statically scanned and added to depends_on. Without this, dbt's
+#    runtime ref() validation rejects any dbt.ref() call in a .py test. Requires a
+#    "def model" entry point (satisfied by _TEST_ENTRYPOINT_WRAPPER below).
+_TEST_ENTRYPOINT_WRAPPER = "\n\n\ndef model(dbt, pl):\n    return test(dbt, pl)\n"
+
+try:
+    import dbt.parser.read_files as _read_files
+    from dbt.node_types import ModelLanguage
+    from dbt.parser.models import ModelParser, verify_python_model_code
+    from dbt.parser.singular_test import SingularTestParser
+
+    _original_get_file_types_for_project = _read_files.get_file_types_for_project
+
+    def _get_file_types_for_project_with_py_tests(project):
+        file_types = _original_get_file_types_for_project(project)
+        try:
+            file_types[_read_files.ParseFileType.SingularTest]["extensions"].append(
+                ".py"
+            )
+        except (KeyError, AttributeError, TypeError):
+            pass
+        return file_types
+
+    _read_files.get_file_types_for_project = _get_file_types_for_project_with_py_tests
+
+    _original_singular_test_render_update = SingularTestParser.render_update
+
+    def _singular_test_render_update(
+        self, node, config, validate_config_call_dict=False
+    ):
+        if node.language == ModelLanguage.python:
+            if "def test(" not in node.raw_code:
+                raise DbtRuntimeError(
+                    f"Python singular test '{node.name}' must define "
+                    + "a function named 'test(dbt, pl)'"
+                )
+            if not node.raw_code.endswith(_TEST_ENTRYPOINT_WRAPPER):
+                node.raw_code += _TEST_ENTRYPOINT_WRAPPER
+            try:
+                verify_python_model_code(node)
+                context = self._context_for(node, config)
+                ModelParser.parse_python_model(self, node, config, context)
+                self.update_parsed_node_config(
+                    node, config, context=context, validate_config_call_dict=True
+                )
+            except ValidationError as exc:
+                raise ModelConfigError(exc, node=node) from exc
+            except Exception:
+                return _original_singular_test_render_update(
+                    self, node, config, validate_config_call_dict
+                )
+            return
+        return _original_singular_test_render_update(
+            self, node, config, validate_config_call_dict
+        )
+
+    SingularTestParser.render_update = _singular_test_render_update  # type: ignore[method-assign]
+except (ImportError, AttributeError):
+    from dbt.adapters.events.logging import AdapterLogger
+
+    AdapterLogger("polars").warning(
+        "Python singular tests are not supported with this version of dbt-core."
+    )
+
+Plugin = AdapterPlugin(
+    adapter=PolarsAdapter,  # type: ignore[arg-type]
+    credentials=PolarsCredentials,
+    include_path=PACKAGE_PATH,
+)

dbt_polars-0.0.1/dbt/adapters/polars/__version__.py

@@ -0,0 +1 @@
+version = "1.7.0"

dbt_polars-0.0.1/dbt/adapters/polars/catalogs/__init__.py

@@ -0,0 +1,24 @@
+from dbt.adapters.polars.catalogs.azureCatalog import (
+    AzureBlobStorageCatalog,
+    AzureBlobStorageCatalogConfig,
+)
+from dbt.adapters.polars.catalogs.baseCatalog import BaseCatalog, CatalogConfig
+from dbt.adapters.polars.catalogs.icebergCatalog import (
+    IcebergCatalog,
+    IcebergCatalogConfig,
+)
+from dbt.adapters.polars.catalogs.localCatalog import LocalCatalog, LocalCatalogConfig
+from dbt.adapters.polars.catalogs.s3Catalog import AWSS3Catalog, S3CatalogConfig
+
+CATALOG_CONFIG_REGISTRY: dict[str, type[CatalogConfig]] = {
+    "local": LocalCatalogConfig,
+    "iceberg": IcebergCatalogConfig,
+    "azure": AzureBlobStorageCatalogConfig,
+    "s3": S3CatalogConfig,
+}
+CATALOG_REGISTRY: dict[str, type[BaseCatalog]] = {
+    "local": LocalCatalog,
+    "iceberg": IcebergCatalog,
+    "azure": AzureBlobStorageCatalog,
+    "s3": AWSS3Catalog,
+}

dbt_polars-0.0.1/dbt/adapters/polars/catalogs/azureCatalog.py

@@ -0,0 +1,291 @@
+from typing import TYPE_CHECKING
+
+from dbt.adapters.contracts.relation import RelationType
+from dbt.adapters.events.logging import AdapterLogger
+from dbt_common.exceptions import DbtRuntimeError
+from deltalake import DeltaTable
+
+from dbt.adapters.polars.catalogs.baseCatalog import CatalogConfig
+from dbt.adapters.polars.catalogs.formats import FILE_FORMATS
+from dbt.adapters.polars.catalogs.storageCatalog import StorageCatalog
+from dbt.adapters.polars.relation import PolarsRelation
+
+if TYPE_CHECKING:
+    from azure.core.credentials import AccessToken
+
+logger = AdapterLogger("polars")
+
+_FILE_FORMATS = frozenset(FILE_FORMATS)
+
+
+class AzureBlobStorageCatalogConfig(CatalogConfig):
+    def __init__(
+        self,
+        name: str,
+        type: str,
+        account_name: str,
+        container: str = "",
+        prefix: str = "",
+        credentials: dict | None = None,
+    ) -> None:
+        self.name = name
+        self.type = type
+        self.account_name = account_name
+        self.container = container
+        self.prefix = prefix.strip("/")
+        self.credentials = credentials or {}
+
+    def unique_field(self) -> str:
+        return f"{self.account_name}/{self.container}/{self.prefix}"
+
+    def connection_keys(self) -> tuple[str, ...]:
+        return ("name", "account_name", "container", "prefix")
+
+
+class _StaticTokenCredential:
+    def __init__(self, token: str) -> None:
+        self._token = token
+
+    def get_token(self, *_args, **_kwargs):
+        import time
+
+        from azure.core.credentials import AccessToken
+
+        return AccessToken(self._token, int(time.time()) + 3600)
+
+
+class AzureBlobStorageCatalog(StorageCatalog):
+    config: AzureBlobStorageCatalogConfig
+
+    def __init__(
+        self, config: AzureBlobStorageCatalogConfig, project_root: str
+    ) -> None:
+        try:
+            import azure.identity  # noqa: F401
+            import azure.storage.filedatalake  # noqa: F401
+        except ImportError as exc:
+            raise DbtRuntimeError(
+                "The azure extra is required for AzureBlobStorageCatalog. "
+                "Install it with: pip install 'dbt-polars[azure]'"
+            ) from exc
+        super().__init__(config, project_root)
+        self._cached_token: AccessToken | None = None
+        self._service_client = None  # DataLakeServiceClient | None
+        self._dac = None  # DefaultAzureCredential | None
+
+    # ── URI and credential helpers ────────────────────────────────────────────
+
+    def _blob_prefix(self, *parts: str) -> str:
+        """Join the catalog prefix with additional path parts."""
+        components = [self.config.prefix] if self.config.prefix else []
+        components.extend(p.strip("/") for p in parts if p)
+        return "/".join(components)
+
+    def _object_path(self, schema: str, *parts: str) -> str:
+        return self._blob_prefix(schema, *parts)
+
+    def _dfs_url(self) -> str:
+        return f"https://{self.config.account_name}.dfs.core.windows.net"
+
+    def _get_uri(self, relation: PolarsRelation) -> str:
+        schema = relation.schema or ""
+        identifier = relation.identifier or ""
+        container = self.config.container
+        path = self._object_path(schema, identifier)
+        if relation.file_format != "delta":
+            path = f"{path}.{relation.file_format}"
+        return f"az://{container}/{path}"
+
+    _DELEGATED_CRED_KEYS = ("bearer_token", "account_key", "sas_token")
+
+    def _get_storage_options(self, _uri: str) -> dict[str, str]:
+        base: dict[str, str] = {
+            "account_name": self.config.account_name,
+            "timeout": "120s",
+        }
+        creds = self.config.credentials
+
+        override = creds.get("storage_options")
+        if override is not None:
+            return {**base, **override}
+
+        direct = {k: str(v) for k, v in creds.items() if k in self._DELEGATED_CRED_KEYS}
+        if direct:
+            return {**base, **direct}
+
+        base["bearer_token"] = self._default_chain_token()
+        return base
+
+    def _get_dac(self):
+        if self._dac is None:
+            from azure.identity import DefaultAzureCredential
+
+            dac_kwargs = {
+                k: v
+                for k, v in self.config.credentials.items()
+                if k != "storage_options"
+            }
+            self._dac = DefaultAzureCredential(**dac_kwargs)
+        return self._dac
+
+    def _default_chain_token(self) -> str:
+        import time
+
+        if (
+            self._cached_token is None
+            or self._cached_token.expires_on < time.time() + 300
+        ):
+            self._cached_token = self._get_dac().get_token(
+                "https://storage.azure.com/.default"
+            )
+        return self._cached_token.token
+
+    def _get_service_client(self):
+        if self._service_client is not None:
+            return self._service_client
+
+        from azure.storage.filedatalake import DataLakeServiceClient
+
+        url = self._dfs_url()
+        creds = self.config.credentials
+        if "bearer_token" in creds:
+            service = DataLakeServiceClient(
+                url, credential=_StaticTokenCredential(creds["bearer_token"])
+            )
+        elif "account_key" in creds:
+            service = DataLakeServiceClient(url, credential=creds["account_key"])
+        elif "sas_token" in creds:
+            service = DataLakeServiceClient(f"{url}?{creds['sas_token']}")
+        else:
+            service = DataLakeServiceClient(url, credential=self._get_dac())
+        self._service_client = service
+        return self._service_client
+
+    def _get_file_system_client(self, file_system: str | None = None):
+        name = file_system or self.config.container
+        return self._get_service_client().get_file_system_client(name)
+
+    def _remove_directory(self, file_system: str, path: str) -> None:
+        from azure.core.exceptions import ResourceNotFoundError
+
+        try:
+            self._get_file_system_client(file_system).get_directory_client(
+                path
+            ).delete_directory()
+        except ResourceNotFoundError:
+            pass
+
+    # ── schema management ─────────────────────────────────────────────────────
+
+    def create_schema(self, relation: PolarsRelation) -> None:
+        schema = relation.schema or ""
+        path = self._object_path(schema)
+        logger.debug(f"create_schema: creating directory {path}")
+        self._get_file_system_client().create_directory(path)
+
+    def drop_schema(self, relation: PolarsRelation) -> None:
+        schema = relation.schema or ""
+        path = self._object_path(schema)
+        logger.debug(f"Dropping schema {relation.catalog}/{schema}")
+        self._remove_directory(self.config.container, path)
+
+    def list_schemas(self) -> list[str]:
+        from azure.core.exceptions import ResourceNotFoundError
+
+        base = self.config.prefix or ""
+        try:
+            return [
+                p.name.split("/")[-1]
+                for p in self._get_file_system_client().get_paths(
+                    path=base, recursive=False
+                )
+                if p.is_directory
+            ]
+        except ResourceNotFoundError:
+            return []
+
+    def table_exists(self, relation: PolarsRelation) -> bool:
+        uri = self._get_uri(relation)
+        opts = self._get_storage_options(uri)
+        if relation.file_format == "delta":
+            return DeltaTable.is_deltatable(uri, storage_options=opts)
+        file_path = uri.removeprefix(f"az://{self.config.container}/")
+        return (
+            self._get_file_system_client(self.config.container)
+            .get_file_client(file_path)
+            .exists()
+        )
+
+    def drop_relation(self, relation: PolarsRelation) -> None:
+        schema = relation.schema or ""
+        identifier = relation.identifier or ""
+        file_system = self.config.container
+        logger.debug(
+            f"Dropping table if exists {relation.catalog}/{schema}/{identifier}"
+        )
+        if relation.file_format == "delta":
+            self._remove_directory(file_system, self._object_path(schema, identifier))
+        else:
+            from azure.core.exceptions import ResourceNotFoundError
+
+            file_path = (
+                f"{self._object_path(schema, identifier)}.{relation.file_format}"
+            )
+            try:
+                self._get_file_system_client(file_system).get_file_client(
+                    file_path
+                ).delete_file()
+            except ResourceNotFoundError:
+                pass
+
+    def list_relations_without_caching(
+        self, schema_relation: PolarsRelation
+    ) -> list[PolarsRelation]:
+        from azure.core.exceptions import ResourceNotFoundError
+
+        schema = schema_relation.schema or ""
+        file_system = self.config.container
+        prefix = self._object_path(schema)
+        relations: list[PolarsRelation] = []
+
+        try:
+            paths = list(
+                self._get_file_system_client(file_system).get_paths(
+                    path=prefix, recursive=False
+                )
+            )
+        except ResourceNotFoundError:
+            return []
+
+        for path in paths:
+            name = path.name.split("/")[-1]
+            if path.is_directory:
+                uri = f"az://{file_system}/{path.name}"
+                if DeltaTable.is_deltatable(
+                    uri, storage_options=self._get_storage_options(uri)
+                ):
+                    relations.append(
+                        schema_relation.create(
+                            database=schema_relation.database,
+                            schema=schema,
+                            identifier=name,
+                            type=RelationType.Table,
+                            catalog=schema_relation.catalog,
+                            file_format="delta",
+                        )
+                    )
+            else:
+                stem, _, ext = name.rpartition(".")
+                if ext in _FILE_FORMATS:
+                    relations.append(
+                        schema_relation.create(
+                            database=schema_relation.database,
+                            schema=schema,
+                            identifier=stem,
+                            type=RelationType.Table,
+                            catalog=schema_relation.catalog,
+                            file_format=ext,
+                        )
+                    )
+
+        return relations