dbt-polars-catalog-snowflake 0.4.0.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,228 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ # Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ # poetry.lock
109
+ # poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ # pdm.lock
116
+ # pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ # pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # Redis
135
+ *.rdb
136
+ *.aof
137
+ *.pid
138
+
139
+ # RabbitMQ
140
+ mnesia/
141
+ rabbitmq/
142
+ rabbitmq-data/
143
+
144
+ # ActiveMQ
145
+ activemq-data/
146
+
147
+ # SageMath parsed files
148
+ *.sage.py
149
+
150
+ # Environments
151
+ .env
152
+ .envrc
153
+ .venv
154
+ env/
155
+ venv/
156
+ ENV/
157
+ env.bak/
158
+ venv.bak/
159
+
160
+ # Spyder project settings
161
+ .spyderproject
162
+ .spyproject
163
+
164
+ # Rope project settings
165
+ .ropeproject
166
+
167
+ # mkdocs documentation
168
+ /site
169
+
170
+ # mypy
171
+ .mypy_cache/
172
+ .dmypy.json
173
+ dmypy.json
174
+
175
+ # Pyre type checker
176
+ .pyre/
177
+
178
+ # pytype static type analyzer
179
+ .pytype/
180
+
181
+ # Cython debug symbols
182
+ cython_debug/
183
+
184
+ # PyCharm
185
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
186
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
187
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
188
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
189
+ # .idea/
190
+
191
+ # Abstra
192
+ # Abstra is an AI-powered process automation framework.
193
+ # Ignore directories containing user credentials, local state, and settings.
194
+ # Learn more at https://abstra.io/docs
195
+ .abstra/
196
+
197
+ # Visual Studio Code
198
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
199
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
200
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
201
+ # you could uncomment the following to ignore the entire vscode folder
202
+ .vscode/
203
+ # Temporary file for partial code execution
204
+ tempCodeRunnerFile.py
205
+
206
+ # Ruff stuff:
207
+ .ruff_cache/
208
+
209
+ # PyPI configuration file
210
+ .pypirc
211
+
212
+ # Marimo
213
+ marimo/_static/
214
+ marimo/_lsp/
215
+ __marimo__/
216
+
217
+ # Streamlit
218
+ .streamlit/secrets.toml
219
+
220
+ # uv.lock
221
+ uv.lock
222
+
223
+ # dbt-polars
224
+ logs/
225
+ test.env
226
+ .claude/
227
+ decisions.md
228
+ test_project*/
@@ -0,0 +1,98 @@
1
+ Metadata-Version: 2.5
2
+ Name: dbt-polars-catalog-snowflake
3
+ Version: 0.4.0.dev0
4
+ Summary: Snowflake Horizon Catalog (Iceberg) catalog for dbt-polars
5
+ Project-URL: Homepage, https://github.com/datamindedbe/incubator-dbt-polars
6
+ Project-URL: Repository, https://github.com/datamindedbe/incubator-dbt-polars
7
+ Project-URL: Changelog, https://github.com/datamindedbe/incubator-dbt-polars/releases
8
+ Author-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
9
+ Maintainer-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
10
+ License-Expression: Apache-2.0
11
+ Keywords: catalog,dbt,horizon,iceberg,polars,snowflake
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Database
20
+ Requires-Python: <3.14,>=3.10
21
+ Requires-Dist: dbt-polars==0.4.0.dev0
22
+ Description-Content-Type: text/markdown
23
+
24
+ # dbt-polars
25
+
26
+ dbt-polars integrates the flexibility and performance of Polars with the data governance and lineage capabilities of dbt.
27
+
28
+ Key features of **dbt-polars**:
29
+
30
+ * **Run anywhere**: Polars runs on any compute platform, so you’re not locked into an expensive data warehouse.
31
+ * **Flexible**: Available catalogs let you connect to local storage, Iceberg catalogs, Databricks, and more.
32
+ * **Extensible**: Create a custom catalog to fit your unique storage and warehouse needs.
33
+ * **First-class Python models**: Run Python models with the same performance and capabilities as SQL models.
34
+ * **Mix Python and SQL**: Choose the language that best fits the transformation.
35
+ * **dbt-native**: Bring lineage, documentation, and data testing into your workflow.
36
+ * **Ingestion**: Build efficient, incremental API ingestion with Python models and keep the entire pipeline visible in the lineage graph.
37
+
38
+ ## Installation
39
+
40
+ ```bash
41
+ pip install dbt-polars
42
+ ```
43
+
44
+ Extra dependencies are required for cloud and Iceberg backends:
45
+
46
+ ```bash
47
+ pip install 'dbt-polars[azure]' # Azure Blob Storage
48
+ pip install 'dbt-polars[databricks]' # Azure Blob Storage
49
+ pip install 'dbt-polars[fabric]' # Microsoft Fabric Lakehouse
50
+ pip install 'dbt-polars[snowflake]' # Snowflake (Iceberg tables)
51
+ pip install 'dbt-polars[iceberg]' # Apache Iceberg (SQLite, REST, Databricks, …)
52
+ ```
53
+
54
+ ## Catalogs
55
+
56
+ dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
57
+
58
+ | Catalog | Backend | Status |
59
+ |---|---|---|
60
+ | `local` | Local filesystem | Usable |
61
+ | `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
62
+ | `iceberg` with REST + Databricks or OneLake | Iceberg REST API (Unity Catalog, Microsoft Fabric OneLake) | Experimental |
63
+ | `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
64
+ | `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
65
+ | `databricks` | Unity Catalog REST API + credential vending | Experimental |
66
+ | `fabric` | Microsoft Fabric Lakehouse via OneLake | Experimental |
67
+ | `snowflake` | Snowflake Horizon Catalog (Iceberg REST) | Experimental |
68
+
69
+ ## Quick start
70
+
71
+ See the [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started) guide.
72
+
73
+ A minimal `profiles.yml` for local development is:
74
+
75
+ ```yaml
76
+ my_project:
77
+ target: dev
78
+ outputs:
79
+ dev:
80
+ type: polars
81
+ schema: dev
82
+
83
+ catalog_type: local
84
+ root: ./data
85
+ ```
86
+
87
+ ## Documentation
88
+
89
+ - [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started)
90
+ - [Catalog overview and profile reference](https://datamindedbe.github.io/incubator-dbt-polars/catalog-overview)
91
+ - [Local catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/local)
92
+ - [Azure catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/azure)
93
+ - [Iceberg catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/iceberg)
94
+ - [Databricks catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/databricks)
95
+ - [Fabric catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/fabric)
96
+ - [Snowflake catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/snowflake)
97
+ - [Custom catalogs](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/custom)
98
+ - [Python models and tests](https://datamindedbe.github.io/incubator-dbt-polars/python-models)
@@ -0,0 +1,75 @@
1
+ # dbt-polars
2
+
3
+ dbt-polars integrates the flexibility and performance of Polars with the data governance and lineage capabilities of dbt.
4
+
5
+ Key features of **dbt-polars**:
6
+
7
+ * **Run anywhere**: Polars runs on any compute platform, so you’re not locked into an expensive data warehouse.
8
+ * **Flexible**: Available catalogs let you connect to local storage, Iceberg catalogs, Databricks, and more.
9
+ * **Extensible**: Create a custom catalog to fit your unique storage and warehouse needs.
10
+ * **First-class Python models**: Run Python models with the same performance and capabilities as SQL models.
11
+ * **Mix Python and SQL**: Choose the language that best fits the transformation.
12
+ * **dbt-native**: Bring lineage, documentation, and data testing into your workflow.
13
+ * **Ingestion**: Build efficient, incremental API ingestion with Python models and keep the entire pipeline visible in the lineage graph.
14
+
15
+ ## Installation
16
+
17
+ ```bash
18
+ pip install dbt-polars
19
+ ```
20
+
21
+ Extra dependencies are required for cloud and Iceberg backends:
22
+
23
+ ```bash
24
+ pip install 'dbt-polars[azure]' # Azure Blob Storage
25
+ pip install 'dbt-polars[databricks]' # Azure Blob Storage
26
+ pip install 'dbt-polars[fabric]' # Microsoft Fabric Lakehouse
27
+ pip install 'dbt-polars[snowflake]' # Snowflake (Iceberg tables)
28
+ pip install 'dbt-polars[iceberg]' # Apache Iceberg (SQLite, REST, Databricks, …)
29
+ ```
30
+
31
+ ## Catalogs
32
+
33
+ dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
34
+
35
+ | Catalog | Backend | Status |
36
+ |---|---|---|
37
+ | `local` | Local filesystem | Usable |
38
+ | `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
39
+ | `iceberg` with REST + Databricks or OneLake | Iceberg REST API (Unity Catalog, Microsoft Fabric OneLake) | Experimental |
40
+ | `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
41
+ | `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
42
+ | `databricks` | Unity Catalog REST API + credential vending | Experimental |
43
+ | `fabric` | Microsoft Fabric Lakehouse via OneLake | Experimental |
44
+ | `snowflake` | Snowflake Horizon Catalog (Iceberg REST) | Experimental |
45
+
46
+ ## Quick start
47
+
48
+ See the [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started) guide.
49
+
50
+ A minimal `profiles.yml` for local development is:
51
+
52
+ ```yaml
53
+ my_project:
54
+ target: dev
55
+ outputs:
56
+ dev:
57
+ type: polars
58
+ schema: dev
59
+
60
+ catalog_type: local
61
+ root: ./data
62
+ ```
63
+
64
+ ## Documentation
65
+
66
+ - [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started)
67
+ - [Catalog overview and profile reference](https://datamindedbe.github.io/incubator-dbt-polars/catalog-overview)
68
+ - [Local catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/local)
69
+ - [Azure catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/azure)
70
+ - [Iceberg catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/iceberg)
71
+ - [Databricks catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/databricks)
72
+ - [Fabric catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/fabric)
73
+ - [Snowflake catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/snowflake)
74
+ - [Custom catalogs](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/custom)
75
+ - [Python models and tests](https://datamindedbe.github.io/incubator-dbt-polars/python-models)
@@ -0,0 +1,14 @@
1
+ from dbt_polars_catalog_snowflake.catalog import (
2
+ SnowflakeCatalog,
3
+ SnowflakeCatalogConfig,
4
+ )
5
+
6
+ config_class = SnowflakeCatalogConfig
7
+ create_catalog = SnowflakeCatalog
8
+
9
+ __all__ = [
10
+ "SnowflakeCatalog",
11
+ "SnowflakeCatalogConfig",
12
+ "config_class",
13
+ "create_catalog",
14
+ ]
@@ -0,0 +1,74 @@
1
+ from dbt.adapters.polars.catalogs import (
2
+ CatalogConfig,
3
+ IcebergCatalog,
4
+ IcebergCatalogConfig,
5
+ )
6
+
7
+ HORIZON_URI = "https://{account}.snowflakecomputing.com/polaris/api/catalog"
8
+
9
+
10
+ class SnowflakeCatalogConfig(CatalogConfig):
11
+ """Config for Snowflake-managed Iceberg tables through Horizon Catalog.
12
+
13
+ `account` is the `<org>-<account>` identifier, `database_name` the Snowflake
14
+ database holding the tables and `role` the role to act as. Every
15
+ other keyword is passed straight through to pyiceberg's REST catalog, so any
16
+ authentication it supports works, e.g. `credential` for a programmatic
17
+ access token or `token` for an OAuth access token.
18
+ """
19
+
20
+ def __init__(
21
+ self,
22
+ *,
23
+ name: str,
24
+ type: str,
25
+ schema: str,
26
+ account: str,
27
+ database_name: str,
28
+ role: str | None = None,
29
+ **kwargs: object,
30
+ ) -> None:
31
+ self.name = name
32
+ self.type = type
33
+ self.schema = schema
34
+ self.account = account
35
+ self.database_name = database_name
36
+ self.role = role
37
+ self.pyiceberg_kwargs: dict[str, object] = kwargs
38
+
39
+ def iceberg_config(self) -> IcebergCatalogConfig:
40
+ role_scope = {"scope": f"session:role:{self.role.upper()}"} if self.role else {}
41
+ return IcebergCatalogConfig(
42
+ name=self.name,
43
+ type="iceberg",
44
+ schema=self.schema,
45
+ pyiceberg_type="rest",
46
+ **{
47
+ "uri": HORIZON_URI.format(account=self.account),
48
+ "warehouse": self.database_name.upper(),
49
+ "header.X-Iceberg-Access-Delegation": "vended-credentials",
50
+ **role_scope,
51
+ **self.pyiceberg_kwargs,
52
+ },
53
+ )
54
+
55
+ def unique_field(self) -> str:
56
+ return f"{self.account}/{self.database_name}"
57
+
58
+ def connection_keys(self) -> tuple[str, ...]:
59
+ return ("name", "account", "database_name", "role")
60
+
61
+
62
+ class SnowflakeCatalog(IcebergCatalog):
63
+ """Snowflake-managed Iceberg tables through Snowflake Horizon Catalog.
64
+
65
+ Uses Horizon's Iceberg REST endpoint with credentials vended by Snowflake,
66
+ so no Snowflake warehouse is needed.
67
+ """
68
+
69
+ config: SnowflakeCatalogConfig # type: ignore[assignment]
70
+ min_commit_interval = 2
71
+
72
+ def __init__(self, config: SnowflakeCatalogConfig, project_root: str) -> None:
73
+ super().__init__(config.iceberg_config(), project_root)
74
+ self.config = config
@@ -0,0 +1,39 @@
1
+ import sys
2
+ from email.parser import Parser
3
+ from pathlib import Path
4
+
5
+ from hatchling.metadata.plugin.interface import MetadataHookInterface
6
+
7
+ if sys.version_info >= (3, 11):
8
+ import tomllib
9
+ else:
10
+ import tomli as tomllib
11
+
12
+
13
+ def core_version(root: Path) -> str:
14
+ """The dbt-polars version, which every catalog package shares."""
15
+ pkg_info = root / "PKG-INFO"
16
+ if pkg_info.exists():
17
+ return Parser().parsestr(pkg_info.read_text())["Version"]
18
+ with open(root.parent.parent / "pyproject.toml", "rb") as file:
19
+ return tomllib.load(file)["project"]["version"]
20
+
21
+
22
+ def core_readme(root: Path) -> str:
23
+ """The dbt-polars README; built from an sdist it's the copy included there."""
24
+ local = root / "README.md"
25
+ return (local if local.exists() else root.parent.parent / "README.md").read_text()
26
+
27
+
28
+ class CustomMetadataHook(MetadataHookInterface):
29
+ def update(self, metadata: dict) -> None:
30
+ version = core_version(Path(self.root))
31
+ metadata["version"] = version
32
+ metadata["readme"] = {
33
+ "content-type": "text/markdown",
34
+ "text": core_readme(Path(self.root)),
35
+ }
36
+ metadata["dependencies"] = [
37
+ f"dbt-polars=={version}",
38
+ *self.config.get("dependencies", []),
39
+ ]
@@ -0,0 +1,70 @@
1
+ [project]
2
+ name = "dbt-polars-catalog-snowflake"
3
+ dynamic = ["version", "dependencies", "readme"]
4
+ description = "Snowflake Horizon Catalog (Iceberg) catalog for dbt-polars"
5
+ requires-python = ">=3.10, <3.14"
6
+ license = "Apache-2.0"
7
+ keywords = ["dbt", "polars", "catalog", "snowflake", "iceberg", "horizon"]
8
+ authors = [
9
+ {name = "Jonas Crevecoeur", email = "jonas.crevecoeur@dataminded.com"}
10
+ ]
11
+ maintainers = [
12
+ {name = "Jonas Crevecoeur", email = "jonas.crevecoeur@dataminded.com"}
13
+ ]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Intended Audience :: Developers",
17
+ "Programming Language :: Python :: 3",
18
+ "Programming Language :: Python :: 3.10",
19
+ "Programming Language :: Python :: 3.11",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Programming Language :: Python :: 3.13",
22
+ "Topic :: Database",
23
+ ]
24
+
25
+ [project.urls]
26
+ Homepage = "https://github.com/datamindedbe/incubator-dbt-polars"
27
+ Repository = "https://github.com/datamindedbe/incubator-dbt-polars"
28
+ Changelog = "https://github.com/datamindedbe/incubator-dbt-polars/releases"
29
+
30
+ [build-system]
31
+ requires = ["hatchling", "tomli; python_version < '3.11'"]
32
+ build-backend = "hatchling.build"
33
+
34
+ # Version, readme and a dbt-polars==<version> pin come from the core project,
35
+ # see hatch_build.py. pyiceberg comes with the core `snowflake` extra.
36
+ [tool.hatch.metadata.hooks.custom]
37
+ dependencies = []
38
+
39
+ [tool.hatch.build.targets.sdist.force-include]
40
+ "../../README.md" = "README.md"
41
+
42
+ [tool.uv]
43
+ cache-keys = [
44
+ { file = "pyproject.toml" },
45
+ { file = "hatch_build.py" },
46
+ { file = "../../pyproject.toml" },
47
+ { file = "../../README.md" },
48
+ ]
49
+
50
+ [tool.uv.sources]
51
+ dbt-polars = { workspace = true }
52
+
53
+ [tool.ruff]
54
+ extend = "../../pyproject.toml"
55
+
56
+ [tool.ruff.lint]
57
+ extend-select = ["TID251"]
58
+
59
+ [tool.ruff.lint.flake8-tidy-imports.banned-api]
60
+ "dbt.adapters.polars.catalogs.baseCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
61
+ "dbt.adapters.polars.catalogs.storageCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
62
+ "dbt.adapters.polars.catalogs.tableFormatCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
63
+ "dbt.adapters.polars.catalogs.formats".msg = "Use the public API from dbt.adapters.polars.catalogs"
64
+ "dbt.adapters.polars.catalogs.icebergCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
65
+ "dbt.adapters.polars.catalogs.iceberg_write_utils".msg = "Use the public API from dbt.adapters.polars.catalogs"
66
+ "dbt.adapters.polars.connections".msg = "Use the public API from dbt.adapters.polars.catalogs"
67
+ "dbt.adapters.polars.impl".msg = "Use the public API from dbt.adapters.polars.catalogs"
68
+ "dbt.adapters.polars.relation".msg = "Use the public API from dbt.adapters.polars.catalogs"
69
+ "dbt.adapters.polars.sql_rewrite".msg = "Use the public API from dbt.adapters.polars.catalogs"
70
+ "dbt.adapters.polars.utils".msg = "Use the public API from dbt.adapters.polars.catalogs"
@@ -0,0 +1,145 @@
1
+ import subprocess
2
+ import sys
3
+ from types import SimpleNamespace
4
+
5
+ import polars as pl
6
+ import pytest
7
+ from dbt.adapters.polars.catalogs import IcebergCatalog, PolarsRelation
8
+ from dbt_common.exceptions import DbtRuntimeError
9
+
10
+ from dbt_polars_catalog_snowflake import SnowflakeCatalog, SnowflakeCatalogConfig
11
+
12
+ DATA = pl.DataFrame({"id": [1]})
13
+
14
+
15
+ def make_config(**kwargs) -> SnowflakeCatalogConfig:
16
+ return SnowflakeCatalogConfig(
17
+ name="snowflake",
18
+ type="snowflake",
19
+ schema="analytics",
20
+ account="myorg-myaccount",
21
+ database_name="analytics_db",
22
+ **kwargs,
23
+ )
24
+
25
+
26
+ def make_catalog(**kwargs) -> SnowflakeCatalog:
27
+ return SnowflakeCatalog(make_config(**kwargs), project_root=".")
28
+
29
+
30
+ def relation(**kwargs) -> PolarsRelation:
31
+ return PolarsRelation.create(
32
+ database="snowflake",
33
+ schema="analytics",
34
+ identifier="orders",
35
+ catalog="snowflake",
36
+ **kwargs,
37
+ )
38
+
39
+
40
+ def test_properties_point_to_horizon():
41
+ properties = make_catalog(role="transformer")._catalog_properties
42
+
43
+ assert properties["type"] == "rest"
44
+ assert properties["uri"] == (
45
+ "https://myorg-myaccount.snowflakecomputing.com/polaris/api/catalog"
46
+ )
47
+ assert properties["warehouse"] == "ANALYTICS_DB"
48
+ assert properties["scope"] == "session:role:TRANSFORMER"
49
+ assert properties["header.X-Iceberg-Access-Delegation"] == "vended-credentials"
50
+
51
+
52
+ def test_other_profile_keys_go_to_pyiceberg():
53
+ properties = make_catalog(
54
+ credential="pat", scope="session:role:OTHER"
55
+ )._catalog_properties
56
+
57
+ assert properties["credential"] == "pat"
58
+ assert properties["scope"] == "session:role:OTHER"
59
+
60
+
61
+ def test_profile_overrides_uri_and_warehouse():
62
+ properties = make_catalog(
63
+ uri="https://private.example/polaris/api/catalog", warehouse="OTHER_DB"
64
+ )._catalog_properties
65
+
66
+ assert properties["uri"] == "https://private.example/polaris/api/catalog"
67
+ assert properties["warehouse"] == "OTHER_DB"
68
+
69
+
70
+ def test_names_are_kept_as_in_dbt():
71
+ pytest.importorskip("pyiceberg")
72
+ catalog = make_catalog()
73
+ listed = []
74
+ catalog._catalog_instance = SimpleNamespace(
75
+ list_tables=lambda namespace: (
76
+ listed.append(namespace)
77
+ or [("Analytics", "MyOrders"), ("Analytics", "customers")]
78
+ ),
79
+ list_namespaces=lambda: [("Analytics",), ("raw",)],
80
+ )
81
+ schema_relation = PolarsRelation.create(
82
+ database="snowflake",
83
+ schema="Analytics",
84
+ identifier="MyOrders",
85
+ catalog="snowflake",
86
+ )
87
+
88
+ names = [
89
+ r.identifier for r in catalog.list_relations_without_caching(schema_relation)
90
+ ]
91
+
92
+ assert catalog._id(schema_relation) == ("Analytics", "MyOrders")
93
+ assert names == ["MyOrders", "customers"]
94
+ assert listed == [("Analytics",)]
95
+ assert catalog.list_schemas() == ["Analytics", "raw"]
96
+
97
+
98
+ @pytest.mark.parametrize(
99
+ "write",
100
+ [
101
+ lambda catalog, rel: catalog.write_relation(rel, DATA, []),
102
+ lambda catalog, rel: catalog.validate_incremental_relation(rel, {}),
103
+ lambda catalog, rel: catalog.apply_snapshot_delta(rel, DATA, DATA),
104
+ ],
105
+ )
106
+ def test_writes_reject_file_format(write):
107
+ with pytest.raises(
108
+ DbtRuntimeError, match="file_format is not supported for the snowflake catalog"
109
+ ):
110
+ write(make_catalog(), relation(file_format="parquet"))
111
+
112
+
113
+ def record_iceberg_calls(monkeypatch, method: str, returns=None) -> list[tuple]:
114
+ """Replace an IcebergCatalog method with one recording its arguments."""
115
+ calls: list[tuple] = []
116
+
117
+ def record(self, *args):
118
+ calls.append(args)
119
+ return returns
120
+
121
+ monkeypatch.setattr(IcebergCatalog, method, record)
122
+ return calls
123
+
124
+
125
+ def test_column_names_are_written_as_they_are_in_dbt(monkeypatch):
126
+ calls = record_iceberg_calls(monkeypatch, "write_relation")
127
+
128
+ make_catalog().write_relation(
129
+ relation(), pl.DataFrame({"id": [1], "CustomerName": ["a"]}), ["id"], {}
130
+ )
131
+
132
+ _, data, partition_by, _ = calls[0]
133
+ assert data.columns == ["id", "CustomerName"]
134
+ assert partition_by == ["id"]
135
+
136
+
137
+ def test_catalog_does_not_import_pyiceberg_until_used():
138
+ code = (
139
+ "import sys\n"
140
+ "from dbt_polars_catalog_snowflake import create_catalog, config_class\n"
141
+ "create_catalog(config_class(name='s', type='snowflake', schema='s', "
142
+ "account='o-a', database_name='d'), '.')\n"
143
+ "assert not [m for m in sys.modules if m.startswith('pyiceberg')]\n"
144
+ )
145
+ subprocess.run([sys.executable, "-c", code], check=True)