dbt-polars-catalog-snowflake 0.4.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_polars_catalog_snowflake-0.4.0.dev0/.gitignore +228 -0
- dbt_polars_catalog_snowflake-0.4.0.dev0/PKG-INFO +98 -0
- dbt_polars_catalog_snowflake-0.4.0.dev0/README.md +75 -0
- dbt_polars_catalog_snowflake-0.4.0.dev0/dbt_polars_catalog_snowflake/__init__.py +14 -0
- dbt_polars_catalog_snowflake-0.4.0.dev0/dbt_polars_catalog_snowflake/catalog.py +74 -0
- dbt_polars_catalog_snowflake-0.4.0.dev0/hatch_build.py +39 -0
- dbt_polars_catalog_snowflake-0.4.0.dev0/pyproject.toml +70 -0
- dbt_polars_catalog_snowflake-0.4.0.dev0/tests/test_snowflake_catalog.py +145 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
# Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
# poetry.lock
|
|
109
|
+
# poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
# pdm.lock
|
|
116
|
+
# pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
# pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# Redis
|
|
135
|
+
*.rdb
|
|
136
|
+
*.aof
|
|
137
|
+
*.pid
|
|
138
|
+
|
|
139
|
+
# RabbitMQ
|
|
140
|
+
mnesia/
|
|
141
|
+
rabbitmq/
|
|
142
|
+
rabbitmq-data/
|
|
143
|
+
|
|
144
|
+
# ActiveMQ
|
|
145
|
+
activemq-data/
|
|
146
|
+
|
|
147
|
+
# SageMath parsed files
|
|
148
|
+
*.sage.py
|
|
149
|
+
|
|
150
|
+
# Environments
|
|
151
|
+
.env
|
|
152
|
+
.envrc
|
|
153
|
+
.venv
|
|
154
|
+
env/
|
|
155
|
+
venv/
|
|
156
|
+
ENV/
|
|
157
|
+
env.bak/
|
|
158
|
+
venv.bak/
|
|
159
|
+
|
|
160
|
+
# Spyder project settings
|
|
161
|
+
.spyderproject
|
|
162
|
+
.spyproject
|
|
163
|
+
|
|
164
|
+
# Rope project settings
|
|
165
|
+
.ropeproject
|
|
166
|
+
|
|
167
|
+
# mkdocs documentation
|
|
168
|
+
/site
|
|
169
|
+
|
|
170
|
+
# mypy
|
|
171
|
+
.mypy_cache/
|
|
172
|
+
.dmypy.json
|
|
173
|
+
dmypy.json
|
|
174
|
+
|
|
175
|
+
# Pyre type checker
|
|
176
|
+
.pyre/
|
|
177
|
+
|
|
178
|
+
# pytype static type analyzer
|
|
179
|
+
.pytype/
|
|
180
|
+
|
|
181
|
+
# Cython debug symbols
|
|
182
|
+
cython_debug/
|
|
183
|
+
|
|
184
|
+
# PyCharm
|
|
185
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
186
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
188
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
|
+
# .idea/
|
|
190
|
+
|
|
191
|
+
# Abstra
|
|
192
|
+
# Abstra is an AI-powered process automation framework.
|
|
193
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
194
|
+
# Learn more at https://abstra.io/docs
|
|
195
|
+
.abstra/
|
|
196
|
+
|
|
197
|
+
# Visual Studio Code
|
|
198
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
199
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
200
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
201
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
202
|
+
.vscode/
|
|
203
|
+
# Temporary file for partial code execution
|
|
204
|
+
tempCodeRunnerFile.py
|
|
205
|
+
|
|
206
|
+
# Ruff stuff:
|
|
207
|
+
.ruff_cache/
|
|
208
|
+
|
|
209
|
+
# PyPI configuration file
|
|
210
|
+
.pypirc
|
|
211
|
+
|
|
212
|
+
# Marimo
|
|
213
|
+
marimo/_static/
|
|
214
|
+
marimo/_lsp/
|
|
215
|
+
__marimo__/
|
|
216
|
+
|
|
217
|
+
# Streamlit
|
|
218
|
+
.streamlit/secrets.toml
|
|
219
|
+
|
|
220
|
+
# uv.lock
|
|
221
|
+
uv.lock
|
|
222
|
+
|
|
223
|
+
# dbt-polars
|
|
224
|
+
logs/
|
|
225
|
+
test.env
|
|
226
|
+
.claude/
|
|
227
|
+
decisions.md
|
|
228
|
+
test_project*/
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: dbt-polars-catalog-snowflake
|
|
3
|
+
Version: 0.4.0.dev0
|
|
4
|
+
Summary: Snowflake Horizon Catalog (Iceberg) catalog for dbt-polars
|
|
5
|
+
Project-URL: Homepage, https://github.com/datamindedbe/incubator-dbt-polars
|
|
6
|
+
Project-URL: Repository, https://github.com/datamindedbe/incubator-dbt-polars
|
|
7
|
+
Project-URL: Changelog, https://github.com/datamindedbe/incubator-dbt-polars/releases
|
|
8
|
+
Author-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
|
|
9
|
+
Maintainer-email: Jonas Crevecoeur <jonas.crevecoeur@dataminded.com>
|
|
10
|
+
License-Expression: Apache-2.0
|
|
11
|
+
Keywords: catalog,dbt,horizon,iceberg,polars,snowflake
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Database
|
|
20
|
+
Requires-Python: <3.14,>=3.10
|
|
21
|
+
Requires-Dist: dbt-polars==0.4.0.dev0
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# dbt-polars
|
|
25
|
+
|
|
26
|
+
dbt-polars integrates the flexibility and performance of Polars with the data governance and lineage capabilities of dbt.
|
|
27
|
+
|
|
28
|
+
Key features of **dbt-polars**:
|
|
29
|
+
|
|
30
|
+
* **Run anywhere**: Polars runs on any compute platform, so you’re not locked into an expensive data warehouse.
|
|
31
|
+
* **Flexible**: Available catalogs let you connect to local storage, Iceberg catalogs, Databricks, and more.
|
|
32
|
+
* **Extensible**: Create a custom catalog to fit your unique storage and warehouse needs.
|
|
33
|
+
* **First-class Python models**: Run Python models with the same performance and capabilities as SQL models.
|
|
34
|
+
* **Mix Python and SQL**: Choose the language that best fits the transformation.
|
|
35
|
+
* **dbt-native**: Bring lineage, documentation, and data testing into your workflow.
|
|
36
|
+
* **Ingestion**: Build efficient, incremental API ingestion with Python models and keep the entire pipeline visible in the lineage graph.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install dbt-polars
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Extra dependencies are required for cloud and Iceberg backends:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install 'dbt-polars[azure]' # Azure Blob Storage
|
|
48
|
+
pip install 'dbt-polars[databricks]' # Azure Blob Storage
|
|
49
|
+
pip install 'dbt-polars[fabric]' # Microsoft Fabric Lakehouse
|
|
50
|
+
pip install 'dbt-polars[snowflake]' # Snowflake (Iceberg tables)
|
|
51
|
+
pip install 'dbt-polars[iceberg]' # Apache Iceberg (SQLite, REST, Databricks, …)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Catalogs
|
|
55
|
+
|
|
56
|
+
dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
|
|
57
|
+
|
|
58
|
+
| Catalog | Backend | Status |
|
|
59
|
+
|---|---|---|
|
|
60
|
+
| `local` | Local filesystem | Usable |
|
|
61
|
+
| `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
|
|
62
|
+
| `iceberg` with REST + Databricks or OneLake | Iceberg REST API (Unity Catalog, Microsoft Fabric OneLake) | Experimental |
|
|
63
|
+
| `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
|
|
64
|
+
| `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
|
|
65
|
+
| `databricks` | Unity Catalog REST API + credential vending | Experimental |
|
|
66
|
+
| `fabric` | Microsoft Fabric Lakehouse via OneLake | Experimental |
|
|
67
|
+
| `snowflake` | Snowflake Horizon Catalog (Iceberg REST) | Experimental |
|
|
68
|
+
|
|
69
|
+
## Quick start
|
|
70
|
+
|
|
71
|
+
See the [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started) guide.
|
|
72
|
+
|
|
73
|
+
A minimal `profiles.yml` for local development is:
|
|
74
|
+
|
|
75
|
+
```yaml
|
|
76
|
+
my_project:
|
|
77
|
+
target: dev
|
|
78
|
+
outputs:
|
|
79
|
+
dev:
|
|
80
|
+
type: polars
|
|
81
|
+
schema: dev
|
|
82
|
+
|
|
83
|
+
catalog_type: local
|
|
84
|
+
root: ./data
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Documentation
|
|
88
|
+
|
|
89
|
+
- [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started)
|
|
90
|
+
- [Catalog overview and profile reference](https://datamindedbe.github.io/incubator-dbt-polars/catalog-overview)
|
|
91
|
+
- [Local catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/local)
|
|
92
|
+
- [Azure catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/azure)
|
|
93
|
+
- [Iceberg catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/iceberg)
|
|
94
|
+
- [Databricks catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/databricks)
|
|
95
|
+
- [Fabric catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/fabric)
|
|
96
|
+
- [Snowflake catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/snowflake)
|
|
97
|
+
- [Custom catalogs](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/custom)
|
|
98
|
+
- [Python models and tests](https://datamindedbe.github.io/incubator-dbt-polars/python-models)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# dbt-polars
|
|
2
|
+
|
|
3
|
+
dbt-polars integrates the flexibility and performance of Polars with the data governance and lineage capabilities of dbt.
|
|
4
|
+
|
|
5
|
+
Key features of **dbt-polars**:
|
|
6
|
+
|
|
7
|
+
* **Run anywhere**: Polars runs on any compute platform, so you’re not locked into an expensive data warehouse.
|
|
8
|
+
* **Flexible**: Available catalogs let you connect to local storage, Iceberg catalogs, Databricks, and more.
|
|
9
|
+
* **Extensible**: Create a custom catalog to fit your unique storage and warehouse needs.
|
|
10
|
+
* **First-class Python models**: Run Python models with the same performance and capabilities as SQL models.
|
|
11
|
+
* **Mix Python and SQL**: Choose the language that best fits the transformation.
|
|
12
|
+
* **dbt-native**: Bring lineage, documentation, and data testing into your workflow.
|
|
13
|
+
* **Ingestion**: Build efficient, incremental API ingestion with Python models and keep the entire pipeline visible in the lineage graph.
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install dbt-polars
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Extra dependencies are required for cloud and Iceberg backends:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install 'dbt-polars[azure]' # Azure Blob Storage
|
|
25
|
+
pip install 'dbt-polars[databricks]' # Azure Blob Storage
|
|
26
|
+
pip install 'dbt-polars[fabric]' # Microsoft Fabric Lakehouse
|
|
27
|
+
pip install 'dbt-polars[snowflake]' # Snowflake (Iceberg tables)
|
|
28
|
+
pip install 'dbt-polars[iceberg]' # Apache Iceberg (SQLite, REST, Databricks, …)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Catalogs
|
|
32
|
+
|
|
33
|
+
dbt-polars uses **catalogs** to define where data is stored. Each catalog maps to a storage backend. You can configure multiple catalogs in a single profile and reference them from your models.
|
|
34
|
+
|
|
35
|
+
| Catalog | Backend | Status |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| `local` | Local filesystem | Usable |
|
|
38
|
+
| `iceberg` with SQLite | Iceberg + SQLite metastore | Usable |
|
|
39
|
+
| `iceberg` with REST + Databricks or OneLake | Iceberg REST API (Unity Catalog, Microsoft Fabric OneLake) | Experimental |
|
|
40
|
+
| `iceberg` with other backends | Any pyiceberg-supported backend | Experimental |
|
|
41
|
+
| `azure` | Azure Blob Storage / ADLS Gen2 | Experimental |
|
|
42
|
+
| `databricks` | Unity Catalog REST API + credential vending | Experimental |
|
|
43
|
+
| `fabric` | Microsoft Fabric Lakehouse via OneLake | Experimental |
|
|
44
|
+
| `snowflake` | Snowflake Horizon Catalog (Iceberg REST) | Experimental |
|
|
45
|
+
|
|
46
|
+
## Quick start
|
|
47
|
+
|
|
48
|
+
See the [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started) guide.
|
|
49
|
+
|
|
50
|
+
A minimal `profiles.yml` for local development is:
|
|
51
|
+
|
|
52
|
+
```yaml
|
|
53
|
+
my_project:
|
|
54
|
+
target: dev
|
|
55
|
+
outputs:
|
|
56
|
+
dev:
|
|
57
|
+
type: polars
|
|
58
|
+
schema: dev
|
|
59
|
+
|
|
60
|
+
catalog_type: local
|
|
61
|
+
root: ./data
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Documentation
|
|
65
|
+
|
|
66
|
+
- [Getting started](https://datamindedbe.github.io/incubator-dbt-polars/demos/getting_started)
|
|
67
|
+
- [Catalog overview and profile reference](https://datamindedbe.github.io/incubator-dbt-polars/catalog-overview)
|
|
68
|
+
- [Local catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/local)
|
|
69
|
+
- [Azure catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/azure)
|
|
70
|
+
- [Iceberg catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/iceberg)
|
|
71
|
+
- [Databricks catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/databricks)
|
|
72
|
+
- [Fabric catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/fabric)
|
|
73
|
+
- [Snowflake catalog](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/snowflake)
|
|
74
|
+
- [Custom catalogs](https://datamindedbe.github.io/incubator-dbt-polars/catalogs/custom)
|
|
75
|
+
- [Python models and tests](https://datamindedbe.github.io/incubator-dbt-polars/python-models)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from dbt_polars_catalog_snowflake.catalog import (
|
|
2
|
+
SnowflakeCatalog,
|
|
3
|
+
SnowflakeCatalogConfig,
|
|
4
|
+
)
|
|
5
|
+
|
|
6
|
+
config_class = SnowflakeCatalogConfig
|
|
7
|
+
create_catalog = SnowflakeCatalog
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"SnowflakeCatalog",
|
|
11
|
+
"SnowflakeCatalogConfig",
|
|
12
|
+
"config_class",
|
|
13
|
+
"create_catalog",
|
|
14
|
+
]
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from dbt.adapters.polars.catalogs import (
|
|
2
|
+
CatalogConfig,
|
|
3
|
+
IcebergCatalog,
|
|
4
|
+
IcebergCatalogConfig,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
HORIZON_URI = "https://{account}.snowflakecomputing.com/polaris/api/catalog"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class SnowflakeCatalogConfig(CatalogConfig):
|
|
11
|
+
"""Config for Snowflake-managed Iceberg tables through Horizon Catalog.
|
|
12
|
+
|
|
13
|
+
`account` is the `<org>-<account>` identifier, `database_name` the Snowflake
|
|
14
|
+
database holding the tables and `role` the role to act as. Every
|
|
15
|
+
other keyword is passed straight through to pyiceberg's REST catalog, so any
|
|
16
|
+
authentication it supports works, e.g. `credential` for a programmatic
|
|
17
|
+
access token or `token` for an OAuth access token.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
*,
|
|
23
|
+
name: str,
|
|
24
|
+
type: str,
|
|
25
|
+
schema: str,
|
|
26
|
+
account: str,
|
|
27
|
+
database_name: str,
|
|
28
|
+
role: str | None = None,
|
|
29
|
+
**kwargs: object,
|
|
30
|
+
) -> None:
|
|
31
|
+
self.name = name
|
|
32
|
+
self.type = type
|
|
33
|
+
self.schema = schema
|
|
34
|
+
self.account = account
|
|
35
|
+
self.database_name = database_name
|
|
36
|
+
self.role = role
|
|
37
|
+
self.pyiceberg_kwargs: dict[str, object] = kwargs
|
|
38
|
+
|
|
39
|
+
def iceberg_config(self) -> IcebergCatalogConfig:
|
|
40
|
+
role_scope = {"scope": f"session:role:{self.role.upper()}"} if self.role else {}
|
|
41
|
+
return IcebergCatalogConfig(
|
|
42
|
+
name=self.name,
|
|
43
|
+
type="iceberg",
|
|
44
|
+
schema=self.schema,
|
|
45
|
+
pyiceberg_type="rest",
|
|
46
|
+
**{
|
|
47
|
+
"uri": HORIZON_URI.format(account=self.account),
|
|
48
|
+
"warehouse": self.database_name.upper(),
|
|
49
|
+
"header.X-Iceberg-Access-Delegation": "vended-credentials",
|
|
50
|
+
**role_scope,
|
|
51
|
+
**self.pyiceberg_kwargs,
|
|
52
|
+
},
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def unique_field(self) -> str:
|
|
56
|
+
return f"{self.account}/{self.database_name}"
|
|
57
|
+
|
|
58
|
+
def connection_keys(self) -> tuple[str, ...]:
|
|
59
|
+
return ("name", "account", "database_name", "role")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class SnowflakeCatalog(IcebergCatalog):
|
|
63
|
+
"""Snowflake-managed Iceberg tables through Snowflake Horizon Catalog.
|
|
64
|
+
|
|
65
|
+
Uses Horizon's Iceberg REST endpoint with credentials vended by Snowflake,
|
|
66
|
+
so no Snowflake warehouse is needed.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
config: SnowflakeCatalogConfig # type: ignore[assignment]
|
|
70
|
+
min_commit_interval = 2
|
|
71
|
+
|
|
72
|
+
def __init__(self, config: SnowflakeCatalogConfig, project_root: str) -> None:
|
|
73
|
+
super().__init__(config.iceberg_config(), project_root)
|
|
74
|
+
self.config = config
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from email.parser import Parser
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from hatchling.metadata.plugin.interface import MetadataHookInterface
|
|
6
|
+
|
|
7
|
+
if sys.version_info >= (3, 11):
|
|
8
|
+
import tomllib
|
|
9
|
+
else:
|
|
10
|
+
import tomli as tomllib
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def core_version(root: Path) -> str:
|
|
14
|
+
"""The dbt-polars version, which every catalog package shares."""
|
|
15
|
+
pkg_info = root / "PKG-INFO"
|
|
16
|
+
if pkg_info.exists():
|
|
17
|
+
return Parser().parsestr(pkg_info.read_text())["Version"]
|
|
18
|
+
with open(root.parent.parent / "pyproject.toml", "rb") as file:
|
|
19
|
+
return tomllib.load(file)["project"]["version"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def core_readme(root: Path) -> str:
|
|
23
|
+
"""The dbt-polars README; built from an sdist it's the copy included there."""
|
|
24
|
+
local = root / "README.md"
|
|
25
|
+
return (local if local.exists() else root.parent.parent / "README.md").read_text()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CustomMetadataHook(MetadataHookInterface):
|
|
29
|
+
def update(self, metadata: dict) -> None:
|
|
30
|
+
version = core_version(Path(self.root))
|
|
31
|
+
metadata["version"] = version
|
|
32
|
+
metadata["readme"] = {
|
|
33
|
+
"content-type": "text/markdown",
|
|
34
|
+
"text": core_readme(Path(self.root)),
|
|
35
|
+
}
|
|
36
|
+
metadata["dependencies"] = [
|
|
37
|
+
f"dbt-polars=={version}",
|
|
38
|
+
*self.config.get("dependencies", []),
|
|
39
|
+
]
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dbt-polars-catalog-snowflake"
|
|
3
|
+
dynamic = ["version", "dependencies", "readme"]
|
|
4
|
+
description = "Snowflake Horizon Catalog (Iceberg) catalog for dbt-polars"
|
|
5
|
+
requires-python = ">=3.10, <3.14"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
keywords = ["dbt", "polars", "catalog", "snowflake", "iceberg", "horizon"]
|
|
8
|
+
authors = [
|
|
9
|
+
{name = "Jonas Crevecoeur", email = "jonas.crevecoeur@dataminded.com"}
|
|
10
|
+
]
|
|
11
|
+
maintainers = [
|
|
12
|
+
{name = "Jonas Crevecoeur", email = "jonas.crevecoeur@dataminded.com"}
|
|
13
|
+
]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.10",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Topic :: Database",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/datamindedbe/incubator-dbt-polars"
|
|
27
|
+
Repository = "https://github.com/datamindedbe/incubator-dbt-polars"
|
|
28
|
+
Changelog = "https://github.com/datamindedbe/incubator-dbt-polars/releases"
|
|
29
|
+
|
|
30
|
+
[build-system]
|
|
31
|
+
requires = ["hatchling", "tomli; python_version < '3.11'"]
|
|
32
|
+
build-backend = "hatchling.build"
|
|
33
|
+
|
|
34
|
+
# Version, readme and a dbt-polars==<version> pin come from the core project,
|
|
35
|
+
# see hatch_build.py. pyiceberg comes with the core `snowflake` extra.
|
|
36
|
+
[tool.hatch.metadata.hooks.custom]
|
|
37
|
+
dependencies = []
|
|
38
|
+
|
|
39
|
+
[tool.hatch.build.targets.sdist.force-include]
|
|
40
|
+
"../../README.md" = "README.md"
|
|
41
|
+
|
|
42
|
+
[tool.uv]
|
|
43
|
+
cache-keys = [
|
|
44
|
+
{ file = "pyproject.toml" },
|
|
45
|
+
{ file = "hatch_build.py" },
|
|
46
|
+
{ file = "../../pyproject.toml" },
|
|
47
|
+
{ file = "../../README.md" },
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[tool.uv.sources]
|
|
51
|
+
dbt-polars = { workspace = true }
|
|
52
|
+
|
|
53
|
+
[tool.ruff]
|
|
54
|
+
extend = "../../pyproject.toml"
|
|
55
|
+
|
|
56
|
+
[tool.ruff.lint]
|
|
57
|
+
extend-select = ["TID251"]
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
|
60
|
+
"dbt.adapters.polars.catalogs.baseCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
61
|
+
"dbt.adapters.polars.catalogs.storageCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
62
|
+
"dbt.adapters.polars.catalogs.tableFormatCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
63
|
+
"dbt.adapters.polars.catalogs.formats".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
64
|
+
"dbt.adapters.polars.catalogs.icebergCatalog".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
65
|
+
"dbt.adapters.polars.catalogs.iceberg_write_utils".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
66
|
+
"dbt.adapters.polars.connections".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
67
|
+
"dbt.adapters.polars.impl".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
68
|
+
"dbt.adapters.polars.relation".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
69
|
+
"dbt.adapters.polars.sql_rewrite".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
70
|
+
"dbt.adapters.polars.utils".msg = "Use the public API from dbt.adapters.polars.catalogs"
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import sys
|
|
3
|
+
from types import SimpleNamespace
|
|
4
|
+
|
|
5
|
+
import polars as pl
|
|
6
|
+
import pytest
|
|
7
|
+
from dbt.adapters.polars.catalogs import IcebergCatalog, PolarsRelation
|
|
8
|
+
from dbt_common.exceptions import DbtRuntimeError
|
|
9
|
+
|
|
10
|
+
from dbt_polars_catalog_snowflake import SnowflakeCatalog, SnowflakeCatalogConfig
|
|
11
|
+
|
|
12
|
+
DATA = pl.DataFrame({"id": [1]})
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def make_config(**kwargs) -> SnowflakeCatalogConfig:
|
|
16
|
+
return SnowflakeCatalogConfig(
|
|
17
|
+
name="snowflake",
|
|
18
|
+
type="snowflake",
|
|
19
|
+
schema="analytics",
|
|
20
|
+
account="myorg-myaccount",
|
|
21
|
+
database_name="analytics_db",
|
|
22
|
+
**kwargs,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def make_catalog(**kwargs) -> SnowflakeCatalog:
|
|
27
|
+
return SnowflakeCatalog(make_config(**kwargs), project_root=".")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def relation(**kwargs) -> PolarsRelation:
|
|
31
|
+
return PolarsRelation.create(
|
|
32
|
+
database="snowflake",
|
|
33
|
+
schema="analytics",
|
|
34
|
+
identifier="orders",
|
|
35
|
+
catalog="snowflake",
|
|
36
|
+
**kwargs,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_properties_point_to_horizon():
|
|
41
|
+
properties = make_catalog(role="transformer")._catalog_properties
|
|
42
|
+
|
|
43
|
+
assert properties["type"] == "rest"
|
|
44
|
+
assert properties["uri"] == (
|
|
45
|
+
"https://myorg-myaccount.snowflakecomputing.com/polaris/api/catalog"
|
|
46
|
+
)
|
|
47
|
+
assert properties["warehouse"] == "ANALYTICS_DB"
|
|
48
|
+
assert properties["scope"] == "session:role:TRANSFORMER"
|
|
49
|
+
assert properties["header.X-Iceberg-Access-Delegation"] == "vended-credentials"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_other_profile_keys_go_to_pyiceberg():
|
|
53
|
+
properties = make_catalog(
|
|
54
|
+
credential="pat", scope="session:role:OTHER"
|
|
55
|
+
)._catalog_properties
|
|
56
|
+
|
|
57
|
+
assert properties["credential"] == "pat"
|
|
58
|
+
assert properties["scope"] == "session:role:OTHER"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_profile_overrides_uri_and_warehouse():
|
|
62
|
+
properties = make_catalog(
|
|
63
|
+
uri="https://private.example/polaris/api/catalog", warehouse="OTHER_DB"
|
|
64
|
+
)._catalog_properties
|
|
65
|
+
|
|
66
|
+
assert properties["uri"] == "https://private.example/polaris/api/catalog"
|
|
67
|
+
assert properties["warehouse"] == "OTHER_DB"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_names_are_kept_as_in_dbt():
|
|
71
|
+
pytest.importorskip("pyiceberg")
|
|
72
|
+
catalog = make_catalog()
|
|
73
|
+
listed = []
|
|
74
|
+
catalog._catalog_instance = SimpleNamespace(
|
|
75
|
+
list_tables=lambda namespace: (
|
|
76
|
+
listed.append(namespace)
|
|
77
|
+
or [("Analytics", "MyOrders"), ("Analytics", "customers")]
|
|
78
|
+
),
|
|
79
|
+
list_namespaces=lambda: [("Analytics",), ("raw",)],
|
|
80
|
+
)
|
|
81
|
+
schema_relation = PolarsRelation.create(
|
|
82
|
+
database="snowflake",
|
|
83
|
+
schema="Analytics",
|
|
84
|
+
identifier="MyOrders",
|
|
85
|
+
catalog="snowflake",
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
names = [
|
|
89
|
+
r.identifier for r in catalog.list_relations_without_caching(schema_relation)
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
assert catalog._id(schema_relation) == ("Analytics", "MyOrders")
|
|
93
|
+
assert names == ["MyOrders", "customers"]
|
|
94
|
+
assert listed == [("Analytics",)]
|
|
95
|
+
assert catalog.list_schemas() == ["Analytics", "raw"]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@pytest.mark.parametrize(
|
|
99
|
+
"write",
|
|
100
|
+
[
|
|
101
|
+
lambda catalog, rel: catalog.write_relation(rel, DATA, []),
|
|
102
|
+
lambda catalog, rel: catalog.validate_incremental_relation(rel, {}),
|
|
103
|
+
lambda catalog, rel: catalog.apply_snapshot_delta(rel, DATA, DATA),
|
|
104
|
+
],
|
|
105
|
+
)
|
|
106
|
+
def test_writes_reject_file_format(write):
|
|
107
|
+
with pytest.raises(
|
|
108
|
+
DbtRuntimeError, match="file_format is not supported for the snowflake catalog"
|
|
109
|
+
):
|
|
110
|
+
write(make_catalog(), relation(file_format="parquet"))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def record_iceberg_calls(monkeypatch, method: str, returns=None) -> list[tuple]:
|
|
114
|
+
"""Replace an IcebergCatalog method with one recording its arguments."""
|
|
115
|
+
calls: list[tuple] = []
|
|
116
|
+
|
|
117
|
+
def record(self, *args):
|
|
118
|
+
calls.append(args)
|
|
119
|
+
return returns
|
|
120
|
+
|
|
121
|
+
monkeypatch.setattr(IcebergCatalog, method, record)
|
|
122
|
+
return calls
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_column_names_are_written_as_they_are_in_dbt(monkeypatch):
|
|
126
|
+
calls = record_iceberg_calls(monkeypatch, "write_relation")
|
|
127
|
+
|
|
128
|
+
make_catalog().write_relation(
|
|
129
|
+
relation(), pl.DataFrame({"id": [1], "CustomerName": ["a"]}), ["id"], {}
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
_, data, partition_by, _ = calls[0]
|
|
133
|
+
assert data.columns == ["id", "CustomerName"]
|
|
134
|
+
assert partition_by == ["id"]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def test_catalog_does_not_import_pyiceberg_until_used():
|
|
138
|
+
code = (
|
|
139
|
+
"import sys\n"
|
|
140
|
+
"from dbt_polars_catalog_snowflake import create_catalog, config_class\n"
|
|
141
|
+
"create_catalog(config_class(name='s', type='snowflake', schema='s', "
|
|
142
|
+
"account='o-a', database_name='d'), '.')\n"
|
|
143
|
+
"assert not [m for m in sys.modules if m.startswith('pyiceberg')]\n"
|
|
144
|
+
)
|
|
145
|
+
subprocess.run([sys.executable, "-c", code], check=True)
|