smallcat-chonk 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smallcat_chonk-0.6.0/.gitignore +218 -0
- smallcat_chonk-0.6.0/LICENSE +7 -0
- smallcat_chonk-0.6.0/PKG-INFO +214 -0
- smallcat_chonk-0.6.0/README.md +190 -0
- smallcat_chonk-0.6.0/pyproject.toml +124 -0
- smallcat_chonk-0.6.0/src/smallcat/__init__.py +112 -0
- smallcat_chonk-0.6.0/src/smallcat/_airflow.py +110 -0
- smallcat_chonk-0.6.0/src/smallcat/catalog.py +642 -0
- smallcat_chonk-0.6.0/src/smallcat/connections.py +524 -0
- smallcat_chonk-0.6.0/src/smallcat/connections_config.py +351 -0
- smallcat_chonk-0.6.0/src/smallcat/datasets/__init__.py +58 -0
- smallcat_chonk-0.6.0/src/smallcat/datasets/base_dataset.py +891 -0
- smallcat_chonk-0.6.0/src/smallcat/datasets/csv_dataset.py +161 -0
- smallcat_chonk-0.6.0/src/smallcat/datasets/delta_table_dataset.py +447 -0
- smallcat_chonk-0.6.0/src/smallcat/datasets/excel_dataset.py +130 -0
- smallcat_chonk-0.6.0/src/smallcat/datasets/parquet_dataset.py +151 -0
- smallcat_chonk-0.6.0/src/smallcat/models/__init__.py +65 -0
- smallcat_chonk-0.6.0/src/smallcat/models/enums.py +72 -0
- smallcat_chonk-0.6.0/src/smallcat/models/errors.py +60 -0
- smallcat_chonk-0.6.0/src/smallcat/models/formats.py +106 -0
- smallcat_chonk-0.6.0/src/smallcat/models/lookups.py +314 -0
- smallcat_chonk-0.6.0/src/smallcat/models/meta.py +280 -0
- smallcat_chonk-0.6.0/src/smallcat/models/model.py +101 -0
- smallcat_chonk-0.6.0/src/smallcat/models/query.py +439 -0
- smallcat_chonk-0.6.0/src/smallcat/models/registry.py +286 -0
- smallcat_chonk-0.6.0/src/smallcat/models/schema.py +318 -0
- smallcat_chonk-0.6.0/src/smallcat/path_utils.py +175 -0
- smallcat_chonk-0.6.0/src/smallcat/py.typed +0 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
# Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
# poetry.lock
|
|
109
|
+
# poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
# pdm.lock
|
|
116
|
+
# pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
# pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# Redis
|
|
135
|
+
*.rdb
|
|
136
|
+
*.aof
|
|
137
|
+
*.pid
|
|
138
|
+
|
|
139
|
+
# RabbitMQ
|
|
140
|
+
mnesia/
|
|
141
|
+
rabbitmq/
|
|
142
|
+
rabbitmq-data/
|
|
143
|
+
|
|
144
|
+
# ActiveMQ
|
|
145
|
+
activemq-data/
|
|
146
|
+
|
|
147
|
+
# SageMath parsed files
|
|
148
|
+
*.sage.py
|
|
149
|
+
|
|
150
|
+
# Environments
|
|
151
|
+
.env
|
|
152
|
+
.envrc
|
|
153
|
+
.venv
|
|
154
|
+
env/
|
|
155
|
+
venv/
|
|
156
|
+
ENV/
|
|
157
|
+
env.bak/
|
|
158
|
+
venv.bak/
|
|
159
|
+
|
|
160
|
+
# Spyder project settings
|
|
161
|
+
.spyderproject
|
|
162
|
+
.spyproject
|
|
163
|
+
|
|
164
|
+
# Rope project settings
|
|
165
|
+
.ropeproject
|
|
166
|
+
|
|
167
|
+
# mkdocs documentation
|
|
168
|
+
/site
|
|
169
|
+
|
|
170
|
+
# mypy
|
|
171
|
+
.mypy_cache/
|
|
172
|
+
.dmypy.json
|
|
173
|
+
dmypy.json
|
|
174
|
+
|
|
175
|
+
# Pyre type checker
|
|
176
|
+
.pyre/
|
|
177
|
+
|
|
178
|
+
# pytype static type analyzer
|
|
179
|
+
.pytype/
|
|
180
|
+
|
|
181
|
+
# Cython debug symbols
|
|
182
|
+
cython_debug/
|
|
183
|
+
|
|
184
|
+
# PyCharm
|
|
185
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
186
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
188
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
|
+
# .idea/
|
|
190
|
+
|
|
191
|
+
# Abstra
|
|
192
|
+
# Abstra is an AI-powered process automation framework.
|
|
193
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
194
|
+
# Learn more at https://abstra.io/docs
|
|
195
|
+
.abstra/
|
|
196
|
+
|
|
197
|
+
# Visual Studio Code
|
|
198
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
199
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
200
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
201
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
202
|
+
# .vscode/
|
|
203
|
+
|
|
204
|
+
# Ruff stuff:
|
|
205
|
+
.ruff_cache/
|
|
206
|
+
|
|
207
|
+
# PyPI configuration file
|
|
208
|
+
.pypirc
|
|
209
|
+
|
|
210
|
+
# Marimo
|
|
211
|
+
marimo/_static/
|
|
212
|
+
marimo/_lsp/
|
|
213
|
+
__marimo__/
|
|
214
|
+
|
|
215
|
+
# Streamlit
|
|
216
|
+
.streamlit/secrets.toml
|
|
217
|
+
|
|
218
|
+
.local
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
Copyright 2025 Deep Kernel Labs S.L.
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: smallcat-chonk
|
|
3
|
+
Version: 0.6.0
|
|
4
|
+
Summary: A small, modular data catalog.
|
|
5
|
+
Project-URL: Homepage, https://github.com/NRGConsulting/smallcat
|
|
6
|
+
Project-URL: Documentation, https://nrgconsulting.github.io/smallcat/
|
|
7
|
+
Project-URL: Repository, https://github.com/NRGConsulting/smallcat
|
|
8
|
+
Project-URL: Issues, https://github.com/NRGConsulting/smallcat/issues
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Requires-Python: !=3.15,>=3.12
|
|
15
|
+
Requires-Dist: deltalake>=1.0
|
|
16
|
+
Requires-Dist: duckdb>=2.0.0
|
|
17
|
+
Requires-Dist: polars>=1.31.0
|
|
18
|
+
Requires-Dist: pyarrow>=16.0.0
|
|
19
|
+
Requires-Dist: pydantic>=2.11.0
|
|
20
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
21
|
+
Provides-Extra: airflow
|
|
22
|
+
Requires-Dist: apache-airflow>=3.3.0; extra == 'airflow'
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
<p align="center">
|
|
26
|
+
<img src="docs/assets/images/smallcat-logo.png" width="140" alt="smallcat logo">
|
|
27
|
+
</p>
|
|
28
|
+
|
|
29
|
+
<h1 align="center">smallcat-chonk</h1>
|
|
30
|
+
<p align="center"><em>A small, modular data catalog.</em></p>
|
|
31
|
+
|
|
32
|
+
<p align="center">
|
|
33
|
+
<a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/v/smallcat-chonk.svg" alt="PyPI"></a>
|
|
34
|
+
<a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/pyversions/smallcat-chonk.svg" alt="Python versions"></a>
|
|
35
|
+
<a href="https://github.com/NRGConsulting/smallcat/actions"><img src="https://img.shields.io/github/actions/workflow/status/NRGConsulting/smallcat/publish.yml?label=CI" alt="CI"></a>
|
|
36
|
+
<a href="https://codecov.io/gh/NRGConsulting/smallcat"><img src="https://img.shields.io/codecov/c/github/NRGConsulting/smallcat" alt="coverage"></a>
|
|
37
|
+
<a href="https://github.com/NRGConsulting/smallcat/blob/main/LICENSE"><img src="https://img.shields.io/github/license/NRGConsulting/smallcat.svg" alt="license"></a>
|
|
38
|
+
<a href="https://pepy.tech/project/smallcat-chonk"><img src="https://static.pepy.tech/badge/smallcat-chonk" alt="downloads"></a>
|
|
39
|
+
<a href="https://nrgconsulting.github.io/smallcat/"><img src="https://img.shields.io/badge/docs-mkdocs%20material-blue" alt="docs"></a>
|
|
40
|
+
</p>
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
```bash
|
|
44
|
+
pip install smallcat-chonk
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
> **Do not install it next to the original `smallcat`.** Both distributions put
|
|
48
|
+
> the same importable package, `smallcat`, in `site-packages`, so they overwrite
|
|
49
|
+
> each other's files, and uninstalling one removes files of the other. Use one
|
|
50
|
+
> or the other in an environment.
|
|
51
|
+
|
|
52
|
+
## Quickstart
|
|
53
|
+
### Define connections
|
|
54
|
+
|
|
55
|
+
Connections live in a file (or in Airflow), never inline in the catalog. Each one
|
|
56
|
+
has a name that catalogs and models refer to; they are validated when loaded.
|
|
57
|
+
|
|
58
|
+
```yaml
|
|
59
|
+
# connections.yaml
|
|
60
|
+
connections:
|
|
61
|
+
local:
|
|
62
|
+
conn_type: fs
|
|
63
|
+
extra:
|
|
64
|
+
base_path: /tmp/smallcat-example/
|
|
65
|
+
lake:
|
|
66
|
+
conn_type: google_cloud_platform
|
|
67
|
+
extra:
|
|
68
|
+
bucket: my-bucket
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### Create Catalog
|
|
72
|
+
|
|
73
|
+
Local catalogs can be kept in YAML files.
|
|
74
|
+
|
|
75
|
+
```yaml
|
|
76
|
+
entries:
|
|
77
|
+
foo:
|
|
78
|
+
file_format: csv
|
|
79
|
+
connection: local
|
|
80
|
+
location: foo.csv
|
|
81
|
+
load_options:
|
|
82
|
+
header: true
|
|
83
|
+
bar:
|
|
84
|
+
file_format: parquet
|
|
85
|
+
connection: lake
|
|
86
|
+
location: bar.parquet
|
|
87
|
+
save_options:
|
|
88
|
+
partition_by:
|
|
89
|
+
- year
|
|
90
|
+
- month
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Standalone
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from smallcat import Connections
|
|
97
|
+
from smallcat.catalog import Catalog
|
|
98
|
+
|
|
99
|
+
conns = Connections.from_yaml("connections.yaml")
|
|
100
|
+
catalog = Catalog.from_yaml("catalog.yaml", connections=conns)
|
|
101
|
+
catalog.save_polars("foo", df)
|
|
102
|
+
df2 = catalog.load_polars("foo")
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
A connection name that is not in `conns` fails when the catalog is loaded, not on
|
|
106
|
+
the first read.
|
|
107
|
+
|
|
108
|
+
### Filter on load
|
|
109
|
+
|
|
110
|
+
`load_polars` (and the lower-level Arrow loaders) accept optional `where` and
|
|
111
|
+
`columns` arguments to push filters and projections down to DuckDB/Arrow when reading:
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
df = catalog.load_polars(
|
|
115
|
+
"bar",
|
|
116
|
+
where="event_date >= '2024-01-01'",
|
|
117
|
+
columns=["event_date", "user_id"],
|
|
118
|
+
)
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Lazy scans
|
|
122
|
+
|
|
123
|
+
`scan_polars` returns a Polars `LazyFrame`: nothing is read until `collect()`, and
|
|
124
|
+
the filters and columns you chain are pushed down to the storage layer. On S3/GCS
|
|
125
|
+
this means only the needed columns and row groups are downloaded.
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
import polars as pl
|
|
129
|
+
|
|
130
|
+
df = (
|
|
131
|
+
catalog.scan_polars("bar")
|
|
132
|
+
.filter(pl.col("event_date") >= pl.date(2024, 1, 1)) # Polars expressions, not SQL
|
|
133
|
+
.select("event_date", "user_id")
|
|
134
|
+
.collect()
|
|
135
|
+
)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
| | `load_polars` | `scan_polars` |
|
|
139
|
+
|---|---|---|
|
|
140
|
+
| Engine | DuckDB | Polars' own reader (Parquet) / delta-rs + Polars (Delta) |
|
|
141
|
+
| Filter | `where="amount > 10"` (SQL string) | `.filter(pl.col("amount") > 10)` |
|
|
142
|
+
| Reads | Everything the query needs, immediately | Nothing until `collect()` |
|
|
143
|
+
| Real lazy scan | n/a | Parquet and Delta. CSV and Excel are loaded eagerly and wrapped, so nothing is skipped |
|
|
144
|
+
|
|
145
|
+
Notes:
|
|
146
|
+
|
|
147
|
+
* Parquet and Delta use the connection's credentials through the same options
|
|
148
|
+
(`aws_*`, `google_service_account*`). For Parquet, the DuckDB-only load options
|
|
149
|
+
`binary_as_string` and `file_row_number` raise a `ValueError` in `scan_polars`;
|
|
150
|
+
use `load_polars` for those.
|
|
151
|
+
* Existing limits apply: IAM role, SSO and `AssumeRole` credentials are unverified
|
|
152
|
+
against real AWS, and GCS needs a service account for Delta/`scan_polars` but HMAC
|
|
153
|
+
keys for DuckDB.
|
|
154
|
+
* Requires `polars>=1.31.0`.
|
|
155
|
+
|
|
156
|
+
### With Airflow
|
|
157
|
+
```python
|
|
158
|
+
from smallcat import Connections
|
|
159
|
+
from smallcat.catalog import Catalog
|
|
160
|
+
|
|
161
|
+
conns = Connections.from_airflow(["lake"]) # resolved and validated now
|
|
162
|
+
catalog = Catalog.from_airflow_variable("example_catalog", connections=conns)
|
|
163
|
+
df = catalog.load_polars("bar")
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Models
|
|
167
|
+
|
|
168
|
+
A model describes a dataset: its columns, where it lives and how it is stored.
|
|
169
|
+
`Model.objects` reads and writes it as Polars DataFrames, validated against the
|
|
170
|
+
model. The model never names a connection: the registry does, per environment.
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
import polars as pl
|
|
174
|
+
from smallcat import Connections, Model, ModelFileFormat, ModelWriteMode, registry
|
|
175
|
+
|
|
176
|
+
class Customer(Model):
|
|
177
|
+
id: int
|
|
178
|
+
name: str
|
|
179
|
+
email: str | None = None
|
|
180
|
+
|
|
181
|
+
class Meta:
|
|
182
|
+
location = "silver/customers"
|
|
183
|
+
file_format = ModelFileFormat.PARQUET
|
|
184
|
+
|
|
185
|
+
registry.load_connections(Connections.from_yaml("connections.yaml"))
|
|
186
|
+
registry.register(Customer, connection="lake")
|
|
187
|
+
|
|
188
|
+
Customer.objects.filter(pl.col("id") > 10).select("id", "name").all() # DataFrame
|
|
189
|
+
Customer.objects.filter(pl.col("id") > 10).lazy() # LazyFrame
|
|
190
|
+
Customer.objects.save(df, mode=ModelWriteMode.OVERWRITE) # mode is required
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
* `mode` is required: `OVERWRITE` replaces the whole dataset, `APPEND` adds rows
|
|
194
|
+
and `MERGE` inserts the rows with a new key and updates the ones whose key
|
|
195
|
+
exists (both Delta only; `MERGE` never deletes). Declare the key in
|
|
196
|
+
`Meta.merge_keys = ["id"]` or pass it with
|
|
197
|
+
`save(df, mode="merge", merge_keys=["id"])`.
|
|
198
|
+
* `allowed_write_modes` in `Meta` limits what a model may do (`[]` is read-only);
|
|
199
|
+
registering can restrict it further per environment, never widen it.
|
|
200
|
+
* `validation_mode` is `STRICT` (types must match) or `COERCE` (cast, failing only on
|
|
201
|
+
values that cannot be cast without loss).
|
|
202
|
+
|
|
203
|
+
See [Models](https://nrgconsulting.github.io/smallcat/models/) and a runnable
|
|
204
|
+
[example](https://nrgconsulting.github.io/smallcat/examples/orm/models/) and a walk-through of
|
|
205
|
+
[querying with `filter`, `where`, `all` and `lazy`](https://nrgconsulting.github.io/smallcat/querying-models/).
|
|
206
|
+
|
|
207
|
+
## Catalog reference
|
|
208
|
+
|
|
209
|
+
`load_options` and `save_options` are optional, `file_format` is required and unknown
|
|
210
|
+
keys are rejected. See the [catalog reference](https://nrgconsulting.github.io/smallcat/catalog-reference/)
|
|
211
|
+
for every option, the validation rules and the credentials each format uses.
|
|
212
|
+
|
|
213
|
+
## Docs
|
|
214
|
+
Read more at [the official docs](https://nrgconsulting.github.io/smallcat/).
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/assets/images/smallcat-logo.png" width="140" alt="smallcat logo">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">smallcat-chonk</h1>
|
|
6
|
+
<p align="center"><em>A small, modular data catalog.</em></p>
|
|
7
|
+
|
|
8
|
+
<p align="center">
|
|
9
|
+
<a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/v/smallcat-chonk.svg" alt="PyPI"></a>
|
|
10
|
+
<a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/pyversions/smallcat-chonk.svg" alt="Python versions"></a>
|
|
11
|
+
<a href="https://github.com/NRGConsulting/smallcat/actions"><img src="https://img.shields.io/github/actions/workflow/status/NRGConsulting/smallcat/publish.yml?label=CI" alt="CI"></a>
|
|
12
|
+
<a href="https://codecov.io/gh/NRGConsulting/smallcat"><img src="https://img.shields.io/codecov/c/github/NRGConsulting/smallcat" alt="coverage"></a>
|
|
13
|
+
<a href="https://github.com/NRGConsulting/smallcat/blob/main/LICENSE"><img src="https://img.shields.io/github/license/NRGConsulting/smallcat.svg" alt="license"></a>
|
|
14
|
+
<a href="https://pepy.tech/project/smallcat-chonk"><img src="https://static.pepy.tech/badge/smallcat-chonk" alt="downloads"></a>
|
|
15
|
+
<a href="https://nrgconsulting.github.io/smallcat/"><img src="https://img.shields.io/badge/docs-mkdocs%20material-blue" alt="docs"></a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
```bash
|
|
20
|
+
pip install smallcat-chonk
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
> **Do not install it next to the original `smallcat`.** Both distributions put
|
|
24
|
+
> the same importable package, `smallcat`, in `site-packages`, so they overwrite
|
|
25
|
+
> each other's files, and uninstalling one removes files of the other. Use one
|
|
26
|
+
> or the other in an environment.
|
|
27
|
+
|
|
28
|
+
## Quickstart
|
|
29
|
+
### Define connections
|
|
30
|
+
|
|
31
|
+
Connections live in a file (or in Airflow), never inline in the catalog. Each one
|
|
32
|
+
has a name that catalogs and models refer to; they are validated when loaded.
|
|
33
|
+
|
|
34
|
+
```yaml
|
|
35
|
+
# connections.yaml
|
|
36
|
+
connections:
|
|
37
|
+
local:
|
|
38
|
+
conn_type: fs
|
|
39
|
+
extra:
|
|
40
|
+
base_path: /tmp/smallcat-example/
|
|
41
|
+
lake:
|
|
42
|
+
conn_type: google_cloud_platform
|
|
43
|
+
extra:
|
|
44
|
+
bucket: my-bucket
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### Create Catalog
|
|
48
|
+
|
|
49
|
+
Local catalogs can be kept in YAML files.
|
|
50
|
+
|
|
51
|
+
```yaml
|
|
52
|
+
entries:
|
|
53
|
+
foo:
|
|
54
|
+
file_format: csv
|
|
55
|
+
connection: local
|
|
56
|
+
location: foo.csv
|
|
57
|
+
load_options:
|
|
58
|
+
header: true
|
|
59
|
+
bar:
|
|
60
|
+
file_format: parquet
|
|
61
|
+
connection: lake
|
|
62
|
+
location: bar.parquet
|
|
63
|
+
save_options:
|
|
64
|
+
partition_by:
|
|
65
|
+
- year
|
|
66
|
+
- month
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### Standalone
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from smallcat import Connections
|
|
73
|
+
from smallcat.catalog import Catalog
|
|
74
|
+
|
|
75
|
+
conns = Connections.from_yaml("connections.yaml")
|
|
76
|
+
catalog = Catalog.from_yaml("catalog.yaml", connections=conns)
|
|
77
|
+
catalog.save_polars("foo", df)
|
|
78
|
+
df2 = catalog.load_polars("foo")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
A connection name that is not in `conns` fails when the catalog is loaded, not on
|
|
82
|
+
the first read.
|
|
83
|
+
|
|
84
|
+
### Filter on load
|
|
85
|
+
|
|
86
|
+
`load_polars` (and the lower-level Arrow loaders) accept optional `where` and
|
|
87
|
+
`columns` arguments to push filters and projections down to DuckDB/Arrow when reading:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
df = catalog.load_polars(
|
|
91
|
+
"bar",
|
|
92
|
+
where="event_date >= '2024-01-01'",
|
|
93
|
+
columns=["event_date", "user_id"],
|
|
94
|
+
)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Lazy scans
|
|
98
|
+
|
|
99
|
+
`scan_polars` returns a Polars `LazyFrame`: nothing is read until `collect()`, and
|
|
100
|
+
the filters and columns you chain are pushed down to the storage layer. On S3/GCS
|
|
101
|
+
this means only the needed columns and row groups are downloaded.
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
import polars as pl
|
|
105
|
+
|
|
106
|
+
df = (
|
|
107
|
+
catalog.scan_polars("bar")
|
|
108
|
+
.filter(pl.col("event_date") >= pl.date(2024, 1, 1)) # Polars expressions, not SQL
|
|
109
|
+
.select("event_date", "user_id")
|
|
110
|
+
.collect()
|
|
111
|
+
)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
| | `load_polars` | `scan_polars` |
|
|
115
|
+
|---|---|---|
|
|
116
|
+
| Engine | DuckDB | Polars' own reader (Parquet) / delta-rs + Polars (Delta) |
|
|
117
|
+
| Filter | `where="amount > 10"` (SQL string) | `.filter(pl.col("amount") > 10)` |
|
|
118
|
+
| Reads | Everything the query needs, immediately | Nothing until `collect()` |
|
|
119
|
+
| Real lazy scan | n/a | Parquet and Delta. CSV and Excel are loaded eagerly and wrapped, so nothing is skipped |
|
|
120
|
+
|
|
121
|
+
Notes:
|
|
122
|
+
|
|
123
|
+
* Parquet and Delta use the connection's credentials through the same options
|
|
124
|
+
(`aws_*`, `google_service_account*`). For Parquet, the DuckDB-only load options
|
|
125
|
+
`binary_as_string` and `file_row_number` raise a `ValueError` in `scan_polars`;
|
|
126
|
+
use `load_polars` for those.
|
|
127
|
+
* Existing limits apply: IAM role, SSO and `AssumeRole` credentials are unverified
|
|
128
|
+
against real AWS, and GCS needs a service account for Delta/`scan_polars` but HMAC
|
|
129
|
+
keys for DuckDB.
|
|
130
|
+
* Requires `polars>=1.31.0`.
|
|
131
|
+
|
|
132
|
+
### With Airflow
|
|
133
|
+
```python
|
|
134
|
+
from smallcat import Connections
|
|
135
|
+
from smallcat.catalog import Catalog
|
|
136
|
+
|
|
137
|
+
conns = Connections.from_airflow(["lake"]) # resolved and validated now
|
|
138
|
+
catalog = Catalog.from_airflow_variable("example_catalog", connections=conns)
|
|
139
|
+
df = catalog.load_polars("bar")
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Models
|
|
143
|
+
|
|
144
|
+
A model describes a dataset: its columns, where it lives and how it is stored.
|
|
145
|
+
`Model.objects` reads and writes it as Polars DataFrames, validated against the
|
|
146
|
+
model. The model never names a connection: the registry does, per environment.
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
import polars as pl
|
|
150
|
+
from smallcat import Connections, Model, ModelFileFormat, ModelWriteMode, registry
|
|
151
|
+
|
|
152
|
+
class Customer(Model):
|
|
153
|
+
id: int
|
|
154
|
+
name: str
|
|
155
|
+
email: str | None = None
|
|
156
|
+
|
|
157
|
+
class Meta:
|
|
158
|
+
location = "silver/customers"
|
|
159
|
+
file_format = ModelFileFormat.PARQUET
|
|
160
|
+
|
|
161
|
+
registry.load_connections(Connections.from_yaml("connections.yaml"))
|
|
162
|
+
registry.register(Customer, connection="lake")
|
|
163
|
+
|
|
164
|
+
Customer.objects.filter(pl.col("id") > 10).select("id", "name").all() # DataFrame
|
|
165
|
+
Customer.objects.filter(pl.col("id") > 10).lazy() # LazyFrame
|
|
166
|
+
Customer.objects.save(df, mode=ModelWriteMode.OVERWRITE) # mode is required
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
* `mode` is required: `OVERWRITE` replaces the whole dataset, `APPEND` adds rows
|
|
170
|
+
and `MERGE` inserts the rows with a new key and updates the ones whose key
|
|
171
|
+
exists (both Delta only; `MERGE` never deletes). Declare the key in
|
|
172
|
+
`Meta.merge_keys = ["id"]` or pass it with
|
|
173
|
+
`save(df, mode="merge", merge_keys=["id"])`.
|
|
174
|
+
* `allowed_write_modes` in `Meta` limits what a model may do (`[]` is read-only);
|
|
175
|
+
registering can restrict it further per environment, never widen it.
|
|
176
|
+
* `validation_mode` is `STRICT` (types must match) or `COERCE` (cast, failing only on
|
|
177
|
+
values that cannot be cast without loss).
|
|
178
|
+
|
|
179
|
+
See [Models](https://nrgconsulting.github.io/smallcat/models/) and a runnable
|
|
180
|
+
[example](https://nrgconsulting.github.io/smallcat/examples/orm/models/) and a walk-through of
|
|
181
|
+
[querying with `filter`, `where`, `all` and `lazy`](https://nrgconsulting.github.io/smallcat/querying-models/).
|
|
182
|
+
|
|
183
|
+
## Catalog reference
|
|
184
|
+
|
|
185
|
+
`load_options` and `save_options` are optional, `file_format` is required and unknown
|
|
186
|
+
keys are rejected. See the [catalog reference](https://nrgconsulting.github.io/smallcat/catalog-reference/)
|
|
187
|
+
for every option, the validation rules and the credentials each format uses.
|
|
188
|
+
|
|
189
|
+
## Docs
|
|
190
|
+
Read more at [the official docs](https://nrgconsulting.github.io/smallcat/).
|