smallcat-chonk 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. smallcat_chonk-0.6.0/.gitignore +218 -0
  2. smallcat_chonk-0.6.0/LICENSE +7 -0
  3. smallcat_chonk-0.6.0/PKG-INFO +214 -0
  4. smallcat_chonk-0.6.0/README.md +190 -0
  5. smallcat_chonk-0.6.0/pyproject.toml +124 -0
  6. smallcat_chonk-0.6.0/src/smallcat/__init__.py +112 -0
  7. smallcat_chonk-0.6.0/src/smallcat/_airflow.py +110 -0
  8. smallcat_chonk-0.6.0/src/smallcat/catalog.py +642 -0
  9. smallcat_chonk-0.6.0/src/smallcat/connections.py +524 -0
  10. smallcat_chonk-0.6.0/src/smallcat/connections_config.py +351 -0
  11. smallcat_chonk-0.6.0/src/smallcat/datasets/__init__.py +58 -0
  12. smallcat_chonk-0.6.0/src/smallcat/datasets/base_dataset.py +891 -0
  13. smallcat_chonk-0.6.0/src/smallcat/datasets/csv_dataset.py +161 -0
  14. smallcat_chonk-0.6.0/src/smallcat/datasets/delta_table_dataset.py +447 -0
  15. smallcat_chonk-0.6.0/src/smallcat/datasets/excel_dataset.py +130 -0
  16. smallcat_chonk-0.6.0/src/smallcat/datasets/parquet_dataset.py +151 -0
  17. smallcat_chonk-0.6.0/src/smallcat/models/__init__.py +65 -0
  18. smallcat_chonk-0.6.0/src/smallcat/models/enums.py +72 -0
  19. smallcat_chonk-0.6.0/src/smallcat/models/errors.py +60 -0
  20. smallcat_chonk-0.6.0/src/smallcat/models/formats.py +106 -0
  21. smallcat_chonk-0.6.0/src/smallcat/models/lookups.py +314 -0
  22. smallcat_chonk-0.6.0/src/smallcat/models/meta.py +280 -0
  23. smallcat_chonk-0.6.0/src/smallcat/models/model.py +101 -0
  24. smallcat_chonk-0.6.0/src/smallcat/models/query.py +439 -0
  25. smallcat_chonk-0.6.0/src/smallcat/models/registry.py +286 -0
  26. smallcat_chonk-0.6.0/src/smallcat/models/schema.py +318 -0
  27. smallcat_chonk-0.6.0/src/smallcat/path_utils.py +175 -0
  28. smallcat_chonk-0.6.0/src/smallcat/py.typed +0 -0
@@ -0,0 +1,218 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ # Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ # poetry.lock
109
+ # poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ # pdm.lock
116
+ # pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ # pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # Redis
135
+ *.rdb
136
+ *.aof
137
+ *.pid
138
+
139
+ # RabbitMQ
140
+ mnesia/
141
+ rabbitmq/
142
+ rabbitmq-data/
143
+
144
+ # ActiveMQ
145
+ activemq-data/
146
+
147
+ # SageMath parsed files
148
+ *.sage.py
149
+
150
+ # Environments
151
+ .env
152
+ .envrc
153
+ .venv
154
+ env/
155
+ venv/
156
+ ENV/
157
+ env.bak/
158
+ venv.bak/
159
+
160
+ # Spyder project settings
161
+ .spyderproject
162
+ .spyproject
163
+
164
+ # Rope project settings
165
+ .ropeproject
166
+
167
+ # mkdocs documentation
168
+ /site
169
+
170
+ # mypy
171
+ .mypy_cache/
172
+ .dmypy.json
173
+ dmypy.json
174
+
175
+ # Pyre type checker
176
+ .pyre/
177
+
178
+ # pytype static type analyzer
179
+ .pytype/
180
+
181
+ # Cython debug symbols
182
+ cython_debug/
183
+
184
+ # PyCharm
185
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
186
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
187
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
188
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
189
+ # .idea/
190
+
191
+ # Abstra
192
+ # Abstra is an AI-powered process automation framework.
193
+ # Ignore directories containing user credentials, local state, and settings.
194
+ # Learn more at https://abstra.io/docs
195
+ .abstra/
196
+
197
+ # Visual Studio Code
198
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
199
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
200
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
201
+ # you could uncomment the following to ignore the entire vscode folder
202
+ # .vscode/
203
+
204
+ # Ruff stuff:
205
+ .ruff_cache/
206
+
207
+ # PyPI configuration file
208
+ .pypirc
209
+
210
+ # Marimo
211
+ marimo/_static/
212
+ marimo/_lsp/
213
+ __marimo__/
214
+
215
+ # Streamlit
216
+ .streamlit/secrets.toml
217
+
218
+ .local
@@ -0,0 +1,7 @@
1
+ Copyright 2025 Deep Kernel Labs S.L.
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4
+
5
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6
+
7
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,214 @@
1
+ Metadata-Version: 2.5
2
+ Name: smallcat-chonk
3
+ Version: 0.6.0
4
+ Summary: A small, modular data catalog.
5
+ Project-URL: Homepage, https://github.com/NRGConsulting/smallcat
6
+ Project-URL: Documentation, https://nrgconsulting.github.io/smallcat/
7
+ Project-URL: Repository, https://github.com/NRGConsulting/smallcat
8
+ Project-URL: Issues, https://github.com/NRGConsulting/smallcat/issues
9
+ License-File: LICENSE
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3 :: Only
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Requires-Python: !=3.15,>=3.12
15
+ Requires-Dist: deltalake>=1.0
16
+ Requires-Dist: duckdb>=2.0.0
17
+ Requires-Dist: polars>=1.31.0
18
+ Requires-Dist: pyarrow>=16.0.0
19
+ Requires-Dist: pydantic>=2.11.0
20
+ Requires-Dist: pyyaml>=6.0.3
21
+ Provides-Extra: airflow
22
+ Requires-Dist: apache-airflow>=3.3.0; extra == 'airflow'
23
+ Description-Content-Type: text/markdown
24
+
25
+ <p align="center">
26
+ <img src="docs/assets/images/smallcat-logo.png" width="140" alt="smallcat logo">
27
+ </p>
28
+
29
+ <h1 align="center">smallcat-chonk</h1>
30
+ <p align="center"><em>A small, modular data catalog.</em></p>
31
+
32
+ <p align="center">
33
+ <a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/v/smallcat-chonk.svg" alt="PyPI"></a>
34
+ <a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/pyversions/smallcat-chonk.svg" alt="Python versions"></a>
35
+ <a href="https://github.com/NRGConsulting/smallcat/actions"><img src="https://img.shields.io/github/actions/workflow/status/NRGConsulting/smallcat/publish.yml?label=CI" alt="CI"></a>
36
+ <a href="https://codecov.io/gh/NRGConsulting/smallcat"><img src="https://img.shields.io/codecov/c/github/NRGConsulting/smallcat" alt="coverage"></a>
37
+ <a href="https://github.com/NRGConsulting/smallcat/blob/main/LICENSE"><img src="https://img.shields.io/github/license/NRGConsulting/smallcat.svg" alt="license"></a>
38
+ <a href="https://pepy.tech/project/smallcat-chonk"><img src="https://static.pepy.tech/badge/smallcat-chonk" alt="downloads"></a>
39
+ <a href="https://nrgconsulting.github.io/smallcat/"><img src="https://img.shields.io/badge/docs-mkdocs%20material-blue" alt="docs"></a>
40
+ </p>
41
+
42
+ ## Install
43
+ ```bash
44
+ pip install smallcat-chonk
45
+ ```
46
+
47
+ > **Do not install it next to the original `smallcat`.** Both distributions put
48
+ > the same importable package, `smallcat`, in `site-packages`, so they overwrite
49
+ > each other's files, and uninstalling one removes files of the other. Use one
50
+ > or the other in an environment.
51
+
52
+ ## Quickstart
53
+ ### Define connections
54
+
55
+ Connections live in a file (or in Airflow), never inline in the catalog. Each one
56
+ has a name that catalogs and models refer to; they are validated when loaded.
57
+
58
+ ```yaml
59
+ # connections.yaml
60
+ connections:
61
+ local:
62
+ conn_type: fs
63
+ extra:
64
+ base_path: /tmp/smallcat-example/
65
+ lake:
66
+ conn_type: google_cloud_platform
67
+ extra:
68
+ bucket: my-bucket
69
+ ```
70
+
71
+ ### Create Catalog
72
+
73
+ Local catalogs can be kept in YAML files.
74
+
75
+ ```yaml
76
+ entries:
77
+ foo:
78
+ file_format: csv
79
+ connection: local
80
+ location: foo.csv
81
+ load_options:
82
+ header: true
83
+ bar:
84
+ file_format: parquet
85
+ connection: lake
86
+ location: bar.parquet
87
+ save_options:
88
+ partition_by:
89
+ - year
90
+ - month
91
+ ```
92
+
93
+ ### Standalone
94
+
95
+ ```python
96
+ from smallcat import Connections
97
+ from smallcat.catalog import Catalog
98
+
99
+ conns = Connections.from_yaml("connections.yaml")
100
+ catalog = Catalog.from_yaml("catalog.yaml", connections=conns)
101
+ catalog.save_polars("foo", df)
102
+ df2 = catalog.load_polars("foo")
103
+ ```
104
+
105
+ A connection name that is not in `conns` fails when the catalog is loaded, not on
106
+ the first read.
107
+
108
+ ### Filter on load
109
+
110
+ `load_polars` (and the lower-level Arrow loaders) accept optional `where` and
111
+ `columns` arguments to push filters and projections down to DuckDB/Arrow when reading:
112
+
113
+ ```python
114
+ df = catalog.load_polars(
115
+ "bar",
116
+ where="event_date >= '2024-01-01'",
117
+ columns=["event_date", "user_id"],
118
+ )
119
+ ```
120
+
121
+ ### Lazy scans
122
+
123
+ `scan_polars` returns a Polars `LazyFrame`: nothing is read until `collect()`, and
124
+ the filters and columns you chain are pushed down to the storage layer. On S3/GCS
125
+ this means only the needed columns and row groups are downloaded.
126
+
127
+ ```python
128
+ import polars as pl
129
+
130
+ df = (
131
+ catalog.scan_polars("bar")
132
+ .filter(pl.col("event_date") >= pl.date(2024, 1, 1)) # Polars expressions, not SQL
133
+ .select("event_date", "user_id")
134
+ .collect()
135
+ )
136
+ ```
137
+
138
+ | | `load_polars` | `scan_polars` |
139
+ |---|---|---|
140
+ | Engine | DuckDB | Polars' own reader (Parquet) / delta-rs + Polars (Delta) |
141
+ | Filter | `where="amount > 10"` (SQL string) | `.filter(pl.col("amount") > 10)` |
142
+ | Reads | Everything the query needs, immediately | Nothing until `collect()` |
143
+ | Real lazy scan | n/a | Parquet and Delta. CSV and Excel are loaded eagerly and wrapped, so nothing is skipped |
144
+
145
+ Notes:
146
+
147
+ * Parquet and Delta use the connection's credentials through the same options
148
+ (`aws_*`, `google_service_account*`). For Parquet, the DuckDB-only load options
149
+ `binary_as_string` and `file_row_number` raise a `ValueError` in `scan_polars`;
150
+ use `load_polars` for those.
151
+ * Existing limits apply: IAM role, SSO and `AssumeRole` credentials are unverified
152
+ against real AWS, and GCS needs a service account for Delta/`scan_polars` but HMAC
153
+ keys for DuckDB.
154
+ * Requires `polars>=1.31.0`.
155
+
156
+ ### With Airflow
157
+ ```python
158
+ from smallcat import Connections
159
+ from smallcat.catalog import Catalog
160
+
161
+ conns = Connections.from_airflow(["lake"]) # resolved and validated now
162
+ catalog = Catalog.from_airflow_variable("example_catalog", connections=conns)
163
+ df = catalog.load_polars("bar")
164
+ ```
165
+
166
+ ## Models
167
+
168
+ A model describes a dataset: its columns, where it lives and how it is stored.
169
+ `Model.objects` reads and writes it as Polars DataFrames, validated against the
170
+ model. The model never names a connection: the registry does, per environment.
171
+
172
+ ```python
173
+ import polars as pl
174
+ from smallcat import Connections, Model, ModelFileFormat, ModelWriteMode, registry
175
+
176
+ class Customer(Model):
177
+ id: int
178
+ name: str
179
+ email: str | None = None
180
+
181
+ class Meta:
182
+ location = "silver/customers"
183
+ file_format = ModelFileFormat.PARQUET
184
+
185
+ registry.load_connections(Connections.from_yaml("connections.yaml"))
186
+ registry.register(Customer, connection="lake")
187
+
188
+ Customer.objects.filter(pl.col("id") > 10).select("id", "name").all() # DataFrame
189
+ Customer.objects.filter(pl.col("id") > 10).lazy() # LazyFrame
190
+ Customer.objects.save(df, mode=ModelWriteMode.OVERWRITE) # mode is required
191
+ ```
192
+
193
+ * `mode` is required: `OVERWRITE` replaces the whole dataset, `APPEND` adds rows
194
+ and `MERGE` inserts the rows with a new key and updates the ones whose key
195
+ exists (both Delta only; `MERGE` never deletes). Declare the key in
196
+ `Meta.merge_keys = ["id"]` or pass it with
197
+ `save(df, mode="merge", merge_keys=["id"])`.
198
+ * `allowed_write_modes` in `Meta` limits what a model may do (`[]` is read-only);
199
+ registering can restrict it further per environment, never widen it.
200
+ * `validation_mode` is `STRICT` (types must match) or `COERCE` (cast, failing only on
201
+ values that cannot be cast without loss).
202
+
203
+ See [Models](https://nrgconsulting.github.io/smallcat/models/) and a runnable
204
+ [example](https://nrgconsulting.github.io/smallcat/examples/orm/models/) and a walk-through of
205
+ [querying with `filter`, `where`, `all` and `lazy`](https://nrgconsulting.github.io/smallcat/querying-models/).
206
+
207
+ ## Catalog reference
208
+
209
+ `load_options` and `save_options` are optional, `file_format` is required and unknown
210
+ keys are rejected. See the [catalog reference](https://nrgconsulting.github.io/smallcat/catalog-reference/)
211
+ for every option, the validation rules and the credentials each format uses.
212
+
213
+ ## Docs
214
+ Read more at [the official docs](https://nrgconsulting.github.io/smallcat/).
@@ -0,0 +1,190 @@
1
+ <p align="center">
2
+ <img src="docs/assets/images/smallcat-logo.png" width="140" alt="smallcat logo">
3
+ </p>
4
+
5
+ <h1 align="center">smallcat-chonk</h1>
6
+ <p align="center"><em>A small, modular data catalog.</em></p>
7
+
8
+ <p align="center">
9
+ <a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/v/smallcat-chonk.svg" alt="PyPI"></a>
10
+ <a href="https://pypi.org/project/smallcat-chonk/"><img src="https://img.shields.io/pypi/pyversions/smallcat-chonk.svg" alt="Python versions"></a>
11
+ <a href="https://github.com/NRGConsulting/smallcat/actions"><img src="https://img.shields.io/github/actions/workflow/status/NRGConsulting/smallcat/publish.yml?label=CI" alt="CI"></a>
12
+ <a href="https://codecov.io/gh/NRGConsulting/smallcat"><img src="https://img.shields.io/codecov/c/github/NRGConsulting/smallcat" alt="coverage"></a>
13
+ <a href="https://github.com/NRGConsulting/smallcat/blob/main/LICENSE"><img src="https://img.shields.io/github/license/NRGConsulting/smallcat.svg" alt="license"></a>
14
+ <a href="https://pepy.tech/project/smallcat-chonk"><img src="https://static.pepy.tech/badge/smallcat-chonk" alt="downloads"></a>
15
+ <a href="https://nrgconsulting.github.io/smallcat/"><img src="https://img.shields.io/badge/docs-mkdocs%20material-blue" alt="docs"></a>
16
+ </p>
17
+
18
+ ## Install
19
+ ```bash
20
+ pip install smallcat-chonk
21
+ ```
22
+
23
+ > **Do not install it next to the original `smallcat`.** Both distributions put
24
+ > the same importable package, `smallcat`, in `site-packages`, so they overwrite
25
+ > each other's files, and uninstalling one removes files of the other. Use one
26
+ > or the other in an environment.
27
+
28
+ ## Quickstart
29
+ ### Define connections
30
+
31
+ Connections live in a file (or in Airflow), never inline in the catalog. Each one
32
+ has a name that catalogs and models refer to; they are validated when loaded.
33
+
34
+ ```yaml
35
+ # connections.yaml
36
+ connections:
37
+ local:
38
+ conn_type: fs
39
+ extra:
40
+ base_path: /tmp/smallcat-example/
41
+ lake:
42
+ conn_type: google_cloud_platform
43
+ extra:
44
+ bucket: my-bucket
45
+ ```
46
+
47
+ ### Create Catalog
48
+
49
+ Local catalogs can be kept in YAML files.
50
+
51
+ ```yaml
52
+ entries:
53
+ foo:
54
+ file_format: csv
55
+ connection: local
56
+ location: foo.csv
57
+ load_options:
58
+ header: true
59
+ bar:
60
+ file_format: parquet
61
+ connection: lake
62
+ location: bar.parquet
63
+ save_options:
64
+ partition_by:
65
+ - year
66
+ - month
67
+ ```
68
+
69
+ ### Standalone
70
+
71
+ ```python
72
+ from smallcat import Connections
73
+ from smallcat.catalog import Catalog
74
+
75
+ conns = Connections.from_yaml("connections.yaml")
76
+ catalog = Catalog.from_yaml("catalog.yaml", connections=conns)
77
+ catalog.save_polars("foo", df)
78
+ df2 = catalog.load_polars("foo")
79
+ ```
80
+
81
+ A connection name that is not in `conns` fails when the catalog is loaded, not on
82
+ the first read.
83
+
84
+ ### Filter on load
85
+
86
+ `load_polars` (and the lower-level Arrow loaders) accept optional `where` and
87
+ `columns` arguments to push filters and projections down to DuckDB/Arrow when reading:
88
+
89
+ ```python
90
+ df = catalog.load_polars(
91
+ "bar",
92
+ where="event_date >= '2024-01-01'",
93
+ columns=["event_date", "user_id"],
94
+ )
95
+ ```
96
+
97
+ ### Lazy scans
98
+
99
+ `scan_polars` returns a Polars `LazyFrame`: nothing is read until `collect()`, and
100
+ the filters and columns you chain are pushed down to the storage layer. On S3/GCS
101
+ this means only the needed columns and row groups are downloaded.
102
+
103
+ ```python
104
+ import polars as pl
105
+
106
+ df = (
107
+ catalog.scan_polars("bar")
108
+ .filter(pl.col("event_date") >= pl.date(2024, 1, 1)) # Polars expressions, not SQL
109
+ .select("event_date", "user_id")
110
+ .collect()
111
+ )
112
+ ```
113
+
114
+ | | `load_polars` | `scan_polars` |
115
+ |---|---|---|
116
+ | Engine | DuckDB | Polars' own reader (Parquet) / delta-rs + Polars (Delta) |
117
+ | Filter | `where="amount > 10"` (SQL string) | `.filter(pl.col("amount") > 10)` |
118
+ | Reads | Everything the query needs, immediately | Nothing until `collect()` |
119
+ | Real lazy scan | n/a | Parquet and Delta. CSV and Excel are loaded eagerly and wrapped, so nothing is skipped |
120
+
121
+ Notes:
122
+
123
+ * Parquet and Delta use the connection's credentials through the same options
124
+ (`aws_*`, `google_service_account*`). For Parquet, the DuckDB-only load options
125
+ `binary_as_string` and `file_row_number` raise a `ValueError` in `scan_polars`;
126
+ use `load_polars` for those.
127
+ * Existing limits apply: IAM role, SSO and `AssumeRole` credentials are unverified
128
+ against real AWS, and GCS needs a service account for Delta/`scan_polars` but HMAC
129
+ keys for DuckDB.
130
+ * Requires `polars>=1.31.0`.
131
+
132
+ ### With Airflow
133
+ ```python
134
+ from smallcat import Connections
135
+ from smallcat.catalog import Catalog
136
+
137
+ conns = Connections.from_airflow(["lake"]) # resolved and validated now
138
+ catalog = Catalog.from_airflow_variable("example_catalog", connections=conns)
139
+ df = catalog.load_polars("bar")
140
+ ```
141
+
142
+ ## Models
143
+
144
+ A model describes a dataset: its columns, where it lives and how it is stored.
145
+ `Model.objects` reads and writes it as Polars DataFrames, validated against the
146
+ model. The model never names a connection: the registry does, per environment.
147
+
148
+ ```python
149
+ import polars as pl
150
+ from smallcat import Connections, Model, ModelFileFormat, ModelWriteMode, registry
151
+
152
+ class Customer(Model):
153
+ id: int
154
+ name: str
155
+ email: str | None = None
156
+
157
+ class Meta:
158
+ location = "silver/customers"
159
+ file_format = ModelFileFormat.PARQUET
160
+
161
+ registry.load_connections(Connections.from_yaml("connections.yaml"))
162
+ registry.register(Customer, connection="lake")
163
+
164
+ Customer.objects.filter(pl.col("id") > 10).select("id", "name").all() # DataFrame
165
+ Customer.objects.filter(pl.col("id") > 10).lazy() # LazyFrame
166
+ Customer.objects.save(df, mode=ModelWriteMode.OVERWRITE) # mode is required
167
+ ```
168
+
169
+ * `mode` is required: `OVERWRITE` replaces the whole dataset, `APPEND` adds rows
170
+ and `MERGE` inserts the rows with a new key and updates the ones whose key
171
+ exists (both Delta only; `MERGE` never deletes). Declare the key in
172
+ `Meta.merge_keys = ["id"]` or pass it with
173
+ `save(df, mode="merge", merge_keys=["id"])`.
174
+ * `allowed_write_modes` in `Meta` limits what a model may do (`[]` is read-only);
175
+ registering can restrict it further per environment, never widen it.
176
+ * `validation_mode` is `STRICT` (types must match) or `COERCE` (cast, failing only on
177
+ values that cannot be cast without loss).
178
+
179
+ See [Models](https://nrgconsulting.github.io/smallcat/models/) and a runnable
180
+ [example](https://nrgconsulting.github.io/smallcat/examples/orm/models/) and a walk-through of
181
+ [querying with `filter`, `where`, `all` and `lazy`](https://nrgconsulting.github.io/smallcat/querying-models/).
182
+
183
+ ## Catalog reference
184
+
185
+ `load_options` and `save_options` are optional, `file_format` is required and unknown
186
+ keys are rejected. See the [catalog reference](https://nrgconsulting.github.io/smallcat/catalog-reference/)
187
+ for every option, the validation rules and the credentials each format uses.
188
+
189
+ ## Docs
190
+ Read more at [the official docs](https://nrgconsulting.github.io/smallcat/).