etlpy-tools 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlpy_tools-1.0.0/LICENSE +21 -0
- etlpy_tools-1.0.0/PKG-INFO +311 -0
- etlpy_tools-1.0.0/README.md +270 -0
- etlpy_tools-1.0.0/pyproject.toml +72 -0
- etlpy_tools-1.0.0/setup.cfg +4 -0
- etlpy_tools-1.0.0/src/etl/__init__.py +0 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/api/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/api/basic_authenticate.py +20 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/api/oauthenticate.py +107 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/clickhouse/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/clickhouse/connect.py +28 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/clickhouse/read.py +34 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/datalake/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/datalake/read.py +35 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/datalake/reads.py +51 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/mongodb/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/mongodb/connect.py +28 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/mongodb/read.py +67 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/mssql/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/mssql/connect.py +29 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/mssql/read.py +33 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/postgresql/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/postgresql/connect.py +24 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/postgresql/read.py +31 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/sqlite/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/sqlite/connect.py +27 -0
- etlpy_tools-1.0.0/src/etl/extractor/steps/sqlite/read.py +25 -0
- etlpy_tools-1.0.0/src/etl/generic/__init__.py +5 -0
- etlpy_tools-1.0.0/src/etl/generic/context.py +36 -0
- etlpy_tools-1.0.0/src/etl/generic/pipeline.py +61 -0
- etlpy_tools-1.0.0/src/etl/generic/step.py +23 -0
- etlpy_tools-1.0.0/src/etl/loader/__init__.py +3 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/__init__.py +6 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/delta.py +52 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/ensuretable.py +83 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/insert.py +25 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/optimize.py +22 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/datalake/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/datalake/archive.py +34 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/datalake/save.py +38 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/__init__.py +5 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/delta.py +42 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/ensureindexes.py +27 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/upsert.py +42 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/mssql/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/mssql/ensuretable.py +46 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/mssql/insert.py +40 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/postgresql/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/postgresql/ensuretable.py +46 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/postgresql/insert.py +34 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/sqlite/__init__.py +4 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/sqlite/ensuretable.py +39 -0
- etlpy_tools-1.0.0/src/etl/loader/steps/sqlite/insert.py +32 -0
- etlpy_tools-1.0.0/src/etl/loader/storage.py +207 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/__init__.py +43 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/add_column.py +30 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/aggregate.py +55 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/cast_types.py +22 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/clear_text.py +39 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/drop_columns.py +36 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/drop_duplicates.py +29 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/drop_nulls.py +28 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/extract_entities.py +43 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/fill_nulls.py +29 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/filter_rows.py +22 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/generate_key.py +52 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/join.py +72 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/normalize_numeric.py +36 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/rename_columns.py +22 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/row_hash.py +32 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/slambda.py +22 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/sql.py +33 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/to_schema.py +67 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/trim_string.py +23 -0
- etlpy_tools-1.0.0/src/etl/transformer/steps/union.py +34 -0
- etlpy_tools-1.0.0/src/etlpy_tools.egg-info/PKG-INFO +311 -0
- etlpy_tools-1.0.0/src/etlpy_tools.egg-info/SOURCES.txt +78 -0
- etlpy_tools-1.0.0/src/etlpy_tools.egg-info/dependency_links.txt +1 -0
- etlpy_tools-1.0.0/src/etlpy_tools.egg-info/requires.txt +26 -0
- etlpy_tools-1.0.0/src/etlpy_tools.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vladimir Novotorzhin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: etlpy-tools
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A composable, async, Polars-native ETL toolkit
|
|
5
|
+
Author-email: Vladimir Novotorzhin <include.dos.h@yandex.ru>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Sque-ak/etlpy
|
|
8
|
+
Project-URL: Repository, https://github.com/Sque-ak/etlpy
|
|
9
|
+
Project-URL: Issues, https://github.com/Sque-ak/etlpy/issues
|
|
10
|
+
Keywords: etl,polars,clickhouse,airflow,data-pipeline
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Topic :: Database
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Requires-Python: >=3.13
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: polars>=1.41.2
|
|
20
|
+
Requires-Dist: polars-hash>=0.6.0
|
|
21
|
+
Provides-Extra: extractor
|
|
22
|
+
Requires-Dist: httpx>=0.28.1; extra == "extractor"
|
|
23
|
+
Requires-Dist: etlpy-tools[loader]; extra == "extractor"
|
|
24
|
+
Provides-Extra: transformer
|
|
25
|
+
Requires-Dist: polars>=1.41.2; extra == "transformer"
|
|
26
|
+
Requires-Dist: pyarrow>=15.0.0; extra == "transformer"
|
|
27
|
+
Provides-Extra: loader
|
|
28
|
+
Requires-Dist: pyarrow>=15.0.0; extra == "loader"
|
|
29
|
+
Provides-Extra: test
|
|
30
|
+
Requires-Dist: pytest>=9.1.0; extra == "test"
|
|
31
|
+
Requires-Dist: pytest-asyncio>=1.4.0; extra == "test"
|
|
32
|
+
Requires-Dist: testcontainers[clickhouse]>=4.14.2; extra == "test"
|
|
33
|
+
Requires-Dist: clickhouse-connect>=1.3.0; extra == "test"
|
|
34
|
+
Requires-Dist: pymongo>=4.9; extra == "test"
|
|
35
|
+
Requires-Dist: aioodbc>=0.5.0; extra == "test"
|
|
36
|
+
Requires-Dist: asyncpg>=0.31.0; extra == "test"
|
|
37
|
+
Requires-Dist: etlpy-tools[all]; extra == "test"
|
|
38
|
+
Provides-Extra: all
|
|
39
|
+
Requires-Dist: etlpy-tools[extractor,loader,transformer]; extra == "all"
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
|
|
42
|
+
<p align="center">
|
|
43
|
+
<img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/logo.png" alt="etlpy" width="360">
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
<p align="center">
|
|
47
|
+
<strong>A composable, async toolkit for building ETL pipelines in Python powered by Polars.</strong>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
<p align="center">
|
|
51
|
+
<img src="https://img.shields.io/badge/python-3.13+-blue.svg" alt="Python 3.13+">
|
|
52
|
+
<img src="https://img.shields.io/badge/engine-Polars-orange.svg" alt="Polars">
|
|
53
|
+
<img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License">
|
|
54
|
+
<img src="https://img.shields.io/badge/status-alpha-yellow.svg" alt="Alpha">
|
|
55
|
+
<img src="https://img.shields.io/badge/coverage-100%25-brightgreen.svg" alt="Coverage 100%">
|
|
56
|
+
<img src="https://img.shields.io/pypi/v/etlpytools.svg" alt="PyPI">
|
|
57
|
+
</p>
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
**ETLPY** is a small, unopinionated set of building blocks for extract / transform / load work.
|
|
62
|
+
You compose pipelines from tiny, single-purpose **steps** and run them - in a script or inside an Airflow task. It is a _toolkit_, not a framework: take what you need, write your own
|
|
63
|
+
steps for the rest.
|
|
64
|
+
|
|
65
|
+
- **Pipeline** - Just a list of async `Step`s. Add, remove, reorder.
|
|
66
|
+
- **Polars-native** - one engine end to end (Arrow under the hood).
|
|
67
|
+
- **Async** - steps are `async`, so you can fan out concurrent API/DB calls inside a step.
|
|
68
|
+
- **Many databases** - ClickHouse, PostgreSQL, MS SQL Server, MongoDB and SQLite connectors, plus a parquet data lake. Read from one, load into another.
|
|
69
|
+
- **Schema-guaranteed** - validate and coerce data against a Pydantic model, so downstream gets exactly the types you declared.
|
|
70
|
+
- **Deduplication** - row-hash change detection loads only new or changed rows.
|
|
71
|
+
- **Airflow-friendly** - one pipeline = one task. etlpy moves the data; Airflow orchestrates.
|
|
72
|
+
|
|
73
|
+
## Installation
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
pip install etlpy-tools # core tools kit
|
|
77
|
+
|
|
78
|
+
# or with the stage extras you need
|
|
79
|
+
pip install "etlpy-tools[all]" # everything
|
|
80
|
+
pip install "etlpy-tools[extractor]" # httpx-based API steps
|
|
81
|
+
pip install "etlpy-tools[transformer]" # Polars transform steps
|
|
82
|
+
pip install "etlpy-tools[loader]" # data lake + DB loaders
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
> Installed as **`etlpy-tools`**, imported as **`etl`** e.g. `from etl.generic import Pipeline`.
|
|
86
|
+
|
|
87
|
+
Database drivers are **not** bundled install the one you use:
|
|
88
|
+
`clickhouse-connect`, `asyncpg`, `aioodbc`, `pymongo`, or `aiosqlite`.
|
|
89
|
+
|
|
90
|
+
Requires Python 3.13+.
|
|
91
|
+
|
|
92
|
+
## Quick start
|
|
93
|
+
|
|
94
|
+
A pipeline threads a Polars `DataFrame` through a list of steps:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
import asyncio
|
|
98
|
+
from etl.generic import Pipeline, Pipestart
|
|
99
|
+
from etl.transformer.steps import ClearText, DropDuplicates, GenerateKey, RowHash
|
|
100
|
+
from etl.loader.steps.datalake import Save
|
|
101
|
+
|
|
102
|
+
@Pipestart
|
|
103
|
+
async def clean_customers(raw_df):
|
|
104
|
+
return Pipeline([
|
|
105
|
+
ClearText(),
|
|
106
|
+
DropDuplicates(),
|
|
107
|
+
GenerateKey(columns=["id"], key_name="pk"),
|
|
108
|
+
RowHash(), # content fingerprint for change detection
|
|
109
|
+
Save(name="customers", layer="raw"), # persist to the data lake
|
|
110
|
+
], dataframe=raw_df)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
asyncio.run(clean_customers(raw_df))
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Load a lake file into ClickHouse, inserting only what actually changed:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from etl.generic import Pipeline
|
|
120
|
+
from etl.extractor.steps.datalake import Read
|
|
121
|
+
from etl.loader.steps.clickhouse import EnsureTable, Delta, Insert
|
|
122
|
+
from etl.extractor.steps.clickhouse import Connect
|
|
123
|
+
from etl.loader.steps.datalake import Archive
|
|
124
|
+
|
|
125
|
+
await Pipeline([
|
|
126
|
+
Connect(host="clickhouse", database="analytics"),
|
|
127
|
+
Read(layer="fact", name="transactions"),
|
|
128
|
+
EnsureTable("fact_transactions",
|
|
129
|
+
engine="ReplacingMergeTree(loaded_at)", order_by=["pk"]),
|
|
130
|
+
Delta("fact_transactions", keys=["pk"]), # skip rows already loaded unchanged
|
|
131
|
+
Insert("fact_transactions"),
|
|
132
|
+
Archive(layer="fact", name="transactions"),
|
|
133
|
+
]).run()
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Any step can halt the pipeline gracefully by raising `StopPipeline`. It is a
|
|
137
|
+
control-flow signal, not an error: `run()` stops, skips the remaining steps and
|
|
138
|
+
returns the data produced so far (no traceback).
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
from etl.generic import Step, StopPipeline
|
|
142
|
+
|
|
143
|
+
class StopIfEmpty(Step):
|
|
144
|
+
async def apply(self, df, data=None):
|
|
145
|
+
if df is None or df.is_empty():
|
|
146
|
+
raise StopPipeline("no rows") # stop, return df as-is
|
|
147
|
+
return df
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Pass `df=` to control what the pipeline returns on stop:
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
raise StopPipeline("threshold hit", df=partial)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## Core concepts
|
|
157
|
+
|
|
158
|
+
| Piece | What it is |
|
|
159
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
160
|
+
| **`Pipeline`** | An ordered list of steps. `run()` threads a DataFrame through them and returns the final one. |
|
|
161
|
+
| **`StopPipeline`** | A step may raise StopPipeline to halt the run early |
|
|
162
|
+
| **`Pipestart`** | A decorator that runs a pipeline-returning function. |
|
|
163
|
+
| **`Step`** | A unit of work: `async def apply(self, df, data) -> df`. Write your own by subclassing. |
|
|
164
|
+
| **`Data`** | A shared context (auth tokens, DB clients, config) passed to every step. Mutated by reference; **DataFrames don't live here** - they flow as the threaded `df` or via the lake. |
|
|
165
|
+
|
|
166
|
+
The split is deliberate: the **`DataFrame`** is threaded and returned, while the **`Data`** context is
|
|
167
|
+
shared state. Heavy tables move through the data lake, not through the context - which keeps each
|
|
168
|
+
pipeline a clean fit for a single Airflow task.
|
|
169
|
+
|
|
170
|
+
## The toolbox
|
|
171
|
+
|
|
172
|
+
Every step is one small class with an `async def apply(self, df, data) -> df`. Below is the full set.
|
|
173
|
+
Missing one? Subclass `Step`, implement `apply`, and drop it into the list.
|
|
174
|
+
|
|
175
|
+
## Schema guarantee with `ToSchema`
|
|
176
|
+
|
|
177
|
+
Raw data lies: missing fields, wrong types, extra columns, values that don't parse.
|
|
178
|
+
`ToSchema` is the gate that turns messy input into exactly the data you declared. It
|
|
179
|
+
validates and coerces **every row against a Pydantic model** and returns a DataFrame whose
|
|
180
|
+
schema comes from that model - what is not in the model is dropped, what cannot be coerced
|
|
181
|
+
raises. Downstream steps and your warehouse table then get clean, typed data by construction.
|
|
182
|
+
|
|
183
|
+
Declare the canon as a Pydantic model:
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
from datetime import date
|
|
187
|
+
from pydantic import BaseModel
|
|
188
|
+
|
|
189
|
+
class Transaction(BaseModel):
|
|
190
|
+
id: int
|
|
191
|
+
amount: float
|
|
192
|
+
currency: str
|
|
193
|
+
txn_date: date
|
|
194
|
+
note: str | None = None # optional -> nullable column
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Drop `ToSchema` into a pipeline:
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
from etl.transformer.steps import ToSchema, GenerateKey, RowHash
|
|
201
|
+
|
|
202
|
+
Pipeline([
|
|
203
|
+
# ... read raw data ...
|
|
204
|
+
ToSchema(Transaction, mapping={"trans_id": "id", "sum": "amount"}), # rename + validate + type
|
|
205
|
+
GenerateKey(columns=["id"], key_name="pk"),
|
|
206
|
+
RowHash(),
|
|
207
|
+
# ... load ...
|
|
208
|
+
])
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
What the step guarantees:
|
|
212
|
+
|
|
213
|
+
- **Exact schema & types** - columns and dtypes come from the model, not from guessing on the
|
|
214
|
+
first N rows: `int` -> `Int64`, `date` -> `Date`, `str | None` -> nullable `String`, and so on.
|
|
215
|
+
- **Extra columns dropped** - anything outside the model is discarded.
|
|
216
|
+
- **Fail fast** - a value that can't be coerced (or a missing required field) raises, so bad
|
|
217
|
+
upstream data never silently reaches the warehouse.
|
|
218
|
+
- **`mapping`** - rename raw source fields to the canonical names (1:1) before validation.
|
|
219
|
+
|
|
220
|
+
`ToSchema` never imports pydantic itself - it only uses the model you pass, so **pydantic
|
|
221
|
+
stays an optional, user-side dependency**. It also works with
|
|
222
|
+
[Patito](https://github.com/JakobGM/patito) models (Pydantic on Polars). Validation runs per
|
|
223
|
+
row - thorough rather than fast; correctness is the point, speed is recovered downstream.
|
|
224
|
+
|
|
225
|
+
### Extract
|
|
226
|
+
|
|
227
|
+
- **`OAuthenticate(url, credentials, fields=OAuthFields(...), send="json", method="POST", auth_header=None, headers=None, store="auth", timeout=60.0)`** - non-interactive OAuth 2.0 token flow (client-credentials). Sends `credentials` (JSON or form via `send`), pulls the token out of the response by dotted paths (`fields`), applies `auth_header` (e.g. `{"Authorization": "Bearer {token}"}`, `{token}` is filled in) to the shared httpx client, and stores the parsed auth in `data["auth"]`. Request-level headers (gateway API keys, `Content-Type`) go in `headers`.
|
|
228
|
+
- **`AuthenticateBasic(user, password, headers=None, timeout=60.0)`** _(HTTP Basic, RFC 7617)_ - sets `httpx.BasicAuth(user, password)` on the shared client so every downstream request carries `Authorization: Basic ...`. No token exchange - the header is static.
|
|
229
|
+
- **`Read(layer, name)`** _(data lake)_ - read `{name}.parquet` from a lake layer into the pipeline df.
|
|
230
|
+
- **`Read(query)`** _(ClickHouse)_ - run a SQL query (client from `data["ch"]`) and return the result as a Polars frame.
|
|
231
|
+
- **`Reads(layer, name, mode=STATIC, date_from=None, date_to=None, pattern="*.parquet", missing_ok=False)`** _(data lake)_ - read and concat every parquet in a lake folder, with an optional inclusive date range on the file name.
|
|
232
|
+
- **`Connect(host, port=8123, database, username, password, secure=False)`** _(ClickHouse)_ - open a clickhouse-connect client (extra kwargs are forwarded) and store it in `data["ch"]`.
|
|
233
|
+
|
|
234
|
+
> **`OAuthFields`** maps where each token-response field lives, e.g.
|
|
235
|
+
> `OAuthFields(access_token="data.access_token", expires_in="data.expires_in")`.
|
|
236
|
+
> Set a field to `None` if the API does not return it.
|
|
237
|
+
|
|
238
|
+
### Transform
|
|
239
|
+
|
|
240
|
+
- **`AddColumn(column_name, value)`** - add a constant-value column.
|
|
241
|
+
- **`CastTypes({column: dtype})`** - cast columns to Polars dtypes (pin types before loading).
|
|
242
|
+
- **`RenameColumns({old: new})`** - rename columns.
|
|
243
|
+
- **`DropColumns(columns, exclude=False)`** - drop the listed columns; `exclude=True` keeps only them.
|
|
244
|
+
- **`DropDuplicates(subset=None)`** - drop duplicate rows (by `subset`, or all columns).
|
|
245
|
+
- **`DropNulls(subset=None)`** - drop rows with a null in any of `subset`.
|
|
246
|
+
- **`FillNulls(value, columns=None)`** - fill nulls; `value` is a scalar or a `{column: value}` dict.
|
|
247
|
+
- **`FilterRows(expr)`** - keep rows matching a Polars boolean expression, e.g. `pl.col("amount") > 0`.
|
|
248
|
+
- **`TrimString(columns=None)`** - strip whitespace from string columns (`None` / `"*"` = all string columns).
|
|
249
|
+
- **`ClearText(columns="*")`** - clean text: collapse newlines to spaces, drop quotes/backslashes, trim.
|
|
250
|
+
- **`NormalizeNumeric(columns, method="minmax")`** - scale numeric columns (`"minmax"` or `"zscore"`).
|
|
251
|
+
- **`GenerateKey(columns, key_name="pk", mode="hash")`** - build a key column. `mode`: `"hash"` (SHA-256 hex), `"hash_int"` (stable 64-bit int), `"sequential"` (1, 2, 3, ...).
|
|
252
|
+
- **`RowHash(exclude=None, separator="||")`** - SHA-256 fingerprint of every row value (minus `row_hash`, `loaded_at`, and `exclude`) written to `row_hash`, for change detection.
|
|
253
|
+
- **`Aggregate(group_by, aggregations)`** - group + aggregate, e.g. `{"amount": ["sum", "mean"]}`; output columns are `{column}_{func}`.
|
|
254
|
+
- **`Join(other, on, how="inner", select=None, prefix=None)`** - join with another frame **or a sub-`Pipeline`** (run to produce the right side); `select` / `prefix` shape the right columns.
|
|
255
|
+
- **`ExtractEntities(sources, defaults=None)`** - stack several column groups into one long table (e.g. sender/receiver columns into a single `party` table).
|
|
256
|
+
- **`ToSchema(model, mapping=None)`** - validate and coerce the df against a Pydantic model; output has the model's exact schema (see [Schema guarantee](#schema-guarantee-with-toschema)).
|
|
257
|
+
- **`Union(other, how="vertical")`** - stack another DataFrame **or a sub-`Pipeline`** onto the df (`vertical` / `diagonal` / `horizontal`).
|
|
258
|
+
- **`SQL(query, view_name="source")`** - run a Polars-SQL query over the df (registered as `view_name`); `SQL.from_file(path)` loads the query from a `.sql` file.
|
|
259
|
+
- **`Lambda(func)`** - apply an arbitrary `df -> df` callable (escape hatch for one-off logic).
|
|
260
|
+
|
|
261
|
+
### Load
|
|
262
|
+
|
|
263
|
+
- **`EnsureTable(table, order_by, engine="MergeTree", partition_by=None, if_exists="append")`** - create `table` from the df schema if it does not exist. `order_by` is required (it is the primary / dedup key); `if_exists="error"` raises when the table already exists.
|
|
264
|
+
- **`Delta(table, keys)`** - keep only new or changed rows by comparing `row_hash` against the table (needs a `ReplacingMergeTree`-family engine). The core of idempotent loads.
|
|
265
|
+
- **`Insert(table)`** - insert the df into a ClickHouse table via Arrow (an empty / `None` frame is skipped).
|
|
266
|
+
- **`Save(name, layer="raw")`** _(data lake)_ - write the df to a lake layer as `{name}.parquet`, overwriting in place.
|
|
267
|
+
- **`Optimize(table)`** _(ClickHouse)_ - `OPTIMIZE TABLE ... FINAL` to collapse ReplacingMergeTree duplicates (expensive; run after a load, not per row).
|
|
268
|
+
- **`Archive(layer, name)`** _(data lake)_ - move a lake file into the `archive` layer (e.g. after a successful load).
|
|
269
|
+
|
|
270
|
+
## Connectors
|
|
271
|
+
|
|
272
|
+
etlpy ships steps for several databases. **Drivers are not bundled** - there are many
|
|
273
|
+
databases, so you install only the one you need. Each `Connect` opens a client and stores it
|
|
274
|
+
in `data[...]`; the read/load steps pick it up from there.
|
|
275
|
+
|
|
276
|
+
| Database | Steps | `data` key | Driver (install yourself) |
|
|
277
|
+
| ----------------- | --------------------------------------------------------------- | ---------------- | ------------------------- |
|
|
278
|
+
| **ClickHouse** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Insert`, `Optimize` | `data["ch"]` | `clickhouse-connect` |
|
|
279
|
+
| **PostgreSQL** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["pg"]` | `asyncpg` |
|
|
280
|
+
| **MS SQL Server** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["mssql"]` | `aioodbc` (+ ODBC driver) |
|
|
281
|
+
| **MongoDB** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Upsert` | `data["mongo"]` | `pymongo` (async) |
|
|
282
|
+
| **SQLite** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["sqlite"]` | `aiosqlite` |
|
|
283
|
+
|
|
284
|
+
Import `Connect`/`Read` from `etl.extractor.steps.<db>` and the write steps from
|
|
285
|
+
`etl.loader.steps.<db>`. SQL reads are parameterized (injection-safe): ClickHouse
|
|
286
|
+
`{name:Type}`, Postgres `$1`, MS SQL / SQLite `?`; MongoDB uses a query dict.
|
|
287
|
+
|
|
288
|
+
> **SQLite** is meant for tests and small local datasets (great as an in-memory `:memory:`
|
|
289
|
+
> backend). It is a single-file, single-writer engine and is **not suitable for storing large
|
|
290
|
+
> data** - use ClickHouse or Postgres for that.
|
|
291
|
+
|
|
292
|
+
## Testing
|
|
293
|
+
|
|
294
|
+
Every step has its own unit tests, and the whole extract / transform / load chain is exercised end to
|
|
295
|
+
end against a real ClickHouse (spun up in Docker via `testcontainers`). A complete pipeline example
|
|
296
|
+
lives in [`generic/tests/test_pipeline.py`](src/etl/generic/tests/test_pipeline.py).
|
|
297
|
+
|
|
298
|
+
```bash
|
|
299
|
+
pip install -e ".[all,test]"
|
|
300
|
+
pytest -q
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
<p align="center"> <img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/100cov.png" alt="100% coverage" width="640"> </p>
|
|
304
|
+
|
|
305
|
+
## Status
|
|
306
|
+
|
|
307
|
+
Alpha, under active development. A PyPI release is planned.
|
|
308
|
+
|
|
309
|
+
## License
|
|
310
|
+
|
|
311
|
+
Released under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/logo.png" alt="etlpy" width="360">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<strong>A composable, async toolkit for building ETL pipelines in Python powered by Polars.</strong>
|
|
7
|
+
</p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<img src="https://img.shields.io/badge/python-3.13+-blue.svg" alt="Python 3.13+">
|
|
11
|
+
<img src="https://img.shields.io/badge/engine-Polars-orange.svg" alt="Polars">
|
|
12
|
+
<img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License">
|
|
13
|
+
<img src="https://img.shields.io/badge/status-alpha-yellow.svg" alt="Alpha">
|
|
14
|
+
<img src="https://img.shields.io/badge/coverage-100%25-brightgreen.svg" alt="Coverage 100%">
|
|
15
|
+
<img src="https://img.shields.io/pypi/v/etlpytools.svg" alt="PyPI">
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
**ETLPY** is a small, unopinionated set of building blocks for extract / transform / load work.
|
|
21
|
+
You compose pipelines from tiny, single-purpose **steps** and run them - in a script or inside an Airflow task. It is a _toolkit_, not a framework: take what you need, write your own
|
|
22
|
+
steps for the rest.
|
|
23
|
+
|
|
24
|
+
- **Pipeline** - Just a list of async `Step`s. Add, remove, reorder.
|
|
25
|
+
- **Polars-native** - one engine end to end (Arrow under the hood).
|
|
26
|
+
- **Async** - steps are `async`, so you can fan out concurrent API/DB calls inside a step.
|
|
27
|
+
- **Many databases** - ClickHouse, PostgreSQL, MS SQL Server, MongoDB and SQLite connectors, plus a parquet data lake. Read from one, load into another.
|
|
28
|
+
- **Schema-guaranteed** - validate and coerce data against a Pydantic model, so downstream gets exactly the types you declared.
|
|
29
|
+
- **Deduplication** - row-hash change detection loads only new or changed rows.
|
|
30
|
+
- **Airflow-friendly** - one pipeline = one task. etlpy moves the data; Airflow orchestrates.
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install etlpy-tools # core tools kit
|
|
36
|
+
|
|
37
|
+
# or with the stage extras you need
|
|
38
|
+
pip install "etlpy-tools[all]" # everything
|
|
39
|
+
pip install "etlpy-tools[extractor]" # httpx-based API steps
|
|
40
|
+
pip install "etlpy-tools[transformer]" # Polars transform steps
|
|
41
|
+
pip install "etlpy-tools[loader]" # data lake + DB loaders
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
> Installed as **`etlpy-tools`**, imported as **`etl`** e.g. `from etl.generic import Pipeline`.
|
|
45
|
+
|
|
46
|
+
Database drivers are **not** bundled install the one you use:
|
|
47
|
+
`clickhouse-connect`, `asyncpg`, `aioodbc`, `pymongo`, or `aiosqlite`.
|
|
48
|
+
|
|
49
|
+
Requires Python 3.13+.
|
|
50
|
+
|
|
51
|
+
## Quick start
|
|
52
|
+
|
|
53
|
+
A pipeline threads a Polars `DataFrame` through a list of steps:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import asyncio
|
|
57
|
+
from etl.generic import Pipeline, Pipestart
|
|
58
|
+
from etl.transformer.steps import ClearText, DropDuplicates, GenerateKey, RowHash
|
|
59
|
+
from etl.loader.steps.datalake import Save
|
|
60
|
+
|
|
61
|
+
@Pipestart
|
|
62
|
+
async def clean_customers(raw_df):
|
|
63
|
+
return Pipeline([
|
|
64
|
+
ClearText(),
|
|
65
|
+
DropDuplicates(),
|
|
66
|
+
GenerateKey(columns=["id"], key_name="pk"),
|
|
67
|
+
RowHash(), # content fingerprint for change detection
|
|
68
|
+
Save(name="customers", layer="raw"), # persist to the data lake
|
|
69
|
+
], dataframe=raw_df)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
asyncio.run(clean_customers(raw_df))
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Load a lake file into ClickHouse, inserting only what actually changed:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from etl.generic import Pipeline
|
|
79
|
+
from etl.extractor.steps.datalake import Read
|
|
80
|
+
from etl.loader.steps.clickhouse import EnsureTable, Delta, Insert
|
|
81
|
+
from etl.extractor.steps.clickhouse import Connect
|
|
82
|
+
from etl.loader.steps.datalake import Archive
|
|
83
|
+
|
|
84
|
+
await Pipeline([
|
|
85
|
+
Connect(host="clickhouse", database="analytics"),
|
|
86
|
+
Read(layer="fact", name="transactions"),
|
|
87
|
+
EnsureTable("fact_transactions",
|
|
88
|
+
engine="ReplacingMergeTree(loaded_at)", order_by=["pk"]),
|
|
89
|
+
Delta("fact_transactions", keys=["pk"]), # skip rows already loaded unchanged
|
|
90
|
+
Insert("fact_transactions"),
|
|
91
|
+
Archive(layer="fact", name="transactions"),
|
|
92
|
+
]).run()
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Any step can halt the pipeline gracefully by raising `StopPipeline`. It is a
|
|
96
|
+
control-flow signal, not an error: `run()` stops, skips the remaining steps and
|
|
97
|
+
returns the data produced so far (no traceback).
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from etl.generic import Step, StopPipeline
|
|
101
|
+
|
|
102
|
+
class StopIfEmpty(Step):
|
|
103
|
+
async def apply(self, df, data=None):
|
|
104
|
+
if df is None or df.is_empty():
|
|
105
|
+
raise StopPipeline("no rows") # stop, return df as-is
|
|
106
|
+
return df
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Pass `df=` to control what the pipeline returns on stop:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
raise StopPipeline("threshold hit", df=partial)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Core concepts
|
|
116
|
+
|
|
117
|
+
| Piece | What it is |
|
|
118
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
119
|
+
| **`Pipeline`** | An ordered list of steps. `run()` threads a DataFrame through them and returns the final one. |
|
|
120
|
+
| **`StopPipeline`** | A step may raise StopPipeline to halt the run early |
|
|
121
|
+
| **`Pipestart`** | A decorator that runs a pipeline-returning function. |
|
|
122
|
+
| **`Step`** | A unit of work: `async def apply(self, df, data) -> df`. Write your own by subclassing. |
|
|
123
|
+
| **`Data`** | A shared context (auth tokens, DB clients, config) passed to every step. Mutated by reference; **DataFrames don't live here** - they flow as the threaded `df` or via the lake. |
|
|
124
|
+
|
|
125
|
+
The split is deliberate: the **`DataFrame`** is threaded and returned, while the **`Data`** context is
|
|
126
|
+
shared state. Heavy tables move through the data lake, not through the context - which keeps each
|
|
127
|
+
pipeline a clean fit for a single Airflow task.
|
|
128
|
+
|
|
129
|
+
## The toolbox
|
|
130
|
+
|
|
131
|
+
Every step is one small class with an `async def apply(self, df, data) -> df`. Below is the full set.
|
|
132
|
+
Missing one? Subclass `Step`, implement `apply`, and drop it into the list.
|
|
133
|
+
|
|
134
|
+
## Schema guarantee with `ToSchema`
|
|
135
|
+
|
|
136
|
+
Raw data lies: missing fields, wrong types, extra columns, values that don't parse.
|
|
137
|
+
`ToSchema` is the gate that turns messy input into exactly the data you declared. It
|
|
138
|
+
validates and coerces **every row against a Pydantic model** and returns a DataFrame whose
|
|
139
|
+
schema comes from that model - what is not in the model is dropped, what cannot be coerced
|
|
140
|
+
raises. Downstream steps and your warehouse table then get clean, typed data by construction.
|
|
141
|
+
|
|
142
|
+
Declare the canon as a Pydantic model:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from datetime import date
|
|
146
|
+
from pydantic import BaseModel
|
|
147
|
+
|
|
148
|
+
class Transaction(BaseModel):
|
|
149
|
+
id: int
|
|
150
|
+
amount: float
|
|
151
|
+
currency: str
|
|
152
|
+
txn_date: date
|
|
153
|
+
note: str | None = None # optional -> nullable column
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Drop `ToSchema` into a pipeline:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
from etl.transformer.steps import ToSchema, GenerateKey, RowHash
|
|
160
|
+
|
|
161
|
+
Pipeline([
|
|
162
|
+
# ... read raw data ...
|
|
163
|
+
ToSchema(Transaction, mapping={"trans_id": "id", "sum": "amount"}), # rename + validate + type
|
|
164
|
+
GenerateKey(columns=["id"], key_name="pk"),
|
|
165
|
+
RowHash(),
|
|
166
|
+
# ... load ...
|
|
167
|
+
])
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
What the step guarantees:
|
|
171
|
+
|
|
172
|
+
- **Exact schema & types** - columns and dtypes come from the model, not from guessing on the
|
|
173
|
+
first N rows: `int` -> `Int64`, `date` -> `Date`, `str | None` -> nullable `String`, and so on.
|
|
174
|
+
- **Extra columns dropped** - anything outside the model is discarded.
|
|
175
|
+
- **Fail fast** - a value that can't be coerced (or a missing required field) raises, so bad
|
|
176
|
+
upstream data never silently reaches the warehouse.
|
|
177
|
+
- **`mapping`** - rename raw source fields to the canonical names (1:1) before validation.
|
|
178
|
+
|
|
179
|
+
`ToSchema` never imports pydantic itself - it only uses the model you pass, so **pydantic
|
|
180
|
+
stays an optional, user-side dependency**. It also works with
|
|
181
|
+
[Patito](https://github.com/JakobGM/patito) models (Pydantic on Polars). Validation runs per
|
|
182
|
+
row - thorough rather than fast; correctness is the point, speed is recovered downstream.
|
|
183
|
+
|
|
184
|
+
### Extract
|
|
185
|
+
|
|
186
|
+
- **`OAuthenticate(url, credentials, fields=OAuthFields(...), send="json", method="POST", auth_header=None, headers=None, store="auth", timeout=60.0)`** - non-interactive OAuth 2.0 token flow (client-credentials). Sends `credentials` (JSON or form via `send`), pulls the token out of the response by dotted paths (`fields`), applies `auth_header` (e.g. `{"Authorization": "Bearer {token}"}`, `{token}` is filled in) to the shared httpx client, and stores the parsed auth in `data["auth"]`. Request-level headers (gateway API keys, `Content-Type`) go in `headers`.
|
|
187
|
+
- **`AuthenticateBasic(user, password, headers=None, timeout=60.0)`** _(HTTP Basic, RFC 7617)_ - sets `httpx.BasicAuth(user, password)` on the shared client so every downstream request carries `Authorization: Basic ...`. No token exchange - the header is static.
|
|
188
|
+
- **`Read(layer, name)`** _(data lake)_ - read `{name}.parquet` from a lake layer into the pipeline df.
|
|
189
|
+
- **`Read(query)`** _(ClickHouse)_ - run a SQL query (client from `data["ch"]`) and return the result as a Polars frame.
|
|
190
|
+
- **`Reads(layer, name, mode=STATIC, date_from=None, date_to=None, pattern="*.parquet", missing_ok=False)`** _(data lake)_ - read and concat every parquet in a lake folder, with an optional inclusive date range on the file name.
|
|
191
|
+
- **`Connect(host, port=8123, database, username, password, secure=False)`** _(ClickHouse)_ - open a clickhouse-connect client (extra kwargs are forwarded) and store it in `data["ch"]`.
|
|
192
|
+
|
|
193
|
+
> **`OAuthFields`** maps where each token-response field lives, e.g.
|
|
194
|
+
> `OAuthFields(access_token="data.access_token", expires_in="data.expires_in")`.
|
|
195
|
+
> Set a field to `None` if the API does not return it.
|
|
196
|
+
|
|
197
|
+
### Transform
|
|
198
|
+
|
|
199
|
+
- **`AddColumn(column_name, value)`** - add a constant-value column.
|
|
200
|
+
- **`CastTypes({column: dtype})`** - cast columns to Polars dtypes (pin types before loading).
|
|
201
|
+
- **`RenameColumns({old: new})`** - rename columns.
|
|
202
|
+
- **`DropColumns(columns, exclude=False)`** - drop the listed columns; `exclude=True` keeps only them.
|
|
203
|
+
- **`DropDuplicates(subset=None)`** - drop duplicate rows (by `subset`, or all columns).
|
|
204
|
+
- **`DropNulls(subset=None)`** - drop rows with a null in any of `subset`.
|
|
205
|
+
- **`FillNulls(value, columns=None)`** - fill nulls; `value` is a scalar or a `{column: value}` dict.
|
|
206
|
+
- **`FilterRows(expr)`** - keep rows matching a Polars boolean expression, e.g. `pl.col("amount") > 0`.
|
|
207
|
+
- **`TrimString(columns=None)`** - strip whitespace from string columns (`None` / `"*"` = all string columns).
|
|
208
|
+
- **`ClearText(columns="*")`** - clean text: collapse newlines to spaces, drop quotes/backslashes, trim.
|
|
209
|
+
- **`NormalizeNumeric(columns, method="minmax")`** - scale numeric columns (`"minmax"` or `"zscore"`).
|
|
210
|
+
- **`GenerateKey(columns, key_name="pk", mode="hash")`** - build a key column. `mode`: `"hash"` (SHA-256 hex), `"hash_int"` (stable 64-bit int), `"sequential"` (1, 2, 3, ...).
|
|
211
|
+
- **`RowHash(exclude=None, separator="||")`** - SHA-256 fingerprint of every row value (minus `row_hash`, `loaded_at`, and `exclude`) written to `row_hash`, for change detection.
|
|
212
|
+
- **`Aggregate(group_by, aggregations)`** - group + aggregate, e.g. `{"amount": ["sum", "mean"]}`; output columns are `{column}_{func}`.
|
|
213
|
+
- **`Join(other, on, how="inner", select=None, prefix=None)`** - join with another frame **or a sub-`Pipeline`** (run to produce the right side); `select` / `prefix` shape the right columns.
|
|
214
|
+
- **`ExtractEntities(sources, defaults=None)`** - stack several column groups into one long table (e.g. sender/receiver columns into a single `party` table).
|
|
215
|
+
- **`ToSchema(model, mapping=None)`** - validate and coerce the df against a Pydantic model; output has the model's exact schema (see [Schema guarantee](#schema-guarantee-with-toschema)).
|
|
216
|
+
- **`Union(other, how="vertical")`** - stack another DataFrame **or a sub-`Pipeline`** onto the df (`vertical` / `diagonal` / `horizontal`).
|
|
217
|
+
- **`SQL(query, view_name="source")`** - run a Polars-SQL query over the df (registered as `view_name`); `SQL.from_file(path)` loads the query from a `.sql` file.
|
|
218
|
+
- **`Lambda(func)`** - apply an arbitrary `df -> df` callable (escape hatch for one-off logic).
|
|
219
|
+
|
|
220
|
+
### Load
|
|
221
|
+
|
|
222
|
+
- **`EnsureTable(table, order_by, engine="MergeTree", partition_by=None, if_exists="append")`** - create `table` from the df schema if it does not exist. `order_by` is required (it is the primary / dedup key); `if_exists="error"` raises when the table already exists.
|
|
223
|
+
- **`Delta(table, keys)`** - keep only new or changed rows by comparing `row_hash` against the table (needs a `ReplacingMergeTree`-family engine). The core of idempotent loads.
|
|
224
|
+
- **`Insert(table)`** - insert the df into a ClickHouse table via Arrow (an empty / `None` frame is skipped).
|
|
225
|
+
- **`Save(name, layer="raw")`** _(data lake)_ - write the df to a lake layer as `{name}.parquet`, overwriting in place.
|
|
226
|
+
- **`Optimize(table)`** _(ClickHouse)_ - `OPTIMIZE TABLE ... FINAL` to collapse ReplacingMergeTree duplicates (expensive; run after a load, not per row).
|
|
227
|
+
- **`Archive(layer, name)`** _(data lake)_ - move a lake file into the `archive` layer (e.g. after a successful load).
|
|
228
|
+
|
|
229
|
+
## Connectors
|
|
230
|
+
|
|
231
|
+
etlpy ships steps for several databases. **Drivers are not bundled** - there are many
|
|
232
|
+
databases, so you install only the one you need. Each `Connect` opens a client and stores it
|
|
233
|
+
in `data[...]`; the read/load steps pick it up from there.
|
|
234
|
+
|
|
235
|
+
| Database | Steps | `data` key | Driver (install yourself) |
|
|
236
|
+
| ----------------- | --------------------------------------------------------------- | ---------------- | ------------------------- |
|
|
237
|
+
| **ClickHouse** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Insert`, `Optimize` | `data["ch"]` | `clickhouse-connect` |
|
|
238
|
+
| **PostgreSQL** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["pg"]` | `asyncpg` |
|
|
239
|
+
| **MS SQL Server** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["mssql"]` | `aioodbc` (+ ODBC driver) |
|
|
240
|
+
| **MongoDB** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Upsert` | `data["mongo"]` | `pymongo` (async) |
|
|
241
|
+
| **SQLite** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["sqlite"]` | `aiosqlite` |
|
|
242
|
+
|
|
243
|
+
Import `Connect`/`Read` from `etl.extractor.steps.<db>` and the write steps from
|
|
244
|
+
`etl.loader.steps.<db>`. SQL reads are parameterized (injection-safe): ClickHouse
|
|
245
|
+
`{name:Type}`, Postgres `$1`, MS SQL / SQLite `?`; MongoDB uses a query dict.
|
|
246
|
+
|
|
247
|
+
> **SQLite** is meant for tests and small local datasets (great as an in-memory `:memory:`
|
|
248
|
+
> backend). It is a single-file, single-writer engine and is **not suitable for storing large
|
|
249
|
+
> data** - use ClickHouse or Postgres for that.
|
|
250
|
+
|
|
251
|
+
## Testing
|
|
252
|
+
|
|
253
|
+
Every step has its own unit tests, and the whole extract / transform / load chain is exercised end to
|
|
254
|
+
end against a real ClickHouse (spun up in Docker via `testcontainers`). A complete pipeline example
|
|
255
|
+
lives in [`generic/tests/test_pipeline.py`](src/etl/generic/tests/test_pipeline.py).
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
pip install -e ".[all,test]"
|
|
259
|
+
pytest -q
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
<p align="center"> <img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/100cov.png" alt="100% coverage" width="640"> </p>
|
|
263
|
+
|
|
264
|
+
## Status
|
|
265
|
+
|
|
266
|
+
Alpha, under active development. A PyPI release is planned.
|
|
267
|
+
|
|
268
|
+
## License
|
|
269
|
+
|
|
270
|
+
Released under the [MIT License](LICENSE).
|