etlpy-tools 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. etlpy_tools-1.0.0/LICENSE +21 -0
  2. etlpy_tools-1.0.0/PKG-INFO +311 -0
  3. etlpy_tools-1.0.0/README.md +270 -0
  4. etlpy_tools-1.0.0/pyproject.toml +72 -0
  5. etlpy_tools-1.0.0/setup.cfg +4 -0
  6. etlpy_tools-1.0.0/src/etl/__init__.py +0 -0
  7. etlpy_tools-1.0.0/src/etl/extractor/steps/api/__init__.py +4 -0
  8. etlpy_tools-1.0.0/src/etl/extractor/steps/api/basic_authenticate.py +20 -0
  9. etlpy_tools-1.0.0/src/etl/extractor/steps/api/oauthenticate.py +107 -0
  10. etlpy_tools-1.0.0/src/etl/extractor/steps/clickhouse/__init__.py +4 -0
  11. etlpy_tools-1.0.0/src/etl/extractor/steps/clickhouse/connect.py +28 -0
  12. etlpy_tools-1.0.0/src/etl/extractor/steps/clickhouse/read.py +34 -0
  13. etlpy_tools-1.0.0/src/etl/extractor/steps/datalake/__init__.py +4 -0
  14. etlpy_tools-1.0.0/src/etl/extractor/steps/datalake/read.py +35 -0
  15. etlpy_tools-1.0.0/src/etl/extractor/steps/datalake/reads.py +51 -0
  16. etlpy_tools-1.0.0/src/etl/extractor/steps/mongodb/__init__.py +4 -0
  17. etlpy_tools-1.0.0/src/etl/extractor/steps/mongodb/connect.py +28 -0
  18. etlpy_tools-1.0.0/src/etl/extractor/steps/mongodb/read.py +67 -0
  19. etlpy_tools-1.0.0/src/etl/extractor/steps/mssql/__init__.py +4 -0
  20. etlpy_tools-1.0.0/src/etl/extractor/steps/mssql/connect.py +29 -0
  21. etlpy_tools-1.0.0/src/etl/extractor/steps/mssql/read.py +33 -0
  22. etlpy_tools-1.0.0/src/etl/extractor/steps/postgresql/__init__.py +4 -0
  23. etlpy_tools-1.0.0/src/etl/extractor/steps/postgresql/connect.py +24 -0
  24. etlpy_tools-1.0.0/src/etl/extractor/steps/postgresql/read.py +31 -0
  25. etlpy_tools-1.0.0/src/etl/extractor/steps/sqlite/__init__.py +4 -0
  26. etlpy_tools-1.0.0/src/etl/extractor/steps/sqlite/connect.py +27 -0
  27. etlpy_tools-1.0.0/src/etl/extractor/steps/sqlite/read.py +25 -0
  28. etlpy_tools-1.0.0/src/etl/generic/__init__.py +5 -0
  29. etlpy_tools-1.0.0/src/etl/generic/context.py +36 -0
  30. etlpy_tools-1.0.0/src/etl/generic/pipeline.py +61 -0
  31. etlpy_tools-1.0.0/src/etl/generic/step.py +23 -0
  32. etlpy_tools-1.0.0/src/etl/loader/__init__.py +3 -0
  33. etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/__init__.py +6 -0
  34. etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/delta.py +52 -0
  35. etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/ensuretable.py +83 -0
  36. etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/insert.py +25 -0
  37. etlpy_tools-1.0.0/src/etl/loader/steps/clickhouse/optimize.py +22 -0
  38. etlpy_tools-1.0.0/src/etl/loader/steps/datalake/__init__.py +4 -0
  39. etlpy_tools-1.0.0/src/etl/loader/steps/datalake/archive.py +34 -0
  40. etlpy_tools-1.0.0/src/etl/loader/steps/datalake/save.py +38 -0
  41. etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/__init__.py +5 -0
  42. etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/delta.py +42 -0
  43. etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/ensureindexes.py +27 -0
  44. etlpy_tools-1.0.0/src/etl/loader/steps/mongodb/upsert.py +42 -0
  45. etlpy_tools-1.0.0/src/etl/loader/steps/mssql/__init__.py +4 -0
  46. etlpy_tools-1.0.0/src/etl/loader/steps/mssql/ensuretable.py +46 -0
  47. etlpy_tools-1.0.0/src/etl/loader/steps/mssql/insert.py +40 -0
  48. etlpy_tools-1.0.0/src/etl/loader/steps/postgresql/__init__.py +4 -0
  49. etlpy_tools-1.0.0/src/etl/loader/steps/postgresql/ensuretable.py +46 -0
  50. etlpy_tools-1.0.0/src/etl/loader/steps/postgresql/insert.py +34 -0
  51. etlpy_tools-1.0.0/src/etl/loader/steps/sqlite/__init__.py +4 -0
  52. etlpy_tools-1.0.0/src/etl/loader/steps/sqlite/ensuretable.py +39 -0
  53. etlpy_tools-1.0.0/src/etl/loader/steps/sqlite/insert.py +32 -0
  54. etlpy_tools-1.0.0/src/etl/loader/storage.py +207 -0
  55. etlpy_tools-1.0.0/src/etl/transformer/steps/__init__.py +43 -0
  56. etlpy_tools-1.0.0/src/etl/transformer/steps/add_column.py +30 -0
  57. etlpy_tools-1.0.0/src/etl/transformer/steps/aggregate.py +55 -0
  58. etlpy_tools-1.0.0/src/etl/transformer/steps/cast_types.py +22 -0
  59. etlpy_tools-1.0.0/src/etl/transformer/steps/clear_text.py +39 -0
  60. etlpy_tools-1.0.0/src/etl/transformer/steps/drop_columns.py +36 -0
  61. etlpy_tools-1.0.0/src/etl/transformer/steps/drop_duplicates.py +29 -0
  62. etlpy_tools-1.0.0/src/etl/transformer/steps/drop_nulls.py +28 -0
  63. etlpy_tools-1.0.0/src/etl/transformer/steps/extract_entities.py +43 -0
  64. etlpy_tools-1.0.0/src/etl/transformer/steps/fill_nulls.py +29 -0
  65. etlpy_tools-1.0.0/src/etl/transformer/steps/filter_rows.py +22 -0
  66. etlpy_tools-1.0.0/src/etl/transformer/steps/generate_key.py +52 -0
  67. etlpy_tools-1.0.0/src/etl/transformer/steps/join.py +72 -0
  68. etlpy_tools-1.0.0/src/etl/transformer/steps/normalize_numeric.py +36 -0
  69. etlpy_tools-1.0.0/src/etl/transformer/steps/rename_columns.py +22 -0
  70. etlpy_tools-1.0.0/src/etl/transformer/steps/row_hash.py +32 -0
  71. etlpy_tools-1.0.0/src/etl/transformer/steps/slambda.py +22 -0
  72. etlpy_tools-1.0.0/src/etl/transformer/steps/sql.py +33 -0
  73. etlpy_tools-1.0.0/src/etl/transformer/steps/to_schema.py +67 -0
  74. etlpy_tools-1.0.0/src/etl/transformer/steps/trim_string.py +23 -0
  75. etlpy_tools-1.0.0/src/etl/transformer/steps/union.py +34 -0
  76. etlpy_tools-1.0.0/src/etlpy_tools.egg-info/PKG-INFO +311 -0
  77. etlpy_tools-1.0.0/src/etlpy_tools.egg-info/SOURCES.txt +78 -0
  78. etlpy_tools-1.0.0/src/etlpy_tools.egg-info/dependency_links.txt +1 -0
  79. etlpy_tools-1.0.0/src/etlpy_tools.egg-info/requires.txt +26 -0
  80. etlpy_tools-1.0.0/src/etlpy_tools.egg-info/top_level.txt +1 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Vladimir Novotorzhin
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,311 @@
1
+ Metadata-Version: 2.4
2
+ Name: etlpy-tools
3
+ Version: 1.0.0
4
+ Summary: A composable, async, Polars-native ETL toolkit
5
+ Author-email: Vladimir Novotorzhin <include.dos.h@yandex.ru>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/Sque-ak/etlpy
8
+ Project-URL: Repository, https://github.com/Sque-ak/etlpy
9
+ Project-URL: Issues, https://github.com/Sque-ak/etlpy/issues
10
+ Keywords: etl,polars,clickhouse,airflow,data-pipeline
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Programming Language :: Python :: 3.13
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Topic :: Database
15
+ Classifier: Topic :: Software Development :: Libraries
16
+ Requires-Python: >=3.13
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: polars>=1.41.2
20
+ Requires-Dist: polars-hash>=0.6.0
21
+ Provides-Extra: extractor
22
+ Requires-Dist: httpx>=0.28.1; extra == "extractor"
23
+ Requires-Dist: etlpy-tools[loader]; extra == "extractor"
24
+ Provides-Extra: transformer
25
+ Requires-Dist: polars>=1.41.2; extra == "transformer"
26
+ Requires-Dist: pyarrow>=15.0.0; extra == "transformer"
27
+ Provides-Extra: loader
28
+ Requires-Dist: pyarrow>=15.0.0; extra == "loader"
29
+ Provides-Extra: test
30
+ Requires-Dist: pytest>=9.1.0; extra == "test"
31
+ Requires-Dist: pytest-asyncio>=1.4.0; extra == "test"
32
+ Requires-Dist: testcontainers[clickhouse]>=4.14.2; extra == "test"
33
+ Requires-Dist: clickhouse-connect>=1.3.0; extra == "test"
34
+ Requires-Dist: pymongo>=4.9; extra == "test"
35
+ Requires-Dist: aioodbc>=0.5.0; extra == "test"
36
+ Requires-Dist: asyncpg>=0.31.0; extra == "test"
37
+ Requires-Dist: etlpy-tools[all]; extra == "test"
38
+ Provides-Extra: all
39
+ Requires-Dist: etlpy-tools[extractor,loader,transformer]; extra == "all"
40
+ Dynamic: license-file
41
+
42
+ <p align="center">
43
+ <img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/logo.png" alt="etlpy" width="360">
44
+ </p>
45
+
46
+ <p align="center">
47
+ <strong>A composable, async toolkit for building ETL pipelines in Python powered by Polars.</strong>
48
+ </p>
49
+
50
+ <p align="center">
51
+ <img src="https://img.shields.io/badge/python-3.13+-blue.svg" alt="Python 3.13+">
52
+ <img src="https://img.shields.io/badge/engine-Polars-orange.svg" alt="Polars">
53
+ <img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License">
54
+ <img src="https://img.shields.io/badge/status-alpha-yellow.svg" alt="Alpha">
55
+ <img src="https://img.shields.io/badge/coverage-100%25-brightgreen.svg" alt="Coverage 100%">
56
+ <img src="https://img.shields.io/pypi/v/etlpytools.svg" alt="PyPI">
57
+ </p>
58
+
59
+ ---
60
+
61
+ **ETLPY** is a small, unopinionated set of building blocks for extract / transform / load work.
62
+ You compose pipelines from tiny, single-purpose **steps** and run them - in a script or inside an Airflow task. It is a _toolkit_, not a framework: take what you need, write your own
63
+ steps for the rest.
64
+
65
+ - **Pipeline** - Just a list of async `Step`s. Add, remove, reorder.
66
+ - **Polars-native** - one engine end to end (Arrow under the hood).
67
+ - **Async** - steps are `async`, so you can fan out concurrent API/DB calls inside a step.
68
+ - **Many databases** - ClickHouse, PostgreSQL, MS SQL Server, MongoDB and SQLite connectors, plus a parquet data lake. Read from one, load into another.
69
+ - **Schema-guaranteed** - validate and coerce data against a Pydantic model, so downstream gets exactly the types you declared.
70
+ - **Deduplication** - row-hash change detection loads only new or changed rows.
71
+ - **Airflow-friendly** - one pipeline = one task. etlpy moves the data; Airflow orchestrates.
72
+
73
+ ## Installation
74
+
75
+ ```bash
76
+ pip install etlpy-tools # core tools kit
77
+
78
+ # or with the stage extras you need
79
+ pip install "etlpy-tools[all]" # everything
80
+ pip install "etlpy-tools[extractor]" # httpx-based API steps
81
+ pip install "etlpy-tools[transformer]" # Polars transform steps
82
+ pip install "etlpy-tools[loader]" # data lake + DB loaders
83
+ ```
84
+
85
+ > Installed as **`etlpy-tools`**, imported as **`etl`** e.g. `from etl.generic import Pipeline`.
86
+
87
+ Database drivers are **not** bundled install the one you use:
88
+ `clickhouse-connect`, `asyncpg`, `aioodbc`, `pymongo`, or `aiosqlite`.
89
+
90
+ Requires Python 3.13+.
91
+
92
+ ## Quick start
93
+
94
+ A pipeline threads a Polars `DataFrame` through a list of steps:
95
+
96
+ ```python
97
+ import asyncio
98
+ from etl.generic import Pipeline, Pipestart
99
+ from etl.transformer.steps import ClearText, DropDuplicates, GenerateKey, RowHash
100
+ from etl.loader.steps.datalake import Save
101
+
102
+ @Pipestart
103
+ async def clean_customers(raw_df):
104
+ return Pipeline([
105
+ ClearText(),
106
+ DropDuplicates(),
107
+ GenerateKey(columns=["id"], key_name="pk"),
108
+ RowHash(), # content fingerprint for change detection
109
+ Save(name="customers", layer="raw"), # persist to the data lake
110
+ ], dataframe=raw_df)
111
+
112
+
113
+ asyncio.run(clean_customers(raw_df))
114
+ ```
115
+
116
+ Load a lake file into ClickHouse, inserting only what actually changed:
117
+
118
+ ```python
119
+ from etl.generic import Pipeline
120
+ from etl.extractor.steps.datalake import Read
121
+ from etl.loader.steps.clickhouse import EnsureTable, Delta, Insert
122
+ from etl.extractor.steps.clickhouse import Connect
123
+ from etl.loader.steps.datalake import Archive
124
+
125
+ await Pipeline([
126
+ Connect(host="clickhouse", database="analytics"),
127
+ Read(layer="fact", name="transactions"),
128
+ EnsureTable("fact_transactions",
129
+ engine="ReplacingMergeTree(loaded_at)", order_by=["pk"]),
130
+ Delta("fact_transactions", keys=["pk"]), # skip rows already loaded unchanged
131
+ Insert("fact_transactions"),
132
+ Archive(layer="fact", name="transactions"),
133
+ ]).run()
134
+ ```
135
+
136
+ Any step can halt the pipeline gracefully by raising `StopPipeline`. It is a
137
+ control-flow signal, not an error: `run()` stops, skips the remaining steps and
138
+ returns the data produced so far (no traceback).
139
+
140
+ ```python
141
+ from etl.generic import Step, StopPipeline
142
+
143
+ class StopIfEmpty(Step):
144
+ async def apply(self, df, data=None):
145
+ if df is None or df.is_empty():
146
+ raise StopPipeline("no rows") # stop, return df as-is
147
+ return df
148
+ ```
149
+
150
+ Pass `df=` to control what the pipeline returns on stop:
151
+
152
+ ```python
153
+ raise StopPipeline("threshold hit", df=partial)
154
+ ```
155
+
156
+ ## Core concepts
157
+
158
+ | Piece | What it is |
159
+ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
160
+ | **`Pipeline`** | An ordered list of steps. `run()` threads a DataFrame through them and returns the final one. |
161
+ | **`StopPipeline`** | A step may raise StopPipeline to halt the run early |
162
+ | **`Pipestart`** | A decorator that runs a pipeline-returning function. |
163
+ | **`Step`** | A unit of work: `async def apply(self, df, data) -> df`. Write your own by subclassing. |
164
+ | **`Data`** | A shared context (auth tokens, DB clients, config) passed to every step. Mutated by reference; **DataFrames don't live here** - they flow as the threaded `df` or via the lake. |
165
+
166
+ The split is deliberate: the **`DataFrame`** is threaded and returned, while the **`Data`** context is
167
+ shared state. Heavy tables move through the data lake, not through the context - which keeps each
168
+ pipeline a clean fit for a single Airflow task.
169
+
170
+ ## The toolbox
171
+
172
+ Every step is one small class with an `async def apply(self, df, data) -> df`. Below is the full set.
173
+ Missing one? Subclass `Step`, implement `apply`, and drop it into the list.
174
+
175
+ ## Schema guarantee with `ToSchema`
176
+
177
+ Raw data lies: missing fields, wrong types, extra columns, values that don't parse.
178
+ `ToSchema` is the gate that turns messy input into exactly the data you declared. It
179
+ validates and coerces **every row against a Pydantic model** and returns a DataFrame whose
180
+ schema comes from that model - what is not in the model is dropped, what cannot be coerced
181
+ raises. Downstream steps and your warehouse table then get clean, typed data by construction.
182
+
183
+ Declare the canon as a Pydantic model:
184
+
185
+ ```python
186
+ from datetime import date
187
+ from pydantic import BaseModel
188
+
189
+ class Transaction(BaseModel):
190
+ id: int
191
+ amount: float
192
+ currency: str
193
+ txn_date: date
194
+ note: str | None = None # optional -> nullable column
195
+ ```
196
+
197
+ Drop `ToSchema` into a pipeline:
198
+
199
+ ```python
200
+ from etl.transformer.steps import ToSchema, GenerateKey, RowHash
201
+
202
+ Pipeline([
203
+ # ... read raw data ...
204
+ ToSchema(Transaction, mapping={"trans_id": "id", "sum": "amount"}), # rename + validate + type
205
+ GenerateKey(columns=["id"], key_name="pk"),
206
+ RowHash(),
207
+ # ... load ...
208
+ ])
209
+ ```
210
+
211
+ What the step guarantees:
212
+
213
+ - **Exact schema & types** - columns and dtypes come from the model, not from guessing on the
214
+ first N rows: `int` -> `Int64`, `date` -> `Date`, `str | None` -> nullable `String`, and so on.
215
+ - **Extra columns dropped** - anything outside the model is discarded.
216
+ - **Fail fast** - a value that can't be coerced (or a missing required field) raises, so bad
217
+ upstream data never silently reaches the warehouse.
218
+ - **`mapping`** - rename raw source fields to the canonical names (1:1) before validation.
219
+
220
+ `ToSchema` never imports pydantic itself - it only uses the model you pass, so **pydantic
221
+ stays an optional, user-side dependency**. It also works with
222
+ [Patito](https://github.com/JakobGM/patito) models (Pydantic on Polars). Validation runs per
223
+ row - thorough rather than fast; correctness is the point, speed is recovered downstream.
224
+
225
+ ### Extract
226
+
227
+ - **`OAuthenticate(url, credentials, fields=OAuthFields(...), send="json", method="POST", auth_header=None, headers=None, store="auth", timeout=60.0)`** - non-interactive OAuth 2.0 token flow (client-credentials). Sends `credentials` (JSON or form via `send`), pulls the token out of the response by dotted paths (`fields`), applies `auth_header` (e.g. `{"Authorization": "Bearer {token}"}`, `{token}` is filled in) to the shared httpx client, and stores the parsed auth in `data["auth"]`. Request-level headers (gateway API keys, `Content-Type`) go in `headers`.
228
+ - **`AuthenticateBasic(user, password, headers=None, timeout=60.0)`** _(HTTP Basic, RFC 7617)_ - sets `httpx.BasicAuth(user, password)` on the shared client so every downstream request carries `Authorization: Basic ...`. No token exchange - the header is static.
229
+ - **`Read(layer, name)`** _(data lake)_ - read `{name}.parquet` from a lake layer into the pipeline df.
230
+ - **`Read(query)`** _(ClickHouse)_ - run a SQL query (client from `data["ch"]`) and return the result as a Polars frame.
231
+ - **`Reads(layer, name, mode=STATIC, date_from=None, date_to=None, pattern="*.parquet", missing_ok=False)`** _(data lake)_ - read and concat every parquet in a lake folder, with an optional inclusive date range on the file name.
232
+ - **`Connect(host, port=8123, database, username, password, secure=False)`** _(ClickHouse)_ - open a clickhouse-connect client (extra kwargs are forwarded) and store it in `data["ch"]`.
233
+
234
+ > **`OAuthFields`** maps where each token-response field lives, e.g.
235
+ > `OAuthFields(access_token="data.access_token", expires_in="data.expires_in")`.
236
+ > Set a field to `None` if the API does not return it.
237
+
238
+ ### Transform
239
+
240
+ - **`AddColumn(column_name, value)`** - add a constant-value column.
241
+ - **`CastTypes({column: dtype})`** - cast columns to Polars dtypes (pin types before loading).
242
+ - **`RenameColumns({old: new})`** - rename columns.
243
+ - **`DropColumns(columns, exclude=False)`** - drop the listed columns; `exclude=True` keeps only them.
244
+ - **`DropDuplicates(subset=None)`** - drop duplicate rows (by `subset`, or all columns).
245
+ - **`DropNulls(subset=None)`** - drop rows with a null in any of `subset`.
246
+ - **`FillNulls(value, columns=None)`** - fill nulls; `value` is a scalar or a `{column: value}` dict.
247
+ - **`FilterRows(expr)`** - keep rows matching a Polars boolean expression, e.g. `pl.col("amount") > 0`.
248
+ - **`TrimString(columns=None)`** - strip whitespace from string columns (`None` / `"*"` = all string columns).
249
+ - **`ClearText(columns="*")`** - clean text: collapse newlines to spaces, drop quotes/backslashes, trim.
250
+ - **`NormalizeNumeric(columns, method="minmax")`** - scale numeric columns (`"minmax"` or `"zscore"`).
251
+ - **`GenerateKey(columns, key_name="pk", mode="hash")`** - build a key column. `mode`: `"hash"` (SHA-256 hex), `"hash_int"` (stable 64-bit int), `"sequential"` (1, 2, 3, ...).
252
+ - **`RowHash(exclude=None, separator="||")`** - SHA-256 fingerprint of every row value (minus `row_hash`, `loaded_at`, and `exclude`) written to `row_hash`, for change detection.
253
+ - **`Aggregate(group_by, aggregations)`** - group + aggregate, e.g. `{"amount": ["sum", "mean"]}`; output columns are `{column}_{func}`.
254
+ - **`Join(other, on, how="inner", select=None, prefix=None)`** - join with another frame **or a sub-`Pipeline`** (run to produce the right side); `select` / `prefix` shape the right columns.
255
+ - **`ExtractEntities(sources, defaults=None)`** - stack several column groups into one long table (e.g. sender/receiver columns into a single `party` table).
256
+ - **`ToSchema(model, mapping=None)`** - validate and coerce the df against a Pydantic model; output has the model's exact schema (see [Schema guarantee](#schema-guarantee-with-toschema)).
257
+ - **`Union(other, how="vertical")`** - stack another DataFrame **or a sub-`Pipeline`** onto the df (`vertical` / `diagonal` / `horizontal`).
258
+ - **`SQL(query, view_name="source")`** - run a Polars-SQL query over the df (registered as `view_name`); `SQL.from_file(path)` loads the query from a `.sql` file.
259
+ - **`Lambda(func)`** - apply an arbitrary `df -> df` callable (escape hatch for one-off logic).
260
+
261
+ ### Load
262
+
263
+ - **`EnsureTable(table, order_by, engine="MergeTree", partition_by=None, if_exists="append")`** - create `table` from the df schema if it does not exist. `order_by` is required (it is the primary / dedup key); `if_exists="error"` raises when the table already exists.
264
+ - **`Delta(table, keys)`** - keep only new or changed rows by comparing `row_hash` against the table (needs a `ReplacingMergeTree`-family engine). The core of idempotent loads.
265
+ - **`Insert(table)`** - insert the df into a ClickHouse table via Arrow (an empty / `None` frame is skipped).
266
+ - **`Save(name, layer="raw")`** _(data lake)_ - write the df to a lake layer as `{name}.parquet`, overwriting in place.
267
+ - **`Optimize(table)`** _(ClickHouse)_ - `OPTIMIZE TABLE ... FINAL` to collapse ReplacingMergeTree duplicates (expensive; run after a load, not per row).
268
+ - **`Archive(layer, name)`** _(data lake)_ - move a lake file into the `archive` layer (e.g. after a successful load).
269
+
270
+ ## Connectors
271
+
272
+ etlpy ships steps for several databases. **Drivers are not bundled** - there are many
273
+ databases, so you install only the one you need. Each `Connect` opens a client and stores it
274
+ in `data[...]`; the read/load steps pick it up from there.
275
+
276
+ | Database | Steps | `data` key | Driver (install yourself) |
277
+ | ----------------- | --------------------------------------------------------------- | ---------------- | ------------------------- |
278
+ | **ClickHouse** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Insert`, `Optimize` | `data["ch"]` | `clickhouse-connect` |
279
+ | **PostgreSQL** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["pg"]` | `asyncpg` |
280
+ | **MS SQL Server** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["mssql"]` | `aioodbc` (+ ODBC driver) |
281
+ | **MongoDB** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Upsert` | `data["mongo"]` | `pymongo` (async) |
282
+ | **SQLite** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["sqlite"]` | `aiosqlite` |
283
+
284
+ Import `Connect`/`Read` from `etl.extractor.steps.<db>` and the write steps from
285
+ `etl.loader.steps.<db>`. SQL reads are parameterized (injection-safe): ClickHouse
286
+ `{name:Type}`, Postgres `$1`, MS SQL / SQLite `?`; MongoDB uses a query dict.
287
+
288
+ > **SQLite** is meant for tests and small local datasets (great as an in-memory `:memory:`
289
+ > backend). It is a single-file, single-writer engine and is **not suitable for storing large
290
+ > data** - use ClickHouse or Postgres for that.
291
+
292
+ ## Testing
293
+
294
+ Every step has its own unit tests, and the whole extract / transform / load chain is exercised end to
295
+ end against a real ClickHouse (spun up in Docker via `testcontainers`). A complete pipeline example
296
+ lives in [`generic/tests/test_pipeline.py`](src/etl/generic/tests/test_pipeline.py).
297
+
298
+ ```bash
299
+ pip install -e ".[all,test]"
300
+ pytest -q
301
+ ```
302
+
303
+ <p align="center"> <img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/100cov.png" alt="100% coverage" width="640"> </p>
304
+
305
+ ## Status
306
+
307
+ Alpha, under active development. A PyPI release is planned.
308
+
309
+ ## License
310
+
311
+ Released under the [MIT License](LICENSE).
@@ -0,0 +1,270 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/logo.png" alt="etlpy" width="360">
3
+ </p>
4
+
5
+ <p align="center">
6
+ <strong>A composable, async toolkit for building ETL pipelines in Python powered by Polars.</strong>
7
+ </p>
8
+
9
+ <p align="center">
10
+ <img src="https://img.shields.io/badge/python-3.13+-blue.svg" alt="Python 3.13+">
11
+ <img src="https://img.shields.io/badge/engine-Polars-orange.svg" alt="Polars">
12
+ <img src="https://img.shields.io/badge/license-MIT-green.svg" alt="MIT License">
13
+ <img src="https://img.shields.io/badge/status-alpha-yellow.svg" alt="Alpha">
14
+ <img src="https://img.shields.io/badge/coverage-100%25-brightgreen.svg" alt="Coverage 100%">
15
+ <img src="https://img.shields.io/pypi/v/etlpytools.svg" alt="PyPI">
16
+ </p>
17
+
18
+ ---
19
+
20
+ **ETLPY** is a small, unopinionated set of building blocks for extract / transform / load work.
21
+ You compose pipelines from tiny, single-purpose **steps** and run them - in a script or inside an Airflow task. It is a _toolkit_, not a framework: take what you need, write your own
22
+ steps for the rest.
23
+
24
+ - **Pipeline** - Just a list of async `Step`s. Add, remove, reorder.
25
+ - **Polars-native** - one engine end to end (Arrow under the hood).
26
+ - **Async** - steps are `async`, so you can fan out concurrent API/DB calls inside a step.
27
+ - **Many databases** - ClickHouse, PostgreSQL, MS SQL Server, MongoDB and SQLite connectors, plus a parquet data lake. Read from one, load into another.
28
+ - **Schema-guaranteed** - validate and coerce data against a Pydantic model, so downstream gets exactly the types you declared.
29
+ - **Deduplication** - row-hash change detection loads only new or changed rows.
30
+ - **Airflow-friendly** - one pipeline = one task. etlpy moves the data; Airflow orchestrates.
31
+
32
+ ## Installation
33
+
34
+ ```bash
35
+ pip install etlpy-tools # core tools kit
36
+
37
+ # or with the stage extras you need
38
+ pip install "etlpy-tools[all]" # everything
39
+ pip install "etlpy-tools[extractor]" # httpx-based API steps
40
+ pip install "etlpy-tools[transformer]" # Polars transform steps
41
+ pip install "etlpy-tools[loader]" # data lake + DB loaders
42
+ ```
43
+
44
+ > Installed as **`etlpy-tools`**, imported as **`etl`** e.g. `from etl.generic import Pipeline`.
45
+
46
+ Database drivers are **not** bundled install the one you use:
47
+ `clickhouse-connect`, `asyncpg`, `aioodbc`, `pymongo`, or `aiosqlite`.
48
+
49
+ Requires Python 3.13+.
50
+
51
+ ## Quick start
52
+
53
+ A pipeline threads a Polars `DataFrame` through a list of steps:
54
+
55
+ ```python
56
+ import asyncio
57
+ from etl.generic import Pipeline, Pipestart
58
+ from etl.transformer.steps import ClearText, DropDuplicates, GenerateKey, RowHash
59
+ from etl.loader.steps.datalake import Save
60
+
61
+ @Pipestart
62
+ async def clean_customers(raw_df):
63
+ return Pipeline([
64
+ ClearText(),
65
+ DropDuplicates(),
66
+ GenerateKey(columns=["id"], key_name="pk"),
67
+ RowHash(), # content fingerprint for change detection
68
+ Save(name="customers", layer="raw"), # persist to the data lake
69
+ ], dataframe=raw_df)
70
+
71
+
72
+ asyncio.run(clean_customers(raw_df))
73
+ ```
74
+
75
+ Load a lake file into ClickHouse, inserting only what actually changed:
76
+
77
+ ```python
78
+ from etl.generic import Pipeline
79
+ from etl.extractor.steps.datalake import Read
80
+ from etl.loader.steps.clickhouse import EnsureTable, Delta, Insert
81
+ from etl.extractor.steps.clickhouse import Connect
82
+ from etl.loader.steps.datalake import Archive
83
+
84
+ await Pipeline([
85
+ Connect(host="clickhouse", database="analytics"),
86
+ Read(layer="fact", name="transactions"),
87
+ EnsureTable("fact_transactions",
88
+ engine="ReplacingMergeTree(loaded_at)", order_by=["pk"]),
89
+ Delta("fact_transactions", keys=["pk"]), # skip rows already loaded unchanged
90
+ Insert("fact_transactions"),
91
+ Archive(layer="fact", name="transactions"),
92
+ ]).run()
93
+ ```
94
+
95
+ Any step can halt the pipeline gracefully by raising `StopPipeline`. It is a
96
+ control-flow signal, not an error: `run()` stops, skips the remaining steps and
97
+ returns the data produced so far (no traceback).
98
+
99
+ ```python
100
+ from etl.generic import Step, StopPipeline
101
+
102
+ class StopIfEmpty(Step):
103
+ async def apply(self, df, data=None):
104
+ if df is None or df.is_empty():
105
+ raise StopPipeline("no rows") # stop, return df as-is
106
+ return df
107
+ ```
108
+
109
+ Pass `df=` to control what the pipeline returns on stop:
110
+
111
+ ```python
112
+ raise StopPipeline("threshold hit", df=partial)
113
+ ```
114
+
115
+ ## Core concepts
116
+
117
+ | Piece | What it is |
118
+ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
119
+ | **`Pipeline`** | An ordered list of steps. `run()` threads a DataFrame through them and returns the final one. |
120
+ | **`StopPipeline`** | A step may raise StopPipeline to halt the run early |
121
+ | **`Pipestart`** | A decorator that runs a pipeline-returning function. |
122
+ | **`Step`** | A unit of work: `async def apply(self, df, data) -> df`. Write your own by subclassing. |
123
+ | **`Data`** | A shared context (auth tokens, DB clients, config) passed to every step. Mutated by reference; **DataFrames don't live here** - they flow as the threaded `df` or via the lake. |
124
+
125
+ The split is deliberate: the **`DataFrame`** is threaded and returned, while the **`Data`** context is
126
+ shared state. Heavy tables move through the data lake, not through the context - which keeps each
127
+ pipeline a clean fit for a single Airflow task.
128
+
129
+ ## The toolbox
130
+
131
+ Every step is one small class with an `async def apply(self, df, data) -> df`. Below is the full set.
132
+ Missing one? Subclass `Step`, implement `apply`, and drop it into the list.
133
+
134
+ ## Schema guarantee with `ToSchema`
135
+
136
+ Raw data lies: missing fields, wrong types, extra columns, values that don't parse.
137
+ `ToSchema` is the gate that turns messy input into exactly the data you declared. It
138
+ validates and coerces **every row against a Pydantic model** and returns a DataFrame whose
139
+ schema comes from that model - what is not in the model is dropped, what cannot be coerced
140
+ raises. Downstream steps and your warehouse table then get clean, typed data by construction.
141
+
142
+ Declare the canon as a Pydantic model:
143
+
144
+ ```python
145
+ from datetime import date
146
+ from pydantic import BaseModel
147
+
148
+ class Transaction(BaseModel):
149
+ id: int
150
+ amount: float
151
+ currency: str
152
+ txn_date: date
153
+ note: str | None = None # optional -> nullable column
154
+ ```
155
+
156
+ Drop `ToSchema` into a pipeline:
157
+
158
+ ```python
159
+ from etl.transformer.steps import ToSchema, GenerateKey, RowHash
160
+
161
+ Pipeline([
162
+ # ... read raw data ...
163
+ ToSchema(Transaction, mapping={"trans_id": "id", "sum": "amount"}), # rename + validate + type
164
+ GenerateKey(columns=["id"], key_name="pk"),
165
+ RowHash(),
166
+ # ... load ...
167
+ ])
168
+ ```
169
+
170
+ What the step guarantees:
171
+
172
+ - **Exact schema & types** - columns and dtypes come from the model, not from guessing on the
173
+ first N rows: `int` -> `Int64`, `date` -> `Date`, `str | None` -> nullable `String`, and so on.
174
+ - **Extra columns dropped** - anything outside the model is discarded.
175
+ - **Fail fast** - a value that can't be coerced (or a missing required field) raises, so bad
176
+ upstream data never silently reaches the warehouse.
177
+ - **`mapping`** - rename raw source fields to the canonical names (1:1) before validation.
178
+
179
+ `ToSchema` never imports pydantic itself - it only uses the model you pass, so **pydantic
180
+ stays an optional, user-side dependency**. It also works with
181
+ [Patito](https://github.com/JakobGM/patito) models (Pydantic on Polars). Validation runs per
182
+ row - thorough rather than fast; correctness is the point, speed is recovered downstream.
183
+
184
+ ### Extract
185
+
186
+ - **`OAuthenticate(url, credentials, fields=OAuthFields(...), send="json", method="POST", auth_header=None, headers=None, store="auth", timeout=60.0)`** - non-interactive OAuth 2.0 token flow (client-credentials). Sends `credentials` (JSON or form via `send`), pulls the token out of the response by dotted paths (`fields`), applies `auth_header` (e.g. `{"Authorization": "Bearer {token}"}`, `{token}` is filled in) to the shared httpx client, and stores the parsed auth in `data["auth"]`. Request-level headers (gateway API keys, `Content-Type`) go in `headers`.
187
+ - **`AuthenticateBasic(user, password, headers=None, timeout=60.0)`** _(HTTP Basic, RFC 7617)_ - sets `httpx.BasicAuth(user, password)` on the shared client so every downstream request carries `Authorization: Basic ...`. No token exchange - the header is static.
188
+ - **`Read(layer, name)`** _(data lake)_ - read `{name}.parquet` from a lake layer into the pipeline df.
189
+ - **`Read(query)`** _(ClickHouse)_ - run a SQL query (client from `data["ch"]`) and return the result as a Polars frame.
190
+ - **`Reads(layer, name, mode=STATIC, date_from=None, date_to=None, pattern="*.parquet", missing_ok=False)`** _(data lake)_ - read and concat every parquet in a lake folder, with an optional inclusive date range on the file name.
191
+ - **`Connect(host, port=8123, database, username, password, secure=False)`** _(ClickHouse)_ - open a clickhouse-connect client (extra kwargs are forwarded) and store it in `data["ch"]`.
192
+
193
+ > **`OAuthFields`** maps where each token-response field lives, e.g.
194
+ > `OAuthFields(access_token="data.access_token", expires_in="data.expires_in")`.
195
+ > Set a field to `None` if the API does not return it.
196
+
197
+ ### Transform
198
+
199
+ - **`AddColumn(column_name, value)`** - add a constant-value column.
200
+ - **`CastTypes({column: dtype})`** - cast columns to Polars dtypes (pin types before loading).
201
+ - **`RenameColumns({old: new})`** - rename columns.
202
+ - **`DropColumns(columns, exclude=False)`** - drop the listed columns; `exclude=True` keeps only them.
203
+ - **`DropDuplicates(subset=None)`** - drop duplicate rows (by `subset`, or all columns).
204
+ - **`DropNulls(subset=None)`** - drop rows with a null in any of `subset`.
205
+ - **`FillNulls(value, columns=None)`** - fill nulls; `value` is a scalar or a `{column: value}` dict.
206
+ - **`FilterRows(expr)`** - keep rows matching a Polars boolean expression, e.g. `pl.col("amount") > 0`.
207
+ - **`TrimString(columns=None)`** - strip whitespace from string columns (`None` / `"*"` = all string columns).
208
+ - **`ClearText(columns="*")`** - clean text: collapse newlines to spaces, drop quotes/backslashes, trim.
209
+ - **`NormalizeNumeric(columns, method="minmax")`** - scale numeric columns (`"minmax"` or `"zscore"`).
210
+ - **`GenerateKey(columns, key_name="pk", mode="hash")`** - build a key column. `mode`: `"hash"` (SHA-256 hex), `"hash_int"` (stable 64-bit int), `"sequential"` (1, 2, 3, ...).
211
+ - **`RowHash(exclude=None, separator="||")`** - SHA-256 fingerprint of every row value (minus `row_hash`, `loaded_at`, and `exclude`) written to `row_hash`, for change detection.
212
+ - **`Aggregate(group_by, aggregations)`** - group + aggregate, e.g. `{"amount": ["sum", "mean"]}`; output columns are `{column}_{func}`.
213
+ - **`Join(other, on, how="inner", select=None, prefix=None)`** - join with another frame **or a sub-`Pipeline`** (run to produce the right side); `select` / `prefix` shape the right columns.
214
+ - **`ExtractEntities(sources, defaults=None)`** - stack several column groups into one long table (e.g. sender/receiver columns into a single `party` table).
215
+ - **`ToSchema(model, mapping=None)`** - validate and coerce the df against a Pydantic model; output has the model's exact schema (see [Schema guarantee](#schema-guarantee-with-toschema)).
216
+ - **`Union(other, how="vertical")`** - stack another DataFrame **or a sub-`Pipeline`** onto the df (`vertical` / `diagonal` / `horizontal`).
217
+ - **`SQL(query, view_name="source")`** - run a Polars-SQL query over the df (registered as `view_name`); `SQL.from_file(path)` loads the query from a `.sql` file.
218
+ - **`Lambda(func)`** - apply an arbitrary `df -> df` callable (escape hatch for one-off logic).
219
+
220
+ ### Load
221
+
222
+ - **`EnsureTable(table, order_by, engine="MergeTree", partition_by=None, if_exists="append")`** - create `table` from the df schema if it does not exist. `order_by` is required (it is the primary / dedup key); `if_exists="error"` raises when the table already exists.
223
+ - **`Delta(table, keys)`** - keep only new or changed rows by comparing `row_hash` against the table (needs a `ReplacingMergeTree`-family engine). The core of idempotent loads.
224
+ - **`Insert(table)`** - insert the df into a ClickHouse table via Arrow (an empty / `None` frame is skipped).
225
+ - **`Save(name, layer="raw")`** _(data lake)_ - write the df to a lake layer as `{name}.parquet`, overwriting in place.
226
+ - **`Optimize(table)`** _(ClickHouse)_ - `OPTIMIZE TABLE ... FINAL` to collapse ReplacingMergeTree duplicates (expensive; run after a load, not per row).
227
+ - **`Archive(layer, name)`** _(data lake)_ - move a lake file into the `archive` layer (e.g. after a successful load).
228
+
229
+ ## Connectors
230
+
231
+ etlpy ships steps for several databases. **Drivers are not bundled** - there are many
232
+ databases, so you install only the one you need. Each `Connect` opens a client and stores it
233
+ in `data[...]`; the read/load steps pick it up from there.
234
+
235
+ | Database | Steps | `data` key | Driver (install yourself) |
236
+ | ----------------- | --------------------------------------------------------------- | ---------------- | ------------------------- |
237
+ | **ClickHouse** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Insert`, `Optimize` | `data["ch"]` | `clickhouse-connect` |
238
+ | **PostgreSQL** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["pg"]` | `asyncpg` |
239
+ | **MS SQL Server** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["mssql"]` | `aioodbc` (+ ODBC driver) |
240
+ | **MongoDB** | `Connect`, `Read`, `EnsureTable`, `Delta`, `Upsert` | `data["mongo"]` | `pymongo` (async) |
241
+ | **SQLite** | `Connect`, `Read`, `EnsureTable`, `Insert` | `data["sqlite"]` | `aiosqlite` |
242
+
243
+ Import `Connect`/`Read` from `etl.extractor.steps.<db>` and the write steps from
244
+ `etl.loader.steps.<db>`. SQL reads are parameterized (injection-safe): ClickHouse
245
+ `{name:Type}`, Postgres `$1`, MS SQL / SQLite `?`; MongoDB uses a query dict.
246
+
247
+ > **SQLite** is meant for tests and small local datasets (great as an in-memory `:memory:`
248
+ > backend). It is a single-file, single-writer engine and is **not suitable for storing large
249
+ > data** - use ClickHouse or Postgres for that.
250
+
251
+ ## Testing
252
+
253
+ Every step has its own unit tests, and the whole extract / transform / load chain is exercised end to
254
+ end against a real ClickHouse (spun up in Docker via `testcontainers`). A complete pipeline example
255
+ lives in [`generic/tests/test_pipeline.py`](src/etl/generic/tests/test_pipeline.py).
256
+
257
+ ```bash
258
+ pip install -e ".[all,test]"
259
+ pytest -q
260
+ ```
261
+
262
+ <p align="center"> <img src="https://raw.githubusercontent.com/Sque-ak/etlpy/main/resources/100cov.png" alt="100% coverage" width="640"> </p>
263
+
264
+ ## Status
265
+
266
+ Alpha, under active development. A PyPI release is planned.
267
+
268
+ ## License
269
+
270
+ Released under the [MIT License](LICENSE).