etlpy-tools 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etl/__init__.py +0 -0
- etl/extractor/steps/api/__init__.py +4 -0
- etl/extractor/steps/api/basic_authenticate.py +20 -0
- etl/extractor/steps/api/oauthenticate.py +107 -0
- etl/extractor/steps/clickhouse/__init__.py +4 -0
- etl/extractor/steps/clickhouse/connect.py +28 -0
- etl/extractor/steps/clickhouse/read.py +34 -0
- etl/extractor/steps/datalake/__init__.py +4 -0
- etl/extractor/steps/datalake/read.py +35 -0
- etl/extractor/steps/datalake/reads.py +51 -0
- etl/extractor/steps/mongodb/__init__.py +4 -0
- etl/extractor/steps/mongodb/connect.py +28 -0
- etl/extractor/steps/mongodb/read.py +67 -0
- etl/extractor/steps/mssql/__init__.py +4 -0
- etl/extractor/steps/mssql/connect.py +29 -0
- etl/extractor/steps/mssql/read.py +33 -0
- etl/extractor/steps/postgresql/__init__.py +4 -0
- etl/extractor/steps/postgresql/connect.py +24 -0
- etl/extractor/steps/postgresql/read.py +31 -0
- etl/extractor/steps/sqlite/__init__.py +4 -0
- etl/extractor/steps/sqlite/connect.py +27 -0
- etl/extractor/steps/sqlite/read.py +25 -0
- etl/generic/__init__.py +5 -0
- etl/generic/context.py +36 -0
- etl/generic/pipeline.py +61 -0
- etl/generic/step.py +23 -0
- etl/loader/__init__.py +3 -0
- etl/loader/steps/clickhouse/__init__.py +6 -0
- etl/loader/steps/clickhouse/delta.py +52 -0
- etl/loader/steps/clickhouse/ensuretable.py +83 -0
- etl/loader/steps/clickhouse/insert.py +25 -0
- etl/loader/steps/clickhouse/optimize.py +22 -0
- etl/loader/steps/datalake/__init__.py +4 -0
- etl/loader/steps/datalake/archive.py +34 -0
- etl/loader/steps/datalake/save.py +38 -0
- etl/loader/steps/mongodb/__init__.py +5 -0
- etl/loader/steps/mongodb/delta.py +42 -0
- etl/loader/steps/mongodb/ensureindexes.py +27 -0
- etl/loader/steps/mongodb/upsert.py +42 -0
- etl/loader/steps/mssql/__init__.py +4 -0
- etl/loader/steps/mssql/ensuretable.py +46 -0
- etl/loader/steps/mssql/insert.py +40 -0
- etl/loader/steps/postgresql/__init__.py +4 -0
- etl/loader/steps/postgresql/ensuretable.py +46 -0
- etl/loader/steps/postgresql/insert.py +34 -0
- etl/loader/steps/sqlite/__init__.py +4 -0
- etl/loader/steps/sqlite/ensuretable.py +39 -0
- etl/loader/steps/sqlite/insert.py +32 -0
- etl/loader/storage.py +207 -0
- etl/transformer/steps/__init__.py +43 -0
- etl/transformer/steps/add_column.py +30 -0
- etl/transformer/steps/aggregate.py +55 -0
- etl/transformer/steps/cast_types.py +22 -0
- etl/transformer/steps/clear_text.py +39 -0
- etl/transformer/steps/drop_columns.py +36 -0
- etl/transformer/steps/drop_duplicates.py +29 -0
- etl/transformer/steps/drop_nulls.py +28 -0
- etl/transformer/steps/extract_entities.py +43 -0
- etl/transformer/steps/fill_nulls.py +29 -0
- etl/transformer/steps/filter_rows.py +22 -0
- etl/transformer/steps/generate_key.py +52 -0
- etl/transformer/steps/join.py +72 -0
- etl/transformer/steps/normalize_numeric.py +36 -0
- etl/transformer/steps/rename_columns.py +22 -0
- etl/transformer/steps/row_hash.py +32 -0
- etl/transformer/steps/slambda.py +22 -0
- etl/transformer/steps/sql.py +33 -0
- etl/transformer/steps/to_schema.py +67 -0
- etl/transformer/steps/trim_string.py +23 -0
- etl/transformer/steps/union.py +34 -0
- etlpy_tools-1.0.0.dist-info/METADATA +311 -0
- etlpy_tools-1.0.0.dist-info/RECORD +75 -0
- etlpy_tools-1.0.0.dist-info/WHEEL +5 -0
- etlpy_tools-1.0.0.dist-info/licenses/LICENSE +21 -0
- etlpy_tools-1.0.0.dist-info/top_level.txt +1 -0
etl/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import httpx
|
|
2
|
+
from etl.generic import Step, Data
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class AuthenticateBasic(Step):
|
|
6
|
+
"""
|
|
7
|
+
HTTP Basic (RFC 7617): token is empty, header is static.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
def __init__(self, user: str, password: str, headers: dict | None = None, timeout: float = 60.0):
|
|
11
|
+
self.user, self.password, self.timeout = user, password, timeout
|
|
12
|
+
self.headers = headers or {}
|
|
13
|
+
|
|
14
|
+
async def apply(self, df, data: Data = None):
|
|
15
|
+
client = data.get("client") or httpx.AsyncClient(timeout=httpx.Timeout(self.timeout))
|
|
16
|
+
client.auth = httpx.BasicAuth(self.user, self.password) # base64 (user:password)
|
|
17
|
+
client.headers.update(self.headers)
|
|
18
|
+
|
|
19
|
+
data["client"] = client
|
|
20
|
+
return df
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass, asdict
|
|
3
|
+
import httpx
|
|
4
|
+
|
|
5
|
+
from etl.generic.step import Step
|
|
6
|
+
from etl.generic.context import Data
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class OAuthFields:
|
|
10
|
+
"""
|
|
11
|
+
Map fields with the Authenticate step can works.
|
|
12
|
+
|
|
13
|
+
Where each OAuth 2.0 token-response field lives (RFC 6749 section 5.1).
|
|
14
|
+
Value = dotted path in th response; None = The API does not return it.
|
|
15
|
+
|
|
16
|
+
OAuthFields(
|
|
17
|
+
access_token="data.access_token",
|
|
18
|
+
expires_in="data.expires_in",
|
|
19
|
+
refresh_token="data.refresh_token",
|
|
20
|
+
)
|
|
21
|
+
"""
|
|
22
|
+
access_token: str = "data.access_token"
|
|
23
|
+
token_type: str | None = None
|
|
24
|
+
expires_in: str | None = "data.expires_in"
|
|
25
|
+
refresh_token: str | None = "data.refresh_token"
|
|
26
|
+
scope: str | None = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class OAuthenticate(Step):
|
|
30
|
+
"""
|
|
31
|
+
Generic API auth: send credentials, extract a token, configure the shared
|
|
32
|
+
client and stash the auth in data.
|
|
33
|
+
|
|
34
|
+
Implements the non-interactive OAuth 2.0 token-endpoint flows, primarily the
|
|
35
|
+
Client Credentials grant (RFC 6749 section 4.4). The access token is applied
|
|
36
|
+
per RFC 6750 (Bearer). Client authentication via request body or HTTP Basic
|
|
37
|
+
(RFC 7617) is selectable through 'send'.
|
|
38
|
+
|
|
39
|
+
- RFC 6749 OAuth 2.0 Authorization Framework https://www.rfc-editor.org/info/rfc6749/
|
|
40
|
+
- RFC 6750 OAuth 2.0 Bearer Token Usage https://www.rfc-editor.org/info/rfc6750/
|
|
41
|
+
- RFC 7617 HTTP Basic Authentication https://www.rfc-editor.org/info/rfc7617/
|
|
42
|
+
- RFC 9700 OAuth 2.0 Security BCP https://www.rfc-editor.org/info/rfc9700/
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
url: str,
|
|
48
|
+
credentials: dict,
|
|
49
|
+
fields: OAuthFields = None,
|
|
50
|
+
send="json",
|
|
51
|
+
method="POST",
|
|
52
|
+
auth_header=None,
|
|
53
|
+
headers=None,
|
|
54
|
+
store="auth",
|
|
55
|
+
timeout=60.0
|
|
56
|
+
):
|
|
57
|
+
self.url = url
|
|
58
|
+
self.credentials = credentials
|
|
59
|
+
self.fields = fields or OAuthFields()
|
|
60
|
+
self.send = send
|
|
61
|
+
self.method = method
|
|
62
|
+
self.headers=headers or {"Content-Type": "application/json"}
|
|
63
|
+
self.auth_header = auth_header or {"Authorization": "Bearer {token}"}
|
|
64
|
+
# auth header for client it can be:
|
|
65
|
+
# {"X-Auth-Token": "{token}"}
|
|
66
|
+
# {"Authorization": "{token}"}
|
|
67
|
+
# for headers to https/http response use headers.
|
|
68
|
+
self.store = store
|
|
69
|
+
self.timeout = timeout
|
|
70
|
+
|
|
71
|
+
async def apply(self, df, data: Data):
|
|
72
|
+
client = data.get("client") or httpx.AsyncClient(timeout=httpx.Timeout(self.timeout))
|
|
73
|
+
response = await client.request(self.method, self.url, headers=self.headers, **{self.send:self.credentials})
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
response.raise_for_status()
|
|
77
|
+
except httpx.HTTPStatusError as error:
|
|
78
|
+
raise httpx.HTTPStatusError(
|
|
79
|
+
f"{error}\nResponse body: {response.text}",
|
|
80
|
+
request=error.request, response=error.response
|
|
81
|
+
) from error
|
|
82
|
+
|
|
83
|
+
body = response.json()
|
|
84
|
+
auth = {}
|
|
85
|
+
for name, path in asdict(self.fields).items():
|
|
86
|
+
if path is None: # API does not return this field
|
|
87
|
+
print(f"API does not return this field: {name}")
|
|
88
|
+
continue
|
|
89
|
+
auth[name] = self._get(body, path, required=(name == "access_token"))
|
|
90
|
+
|
|
91
|
+
token = auth["access_token"]
|
|
92
|
+
headers = {key: value.format(token=token) for key, value in self.auth_header.items()}
|
|
93
|
+
client.headers.update(headers) # downstream uses data["client"]
|
|
94
|
+
|
|
95
|
+
data["client"] = client
|
|
96
|
+
data[self.store] = auth
|
|
97
|
+
return df
|
|
98
|
+
|
|
99
|
+
def _get(self, body, path, required=True):
|
|
100
|
+
obj = body
|
|
101
|
+
for key in path.split("."):
|
|
102
|
+
if not isinstance(obj, dict) or key not in obj:
|
|
103
|
+
if required:
|
|
104
|
+
raise KeyError(f"path '{path}' not found in auth response")
|
|
105
|
+
return None
|
|
106
|
+
obj = obj[key]
|
|
107
|
+
return obj
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from etl.generic.step import Step
|
|
3
|
+
from polars import DataFrame
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Connect(Step):
|
|
7
|
+
"""
|
|
8
|
+
Open a ClickHouse connection and put the client into data["ch"] for
|
|
9
|
+
downstream steps (EnsureTable, Delta, Insert).
|
|
10
|
+
|
|
11
|
+
Usually the first step of a load pipeline.
|
|
12
|
+
|
|
13
|
+
:param host/port/database/username/password/secure: clickhouse-connect params.
|
|
14
|
+
:param kwargs: extra arguments forwarded to clickhouse_connect.get_client.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, host="localhost", port=8123, database="default",
|
|
18
|
+
username="default", password="", secure=False, **kwargs):
|
|
19
|
+
self.config = dict(host=host, port=port, database=database,
|
|
20
|
+
username=username, password=password, secure=secure, **kwargs)
|
|
21
|
+
|
|
22
|
+
async def apply(self, df: DataFrame, data=None):
|
|
23
|
+
import clickhouse_connect
|
|
24
|
+
data["ch"] = await asyncio.to_thread(clickhouse_connect.get_client, **self.config)
|
|
25
|
+
return df
|
|
26
|
+
|
|
27
|
+
def __repr__(self):
|
|
28
|
+
return f"ConnectClickHouse(host={self.config['host']!r}, database={self.config['database']!r})"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import asyncio, polars as pl
|
|
2
|
+
from etl.generic.step import Step
|
|
3
|
+
|
|
4
|
+
class Read(Step):
|
|
5
|
+
'''
|
|
6
|
+
Run a SQL query against ClickHouse and make the result the pipeline df.
|
|
7
|
+
|
|
8
|
+
Source step: reads the client from data["ch"] (set by ConnectClickHouse),
|
|
9
|
+
runs the query, returns a Polars DataFrame. The incoming df is ignored.
|
|
10
|
+
|
|
11
|
+
:param query: SQL SELECT to execute.
|
|
12
|
+
:param parameters: SQL parameters
|
|
13
|
+
|
|
14
|
+
Example:
|
|
15
|
+
Read(
|
|
16
|
+
query="""
|
|
17
|
+
SELECT *
|
|
18
|
+
FROM transactions
|
|
19
|
+
WHERE txn_date >= {start:Date}
|
|
20
|
+
AND bank = {bank:String}
|
|
21
|
+
""",
|
|
22
|
+
parameters={"start": "2026-06-17", "bank": "acme"},
|
|
23
|
+
)
|
|
24
|
+
'''
|
|
25
|
+
|
|
26
|
+
def __init__(self, query: str, parameters: dict | None = None):
|
|
27
|
+
self.query, self.parameters = query, parameters
|
|
28
|
+
|
|
29
|
+
async def apply(self, df=None, data=None) -> pl.DataFrame:
|
|
30
|
+
table = await asyncio.to_thread(data["ch"].query_arrow, self.query, parameters=self.parameters)
|
|
31
|
+
return pl.from_arrow(table)
|
|
32
|
+
|
|
33
|
+
def __repr__(self):
|
|
34
|
+
return f"Read(query={self.query!r}, parameters={self.parameters})"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from etl.generic.step import Step, StopPipeline
|
|
3
|
+
from etl.loader.storage import read, Layer, Mode
|
|
4
|
+
from polars import DataFrame
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Read(Step):
|
|
8
|
+
"""
|
|
9
|
+
Read a parquet file from a data lake layer and make it the pipeline df.
|
|
10
|
+
|
|
11
|
+
Source step of a load pipeline: the incoming df is ignored, the read frame
|
|
12
|
+
is returned and flows downstream.
|
|
13
|
+
|
|
14
|
+
:param layer: lake layer to read from (e.g. Storage.Layer.FACT).
|
|
15
|
+
:param name: file stem; ".parquet" is appended.
|
|
16
|
+
:param date: date partition; None = today.
|
|
17
|
+
:param mode: read from static, or date
|
|
18
|
+
:param missing_ok: if is true then pipeline must be continue.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, layer: Layer, name: str, date: str | None = None, mode: Mode | None = None , missing_ok: bool = False):
|
|
22
|
+
self.layer, self.name, self.date, self.mode, self.missing_ok = layer, name, date, mode, missing_ok
|
|
23
|
+
|
|
24
|
+
async def apply(self, df: DataFrame=None, data=None):
|
|
25
|
+
try:
|
|
26
|
+
return await asyncio.to_thread(read, self.layer, f"{self.name}.parquet", self.date, self.mode)
|
|
27
|
+
except FileNotFoundError:
|
|
28
|
+
if self.missing_ok:
|
|
29
|
+
print(f" [read] file not found, returning empty df: {self.layer}/{self.name}.parquet")
|
|
30
|
+
return DataFrame()
|
|
31
|
+
raise StopPipeline(message="The file wasn't found.")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def __repr__(self):
|
|
35
|
+
return f"Read(layer={self.layer}, name={self.name!r}, date={self.date!r})"
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from etl.generic import Step, StopPipeline
|
|
2
|
+
from etl.loader.storage import path, Mode, Layer
|
|
3
|
+
import asyncio, polars as pl
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Reads(Step):
|
|
7
|
+
"""
|
|
8
|
+
Read every parquet file in one lake folder and concat them into a single df.
|
|
9
|
+
Useful for datasets split across many files, e.g. manual_activity/<date>.parquet.
|
|
10
|
+
|
|
11
|
+
Files are matched by `pattern` inside the folder and optionally filtered by an
|
|
12
|
+
inclusive date range compared against the file stem (name without extension).
|
|
13
|
+
Empty result: return an empty df when `missing_ok`, otherwise stop the pipeline.
|
|
14
|
+
|
|
15
|
+
:param layer: lake layer to read from (e.g. Storage.Layer.STG).
|
|
16
|
+
:param name: folder name inside the layer.
|
|
17
|
+
:param mode: STATIC (static subdir) or DATE (dated subdir).
|
|
18
|
+
:param date_from: lower bound "YYYY-MM-DD", inclusive, matched on file stem.
|
|
19
|
+
:param date_to: upper bound "YYYY-MM-DD", inclusive.
|
|
20
|
+
:param pattern: glob for files inside the folder (default "*.parquet").
|
|
21
|
+
:param missing_ok: if True, return an empty df instead of raising StopPipeline.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def __init__(self, layer, name, mode=Mode.STATIC, date_from=None, date_to=None,
|
|
25
|
+
pattern="*.parquet", missing_ok=False):
|
|
26
|
+
self.layer, self.name, self.mode = layer, name, mode
|
|
27
|
+
self.date_from, self.date_to = date_from, date_to
|
|
28
|
+
self.pattern, self.missing_ok = pattern, missing_ok
|
|
29
|
+
|
|
30
|
+
async def apply(self, df, data = None):
|
|
31
|
+
folder = path(self.layer, self.name, mode=self.mode)
|
|
32
|
+
files = sorted(folder.glob(self.pattern)) if folder.exists() else []
|
|
33
|
+
|
|
34
|
+
if self.date_from:
|
|
35
|
+
files = [f for f in files if f.stem >= self.date_from]
|
|
36
|
+
|
|
37
|
+
if self.date_to:
|
|
38
|
+
files = [f for f in files if f.stem <= self.date_to]
|
|
39
|
+
|
|
40
|
+
if not files:
|
|
41
|
+
if self.missing_ok:
|
|
42
|
+
print(f" No files: {folder}")
|
|
43
|
+
return pl.DataFrame()
|
|
44
|
+
raise StopPipeline(message=f"No files found: {folder}")
|
|
45
|
+
|
|
46
|
+
return await asyncio.to_thread(
|
|
47
|
+
lambda: pl.concat([pl.read_parquet(file) for file in files], how="vertical_relaxed")
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def __repr__(self):
|
|
51
|
+
return f"Reads(name={self.name!r}, from={self.date_from}, to={self.date_to})"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from etl.generic.step import Step
|
|
2
|
+
from polars import DataFrame
|
|
3
|
+
|
|
4
|
+
class Connect(Step):
|
|
5
|
+
"""
|
|
6
|
+
Open a MongoDB connection and put the database handle into data["mongo"]
|
|
7
|
+
for downstream steps (Read, EnsureIndexes, Upsert, Delta).
|
|
8
|
+
|
|
9
|
+
Usually the first step of a MongoDB pipeline.
|
|
10
|
+
|
|
11
|
+
:param host/port/database/username/password: pymongo connection params.
|
|
12
|
+
:param kwargs: extra args forwarded to AsyncMongoClient (authSource, tls, replicaSet, ...).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def __init__(self, host="localhost", port=27017, database="admin",
|
|
16
|
+
username=None, password=None, **kwargs):
|
|
17
|
+
self.database = database
|
|
18
|
+
self.config = dict(host=host, port=port, username=username, password=password, **kwargs)
|
|
19
|
+
|
|
20
|
+
async def apply(self, df: DataFrame, data=None):
|
|
21
|
+
from pymongo import AsyncMongoClient
|
|
22
|
+
|
|
23
|
+
client = AsyncMongoClient(**self.config)
|
|
24
|
+
data["mongo"] = client[self.database]
|
|
25
|
+
return df
|
|
26
|
+
|
|
27
|
+
def __repr__(self):
|
|
28
|
+
return f"ConnectMongo(host={self.config['host']!r}, database={self.database!r})"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
from etl.generic.step import Step
|
|
3
|
+
|
|
4
|
+
class Read(Step):
|
|
5
|
+
"""
|
|
6
|
+
Query a MongoDB collection and make the result the pipeline df.
|
|
7
|
+
|
|
8
|
+
Source step: reads the database handle from data["mongo"] (set by Connect).
|
|
9
|
+
Two modes: by default runs collection.find(filter, projection); if `pipeline`
|
|
10
|
+
is given, runs collection.aggregate(pipeline) instead (for $lookup joins etc.).
|
|
11
|
+
The incoming df is ignored.
|
|
12
|
+
|
|
13
|
+
:param collection: collection name to read from.
|
|
14
|
+
:param filter: query dict, e.g. {"txn_date": {"$gte": date}}. None = all. (find mode)
|
|
15
|
+
:param projection: fields to include/exclude. None defaults to {"_id": 0}. (find mode)
|
|
16
|
+
:param sort: list of (field, direction), e.g. [("txn_date", 1)]. (find mode)
|
|
17
|
+
:param limit: max documents (0 = no limit). (find mode)
|
|
18
|
+
:param pipeline: aggregation pipeline; if set, aggregate() is used and
|
|
19
|
+
filter/projection/sort/limit are ignored. In this mode YOU own
|
|
20
|
+
_id handling - convert/drop it, e.g. a final
|
|
21
|
+
{"$project": {"id": {"$toString": "$_id"}, "_id": 0}},
|
|
22
|
+
or the raw ObjectId column will break Polars.
|
|
23
|
+
|
|
24
|
+
Example (find):
|
|
25
|
+
Read(
|
|
26
|
+
collection="transactions",
|
|
27
|
+
filter={"txn_date": {"$gte": "2026-06-17"}, "bank": "acme"},
|
|
28
|
+
projection={"_id": 0},
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
Example (aggregate with joins):
|
|
32
|
+
Read(
|
|
33
|
+
collection="transactions",
|
|
34
|
+
pipeline=[
|
|
35
|
+
{"$lookup": {...}}, # accounts
|
|
36
|
+
{"$lookup": {...}}, # organizations
|
|
37
|
+
{"$project": {"id": {"$toString": "$_id"}, "_id": 0}},
|
|
38
|
+
],
|
|
39
|
+
)
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self, collection, filter=None, projection=None,
|
|
43
|
+
sort=None, limit=0, pipeline=None):
|
|
44
|
+
self.collection = collection
|
|
45
|
+
self.filter = filter or {}
|
|
46
|
+
self.projection = projection if projection else {"_id": 0}
|
|
47
|
+
self.sort = sort
|
|
48
|
+
self.limit = limit
|
|
49
|
+
self.pipeline = pipeline
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def apply(self, df=None, data=None) -> pl.DataFrame:
|
|
53
|
+
coll = data["mongo"][self.collection]
|
|
54
|
+
if self.pipeline is not None:
|
|
55
|
+
cursor = await coll.aggregate(self.pipeline)
|
|
56
|
+
else:
|
|
57
|
+
cursor = coll.find(self.filter, self.projection)
|
|
58
|
+
if self.sort:
|
|
59
|
+
cursor = cursor.sort(self.sort)
|
|
60
|
+
if self.limit:
|
|
61
|
+
cursor = cursor.limit(self.limit)
|
|
62
|
+
docs = await cursor.to_list()
|
|
63
|
+
return pl.DataFrame(docs, infer_schema_length=None)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def __repr__(self):
|
|
67
|
+
return f"Read(collection={self.collection!r}, filter={self.filter})"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from etl.generic.step import Step
|
|
2
|
+
from polars import DataFrame
|
|
3
|
+
|
|
4
|
+
class Connect(Step):
|
|
5
|
+
"""
|
|
6
|
+
Open a Microsoft SQL Server connection and put it into data["mssql"].
|
|
7
|
+
Requires the Microsoft ODBC Driver for SQL Server installed on the host,
|
|
8
|
+
and the 'aioodbc' package (pip install aioodbc).
|
|
9
|
+
|
|
10
|
+
:param host/port/database/user/password: connection params.
|
|
11
|
+
:param driver: installed ODBC driver name.
|
|
12
|
+
:param kwargs: extra ODBC keywords appended to the connection string
|
|
13
|
+
(e.g. TrustServerCertificate="yes", Encrypt="no").
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, host="localhost", port=1433, database="master",
|
|
17
|
+
user="sa", password="", driver="ODBC Driver 18 for SQL Server", **kwargs):
|
|
18
|
+
self.host, self.database = host, database
|
|
19
|
+
extra = "".join(f"{k}={v};" for k, v in kwargs.items())
|
|
20
|
+
self.dsn = (f"Driver={{{driver}}};Server={host},{port};Database={database};"
|
|
21
|
+
f"UID={user};PWD={password};{extra}")
|
|
22
|
+
|
|
23
|
+
async def apply(self, df: DataFrame = None, data=None):
|
|
24
|
+
import aioodbc
|
|
25
|
+
data["mssql"] = await aioodbc.connect(dsn=self.dsn, autocommit=True)
|
|
26
|
+
return df
|
|
27
|
+
|
|
28
|
+
def __repr__(self):
|
|
29
|
+
return f"ConnectMSSQL(host={self.host!r}, database={self.database!r})"
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
from etl.generic.step import Step
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Read(Step):
|
|
6
|
+
"""
|
|
7
|
+
Run a SQL query against SQL Server and make the result the pipeline df.
|
|
8
|
+
|
|
9
|
+
Reads the connection from data["mssql"] (set by Connect). Incoming df ignored.
|
|
10
|
+
|
|
11
|
+
:param query: T-SQL SELECT with ? placeholders for parameters.
|
|
12
|
+
:param parameters: values bound to the placeholders (injection-safe).
|
|
13
|
+
|
|
14
|
+
Example:
|
|
15
|
+
Read(
|
|
16
|
+
query="SELECT * FROM transactions WHERE txn_date >= ? AND bank = ?",
|
|
17
|
+
parameters=["2026-06-17", "acme"],
|
|
18
|
+
)
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, query: str, parameters: list | None = None):
|
|
22
|
+
self.query = query
|
|
23
|
+
self.parameters = parameters or []
|
|
24
|
+
|
|
25
|
+
async def apply(self, df=None, data=None) -> pl.DataFrame:
|
|
26
|
+
async with data["mssql"].cursor() as cur:
|
|
27
|
+
await cur.execute(self.query, *self.parameters)
|
|
28
|
+
rows = await cur.fetchall()
|
|
29
|
+
cols = [d[0] for d in cur.description]
|
|
30
|
+
return pl.DataFrame([tuple(r) for r in rows], schema=cols, orient="row", infer_schema_length=None)
|
|
31
|
+
|
|
32
|
+
def __repr__(self):
|
|
33
|
+
return f"Read(query={self.query!r}, parameters={self.parameters})"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from etl.generic.step import Step
|
|
2
|
+
from polars import DataFrame
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Connect(Step):
|
|
6
|
+
"""
|
|
7
|
+
Open a PostgreSQL connection and put it into data["pg"] for downstream Read.
|
|
8
|
+
|
|
9
|
+
:param host/port/database/user/password: asyncpg connection params.
|
|
10
|
+
:param kwargs: extra args forwarded to asyncpg.connect (ssl, timeout, ...).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
def __init__(self, host="localhost", port=5432, database="postgres",
|
|
14
|
+
user="postgres", password="", **kwargs):
|
|
15
|
+
self.config = dict(host=host, port=port, database=database,
|
|
16
|
+
user=user, password=password, **kwargs)
|
|
17
|
+
|
|
18
|
+
async def apply(self, df: DataFrame = None, data=None):
|
|
19
|
+
import asyncpg
|
|
20
|
+
data["pg"] = await asyncpg.connect(**self.config)
|
|
21
|
+
return df
|
|
22
|
+
|
|
23
|
+
def __repr__(self):
|
|
24
|
+
return f"ConnectPostgres(host={self.config['host']!r}, database={self.config['database']!r})"
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
from etl.generic.step import Step
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Read(Step):
|
|
6
|
+
"""
|
|
7
|
+
Run a SQL query against PostgreSQL and make the result the pipeline df.
|
|
8
|
+
|
|
9
|
+
Source step: reads the connection from data["pg"] (set by Connect),
|
|
10
|
+
runs the query, returns a Polars DataFrame. The incoming df is ignored.
|
|
11
|
+
|
|
12
|
+
:param query: SQL SELECT with $1, $2 placeholders.
|
|
13
|
+
:param parameters: values bound to the placeholders (injection-safe).
|
|
14
|
+
|
|
15
|
+
Example:
|
|
16
|
+
Read(
|
|
17
|
+
query="SELECT * FROM transactions WHERE txn_date >= $1 AND bank = $2",
|
|
18
|
+
parameters=["2026-06-17", "acme"],
|
|
19
|
+
)
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, query: str, parameters: list | None = None):
|
|
23
|
+
self.query = query
|
|
24
|
+
self.parameters = parameters or []
|
|
25
|
+
|
|
26
|
+
async def apply(self, df=None, data=None) -> pl.DataFrame:
|
|
27
|
+
rows = await data["pg"].fetch(self.query, *self.parameters)
|
|
28
|
+
return pl.DataFrame([dict(r) for r in rows], infer_schema_length=None)
|
|
29
|
+
|
|
30
|
+
def __repr__(self):
|
|
31
|
+
return f"Read(query={self.query!r}, parameters={self.parameters})"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from etl.generic.step import Step
|
|
2
|
+
from polars import DataFrame
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Connect(Step):
|
|
6
|
+
"""
|
|
7
|
+
Open a SQLite connection and put it into data["sqlite"].
|
|
8
|
+
|
|
9
|
+
WARNING: SQLite is meant for tests and small local datasets. It is a
|
|
10
|
+
single-file, single-writer engine and is NOT suitable for storing large
|
|
11
|
+
data - use ClickHouse or Postgres for that. Great as an in-memory test
|
|
12
|
+
backend (path=":memory:").
|
|
13
|
+
|
|
14
|
+
:param path: database file path, or ":memory:" for an in-memory DB.
|
|
15
|
+
:param kwargs: extra args forwarded to aiosqlite.connect.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self, path: str = ":memory:", **kwargs):
|
|
19
|
+
self.path, self.kwargs = path, kwargs
|
|
20
|
+
|
|
21
|
+
async def apply(self, df: DataFrame = None, data=None):
|
|
22
|
+
import aiosqlite
|
|
23
|
+
data["sqlite"] = await aiosqlite.connect(self.path, **self.kwargs)
|
|
24
|
+
return df
|
|
25
|
+
|
|
26
|
+
def __repr__(self):
|
|
27
|
+
return f"ConnectSQLite(path={self.path!r})"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
from etl.generic.step import Step
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Read(Step):
|
|
6
|
+
"""
|
|
7
|
+
Run a SQL query against SQLite and make the result the pipeline df.
|
|
8
|
+
Reads the connection from data["sqlite"] (set by Connect).
|
|
9
|
+
|
|
10
|
+
:param query: SQL SELECT with ? placeholders.
|
|
11
|
+
:param parameters: values bound to the placeholders (injection-safe).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def __init__(self, query: str, parameters: list | None = None):
|
|
15
|
+
self.query = query
|
|
16
|
+
self.parameters = parameters or []
|
|
17
|
+
|
|
18
|
+
async def apply(self, df=None, data=None) -> pl.DataFrame:
|
|
19
|
+
async with data["sqlite"].execute(self.query, self.parameters) as cursor:
|
|
20
|
+
rows = await cursor.fetchall()
|
|
21
|
+
cols = [d[0] for d in cursor.description]
|
|
22
|
+
return pl.DataFrame([tuple(r) for r in rows], schema=cols, orient="row", infer_schema_length=None)
|
|
23
|
+
|
|
24
|
+
def __repr__(self):
|
|
25
|
+
return f"Read(query={self.query!r}, parameters={self.parameters})"
|
etl/generic/__init__.py
ADDED
etl/generic/context.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from typing import Any
|
|
3
|
+
from polars import DataFrame, LazyFrame
|
|
4
|
+
|
|
5
|
+
class Data:
|
|
6
|
+
'''
|
|
7
|
+
General data within the pipline; all steps communicate thought this context.
|
|
8
|
+
|
|
9
|
+
Anti-pattern: Only simple data should be stored here; heavy dataframes must not be placed here.
|
|
10
|
+
Solution: Extract dataframe to data lake and read them frome there.
|
|
11
|
+
This is required for auditing raw data.
|
|
12
|
+
'''
|
|
13
|
+
|
|
14
|
+
def __init__(self, **values: Any) -> None:
|
|
15
|
+
self.store: dict[str, Any] = dict(values)
|
|
16
|
+
|
|
17
|
+
def get(self, key: str, default: Any = None) -> Any:
|
|
18
|
+
return self.store.get(key, default)
|
|
19
|
+
|
|
20
|
+
def __getitem__(self, key: str) -> Any:
|
|
21
|
+
return self.store[key]
|
|
22
|
+
|
|
23
|
+
def __setitem__(self, key:str, value: Any) -> None:
|
|
24
|
+
if isinstance(value, (DataFrame, LazyFrame)):
|
|
25
|
+
raise TypeError(
|
|
26
|
+
f"Data['{key}'] is Dataframe is antipattern,"
|
|
27
|
+
f"should be exchanged between steps via raw exports.")
|
|
28
|
+
|
|
29
|
+
self.store[key] = value
|
|
30
|
+
|
|
31
|
+
def __contains__(self, key: str) -> bool:
|
|
32
|
+
return key in self.store
|
|
33
|
+
|
|
34
|
+
def __repr__(self) -> str:
|
|
35
|
+
body = ", ".join(f"{key}: {type(value).__name__}" for key,value in self.store.items())
|
|
36
|
+
return f"Data({body})"
|