etlpy-tools 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. etl/__init__.py +0 -0
  2. etl/extractor/steps/api/__init__.py +4 -0
  3. etl/extractor/steps/api/basic_authenticate.py +20 -0
  4. etl/extractor/steps/api/oauthenticate.py +107 -0
  5. etl/extractor/steps/clickhouse/__init__.py +4 -0
  6. etl/extractor/steps/clickhouse/connect.py +28 -0
  7. etl/extractor/steps/clickhouse/read.py +34 -0
  8. etl/extractor/steps/datalake/__init__.py +4 -0
  9. etl/extractor/steps/datalake/read.py +35 -0
  10. etl/extractor/steps/datalake/reads.py +51 -0
  11. etl/extractor/steps/mongodb/__init__.py +4 -0
  12. etl/extractor/steps/mongodb/connect.py +28 -0
  13. etl/extractor/steps/mongodb/read.py +67 -0
  14. etl/extractor/steps/mssql/__init__.py +4 -0
  15. etl/extractor/steps/mssql/connect.py +29 -0
  16. etl/extractor/steps/mssql/read.py +33 -0
  17. etl/extractor/steps/postgresql/__init__.py +4 -0
  18. etl/extractor/steps/postgresql/connect.py +24 -0
  19. etl/extractor/steps/postgresql/read.py +31 -0
  20. etl/extractor/steps/sqlite/__init__.py +4 -0
  21. etl/extractor/steps/sqlite/connect.py +27 -0
  22. etl/extractor/steps/sqlite/read.py +25 -0
  23. etl/generic/__init__.py +5 -0
  24. etl/generic/context.py +36 -0
  25. etl/generic/pipeline.py +61 -0
  26. etl/generic/step.py +23 -0
  27. etl/loader/__init__.py +3 -0
  28. etl/loader/steps/clickhouse/__init__.py +6 -0
  29. etl/loader/steps/clickhouse/delta.py +52 -0
  30. etl/loader/steps/clickhouse/ensuretable.py +83 -0
  31. etl/loader/steps/clickhouse/insert.py +25 -0
  32. etl/loader/steps/clickhouse/optimize.py +22 -0
  33. etl/loader/steps/datalake/__init__.py +4 -0
  34. etl/loader/steps/datalake/archive.py +34 -0
  35. etl/loader/steps/datalake/save.py +38 -0
  36. etl/loader/steps/mongodb/__init__.py +5 -0
  37. etl/loader/steps/mongodb/delta.py +42 -0
  38. etl/loader/steps/mongodb/ensureindexes.py +27 -0
  39. etl/loader/steps/mongodb/upsert.py +42 -0
  40. etl/loader/steps/mssql/__init__.py +4 -0
  41. etl/loader/steps/mssql/ensuretable.py +46 -0
  42. etl/loader/steps/mssql/insert.py +40 -0
  43. etl/loader/steps/postgresql/__init__.py +4 -0
  44. etl/loader/steps/postgresql/ensuretable.py +46 -0
  45. etl/loader/steps/postgresql/insert.py +34 -0
  46. etl/loader/steps/sqlite/__init__.py +4 -0
  47. etl/loader/steps/sqlite/ensuretable.py +39 -0
  48. etl/loader/steps/sqlite/insert.py +32 -0
  49. etl/loader/storage.py +207 -0
  50. etl/transformer/steps/__init__.py +43 -0
  51. etl/transformer/steps/add_column.py +30 -0
  52. etl/transformer/steps/aggregate.py +55 -0
  53. etl/transformer/steps/cast_types.py +22 -0
  54. etl/transformer/steps/clear_text.py +39 -0
  55. etl/transformer/steps/drop_columns.py +36 -0
  56. etl/transformer/steps/drop_duplicates.py +29 -0
  57. etl/transformer/steps/drop_nulls.py +28 -0
  58. etl/transformer/steps/extract_entities.py +43 -0
  59. etl/transformer/steps/fill_nulls.py +29 -0
  60. etl/transformer/steps/filter_rows.py +22 -0
  61. etl/transformer/steps/generate_key.py +52 -0
  62. etl/transformer/steps/join.py +72 -0
  63. etl/transformer/steps/normalize_numeric.py +36 -0
  64. etl/transformer/steps/rename_columns.py +22 -0
  65. etl/transformer/steps/row_hash.py +32 -0
  66. etl/transformer/steps/slambda.py +22 -0
  67. etl/transformer/steps/sql.py +33 -0
  68. etl/transformer/steps/to_schema.py +67 -0
  69. etl/transformer/steps/trim_string.py +23 -0
  70. etl/transformer/steps/union.py +34 -0
  71. etlpy_tools-1.0.0.dist-info/METADATA +311 -0
  72. etlpy_tools-1.0.0.dist-info/RECORD +75 -0
  73. etlpy_tools-1.0.0.dist-info/WHEEL +5 -0
  74. etlpy_tools-1.0.0.dist-info/licenses/LICENSE +21 -0
  75. etlpy_tools-1.0.0.dist-info/top_level.txt +1 -0
etl/__init__.py ADDED
File without changes
@@ -0,0 +1,4 @@
1
+ from etl.extractor.steps.api.oauthenticate import OAuthenticate, OAuthFields
2
+ from etl.extractor.steps.api.basic_authenticate import AuthenticateBasic
3
+
4
+ __all__ = ["OAuthenticate", "OAuthFields", "AuthenticateBasic"]
@@ -0,0 +1,20 @@
1
+ import httpx
2
+ from etl.generic import Step, Data
3
+
4
+
5
+ class AuthenticateBasic(Step):
6
+ """
7
+ HTTP Basic (RFC 7617): token is empty, header is static.
8
+ """
9
+
10
+ def __init__(self, user: str, password: str, headers: dict | None = None, timeout: float = 60.0):
11
+ self.user, self.password, self.timeout = user, password, timeout
12
+ self.headers = headers or {}
13
+
14
+ async def apply(self, df, data: Data = None):
15
+ client = data.get("client") or httpx.AsyncClient(timeout=httpx.Timeout(self.timeout))
16
+ client.auth = httpx.BasicAuth(self.user, self.password) # base64 (user:password)
17
+ client.headers.update(self.headers)
18
+
19
+ data["client"] = client
20
+ return df
@@ -0,0 +1,107 @@
1
+ from __future__ import annotations
2
+ from dataclasses import dataclass, asdict
3
+ import httpx
4
+
5
+ from etl.generic.step import Step
6
+ from etl.generic.context import Data
7
+
8
+ @dataclass
9
+ class OAuthFields:
10
+ """
11
+ Map fields with the Authenticate step can works.
12
+
13
+ Where each OAuth 2.0 token-response field lives (RFC 6749 section 5.1).
14
+ Value = dotted path in th response; None = The API does not return it.
15
+
16
+ OAuthFields(
17
+ access_token="data.access_token",
18
+ expires_in="data.expires_in",
19
+ refresh_token="data.refresh_token",
20
+ )
21
+ """
22
+ access_token: str = "data.access_token"
23
+ token_type: str | None = None
24
+ expires_in: str | None = "data.expires_in"
25
+ refresh_token: str | None = "data.refresh_token"
26
+ scope: str | None = None
27
+
28
+
29
+ class OAuthenticate(Step):
30
+ """
31
+ Generic API auth: send credentials, extract a token, configure the shared
32
+ client and stash the auth in data.
33
+
34
+ Implements the non-interactive OAuth 2.0 token-endpoint flows, primarily the
35
+ Client Credentials grant (RFC 6749 section 4.4). The access token is applied
36
+ per RFC 6750 (Bearer). Client authentication via request body or HTTP Basic
37
+ (RFC 7617) is selectable through 'send'.
38
+
39
+ - RFC 6749 OAuth 2.0 Authorization Framework https://www.rfc-editor.org/info/rfc6749/
40
+ - RFC 6750 OAuth 2.0 Bearer Token Usage https://www.rfc-editor.org/info/rfc6750/
41
+ - RFC 7617 HTTP Basic Authentication https://www.rfc-editor.org/info/rfc7617/
42
+ - RFC 9700 OAuth 2.0 Security BCP https://www.rfc-editor.org/info/rfc9700/
43
+ """
44
+
45
+ def __init__(
46
+ self,
47
+ url: str,
48
+ credentials: dict,
49
+ fields: OAuthFields = None,
50
+ send="json",
51
+ method="POST",
52
+ auth_header=None,
53
+ headers=None,
54
+ store="auth",
55
+ timeout=60.0
56
+ ):
57
+ self.url = url
58
+ self.credentials = credentials
59
+ self.fields = fields or OAuthFields()
60
+ self.send = send
61
+ self.method = method
62
+ self.headers=headers or {"Content-Type": "application/json"}
63
+ self.auth_header = auth_header or {"Authorization": "Bearer {token}"}
64
+ # auth header for client it can be:
65
+ # {"X-Auth-Token": "{token}"}
66
+ # {"Authorization": "{token}"}
67
+ # for headers to https/http response use headers.
68
+ self.store = store
69
+ self.timeout = timeout
70
+
71
+ async def apply(self, df, data: Data):
72
+ client = data.get("client") or httpx.AsyncClient(timeout=httpx.Timeout(self.timeout))
73
+ response = await client.request(self.method, self.url, headers=self.headers, **{self.send:self.credentials})
74
+
75
+ try:
76
+ response.raise_for_status()
77
+ except httpx.HTTPStatusError as error:
78
+ raise httpx.HTTPStatusError(
79
+ f"{error}\nResponse body: {response.text}",
80
+ request=error.request, response=error.response
81
+ ) from error
82
+
83
+ body = response.json()
84
+ auth = {}
85
+ for name, path in asdict(self.fields).items():
86
+ if path is None: # API does not return this field
87
+ print(f"API does not return this field: {name}")
88
+ continue
89
+ auth[name] = self._get(body, path, required=(name == "access_token"))
90
+
91
+ token = auth["access_token"]
92
+ headers = {key: value.format(token=token) for key, value in self.auth_header.items()}
93
+ client.headers.update(headers) # downstream uses data["client"]
94
+
95
+ data["client"] = client
96
+ data[self.store] = auth
97
+ return df
98
+
99
+ def _get(self, body, path, required=True):
100
+ obj = body
101
+ for key in path.split("."):
102
+ if not isinstance(obj, dict) or key not in obj:
103
+ if required:
104
+ raise KeyError(f"path '{path}' not found in auth response")
105
+ return None
106
+ obj = obj[key]
107
+ return obj
@@ -0,0 +1,4 @@
1
+ from etl.extractor.steps.clickhouse.read import Read
2
+ from etl.extractor.steps.clickhouse.connect import Connect
3
+
4
+ __all__ = ["Read", "Connect"]
@@ -0,0 +1,28 @@
1
+ import asyncio
2
+ from etl.generic.step import Step
3
+ from polars import DataFrame
4
+
5
+
6
+ class Connect(Step):
7
+ """
8
+ Open a ClickHouse connection and put the client into data["ch"] for
9
+ downstream steps (EnsureTable, Delta, Insert).
10
+
11
+ Usually the first step of a load pipeline.
12
+
13
+ :param host/port/database/username/password/secure: clickhouse-connect params.
14
+ :param kwargs: extra arguments forwarded to clickhouse_connect.get_client.
15
+ """
16
+
17
+ def __init__(self, host="localhost", port=8123, database="default",
18
+ username="default", password="", secure=False, **kwargs):
19
+ self.config = dict(host=host, port=port, database=database,
20
+ username=username, password=password, secure=secure, **kwargs)
21
+
22
+ async def apply(self, df: DataFrame, data=None):
23
+ import clickhouse_connect
24
+ data["ch"] = await asyncio.to_thread(clickhouse_connect.get_client, **self.config)
25
+ return df
26
+
27
+ def __repr__(self):
28
+ return f"ConnectClickHouse(host={self.config['host']!r}, database={self.config['database']!r})"
@@ -0,0 +1,34 @@
1
+ import asyncio, polars as pl
2
+ from etl.generic.step import Step
3
+
4
+ class Read(Step):
5
+ '''
6
+ Run a SQL query against ClickHouse and make the result the pipeline df.
7
+
8
+ Source step: reads the client from data["ch"] (set by ConnectClickHouse),
9
+ runs the query, returns a Polars DataFrame. The incoming df is ignored.
10
+
11
+ :param query: SQL SELECT to execute.
12
+ :param parameters: SQL parameters
13
+
14
+ Example:
15
+ Read(
16
+ query="""
17
+ SELECT *
18
+ FROM transactions
19
+ WHERE txn_date >= {start:Date}
20
+ AND bank = {bank:String}
21
+ """,
22
+ parameters={"start": "2026-06-17", "bank": "acme"},
23
+ )
24
+ '''
25
+
26
+ def __init__(self, query: str, parameters: dict | None = None):
27
+ self.query, self.parameters = query, parameters
28
+
29
+ async def apply(self, df=None, data=None) -> pl.DataFrame:
30
+ table = await asyncio.to_thread(data["ch"].query_arrow, self.query, parameters=self.parameters)
31
+ return pl.from_arrow(table)
32
+
33
+ def __repr__(self):
34
+ return f"Read(query={self.query!r}, parameters={self.parameters})"
@@ -0,0 +1,4 @@
1
+ from etl.extractor.steps.datalake.read import Read
2
+ from etl.extractor.steps.datalake.reads import Reads
3
+
4
+ __all__ = ["Read", "Reads"]
@@ -0,0 +1,35 @@
1
+ import asyncio
2
+ from etl.generic.step import Step, StopPipeline
3
+ from etl.loader.storage import read, Layer, Mode
4
+ from polars import DataFrame
5
+
6
+
7
+ class Read(Step):
8
+ """
9
+ Read a parquet file from a data lake layer and make it the pipeline df.
10
+
11
+ Source step of a load pipeline: the incoming df is ignored, the read frame
12
+ is returned and flows downstream.
13
+
14
+ :param layer: lake layer to read from (e.g. Storage.Layer.FACT).
15
+ :param name: file stem; ".parquet" is appended.
16
+ :param date: date partition; None = today.
17
+ :param mode: read from static, or date
18
+ :param missing_ok: if is true then pipeline must be continue.
19
+ """
20
+
21
+ def __init__(self, layer: Layer, name: str, date: str | None = None, mode: Mode | None = None , missing_ok: bool = False):
22
+ self.layer, self.name, self.date, self.mode, self.missing_ok = layer, name, date, mode, missing_ok
23
+
24
+ async def apply(self, df: DataFrame=None, data=None):
25
+ try:
26
+ return await asyncio.to_thread(read, self.layer, f"{self.name}.parquet", self.date, self.mode)
27
+ except FileNotFoundError:
28
+ if self.missing_ok:
29
+ print(f" [read] file not found, returning empty df: {self.layer}/{self.name}.parquet")
30
+ return DataFrame()
31
+ raise StopPipeline(message="The file wasn't found.")
32
+
33
+
34
+ def __repr__(self):
35
+ return f"Read(layer={self.layer}, name={self.name!r}, date={self.date!r})"
@@ -0,0 +1,51 @@
1
+ from etl.generic import Step, StopPipeline
2
+ from etl.loader.storage import path, Mode, Layer
3
+ import asyncio, polars as pl
4
+
5
+
6
+ class Reads(Step):
7
+ """
8
+ Read every parquet file in one lake folder and concat them into a single df.
9
+ Useful for datasets split across many files, e.g. manual_activity/<date>.parquet.
10
+
11
+ Files are matched by `pattern` inside the folder and optionally filtered by an
12
+ inclusive date range compared against the file stem (name without extension).
13
+ Empty result: return an empty df when `missing_ok`, otherwise stop the pipeline.
14
+
15
+ :param layer: lake layer to read from (e.g. Storage.Layer.STG).
16
+ :param name: folder name inside the layer.
17
+ :param mode: STATIC (static subdir) or DATE (dated subdir).
18
+ :param date_from: lower bound "YYYY-MM-DD", inclusive, matched on file stem.
19
+ :param date_to: upper bound "YYYY-MM-DD", inclusive.
20
+ :param pattern: glob for files inside the folder (default "*.parquet").
21
+ :param missing_ok: if True, return an empty df instead of raising StopPipeline.
22
+ """
23
+
24
+ def __init__(self, layer, name, mode=Mode.STATIC, date_from=None, date_to=None,
25
+ pattern="*.parquet", missing_ok=False):
26
+ self.layer, self.name, self.mode = layer, name, mode
27
+ self.date_from, self.date_to = date_from, date_to
28
+ self.pattern, self.missing_ok = pattern, missing_ok
29
+
30
+ async def apply(self, df, data = None):
31
+ folder = path(self.layer, self.name, mode=self.mode)
32
+ files = sorted(folder.glob(self.pattern)) if folder.exists() else []
33
+
34
+ if self.date_from:
35
+ files = [f for f in files if f.stem >= self.date_from]
36
+
37
+ if self.date_to:
38
+ files = [f for f in files if f.stem <= self.date_to]
39
+
40
+ if not files:
41
+ if self.missing_ok:
42
+ print(f" No files: {folder}")
43
+ return pl.DataFrame()
44
+ raise StopPipeline(message=f"No files found: {folder}")
45
+
46
+ return await asyncio.to_thread(
47
+ lambda: pl.concat([pl.read_parquet(file) for file in files], how="vertical_relaxed")
48
+ )
49
+
50
+ def __repr__(self):
51
+ return f"Reads(name={self.name!r}, from={self.date_from}, to={self.date_to})"
@@ -0,0 +1,4 @@
1
+ from etl.extractor.steps.mongodb.read import Read
2
+ from etl.extractor.steps.mongodb.connect import Connect
3
+
4
+ __all__ = ["Read", "Connect"]
@@ -0,0 +1,28 @@
1
+ from etl.generic.step import Step
2
+ from polars import DataFrame
3
+
4
+ class Connect(Step):
5
+ """
6
+ Open a MongoDB connection and put the database handle into data["mongo"]
7
+ for downstream steps (Read, EnsureIndexes, Upsert, Delta).
8
+
9
+ Usually the first step of a MongoDB pipeline.
10
+
11
+ :param host/port/database/username/password: pymongo connection params.
12
+ :param kwargs: extra args forwarded to AsyncMongoClient (authSource, tls, replicaSet, ...).
13
+ """
14
+
15
+ def __init__(self, host="localhost", port=27017, database="admin",
16
+ username=None, password=None, **kwargs):
17
+ self.database = database
18
+ self.config = dict(host=host, port=port, username=username, password=password, **kwargs)
19
+
20
+ async def apply(self, df: DataFrame, data=None):
21
+ from pymongo import AsyncMongoClient
22
+
23
+ client = AsyncMongoClient(**self.config)
24
+ data["mongo"] = client[self.database]
25
+ return df
26
+
27
+ def __repr__(self):
28
+ return f"ConnectMongo(host={self.config['host']!r}, database={self.database!r})"
@@ -0,0 +1,67 @@
1
+ import polars as pl
2
+ from etl.generic.step import Step
3
+
4
+ class Read(Step):
5
+ """
6
+ Query a MongoDB collection and make the result the pipeline df.
7
+
8
+ Source step: reads the database handle from data["mongo"] (set by Connect).
9
+ Two modes: by default runs collection.find(filter, projection); if `pipeline`
10
+ is given, runs collection.aggregate(pipeline) instead (for $lookup joins etc.).
11
+ The incoming df is ignored.
12
+
13
+ :param collection: collection name to read from.
14
+ :param filter: query dict, e.g. {"txn_date": {"$gte": date}}. None = all. (find mode)
15
+ :param projection: fields to include/exclude. None defaults to {"_id": 0}. (find mode)
16
+ :param sort: list of (field, direction), e.g. [("txn_date", 1)]. (find mode)
17
+ :param limit: max documents (0 = no limit). (find mode)
18
+ :param pipeline: aggregation pipeline; if set, aggregate() is used and
19
+ filter/projection/sort/limit are ignored. In this mode YOU own
20
+ _id handling - convert/drop it, e.g. a final
21
+ {"$project": {"id": {"$toString": "$_id"}, "_id": 0}},
22
+ or the raw ObjectId column will break Polars.
23
+
24
+ Example (find):
25
+ Read(
26
+ collection="transactions",
27
+ filter={"txn_date": {"$gte": "2026-06-17"}, "bank": "acme"},
28
+ projection={"_id": 0},
29
+ )
30
+
31
+ Example (aggregate with joins):
32
+ Read(
33
+ collection="transactions",
34
+ pipeline=[
35
+ {"$lookup": {...}}, # accounts
36
+ {"$lookup": {...}}, # organizations
37
+ {"$project": {"id": {"$toString": "$_id"}, "_id": 0}},
38
+ ],
39
+ )
40
+ """
41
+
42
+ def __init__(self, collection, filter=None, projection=None,
43
+ sort=None, limit=0, pipeline=None):
44
+ self.collection = collection
45
+ self.filter = filter or {}
46
+ self.projection = projection if projection else {"_id": 0}
47
+ self.sort = sort
48
+ self.limit = limit
49
+ self.pipeline = pipeline
50
+
51
+
52
+ async def apply(self, df=None, data=None) -> pl.DataFrame:
53
+ coll = data["mongo"][self.collection]
54
+ if self.pipeline is not None:
55
+ cursor = await coll.aggregate(self.pipeline)
56
+ else:
57
+ cursor = coll.find(self.filter, self.projection)
58
+ if self.sort:
59
+ cursor = cursor.sort(self.sort)
60
+ if self.limit:
61
+ cursor = cursor.limit(self.limit)
62
+ docs = await cursor.to_list()
63
+ return pl.DataFrame(docs, infer_schema_length=None)
64
+
65
+
66
+ def __repr__(self):
67
+ return f"Read(collection={self.collection!r}, filter={self.filter})"
@@ -0,0 +1,4 @@
1
+ from etl.extractor.steps.mssql.connect import Connect
2
+ from etl.extractor.steps.mssql.read import Read
3
+
4
+ __all__ = ["Connect", "Read"]
@@ -0,0 +1,29 @@
1
+ from etl.generic.step import Step
2
+ from polars import DataFrame
3
+
4
+ class Connect(Step):
5
+ """
6
+ Open a Microsoft SQL Server connection and put it into data["mssql"].
7
+ Requires the Microsoft ODBC Driver for SQL Server installed on the host,
8
+ and the 'aioodbc' package (pip install aioodbc).
9
+
10
+ :param host/port/database/user/password: connection params.
11
+ :param driver: installed ODBC driver name.
12
+ :param kwargs: extra ODBC keywords appended to the connection string
13
+ (e.g. TrustServerCertificate="yes", Encrypt="no").
14
+ """
15
+
16
+ def __init__(self, host="localhost", port=1433, database="master",
17
+ user="sa", password="", driver="ODBC Driver 18 for SQL Server", **kwargs):
18
+ self.host, self.database = host, database
19
+ extra = "".join(f"{k}={v};" for k, v in kwargs.items())
20
+ self.dsn = (f"Driver={{{driver}}};Server={host},{port};Database={database};"
21
+ f"UID={user};PWD={password};{extra}")
22
+
23
+ async def apply(self, df: DataFrame = None, data=None):
24
+ import aioodbc
25
+ data["mssql"] = await aioodbc.connect(dsn=self.dsn, autocommit=True)
26
+ return df
27
+
28
+ def __repr__(self):
29
+ return f"ConnectMSSQL(host={self.host!r}, database={self.database!r})"
@@ -0,0 +1,33 @@
1
+ import polars as pl
2
+ from etl.generic.step import Step
3
+
4
+
5
+ class Read(Step):
6
+ """
7
+ Run a SQL query against SQL Server and make the result the pipeline df.
8
+
9
+ Reads the connection from data["mssql"] (set by Connect). Incoming df ignored.
10
+
11
+ :param query: T-SQL SELECT with ? placeholders for parameters.
12
+ :param parameters: values bound to the placeholders (injection-safe).
13
+
14
+ Example:
15
+ Read(
16
+ query="SELECT * FROM transactions WHERE txn_date >= ? AND bank = ?",
17
+ parameters=["2026-06-17", "acme"],
18
+ )
19
+ """
20
+
21
+ def __init__(self, query: str, parameters: list | None = None):
22
+ self.query = query
23
+ self.parameters = parameters or []
24
+
25
+ async def apply(self, df=None, data=None) -> pl.DataFrame:
26
+ async with data["mssql"].cursor() as cur:
27
+ await cur.execute(self.query, *self.parameters)
28
+ rows = await cur.fetchall()
29
+ cols = [d[0] for d in cur.description]
30
+ return pl.DataFrame([tuple(r) for r in rows], schema=cols, orient="row", infer_schema_length=None)
31
+
32
+ def __repr__(self):
33
+ return f"Read(query={self.query!r}, parameters={self.parameters})"
@@ -0,0 +1,4 @@
1
+ from etl.extractor.steps.postgresql.connect import Connect
2
+ from etl.extractor.steps.postgresql.read import Read
3
+
4
+ __all__ = ["Connect", "Read"]
@@ -0,0 +1,24 @@
1
+ from etl.generic.step import Step
2
+ from polars import DataFrame
3
+
4
+
5
+ class Connect(Step):
6
+ """
7
+ Open a PostgreSQL connection and put it into data["pg"] for downstream Read.
8
+
9
+ :param host/port/database/user/password: asyncpg connection params.
10
+ :param kwargs: extra args forwarded to asyncpg.connect (ssl, timeout, ...).
11
+ """
12
+
13
+ def __init__(self, host="localhost", port=5432, database="postgres",
14
+ user="postgres", password="", **kwargs):
15
+ self.config = dict(host=host, port=port, database=database,
16
+ user=user, password=password, **kwargs)
17
+
18
+ async def apply(self, df: DataFrame = None, data=None):
19
+ import asyncpg
20
+ data["pg"] = await asyncpg.connect(**self.config)
21
+ return df
22
+
23
+ def __repr__(self):
24
+ return f"ConnectPostgres(host={self.config['host']!r}, database={self.config['database']!r})"
@@ -0,0 +1,31 @@
1
+ import polars as pl
2
+ from etl.generic.step import Step
3
+
4
+
5
+ class Read(Step):
6
+ """
7
+ Run a SQL query against PostgreSQL and make the result the pipeline df.
8
+
9
+ Source step: reads the connection from data["pg"] (set by Connect),
10
+ runs the query, returns a Polars DataFrame. The incoming df is ignored.
11
+
12
+ :param query: SQL SELECT with $1, $2 placeholders.
13
+ :param parameters: values bound to the placeholders (injection-safe).
14
+
15
+ Example:
16
+ Read(
17
+ query="SELECT * FROM transactions WHERE txn_date >= $1 AND bank = $2",
18
+ parameters=["2026-06-17", "acme"],
19
+ )
20
+ """
21
+
22
+ def __init__(self, query: str, parameters: list | None = None):
23
+ self.query = query
24
+ self.parameters = parameters or []
25
+
26
+ async def apply(self, df=None, data=None) -> pl.DataFrame:
27
+ rows = await data["pg"].fetch(self.query, *self.parameters)
28
+ return pl.DataFrame([dict(r) for r in rows], infer_schema_length=None)
29
+
30
+ def __repr__(self):
31
+ return f"Read(query={self.query!r}, parameters={self.parameters})"
@@ -0,0 +1,4 @@
1
+ from etl.extractor.steps.sqlite.connect import Connect
2
+ from etl.extractor.steps.sqlite.read import Read
3
+
4
+ __all__ = ["Connect", "Read"]
@@ -0,0 +1,27 @@
1
+ from etl.generic.step import Step
2
+ from polars import DataFrame
3
+
4
+
5
+ class Connect(Step):
6
+ """
7
+ Open a SQLite connection and put it into data["sqlite"].
8
+
9
+ WARNING: SQLite is meant for tests and small local datasets. It is a
10
+ single-file, single-writer engine and is NOT suitable for storing large
11
+ data - use ClickHouse or Postgres for that. Great as an in-memory test
12
+ backend (path=":memory:").
13
+
14
+ :param path: database file path, or ":memory:" for an in-memory DB.
15
+ :param kwargs: extra args forwarded to aiosqlite.connect.
16
+ """
17
+
18
+ def __init__(self, path: str = ":memory:", **kwargs):
19
+ self.path, self.kwargs = path, kwargs
20
+
21
+ async def apply(self, df: DataFrame = None, data=None):
22
+ import aiosqlite
23
+ data["sqlite"] = await aiosqlite.connect(self.path, **self.kwargs)
24
+ return df
25
+
26
+ def __repr__(self):
27
+ return f"ConnectSQLite(path={self.path!r})"
@@ -0,0 +1,25 @@
1
+ import polars as pl
2
+ from etl.generic.step import Step
3
+
4
+
5
+ class Read(Step):
6
+ """
7
+ Run a SQL query against SQLite and make the result the pipeline df.
8
+ Reads the connection from data["sqlite"] (set by Connect).
9
+
10
+ :param query: SQL SELECT with ? placeholders.
11
+ :param parameters: values bound to the placeholders (injection-safe).
12
+ """
13
+
14
+ def __init__(self, query: str, parameters: list | None = None):
15
+ self.query = query
16
+ self.parameters = parameters or []
17
+
18
+ async def apply(self, df=None, data=None) -> pl.DataFrame:
19
+ async with data["sqlite"].execute(self.query, self.parameters) as cursor:
20
+ rows = await cursor.fetchall()
21
+ cols = [d[0] for d in cursor.description]
22
+ return pl.DataFrame([tuple(r) for r in rows], schema=cols, orient="row", infer_schema_length=None)
23
+
24
+ def __repr__(self):
25
+ return f"Read(query={self.query!r}, parameters={self.parameters})"
@@ -0,0 +1,5 @@
1
+ from etl.generic.pipeline import Pipeline, Pipestart
2
+ from etl.generic.context import Data
3
+ from etl.generic.step import Step, StopPipeline
4
+
5
+ __all__ = ["Step", "Data", "Pipeline", "Pipestart", "StopPipeline"]
etl/generic/context.py ADDED
@@ -0,0 +1,36 @@
1
+ from __future__ import annotations
2
+ from typing import Any
3
+ from polars import DataFrame, LazyFrame
4
+
5
+ class Data:
6
+ '''
7
+ General data within the pipline; all steps communicate thought this context.
8
+
9
+ Anti-pattern: Only simple data should be stored here; heavy dataframes must not be placed here.
10
+ Solution: Extract dataframe to data lake and read them frome there.
11
+ This is required for auditing raw data.
12
+ '''
13
+
14
+ def __init__(self, **values: Any) -> None:
15
+ self.store: dict[str, Any] = dict(values)
16
+
17
+ def get(self, key: str, default: Any = None) -> Any:
18
+ return self.store.get(key, default)
19
+
20
+ def __getitem__(self, key: str) -> Any:
21
+ return self.store[key]
22
+
23
+ def __setitem__(self, key:str, value: Any) -> None:
24
+ if isinstance(value, (DataFrame, LazyFrame)):
25
+ raise TypeError(
26
+ f"Data['{key}'] is Dataframe is antipattern,"
27
+ f"should be exchanged between steps via raw exports.")
28
+
29
+ self.store[key] = value
30
+
31
+ def __contains__(self, key: str) -> bool:
32
+ return key in self.store
33
+
34
+ def __repr__(self) -> str:
35
+ body = ", ".join(f"{key}: {type(value).__name__}" for key,value in self.store.items())
36
+ return f"Data({body})"