etlplus 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/README.md +37 -0
- etlplus/__init__.py +1 -26
- etlplus/api/README.md +51 -3
- etlplus/api/__init__.py +10 -0
- etlplus/api/config.py +39 -28
- etlplus/api/endpoint_client.py +3 -3
- etlplus/api/enums.py +51 -0
- etlplus/api/pagination/client.py +1 -1
- etlplus/api/rate_limiting/config.py +13 -1
- etlplus/api/rate_limiting/rate_limiter.py +8 -11
- etlplus/api/request_manager.py +11 -6
- etlplus/api/transport.py +14 -2
- etlplus/api/types.py +96 -6
- etlplus/{run_helpers.py → api/utils.py} +209 -153
- etlplus/cli/README.md +40 -0
- etlplus/cli/commands.py +94 -61
- etlplus/cli/constants.py +1 -1
- etlplus/cli/handlers.py +40 -12
- etlplus/cli/io.py +2 -2
- etlplus/cli/main.py +1 -1
- etlplus/cli/state.py +4 -7
- etlplus/database/README.md +48 -0
- etlplus/database/ddl.py +1 -1
- etlplus/database/engine.py +19 -3
- etlplus/database/orm.py +2 -0
- etlplus/database/schema.py +1 -1
- etlplus/enums.py +1 -107
- etlplus/file/README.md +105 -0
- etlplus/file/__init__.py +25 -0
- etlplus/file/_imports.py +141 -0
- etlplus/file/_io.py +160 -0
- etlplus/file/accdb.py +78 -0
- etlplus/file/arrow.py +78 -0
- etlplus/file/avro.py +176 -0
- etlplus/file/bson.py +77 -0
- etlplus/file/cbor.py +78 -0
- etlplus/file/cfg.py +79 -0
- etlplus/file/conf.py +80 -0
- etlplus/file/core.py +322 -0
- etlplus/file/csv.py +79 -0
- etlplus/file/dat.py +78 -0
- etlplus/file/dta.py +77 -0
- etlplus/file/duckdb.py +78 -0
- etlplus/file/enums.py +343 -0
- etlplus/file/feather.py +111 -0
- etlplus/file/fwf.py +77 -0
- etlplus/file/gz.py +123 -0
- etlplus/file/hbs.py +78 -0
- etlplus/file/hdf5.py +78 -0
- etlplus/file/ini.py +79 -0
- etlplus/file/ion.py +78 -0
- etlplus/file/jinja2.py +78 -0
- etlplus/file/json.py +98 -0
- etlplus/file/log.py +78 -0
- etlplus/file/mat.py +78 -0
- etlplus/file/mdb.py +78 -0
- etlplus/file/msgpack.py +78 -0
- etlplus/file/mustache.py +78 -0
- etlplus/file/nc.py +78 -0
- etlplus/file/ndjson.py +108 -0
- etlplus/file/numbers.py +75 -0
- etlplus/file/ods.py +79 -0
- etlplus/file/orc.py +111 -0
- etlplus/file/parquet.py +113 -0
- etlplus/file/pb.py +78 -0
- etlplus/file/pbf.py +77 -0
- etlplus/file/properties.py +78 -0
- etlplus/file/proto.py +77 -0
- etlplus/file/psv.py +79 -0
- etlplus/file/rda.py +78 -0
- etlplus/file/rds.py +78 -0
- etlplus/file/sas7bdat.py +78 -0
- etlplus/file/sav.py +77 -0
- etlplus/file/sqlite.py +78 -0
- etlplus/file/stub.py +84 -0
- etlplus/file/sylk.py +77 -0
- etlplus/file/tab.py +81 -0
- etlplus/file/toml.py +78 -0
- etlplus/file/tsv.py +80 -0
- etlplus/file/txt.py +102 -0
- etlplus/file/vm.py +78 -0
- etlplus/file/wks.py +77 -0
- etlplus/file/xls.py +88 -0
- etlplus/file/xlsm.py +79 -0
- etlplus/file/xlsx.py +99 -0
- etlplus/file/xml.py +185 -0
- etlplus/file/xpt.py +78 -0
- etlplus/file/yaml.py +95 -0
- etlplus/file/zip.py +175 -0
- etlplus/file/zsav.py +77 -0
- etlplus/ops/README.md +50 -0
- etlplus/ops/__init__.py +61 -0
- etlplus/{extract.py → ops/extract.py} +81 -99
- etlplus/{load.py → ops/load.py} +78 -101
- etlplus/{run.py → ops/run.py} +159 -127
- etlplus/{transform.py → ops/transform.py} +75 -68
- etlplus/{validation → ops}/utils.py +53 -17
- etlplus/{validate.py → ops/validate.py} +22 -12
- etlplus/templates/README.md +46 -0
- etlplus/types.py +5 -4
- etlplus/utils.py +136 -2
- etlplus/workflow/README.md +52 -0
- etlplus/{config → workflow}/__init__.py +10 -23
- etlplus/{config → workflow}/connector.py +58 -44
- etlplus/workflow/dag.py +105 -0
- etlplus/{config → workflow}/jobs.py +105 -32
- etlplus/{config → workflow}/pipeline.py +59 -51
- etlplus/{config → workflow}/profile.py +8 -5
- etlplus/workflow/types.py +115 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/METADATA +210 -17
- etlplus-0.9.2.dist-info/RECORD +134 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/WHEEL +1 -1
- etlplus/config/types.py +0 -204
- etlplus/config/utils.py +0 -120
- etlplus/file.py +0 -657
- etlplus/validation/__init__.py +0 -44
- etlplus-0.9.0.dist-info/RECORD +0 -65
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/entry_points.txt +0 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/top_level.txt +0 -0
etlplus/api/types.py
CHANGED
|
@@ -20,8 +20,11 @@ Examples
|
|
|
20
20
|
from __future__ import annotations
|
|
21
21
|
|
|
22
22
|
from collections.abc import Callable
|
|
23
|
+
from collections.abc import Mapping
|
|
23
24
|
from dataclasses import dataclass
|
|
24
25
|
from typing import Any
|
|
26
|
+
from typing import Self
|
|
27
|
+
from typing import TypedDict
|
|
25
28
|
from typing import cast
|
|
26
29
|
|
|
27
30
|
from ..types import JSONData
|
|
@@ -39,6 +42,11 @@ __all__ = [
|
|
|
39
42
|
'Headers',
|
|
40
43
|
'Params',
|
|
41
44
|
'Url',
|
|
45
|
+
# Typed Dicts
|
|
46
|
+
'ApiConfigMap',
|
|
47
|
+
'ApiProfileConfigMap',
|
|
48
|
+
'ApiProfileDefaultsMap',
|
|
49
|
+
'EndpointMap',
|
|
42
50
|
]
|
|
43
51
|
|
|
44
52
|
|
|
@@ -48,6 +56,88 @@ __all__ = [
|
|
|
48
56
|
_UNSET = object()
|
|
49
57
|
|
|
50
58
|
|
|
59
|
+
# SECTION: TYPED DICTS ====================================================== #
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ApiConfigMap(TypedDict, total=False):
|
|
63
|
+
"""
|
|
64
|
+
Top-level API config shape parsed by ApiConfig.from_obj.
|
|
65
|
+
|
|
66
|
+
Either provide a ``base_url`` with optional ``headers`` and ``endpoints``,
|
|
67
|
+
or provide ``profiles`` with at least one profile having a ``base_url``.
|
|
68
|
+
|
|
69
|
+
See Also
|
|
70
|
+
--------
|
|
71
|
+
- :class:`etlplus.api.config.ApiConfig`
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
base_url: str
|
|
75
|
+
headers: StrAnyMap
|
|
76
|
+
endpoints: Mapping[str, EndpointMap | str]
|
|
77
|
+
profiles: Mapping[str, ApiProfileConfigMap]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class ApiProfileConfigMap(TypedDict, total=False):
|
|
81
|
+
"""
|
|
82
|
+
Shape accepted for a profile entry under ApiConfigMap.profiles.
|
|
83
|
+
|
|
84
|
+
Notes
|
|
85
|
+
-----
|
|
86
|
+
``base_url`` is required at runtime when profiles are provided.
|
|
87
|
+
|
|
88
|
+
See Also
|
|
89
|
+
--------
|
|
90
|
+
- :class:`etlplus.api.config.ApiProfileConfig`
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
base_url: str
|
|
94
|
+
headers: StrAnyMap
|
|
95
|
+
base_path: str
|
|
96
|
+
auth: StrAnyMap
|
|
97
|
+
defaults: ApiProfileDefaultsMap
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class ApiProfileDefaultsMap(TypedDict, total=False):
|
|
101
|
+
"""
|
|
102
|
+
Defaults block available under a profile (all keys optional).
|
|
103
|
+
|
|
104
|
+
Notes
|
|
105
|
+
-----
|
|
106
|
+
Runtime expects header values to be str; typing remains permissive.
|
|
107
|
+
|
|
108
|
+
See Also
|
|
109
|
+
--------
|
|
110
|
+
- :class:`etlplus.api.config.ApiProfileConfig`
|
|
111
|
+
- :class:`etlplus.api.pagination.PaginationConfig`
|
|
112
|
+
- :class:`etlplus.api.rate_limiting.RateLimitConfig`
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
headers: StrAnyMap
|
|
116
|
+
pagination: Any
|
|
117
|
+
rate_limit: Any
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class EndpointMap(TypedDict, total=False):
|
|
121
|
+
"""
|
|
122
|
+
Shape accepted by EndpointConfig.from_obj.
|
|
123
|
+
|
|
124
|
+
One of ``path`` or ``url`` should be provided.
|
|
125
|
+
|
|
126
|
+
See Also
|
|
127
|
+
--------
|
|
128
|
+
- :class:`etlplus.api.config.EndpointConfig`
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
path: str
|
|
132
|
+
url: str
|
|
133
|
+
method: str
|
|
134
|
+
path_params: StrAnyMap
|
|
135
|
+
query_params: StrAnyMap
|
|
136
|
+
body: Any
|
|
137
|
+
pagination: Any
|
|
138
|
+
rate_limit: Any
|
|
139
|
+
|
|
140
|
+
|
|
51
141
|
# SECTION: DATA CLASSES ===================================================== #
|
|
52
142
|
|
|
53
143
|
|
|
@@ -75,9 +165,9 @@ class RequestOptions:
|
|
|
75
165
|
# -- Magic Methods (Object Lifecycle) -- #
|
|
76
166
|
|
|
77
167
|
def __post_init__(self) -> None:
|
|
78
|
-
if self.params:
|
|
168
|
+
if self.params is not None:
|
|
79
169
|
object.__setattr__(self, 'params', dict(self.params))
|
|
80
|
-
if self.headers:
|
|
170
|
+
if self.headers is not None:
|
|
81
171
|
object.__setattr__(self, 'headers', dict(self.headers))
|
|
82
172
|
|
|
83
173
|
# -- Instance Methods -- #
|
|
@@ -92,9 +182,9 @@ class RequestOptions:
|
|
|
92
182
|
Keyword arguments for ``requests`` methods.
|
|
93
183
|
"""
|
|
94
184
|
kw: dict[str, Any] = {}
|
|
95
|
-
if self.params:
|
|
185
|
+
if self.params is not None:
|
|
96
186
|
kw['params'] = dict(self.params)
|
|
97
|
-
if self.headers:
|
|
187
|
+
if self.headers is not None:
|
|
98
188
|
kw['headers'] = dict(self.headers)
|
|
99
189
|
if self.timeout is not None:
|
|
100
190
|
kw['timeout'] = self.timeout
|
|
@@ -106,7 +196,7 @@ class RequestOptions:
|
|
|
106
196
|
params: Params | None | object = _UNSET,
|
|
107
197
|
headers: Headers | None | object = _UNSET,
|
|
108
198
|
timeout: float | None | object = _UNSET,
|
|
109
|
-
) ->
|
|
199
|
+
) -> Self:
|
|
110
200
|
"""
|
|
111
201
|
Return a copy with the provided fields replaced.
|
|
112
202
|
|
|
@@ -146,7 +236,7 @@ class RequestOptions:
|
|
|
146
236
|
else:
|
|
147
237
|
next_timeout = cast(float | None, timeout)
|
|
148
238
|
|
|
149
|
-
return
|
|
239
|
+
return self.__class__(
|
|
150
240
|
params=next_params,
|
|
151
241
|
headers=next_headers,
|
|
152
242
|
timeout=next_timeout,
|