etlplus 0.16.4__py3-none-any.whl → 0.16.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/api/__init__.py +14 -14
- etlplus/api/auth.py +9 -6
- etlplus/api/config.py +6 -6
- etlplus/api/endpoint_client.py +16 -16
- etlplus/api/errors.py +4 -4
- etlplus/api/pagination/__init__.py +6 -6
- etlplus/api/pagination/config.py +11 -9
- etlplus/api/rate_limiting/__init__.py +2 -2
- etlplus/api/rate_limiting/config.py +10 -10
- etlplus/api/rate_limiting/rate_limiter.py +2 -2
- etlplus/api/request_manager.py +4 -4
- etlplus/api/retry_manager.py +6 -6
- etlplus/api/transport.py +10 -10
- etlplus/api/types.py +15 -15
- etlplus/api/utils.py +49 -49
- etlplus/cli/handlers.py +4 -4
- etlplus/connector/__init__.py +6 -6
- etlplus/connector/api.py +7 -7
- etlplus/connector/database.py +3 -3
- etlplus/connector/file.py +3 -3
- etlplus/connector/types.py +2 -2
- etlplus/ops/extract.py +2 -2
- etlplus/ops/utils.py +5 -5
- etlplus/ops/validate.py +13 -13
- {etlplus-0.16.4.dist-info → etlplus-0.16.5.dist-info}/METADATA +1 -1
- {etlplus-0.16.4.dist-info → etlplus-0.16.5.dist-info}/RECORD +30 -30
- {etlplus-0.16.4.dist-info → etlplus-0.16.5.dist-info}/WHEEL +0 -0
- {etlplus-0.16.4.dist-info → etlplus-0.16.5.dist-info}/entry_points.txt +0 -0
- {etlplus-0.16.4.dist-info → etlplus-0.16.5.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.16.4.dist-info → etlplus-0.16.5.dist-info}/top_level.txt +0 -0
etlplus/api/types.py
CHANGED
|
@@ -43,10 +43,10 @@ __all__ = [
|
|
|
43
43
|
'Params',
|
|
44
44
|
'Url',
|
|
45
45
|
# Typed Dicts
|
|
46
|
-
'
|
|
47
|
-
'
|
|
48
|
-
'
|
|
49
|
-
'
|
|
46
|
+
'ApiConfigDict',
|
|
47
|
+
'ApiProfileConfigDict',
|
|
48
|
+
'ApiProfileDefaultsDict',
|
|
49
|
+
'EndpointConfigDict',
|
|
50
50
|
]
|
|
51
51
|
|
|
52
52
|
|
|
@@ -83,7 +83,7 @@ def _to_dict(
|
|
|
83
83
|
# SECTION: TYPED DICTS ====================================================== #
|
|
84
84
|
|
|
85
85
|
|
|
86
|
-
class
|
|
86
|
+
class ApiConfigDict(TypedDict, total=False):
|
|
87
87
|
"""
|
|
88
88
|
Top-level API config shape parsed by
|
|
89
89
|
:meth:`etlplus.api.config.ApiConfig.from_obj`.
|
|
@@ -99,11 +99,11 @@ class ApiConfigMap(TypedDict, total=False):
|
|
|
99
99
|
|
|
100
100
|
base_url: str
|
|
101
101
|
headers: StrAnyMap
|
|
102
|
-
endpoints: Mapping[str,
|
|
103
|
-
profiles: Mapping[str,
|
|
102
|
+
endpoints: Mapping[str, EndpointConfigDict | str]
|
|
103
|
+
profiles: Mapping[str, ApiProfileConfigDict]
|
|
104
104
|
|
|
105
105
|
|
|
106
|
-
class
|
|
106
|
+
class ApiProfileConfigDict(TypedDict, total=False):
|
|
107
107
|
"""
|
|
108
108
|
Shape accepted for a profile entry under
|
|
109
109
|
:meth:`etlplus.api.config.ApiConfig.from_obj`.
|
|
@@ -123,10 +123,10 @@ class ApiProfileConfigMap(TypedDict, total=False):
|
|
|
123
123
|
headers: StrAnyMap
|
|
124
124
|
base_path: str
|
|
125
125
|
auth: StrAnyMap
|
|
126
|
-
defaults:
|
|
126
|
+
defaults: ApiProfileDefaultsDict
|
|
127
127
|
|
|
128
128
|
|
|
129
|
-
class
|
|
129
|
+
class ApiProfileDefaultsDict(TypedDict, total=False):
|
|
130
130
|
"""
|
|
131
131
|
Defaults block available under a profile (all keys optional).
|
|
132
132
|
|
|
@@ -147,11 +147,11 @@ class ApiProfileDefaultsMap(TypedDict, total=False):
|
|
|
147
147
|
"""
|
|
148
148
|
|
|
149
149
|
headers: StrAnyMap
|
|
150
|
-
pagination: StrAnyMap #
|
|
151
|
-
rate_limit: StrAnyMap #
|
|
150
|
+
pagination: StrAnyMap # PaginationConfigDict | StrAnyMap
|
|
151
|
+
rate_limit: StrAnyMap # RateLimitConfigDict | StrAnyMap
|
|
152
152
|
|
|
153
153
|
|
|
154
|
-
class
|
|
154
|
+
class EndpointConfigDict(TypedDict, total=False):
|
|
155
155
|
"""
|
|
156
156
|
Shape accepted by :meth:`etlplus.api.config.EndpointConfig.from_obj`.
|
|
157
157
|
|
|
@@ -168,8 +168,8 @@ class EndpointMap(TypedDict, total=False):
|
|
|
168
168
|
path_params: StrAnyMap
|
|
169
169
|
query_params: StrAnyMap
|
|
170
170
|
body: Any
|
|
171
|
-
pagination: StrAnyMap #
|
|
172
|
-
rate_limit: StrAnyMap #
|
|
171
|
+
pagination: StrAnyMap # PaginationConfigDict | StrAnyMap
|
|
172
|
+
rate_limit: StrAnyMap # RateLimitConfigDict | StrAnyMap
|
|
173
173
|
|
|
174
174
|
|
|
175
175
|
# SECTION: DATA CLASSES ===================================================== #
|
etlplus/api/utils.py
CHANGED
|
@@ -22,11 +22,11 @@ from .config import EndpointConfig
|
|
|
22
22
|
from .endpoint_client import EndpointClient
|
|
23
23
|
from .enums import HttpMethod
|
|
24
24
|
from .pagination import PaginationConfig
|
|
25
|
-
from .pagination import
|
|
25
|
+
from .pagination import PaginationConfigDict
|
|
26
26
|
from .rate_limiting import RateLimitConfig
|
|
27
|
-
from .rate_limiting import
|
|
27
|
+
from .rate_limiting import RateLimitConfigDict
|
|
28
28
|
from .rate_limiting import RateLimiter
|
|
29
|
-
from .retry_manager import
|
|
29
|
+
from .retry_manager import RetryPolicyDict
|
|
30
30
|
from .types import Headers
|
|
31
31
|
from .types import Params
|
|
32
32
|
from .types import Url
|
|
@@ -53,16 +53,16 @@ __all__ = [
|
|
|
53
53
|
'paginate_with_client',
|
|
54
54
|
'resolve_request',
|
|
55
55
|
# Typed Dicts
|
|
56
|
-
'
|
|
57
|
-
'
|
|
58
|
-
'
|
|
56
|
+
'ApiRequestEnvDict',
|
|
57
|
+
'ApiTargetEnvDict',
|
|
58
|
+
'SessionConfigDict',
|
|
59
59
|
]
|
|
60
60
|
|
|
61
61
|
|
|
62
62
|
# SECTION: TYPED DICTS ====================================================== #
|
|
63
63
|
|
|
64
64
|
|
|
65
|
-
class
|
|
65
|
+
class BaseApiHttpEnvDict(TypedDict, total=False):
|
|
66
66
|
"""
|
|
67
67
|
Common HTTP request environment for API interactions.
|
|
68
68
|
|
|
@@ -78,7 +78,7 @@ class BaseApiHttpEnv(TypedDict, total=False):
|
|
|
78
78
|
session: requests.Session | None
|
|
79
79
|
|
|
80
80
|
|
|
81
|
-
class
|
|
81
|
+
class ApiRequestEnvDict(BaseApiHttpEnvDict, total=False):
|
|
82
82
|
"""
|
|
83
83
|
Composed HTTP request environment configuration for REST API sources.
|
|
84
84
|
|
|
@@ -96,15 +96,15 @@ class ApiRequestEnv(BaseApiHttpEnv, total=False):
|
|
|
96
96
|
|
|
97
97
|
# Request
|
|
98
98
|
params: dict[str, Any]
|
|
99
|
-
pagination:
|
|
99
|
+
pagination: PaginationConfigDict | None
|
|
100
100
|
sleep_seconds: float
|
|
101
101
|
|
|
102
102
|
# Reliability
|
|
103
|
-
retry:
|
|
103
|
+
retry: RetryPolicyDict | None
|
|
104
104
|
retry_network_errors: bool
|
|
105
105
|
|
|
106
106
|
|
|
107
|
-
class
|
|
107
|
+
class ApiTargetEnvDict(BaseApiHttpEnvDict, total=False):
|
|
108
108
|
"""
|
|
109
109
|
Composed HTTP request environment configuration for REST API targets.
|
|
110
110
|
|
|
@@ -126,7 +126,7 @@ class ApiTargetEnv(BaseApiHttpEnv, total=False):
|
|
|
126
126
|
method: str | None
|
|
127
127
|
|
|
128
128
|
|
|
129
|
-
class
|
|
129
|
+
class SessionConfigDict(TypedDict, total=False):
|
|
130
130
|
"""
|
|
131
131
|
Minimal session configuration schema accepted by the
|
|
132
132
|
:class:`requests.Session` runner.
|
|
@@ -148,14 +148,14 @@ class SessionConfig(TypedDict, total=False):
|
|
|
148
148
|
|
|
149
149
|
|
|
150
150
|
def _build_session_optional(
|
|
151
|
-
cfg:
|
|
151
|
+
cfg: SessionConfigDict | None,
|
|
152
152
|
) -> requests.Session | None:
|
|
153
153
|
"""
|
|
154
154
|
Return a configured session when *cfg* is a mapping.
|
|
155
155
|
|
|
156
156
|
Parameters
|
|
157
157
|
----------
|
|
158
|
-
cfg :
|
|
158
|
+
cfg : SessionConfigDict | None
|
|
159
159
|
Session configuration mapping.
|
|
160
160
|
|
|
161
161
|
Returns
|
|
@@ -164,7 +164,7 @@ def _build_session_optional(
|
|
|
164
164
|
Configured session or ``None``.
|
|
165
165
|
"""
|
|
166
166
|
if isinstance(cfg, Mapping):
|
|
167
|
-
return build_session(cast(
|
|
167
|
+
return build_session(cast(SessionConfigDict, cfg))
|
|
168
168
|
return None
|
|
169
169
|
|
|
170
170
|
|
|
@@ -233,9 +233,9 @@ def _inherit_http_from_api_endpoint(
|
|
|
233
233
|
ep: EndpointConfig,
|
|
234
234
|
url: Url | None,
|
|
235
235
|
headers: dict[str, str],
|
|
236
|
-
session_cfg:
|
|
236
|
+
session_cfg: SessionConfigDict | None,
|
|
237
237
|
force_url: bool = False,
|
|
238
|
-
) -> tuple[Url | None, dict[str, str],
|
|
238
|
+
) -> tuple[Url | None, dict[str, str], SessionConfigDict | None]:
|
|
239
239
|
"""
|
|
240
240
|
Return HTTP settings inherited from API + endpoint definitions.
|
|
241
241
|
|
|
@@ -249,14 +249,14 @@ def _inherit_http_from_api_endpoint(
|
|
|
249
249
|
Existing URL to use when not forcing endpoint URL.
|
|
250
250
|
headers : dict[str, str]
|
|
251
251
|
Existing headers to augment.
|
|
252
|
-
session_cfg :
|
|
252
|
+
session_cfg : SessionConfigDict | None
|
|
253
253
|
Existing session configuration to augment.
|
|
254
254
|
force_url : bool, optional
|
|
255
255
|
Whether to always use the endpoint URL.
|
|
256
256
|
|
|
257
257
|
Returns
|
|
258
258
|
-------
|
|
259
|
-
tuple[Url | None, dict[str, str],
|
|
259
|
+
tuple[Url | None, dict[str, str], SessionConfigDict | None]
|
|
260
260
|
Resolved URL, headers, and session configuration.
|
|
261
261
|
"""
|
|
262
262
|
if force_url or not url:
|
|
@@ -269,8 +269,8 @@ def _inherit_http_from_api_endpoint(
|
|
|
269
269
|
def _merge_session_cfg_three(
|
|
270
270
|
api_cfg: ApiConfig,
|
|
271
271
|
ep: EndpointConfig,
|
|
272
|
-
source_session_cfg:
|
|
273
|
-
) ->
|
|
272
|
+
source_session_cfg: SessionConfigDict | None,
|
|
273
|
+
) -> SessionConfigDict | None:
|
|
274
274
|
"""
|
|
275
275
|
Merge session configurations from API, endpoint, and source.
|
|
276
276
|
|
|
@@ -280,12 +280,12 @@ def _merge_session_cfg_three(
|
|
|
280
280
|
API configuration.
|
|
281
281
|
ep : EndpointConfig
|
|
282
282
|
Endpoint configuration.
|
|
283
|
-
source_session_cfg :
|
|
283
|
+
source_session_cfg : SessionConfigDict | None
|
|
284
284
|
Source session configuration.
|
|
285
285
|
|
|
286
286
|
Returns
|
|
287
287
|
-------
|
|
288
|
-
|
|
288
|
+
SessionConfigDict | None
|
|
289
289
|
Merged session configuration.
|
|
290
290
|
"""
|
|
291
291
|
api_sess = getattr(api_cfg, 'session', None)
|
|
@@ -297,7 +297,7 @@ def _merge_session_cfg_three(
|
|
|
297
297
|
merged.update(ep_sess)
|
|
298
298
|
if isinstance(source_session_cfg, Mapping):
|
|
299
299
|
merged.update(source_session_cfg)
|
|
300
|
-
return cast(
|
|
300
|
+
return cast(SessionConfigDict | None, (merged or None))
|
|
301
301
|
|
|
302
302
|
|
|
303
303
|
def _update_mapping(
|
|
@@ -361,7 +361,7 @@ def compose_api_request_env(
|
|
|
361
361
|
cfg: Any,
|
|
362
362
|
source_obj: Any,
|
|
363
363
|
ex_opts: Mapping[str, Any] | None,
|
|
364
|
-
) ->
|
|
364
|
+
) -> ApiRequestEnvDict:
|
|
365
365
|
"""
|
|
366
366
|
Compose the API request environment.
|
|
367
367
|
|
|
@@ -376,7 +376,7 @@ def compose_api_request_env(
|
|
|
376
376
|
|
|
377
377
|
Returns
|
|
378
378
|
-------
|
|
379
|
-
|
|
379
|
+
ApiRequestEnvDict
|
|
380
380
|
The composed API request environment.
|
|
381
381
|
"""
|
|
382
382
|
ex_opts = ex_opts or {}
|
|
@@ -393,13 +393,13 @@ def compose_api_request_env(
|
|
|
393
393
|
headers: dict[str, str] = cast(dict[str, str], coerce_dict(source_headers))
|
|
394
394
|
pagination = getattr(source_obj, 'pagination', None)
|
|
395
395
|
rate_limit = getattr(source_obj, 'rate_limit', None)
|
|
396
|
-
retry:
|
|
397
|
-
|
|
396
|
+
retry: RetryPolicyDict | None = cast(
|
|
397
|
+
RetryPolicyDict | None,
|
|
398
398
|
getattr(source_obj, 'retry', None),
|
|
399
399
|
)
|
|
400
400
|
retry_network_errors = getattr(source_obj, 'retry_network_errors', None)
|
|
401
401
|
session_cfg = cast(
|
|
402
|
-
|
|
402
|
+
SessionConfigDict | None,
|
|
403
403
|
getattr(source_obj, 'session', None),
|
|
404
404
|
)
|
|
405
405
|
api_name = getattr(source_obj, 'api', None)
|
|
@@ -435,7 +435,7 @@ def compose_api_request_env(
|
|
|
435
435
|
api_cfg.effective_rate_limit_defaults(),
|
|
436
436
|
)
|
|
437
437
|
retry = cast(
|
|
438
|
-
|
|
438
|
+
RetryPolicyDict | None,
|
|
439
439
|
_coalesce(
|
|
440
440
|
retry,
|
|
441
441
|
getattr(ep, 'retry', None),
|
|
@@ -465,8 +465,8 @@ def compose_api_request_env(
|
|
|
465
465
|
timeout: Timeout = ex_opts.get('timeout')
|
|
466
466
|
pag_ov = ex_opts.get('pagination', {})
|
|
467
467
|
rl_ov = ex_opts.get('rate_limit', {})
|
|
468
|
-
rty_ov:
|
|
469
|
-
|
|
468
|
+
rty_ov: RetryPolicyDict | None = cast(
|
|
469
|
+
RetryPolicyDict | None,
|
|
470
470
|
(ex_opts.get('retry') if 'retry' in ex_opts else None),
|
|
471
471
|
)
|
|
472
472
|
rne_ov = (
|
|
@@ -474,7 +474,7 @@ def compose_api_request_env(
|
|
|
474
474
|
if 'retry_network_errors' in ex_opts
|
|
475
475
|
else None
|
|
476
476
|
)
|
|
477
|
-
sess_ov = cast(
|
|
477
|
+
sess_ov = cast(SessionConfigDict | None, ex_opts.get('session'))
|
|
478
478
|
sleep_s = compute_rl_sleep_seconds(rate_limit, rl_ov) or 0.0
|
|
479
479
|
if rty_ov is not None:
|
|
480
480
|
retry = rty_ov
|
|
@@ -485,8 +485,8 @@ def compose_api_request_env(
|
|
|
485
485
|
cast(Mapping[str, Any], session_cfg or {}),
|
|
486
486
|
)
|
|
487
487
|
base_cfg.update(sess_ov)
|
|
488
|
-
session_cfg = cast(
|
|
489
|
-
pag_cfg:
|
|
488
|
+
session_cfg = cast(SessionConfigDict, base_cfg)
|
|
489
|
+
pag_cfg: PaginationConfigDict | None = build_pagination_cfg(
|
|
490
490
|
pagination,
|
|
491
491
|
pag_ov,
|
|
492
492
|
)
|
|
@@ -513,7 +513,7 @@ def compose_api_target_env(
|
|
|
513
513
|
cfg: Any,
|
|
514
514
|
target_obj: Any,
|
|
515
515
|
overrides: Mapping[str, Any] | None,
|
|
516
|
-
) ->
|
|
516
|
+
) -> ApiTargetEnvDict:
|
|
517
517
|
"""
|
|
518
518
|
Compose the API target environment.
|
|
519
519
|
|
|
@@ -528,7 +528,7 @@ def compose_api_target_env(
|
|
|
528
528
|
|
|
529
529
|
Returns
|
|
530
530
|
-------
|
|
531
|
-
|
|
531
|
+
ApiTargetEnvDict
|
|
532
532
|
Composed API target environment.
|
|
533
533
|
"""
|
|
534
534
|
ov = overrides or {}
|
|
@@ -553,8 +553,8 @@ def compose_api_target_env(
|
|
|
553
553
|
timeout: Timeout = (
|
|
554
554
|
cast(Timeout, ov.get('timeout')) if 'timeout' in ov else None
|
|
555
555
|
)
|
|
556
|
-
sess_cfg:
|
|
557
|
-
|
|
556
|
+
sess_cfg: SessionConfigDict | None = cast(
|
|
557
|
+
SessionConfigDict | None,
|
|
558
558
|
ov.get('session'),
|
|
559
559
|
)
|
|
560
560
|
api_name = getattr(target_obj, 'api', None)
|
|
@@ -583,7 +583,7 @@ def compose_api_target_env(
|
|
|
583
583
|
def build_pagination_cfg(
|
|
584
584
|
pagination: PaginationConfig | None,
|
|
585
585
|
overrides: Mapping[str, Any] | None,
|
|
586
|
-
) ->
|
|
586
|
+
) -> PaginationConfigDict | None:
|
|
587
587
|
"""
|
|
588
588
|
Build pagination configuration.
|
|
589
589
|
|
|
@@ -596,7 +596,7 @@ def build_pagination_cfg(
|
|
|
596
596
|
|
|
597
597
|
Returns
|
|
598
598
|
-------
|
|
599
|
-
|
|
599
|
+
PaginationConfigDict | None
|
|
600
600
|
Pagination configuration.
|
|
601
601
|
"""
|
|
602
602
|
ptype: str | None = None
|
|
@@ -683,7 +683,7 @@ def build_pagination_cfg(
|
|
|
683
683
|
case _:
|
|
684
684
|
pass
|
|
685
685
|
|
|
686
|
-
return cast(
|
|
686
|
+
return cast(PaginationConfigDict, cfg)
|
|
687
687
|
|
|
688
688
|
|
|
689
689
|
def paginate_with_client(
|
|
@@ -692,7 +692,7 @@ def paginate_with_client(
|
|
|
692
692
|
params: Params | None,
|
|
693
693
|
headers: Headers | None,
|
|
694
694
|
timeout: Timeout,
|
|
695
|
-
pagination:
|
|
695
|
+
pagination: PaginationConfigDict | None,
|
|
696
696
|
sleep_seconds: float | None,
|
|
697
697
|
) -> Any:
|
|
698
698
|
"""
|
|
@@ -710,7 +710,7 @@ def paginate_with_client(
|
|
|
710
710
|
Headers to include in the API request.
|
|
711
711
|
timeout : Timeout
|
|
712
712
|
Timeout configuration for the API request.
|
|
713
|
-
pagination :
|
|
713
|
+
pagination : PaginationConfigDict | None
|
|
714
714
|
Pagination configuration for the API request.
|
|
715
715
|
sleep_seconds : float | None
|
|
716
716
|
Sleep duration between API requests.
|
|
@@ -771,9 +771,9 @@ def compute_rl_sleep_seconds(
|
|
|
771
771
|
else:
|
|
772
772
|
rl_map = cast(Mapping[str, Any] | None, rate_limit)
|
|
773
773
|
|
|
774
|
-
rl_mapping = cast(
|
|
774
|
+
rl_mapping = cast(RateLimitConfigDict | None, rl_map)
|
|
775
775
|
|
|
776
|
-
typed_override:
|
|
776
|
+
typed_override: RateLimitConfigDict | None = None
|
|
777
777
|
if overrides:
|
|
778
778
|
filtered: dict[str, float | None] = {}
|
|
779
779
|
if 'sleep_seconds' in overrides:
|
|
@@ -787,7 +787,7 @@ def compute_rl_sleep_seconds(
|
|
|
787
787
|
overrides.get('max_per_sec'),
|
|
788
788
|
)
|
|
789
789
|
if filtered:
|
|
790
|
-
typed_override = cast(
|
|
790
|
+
typed_override = cast(RateLimitConfigDict, filtered)
|
|
791
791
|
|
|
792
792
|
return RateLimiter.resolve_sleep_seconds(
|
|
793
793
|
rate_limit=rl_mapping,
|
|
@@ -796,14 +796,14 @@ def compute_rl_sleep_seconds(
|
|
|
796
796
|
|
|
797
797
|
|
|
798
798
|
def build_session(
|
|
799
|
-
cfg:
|
|
799
|
+
cfg: SessionConfigDict | None,
|
|
800
800
|
) -> requests.Session:
|
|
801
801
|
"""
|
|
802
802
|
Build a requests.Session object with the given configuration.
|
|
803
803
|
|
|
804
804
|
Parameters
|
|
805
805
|
----------
|
|
806
|
-
cfg :
|
|
806
|
+
cfg : SessionConfigDict | None
|
|
807
807
|
Session configuration.
|
|
808
808
|
|
|
809
809
|
Returns
|
etlplus/cli/handlers.py
CHANGED
|
@@ -24,7 +24,7 @@ from ..ops import load
|
|
|
24
24
|
from ..ops import run
|
|
25
25
|
from ..ops import transform
|
|
26
26
|
from ..ops import validate
|
|
27
|
-
from ..ops.validate import
|
|
27
|
+
from ..ops.validate import FieldRulesDict
|
|
28
28
|
from ..types import JSONData
|
|
29
29
|
from ..types import TemplateKey
|
|
30
30
|
from . import io as cli_io
|
|
@@ -661,7 +661,7 @@ def validate_handler(
|
|
|
661
661
|
if not isinstance(rules_payload, dict):
|
|
662
662
|
raise ValueError('rules must resolve to a mapping of field rules')
|
|
663
663
|
|
|
664
|
-
field_rules = cast(Mapping[str,
|
|
664
|
+
field_rules = cast(Mapping[str, FieldRulesDict], rules_payload)
|
|
665
665
|
result = validate(payload, field_rules)
|
|
666
666
|
|
|
667
667
|
if target and target != '-':
|
|
@@ -670,11 +670,11 @@ def validate_handler(
|
|
|
670
670
|
cli_io.write_json_output(
|
|
671
671
|
validated_data,
|
|
672
672
|
target,
|
|
673
|
-
success_message='
|
|
673
|
+
success_message='ValidationDict result saved to',
|
|
674
674
|
)
|
|
675
675
|
else:
|
|
676
676
|
print(
|
|
677
|
-
f'
|
|
677
|
+
f'ValidationDict failed, no data to save for {target}',
|
|
678
678
|
file=sys.stderr,
|
|
679
679
|
)
|
|
680
680
|
else:
|
etlplus/connector/__init__.py
CHANGED
|
@@ -7,15 +7,15 @@ Connector configuration types and enums.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
from .api import ConnectorApi
|
|
10
|
-
from .api import
|
|
10
|
+
from .api import ConnectorApiConfigDict
|
|
11
11
|
from .connector import Connector
|
|
12
12
|
from .core import ConnectorBase
|
|
13
13
|
from .core import ConnectorProtocol
|
|
14
14
|
from .database import ConnectorDb
|
|
15
|
-
from .database import
|
|
15
|
+
from .database import ConnectorDbConfigDict
|
|
16
16
|
from .enums import DataConnectorType
|
|
17
17
|
from .file import ConnectorFile
|
|
18
|
-
from .file import
|
|
18
|
+
from .file import ConnectorFileConfigDict
|
|
19
19
|
from .types import ConnectorType
|
|
20
20
|
from .utils import parse_connector
|
|
21
21
|
|
|
@@ -37,7 +37,7 @@ __all__ = [
|
|
|
37
37
|
'ConnectorProtocol',
|
|
38
38
|
'ConnectorType',
|
|
39
39
|
# Typed Dicts
|
|
40
|
-
'
|
|
41
|
-
'
|
|
42
|
-
'
|
|
40
|
+
'ConnectorApiConfigDict',
|
|
41
|
+
'ConnectorDbConfigDict',
|
|
42
|
+
'ConnectorFileConfigDict',
|
|
43
43
|
]
|
etlplus/connector/api.py
CHANGED
|
@@ -22,9 +22,9 @@ from typing import TypedDict
|
|
|
22
22
|
from typing import overload
|
|
23
23
|
|
|
24
24
|
from ..api import PaginationConfig
|
|
25
|
-
from ..api import
|
|
25
|
+
from ..api import PaginationConfigDict
|
|
26
26
|
from ..api import RateLimitConfig
|
|
27
|
-
from ..api import
|
|
27
|
+
from ..api import RateLimitConfigDict
|
|
28
28
|
from ..types import StrAnyMap
|
|
29
29
|
from ..types import StrStrMap
|
|
30
30
|
from ..utils import cast_str_dict
|
|
@@ -39,14 +39,14 @@ from .types import ConnectorType
|
|
|
39
39
|
|
|
40
40
|
__all__ = [
|
|
41
41
|
'ConnectorApi',
|
|
42
|
-
'
|
|
42
|
+
'ConnectorApiConfigDict',
|
|
43
43
|
]
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
# SECTION: TYPED DICTS ====================================================== #
|
|
47
47
|
|
|
48
48
|
|
|
49
|
-
class
|
|
49
|
+
class ConnectorApiConfigDict(TypedDict, total=False):
|
|
50
50
|
"""
|
|
51
51
|
Shape accepted by :meth:`ConnectorApi.from_obj` (all keys optional).
|
|
52
52
|
|
|
@@ -61,8 +61,8 @@ class ConnectorApiConfigMap(TypedDict, total=False):
|
|
|
61
61
|
method: str
|
|
62
62
|
headers: StrStrMap
|
|
63
63
|
query_params: StrAnyMap
|
|
64
|
-
pagination:
|
|
65
|
-
rate_limit:
|
|
64
|
+
pagination: PaginationConfigDict
|
|
65
|
+
rate_limit: RateLimitConfigDict
|
|
66
66
|
api: str
|
|
67
67
|
endpoint: str
|
|
68
68
|
|
|
@@ -121,7 +121,7 @@ class ConnectorApi(ConnectorBase):
|
|
|
121
121
|
|
|
122
122
|
@classmethod
|
|
123
123
|
@overload
|
|
124
|
-
def from_obj(cls, obj:
|
|
124
|
+
def from_obj(cls, obj: ConnectorApiConfigDict) -> Self: ...
|
|
125
125
|
|
|
126
126
|
@classmethod
|
|
127
127
|
@overload
|
etlplus/connector/database.py
CHANGED
|
@@ -29,14 +29,14 @@ from .types import ConnectorType
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
31
|
'ConnectorDb',
|
|
32
|
-
'
|
|
32
|
+
'ConnectorDbConfigDict',
|
|
33
33
|
]
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
# SECTION: TYPED DICTS ====================================================== #
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
class
|
|
39
|
+
class ConnectorDbConfigDict(TypedDict, total=False):
|
|
40
40
|
"""
|
|
41
41
|
Shape accepted by :meth:`ConnectorDb.from_obj` (all keys optional).
|
|
42
42
|
|
|
@@ -87,7 +87,7 @@ class ConnectorDb(ConnectorBase):
|
|
|
87
87
|
|
|
88
88
|
@classmethod
|
|
89
89
|
@overload
|
|
90
|
-
def from_obj(cls, obj:
|
|
90
|
+
def from_obj(cls, obj: ConnectorDbConfigDict) -> Self: ...
|
|
91
91
|
|
|
92
92
|
@classmethod
|
|
93
93
|
@overload
|
etlplus/connector/file.py
CHANGED
|
@@ -32,14 +32,14 @@ from .types import ConnectorType
|
|
|
32
32
|
|
|
33
33
|
__all__ = [
|
|
34
34
|
'ConnectorFile',
|
|
35
|
-
'
|
|
35
|
+
'ConnectorFileConfigDict',
|
|
36
36
|
]
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
# SECTION: TYPED DICTS ====================================================== #
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class
|
|
42
|
+
class ConnectorFileConfigDict(TypedDict, total=False):
|
|
43
43
|
"""
|
|
44
44
|
Shape accepted by :meth:`ConnectorFile.from_obj` (all keys optional).
|
|
45
45
|
|
|
@@ -86,7 +86,7 @@ class ConnectorFile(ConnectorBase):
|
|
|
86
86
|
|
|
87
87
|
@classmethod
|
|
88
88
|
@overload
|
|
89
|
-
def from_obj(cls, obj:
|
|
89
|
+
def from_obj(cls, obj: ConnectorFileConfigDict) -> Self: ...
|
|
90
90
|
|
|
91
91
|
@classmethod
|
|
92
92
|
@overload
|
etlplus/connector/types.py
CHANGED
|
@@ -14,8 +14,8 @@ Examples
|
|
|
14
14
|
>>> "type": "database",
|
|
15
15
|
>>> "connection_string": "postgresql://user:pass@localhost/db",
|
|
16
16
|
>>> }
|
|
17
|
-
>>> from etlplus.api import
|
|
18
|
-
>>> rp:
|
|
17
|
+
>>> from etlplus.api import RetryPolicyDict
|
|
18
|
+
>>> rp: RetryPolicyDict = {"max_attempts": 3, "backoff": 0.5}
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
from __future__ import annotations
|
etlplus/ops/extract.py
CHANGED
|
@@ -15,7 +15,7 @@ from urllib.parse import urlunsplit
|
|
|
15
15
|
|
|
16
16
|
from ..api import EndpointClient
|
|
17
17
|
from ..api import HttpMethod
|
|
18
|
-
from ..api import
|
|
18
|
+
from ..api import PaginationConfigDict
|
|
19
19
|
from ..api import RequestOptions
|
|
20
20
|
from ..api import compose_api_request_env
|
|
21
21
|
from ..api import paginate_with_client
|
|
@@ -160,7 +160,7 @@ def _extract_from_api_env(
|
|
|
160
160
|
|
|
161
161
|
return client.paginate_url(
|
|
162
162
|
cast(str, url),
|
|
163
|
-
cast(
|
|
163
|
+
cast(PaginationConfigDict | None, env.get('pagination')),
|
|
164
164
|
request=request_options,
|
|
165
165
|
sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
|
|
166
166
|
)
|
etlplus/ops/utils.py
CHANGED
|
@@ -26,7 +26,7 @@ from ..utils import normalize_choice
|
|
|
26
26
|
# SECTION: TYPED DICTIONARIES =============================================== #
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
class
|
|
29
|
+
class ValidationResultDict(TypedDict, total=False):
|
|
30
30
|
"""Shape returned by ``validate_fn`` callables."""
|
|
31
31
|
|
|
32
32
|
valid: bool
|
|
@@ -44,7 +44,7 @@ type ValidationPhase = Literal['before_transform', 'after_transform']
|
|
|
44
44
|
type ValidationWindow = Literal['before_transform', 'after_transform', 'both']
|
|
45
45
|
type ValidationSeverity = Literal['warn', 'error']
|
|
46
46
|
|
|
47
|
-
type ValidateFn = Callable[[Any, Ruleset],
|
|
47
|
+
type ValidateFn = Callable[[Any, Ruleset], ValidationResultDict]
|
|
48
48
|
type PrintFn = Callable[[Any], None]
|
|
49
49
|
|
|
50
50
|
|
|
@@ -198,7 +198,7 @@ def maybe_validate(
|
|
|
198
198
|
Failure severity (``"warn"`` or ``"error"``).
|
|
199
199
|
validate_fn : ValidateFn
|
|
200
200
|
Engine that performs validation and returns a
|
|
201
|
-
:class:`
|
|
201
|
+
:class:`ValidationResultDict` instance.
|
|
202
202
|
print_json_fn : PrintFn
|
|
203
203
|
Structured logger invoked when validation fails.
|
|
204
204
|
|
|
@@ -270,7 +270,7 @@ def _log_failure(
|
|
|
270
270
|
phase: ValidationPhase,
|
|
271
271
|
window: ValidationWindow,
|
|
272
272
|
ruleset_name: str | None,
|
|
273
|
-
result:
|
|
273
|
+
result: ValidationResultDict,
|
|
274
274
|
) -> None:
|
|
275
275
|
"""
|
|
276
276
|
Emit a structured message describing the failed validation.
|
|
@@ -285,7 +285,7 @@ def _log_failure(
|
|
|
285
285
|
Configured validation window.
|
|
286
286
|
ruleset_name : str | None
|
|
287
287
|
Name of the validation ruleset.
|
|
288
|
-
result :
|
|
288
|
+
result : ValidationResultDict
|
|
289
289
|
Result of the failed validation.
|
|
290
290
|
"""
|
|
291
291
|
printer(
|