etlplus 0.12.13__py3-none-any.whl → 0.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/README.md +1 -1
- etlplus/__init__.py +1 -26
- etlplus/api/__init__.py +8 -0
- etlplus/api/endpoint_client.py +3 -3
- etlplus/{run_helpers.py → api/utils.py} +121 -79
- etlplus/cli/handlers.py +17 -7
- etlplus/config/jobs.py +14 -4
- etlplus/dag.py +103 -0
- etlplus/{validation → ops}/README.md +2 -2
- etlplus/ops/__init__.py +57 -0
- etlplus/{extract.py → ops/extract.py} +78 -94
- etlplus/{load.py → ops/load.py} +73 -93
- etlplus/{run.py → ops/run.py} +14 -103
- etlplus/{transform.py → ops/transform.py} +75 -68
- etlplus/{validation → ops}/utils.py +62 -15
- etlplus/{validate.py → ops/validate.py} +19 -9
- etlplus/types.py +2 -2
- {etlplus-0.12.13.dist-info → etlplus-0.14.0.dist-info}/METADATA +4 -4
- {etlplus-0.12.13.dist-info → etlplus-0.14.0.dist-info}/RECORD +23 -22
- etlplus/validation/__init__.py +0 -44
- {etlplus-0.12.13.dist-info → etlplus-0.14.0.dist-info}/WHEEL +0 -0
- {etlplus-0.12.13.dist-info → etlplus-0.14.0.dist-info}/entry_points.txt +0 -0
- {etlplus-0.12.13.dist-info → etlplus-0.14.0.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.12.13.dist-info → etlplus-0.14.0.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""
|
|
2
|
-
:mod:`etlplus.transform` module.
|
|
2
|
+
:mod:`etlplus.ops.transform` module.
|
|
3
3
|
|
|
4
4
|
Helpers to filter, map/rename, select, sort, aggregate, and otherwise
|
|
5
5
|
transform JSON-like records (dicts and lists of dicts).
|
|
@@ -24,7 +24,7 @@ Basic pipeline with strings::
|
|
|
24
24
|
|
|
25
25
|
Using enums for keys and functions::
|
|
26
26
|
|
|
27
|
-
from .enums import PipelineStep, OperatorName, AggregateName
|
|
27
|
+
from etlplus.enums import PipelineStep, OperatorName, AggregateName
|
|
28
28
|
ops = {
|
|
29
29
|
PipelineStep.FILTER: {
|
|
30
30
|
'field': 'age', 'op': OperatorName.GTE, 'value': 18
|
|
@@ -44,28 +44,28 @@ from collections.abc import Sequence
|
|
|
44
44
|
from typing import Any
|
|
45
45
|
from typing import cast
|
|
46
46
|
|
|
47
|
-
from
|
|
48
|
-
from
|
|
49
|
-
from
|
|
47
|
+
from ..enums import AggregateName
|
|
48
|
+
from ..enums import OperatorName
|
|
49
|
+
from ..enums import PipelineStep
|
|
50
|
+
from ..types import AggregateFunc
|
|
51
|
+
from ..types import AggregateSpec
|
|
52
|
+
from ..types import FieldName
|
|
53
|
+
from ..types import Fields
|
|
54
|
+
from ..types import FilterSpec
|
|
55
|
+
from ..types import JSONData
|
|
56
|
+
from ..types import JSONDict
|
|
57
|
+
from ..types import JSONList
|
|
58
|
+
from ..types import MapSpec
|
|
59
|
+
from ..types import OperatorFunc
|
|
60
|
+
from ..types import PipelineConfig
|
|
61
|
+
from ..types import PipelineStepName
|
|
62
|
+
from ..types import SortKey
|
|
63
|
+
from ..types import StepApplier
|
|
64
|
+
from ..types import StepOrSteps
|
|
65
|
+
from ..types import StepSpec
|
|
66
|
+
from ..types import StrPath
|
|
67
|
+
from ..utils import to_number
|
|
50
68
|
from .load import load_data
|
|
51
|
-
from .types import AggregateFunc
|
|
52
|
-
from .types import AggregateSpec
|
|
53
|
-
from .types import FieldName
|
|
54
|
-
from .types import Fields
|
|
55
|
-
from .types import FilterSpec
|
|
56
|
-
from .types import JSONData
|
|
57
|
-
from .types import JSONDict
|
|
58
|
-
from .types import JSONList
|
|
59
|
-
from .types import MapSpec
|
|
60
|
-
from .types import OperatorFunc
|
|
61
|
-
from .types import PipelineConfig
|
|
62
|
-
from .types import PipelineStepName
|
|
63
|
-
from .types import SortKey
|
|
64
|
-
from .types import StepApplier
|
|
65
|
-
from .types import StepOrSteps
|
|
66
|
-
from .types import StepSpec
|
|
67
|
-
from .types import StrPath
|
|
68
|
-
from .utils import to_number
|
|
69
69
|
|
|
70
70
|
# SECTION: EXPORTS ========================================================== #
|
|
71
71
|
|
|
@@ -730,15 +730,16 @@ def _is_plain_fields_list(obj: Any) -> bool:
|
|
|
730
730
|
|
|
731
731
|
|
|
732
732
|
_PIPELINE_STEPS: tuple[PipelineStepName, ...] = (
|
|
733
|
+
'aggregate',
|
|
733
734
|
'filter',
|
|
734
735
|
'map',
|
|
735
736
|
'select',
|
|
736
737
|
'sort',
|
|
737
|
-
'aggregate',
|
|
738
738
|
)
|
|
739
739
|
|
|
740
740
|
|
|
741
741
|
_STEP_APPLIERS: dict[PipelineStepName, StepApplier] = {
|
|
742
|
+
'aggregate': _apply_aggregate_step,
|
|
742
743
|
'filter': _apply_filter_step,
|
|
743
744
|
'map': _apply_map_step,
|
|
744
745
|
'select': _apply_select_step,
|
|
@@ -746,7 +747,54 @@ _STEP_APPLIERS: dict[PipelineStepName, StepApplier] = {
|
|
|
746
747
|
}
|
|
747
748
|
|
|
748
749
|
|
|
749
|
-
# SECTION:
|
|
750
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
# -- Helpers -- #
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def apply_aggregate(
|
|
757
|
+
records: JSONList,
|
|
758
|
+
operation: AggregateSpec,
|
|
759
|
+
) -> JSONDict:
|
|
760
|
+
"""
|
|
761
|
+
Aggregate a numeric field or count presence.
|
|
762
|
+
|
|
763
|
+
Parameters
|
|
764
|
+
----------
|
|
765
|
+
records : JSONList
|
|
766
|
+
Records to aggregate.
|
|
767
|
+
operation : AggregateSpec
|
|
768
|
+
Dict with keys ``field`` and ``func``. ``func`` is one of
|
|
769
|
+
``'sum'``, ``'avg'``, ``'min'``, ``'max'``, or ``'count'``.
|
|
770
|
+
A callable may also be supplied for ``func``. Optionally, set
|
|
771
|
+
``alias`` to control the output key name.
|
|
772
|
+
|
|
773
|
+
Returns
|
|
774
|
+
-------
|
|
775
|
+
JSONDict
|
|
776
|
+
A single-row result like ``{"sum_age": 42}``.
|
|
777
|
+
|
|
778
|
+
Notes
|
|
779
|
+
-----
|
|
780
|
+
Numeric operations ignore non-numeric values but count their presence
|
|
781
|
+
for ``'count'``.
|
|
782
|
+
"""
|
|
783
|
+
field = operation.get('field')
|
|
784
|
+
func = operation.get('func')
|
|
785
|
+
alias = operation.get('alias')
|
|
786
|
+
|
|
787
|
+
if not field or func is None:
|
|
788
|
+
return {'error': 'Invalid aggregation operation'}
|
|
789
|
+
|
|
790
|
+
try:
|
|
791
|
+
aggregator = _resolve_aggregator(func)
|
|
792
|
+
except TypeError:
|
|
793
|
+
return {'error': f'Unknown aggregation function: {func}'}
|
|
794
|
+
|
|
795
|
+
nums, present = _collect_numeric_and_presence(records, field)
|
|
796
|
+
key_name = _derive_agg_key(func, field, alias)
|
|
797
|
+
return {key_name: aggregator(nums, present)}
|
|
750
798
|
|
|
751
799
|
|
|
752
800
|
def apply_filter(
|
|
@@ -894,48 +942,7 @@ def apply_sort(
|
|
|
894
942
|
)
|
|
895
943
|
|
|
896
944
|
|
|
897
|
-
|
|
898
|
-
records: JSONList,
|
|
899
|
-
operation: AggregateSpec,
|
|
900
|
-
) -> JSONDict:
|
|
901
|
-
"""
|
|
902
|
-
Aggregate a numeric field or count presence.
|
|
903
|
-
|
|
904
|
-
Parameters
|
|
905
|
-
----------
|
|
906
|
-
records : JSONList
|
|
907
|
-
Records to aggregate.
|
|
908
|
-
operation : AggregateSpec
|
|
909
|
-
Dict with keys ``field`` and ``func``. ``func`` is one of
|
|
910
|
-
``'sum'``, ``'avg'``, ``'min'``, ``'max'``, or ``'count'``.
|
|
911
|
-
A callable may also be supplied for ``func``. Optionally, set
|
|
912
|
-
``alias`` to control the output key name.
|
|
913
|
-
|
|
914
|
-
Returns
|
|
915
|
-
-------
|
|
916
|
-
JSONDict
|
|
917
|
-
A single-row result like ``{"sum_age": 42}``.
|
|
918
|
-
|
|
919
|
-
Notes
|
|
920
|
-
-----
|
|
921
|
-
Numeric operations ignore non-numeric values but count their presence
|
|
922
|
-
for ``'count'``.
|
|
923
|
-
"""
|
|
924
|
-
field = operation.get('field')
|
|
925
|
-
func = operation.get('func')
|
|
926
|
-
alias = operation.get('alias')
|
|
927
|
-
|
|
928
|
-
if not field or func is None:
|
|
929
|
-
return {'error': 'Invalid aggregation operation'}
|
|
930
|
-
|
|
931
|
-
try:
|
|
932
|
-
aggregator = _resolve_aggregator(func)
|
|
933
|
-
except TypeError:
|
|
934
|
-
return {'error': f'Unknown aggregation function: {func}'}
|
|
935
|
-
|
|
936
|
-
nums, present = _collect_numeric_and_presence(records, field)
|
|
937
|
-
key_name = _derive_agg_key(func, field, alias)
|
|
938
|
-
return {key_name: aggregator(nums, present)}
|
|
945
|
+
# -- Orchestration -- #
|
|
939
946
|
|
|
940
947
|
|
|
941
948
|
def transform(
|
|
@@ -982,7 +989,7 @@ def transform(
|
|
|
982
989
|
|
|
983
990
|
Using enums for keys and functions::
|
|
984
991
|
|
|
985
|
-
from .enums import PipelineStep, OperatorName, AggregateName
|
|
992
|
+
from etlplus.enums import PipelineStep, OperatorName, AggregateName
|
|
986
993
|
ops = {
|
|
987
994
|
PipelineStep.FILTER: {
|
|
988
995
|
'field': 'age', 'op': OperatorName.GTE, 'value': 18
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
|
-
:mod:`etlplus.
|
|
2
|
+
:mod:`etlplus.ops.utils` module.
|
|
3
3
|
|
|
4
|
-
Utility helpers for conditional
|
|
4
|
+
Utility helpers for conditional data ops orchestration.
|
|
5
5
|
|
|
6
6
|
The helpers defined here embrace a "high cohesion, low coupling" design by
|
|
7
7
|
isolating normalization, configuration, and logging responsibilities. The
|
|
@@ -13,11 +13,13 @@ offloading ancillary concerns to composable helpers.
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
15
|
from collections.abc import Callable
|
|
16
|
+
from collections.abc import Mapping
|
|
16
17
|
from dataclasses import dataclass
|
|
17
18
|
from typing import Any
|
|
18
19
|
from typing import Literal
|
|
19
20
|
from typing import Self
|
|
20
21
|
from typing import TypedDict
|
|
22
|
+
from typing import cast
|
|
21
23
|
|
|
22
24
|
from ..types import StrAnyMap
|
|
23
25
|
from ..utils import normalized_str
|
|
@@ -291,11 +293,17 @@ def _normalize_phase(
|
|
|
291
293
|
Normalized validation phase. Defaults to ``"before_transform"`` when
|
|
292
294
|
unspecified.
|
|
293
295
|
"""
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
296
|
+
return cast(
|
|
297
|
+
ValidationPhase,
|
|
298
|
+
_normalize_choice(
|
|
299
|
+
value,
|
|
300
|
+
mapping={
|
|
301
|
+
'before_transform': 'before_transform',
|
|
302
|
+
'after_transform': 'after_transform',
|
|
303
|
+
},
|
|
304
|
+
default='before_transform',
|
|
305
|
+
),
|
|
306
|
+
)
|
|
299
307
|
|
|
300
308
|
|
|
301
309
|
def _normalize_severity(
|
|
@@ -314,7 +322,14 @@ def _normalize_severity(
|
|
|
314
322
|
ValidationSeverity
|
|
315
323
|
Normalized severity. Defaults to ``"error"`` when unspecified.
|
|
316
324
|
"""
|
|
317
|
-
return
|
|
325
|
+
return cast(
|
|
326
|
+
ValidationSeverity,
|
|
327
|
+
_normalize_choice(
|
|
328
|
+
value,
|
|
329
|
+
mapping={'warn': 'warn'},
|
|
330
|
+
default='error',
|
|
331
|
+
),
|
|
332
|
+
)
|
|
318
333
|
|
|
319
334
|
|
|
320
335
|
def _normalize_window(
|
|
@@ -333,13 +348,45 @@ def _normalize_window(
|
|
|
333
348
|
ValidationWindow
|
|
334
349
|
Normalized validation window. Defaults to ``"both"`` when unspecified.
|
|
335
350
|
"""
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
351
|
+
return cast(
|
|
352
|
+
ValidationWindow,
|
|
353
|
+
_normalize_choice(
|
|
354
|
+
value,
|
|
355
|
+
mapping={
|
|
356
|
+
'before_transform': 'before_transform',
|
|
357
|
+
'after_transform': 'after_transform',
|
|
358
|
+
'both': 'both',
|
|
359
|
+
},
|
|
360
|
+
default='both',
|
|
361
|
+
),
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _normalize_choice(
|
|
366
|
+
value: str | None,
|
|
367
|
+
*,
|
|
368
|
+
mapping: Mapping[str, str],
|
|
369
|
+
default: str,
|
|
370
|
+
) -> str:
|
|
371
|
+
"""
|
|
372
|
+
Normalize a text value against a mapping with a default fallback.
|
|
373
|
+
|
|
374
|
+
Parameters
|
|
375
|
+
----------
|
|
376
|
+
value : str | None
|
|
377
|
+
Input text to normalize.
|
|
378
|
+
mapping : Mapping[str, str]
|
|
379
|
+
Mapping of accepted values to normalized outputs.
|
|
380
|
+
default : str
|
|
381
|
+
Default to return when input is missing or unrecognized.
|
|
382
|
+
|
|
383
|
+
Returns
|
|
384
|
+
-------
|
|
385
|
+
str
|
|
386
|
+
Normalized value.
|
|
387
|
+
"""
|
|
388
|
+
normalized = normalized_str(value)
|
|
389
|
+
return mapping.get(normalized, default)
|
|
343
390
|
|
|
344
391
|
|
|
345
392
|
def _rule_name(
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
"""
|
|
2
|
-
:mod:`etlplus.
|
|
2
|
+
:mod:`etlplus.ops.validate` module.
|
|
3
3
|
|
|
4
4
|
Validate dicts and lists of dicts using simple, schema-like rules.
|
|
5
5
|
|
|
@@ -34,11 +34,11 @@ from typing import Final
|
|
|
34
34
|
from typing import Literal
|
|
35
35
|
from typing import TypedDict
|
|
36
36
|
|
|
37
|
+
from ..types import JSONData
|
|
38
|
+
from ..types import Record
|
|
39
|
+
from ..types import StrAnyMap
|
|
40
|
+
from ..types import StrPath
|
|
37
41
|
from .load import load_data
|
|
38
|
-
from .types import JSONData
|
|
39
|
-
from .types import Record
|
|
40
|
-
from .types import StrAnyMap
|
|
41
|
-
from .types import StrPath
|
|
42
42
|
|
|
43
43
|
# SECTION: EXPORTS ========================================================== #
|
|
44
44
|
|
|
@@ -279,11 +279,15 @@ def _type_matches(
|
|
|
279
279
|
bool
|
|
280
280
|
``True`` if the value matches the expected type; ``False`` if not.
|
|
281
281
|
"""
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
282
|
+
if expected == 'number':
|
|
283
|
+
return _is_number(value)
|
|
284
|
+
if expected == 'integer':
|
|
285
|
+
return isinstance(value, int) and not isinstance(value, bool)
|
|
286
|
+
if expected == 'boolean':
|
|
287
|
+
return isinstance(value, bool)
|
|
285
288
|
|
|
286
|
-
|
|
289
|
+
py_type = TYPE_MAP.get(expected)
|
|
290
|
+
return isinstance(value, py_type) if py_type else False
|
|
287
291
|
|
|
288
292
|
|
|
289
293
|
def _validate_record(
|
|
@@ -330,6 +334,9 @@ def _validate_record(
|
|
|
330
334
|
# SECTION: FUNCTIONS ======================================================== #
|
|
331
335
|
|
|
332
336
|
|
|
337
|
+
# -- Helpers -- #
|
|
338
|
+
|
|
339
|
+
|
|
333
340
|
def validate_field(
|
|
334
341
|
value: Any,
|
|
335
342
|
rules: StrAnyMap | FieldRules,
|
|
@@ -425,6 +432,9 @@ def validate_field(
|
|
|
425
432
|
return {'valid': len(errors) == 0, 'errors': errors}
|
|
426
433
|
|
|
427
434
|
|
|
435
|
+
# -- Orchestration -- #
|
|
436
|
+
|
|
437
|
+
|
|
428
438
|
def validate(
|
|
429
439
|
source: StrPath | JSONData,
|
|
430
440
|
rules: RulesMap | None = None,
|
etlplus/types.py
CHANGED
|
@@ -193,8 +193,8 @@ type AggregateSpec = StrAnyMap
|
|
|
193
193
|
|
|
194
194
|
# -- Pipelines-- #
|
|
195
195
|
|
|
196
|
-
# Unified pipeline step spec consumed by :mod:`etlplus.transform`.
|
|
197
|
-
type StepSpec = FilterSpec | MapSpec | SelectSpec | SortSpec
|
|
196
|
+
# Unified pipeline step spec consumed by :mod:`etlplus.ops.transform`.
|
|
197
|
+
type StepSpec = AggregateSpec | FilterSpec | MapSpec | SelectSpec | SortSpec
|
|
198
198
|
|
|
199
199
|
# Collections of steps
|
|
200
200
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.14.0
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -196,7 +196,7 @@ etlplus extract file examples/data/sample.csv \
|
|
|
196
196
|
[Python API](#python-api):
|
|
197
197
|
|
|
198
198
|
```python
|
|
199
|
-
from etlplus import extract, transform, validate, load
|
|
199
|
+
from etlplus.ops import extract, transform, validate, load
|
|
200
200
|
|
|
201
201
|
data = extract("file", "input.csv")
|
|
202
202
|
ops = {"filter": {"field": "age", "op": "gt", "value": 25}, "select": ["name", "email"]}
|
|
@@ -531,7 +531,7 @@ cat examples/data/sample.json \
|
|
|
531
531
|
Use ETLPlus as a Python library:
|
|
532
532
|
|
|
533
533
|
```python
|
|
534
|
-
from etlplus import extract, validate, transform, load
|
|
534
|
+
from etlplus.ops import extract, validate, transform, load
|
|
535
535
|
|
|
536
536
|
# Extract data
|
|
537
537
|
data = extract("file", "data.json")
|
|
@@ -726,7 +726,7 @@ We split tests into two layers:
|
|
|
726
726
|
pagination + rate limit defaults, file/API connector interactions) may touch temp files and use
|
|
727
727
|
fake clients.
|
|
728
728
|
|
|
729
|
-
If a test calls `etlplus.cli.main()` or `etlplus.run.run()` it’s integration by default.
|
|
729
|
+
If a test calls `etlplus.cli.main()` or `etlplus.ops.run.run()` it’s integration by default. Full
|
|
730
730
|
criteria: [`CONTRIBUTING.md#testing`](CONTRIBUTING.md#testing).
|
|
731
731
|
|
|
732
732
|
### Code Coverage
|
|
@@ -1,28 +1,24 @@
|
|
|
1
|
-
etlplus/README.md,sha256=
|
|
2
|
-
etlplus/__init__.py,sha256=
|
|
1
|
+
etlplus/README.md,sha256=HwDt6eh6p4422FOJhrpUBhfPGC82QzpsJNecs3zEtUU,1459
|
|
2
|
+
etlplus/__init__.py,sha256=mgTP4PJmRmsEjTCAizzzdtzAmhuHtarmPzphzdjvLgM,277
|
|
3
3
|
etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
|
|
4
4
|
etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
|
|
5
|
+
etlplus/dag.py,sha256=4EYmBsJax3y4clHv10jjdp3GrBBD_WblvtxUb_JxGCQ,2464
|
|
5
6
|
etlplus/enums.py,sha256=WyxpUEUPdYdXlueKDXGaSEo7o9OqCXyzjDOOPqmW8tw,8326
|
|
6
|
-
etlplus/extract.py,sha256=LOyL8_KCOaIGemTxSnKbN_ttfLWUljqT4OQxANe7G3k,6089
|
|
7
|
-
etlplus/load.py,sha256=aufl-2CpuI_J1hKBY1uFsoVf9Gfl9bKQjs233dYFf00,8631
|
|
8
7
|
etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
|
|
9
8
|
etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
-
etlplus/
|
|
11
|
-
etlplus/run_helpers.py,sha256=bj6MkaeFxjl3CeKG1HoXKx5DwAlXNERVW-GX-z1P_qQ,24373
|
|
12
|
-
etlplus/transform.py,sha256=uAUVDDHYCgx7GpVez9IK3OAZM-CnCuMa9iox3vwGGJA,25296
|
|
13
|
-
etlplus/types.py,sha256=1hsDlnF6r76zAwaUYay-i6pCM-Y0IU5nP7Crj8PLCQ4,6157
|
|
9
|
+
etlplus/types.py,sha256=LhyKU67CW7CcLEshE_1OIvEqWr-QVMJVYByf1221miY,6161
|
|
14
10
|
etlplus/utils.py,sha256=BMLTWAvCJj3zLEcffBgURYnu0UGhhXsfH2WWpAt7fV8,13363
|
|
15
|
-
etlplus/validate.py,sha256=7rJoEI_SIILdPpoBqqh2UJqg9oeReDz34mYSlc3t7Qg,12989
|
|
16
11
|
etlplus/api/README.md,sha256=aGTbcL-EqaiMTS7GToibmeTIzjX-viP2OtUDfIJnZEo,7913
|
|
17
|
-
etlplus/api/__init__.py,sha256=
|
|
12
|
+
etlplus/api/__init__.py,sha256=bWa5a9PjYLVnyd9yh3SCgS-4-WOFSKXsH27zeYMTnJc,4567
|
|
18
13
|
etlplus/api/auth.py,sha256=GOO5on-LoMS1GXTAhtK9rFcfpjbBcNeA6NE5UZwIq0g,12158
|
|
19
14
|
etlplus/api/config.py,sha256=wRpOaZ31sPReVzEMme0jKl_37nqgraESwuYSNxP_xDo,17397
|
|
20
|
-
etlplus/api/endpoint_client.py,sha256=
|
|
15
|
+
etlplus/api/endpoint_client.py,sha256=OPAvw3NkpSzeITqgRP1reyqX9Nc_XC8lAj6Yp7nV4Tw,30705
|
|
21
16
|
etlplus/api/errors.py,sha256=XjI2xW-sypMUNUbqfc2S57-IGyWnH3oCDFhCmKYYI_Q,4648
|
|
22
17
|
etlplus/api/request_manager.py,sha256=YkDz803HM3BBzamsEZdSdE9fbVT0avMbTaLAgar9Wzo,18481
|
|
23
18
|
etlplus/api/retry_manager.py,sha256=0GDhJVyIlb1Ww35JUWlYoa8QYUPjKLBtxQeZj3TdLbY,11306
|
|
24
19
|
etlplus/api/transport.py,sha256=LRsQEPxIYrvXQQMvgPPkIl_57YCmanzsWNEnSYdP_d8,9164
|
|
25
20
|
etlplus/api/types.py,sha256=687JigIf3qfYxgGTNBaMNsQsrza5Pja6DcK5llM9oRU,4591
|
|
21
|
+
etlplus/api/utils.py,sha256=hUrVALJkq4sRQD-UIYyBUgWOmK5cgrs6P_BqXDtErPw,25874
|
|
26
22
|
etlplus/api/pagination/__init__.py,sha256=a4UX2J0AG8RMvmHt_CCofUm5vSmFo6GAfkb8XnSXypM,1395
|
|
27
23
|
etlplus/api/pagination/client.py,sha256=42cG442od3mQkw_JsvGvxT_w7y9J4HPM5PB4tFFU6EQ,5383
|
|
28
24
|
etlplus/api/pagination/config.py,sha256=3dXDJ-nMbO9Zk6i344n4roBFbUlHsa294D1_plPmm6E,13579
|
|
@@ -34,7 +30,7 @@ etlplus/cli/README.md,sha256=rl9VYNH5MluV0rh-eP7TbxJZ5BTMEIaksxhl_JXpYio,1233
|
|
|
34
30
|
etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
|
|
35
31
|
etlplus/cli/commands.py,sha256=g8_m3A8HEMyTRu2HctNiRoi2gtB5oSZCUEcyq-PIXos,24669
|
|
36
32
|
etlplus/cli/constants.py,sha256=E6Uy4WauLa_0zkzxqImXh-bb1gKdb9sBZQVc8QOzr2Q,1943
|
|
37
|
-
etlplus/cli/handlers.py,sha256=
|
|
33
|
+
etlplus/cli/handlers.py,sha256=gbW1jH348QmX65g4gffxrtMt2obFIw8X0dEKnwXQKPQ,18170
|
|
38
34
|
etlplus/cli/io.py,sha256=EFaBPYaBOyOllfMQWXgTjy-MPiGfNejicpD7ROrPyAE,7840
|
|
39
35
|
etlplus/cli/main.py,sha256=IgeqxypixfwLHR-QcpgVMQ7vMZ865bXOh2oO9v-BWeM,5234
|
|
40
36
|
etlplus/cli/options.py,sha256=vfXT3YLh7wG1iC-aTdSg6ItMC8l6n0Lozmy53XjqLbA,1199
|
|
@@ -43,7 +39,7 @@ etlplus/cli/types.py,sha256=tclhKVJXDqHzlTQBYKARfqMgDOcuBJ-Zej2pvFy96WM,652
|
|
|
43
39
|
etlplus/config/README.md,sha256=ot6oFZxTz4x83mj1_FrQ13dO0z2QkRFDnkCkx7NPsSs,1636
|
|
44
40
|
etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
|
|
45
41
|
etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
|
|
46
|
-
etlplus/config/jobs.py,sha256=
|
|
42
|
+
etlplus/config/jobs.py,sha256=oa2rNwacy2b_1HP_iFDLarGnn812ZVP2k5cHt4eqBIg,7843
|
|
47
43
|
etlplus/config/pipeline.py,sha256=m4Jh0ctFcKrIx6zR7LEC0sYY5wq0o8NqOruWPlz6qmA,9494
|
|
48
44
|
etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,1898
|
|
49
45
|
etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
|
|
@@ -118,16 +114,21 @@ etlplus/file/xpt.py,sha256=JLqOkZ60awNsPXSqLKcPUwqZLPhPR05zk4EVRdEfvoU,1702
|
|
|
118
114
|
etlplus/file/yaml.py,sha256=8BEqCELaTQ_nRu1iksLBHVj19P6JmcUf5__fe0yVigw,2449
|
|
119
115
|
etlplus/file/zip.py,sha256=nd26V3S0edklriKnKOGDTLlO8RBXTda_zLLEQrJgKL4,4185
|
|
120
116
|
etlplus/file/zsav.py,sha256=2WxjXamvzj0adDbWGMWM3-cj_LsCRjyZz4J907lNkPk,1664
|
|
117
|
+
etlplus/ops/README.md,sha256=8omi7DYZhelc26JKk8Cm8QR8I3OGwziysPj1ivx41iQ,1380
|
|
118
|
+
etlplus/ops/__init__.py,sha256=43nKiR5aWhR3rcpLfluzA972vX7V5DPKvzU7y0XmFJQ,1524
|
|
119
|
+
etlplus/ops/extract.py,sha256=s3CaLXMY_loMoPU47rvvzyp1buk84uCzDqqaI7l3Dwg,5785
|
|
120
|
+
etlplus/ops/load.py,sha256=2jMewdjaNS1GKZ9cn94ISRSzDfqO3_s5fb_YNo_CqIc,8330
|
|
121
|
+
etlplus/ops/run.py,sha256=u-YUh6ht_jtPHN4d0-ybH6yn4UboNfTKZ7RrBwfM5-A,9829
|
|
122
|
+
etlplus/ops/transform.py,sha256=1P43WYUaw872JDU86FhbbbkP8xnBWpgEPn2Q-z4ywls,25421
|
|
123
|
+
etlplus/ops/utils.py,sha256=9cN-bmcYs87R84GllmW7fSiUEmXFqIkGbSFiPYStsbA,11198
|
|
124
|
+
etlplus/ops/validate.py,sha256=gLa5zcwhxGbaIhH-OqTwDsQAgAYZSajLVGwwyeT2OZk,13257
|
|
121
125
|
etlplus/templates/README.md,sha256=kHSZ8FWcrlrcWz0hBIbho-k1Bi-PL-DQ7g02o-g70c8,1355
|
|
122
126
|
etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
|
|
123
127
|
etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
|
|
124
128
|
etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
|
|
125
|
-
etlplus/
|
|
126
|
-
etlplus/
|
|
127
|
-
etlplus/
|
|
128
|
-
etlplus-0.
|
|
129
|
-
etlplus-0.
|
|
130
|
-
etlplus-0.
|
|
131
|
-
etlplus-0.12.13.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
132
|
-
etlplus-0.12.13.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
133
|
-
etlplus-0.12.13.dist-info/RECORD,,
|
|
129
|
+
etlplus-0.14.0.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
|
|
130
|
+
etlplus-0.14.0.dist-info/METADATA,sha256=gKu31Fe8g5vk461qZsuqwsetPGKENRGwDIH2bbk4PxA,28115
|
|
131
|
+
etlplus-0.14.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
132
|
+
etlplus-0.14.0.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
133
|
+
etlplus-0.14.0.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
134
|
+
etlplus-0.14.0.dist-info/RECORD,,
|
etlplus/validation/__init__.py
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.validation` package.
|
|
3
|
-
|
|
4
|
-
Conditional validation utilities used across the ETL pipeline.
|
|
5
|
-
|
|
6
|
-
The package intentionally exposes a single helper, :func:`maybe_validate`, to
|
|
7
|
-
keep the public API compact and predictable. Supporting logic lives in
|
|
8
|
-
``etlplus.validation.utils`` where validation configuration is normalized,
|
|
9
|
-
reducing the likelihood of phase/option mismatches.
|
|
10
|
-
|
|
11
|
-
Examples
|
|
12
|
-
--------
|
|
13
|
-
>>> from etlplus.validation import maybe_validate
|
|
14
|
-
>>> payload = {'name': 'Alice'}
|
|
15
|
-
>>> rules = {'required': ['name']}
|
|
16
|
-
>>> def validator(data, config):
|
|
17
|
-
... missing = [field for field in config['required'] if field not in data]
|
|
18
|
-
... return {'valid': not missing, 'errors': missing, 'data': data}
|
|
19
|
-
>>> maybe_validate(
|
|
20
|
-
... payload,
|
|
21
|
-
... when='both',
|
|
22
|
-
... enabled=True,
|
|
23
|
-
... rules=rules,
|
|
24
|
-
... phase='before_transform',
|
|
25
|
-
... severity='warn',
|
|
26
|
-
... validate_fn=validator,
|
|
27
|
-
... print_json_fn=lambda message: message,
|
|
28
|
-
... )
|
|
29
|
-
{'name': 'Alice'}
|
|
30
|
-
|
|
31
|
-
See Also
|
|
32
|
-
--------
|
|
33
|
-
- :mod:`etlplus.validation.utils` for implementation details and helper
|
|
34
|
-
utilities.
|
|
35
|
-
"""
|
|
36
|
-
|
|
37
|
-
from __future__ import annotations
|
|
38
|
-
|
|
39
|
-
from .utils import maybe_validate
|
|
40
|
-
|
|
41
|
-
# SECTION: EXPORTS ========================================================== #
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
__all__ = ['maybe_validate']
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|