etlplus 0.16.3__py3-none-any.whl → 0.16.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/README.md +22 -0
- etlplus/__init__.py +2 -0
- etlplus/api/__init__.py +14 -14
- etlplus/api/auth.py +9 -6
- etlplus/api/config.py +6 -6
- etlplus/api/endpoint_client.py +16 -16
- etlplus/api/errors.py +4 -4
- etlplus/api/pagination/__init__.py +6 -6
- etlplus/api/pagination/config.py +11 -9
- etlplus/api/rate_limiting/__init__.py +2 -2
- etlplus/api/rate_limiting/config.py +10 -10
- etlplus/api/rate_limiting/rate_limiter.py +2 -2
- etlplus/api/request_manager.py +4 -4
- etlplus/api/retry_manager.py +6 -6
- etlplus/api/transport.py +10 -10
- etlplus/api/types.py +15 -15
- etlplus/api/utils.py +49 -49
- etlplus/cli/commands.py +22 -22
- etlplus/cli/handlers.py +12 -13
- etlplus/{workflow/pipeline.py → config.py} +17 -37
- etlplus/connector/__init__.py +6 -6
- etlplus/connector/api.py +7 -7
- etlplus/connector/database.py +3 -3
- etlplus/connector/file.py +3 -3
- etlplus/connector/types.py +2 -2
- etlplus/ops/extract.py +2 -2
- etlplus/ops/run.py +2 -2
- etlplus/ops/utils.py +5 -5
- etlplus/ops/validate.py +13 -13
- etlplus/types.py +2 -1
- etlplus/workflow/README.md +0 -24
- etlplus/workflow/__init__.py +0 -4
- etlplus/workflow/jobs.py +0 -2
- {etlplus-0.16.3.dist-info → etlplus-0.16.5.dist-info}/METADATA +1 -1
- {etlplus-0.16.3.dist-info → etlplus-0.16.5.dist-info}/RECORD +39 -39
- {etlplus-0.16.3.dist-info → etlplus-0.16.5.dist-info}/WHEEL +0 -0
- {etlplus-0.16.3.dist-info → etlplus-0.16.5.dist-info}/entry_points.txt +0 -0
- {etlplus-0.16.3.dist-info → etlplus-0.16.5.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.16.3.dist-info → etlplus-0.16.5.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
|
-
:mod:`etlplus.
|
|
2
|
+
:mod:`etlplus.config` module.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
Configuration model and helpers for job pipeline orchestration.
|
|
5
5
|
|
|
6
6
|
Notes
|
|
7
7
|
-----
|
|
@@ -24,26 +24,24 @@ from pathlib import Path
|
|
|
24
24
|
from typing import Any
|
|
25
25
|
from typing import Self
|
|
26
26
|
|
|
27
|
-
from
|
|
28
|
-
from
|
|
29
|
-
from
|
|
30
|
-
from
|
|
31
|
-
from
|
|
32
|
-
from
|
|
33
|
-
from
|
|
34
|
-
from
|
|
35
|
-
from
|
|
36
|
-
from .jobs import JobConfig
|
|
37
|
-
from .profile import ProfileConfig
|
|
27
|
+
from .api import ApiConfig
|
|
28
|
+
from .connector import Connector
|
|
29
|
+
from .connector import parse_connector
|
|
30
|
+
from .file import File
|
|
31
|
+
from .file import FileFormat
|
|
32
|
+
from .types import StrAnyMap
|
|
33
|
+
from .utils import coerce_dict
|
|
34
|
+
from .utils import deep_substitute
|
|
35
|
+
from .utils import maybe_mapping
|
|
36
|
+
from .workflow.jobs import JobConfig
|
|
37
|
+
from .workflow.profile import ProfileConfig
|
|
38
38
|
|
|
39
39
|
# SECTION: EXPORTS ========================================================== #
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
__all__ = [
|
|
43
43
|
# Data Classes
|
|
44
|
-
'
|
|
45
|
-
# Functions
|
|
46
|
-
'load_pipeline_config',
|
|
44
|
+
'Config',
|
|
47
45
|
]
|
|
48
46
|
|
|
49
47
|
|
|
@@ -126,29 +124,11 @@ def _parse_connector_entry(
|
|
|
126
124
|
return None
|
|
127
125
|
|
|
128
126
|
|
|
129
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
def load_pipeline_config(
|
|
133
|
-
path: Path | str,
|
|
134
|
-
*,
|
|
135
|
-
substitute: bool = False,
|
|
136
|
-
env: Mapping[str, str] | None = None,
|
|
137
|
-
) -> PipelineConfig:
|
|
138
|
-
"""
|
|
139
|
-
Load a pipeline YAML file into a ``PipelineConfig`` instance.
|
|
140
|
-
|
|
141
|
-
Delegates to ``PipelineConfig.from_yaml`` for construction and optional
|
|
142
|
-
variable substitution.
|
|
143
|
-
"""
|
|
144
|
-
return PipelineConfig.from_yaml(path, substitute=substitute, env=env)
|
|
145
|
-
|
|
146
|
-
|
|
147
127
|
# SECTION: DATA CLASSES ===================================================== #
|
|
148
128
|
|
|
149
129
|
|
|
150
130
|
@dataclass(kw_only=True, slots=True)
|
|
151
|
-
class
|
|
131
|
+
class Config:
|
|
152
132
|
"""
|
|
153
133
|
Configuration for the data processing pipeline.
|
|
154
134
|
|
|
@@ -211,7 +191,7 @@ class PipelineConfig:
|
|
|
211
191
|
env: Mapping[str, str] | None = None,
|
|
212
192
|
) -> Self:
|
|
213
193
|
"""
|
|
214
|
-
Parse a YAML file into a ``
|
|
194
|
+
Parse a YAML file into a ``Config`` instance.
|
|
215
195
|
|
|
216
196
|
Parameters
|
|
217
197
|
----------
|
|
@@ -259,7 +239,7 @@ class PipelineConfig:
|
|
|
259
239
|
raw: StrAnyMap,
|
|
260
240
|
) -> Self:
|
|
261
241
|
"""
|
|
262
|
-
Parse a mapping into a ``
|
|
242
|
+
Parse a mapping into a ``Config`` instance.
|
|
263
243
|
|
|
264
244
|
Parameters
|
|
265
245
|
----------
|
etlplus/connector/__init__.py
CHANGED
|
@@ -7,15 +7,15 @@ Connector configuration types and enums.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
from .api import ConnectorApi
|
|
10
|
-
from .api import
|
|
10
|
+
from .api import ConnectorApiConfigDict
|
|
11
11
|
from .connector import Connector
|
|
12
12
|
from .core import ConnectorBase
|
|
13
13
|
from .core import ConnectorProtocol
|
|
14
14
|
from .database import ConnectorDb
|
|
15
|
-
from .database import
|
|
15
|
+
from .database import ConnectorDbConfigDict
|
|
16
16
|
from .enums import DataConnectorType
|
|
17
17
|
from .file import ConnectorFile
|
|
18
|
-
from .file import
|
|
18
|
+
from .file import ConnectorFileConfigDict
|
|
19
19
|
from .types import ConnectorType
|
|
20
20
|
from .utils import parse_connector
|
|
21
21
|
|
|
@@ -37,7 +37,7 @@ __all__ = [
|
|
|
37
37
|
'ConnectorProtocol',
|
|
38
38
|
'ConnectorType',
|
|
39
39
|
# Typed Dicts
|
|
40
|
-
'
|
|
41
|
-
'
|
|
42
|
-
'
|
|
40
|
+
'ConnectorApiConfigDict',
|
|
41
|
+
'ConnectorDbConfigDict',
|
|
42
|
+
'ConnectorFileConfigDict',
|
|
43
43
|
]
|
etlplus/connector/api.py
CHANGED
|
@@ -22,9 +22,9 @@ from typing import TypedDict
|
|
|
22
22
|
from typing import overload
|
|
23
23
|
|
|
24
24
|
from ..api import PaginationConfig
|
|
25
|
-
from ..api import
|
|
25
|
+
from ..api import PaginationConfigDict
|
|
26
26
|
from ..api import RateLimitConfig
|
|
27
|
-
from ..api import
|
|
27
|
+
from ..api import RateLimitConfigDict
|
|
28
28
|
from ..types import StrAnyMap
|
|
29
29
|
from ..types import StrStrMap
|
|
30
30
|
from ..utils import cast_str_dict
|
|
@@ -39,14 +39,14 @@ from .types import ConnectorType
|
|
|
39
39
|
|
|
40
40
|
__all__ = [
|
|
41
41
|
'ConnectorApi',
|
|
42
|
-
'
|
|
42
|
+
'ConnectorApiConfigDict',
|
|
43
43
|
]
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
# SECTION: TYPED DICTS ====================================================== #
|
|
47
47
|
|
|
48
48
|
|
|
49
|
-
class
|
|
49
|
+
class ConnectorApiConfigDict(TypedDict, total=False):
|
|
50
50
|
"""
|
|
51
51
|
Shape accepted by :meth:`ConnectorApi.from_obj` (all keys optional).
|
|
52
52
|
|
|
@@ -61,8 +61,8 @@ class ConnectorApiConfigMap(TypedDict, total=False):
|
|
|
61
61
|
method: str
|
|
62
62
|
headers: StrStrMap
|
|
63
63
|
query_params: StrAnyMap
|
|
64
|
-
pagination:
|
|
65
|
-
rate_limit:
|
|
64
|
+
pagination: PaginationConfigDict
|
|
65
|
+
rate_limit: RateLimitConfigDict
|
|
66
66
|
api: str
|
|
67
67
|
endpoint: str
|
|
68
68
|
|
|
@@ -121,7 +121,7 @@ class ConnectorApi(ConnectorBase):
|
|
|
121
121
|
|
|
122
122
|
@classmethod
|
|
123
123
|
@overload
|
|
124
|
-
def from_obj(cls, obj:
|
|
124
|
+
def from_obj(cls, obj: ConnectorApiConfigDict) -> Self: ...
|
|
125
125
|
|
|
126
126
|
@classmethod
|
|
127
127
|
@overload
|
etlplus/connector/database.py
CHANGED
|
@@ -29,14 +29,14 @@ from .types import ConnectorType
|
|
|
29
29
|
|
|
30
30
|
__all__ = [
|
|
31
31
|
'ConnectorDb',
|
|
32
|
-
'
|
|
32
|
+
'ConnectorDbConfigDict',
|
|
33
33
|
]
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
# SECTION: TYPED DICTS ====================================================== #
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
class
|
|
39
|
+
class ConnectorDbConfigDict(TypedDict, total=False):
|
|
40
40
|
"""
|
|
41
41
|
Shape accepted by :meth:`ConnectorDb.from_obj` (all keys optional).
|
|
42
42
|
|
|
@@ -87,7 +87,7 @@ class ConnectorDb(ConnectorBase):
|
|
|
87
87
|
|
|
88
88
|
@classmethod
|
|
89
89
|
@overload
|
|
90
|
-
def from_obj(cls, obj:
|
|
90
|
+
def from_obj(cls, obj: ConnectorDbConfigDict) -> Self: ...
|
|
91
91
|
|
|
92
92
|
@classmethod
|
|
93
93
|
@overload
|
etlplus/connector/file.py
CHANGED
|
@@ -32,14 +32,14 @@ from .types import ConnectorType
|
|
|
32
32
|
|
|
33
33
|
__all__ = [
|
|
34
34
|
'ConnectorFile',
|
|
35
|
-
'
|
|
35
|
+
'ConnectorFileConfigDict',
|
|
36
36
|
]
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
# SECTION: TYPED DICTS ====================================================== #
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class
|
|
42
|
+
class ConnectorFileConfigDict(TypedDict, total=False):
|
|
43
43
|
"""
|
|
44
44
|
Shape accepted by :meth:`ConnectorFile.from_obj` (all keys optional).
|
|
45
45
|
|
|
@@ -86,7 +86,7 @@ class ConnectorFile(ConnectorBase):
|
|
|
86
86
|
|
|
87
87
|
@classmethod
|
|
88
88
|
@overload
|
|
89
|
-
def from_obj(cls, obj:
|
|
89
|
+
def from_obj(cls, obj: ConnectorFileConfigDict) -> Self: ...
|
|
90
90
|
|
|
91
91
|
@classmethod
|
|
92
92
|
@overload
|
etlplus/connector/types.py
CHANGED
|
@@ -14,8 +14,8 @@ Examples
|
|
|
14
14
|
>>> "type": "database",
|
|
15
15
|
>>> "connection_string": "postgresql://user:pass@localhost/db",
|
|
16
16
|
>>> }
|
|
17
|
-
>>> from etlplus.api import
|
|
18
|
-
>>> rp:
|
|
17
|
+
>>> from etlplus.api import RetryPolicyDict
|
|
18
|
+
>>> rp: RetryPolicyDict = {"max_attempts": 3, "backoff": 0.5}
|
|
19
19
|
"""
|
|
20
20
|
|
|
21
21
|
from __future__ import annotations
|
etlplus/ops/extract.py
CHANGED
|
@@ -15,7 +15,7 @@ from urllib.parse import urlunsplit
|
|
|
15
15
|
|
|
16
16
|
from ..api import EndpointClient
|
|
17
17
|
from ..api import HttpMethod
|
|
18
|
-
from ..api import
|
|
18
|
+
from ..api import PaginationConfigDict
|
|
19
19
|
from ..api import RequestOptions
|
|
20
20
|
from ..api import compose_api_request_env
|
|
21
21
|
from ..api import paginate_with_client
|
|
@@ -160,7 +160,7 @@ def _extract_from_api_env(
|
|
|
160
160
|
|
|
161
161
|
return client.paginate_url(
|
|
162
162
|
cast(str, url),
|
|
163
|
-
cast(
|
|
163
|
+
cast(PaginationConfigDict | None, env.get('pagination')),
|
|
164
164
|
request=request_options,
|
|
165
165
|
sleep_seconds=cast(float, env.get('sleep_seconds', 0.0)),
|
|
166
166
|
)
|
etlplus/ops/run.py
CHANGED
|
@@ -11,6 +11,7 @@ from typing import Final
|
|
|
11
11
|
from typing import cast
|
|
12
12
|
|
|
13
13
|
from ..api import HttpMethod
|
|
14
|
+
from ..config import Config
|
|
14
15
|
from ..connector import DataConnectorType
|
|
15
16
|
from ..file import FileFormat
|
|
16
17
|
from ..ops.types import PipelineConfig
|
|
@@ -18,7 +19,6 @@ from ..types import JSONData
|
|
|
18
19
|
from ..types import JSONDict
|
|
19
20
|
from ..types import StrPath
|
|
20
21
|
from ..utils import print_json
|
|
21
|
-
from ..workflow import load_pipeline_config
|
|
22
22
|
from .extract import extract
|
|
23
23
|
from .extract import extract_from_api_source
|
|
24
24
|
from .load import load
|
|
@@ -176,7 +176,7 @@ def run(
|
|
|
176
176
|
If the job is not found or if there are configuration issues.
|
|
177
177
|
"""
|
|
178
178
|
cfg_path = config_path or DEFAULT_CONFIG_PATH
|
|
179
|
-
cfg =
|
|
179
|
+
cfg = Config.from_yaml(cfg_path, substitute=True)
|
|
180
180
|
|
|
181
181
|
# Lookup job by name
|
|
182
182
|
if not (job_obj := next((j for j in cfg.jobs if j.name == job), None)):
|
etlplus/ops/utils.py
CHANGED
|
@@ -26,7 +26,7 @@ from ..utils import normalize_choice
|
|
|
26
26
|
# SECTION: TYPED DICTIONARIES =============================================== #
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
class
|
|
29
|
+
class ValidationResultDict(TypedDict, total=False):
|
|
30
30
|
"""Shape returned by ``validate_fn`` callables."""
|
|
31
31
|
|
|
32
32
|
valid: bool
|
|
@@ -44,7 +44,7 @@ type ValidationPhase = Literal['before_transform', 'after_transform']
|
|
|
44
44
|
type ValidationWindow = Literal['before_transform', 'after_transform', 'both']
|
|
45
45
|
type ValidationSeverity = Literal['warn', 'error']
|
|
46
46
|
|
|
47
|
-
type ValidateFn = Callable[[Any, Ruleset],
|
|
47
|
+
type ValidateFn = Callable[[Any, Ruleset], ValidationResultDict]
|
|
48
48
|
type PrintFn = Callable[[Any], None]
|
|
49
49
|
|
|
50
50
|
|
|
@@ -198,7 +198,7 @@ def maybe_validate(
|
|
|
198
198
|
Failure severity (``"warn"`` or ``"error"``).
|
|
199
199
|
validate_fn : ValidateFn
|
|
200
200
|
Engine that performs validation and returns a
|
|
201
|
-
:class:`
|
|
201
|
+
:class:`ValidationResultDict` instance.
|
|
202
202
|
print_json_fn : PrintFn
|
|
203
203
|
Structured logger invoked when validation fails.
|
|
204
204
|
|
|
@@ -270,7 +270,7 @@ def _log_failure(
|
|
|
270
270
|
phase: ValidationPhase,
|
|
271
271
|
window: ValidationWindow,
|
|
272
272
|
ruleset_name: str | None,
|
|
273
|
-
result:
|
|
273
|
+
result: ValidationResultDict,
|
|
274
274
|
) -> None:
|
|
275
275
|
"""
|
|
276
276
|
Emit a structured message describing the failed validation.
|
|
@@ -285,7 +285,7 @@ def _log_failure(
|
|
|
285
285
|
Configured validation window.
|
|
286
286
|
ruleset_name : str | None
|
|
287
287
|
Name of the validation ruleset.
|
|
288
|
-
result :
|
|
288
|
+
result : ValidationResultDict
|
|
289
289
|
Result of the failed validation.
|
|
290
290
|
"""
|
|
291
291
|
printer(
|
etlplus/ops/validate.py
CHANGED
|
@@ -44,9 +44,9 @@ from .load import load_data
|
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
__all__ = [
|
|
47
|
-
'
|
|
48
|
-
'
|
|
49
|
-
'
|
|
47
|
+
'FieldRulesDict',
|
|
48
|
+
'FieldValidationDict',
|
|
49
|
+
'ValidationDict',
|
|
50
50
|
'validate_field',
|
|
51
51
|
'validate',
|
|
52
52
|
]
|
|
@@ -69,7 +69,7 @@ TYPE_MAP: Final[dict[str, type | tuple[type, ...]]] = {
|
|
|
69
69
|
# SECTION: TYPED DICTS ====================================================== #
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
class
|
|
72
|
+
class FieldRulesDict(TypedDict, total=False):
|
|
73
73
|
"""
|
|
74
74
|
Validation rules for a single field.
|
|
75
75
|
|
|
@@ -93,7 +93,7 @@ class FieldRules(TypedDict, total=False):
|
|
|
93
93
|
enum: list[Any]
|
|
94
94
|
|
|
95
95
|
|
|
96
|
-
class
|
|
96
|
+
class FieldValidationDict(TypedDict):
|
|
97
97
|
"""
|
|
98
98
|
Validation result for a single field.
|
|
99
99
|
|
|
@@ -109,7 +109,7 @@ class FieldValidation(TypedDict):
|
|
|
109
109
|
errors: list[str]
|
|
110
110
|
|
|
111
111
|
|
|
112
|
-
class
|
|
112
|
+
class ValidationDict(TypedDict):
|
|
113
113
|
"""
|
|
114
114
|
Validation result for a complete data structure.
|
|
115
115
|
|
|
@@ -134,7 +134,7 @@ class Validation(TypedDict):
|
|
|
134
134
|
# SECTION: TYPE ALIASES ===================================================== #
|
|
135
135
|
|
|
136
136
|
|
|
137
|
-
type RulesMap = Mapping[str,
|
|
137
|
+
type RulesMap = Mapping[str, FieldRulesDict]
|
|
138
138
|
|
|
139
139
|
|
|
140
140
|
# SECTION: INTERNAL FUNCTIONS ============================================== #
|
|
@@ -339,8 +339,8 @@ def _validate_record(
|
|
|
339
339
|
|
|
340
340
|
def validate_field(
|
|
341
341
|
value: Any,
|
|
342
|
-
rules: StrAnyMap |
|
|
343
|
-
) ->
|
|
342
|
+
rules: StrAnyMap | FieldRulesDict,
|
|
343
|
+
) -> FieldValidationDict:
|
|
344
344
|
"""
|
|
345
345
|
Validate a single value against field rules.
|
|
346
346
|
|
|
@@ -348,14 +348,14 @@ def validate_field(
|
|
|
348
348
|
----------
|
|
349
349
|
value : Any
|
|
350
350
|
The value to validate. ``None`` is treated as missing.
|
|
351
|
-
rules : StrAnyMap |
|
|
351
|
+
rules : StrAnyMap | FieldRulesDict
|
|
352
352
|
Rule dictionary. Supported keys include ``required``, ``type``,
|
|
353
353
|
``min``, ``max``, ``minLength``, ``maxLength``, ``pattern``, and
|
|
354
354
|
``enum``.
|
|
355
355
|
|
|
356
356
|
Returns
|
|
357
357
|
-------
|
|
358
|
-
|
|
358
|
+
FieldValidationDict
|
|
359
359
|
Result with ``valid`` and a list of ``errors``.
|
|
360
360
|
|
|
361
361
|
Notes
|
|
@@ -438,7 +438,7 @@ def validate_field(
|
|
|
438
438
|
def validate(
|
|
439
439
|
source: StrPath | JSONData,
|
|
440
440
|
rules: RulesMap | None = None,
|
|
441
|
-
) ->
|
|
441
|
+
) -> ValidationDict:
|
|
442
442
|
"""
|
|
443
443
|
Validate data against rules.
|
|
444
444
|
|
|
@@ -452,7 +452,7 @@ def validate(
|
|
|
452
452
|
|
|
453
453
|
Returns
|
|
454
454
|
-------
|
|
455
|
-
|
|
455
|
+
ValidationDict
|
|
456
456
|
Structured result with keys ``valid``, ``errors``, ``field_errors``,
|
|
457
457
|
and ``data``. If loading fails, ``data`` is ``None`` and an error is
|
|
458
458
|
reported in ``errors``.
|
etlplus/types.py
CHANGED
|
@@ -16,7 +16,8 @@ See Also
|
|
|
16
16
|
|
|
17
17
|
Examples
|
|
18
18
|
--------
|
|
19
|
-
>>> from etlplus.types import JSONDict
|
|
19
|
+
>>> from etlplus.types import JSONDict
|
|
20
|
+
>>> from etlplus.ops.types import PipelineConfig
|
|
20
21
|
>>> payload: JSONDict = {'id': 1, 'name': 'Ada'}
|
|
21
22
|
>>> isinstance(payload, dict)
|
|
22
23
|
True
|
etlplus/workflow/README.md
CHANGED
|
@@ -12,8 +12,6 @@ Back to project overview: see the top-level [README](../../README.md).
|
|
|
12
12
|
|
|
13
13
|
- [`etlplus.workflow` Subpackage](#etlplusworkflow-subpackage)
|
|
14
14
|
- [Supported Configuration Types](#supported-configuration-types)
|
|
15
|
-
- [Loading and Validating Configs](#loading-and-validating-configs)
|
|
16
|
-
- [Example: Loading a Pipeline Config](#example-loading-a-pipeline-config)
|
|
17
15
|
- [See Also](#see-also)
|
|
18
16
|
|
|
19
17
|
## Supported Configuration Types
|
|
@@ -23,28 +21,6 @@ Back to project overview: see the top-level [README](../../README.md).
|
|
|
23
21
|
- **Pipeline**: End-to-end pipeline configuration
|
|
24
22
|
- **Profile**: User or environment-specific settings
|
|
25
23
|
|
|
26
|
-
## Loading and Validating Configs
|
|
27
|
-
|
|
28
|
-
Use the provided classes to load and validate configuration files:
|
|
29
|
-
|
|
30
|
-
```python
|
|
31
|
-
from etlplus.workflow import PipelineConfig
|
|
32
|
-
|
|
33
|
-
cfg = PipelineConfig.from_yaml("pipeline.yml")
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
- Supports YAML and JSON formats
|
|
37
|
-
- Validates against expected schema
|
|
38
|
-
|
|
39
|
-
## Example: Loading a Pipeline Config
|
|
40
|
-
|
|
41
|
-
```python
|
|
42
|
-
from etlplus.workflow import PipelineConfig
|
|
43
|
-
|
|
44
|
-
pipeline = PipelineConfig.from_yaml("configs/pipeline.yml")
|
|
45
|
-
print(pipeline)
|
|
46
|
-
```
|
|
47
|
-
|
|
48
24
|
## See Also
|
|
49
25
|
|
|
50
26
|
- Top-level CLI and library usage in the main [README](../../README.md)
|
etlplus/workflow/__init__.py
CHANGED
|
@@ -12,8 +12,6 @@ from .jobs import JobConfig
|
|
|
12
12
|
from .jobs import LoadRef
|
|
13
13
|
from .jobs import TransformRef
|
|
14
14
|
from .jobs import ValidationRef
|
|
15
|
-
from .pipeline import PipelineConfig
|
|
16
|
-
from .pipeline import load_pipeline_config
|
|
17
15
|
from .profile import ProfileConfig
|
|
18
16
|
|
|
19
17
|
# SECTION: EXPORTS ========================================================== #
|
|
@@ -24,11 +22,9 @@ __all__ = [
|
|
|
24
22
|
'ExtractRef',
|
|
25
23
|
'JobConfig',
|
|
26
24
|
'LoadRef',
|
|
27
|
-
'PipelineConfig',
|
|
28
25
|
'ProfileConfig',
|
|
29
26
|
'TransformRef',
|
|
30
27
|
'ValidationRef',
|
|
31
28
|
# Functions
|
|
32
|
-
'load_pipeline_config',
|
|
33
29
|
'topological_sort_jobs',
|
|
34
30
|
]
|
etlplus/workflow/jobs.py
CHANGED
|
@@ -1,49 +1,50 @@
|
|
|
1
|
-
etlplus/README.md,sha256=
|
|
2
|
-
etlplus/__init__.py,sha256=
|
|
1
|
+
etlplus/README.md,sha256=L3un9q7Q7Mstfh0dmSjpsZMZzsOz2tvlWw4_-Y_LZEs,1887
|
|
2
|
+
etlplus/__init__.py,sha256=yHZt-sjjfPjB_CrNhcT9bcMlZCfwiul39ZhQ2VfxpRs,318
|
|
3
3
|
etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
|
|
4
4
|
etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
|
|
5
|
+
etlplus/config.py,sha256=6BCI9hC1yWYAy5WclIHJlg90FYeWn5vAVrT1NWUTwpE,8817
|
|
5
6
|
etlplus/enums.py,sha256=MfQhy3XDpN7oqLrF7_WwZojl7n8cW3RAzsZGRnAbWgc,4073
|
|
6
7
|
etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
|
|
7
8
|
etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
etlplus/types.py,sha256=
|
|
9
|
+
etlplus/types.py,sha256=Op2H1dcmv0Srm9prFnBZjt7f1S4Mqrus7XrdsjoZkIM,3461
|
|
9
10
|
etlplus/utils.py,sha256=X-k_Y8i6oDjlE5aQu9sw3gPw7O2ikiSn4uoheVv_ERc,17091
|
|
10
11
|
etlplus/api/README.md,sha256=amxS_eIcsnNuVvD0x_w8nkyfedOTYbhlY0gGhaFg0DE,8705
|
|
11
|
-
etlplus/api/__init__.py,sha256=
|
|
12
|
-
etlplus/api/auth.py,sha256=
|
|
13
|
-
etlplus/api/config.py,sha256=
|
|
14
|
-
etlplus/api/endpoint_client.py,sha256=
|
|
12
|
+
etlplus/api/__init__.py,sha256=eIHkdNBZv6ViB_5MhW3f3vWMYJLFoF4Tr3Wnb3O7B4E,4647
|
|
13
|
+
etlplus/api/auth.py,sha256=PZEJIBwLwnUGfF76s32a5GnLcpDvu4ghEd-wEAHx4rU,12260
|
|
14
|
+
etlplus/api/config.py,sha256=WmH1GOQxoBAr3vUsmYIyMbXSt7kiyNbtKjyMS1dqt-A,17653
|
|
15
|
+
etlplus/api/endpoint_client.py,sha256=UtvK4h_-fxINM-5QcumkcLJsL0Uw4L0L_4RMxC024Yk,30737
|
|
15
16
|
etlplus/api/enums.py,sha256=Tvkru6V8fzQh2JM2FDLcA_yaPENOKz5JgzxLhieqEvc,1141
|
|
16
|
-
etlplus/api/errors.py,sha256=
|
|
17
|
-
etlplus/api/request_manager.py,sha256=
|
|
18
|
-
etlplus/api/retry_manager.py,sha256=
|
|
19
|
-
etlplus/api/transport.py,sha256=
|
|
20
|
-
etlplus/api/types.py,sha256=
|
|
21
|
-
etlplus/api/utils.py,sha256=
|
|
22
|
-
etlplus/api/pagination/__init__.py,sha256=
|
|
17
|
+
etlplus/api/errors.py,sha256=8LuZfExUpZ67PPqPr6SdAmFA-wc0ocw4JHoBYyEcg0s,4664
|
|
18
|
+
etlplus/api/request_manager.py,sha256=K3tlRFflUM-_S-optnHzJx_AWcbd0ZQGVX_NytkN4zg,18690
|
|
19
|
+
etlplus/api/retry_manager.py,sha256=RV5xkmbExNHF_b-IN5RWx2wawiJzeklc9miGXPQb7U4,11326
|
|
20
|
+
etlplus/api/transport.py,sha256=f9hgtlnz5fFUsMYWdwAkSsdULJSv-276ekP8u_k3e4Y,9448
|
|
21
|
+
etlplus/api/types.py,sha256=gxwmgPDra7OfAPjJv2Mq45axd_s4TiGAqYtMCx167UM,8008
|
|
22
|
+
etlplus/api/utils.py,sha256=cxQYV0iAbhr65_yvhSii5jhfNcVQGgdwuwO86PJVD-s,26415
|
|
23
|
+
etlplus/api/pagination/__init__.py,sha256=K76nkMnMyM81wzbKTQynq1rRv81ndEUVEuF6cmPzvKQ,1401
|
|
23
24
|
etlplus/api/pagination/client.py,sha256=yMEpWqRxTCD4zRc9OYtEyUtShpGH5atiHFEAt95v2FE,5394
|
|
24
|
-
etlplus/api/pagination/config.py,sha256=
|
|
25
|
+
etlplus/api/pagination/config.py,sha256=pt4yAGr03Esr1gEZg3Nm6kfBOM1FpWwf_t6XTpvBd4s,13596
|
|
25
26
|
etlplus/api/pagination/paginator.py,sha256=B0OK_0FVmUz3-lCDeKgDOqYJOoEQtjO6I5eSmK58tbY,24433
|
|
26
|
-
etlplus/api/rate_limiting/__init__.py,sha256=
|
|
27
|
-
etlplus/api/rate_limiting/config.py,sha256=
|
|
28
|
-
etlplus/api/rate_limiting/rate_limiter.py,sha256=
|
|
27
|
+
etlplus/api/rate_limiting/__init__.py,sha256=8VIjkW2wGjTFJjjAqOBSFKcKsBFuYndS4o33PLSo_q8,1072
|
|
28
|
+
etlplus/api/rate_limiting/config.py,sha256=U8T8BxrX87uFR-ksTG1FkmxoVr-FTaLIa0wWeXIcZss,9775
|
|
29
|
+
etlplus/api/rate_limiting/rate_limiter.py,sha256=qmOf15qvhFk2htMsGcHfgzATqOBd2BdrG8ShadfOIgY,7035
|
|
29
30
|
etlplus/cli/README.md,sha256=8H_G2d3HteYIU6ReX9K9DM485QjWDT5vHMQbGD_vv20,1237
|
|
30
31
|
etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
|
|
31
|
-
etlplus/cli/commands.py,sha256=
|
|
32
|
+
etlplus/cli/commands.py,sha256=Mbnu_YYUrOumbDjkul9x5VjP8VXW5u08xNi4nLF9Yyo,25048
|
|
32
33
|
etlplus/cli/constants.py,sha256=0F7dXIQKWUhhVu2Us527GJeknJIWpBqz7CK2e5OQgcE,1947
|
|
33
|
-
etlplus/cli/handlers.py,sha256=
|
|
34
|
+
etlplus/cli/handlers.py,sha256=JdN7W7mqmQL9xyU7PkBtsQf7eu3j5-E2AhAQvbfy-4g,18470
|
|
34
35
|
etlplus/cli/io.py,sha256=tGGNQ4ecezqj-mD285fgBVrYdphdeqApsyV9VojOj1I,7836
|
|
35
36
|
etlplus/cli/main.py,sha256=68_uJwmWajhOC9o4R_ns8IQloC9BFmAKC_9GlQOxKWg,5239
|
|
36
37
|
etlplus/cli/options.py,sha256=vfXT3YLh7wG1iC-aTdSg6ItMC8l6n0Lozmy53XjqLbA,1199
|
|
37
38
|
etlplus/cli/state.py,sha256=3Dq5BKct0uAvRajtc2yHbsX7wqepZOwlAMKsyvQcnqk,7918
|
|
38
39
|
etlplus/cli/types.py,sha256=tclhKVJXDqHzlTQBYKARfqMgDOcuBJ-Zej2pvFy96WM,652
|
|
39
|
-
etlplus/connector/__init__.py,sha256=
|
|
40
|
-
etlplus/connector/api.py,sha256=
|
|
40
|
+
etlplus/connector/__init__.py,sha256=C1nMuEvCnYS07kpoP91aWEg1utV1D4bO603OzveF7s0,1012
|
|
41
|
+
etlplus/connector/api.py,sha256=uXoQvOGGnZEP96H0O19nMgu7uiBfTqEFNuFl0r9oBbQ,4531
|
|
41
42
|
etlplus/connector/connector.py,sha256=OQP4kK_1O6g9FnDBrE3L58LOSfImsK5EBHtKI4N71u8,574
|
|
42
43
|
etlplus/connector/core.py,sha256=0GeXjlZFnyS-4j7jR_AtclQtELE6x-vodHJ4rfjFLL8,2795
|
|
43
|
-
etlplus/connector/database.py,sha256=
|
|
44
|
+
etlplus/connector/database.py,sha256=Y8fzcbzucIZW64g9zkqO-M_H02CiW5lB0IDk0RynWJo,3014
|
|
44
45
|
etlplus/connector/enums.py,sha256=43NziUOpol4YvBtM13WJJzY1EAQOjaWESxLl7J2ZT8U,1069
|
|
45
|
-
etlplus/connector/file.py,sha256=
|
|
46
|
-
etlplus/connector/types.py,sha256=
|
|
46
|
+
etlplus/connector/file.py,sha256=AsEXUHOokP2s5NQoTazF3Skz7qMj-1FNdvRZj9LfdbM,2858
|
|
47
|
+
etlplus/connector/types.py,sha256=51UPD4edtMRiRL35ZVfbmKTodhrLUiRP5P067SZGzms,953
|
|
47
48
|
etlplus/connector/utils.py,sha256=fS2hPAfuhKTg_L2xDxF5fJnsO1SuuDIiEWU7GuaJKUM,2933
|
|
48
49
|
etlplus/database/README.md,sha256=3Af5BEGLkBmMoGOLtS1GQuj4wKPh_CwSp5NEPMf2uaY,1435
|
|
49
50
|
etlplus/database/__init__.py,sha256=AKJsDl2RHuRGPS-eXgNJeh4aSncJP5Y0yLApBF6i7i8,1052
|
|
@@ -118,26 +119,25 @@ etlplus/file/zsav.py,sha256=5hMuBjYeHw--UL2ZCCDn6TzJkr_YNhdQhvKI6nr3WW0,1674
|
|
|
118
119
|
etlplus/ops/README.md,sha256=8omi7DYZhelc26JKk8Cm8QR8I3OGwziysPj1ivx41iQ,1380
|
|
119
120
|
etlplus/ops/__init__.py,sha256=r5_-pPhSLCD1nq1EbN0rQrLOGpudueeIxCH_JvT2bt0,1718
|
|
120
121
|
etlplus/ops/enums.py,sha256=dC_8CfaTiB2i83Az-oG-2hkjMuAfDADNbcMF2f94UeU,4014
|
|
121
|
-
etlplus/ops/extract.py,sha256=
|
|
122
|
+
etlplus/ops/extract.py,sha256=fPk8LLjEmCZ5U59IUm15vG5aXTmduteCqtsVIlxvvxI,11022
|
|
122
123
|
etlplus/ops/load.py,sha256=yicciVwomUKkdbhuRqbavKBNpT2Hg813BnQzG6IgF4o,10811
|
|
123
|
-
etlplus/ops/run.py,sha256=
|
|
124
|
+
etlplus/ops/run.py,sha256=4HWelMevW0pW_76lJkoMcbzeQMiThMbxzO09wx6yoHg,11278
|
|
124
125
|
etlplus/ops/transform.py,sha256=-41uw_pwOGsMTUYxtXaeYOmTF_fTkN-L4Q9KT1OFe78,25671
|
|
125
126
|
etlplus/ops/types.py,sha256=Cvp8AJzJhJ1iYjyHd7j9ZLioxE2NdK__3g6fOI0qq6Q,4198
|
|
126
|
-
etlplus/ops/utils.py,sha256=
|
|
127
|
-
etlplus/ops/validate.py,sha256
|
|
127
|
+
etlplus/ops/utils.py,sha256=9UXym1W4qCMxBkcqCPUmI1QJ27yh1kOAbVnI1KsAGwE,10855
|
|
128
|
+
etlplus/ops/validate.py,sha256=VtMhrH6itd_PFH4IhBOndvJpxxOPI56OAJhnrSyT_6U,13323
|
|
128
129
|
etlplus/templates/README.md,sha256=IfPXlj1TGVA-uFWosHJhE2rabFW-znxOlOMazO9Z5cE,1361
|
|
129
130
|
etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
|
|
130
131
|
etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
|
|
131
132
|
etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
|
|
132
|
-
etlplus/workflow/README.md,sha256=
|
|
133
|
-
etlplus/workflow/__init__.py,sha256=
|
|
133
|
+
etlplus/workflow/README.md,sha256=QelyVFGX-sZM9mx3v6BXkzX36hv6MI1yK4eCPSOKNwI,1050
|
|
134
|
+
etlplus/workflow/__init__.py,sha256=XgCQr684om0rONrQZ61yQ0r4qqFQL0iLAAB2Mn2BRSE,594
|
|
134
135
|
etlplus/workflow/dag.py,sha256=-f1x8N1eb-PUuiOwEvFLmJwfR7JaMDJihlCHlhrFhgE,2937
|
|
135
|
-
etlplus/workflow/jobs.py,sha256=
|
|
136
|
-
etlplus/workflow/pipeline.py,sha256=PA5zhcfrk--pAg3b3x4oBf29WMj5HqR8zOozz4oEmg8,9387
|
|
136
|
+
etlplus/workflow/jobs.py,sha256=hLE9QJUzQaI0aOEon0P-xxxa6xHp997ANei4F310WRY,8711
|
|
137
137
|
etlplus/workflow/profile.py,sha256=FQU3bzBZ9_yjKC9kCXKN1FQDS9zjNUjtWB1r3UL95_Q,1993
|
|
138
|
-
etlplus-0.16.
|
|
139
|
-
etlplus-0.16.
|
|
140
|
-
etlplus-0.16.
|
|
141
|
-
etlplus-0.16.
|
|
142
|
-
etlplus-0.16.
|
|
143
|
-
etlplus-0.16.
|
|
138
|
+
etlplus-0.16.5.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
|
|
139
|
+
etlplus-0.16.5.dist-info/METADATA,sha256=vtUzae0iSsY6i5PqpcDQQ8q_tBXbFVT5Xy8AD_A9g8o,28114
|
|
140
|
+
etlplus-0.16.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
141
|
+
etlplus-0.16.5.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
142
|
+
etlplus-0.16.5.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
143
|
+
etlplus-0.16.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|