etlplus 0.16.0__py3-none-any.whl → 0.16.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. etlplus/README.md +22 -0
  2. etlplus/__init__.py +2 -0
  3. etlplus/api/__init__.py +14 -14
  4. etlplus/api/auth.py +9 -6
  5. etlplus/api/config.py +6 -6
  6. etlplus/api/endpoint_client.py +16 -16
  7. etlplus/api/errors.py +4 -4
  8. etlplus/api/pagination/__init__.py +6 -6
  9. etlplus/api/pagination/config.py +11 -9
  10. etlplus/api/rate_limiting/__init__.py +2 -2
  11. etlplus/api/rate_limiting/config.py +10 -10
  12. etlplus/api/rate_limiting/rate_limiter.py +2 -2
  13. etlplus/api/request_manager.py +4 -4
  14. etlplus/api/retry_manager.py +6 -6
  15. etlplus/api/transport.py +10 -10
  16. etlplus/api/types.py +47 -26
  17. etlplus/api/utils.py +49 -49
  18. etlplus/cli/commands.py +22 -22
  19. etlplus/cli/handlers.py +12 -13
  20. etlplus/{workflow/pipeline.py → config.py} +54 -91
  21. etlplus/connector/__init__.py +6 -6
  22. etlplus/connector/api.py +7 -7
  23. etlplus/connector/database.py +3 -3
  24. etlplus/connector/file.py +3 -3
  25. etlplus/connector/types.py +2 -2
  26. etlplus/enums.py +35 -167
  27. etlplus/ops/__init__.py +1 -0
  28. etlplus/ops/enums.py +173 -0
  29. etlplus/ops/extract.py +209 -22
  30. etlplus/ops/load.py +140 -34
  31. etlplus/ops/run.py +88 -103
  32. etlplus/ops/transform.py +46 -27
  33. etlplus/ops/types.py +147 -0
  34. etlplus/ops/utils.py +5 -5
  35. etlplus/ops/validate.py +13 -13
  36. etlplus/types.py +5 -102
  37. etlplus/workflow/README.md +0 -24
  38. etlplus/workflow/__init__.py +2 -4
  39. etlplus/workflow/dag.py +23 -1
  40. etlplus/workflow/jobs.py +15 -28
  41. etlplus/workflow/profile.py +4 -2
  42. {etlplus-0.16.0.dist-info → etlplus-0.16.6.dist-info}/METADATA +1 -1
  43. {etlplus-0.16.0.dist-info → etlplus-0.16.6.dist-info}/RECORD +47 -45
  44. {etlplus-0.16.0.dist-info → etlplus-0.16.6.dist-info}/WHEEL +0 -0
  45. {etlplus-0.16.0.dist-info → etlplus-0.16.6.dist-info}/entry_points.txt +0 -0
  46. {etlplus-0.16.0.dist-info → etlplus-0.16.6.dist-info}/licenses/LICENSE +0 -0
  47. {etlplus-0.16.0.dist-info → etlplus-0.16.6.dist-info}/top_level.txt +0 -0
etlplus/enums.py CHANGED
@@ -1,18 +1,14 @@
1
1
  """
2
2
  :mod:`etlplus.enums` module.
3
3
 
4
- Shared enumeration types used across ETLPlus modules.
4
+ Shared enumeration base class.
5
5
  """
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import enum
10
- import operator as _op
11
- from statistics import fmean
12
10
  from typing import Self
13
11
 
14
- from .types import AggregateFunc
15
- from .types import OperatorFunc
16
12
  from .types import StrStrMap
17
13
 
18
14
  # SECTION: EXPORTS ========================================================== #
@@ -20,10 +16,7 @@ from .types import StrStrMap
20
16
 
21
17
  __all__ = [
22
18
  # Enums
23
- 'AggregateName',
24
19
  'CoercibleStrEnum',
25
- 'OperatorName',
26
- 'PipelineStep',
27
20
  ]
28
21
 
29
22
 
@@ -41,6 +34,7 @@ class CoercibleStrEnum(enum.StrEnum):
41
34
  Notes
42
35
  -----
43
36
  - Values are normalized via ``str(value).strip().casefold()``.
37
+ - If value matching fails, the raw string is tried as a member name.
44
38
  - Error messages enumerate allowed values for easier debugging.
45
39
  """
46
40
 
@@ -56,7 +50,13 @@ class CoercibleStrEnum(enum.StrEnum):
56
50
  Returns
57
51
  -------
58
52
  StrStrMap
59
- A mapping of alias names to their corresponding enum member names.
53
+ A mapping of alias strings to their corresponding enum member
54
+ values or names.
55
+
56
+ Notes
57
+ -----
58
+ - Alias keys are normalized via ``str(key).strip().casefold()``.
59
+ - Alias values should be member values or member names.
60
60
  """
61
61
  return {}
62
62
 
@@ -80,7 +80,7 @@ class CoercibleStrEnum(enum.StrEnum):
80
80
  Parameters
81
81
  ----------
82
82
  value : Self | str | object
83
- An existing enum member or a text value to normalize.
83
+ An existing enum member or a string-like value to normalize.
84
84
 
85
85
  Returns
86
86
  -------
@@ -95,10 +95,26 @@ class CoercibleStrEnum(enum.StrEnum):
95
95
  if isinstance(value, cls):
96
96
  return value
97
97
  try:
98
- normalized = str(value).strip().casefold()
99
- resolved = cls.aliases().get(normalized, normalized)
100
- return cls(resolved) # type: ignore[arg-type]
101
- except (ValueError, TypeError) as e:
98
+ raw = str(value).strip()
99
+ normalized = raw.casefold()
100
+ aliases = {
101
+ str(key).strip().casefold(): alias
102
+ for key, alias in cls.aliases().items()
103
+ }
104
+ resolved = aliases.get(normalized)
105
+ if resolved is None:
106
+ try:
107
+ return cls(normalized) # type: ignore[arg-type]
108
+ except (ValueError, TypeError):
109
+ return cls[raw] # type: ignore[index]
110
+ if isinstance(resolved, cls):
111
+ return resolved
112
+ try:
113
+ return cls(resolved) # type: ignore[arg-type]
114
+ except (ValueError, TypeError):
115
+ # Allow aliases to reference member names.
116
+ return cls[resolved] # type: ignore[index]
117
+ except (ValueError, TypeError, KeyError) as e:
102
118
  allowed = ', '.join(cls.choices())
103
119
  raise ValueError(
104
120
  f'Invalid {cls.__name__} value: {value!r}. Allowed: {allowed}',
@@ -107,15 +123,15 @@ class CoercibleStrEnum(enum.StrEnum):
107
123
  @classmethod
108
124
  def try_coerce(
109
125
  cls,
110
- value: object,
126
+ value: Self | str | object,
111
127
  ) -> Self | None:
112
128
  """
113
- Best-effort parse; return ``None`` on failure instead of raising.
129
+ Attempt to coerce a value into the enum; return ``None`` on failure.
114
130
 
115
131
  Parameters
116
132
  ----------
117
- value : object
118
- An existing enum member or a text value to normalize.
133
+ value : Self | str | object
134
+ An existing enum member or a string-like value to normalize.
119
135
 
120
136
  Returns
121
137
  -------
@@ -124,153 +140,5 @@ class CoercibleStrEnum(enum.StrEnum):
124
140
  """
125
141
  try:
126
142
  return cls.coerce(value)
127
- except ValueError:
143
+ except (ValueError, TypeError, KeyError):
128
144
  return None
129
-
130
-
131
- # SECTION: ENUMS ============================================================ #
132
-
133
-
134
- class AggregateName(CoercibleStrEnum):
135
- """Supported aggregations with helpers."""
136
-
137
- # -- Constants -- #
138
-
139
- AVG = 'avg'
140
- COUNT = 'count'
141
- MAX = 'max'
142
- MIN = 'min'
143
- SUM = 'sum'
144
-
145
- # -- Class Methods -- #
146
-
147
- @property
148
- def func(self) -> AggregateFunc:
149
- """
150
- Get the aggregation function for this aggregation type.
151
-
152
- Returns
153
- -------
154
- AggregateFunc
155
- The aggregation function corresponding to this aggregation type.
156
- """
157
- if self is AggregateName.COUNT:
158
- return lambda xs, n: n
159
- if self is AggregateName.MAX:
160
- return lambda xs, n: (max(xs) if xs else None)
161
- if self is AggregateName.MIN:
162
- return lambda xs, n: (min(xs) if xs else None)
163
- if self is AggregateName.SUM:
164
- return lambda xs, n: sum(xs)
165
-
166
- # AVG
167
- return lambda xs, n: (fmean(xs) if xs else 0.0)
168
-
169
-
170
- class OperatorName(CoercibleStrEnum):
171
- """Supported comparison operators with helpers."""
172
-
173
- # -- Constants -- #
174
-
175
- EQ = 'eq'
176
- NE = 'ne'
177
- GT = 'gt'
178
- GTE = 'gte'
179
- LT = 'lt'
180
- LTE = 'lte'
181
- IN = 'in'
182
- CONTAINS = 'contains'
183
-
184
- # -- Getters -- #
185
-
186
- @property
187
- def func(self) -> OperatorFunc:
188
- """
189
- Get the comparison function for this operator.
190
-
191
- Returns
192
- -------
193
- OperatorFunc
194
- The comparison function corresponding to this operator.
195
- """
196
- match self:
197
- case OperatorName.EQ:
198
- return _op.eq
199
- case OperatorName.NE:
200
- return _op.ne
201
- case OperatorName.GT:
202
- return _op.gt
203
- case OperatorName.GTE:
204
- return _op.ge
205
- case OperatorName.LT:
206
- return _op.lt
207
- case OperatorName.LTE:
208
- return _op.le
209
- case OperatorName.IN:
210
- return lambda a, b: a in b
211
- case OperatorName.CONTAINS:
212
- return lambda a, b: b in a
213
-
214
- # -- Class Methods -- #
215
-
216
- @classmethod
217
- def aliases(cls) -> StrStrMap:
218
- """
219
- Return a mapping of common aliases for each enum member.
220
-
221
- Returns
222
- -------
223
- StrStrMap
224
- A mapping of alias names to their corresponding enum member names.
225
- """
226
- return {
227
- '==': 'eq',
228
- '=': 'eq',
229
- '!=': 'ne',
230
- '<>': 'ne',
231
- '>=': 'gte',
232
- '≥': 'gte',
233
- '<=': 'lte',
234
- '≤': 'lte',
235
- '>': 'gt',
236
- '<': 'lt',
237
- }
238
-
239
-
240
- class PipelineStep(CoercibleStrEnum):
241
- """Pipeline step names as an enum for internal orchestration."""
242
-
243
- # -- Constants -- #
244
-
245
- FILTER = 'filter'
246
- MAP = 'map'
247
- SELECT = 'select'
248
- SORT = 'sort'
249
- AGGREGATE = 'aggregate'
250
-
251
- # -- Getters -- #
252
-
253
- @property
254
- def order(self) -> int:
255
- """
256
- Get the execution order of this pipeline step.
257
-
258
- Returns
259
- -------
260
- int
261
- The execution order of this pipeline step.
262
- """
263
- return _PIPELINE_ORDER_INDEX[self]
264
-
265
-
266
- # SECTION: INTERNAL CONSTANTS ============================================== #
267
-
268
-
269
- # Precomputed order index for PipelineStep; avoids recomputing on each access.
270
- _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
271
- PipelineStep.FILTER: 0,
272
- PipelineStep.MAP: 1,
273
- PipelineStep.SELECT: 2,
274
- PipelineStep.SORT: 3,
275
- PipelineStep.AGGREGATE: 4,
276
- }
etlplus/ops/__init__.py CHANGED
@@ -52,6 +52,7 @@ from .validate import validate
52
52
 
53
53
 
54
54
  __all__ = [
55
+ # Functions
55
56
  'extract',
56
57
  'load',
57
58
  'run',
etlplus/ops/enums.py ADDED
@@ -0,0 +1,173 @@
1
+ """
2
+ :mod:`etlplus.ops.enums` module.
3
+
4
+ Operation-specific enums and helpers.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import operator as _op
10
+ from statistics import fmean
11
+
12
+ from ..enums import CoercibleStrEnum
13
+ from ..types import StrStrMap
14
+ from .types import AggregateFunc
15
+ from .types import OperatorFunc
16
+
17
+ # SECTION: EXPORTS ========================================================= #
18
+
19
+
20
+ __all__ = [
21
+ # Enums
22
+ 'AggregateName',
23
+ 'OperatorName',
24
+ 'PipelineStep',
25
+ ]
26
+
27
+
28
+ # SECTION: ENUMS ============================================================ #
29
+
30
+
31
+ class AggregateName(CoercibleStrEnum):
32
+ """Supported aggregations with helpers."""
33
+
34
+ # -- Constants -- #
35
+
36
+ AVG = 'avg'
37
+ COUNT = 'count'
38
+ MAX = 'max'
39
+ MIN = 'min'
40
+ SUM = 'sum'
41
+
42
+ # -- Class Methods -- #
43
+
44
+ @property
45
+ def func(self) -> AggregateFunc:
46
+ """
47
+ Get the aggregation function for this aggregation type.
48
+
49
+ Returns
50
+ -------
51
+ AggregateFunc
52
+ The aggregation function corresponding to this aggregation type.
53
+ """
54
+ if self is AggregateName.COUNT:
55
+ return lambda xs, n: n
56
+ if self is AggregateName.MAX:
57
+ return lambda xs, n: (max(xs) if xs else None)
58
+ if self is AggregateName.MIN:
59
+ return lambda xs, n: (min(xs) if xs else None)
60
+ if self is AggregateName.SUM:
61
+ return lambda xs, n: sum(xs)
62
+
63
+ # AVG
64
+ return lambda xs, n: (fmean(xs) if xs else 0.0)
65
+
66
+
67
+ class OperatorName(CoercibleStrEnum):
68
+ """Supported comparison operators with helpers."""
69
+
70
+ # -- Constants -- #
71
+
72
+ EQ = 'eq'
73
+ NE = 'ne'
74
+ GT = 'gt'
75
+ GTE = 'gte'
76
+ LT = 'lt'
77
+ LTE = 'lte'
78
+ IN = 'in'
79
+ CONTAINS = 'contains'
80
+
81
+ # -- Getters -- #
82
+
83
+ @property
84
+ def func(self) -> OperatorFunc:
85
+ """
86
+ Get the comparison function for this operator.
87
+
88
+ Returns
89
+ -------
90
+ OperatorFunc
91
+ The comparison function corresponding to this operator.
92
+ """
93
+ match self:
94
+ case OperatorName.EQ:
95
+ return _op.eq
96
+ case OperatorName.NE:
97
+ return _op.ne
98
+ case OperatorName.GT:
99
+ return _op.gt
100
+ case OperatorName.GTE:
101
+ return _op.ge
102
+ case OperatorName.LT:
103
+ return _op.lt
104
+ case OperatorName.LTE:
105
+ return _op.le
106
+ case OperatorName.IN:
107
+ return lambda a, b: a in b
108
+ case OperatorName.CONTAINS:
109
+ return lambda a, b: b in a
110
+
111
+ # -- Class Methods -- #
112
+
113
+ @classmethod
114
+ def aliases(cls) -> StrStrMap:
115
+ """
116
+ Return a mapping of common aliases for each enum member.
117
+
118
+ Returns
119
+ -------
120
+ StrStrMap
121
+ A mapping of alias names to their corresponding enum member names.
122
+ """
123
+ return {
124
+ '==': 'eq',
125
+ '=': 'eq',
126
+ '!=': 'ne',
127
+ '<>': 'ne',
128
+ '>=': 'gte',
129
+ '≥': 'gte',
130
+ '<=': 'lte',
131
+ '≤': 'lte',
132
+ '>': 'gt',
133
+ '<': 'lt',
134
+ }
135
+
136
+
137
+ class PipelineStep(CoercibleStrEnum):
138
+ """Pipeline step names as an enum for internal orchestration."""
139
+
140
+ # -- Constants -- #
141
+
142
+ FILTER = 'filter'
143
+ MAP = 'map'
144
+ SELECT = 'select'
145
+ SORT = 'sort'
146
+ AGGREGATE = 'aggregate'
147
+
148
+ # -- Getters -- #
149
+
150
+ @property
151
+ def order(self) -> int:
152
+ """
153
+ Get the execution order of this pipeline step.
154
+
155
+ Returns
156
+ -------
157
+ int
158
+ The execution order of this pipeline step.
159
+ """
160
+ return _PIPELINE_ORDER_INDEX[self]
161
+
162
+
163
+ # SECTION: INTERNAL CONSTANTS ============================================== #
164
+
165
+
166
+ # Precomputed order index for PipelineStep; avoids recomputing on each access.
167
+ _PIPELINE_ORDER_INDEX: dict[PipelineStep, int] = {
168
+ PipelineStep.FILTER: 0,
169
+ PipelineStep.MAP: 1,
170
+ PipelineStep.SELECT: 2,
171
+ PipelineStep.SORT: 3,
172
+ PipelineStep.AGGREGATE: 4,
173
+ }