etlplus 0.5.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. etlplus/__init__.py +43 -0
  2. etlplus/__main__.py +22 -0
  3. etlplus/__version__.py +14 -0
  4. etlplus/api/README.md +237 -0
  5. etlplus/api/__init__.py +136 -0
  6. etlplus/api/auth.py +432 -0
  7. etlplus/api/config.py +633 -0
  8. etlplus/api/endpoint_client.py +885 -0
  9. etlplus/api/errors.py +170 -0
  10. etlplus/api/pagination/__init__.py +47 -0
  11. etlplus/api/pagination/client.py +188 -0
  12. etlplus/api/pagination/config.py +440 -0
  13. etlplus/api/pagination/paginator.py +775 -0
  14. etlplus/api/rate_limiting/__init__.py +38 -0
  15. etlplus/api/rate_limiting/config.py +343 -0
  16. etlplus/api/rate_limiting/rate_limiter.py +266 -0
  17. etlplus/api/request_manager.py +589 -0
  18. etlplus/api/retry_manager.py +430 -0
  19. etlplus/api/transport.py +325 -0
  20. etlplus/api/types.py +172 -0
  21. etlplus/cli/__init__.py +15 -0
  22. etlplus/cli/app.py +1367 -0
  23. etlplus/cli/handlers.py +771 -0
  24. etlplus/cli/main.py +616 -0
  25. etlplus/config/__init__.py +56 -0
  26. etlplus/config/connector.py +372 -0
  27. etlplus/config/jobs.py +311 -0
  28. etlplus/config/pipeline.py +339 -0
  29. etlplus/config/profile.py +78 -0
  30. etlplus/config/types.py +204 -0
  31. etlplus/config/utils.py +120 -0
  32. etlplus/ddl.py +197 -0
  33. etlplus/enums.py +414 -0
  34. etlplus/extract.py +218 -0
  35. etlplus/file.py +657 -0
  36. etlplus/load.py +336 -0
  37. etlplus/mixins.py +62 -0
  38. etlplus/py.typed +0 -0
  39. etlplus/run.py +368 -0
  40. etlplus/run_helpers.py +843 -0
  41. etlplus/templates/__init__.py +5 -0
  42. etlplus/templates/ddl.sql.j2 +128 -0
  43. etlplus/templates/view.sql.j2 +69 -0
  44. etlplus/transform.py +1049 -0
  45. etlplus/types.py +227 -0
  46. etlplus/utils.py +638 -0
  47. etlplus/validate.py +493 -0
  48. etlplus/validation/__init__.py +44 -0
  49. etlplus/validation/utils.py +389 -0
  50. etlplus-0.5.2.dist-info/METADATA +608 -0
  51. etlplus-0.5.2.dist-info/RECORD +55 -0
  52. etlplus-0.5.2.dist-info/WHEEL +5 -0
  53. etlplus-0.5.2.dist-info/entry_points.txt +2 -0
  54. etlplus-0.5.2.dist-info/licenses/LICENSE +21 -0
  55. etlplus-0.5.2.dist-info/top_level.txt +1 -0
etlplus/utils.py ADDED
@@ -0,0 +1,638 @@
1
+ """
2
+ :mod:`etlplus.utils` module.
3
+
4
+ Small shared helpers used across modules.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import json
11
+ from collections.abc import Callable
12
+ from collections.abc import Mapping
13
+ from typing import Any
14
+ from typing import TypeVar
15
+
16
+ from .types import JSONData
17
+ from .types import StrAnyMap
18
+
19
+ # SECTION: EXPORTS ========================================================== #
20
+
21
+
22
+ __all__ = [
23
+ # Data utilities
24
+ 'count_records',
25
+ 'json_type',
26
+ 'print_json',
27
+ # Mapping utilities
28
+ 'cast_str_dict',
29
+ 'coerce_dict',
30
+ 'maybe_mapping',
31
+ # Float coercion
32
+ 'to_float',
33
+ 'to_maximum_float',
34
+ 'to_minimum_float',
35
+ 'to_positive_float',
36
+ # Int coercion
37
+ 'to_int',
38
+ 'to_maximum_int',
39
+ 'to_minimum_int',
40
+ 'to_positive_int',
41
+ # Generic number coercion
42
+ 'to_number',
43
+ # Text processing
44
+ 'normalized_str',
45
+ ]
46
+
47
+
48
+ # SECTION: TYPE VARS ======================================================== #
49
+
50
+
51
+ Num = TypeVar('Num', int, float)
52
+ # type Num = int | float
53
+
54
+
55
+ # SECTION: FUNCTIONS ======================================================== #
56
+
57
+
58
+ # -- Data Utilities -- #
59
+
60
+
61
+ def cast_str_dict(
62
+ mapping: StrAnyMap | None,
63
+ ) -> dict[str, str]:
64
+ """
65
+ Return a new ``dict`` with keys and values coerced to ``str``.
66
+
67
+ Parameters
68
+ ----------
69
+ mapping : StrAnyMap | None
70
+ Mapping to normalize; ``None`` yields ``{}``.
71
+
72
+ Returns
73
+ -------
74
+ dict[str, str]
75
+ Dictionary of the original key/value pairs converted via ``str()``.
76
+ """
77
+ if not mapping:
78
+ return {}
79
+ return {str(key): str(value) for key, value in mapping.items()}
80
+
81
+
82
+ def coerce_dict(
83
+ value: Any,
84
+ ) -> dict[str, Any]:
85
+ """
86
+ Return a ``dict`` copy when ``value`` is mapping-like.
87
+
88
+ Parameters
89
+ ----------
90
+ value : Any
91
+ Mapping-like object to copy. ``None`` returns an empty dict.
92
+
93
+ Returns
94
+ -------
95
+ dict[str, Any]
96
+ Shallow copy of ``value`` converted to a standard ``dict``.
97
+ """
98
+ return dict(value) if isinstance(value, Mapping) else {}
99
+
100
+
101
+ def count_records(
102
+ data: JSONData,
103
+ ) -> int:
104
+ """
105
+ Return a consistent record count for JSON-like data payloads.
106
+
107
+ Lists are treated as multiple records; dicts as a single record.
108
+
109
+ Parameters
110
+ ----------
111
+ data : JSONData
112
+ Data payload to count records for.
113
+
114
+ Returns
115
+ -------
116
+ int
117
+ Number of records in `data`.
118
+ """
119
+ return len(data) if isinstance(data, list) else 1
120
+
121
+
122
+ def json_type(
123
+ option: str,
124
+ ) -> Any:
125
+ """
126
+ Argparse ``type=`` hook that parses a JSON string.
127
+
128
+ Parameters
129
+ ----------
130
+ option : str
131
+ Raw CLI string to parse as JSON.
132
+
133
+ Returns
134
+ -------
135
+ Any
136
+ Parsed JSON value.
137
+
138
+ Raises
139
+ ------
140
+ argparse.ArgumentTypeError
141
+ If the input cannot be parsed as JSON.
142
+ """
143
+ try:
144
+ return json.loads(option)
145
+ except json.JSONDecodeError as e: # pragma: no cover - argparse path
146
+ raise argparse.ArgumentTypeError(
147
+ f'Invalid JSON: {e.msg} (pos {e.pos})',
148
+ ) from e
149
+
150
+
151
+ def maybe_mapping(
152
+ value: Any,
153
+ ) -> StrAnyMap | None:
154
+ """
155
+ Return ``value`` when it is mapping-like; otherwise ``None``.
156
+
157
+ Parameters
158
+ ----------
159
+ value : Any
160
+ Value to test.
161
+
162
+ Returns
163
+ -------
164
+ StrAnyMap | None
165
+ The input value if it is a mapping; ``None`` if not.
166
+ """
167
+ return value if isinstance(value, Mapping) else None
168
+
169
+
170
+ def print_json(
171
+ obj: Any,
172
+ ) -> None:
173
+ """
174
+ Pretty-print ``obj`` as UTF-8 JSON without ASCII escaping.
175
+
176
+ Parameters
177
+ ----------
178
+ obj : Any
179
+ Object to serialize as JSON.
180
+
181
+ Returns
182
+ -------
183
+ None
184
+ This helper writes directly to ``stdout``.
185
+ """
186
+ print(json.dumps(obj, indent=2, ensure_ascii=False))
187
+
188
+
189
+ # -- Float Coercion -- #
190
+
191
+
192
+ def to_float(
193
+ value: Any,
194
+ default: float | None = None,
195
+ minimum: float | None = None,
196
+ maximum: float | None = None,
197
+ ) -> float | None:
198
+ """
199
+ Coerce ``value`` to a float with optional fallback and bounds.
200
+
201
+ Notes
202
+ -----
203
+ For strings, leading/trailing whitespace is ignored. Returns ``None``
204
+ when coercion fails and no ``default`` is provided.
205
+ """
206
+ return _normalize_number(
207
+ _coerce_float,
208
+ value,
209
+ default=default,
210
+ minimum=minimum,
211
+ maximum=maximum,
212
+ )
213
+
214
+
215
+ def to_maximum_float(
216
+ value: Any,
217
+ default: float,
218
+ ) -> float:
219
+ """
220
+ Return the greater of ``default`` and ``value`` after float coercion.
221
+
222
+ Parameters
223
+ ----------
224
+ value : Any
225
+ Candidate input coerced with :func:`to_float`.
226
+ default : float
227
+ Baseline float value that acts as the lower bound.
228
+
229
+ Returns
230
+ -------
231
+ float
232
+ ``default`` if coercion fails; else ``max(coerced, default)``.
233
+ """
234
+ result = to_float(value, default)
235
+ return max(_value_or_default(result, default), default)
236
+
237
+
238
+ def to_minimum_float(
239
+ value: Any,
240
+ default: float,
241
+ ) -> float:
242
+ """
243
+ Return the lesser of ``default`` and ``value`` after float coercion.
244
+
245
+ Parameters
246
+ ----------
247
+ value : Any
248
+ Candidate input coerced with :func:`to_float`.
249
+ default : float
250
+ Baseline float value that acts as the upper bound.
251
+
252
+ Returns
253
+ -------
254
+ float
255
+ ``default`` if coercion fails; else ``min(coerced, default)``.
256
+ """
257
+ result = to_float(value, default)
258
+ return min(_value_or_default(result, default), default)
259
+
260
+
261
+ def to_positive_float(value: Any) -> float | None:
262
+ """
263
+ Return a positive float when coercion succeeds.
264
+
265
+ Parameters
266
+ ----------
267
+ value : Any
268
+ Value coerced using :func:`to_float`.
269
+
270
+ Returns
271
+ -------
272
+ float | None
273
+ Positive float if coercion succeeds and ``value > 0``; else ``None``.
274
+ """
275
+ result = to_float(value)
276
+ if result is None or result <= 0:
277
+ return None
278
+ return result
279
+
280
+
281
+ # -- Int Coercion -- #
282
+
283
+
284
+ def to_int(
285
+ value: Any,
286
+ default: int | None = None,
287
+ minimum: int | None = None,
288
+ maximum: int | None = None,
289
+ ) -> int | None:
290
+ """
291
+ Coerce ``value`` to an integer with optional fallback and bounds.
292
+
293
+ Notes
294
+ -----
295
+ For strings, leading/trailing whitespace is ignored. Returns ``None``
296
+ when coercion fails and no ``default`` is provided.
297
+ """
298
+ return _normalize_number(
299
+ _coerce_int,
300
+ value,
301
+ default=default,
302
+ minimum=minimum,
303
+ maximum=maximum,
304
+ )
305
+
306
+
307
+ def to_maximum_int(
308
+ value: Any,
309
+ default: int,
310
+ ) -> int:
311
+ """
312
+ Return the greater of ``default`` and ``value`` after integer coercion.
313
+
314
+ Parameters
315
+ ----------
316
+ value : Any
317
+ Candidate input coerced with :func:`to_int`.
318
+ default : int
319
+ Baseline integer that acts as the lower bound.
320
+
321
+ Returns
322
+ -------
323
+ int
324
+ ``default`` if coercion fails; else ``max(coerced, default)``.
325
+ """
326
+ result = to_int(value, default)
327
+ return max(_value_or_default(result, default), default)
328
+
329
+
330
+ def to_minimum_int(
331
+ value: Any,
332
+ default: int,
333
+ ) -> int:
334
+ """
335
+ Return the lesser of ``default`` and ``value`` after integer coercion.
336
+
337
+ Parameters
338
+ ----------
339
+ value : Any
340
+ Candidate input coerced with :func:`to_int`.
341
+ default : int
342
+ Baseline integer acting as the upper bound.
343
+
344
+ Returns
345
+ -------
346
+ int
347
+ ``default`` if coercion fails; else ``min(coerced, default)``.
348
+ """
349
+ result = to_int(value, default)
350
+ return min(_value_or_default(result, default), default)
351
+
352
+
353
+ def to_positive_int(
354
+ value: Any,
355
+ default: int,
356
+ *,
357
+ minimum: int = 1,
358
+ ) -> int:
359
+ """
360
+ Return a positive integer, falling back to ``minimum`` when needed.
361
+
362
+ Parameters
363
+ ----------
364
+ value : Any
365
+ Candidate input coerced with :func:`to_int`.
366
+ default : int
367
+ Fallback value when coercion fails; clamped by ``minimum``.
368
+ minimum : int
369
+ Inclusive lower bound for the result. Defaults to ``1``.
370
+
371
+ Returns
372
+ -------
373
+ int
374
+ Positive integer respecting ``minimum``.
375
+ """
376
+ result = to_int(value, default, minimum=minimum)
377
+ return _value_or_default(result, minimum)
378
+
379
+
380
+ # -- Generic Number Coercion -- #
381
+
382
+
383
+ def to_number(
384
+ value: object,
385
+ ) -> float | None:
386
+ """
387
+ Coerce ``value`` to a ``float`` using the internal float coercer.
388
+
389
+ Parameters
390
+ ----------
391
+ value : object
392
+ Value that may be numeric or a numeric string. Booleans and blanks
393
+ return ``None`` for consistency with :func:`to_float`.
394
+
395
+ Returns
396
+ -------
397
+ float | None
398
+ ``float(value)`` if coercion succeeds; else ``None``.
399
+ """
400
+ return _coerce_float(value)
401
+
402
+
403
+ # -- Text Processing -- #
404
+
405
+
406
+ def normalized_str(
407
+ value: str | None,
408
+ ) -> str:
409
+ """
410
+ Return lower-cased, trimmed text for normalization helpers.
411
+
412
+ Parameters
413
+ ----------
414
+ value : str | None
415
+ Optional user-provided text.
416
+
417
+ Returns
418
+ -------
419
+ str
420
+ Normalized string with surrounding whitespace removed and converted
421
+ to lowercase. ``""`` when *value* is ``None``.
422
+ """
423
+ return (value or '').strip().lower()
424
+
425
+
426
+ # SECTION: INTERNAL FUNCTIONS =============================================== #
427
+
428
+
429
+ def _clamp(
430
+ value: Num,
431
+ minimum: Num | None,
432
+ maximum: Num | None,
433
+ ) -> Num:
434
+ """
435
+ Return ``value`` constrained to the interval ``[minimum, maximum]``.
436
+
437
+ Parameters
438
+ ----------
439
+ value : Num
440
+ Value to clamp.
441
+ minimum : Num | None
442
+ Minimum allowed value.
443
+ maximum : Num | None
444
+ Maximum allowed value.
445
+
446
+ Returns
447
+ -------
448
+ Num
449
+ Clamped value.
450
+ """
451
+ minimum, maximum = _validate_bounds(minimum, maximum)
452
+ if minimum is not None:
453
+ value = max(value, minimum)
454
+ if maximum is not None:
455
+ value = min(value, maximum)
456
+ return value
457
+
458
+
459
+ def _coerce_float(
460
+ value: object,
461
+ ) -> float | None:
462
+ """
463
+ Best-effort float coercion that ignores booleans and blanks.
464
+
465
+ Parameters
466
+ ----------
467
+ value : object
468
+ Value to coerce.
469
+
470
+ Returns
471
+ -------
472
+ float | None
473
+ Coerced float or ``None`` when coercion fails.
474
+ """
475
+ match value:
476
+ case None | bool():
477
+ return None
478
+ case float():
479
+ return value
480
+ case int():
481
+ return float(value)
482
+ case str():
483
+ text = value.strip()
484
+ if not text:
485
+ return None
486
+ try:
487
+ return float(text)
488
+ except ValueError:
489
+ return None
490
+ case _:
491
+ try:
492
+ return float(value) # type: ignore[arg-type]
493
+ except (TypeError, ValueError):
494
+ return None
495
+
496
+
497
+ def _coerce_int(
498
+ value: object,
499
+ ) -> int | None:
500
+ """
501
+ Best-effort integer coercion allowing floats only when integral.
502
+
503
+ Parameters
504
+ ----------
505
+ value : object
506
+ Value to coerce.
507
+
508
+ Returns
509
+ -------
510
+ int | None
511
+ Coerced integer or ``None`` when coercion fails.
512
+ """
513
+ match value:
514
+ case None | bool():
515
+ return None
516
+ case int():
517
+ return value
518
+ case float() if value.is_integer():
519
+ return int(value)
520
+ case str():
521
+ text = value.strip()
522
+ if not text:
523
+ return None
524
+ try:
525
+ return int(text)
526
+ except ValueError:
527
+ return _integral_from_float(_coerce_float(text))
528
+ case _:
529
+ return _integral_from_float(_coerce_float(value))
530
+
531
+
532
+ def _integral_from_float(
533
+ candidate: float | None,
534
+ ) -> int | None:
535
+ """
536
+ Return ``int(candidate)`` when ``candidate`` is integral.
537
+
538
+ Parameters
539
+ ----------
540
+ candidate : float | None
541
+ Float to convert when representing a whole number.
542
+
543
+ Returns
544
+ -------
545
+ int | None
546
+ Integer form of ``candidate``; else ``None`` if not integral.
547
+ """
548
+ if candidate is None or not candidate.is_integer():
549
+ return None
550
+ return int(candidate)
551
+
552
+
553
+ def _normalize_number(
554
+ coercer: Callable[[object], Num | None],
555
+ value: object,
556
+ *,
557
+ default: Num | None = None,
558
+ minimum: Num | None = None,
559
+ maximum: Num | None = None,
560
+ ) -> Num | None:
561
+ """
562
+ Coerce ``value`` with ``coercer`` and optionally clamp it.
563
+
564
+ Parameters
565
+ ----------
566
+ coercer : Callable[[object], Num | None]
567
+ Function that attempts coercion.
568
+ value : object
569
+ Value to normalize.
570
+ default : Num | None, optional
571
+ Fallback returned when coercion fails. Defaults to ``None``.
572
+ minimum : Num | None, optional
573
+ Lower bound, inclusive.
574
+ maximum : Num | None, optional
575
+ Upper bound, inclusive.
576
+
577
+ Returns
578
+ -------
579
+ Num | None
580
+ Normalized value or ``None`` when coercion fails.
581
+ """
582
+ result = coercer(value)
583
+ if result is None:
584
+ result = default
585
+ if result is None:
586
+ return None
587
+ return _clamp(result, minimum, maximum)
588
+
589
+
590
+ def _validate_bounds(
591
+ minimum: Num | None,
592
+ maximum: Num | None,
593
+ ) -> tuple[Num | None, Num | None]:
594
+ """
595
+ Ensure ``minimum`` does not exceed ``maximum``.
596
+
597
+ Parameters
598
+ ----------
599
+ minimum : Num | None
600
+ Candidate lower bound.
601
+ maximum : Num | None
602
+ Candidate upper bound.
603
+
604
+ Returns
605
+ -------
606
+ tuple[Num | None, Num | None]
607
+ Normalized ``(minimum, maximum)`` pair.
608
+
609
+ Raises
610
+ ------
611
+ ValueError
612
+ If both bounds are provided and ``minimum > maximum``.
613
+ """
614
+ if minimum is not None and maximum is not None and minimum > maximum:
615
+ raise ValueError('minimum cannot exceed maximum')
616
+ return minimum, maximum
617
+
618
+
619
+ def _value_or_default(
620
+ value: Num | None,
621
+ default: Num,
622
+ ) -> Num:
623
+ """
624
+ Return ``value`` if not ``None``; else ``default``.
625
+
626
+ Parameters
627
+ ----------
628
+ value : Num | None
629
+ Candidate value.
630
+ default : Num
631
+ Fallback value.
632
+
633
+ Returns
634
+ -------
635
+ Num
636
+ ``value`` or ``default``.
637
+ """
638
+ return default if value is None else value