etlplus 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. etlplus/README.md +37 -0
  2. etlplus/__init__.py +1 -26
  3. etlplus/api/README.md +51 -3
  4. etlplus/api/__init__.py +10 -0
  5. etlplus/api/config.py +39 -28
  6. etlplus/api/endpoint_client.py +3 -3
  7. etlplus/api/enums.py +51 -0
  8. etlplus/api/pagination/client.py +1 -1
  9. etlplus/api/rate_limiting/config.py +13 -1
  10. etlplus/api/rate_limiting/rate_limiter.py +8 -11
  11. etlplus/api/request_manager.py +11 -6
  12. etlplus/api/transport.py +14 -2
  13. etlplus/api/types.py +96 -6
  14. etlplus/{run_helpers.py → api/utils.py} +209 -153
  15. etlplus/cli/README.md +40 -0
  16. etlplus/cli/commands.py +94 -61
  17. etlplus/cli/constants.py +1 -1
  18. etlplus/cli/handlers.py +40 -12
  19. etlplus/cli/io.py +2 -2
  20. etlplus/cli/main.py +1 -1
  21. etlplus/cli/state.py +4 -7
  22. etlplus/database/README.md +48 -0
  23. etlplus/database/ddl.py +1 -1
  24. etlplus/database/engine.py +19 -3
  25. etlplus/database/orm.py +2 -0
  26. etlplus/database/schema.py +1 -1
  27. etlplus/enums.py +1 -107
  28. etlplus/file/README.md +105 -0
  29. etlplus/file/__init__.py +25 -0
  30. etlplus/file/_imports.py +141 -0
  31. etlplus/file/_io.py +160 -0
  32. etlplus/file/accdb.py +78 -0
  33. etlplus/file/arrow.py +78 -0
  34. etlplus/file/avro.py +176 -0
  35. etlplus/file/bson.py +77 -0
  36. etlplus/file/cbor.py +78 -0
  37. etlplus/file/cfg.py +79 -0
  38. etlplus/file/conf.py +80 -0
  39. etlplus/file/core.py +322 -0
  40. etlplus/file/csv.py +79 -0
  41. etlplus/file/dat.py +78 -0
  42. etlplus/file/dta.py +77 -0
  43. etlplus/file/duckdb.py +78 -0
  44. etlplus/file/enums.py +343 -0
  45. etlplus/file/feather.py +111 -0
  46. etlplus/file/fwf.py +77 -0
  47. etlplus/file/gz.py +123 -0
  48. etlplus/file/hbs.py +78 -0
  49. etlplus/file/hdf5.py +78 -0
  50. etlplus/file/ini.py +79 -0
  51. etlplus/file/ion.py +78 -0
  52. etlplus/file/jinja2.py +78 -0
  53. etlplus/file/json.py +98 -0
  54. etlplus/file/log.py +78 -0
  55. etlplus/file/mat.py +78 -0
  56. etlplus/file/mdb.py +78 -0
  57. etlplus/file/msgpack.py +78 -0
  58. etlplus/file/mustache.py +78 -0
  59. etlplus/file/nc.py +78 -0
  60. etlplus/file/ndjson.py +108 -0
  61. etlplus/file/numbers.py +75 -0
  62. etlplus/file/ods.py +79 -0
  63. etlplus/file/orc.py +111 -0
  64. etlplus/file/parquet.py +113 -0
  65. etlplus/file/pb.py +78 -0
  66. etlplus/file/pbf.py +77 -0
  67. etlplus/file/properties.py +78 -0
  68. etlplus/file/proto.py +77 -0
  69. etlplus/file/psv.py +79 -0
  70. etlplus/file/rda.py +78 -0
  71. etlplus/file/rds.py +78 -0
  72. etlplus/file/sas7bdat.py +78 -0
  73. etlplus/file/sav.py +77 -0
  74. etlplus/file/sqlite.py +78 -0
  75. etlplus/file/stub.py +84 -0
  76. etlplus/file/sylk.py +77 -0
  77. etlplus/file/tab.py +81 -0
  78. etlplus/file/toml.py +78 -0
  79. etlplus/file/tsv.py +80 -0
  80. etlplus/file/txt.py +102 -0
  81. etlplus/file/vm.py +78 -0
  82. etlplus/file/wks.py +77 -0
  83. etlplus/file/xls.py +88 -0
  84. etlplus/file/xlsm.py +79 -0
  85. etlplus/file/xlsx.py +99 -0
  86. etlplus/file/xml.py +185 -0
  87. etlplus/file/xpt.py +78 -0
  88. etlplus/file/yaml.py +95 -0
  89. etlplus/file/zip.py +175 -0
  90. etlplus/file/zsav.py +77 -0
  91. etlplus/ops/README.md +50 -0
  92. etlplus/ops/__init__.py +61 -0
  93. etlplus/{extract.py → ops/extract.py} +81 -99
  94. etlplus/{load.py → ops/load.py} +78 -101
  95. etlplus/{run.py → ops/run.py} +159 -127
  96. etlplus/{transform.py → ops/transform.py} +75 -68
  97. etlplus/{validation → ops}/utils.py +53 -17
  98. etlplus/{validate.py → ops/validate.py} +22 -12
  99. etlplus/templates/README.md +46 -0
  100. etlplus/types.py +5 -4
  101. etlplus/utils.py +136 -2
  102. etlplus/workflow/README.md +52 -0
  103. etlplus/{config → workflow}/__init__.py +10 -23
  104. etlplus/{config → workflow}/connector.py +58 -44
  105. etlplus/workflow/dag.py +105 -0
  106. etlplus/{config → workflow}/jobs.py +105 -32
  107. etlplus/{config → workflow}/pipeline.py +59 -51
  108. etlplus/{config → workflow}/profile.py +8 -5
  109. etlplus/workflow/types.py +115 -0
  110. {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/METADATA +210 -17
  111. etlplus-0.9.2.dist-info/RECORD +134 -0
  112. {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/WHEEL +1 -1
  113. etlplus/config/types.py +0 -204
  114. etlplus/config/utils.py +0 -120
  115. etlplus/file.py +0 -657
  116. etlplus/validation/__init__.py +0 -44
  117. etlplus-0.9.0.dist-info/RECORD +0 -65
  118. {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/entry_points.txt +0 -0
  119. {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/licenses/LICENSE +0 -0
  120. {etlplus-0.9.0.dist-info → etlplus-0.9.2.dist-info}/top_level.txt +0 -0
etlplus/api/types.py CHANGED
@@ -20,8 +20,11 @@ Examples
20
20
  from __future__ import annotations
21
21
 
22
22
  from collections.abc import Callable
23
+ from collections.abc import Mapping
23
24
  from dataclasses import dataclass
24
25
  from typing import Any
26
+ from typing import Self
27
+ from typing import TypedDict
25
28
  from typing import cast
26
29
 
27
30
  from ..types import JSONData
@@ -39,6 +42,11 @@ __all__ = [
39
42
  'Headers',
40
43
  'Params',
41
44
  'Url',
45
+ # Typed Dicts
46
+ 'ApiConfigMap',
47
+ 'ApiProfileConfigMap',
48
+ 'ApiProfileDefaultsMap',
49
+ 'EndpointMap',
42
50
  ]
43
51
 
44
52
 
@@ -48,6 +56,88 @@ __all__ = [
48
56
  _UNSET = object()
49
57
 
50
58
 
59
+ # SECTION: TYPED DICTS ====================================================== #
60
+
61
+
62
+ class ApiConfigMap(TypedDict, total=False):
63
+ """
64
+ Top-level API config shape parsed by ApiConfig.from_obj.
65
+
66
+ Either provide a ``base_url`` with optional ``headers`` and ``endpoints``,
67
+ or provide ``profiles`` with at least one profile having a ``base_url``.
68
+
69
+ See Also
70
+ --------
71
+ - :class:`etlplus.api.config.ApiConfig`
72
+ """
73
+
74
+ base_url: str
75
+ headers: StrAnyMap
76
+ endpoints: Mapping[str, EndpointMap | str]
77
+ profiles: Mapping[str, ApiProfileConfigMap]
78
+
79
+
80
+ class ApiProfileConfigMap(TypedDict, total=False):
81
+ """
82
+ Shape accepted for a profile entry under ApiConfigMap.profiles.
83
+
84
+ Notes
85
+ -----
86
+ ``base_url`` is required at runtime when profiles are provided.
87
+
88
+ See Also
89
+ --------
90
+ - :class:`etlplus.api.config.ApiProfileConfig`
91
+ """
92
+
93
+ base_url: str
94
+ headers: StrAnyMap
95
+ base_path: str
96
+ auth: StrAnyMap
97
+ defaults: ApiProfileDefaultsMap
98
+
99
+
100
+ class ApiProfileDefaultsMap(TypedDict, total=False):
101
+ """
102
+ Defaults block available under a profile (all keys optional).
103
+
104
+ Notes
105
+ -----
106
+ Runtime expects header values to be str; typing remains permissive.
107
+
108
+ See Also
109
+ --------
110
+ - :class:`etlplus.api.config.ApiProfileConfig`
111
+ - :class:`etlplus.api.pagination.PaginationConfig`
112
+ - :class:`etlplus.api.rate_limiting.RateLimitConfig`
113
+ """
114
+
115
+ headers: StrAnyMap
116
+ pagination: Any
117
+ rate_limit: Any
118
+
119
+
120
+ class EndpointMap(TypedDict, total=False):
121
+ """
122
+ Shape accepted by EndpointConfig.from_obj.
123
+
124
+ One of ``path`` or ``url`` should be provided.
125
+
126
+ See Also
127
+ --------
128
+ - :class:`etlplus.api.config.EndpointConfig`
129
+ """
130
+
131
+ path: str
132
+ url: str
133
+ method: str
134
+ path_params: StrAnyMap
135
+ query_params: StrAnyMap
136
+ body: Any
137
+ pagination: Any
138
+ rate_limit: Any
139
+
140
+
51
141
  # SECTION: DATA CLASSES ===================================================== #
52
142
 
53
143
 
@@ -75,9 +165,9 @@ class RequestOptions:
75
165
  # -- Magic Methods (Object Lifecycle) -- #
76
166
 
77
167
  def __post_init__(self) -> None:
78
- if self.params:
168
+ if self.params is not None:
79
169
  object.__setattr__(self, 'params', dict(self.params))
80
- if self.headers:
170
+ if self.headers is not None:
81
171
  object.__setattr__(self, 'headers', dict(self.headers))
82
172
 
83
173
  # -- Instance Methods -- #
@@ -92,9 +182,9 @@ class RequestOptions:
92
182
  Keyword arguments for ``requests`` methods.
93
183
  """
94
184
  kw: dict[str, Any] = {}
95
- if self.params:
185
+ if self.params is not None:
96
186
  kw['params'] = dict(self.params)
97
- if self.headers:
187
+ if self.headers is not None:
98
188
  kw['headers'] = dict(self.headers)
99
189
  if self.timeout is not None:
100
190
  kw['timeout'] = self.timeout
@@ -106,7 +196,7 @@ class RequestOptions:
106
196
  params: Params | None | object = _UNSET,
107
197
  headers: Headers | None | object = _UNSET,
108
198
  timeout: float | None | object = _UNSET,
109
- ) -> RequestOptions:
199
+ ) -> Self:
110
200
  """
111
201
  Return a copy with the provided fields replaced.
112
202
 
@@ -146,7 +236,7 @@ class RequestOptions:
146
236
  else:
147
237
  next_timeout = cast(float | None, timeout)
148
238
 
149
- return RequestOptions(
239
+ return self.__class__(
150
240
  params=next_params,
151
241
  headers=next_headers,
152
242
  timeout=next_timeout,