etlplus 0.5.2__py3-none-any.whl → 0.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/api/README.md +24 -26
- etlplus/cli/commands.py +924 -0
- etlplus/cli/constants.py +71 -0
- etlplus/cli/handlers.py +369 -484
- etlplus/cli/io.py +336 -0
- etlplus/cli/main.py +16 -418
- etlplus/cli/options.py +49 -0
- etlplus/cli/state.py +336 -0
- etlplus/cli/types.py +33 -0
- etlplus/database/__init__.py +44 -0
- etlplus/database/ddl.py +319 -0
- etlplus/database/engine.py +151 -0
- etlplus/database/orm.py +354 -0
- etlplus/database/schema.py +274 -0
- etlplus/database/types.py +33 -0
- etlplus/enums.py +51 -1
- etlplus/load.py +1 -1
- etlplus/run.py +2 -4
- etlplus/types.py +5 -0
- etlplus/utils.py +1 -32
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/METADATA +84 -40
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/RECORD +26 -16
- etlplus/cli/app.py +0 -1367
- etlplus/ddl.py +0 -197
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/WHEEL +0 -0
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/entry_points.txt +0 -0
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/top_level.txt +0 -0
etlplus/cli/state.py
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.cli.state` module.
|
|
3
|
+
|
|
4
|
+
Shared state and helper utilities for the ``etlplus`` command-line interface
|
|
5
|
+
(CLI).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
from collections.abc import Collection
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Final
|
|
15
|
+
|
|
16
|
+
import typer
|
|
17
|
+
|
|
18
|
+
from .constants import DATA_CONNECTORS
|
|
19
|
+
|
|
20
|
+
# SECTION: EXPORTS ========================================================== #
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
# Classes
|
|
24
|
+
'CliState',
|
|
25
|
+
# Functions
|
|
26
|
+
'ensure_state',
|
|
27
|
+
'infer_resource_type',
|
|
28
|
+
'infer_resource_type_or_exit',
|
|
29
|
+
'infer_resource_type_soft',
|
|
30
|
+
'log_inferred_resource',
|
|
31
|
+
'optional_choice',
|
|
32
|
+
'resolve_resource_type',
|
|
33
|
+
'validate_choice',
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# SECTION: INTERNAL CONSTANTS =============================================== #
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_DB_SCHEMES: Final[tuple[str, ...]] = (
|
|
41
|
+
'postgres://',
|
|
42
|
+
'postgresql://',
|
|
43
|
+
'mysql://',
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# SECTION: DATA CLASSES ===================================================== #
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(slots=True)
|
|
51
|
+
class CliState:
|
|
52
|
+
"""
|
|
53
|
+
Mutable container for runtime CLI toggles.
|
|
54
|
+
|
|
55
|
+
Attributes
|
|
56
|
+
----------
|
|
57
|
+
pretty : bool
|
|
58
|
+
Whether to pretty-print output.
|
|
59
|
+
quiet : bool
|
|
60
|
+
Whether to suppress non-error output.
|
|
61
|
+
verbose : bool
|
|
62
|
+
Whether to enable verbose logging.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
pretty: bool = True
|
|
66
|
+
quiet: bool = False
|
|
67
|
+
verbose: bool = False
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def ensure_state(
|
|
74
|
+
ctx: typer.Context,
|
|
75
|
+
) -> CliState:
|
|
76
|
+
"""
|
|
77
|
+
Return the :class:`CliState` stored on the :mod:`typer` context.
|
|
78
|
+
|
|
79
|
+
Initializes a new :class:`CliState` if none exists.
|
|
80
|
+
|
|
81
|
+
Parameters
|
|
82
|
+
----------
|
|
83
|
+
ctx : typer.Context
|
|
84
|
+
The Typer command context.
|
|
85
|
+
|
|
86
|
+
Returns
|
|
87
|
+
-------
|
|
88
|
+
CliState
|
|
89
|
+
The CLI state object.
|
|
90
|
+
"""
|
|
91
|
+
if not isinstance(getattr(ctx, 'obj', None), CliState):
|
|
92
|
+
ctx.obj = CliState()
|
|
93
|
+
return ctx.obj
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def infer_resource_type(
|
|
97
|
+
value: str,
|
|
98
|
+
) -> str:
|
|
99
|
+
"""
|
|
100
|
+
Infer the resource type from a path, URL, or DSN string.
|
|
101
|
+
|
|
102
|
+
Parameters
|
|
103
|
+
----------
|
|
104
|
+
value : str
|
|
105
|
+
The resource identifier (path, URL, or DSN).
|
|
106
|
+
|
|
107
|
+
Returns
|
|
108
|
+
-------
|
|
109
|
+
str
|
|
110
|
+
The inferred resource type: ``file``, ``api``, or ``database``.
|
|
111
|
+
|
|
112
|
+
Raises
|
|
113
|
+
------
|
|
114
|
+
ValueError
|
|
115
|
+
If inference fails.
|
|
116
|
+
"""
|
|
117
|
+
val = (value or '').strip()
|
|
118
|
+
low = val.lower()
|
|
119
|
+
|
|
120
|
+
match (val, low):
|
|
121
|
+
case ('-', _):
|
|
122
|
+
return 'file'
|
|
123
|
+
case (_, inferred) if inferred.startswith(('http://', 'https://')):
|
|
124
|
+
return 'api'
|
|
125
|
+
case (_, inferred) if inferred.startswith(_DB_SCHEMES):
|
|
126
|
+
return 'database'
|
|
127
|
+
|
|
128
|
+
path = Path(val)
|
|
129
|
+
if path.exists() or path.suffix:
|
|
130
|
+
return 'file'
|
|
131
|
+
|
|
132
|
+
raise ValueError(
|
|
133
|
+
'Could not infer resource type. '
|
|
134
|
+
'Use --source-type/--target-type to specify it.',
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def infer_resource_type_or_exit(
|
|
139
|
+
value: str,
|
|
140
|
+
) -> str:
|
|
141
|
+
"""
|
|
142
|
+
Infer a resource type and map ``ValueError`` to ``BadParameter``.
|
|
143
|
+
|
|
144
|
+
Parameters
|
|
145
|
+
----------
|
|
146
|
+
value : str
|
|
147
|
+
The resource identifier (path, URL, or DSN).
|
|
148
|
+
|
|
149
|
+
Returns
|
|
150
|
+
-------
|
|
151
|
+
str
|
|
152
|
+
The inferred resource type: ``file``, ``api``, or ``database``.
|
|
153
|
+
|
|
154
|
+
Raises
|
|
155
|
+
------
|
|
156
|
+
typer.BadParameter
|
|
157
|
+
If inference fails.
|
|
158
|
+
"""
|
|
159
|
+
try:
|
|
160
|
+
return infer_resource_type(value)
|
|
161
|
+
except ValueError as exc: # pragma: no cover - exercised indirectly
|
|
162
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def infer_resource_type_soft(
|
|
166
|
+
value: str | None,
|
|
167
|
+
) -> str | None:
|
|
168
|
+
"""
|
|
169
|
+
Make a best-effort inference that tolerates inline payloads.
|
|
170
|
+
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
value : str | None
|
|
174
|
+
The resource identifier (path, URL, DSN, or inline payload).
|
|
175
|
+
|
|
176
|
+
Returns
|
|
177
|
+
-------
|
|
178
|
+
str | None
|
|
179
|
+
The inferred resource type, or ``None`` if inference failed.
|
|
180
|
+
"""
|
|
181
|
+
if value is None:
|
|
182
|
+
return None
|
|
183
|
+
try:
|
|
184
|
+
return infer_resource_type(value)
|
|
185
|
+
except ValueError:
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def log_inferred_resource(
|
|
190
|
+
state: CliState,
|
|
191
|
+
*,
|
|
192
|
+
role: str,
|
|
193
|
+
value: str,
|
|
194
|
+
resource_type: str | None,
|
|
195
|
+
) -> None:
|
|
196
|
+
"""
|
|
197
|
+
Emit a uniform verbose message for inferred resource types.
|
|
198
|
+
|
|
199
|
+
Parameters
|
|
200
|
+
----------
|
|
201
|
+
state : CliState
|
|
202
|
+
The current CLI state.
|
|
203
|
+
role : str
|
|
204
|
+
The resource role, e.g., ``source`` or ``target``.
|
|
205
|
+
value : str
|
|
206
|
+
The resource identifier (path, URL, or DSN).
|
|
207
|
+
resource_type : str | None
|
|
208
|
+
The inferred resource type, or ``None`` if inference failed.
|
|
209
|
+
"""
|
|
210
|
+
if state.quiet or not state.verbose or resource_type is None:
|
|
211
|
+
return
|
|
212
|
+
print(
|
|
213
|
+
f'Inferred {role}_type={resource_type} for {role}={value}',
|
|
214
|
+
file=sys.stderr,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def optional_choice(
|
|
219
|
+
value: str | None,
|
|
220
|
+
choices: Collection[str],
|
|
221
|
+
*,
|
|
222
|
+
label: str,
|
|
223
|
+
) -> str | None:
|
|
224
|
+
"""
|
|
225
|
+
Validate optional CLI choice inputs while preserving ``None``.
|
|
226
|
+
|
|
227
|
+
Parameters
|
|
228
|
+
----------
|
|
229
|
+
value : str | None
|
|
230
|
+
The input value to validate, or ``None``.
|
|
231
|
+
choices : Collection[str]
|
|
232
|
+
The set of valid choices.
|
|
233
|
+
label : str
|
|
234
|
+
The label for error messages.
|
|
235
|
+
|
|
236
|
+
Returns
|
|
237
|
+
-------
|
|
238
|
+
str | None
|
|
239
|
+
The validated choice, or ``None`` if input was ``None``.
|
|
240
|
+
"""
|
|
241
|
+
if value is None:
|
|
242
|
+
return None
|
|
243
|
+
return validate_choice(value, choices, label=label)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def resolve_resource_type(
|
|
247
|
+
*,
|
|
248
|
+
explicit_type: str | None,
|
|
249
|
+
override_type: str | None,
|
|
250
|
+
value: str,
|
|
251
|
+
label: str,
|
|
252
|
+
conflict_error: str | None = None,
|
|
253
|
+
legacy_file_error: str | None = None,
|
|
254
|
+
) -> str:
|
|
255
|
+
"""
|
|
256
|
+
Resolve resource type preference order and validate it.
|
|
257
|
+
|
|
258
|
+
Parameters
|
|
259
|
+
----------
|
|
260
|
+
explicit_type : str | None
|
|
261
|
+
The explicit resource type from the CLI, or ``None`` if not provided.
|
|
262
|
+
override_type : str | None
|
|
263
|
+
The override resource type from the CLI, or ``None`` if not provided.
|
|
264
|
+
value : str
|
|
265
|
+
The resource identifier (path, URL, or DSN).
|
|
266
|
+
label : str
|
|
267
|
+
The label for error messages.
|
|
268
|
+
conflict_error : str | None, optional
|
|
269
|
+
The error message to raise if both explicit and override types are
|
|
270
|
+
provided, by default ``None``.
|
|
271
|
+
legacy_file_error : str | None, optional
|
|
272
|
+
The error message to raise if the explicit type is ``file``, by default
|
|
273
|
+
``None``.
|
|
274
|
+
|
|
275
|
+
Returns
|
|
276
|
+
-------
|
|
277
|
+
str
|
|
278
|
+
The resolved and validated resource type.
|
|
279
|
+
|
|
280
|
+
Raises
|
|
281
|
+
------
|
|
282
|
+
typer.BadParameter
|
|
283
|
+
If there is a conflict between explicit and override types, or if the
|
|
284
|
+
explicit type is ``file`` when disallowed.
|
|
285
|
+
"""
|
|
286
|
+
if explicit_type is not None:
|
|
287
|
+
if override_type is not None and conflict_error:
|
|
288
|
+
raise typer.BadParameter(conflict_error)
|
|
289
|
+
if legacy_file_error and explicit_type.strip().lower() == 'file':
|
|
290
|
+
raise typer.BadParameter(legacy_file_error)
|
|
291
|
+
candidate = explicit_type
|
|
292
|
+
else:
|
|
293
|
+
candidate = override_type or infer_resource_type_or_exit(value)
|
|
294
|
+
return validate_choice(candidate, DATA_CONNECTORS, label=label)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def validate_choice(
|
|
298
|
+
value: str | object,
|
|
299
|
+
choices: Collection[str],
|
|
300
|
+
*,
|
|
301
|
+
label: str,
|
|
302
|
+
) -> str:
|
|
303
|
+
"""
|
|
304
|
+
Validate CLI input against a whitelist of choices.
|
|
305
|
+
|
|
306
|
+
Parameters
|
|
307
|
+
----------
|
|
308
|
+
value : str | object
|
|
309
|
+
The input value to validate.
|
|
310
|
+
choices : Collection[str]
|
|
311
|
+
The set of valid choices.
|
|
312
|
+
label : str
|
|
313
|
+
The label for error messages.
|
|
314
|
+
|
|
315
|
+
Returns
|
|
316
|
+
-------
|
|
317
|
+
str
|
|
318
|
+
The validated choice.
|
|
319
|
+
|
|
320
|
+
Raises
|
|
321
|
+
------
|
|
322
|
+
typer.BadParameter
|
|
323
|
+
If the input value is not in the set of valid choices.
|
|
324
|
+
"""
|
|
325
|
+
v = str(value or '').strip().lower()
|
|
326
|
+
normalized_choices = {c.lower() for c in choices}
|
|
327
|
+
if v in normalized_choices:
|
|
328
|
+
# Preserve original casing from choices when possible for messages
|
|
329
|
+
for choice in choices:
|
|
330
|
+
if choice.lower() == v:
|
|
331
|
+
return choice
|
|
332
|
+
return v
|
|
333
|
+
allowed = ', '.join(sorted(choices))
|
|
334
|
+
raise typer.BadParameter(
|
|
335
|
+
f"Invalid {label} '{value}'. Choose from: {allowed}",
|
|
336
|
+
)
|
etlplus/cli/types.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.cli.types` module.
|
|
3
|
+
|
|
4
|
+
Type aliases for :mod:`etlplus.cli` helpers.
|
|
5
|
+
|
|
6
|
+
Notes
|
|
7
|
+
-----
|
|
8
|
+
- Keeps other modules decoupled from ``typing`` details.
|
|
9
|
+
|
|
10
|
+
Examples
|
|
11
|
+
--------
|
|
12
|
+
>>> from etlplus.cli.types import DataConnectorContext
|
|
13
|
+
>>> connector: DataConnectorContext = 'source'
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import Literal
|
|
19
|
+
|
|
20
|
+
# SECTION: EXPORTS ========================================================== #
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
# Type Aliases
|
|
25
|
+
'DataConnectorContext',
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# SECTION: TYPE ALIASES ===================================================== #
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# Data connector context.
|
|
33
|
+
type DataConnectorContext = Literal['source', 'target']
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.database` package.
|
|
3
|
+
|
|
4
|
+
Database utilities for:
|
|
5
|
+
- DDL rendering and schema management.
|
|
6
|
+
- Schema parsing from configuration files.
|
|
7
|
+
- Dynamic ORM generation.
|
|
8
|
+
- Database engine/session management.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from .ddl import load_table_spec
|
|
14
|
+
from .ddl import render_table_sql
|
|
15
|
+
from .ddl import render_tables
|
|
16
|
+
from .ddl import render_tables_to_string
|
|
17
|
+
from .engine import engine
|
|
18
|
+
from .engine import load_database_url_from_config
|
|
19
|
+
from .engine import make_engine
|
|
20
|
+
from .engine import session
|
|
21
|
+
from .orm import Base
|
|
22
|
+
from .orm import build_models
|
|
23
|
+
from .orm import load_and_build_models
|
|
24
|
+
from .schema import load_table_specs
|
|
25
|
+
|
|
26
|
+
# SECTION: EXPORTS ========================================================== #
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
# Functions
|
|
31
|
+
'build_models',
|
|
32
|
+
'load_and_build_models',
|
|
33
|
+
'load_database_url_from_config',
|
|
34
|
+
'load_table_spec',
|
|
35
|
+
'load_table_specs',
|
|
36
|
+
'make_engine',
|
|
37
|
+
'render_table_sql',
|
|
38
|
+
'render_tables',
|
|
39
|
+
'render_tables_to_string',
|
|
40
|
+
'Base',
|
|
41
|
+
# Singletons
|
|
42
|
+
'engine',
|
|
43
|
+
'session',
|
|
44
|
+
]
|