etlplus 0.4.6__py3-none-any.whl → 0.4.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/cli/app.py +19 -3
- etlplus/cli/handlers.py +12 -0
- etlplus/cli/main.py +146 -28
- {etlplus-0.4.6.dist-info → etlplus-0.4.8.dist-info}/METADATA +14 -1
- {etlplus-0.4.6.dist-info → etlplus-0.4.8.dist-info}/RECORD +9 -9
- {etlplus-0.4.6.dist-info → etlplus-0.4.8.dist-info}/WHEEL +0 -0
- {etlplus-0.4.6.dist-info → etlplus-0.4.8.dist-info}/entry_points.txt +0 -0
- {etlplus-0.4.6.dist-info → etlplus-0.4.8.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.4.6.dist-info → etlplus-0.4.8.dist-info}/top_level.txt +0 -0
etlplus/cli/app.py
CHANGED
|
@@ -782,8 +782,21 @@ def list_cmd(
|
|
|
782
782
|
'--pipelines',
|
|
783
783
|
help='List ETL pipelines',
|
|
784
784
|
),
|
|
785
|
-
sources: bool = typer.Option(
|
|
786
|
-
|
|
785
|
+
sources: bool = typer.Option(
|
|
786
|
+
False,
|
|
787
|
+
'--sources',
|
|
788
|
+
help='List data sources',
|
|
789
|
+
),
|
|
790
|
+
summary: bool = typer.Option(
|
|
791
|
+
False,
|
|
792
|
+
'--summary',
|
|
793
|
+
help='Show pipeline summary (name, version, sources, targets, jobs)',
|
|
794
|
+
),
|
|
795
|
+
targets: bool = typer.Option(
|
|
796
|
+
False,
|
|
797
|
+
'--targets',
|
|
798
|
+
help='List data targets',
|
|
799
|
+
),
|
|
787
800
|
transforms: bool = typer.Option(
|
|
788
801
|
False,
|
|
789
802
|
'--transforms',
|
|
@@ -805,6 +818,8 @@ def list_cmd(
|
|
|
805
818
|
If True, list ETL pipelines.
|
|
806
819
|
sources : bool, optional
|
|
807
820
|
If True, list data sources.
|
|
821
|
+
summary : bool, optional
|
|
822
|
+
If True, show pipeline summary (name, version, sources, targets, jobs).
|
|
808
823
|
targets : bool, optional
|
|
809
824
|
If True, list data targets.
|
|
810
825
|
transforms : bool, optional
|
|
@@ -820,6 +835,7 @@ def list_cmd(
|
|
|
820
835
|
state,
|
|
821
836
|
command='list',
|
|
822
837
|
config=config,
|
|
838
|
+
summary=summary,
|
|
823
839
|
pipelines=pipelines,
|
|
824
840
|
jobs=jobs,
|
|
825
841
|
sources=sources,
|
|
@@ -953,7 +969,7 @@ def pipeline_cmd(
|
|
|
953
969
|
),
|
|
954
970
|
) -> int:
|
|
955
971
|
"""
|
|
956
|
-
|
|
972
|
+
Deprecated wrapper to inspect or run a pipeline YAML configuration.
|
|
957
973
|
|
|
958
974
|
Parameters
|
|
959
975
|
----------
|
etlplus/cli/handlers.py
CHANGED
|
@@ -117,6 +117,8 @@ def _list_sections(
|
|
|
117
117
|
Metadata output for the list command.
|
|
118
118
|
"""
|
|
119
119
|
sections: dict[str, Any] = {}
|
|
120
|
+
if getattr(args, 'jobs', False):
|
|
121
|
+
sections['jobs'] = _pipeline_summary(cfg)['jobs']
|
|
120
122
|
if getattr(args, 'pipelines', False):
|
|
121
123
|
sections['pipelines'] = [cfg.name]
|
|
122
124
|
if getattr(args, 'sources', False):
|
|
@@ -598,6 +600,12 @@ def cmd_pipeline(
|
|
|
598
600
|
int
|
|
599
601
|
Zero on success.
|
|
600
602
|
"""
|
|
603
|
+
print(
|
|
604
|
+
'DEPRECATED: use "etlplus list --summary|--jobs" or '
|
|
605
|
+
'"etlplus run --job/--pipeline" instead of "etlplus pipeline".',
|
|
606
|
+
file=sys.stderr,
|
|
607
|
+
)
|
|
608
|
+
|
|
601
609
|
cfg = load_pipeline_config(args.config, substitute=True)
|
|
602
610
|
|
|
603
611
|
list_flag = getattr(args, 'list', False) or getattr(args, 'jobs', False)
|
|
@@ -635,6 +643,10 @@ def cmd_list(args: argparse.Namespace) -> int:
|
|
|
635
643
|
Zero on success.
|
|
636
644
|
"""
|
|
637
645
|
cfg = load_pipeline_config(args.config, substitute=True)
|
|
646
|
+
if getattr(args, 'summary', False):
|
|
647
|
+
print_json(_pipeline_summary(cfg))
|
|
648
|
+
return 0
|
|
649
|
+
|
|
638
650
|
print_json(_list_sections(cfg, args))
|
|
639
651
|
return 0
|
|
640
652
|
|
etlplus/cli/main.py
CHANGED
|
@@ -10,10 +10,12 @@ This module exposes :func:`main` for the console script as well as
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import argparse
|
|
13
|
+
import contextlib
|
|
13
14
|
import sys
|
|
14
15
|
from collections.abc import Sequence
|
|
15
16
|
from typing import Literal
|
|
16
17
|
|
|
18
|
+
import click
|
|
17
19
|
import typer
|
|
18
20
|
|
|
19
21
|
from .. import __version__
|
|
@@ -68,6 +70,32 @@ class _FormatAction(argparse.Action):
|
|
|
68
70
|
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
69
71
|
|
|
70
72
|
|
|
73
|
+
def _add_boolean_flag(
|
|
74
|
+
parser: argparse.ArgumentParser,
|
|
75
|
+
*,
|
|
76
|
+
name: str,
|
|
77
|
+
help_text: str,
|
|
78
|
+
) -> None:
|
|
79
|
+
"""Add a toggle that also supports the ``--no-`` prefix via 3.13.
|
|
80
|
+
|
|
81
|
+
Parameters
|
|
82
|
+
----------
|
|
83
|
+
parser : argparse.ArgumentParser
|
|
84
|
+
Parser receiving the flag.
|
|
85
|
+
name : str
|
|
86
|
+
Primary flag name without leading dashes.
|
|
87
|
+
help_text : str
|
|
88
|
+
Help text rendered in ``--help`` output.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
parser.add_argument(
|
|
92
|
+
f'--{name}',
|
|
93
|
+
action=argparse.BooleanOptionalAction,
|
|
94
|
+
default=False,
|
|
95
|
+
help=help_text,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
71
99
|
def _add_config_option(
|
|
72
100
|
parser: argparse.ArgumentParser,
|
|
73
101
|
*,
|
|
@@ -129,32 +157,6 @@ def _add_format_options(
|
|
|
129
157
|
)
|
|
130
158
|
|
|
131
159
|
|
|
132
|
-
def _add_boolean_flag(
|
|
133
|
-
parser: argparse.ArgumentParser,
|
|
134
|
-
*,
|
|
135
|
-
name: str,
|
|
136
|
-
help_text: str,
|
|
137
|
-
) -> None:
|
|
138
|
-
"""Add a toggle that also supports the ``--no-`` prefix via 3.13.
|
|
139
|
-
|
|
140
|
-
Parameters
|
|
141
|
-
----------
|
|
142
|
-
parser : argparse.ArgumentParser
|
|
143
|
-
Parser receiving the flag.
|
|
144
|
-
name : str
|
|
145
|
-
Primary flag name without leading dashes.
|
|
146
|
-
help_text : str
|
|
147
|
-
Help text rendered in ``--help`` output.
|
|
148
|
-
"""
|
|
149
|
-
|
|
150
|
-
parser.add_argument(
|
|
151
|
-
f'--{name}',
|
|
152
|
-
action=argparse.BooleanOptionalAction,
|
|
153
|
-
default=False,
|
|
154
|
-
help=help_text,
|
|
155
|
-
)
|
|
156
|
-
|
|
157
|
-
|
|
158
160
|
def _cli_description() -> str:
|
|
159
161
|
return '\n'.join(
|
|
160
162
|
[
|
|
@@ -188,6 +190,93 @@ def _cli_epilog() -> str:
|
|
|
188
190
|
)
|
|
189
191
|
|
|
190
192
|
|
|
193
|
+
def _emit_context_help(
|
|
194
|
+
ctx: click.Context | None,
|
|
195
|
+
) -> bool:
|
|
196
|
+
"""
|
|
197
|
+
Mirror Click help output for the provided context onto stderr.
|
|
198
|
+
|
|
199
|
+
Parameters
|
|
200
|
+
----------
|
|
201
|
+
ctx : click.Context | None
|
|
202
|
+
The Click context to emit help for.
|
|
203
|
+
|
|
204
|
+
Returns
|
|
205
|
+
-------
|
|
206
|
+
bool
|
|
207
|
+
``True`` when help was emitted, ``False`` when ``ctx`` was ``None``.
|
|
208
|
+
"""
|
|
209
|
+
if ctx is None:
|
|
210
|
+
return False
|
|
211
|
+
|
|
212
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
213
|
+
ctx.get_help()
|
|
214
|
+
return True
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _emit_root_help(
|
|
218
|
+
command: click.Command,
|
|
219
|
+
) -> None:
|
|
220
|
+
"""
|
|
221
|
+
Print the root ``etlplus`` help text to stderr.
|
|
222
|
+
|
|
223
|
+
Parameters
|
|
224
|
+
----------
|
|
225
|
+
command : click.Command
|
|
226
|
+
The root Typer/Click command.
|
|
227
|
+
"""
|
|
228
|
+
ctx = command.make_context('etlplus', [], resilient_parsing=True)
|
|
229
|
+
try:
|
|
230
|
+
_emit_context_help(ctx)
|
|
231
|
+
finally:
|
|
232
|
+
ctx.close()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _is_illegal_option_error(
|
|
236
|
+
exc: click.exceptions.UsageError,
|
|
237
|
+
) -> bool:
|
|
238
|
+
"""
|
|
239
|
+
Return ``True`` when usage errors stem from invalid options.
|
|
240
|
+
|
|
241
|
+
Parameters
|
|
242
|
+
----------
|
|
243
|
+
exc : click.exceptions.UsageError
|
|
244
|
+
The usage error to inspect.
|
|
245
|
+
|
|
246
|
+
Returns
|
|
247
|
+
-------
|
|
248
|
+
bool
|
|
249
|
+
``True`` when the error indicates illegal options.
|
|
250
|
+
"""
|
|
251
|
+
return isinstance(
|
|
252
|
+
exc,
|
|
253
|
+
(
|
|
254
|
+
click.exceptions.BadOptionUsage,
|
|
255
|
+
click.exceptions.NoSuchOption,
|
|
256
|
+
),
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _is_unknown_command_error(
|
|
261
|
+
exc: click.exceptions.UsageError,
|
|
262
|
+
) -> bool:
|
|
263
|
+
"""
|
|
264
|
+
Return ``True`` when a :class:`UsageError` indicates bad subcommand.
|
|
265
|
+
|
|
266
|
+
Parameters
|
|
267
|
+
----------
|
|
268
|
+
exc : click.exceptions.UsageError
|
|
269
|
+
The usage error to inspect.
|
|
270
|
+
|
|
271
|
+
Returns
|
|
272
|
+
-------
|
|
273
|
+
bool
|
|
274
|
+
``True`` when the error indicates an unknown command.
|
|
275
|
+
"""
|
|
276
|
+
message = getattr(exc, 'message', None) or str(exc)
|
|
277
|
+
return message.startswith('No such command ')
|
|
278
|
+
|
|
279
|
+
|
|
191
280
|
# SECTION: FUNCTIONS ======================================================== #
|
|
192
281
|
|
|
193
282
|
|
|
@@ -333,8 +422,9 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
333
422
|
pipe_parser = subparsers.add_parser(
|
|
334
423
|
'pipeline',
|
|
335
424
|
help=(
|
|
336
|
-
'
|
|
337
|
-
|
|
425
|
+
'DEPRECATED: use "list" (for summary/jobs) or "run" (to execute); '
|
|
426
|
+
'see '
|
|
427
|
+
f'{PROJECT_URL}/blob/main/docs/pipeline-guide.md'
|
|
338
428
|
),
|
|
339
429
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
340
430
|
)
|
|
@@ -357,6 +447,11 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
357
447
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
358
448
|
)
|
|
359
449
|
_add_config_option(list_parser)
|
|
450
|
+
_add_boolean_flag(
|
|
451
|
+
list_parser,
|
|
452
|
+
name='jobs',
|
|
453
|
+
help_text='List ETL jobs',
|
|
454
|
+
)
|
|
360
455
|
_add_boolean_flag(
|
|
361
456
|
list_parser,
|
|
362
457
|
name='pipelines',
|
|
@@ -367,6 +462,13 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
367
462
|
name='sources',
|
|
368
463
|
help_text='List data sources',
|
|
369
464
|
)
|
|
465
|
+
_add_boolean_flag(
|
|
466
|
+
list_parser,
|
|
467
|
+
name='summary',
|
|
468
|
+
help_text=(
|
|
469
|
+
'Show pipeline summary (name, version, sources, targets, jobs)'
|
|
470
|
+
),
|
|
471
|
+
)
|
|
370
472
|
_add_boolean_flag(
|
|
371
473
|
list_parser,
|
|
372
474
|
name='targets',
|
|
@@ -422,6 +524,9 @@ def main(
|
|
|
422
524
|
|
|
423
525
|
Raises
|
|
424
526
|
------
|
|
527
|
+
click.exceptions.UsageError
|
|
528
|
+
Re-raises Typer/Click usage errors after printing help for unknown
|
|
529
|
+
commands.
|
|
425
530
|
SystemExit
|
|
426
531
|
Re-raises SystemExit exceptions to preserve exit codes.
|
|
427
532
|
|
|
@@ -442,6 +547,19 @@ def main(
|
|
|
442
547
|
)
|
|
443
548
|
return int(result or 0)
|
|
444
549
|
|
|
550
|
+
except click.exceptions.UsageError as exc:
|
|
551
|
+
if _is_unknown_command_error(exc):
|
|
552
|
+
typer.echo(f'Error: {exc}', err=True)
|
|
553
|
+
_emit_root_help(command)
|
|
554
|
+
return int(getattr(exc, 'exit_code', 2))
|
|
555
|
+
if _is_illegal_option_error(exc):
|
|
556
|
+
typer.echo(f'Error: {exc}', err=True)
|
|
557
|
+
if not _emit_context_help(exc.ctx):
|
|
558
|
+
_emit_root_help(command)
|
|
559
|
+
return int(getattr(exc, 'exit_code', 2))
|
|
560
|
+
|
|
561
|
+
raise
|
|
562
|
+
|
|
445
563
|
except typer.Exit as exc:
|
|
446
564
|
return int(exc.exit_code)
|
|
447
565
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.8
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -306,6 +306,19 @@ For YAML-driven pipelines executed end-to-end (extract → validate → transfor
|
|
|
306
306
|
- Authoring: [`docs/pipeline-guide.md`](docs/pipeline-guide.md)
|
|
307
307
|
- Runner API and internals: [`docs/run-module.md`](docs/run-module.md)
|
|
308
308
|
|
|
309
|
+
CLI quick reference for pipelines:
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
# List jobs or show a pipeline summary
|
|
313
|
+
etlplus list --config examples/configs/pipeline.yml --jobs
|
|
314
|
+
etlplus list --config examples/configs/pipeline.yml --summary
|
|
315
|
+
|
|
316
|
+
# Run a job
|
|
317
|
+
etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
318
|
+
|
|
319
|
+
# Deprecated shim (will be removed): etlplus pipeline
|
|
320
|
+
```
|
|
321
|
+
|
|
309
322
|
### Complete ETL Pipeline Example
|
|
310
323
|
|
|
311
324
|
```bash
|
|
@@ -31,9 +31,9 @@ etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-
|
|
|
31
31
|
etlplus/api/rate_limiting/config.py,sha256=2b4wIynblN-1EyMqI4aXa71SljzSjXYh5N1Nngr3jOg,9406
|
|
32
32
|
etlplus/api/rate_limiting/rate_limiter.py,sha256=Uxozqd_Ej5Lsj-M-mLT2WexChgWh7x35_YP10yqYPQA,7159
|
|
33
33
|
etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
|
|
34
|
-
etlplus/cli/app.py,sha256=
|
|
35
|
-
etlplus/cli/handlers.py,sha256=
|
|
36
|
-
etlplus/cli/main.py,sha256=
|
|
34
|
+
etlplus/cli/app.py,sha256=A8MxgKpJs9_KeyAgkopLrdJnB0SH24Mlxituh9Uwmrw,32991
|
|
35
|
+
etlplus/cli/handlers.py,sha256=mObms9k-3XoImECNW-CpOqQkJzlRtDlobU3sHMu4itM,16190
|
|
36
|
+
etlplus/cli/main.py,sha256=rl0NmZdzWwlGQSVyzWYutbBwHWOmBFYJcEjl6fYhwm0,15395
|
|
37
37
|
etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
|
|
38
38
|
etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
|
|
39
39
|
etlplus/config/jobs.py,sha256=hmzRCqt0OvCEZZR4ONKrd3lvSv0OmayjLc4yOBk3ug8,7399
|
|
@@ -43,9 +43,9 @@ etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
|
|
|
43
43
|
etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
|
|
44
44
|
etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
|
|
45
45
|
etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
|
|
46
|
-
etlplus-0.4.
|
|
47
|
-
etlplus-0.4.
|
|
48
|
-
etlplus-0.4.
|
|
49
|
-
etlplus-0.4.
|
|
50
|
-
etlplus-0.4.
|
|
51
|
-
etlplus-0.4.
|
|
46
|
+
etlplus-0.4.8.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
|
|
47
|
+
etlplus-0.4.8.dist-info/METADATA,sha256=FtgFi5AgfGMJ18giygT2oaG9VmtQREG4Ma3HnA_6qyA,17635
|
|
48
|
+
etlplus-0.4.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
49
|
+
etlplus-0.4.8.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
50
|
+
etlplus-0.4.8.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
51
|
+
etlplus-0.4.8.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|