etlplus 0.4.7__py3-none-any.whl → 0.8.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/api/README.md +24 -26
- etlplus/cli/commands.py +870 -0
- etlplus/cli/constants.py +65 -0
- etlplus/cli/handlers.py +426 -434
- etlplus/cli/io.py +320 -0
- etlplus/cli/main.py +14 -367
- etlplus/cli/options.py +49 -0
- etlplus/cli/state.py +335 -0
- etlplus/cli/types.py +33 -0
- etlplus/config/pipeline.py +11 -0
- etlplus/database/__init__.py +44 -0
- etlplus/database/ddl.py +319 -0
- etlplus/database/engine.py +151 -0
- etlplus/database/orm.py +354 -0
- etlplus/database/schema.py +274 -0
- etlplus/database/types.py +33 -0
- etlplus/run.py +2 -4
- etlplus/templates/__init__.py +5 -0
- etlplus/templates/ddl.sql.j2 +128 -0
- etlplus/templates/view.sql.j2 +69 -0
- etlplus/types.py +5 -0
- etlplus/utils.py +0 -31
- {etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/METADATA +66 -1
- {etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/RECORD +28 -14
- etlplus/cli/app.py +0 -1239
- {etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/WHEEL +0 -0
- {etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/entry_points.txt +0 -0
- {etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.4.7.dist-info → etlplus-0.8.3.dist-info}/top_level.txt +0 -0
etlplus/cli/main.py
CHANGED
|
@@ -9,187 +9,27 @@ This module exposes :func:`main` for the console script as well as
|
|
|
9
9
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
|
-
import argparse
|
|
13
12
|
import contextlib
|
|
14
13
|
import sys
|
|
15
|
-
|
|
16
|
-
from typing import Literal
|
|
14
|
+
import warnings
|
|
17
15
|
|
|
18
16
|
import click
|
|
19
17
|
import typer
|
|
20
18
|
|
|
21
|
-
from
|
|
22
|
-
from ..enums import DataConnectorType
|
|
23
|
-
from ..enums import FileFormat
|
|
24
|
-
from ..utils import json_type
|
|
25
|
-
from .app import PROJECT_URL
|
|
26
|
-
from .app import app
|
|
27
|
-
from .handlers import cmd_extract
|
|
28
|
-
from .handlers import cmd_list
|
|
29
|
-
from .handlers import cmd_load
|
|
30
|
-
from .handlers import cmd_pipeline
|
|
31
|
-
from .handlers import cmd_run
|
|
32
|
-
from .handlers import cmd_transform
|
|
33
|
-
from .handlers import cmd_validate
|
|
19
|
+
from .commands import app
|
|
34
20
|
|
|
35
21
|
# SECTION: EXPORTS ========================================================== #
|
|
36
22
|
|
|
37
23
|
|
|
38
24
|
__all__ = [
|
|
39
25
|
# Functions
|
|
40
|
-
'create_parser',
|
|
41
26
|
'main',
|
|
42
27
|
]
|
|
43
28
|
|
|
44
29
|
|
|
45
|
-
# SECTION: TYPE ALIASES ===================================================== #
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
type FormatContext = Literal['source', 'target']
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
# SECTION: INTERNAL CLASSES ================================================= #
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
class _FormatAction(argparse.Action):
|
|
55
|
-
"""
|
|
56
|
-
Argparse action that records when ``--source-format`` or
|
|
57
|
-
``--target-format`` is provided."""
|
|
58
|
-
|
|
59
|
-
def __call__(
|
|
60
|
-
self,
|
|
61
|
-
parser: argparse.ArgumentParser,
|
|
62
|
-
namespace: argparse.Namespace,
|
|
63
|
-
values: str | Sequence[object] | None,
|
|
64
|
-
option_string: str | None = None,
|
|
65
|
-
) -> None: # pragma: no cover
|
|
66
|
-
setattr(namespace, self.dest, values)
|
|
67
|
-
namespace._format_explicit = True
|
|
68
|
-
|
|
69
|
-
|
|
70
30
|
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
71
31
|
|
|
72
32
|
|
|
73
|
-
def _add_boolean_flag(
|
|
74
|
-
parser: argparse.ArgumentParser,
|
|
75
|
-
*,
|
|
76
|
-
name: str,
|
|
77
|
-
help_text: str,
|
|
78
|
-
) -> None:
|
|
79
|
-
"""Add a toggle that also supports the ``--no-`` prefix via 3.13.
|
|
80
|
-
|
|
81
|
-
Parameters
|
|
82
|
-
----------
|
|
83
|
-
parser : argparse.ArgumentParser
|
|
84
|
-
Parser receiving the flag.
|
|
85
|
-
name : str
|
|
86
|
-
Primary flag name without leading dashes.
|
|
87
|
-
help_text : str
|
|
88
|
-
Help text rendered in ``--help`` output.
|
|
89
|
-
"""
|
|
90
|
-
|
|
91
|
-
parser.add_argument(
|
|
92
|
-
f'--{name}',
|
|
93
|
-
action=argparse.BooleanOptionalAction,
|
|
94
|
-
default=False,
|
|
95
|
-
help=help_text,
|
|
96
|
-
)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def _add_config_option(
|
|
100
|
-
parser: argparse.ArgumentParser,
|
|
101
|
-
*,
|
|
102
|
-
required: bool = True,
|
|
103
|
-
) -> None:
|
|
104
|
-
"""Attach the shared ``--config`` option used by legacy commands.
|
|
105
|
-
|
|
106
|
-
Parameters
|
|
107
|
-
----------
|
|
108
|
-
parser : argparse.ArgumentParser
|
|
109
|
-
Parser receiving the option.
|
|
110
|
-
required : bool, optional
|
|
111
|
-
Whether the flag must be provided. Defaults to ``True``.
|
|
112
|
-
"""
|
|
113
|
-
|
|
114
|
-
parser.add_argument(
|
|
115
|
-
'--config',
|
|
116
|
-
required=required,
|
|
117
|
-
help='Path to pipeline YAML configuration file',
|
|
118
|
-
)
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
def _add_format_options(
|
|
122
|
-
parser: argparse.ArgumentParser,
|
|
123
|
-
*,
|
|
124
|
-
context: FormatContext,
|
|
125
|
-
) -> None:
|
|
126
|
-
"""
|
|
127
|
-
Attach shared ``--source-format`` or ``--target-format`` options to
|
|
128
|
-
extract/load parsers.
|
|
129
|
-
|
|
130
|
-
Parameters
|
|
131
|
-
----------
|
|
132
|
-
parser : argparse.ArgumentParser
|
|
133
|
-
Parser to augment.
|
|
134
|
-
context : FormatContext
|
|
135
|
-
Context for the format option: either ``'source'`` or ``'target'``
|
|
136
|
-
"""
|
|
137
|
-
parser.set_defaults(_format_explicit=False)
|
|
138
|
-
parser.add_argument(
|
|
139
|
-
'--source-format',
|
|
140
|
-
choices=list(FileFormat.choices()),
|
|
141
|
-
default='json',
|
|
142
|
-
action=_FormatAction,
|
|
143
|
-
help=(
|
|
144
|
-
f'Format of the {context}. Overrides filename-based inference '
|
|
145
|
-
'when provided.'
|
|
146
|
-
),
|
|
147
|
-
)
|
|
148
|
-
parser.add_argument(
|
|
149
|
-
'--target-format',
|
|
150
|
-
choices=list(FileFormat.choices()),
|
|
151
|
-
default='json',
|
|
152
|
-
action=_FormatAction,
|
|
153
|
-
help=(
|
|
154
|
-
f'Format of the {context}. Overrides filename-based inference '
|
|
155
|
-
'when provided.'
|
|
156
|
-
),
|
|
157
|
-
)
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
def _cli_description() -> str:
|
|
161
|
-
return '\n'.join(
|
|
162
|
-
[
|
|
163
|
-
'ETLPlus - A Swiss Army knife for simple ETL operations.',
|
|
164
|
-
'',
|
|
165
|
-
' Provide a subcommand and options. Examples:',
|
|
166
|
-
'',
|
|
167
|
-
' etlplus extract file in.csv > out.json',
|
|
168
|
-
' etlplus validate in.json --rules \'{"required": ["id"]}\'',
|
|
169
|
-
(
|
|
170
|
-
' etlplus transform --from file in.csv --operations '
|
|
171
|
-
'\'{"select": ["id"]}\' --to file -o out.json'
|
|
172
|
-
),
|
|
173
|
-
' etlplus extract in.csv | etlplus load --to file out.json',
|
|
174
|
-
'',
|
|
175
|
-
' Override format inference when extensions are misleading:',
|
|
176
|
-
'',
|
|
177
|
-
' etlplus extract data.txt --source-format csv',
|
|
178
|
-
' etlplus load payload.bin --target-format json',
|
|
179
|
-
],
|
|
180
|
-
)
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
def _cli_epilog() -> str:
|
|
184
|
-
return '\n'.join(
|
|
185
|
-
[
|
|
186
|
-
'Tip:',
|
|
187
|
-
' --source-format and --target-format override format '
|
|
188
|
-
'inference based on filename extensions when needed.',
|
|
189
|
-
],
|
|
190
|
-
)
|
|
191
|
-
|
|
192
|
-
|
|
193
33
|
def _emit_context_help(
|
|
194
34
|
ctx: click.Context | None,
|
|
195
35
|
) -> bool:
|
|
@@ -280,216 +120,23 @@ def _is_unknown_command_error(
|
|
|
280
120
|
# SECTION: FUNCTIONS ======================================================== #
|
|
281
121
|
|
|
282
122
|
|
|
283
|
-
def create_parser() ->
|
|
123
|
+
def create_parser() -> object:
|
|
284
124
|
"""
|
|
285
|
-
|
|
125
|
+
Deprecated legacy entrypoint.
|
|
286
126
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
argparse.ArgumentParser
|
|
290
|
-
Parser compatible with historical ``etlplus`` entry points.
|
|
127
|
+
The argparse-based parser has been removed. Use the Typer-powered
|
|
128
|
+
``etlplus`` CLI instead (``etlplus.cli.commands.app``).
|
|
291
129
|
"""
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
298
|
-
)
|
|
299
|
-
|
|
300
|
-
parser.add_argument(
|
|
301
|
-
'-V',
|
|
302
|
-
'--version',
|
|
303
|
-
action='version',
|
|
304
|
-
version=f'%(prog)s {__version__}',
|
|
305
|
-
)
|
|
306
|
-
|
|
307
|
-
subparsers = parser.add_subparsers(
|
|
308
|
-
dest='command',
|
|
309
|
-
help='Available commands',
|
|
310
|
-
)
|
|
311
|
-
subparsers.required = True
|
|
312
|
-
|
|
313
|
-
extract_parser = subparsers.add_parser(
|
|
314
|
-
'extract',
|
|
315
|
-
help='Extract data from sources (files, databases, REST APIs)',
|
|
316
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
317
|
-
)
|
|
318
|
-
extract_parser.add_argument(
|
|
319
|
-
'source_type',
|
|
320
|
-
choices=list(DataConnectorType.choices()),
|
|
321
|
-
help='Type of source to extract from',
|
|
322
|
-
)
|
|
323
|
-
extract_parser.add_argument(
|
|
324
|
-
'source',
|
|
325
|
-
help=(
|
|
326
|
-
'Source location (file path, database connection string, '
|
|
327
|
-
'or API URL)'
|
|
328
|
-
),
|
|
329
|
-
)
|
|
330
|
-
_add_format_options(extract_parser, context='source')
|
|
331
|
-
extract_parser.set_defaults(func=cmd_extract)
|
|
332
|
-
|
|
333
|
-
validate_parser = subparsers.add_parser(
|
|
334
|
-
'validate',
|
|
335
|
-
help='Validate data from sources',
|
|
336
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
337
|
-
)
|
|
338
|
-
validate_parser.add_argument(
|
|
339
|
-
'source',
|
|
340
|
-
help='Data source to validate (file path or JSON string)',
|
|
341
|
-
)
|
|
342
|
-
validate_parser.add_argument(
|
|
343
|
-
'--rules',
|
|
344
|
-
type=json_type,
|
|
345
|
-
default={},
|
|
346
|
-
help='Validation rules as JSON string',
|
|
347
|
-
)
|
|
348
|
-
validate_parser.set_defaults(func=cmd_validate)
|
|
349
|
-
|
|
350
|
-
transform_parser = subparsers.add_parser(
|
|
351
|
-
'transform',
|
|
352
|
-
help='Transform data',
|
|
353
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
354
|
-
)
|
|
355
|
-
transform_parser.add_argument(
|
|
356
|
-
'source',
|
|
357
|
-
help='Data source to transform (file path or JSON string)',
|
|
358
|
-
)
|
|
359
|
-
transform_parser.add_argument(
|
|
360
|
-
'--operations',
|
|
361
|
-
type=json_type,
|
|
362
|
-
default={},
|
|
363
|
-
help='Transformation operations as JSON string',
|
|
364
|
-
)
|
|
365
|
-
transform_parser.add_argument(
|
|
366
|
-
'--from',
|
|
367
|
-
dest='from_',
|
|
368
|
-
choices=list(DataConnectorType.choices()),
|
|
369
|
-
help='Override the inferred source type (file, database, api).',
|
|
370
|
-
)
|
|
371
|
-
transform_parser.add_argument(
|
|
372
|
-
'--to',
|
|
373
|
-
dest='to',
|
|
374
|
-
choices=list(DataConnectorType.choices()),
|
|
375
|
-
help='Override the inferred target type (file, database, api).',
|
|
376
|
-
)
|
|
377
|
-
transform_parser.add_argument(
|
|
378
|
-
'--source-format',
|
|
379
|
-
choices=list(FileFormat.choices()),
|
|
380
|
-
dest='source_format',
|
|
381
|
-
help=(
|
|
382
|
-
'Input payload format when SOURCE is - or a literal payload. '
|
|
383
|
-
'File sources infer format from the extension.'
|
|
384
|
-
),
|
|
130
|
+
warnings.warn(
|
|
131
|
+
'create_parser is deprecated and no longer returns an argparse '
|
|
132
|
+
'parser. Use the Typer CLI entrypoint instead.',
|
|
133
|
+
DeprecationWarning,
|
|
134
|
+
stacklevel=2,
|
|
385
135
|
)
|
|
386
|
-
|
|
387
|
-
'
|
|
388
|
-
|
|
389
|
-
choices=list(FileFormat.choices()),
|
|
390
|
-
help=(
|
|
391
|
-
'Output payload format '
|
|
392
|
-
'when writing to stdout or non-file targets. '
|
|
393
|
-
'File targets infer format from the extension.'
|
|
394
|
-
),
|
|
395
|
-
)
|
|
396
|
-
transform_parser.set_defaults(func=cmd_transform)
|
|
397
|
-
|
|
398
|
-
load_parser = subparsers.add_parser(
|
|
399
|
-
'load',
|
|
400
|
-
help='Load data to targets (files, databases, REST APIs)',
|
|
401
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
402
|
-
)
|
|
403
|
-
load_parser.add_argument(
|
|
404
|
-
'source',
|
|
405
|
-
help='Data source to load (file path or JSON string)',
|
|
406
|
-
)
|
|
407
|
-
load_parser.add_argument(
|
|
408
|
-
'target_type',
|
|
409
|
-
choices=list(DataConnectorType.choices()),
|
|
410
|
-
help='Type of target to load to',
|
|
411
|
-
)
|
|
412
|
-
load_parser.add_argument(
|
|
413
|
-
'target',
|
|
414
|
-
help=(
|
|
415
|
-
'Target location (file path, database connection string, '
|
|
416
|
-
'or API URL)'
|
|
417
|
-
),
|
|
418
|
-
)
|
|
419
|
-
_add_format_options(load_parser, context='target')
|
|
420
|
-
load_parser.set_defaults(func=cmd_load)
|
|
421
|
-
|
|
422
|
-
pipe_parser = subparsers.add_parser(
|
|
423
|
-
'pipeline',
|
|
424
|
-
help=(
|
|
425
|
-
'Inspect or run pipeline YAML (see '
|
|
426
|
-
f'{PROJECT_URL}/blob/main/docs/pipeline-guide.md)'
|
|
427
|
-
),
|
|
428
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
136
|
+
raise RuntimeError(
|
|
137
|
+
'The legacy argparse parser has been removed. Invoke the Typer-based '
|
|
138
|
+
'CLI via `etlplus` or import `etlplus.cli.commands.app`.',
|
|
429
139
|
)
|
|
430
|
-
_add_config_option(pipe_parser)
|
|
431
|
-
pipe_parser.add_argument(
|
|
432
|
-
'--list',
|
|
433
|
-
action='store_true',
|
|
434
|
-
help='List available job names and exit',
|
|
435
|
-
)
|
|
436
|
-
pipe_parser.add_argument(
|
|
437
|
-
'--run',
|
|
438
|
-
metavar='JOB',
|
|
439
|
-
help='Run a specific job by name',
|
|
440
|
-
)
|
|
441
|
-
pipe_parser.set_defaults(func=cmd_pipeline)
|
|
442
|
-
|
|
443
|
-
list_parser = subparsers.add_parser(
|
|
444
|
-
'list',
|
|
445
|
-
help='List ETL pipeline metadata',
|
|
446
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
447
|
-
)
|
|
448
|
-
_add_config_option(list_parser)
|
|
449
|
-
_add_boolean_flag(
|
|
450
|
-
list_parser,
|
|
451
|
-
name='pipelines',
|
|
452
|
-
help_text='List ETL pipelines',
|
|
453
|
-
)
|
|
454
|
-
_add_boolean_flag(
|
|
455
|
-
list_parser,
|
|
456
|
-
name='sources',
|
|
457
|
-
help_text='List data sources',
|
|
458
|
-
)
|
|
459
|
-
_add_boolean_flag(
|
|
460
|
-
list_parser,
|
|
461
|
-
name='targets',
|
|
462
|
-
help_text='List data targets',
|
|
463
|
-
)
|
|
464
|
-
_add_boolean_flag(
|
|
465
|
-
list_parser,
|
|
466
|
-
name='transforms',
|
|
467
|
-
help_text='List data transforms',
|
|
468
|
-
)
|
|
469
|
-
list_parser.set_defaults(func=cmd_list)
|
|
470
|
-
|
|
471
|
-
run_parser = subparsers.add_parser(
|
|
472
|
-
'run',
|
|
473
|
-
help=(
|
|
474
|
-
'Run an ETL pipeline '
|
|
475
|
-
f'(see {PROJECT_URL}/blob/main/docs/run-module.md)'
|
|
476
|
-
),
|
|
477
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
478
|
-
)
|
|
479
|
-
_add_config_option(run_parser)
|
|
480
|
-
run_parser.add_argument(
|
|
481
|
-
'-j',
|
|
482
|
-
'--job',
|
|
483
|
-
help='Name of the job to run',
|
|
484
|
-
)
|
|
485
|
-
run_parser.add_argument(
|
|
486
|
-
'-p',
|
|
487
|
-
'--pipeline',
|
|
488
|
-
help='Name of the pipeline to run',
|
|
489
|
-
)
|
|
490
|
-
run_parser.set_defaults(func=cmd_run)
|
|
491
|
-
|
|
492
|
-
return parser
|
|
493
140
|
|
|
494
141
|
|
|
495
142
|
def main(
|
etlplus/cli/options.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.cli.options` module.
|
|
3
|
+
|
|
4
|
+
Shared Typer helper utilities for command-line interface (CLI) option
|
|
5
|
+
configuration.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from .types import DataConnectorContext
|
|
11
|
+
|
|
12
|
+
# SECTION: EXPORTS ========================================================== #
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
# Functions
|
|
17
|
+
'typer_format_option_kwargs',
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def typer_format_option_kwargs(
|
|
22
|
+
*,
|
|
23
|
+
context: DataConnectorContext,
|
|
24
|
+
rich_help_panel: str = 'Format overrides',
|
|
25
|
+
) -> dict[str, object]:
|
|
26
|
+
"""
|
|
27
|
+
Return common Typer option kwargs for format overrides.
|
|
28
|
+
|
|
29
|
+
Parameters
|
|
30
|
+
----------
|
|
31
|
+
context : DataConnectorContext
|
|
32
|
+
Either ``'source'`` or ``'target'`` to tailor help text.
|
|
33
|
+
rich_help_panel : str, optional
|
|
34
|
+
The rich help panel name. Default is ``'Format overrides'``.
|
|
35
|
+
|
|
36
|
+
Returns
|
|
37
|
+
-------
|
|
38
|
+
dict[str, object]
|
|
39
|
+
The Typer option keyword arguments.
|
|
40
|
+
"""
|
|
41
|
+
return {
|
|
42
|
+
'metavar': 'FORMAT',
|
|
43
|
+
'show_default': False,
|
|
44
|
+
'rich_help_panel': rich_help_panel,
|
|
45
|
+
'help': (
|
|
46
|
+
f'Payload format when the {context} is stdin/inline or a '
|
|
47
|
+
'non-file connector. File connectors infer from extensions.'
|
|
48
|
+
),
|
|
49
|
+
}
|