etlplus 0.7.2__py3-none-any.whl → 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/cli/app.py +0 -55
- etlplus/cli/handlers.py +4 -48
- etlplus/cli/main.py +0 -23
- etlplus/database/ddl.py +1 -1
- etlplus/database/types.py +0 -5
- etlplus/types.py +5 -0
- {etlplus-0.7.2.dist-info → etlplus-0.8.0.dist-info}/METADATA +1 -3
- {etlplus-0.7.2.dist-info → etlplus-0.8.0.dist-info}/RECORD +12 -12
- {etlplus-0.7.2.dist-info → etlplus-0.8.0.dist-info}/WHEEL +0 -0
- {etlplus-0.7.2.dist-info → etlplus-0.8.0.dist-info}/entry_points.txt +0 -0
- {etlplus-0.7.2.dist-info → etlplus-0.8.0.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.7.2.dist-info → etlplus-0.8.0.dist-info}/top_level.txt +0 -0
etlplus/cli/app.py
CHANGED
|
@@ -60,7 +60,6 @@ from ..utils import json_type
|
|
|
60
60
|
from .handlers import check_handler
|
|
61
61
|
from .handlers import extract_handler
|
|
62
62
|
from .handlers import load_handler
|
|
63
|
-
from .handlers import pipeline_handler
|
|
64
63
|
from .handlers import render_handler
|
|
65
64
|
from .handlers import run_handler
|
|
66
65
|
from .handlers import transform_handler
|
|
@@ -1010,60 +1009,6 @@ def load_cmd(
|
|
|
1010
1009
|
return int(load_handler(ns))
|
|
1011
1010
|
|
|
1012
1011
|
|
|
1013
|
-
@app.command('pipeline')
|
|
1014
|
-
def pipeline_cmd(
|
|
1015
|
-
ctx: typer.Context,
|
|
1016
|
-
config: PipelineConfigOption,
|
|
1017
|
-
job: str | None = typer.Option(
|
|
1018
|
-
None,
|
|
1019
|
-
'--job',
|
|
1020
|
-
metavar='JOB',
|
|
1021
|
-
help='Run a specific job by name',
|
|
1022
|
-
),
|
|
1023
|
-
jobs: bool = typer.Option(
|
|
1024
|
-
False,
|
|
1025
|
-
'--jobs',
|
|
1026
|
-
help='List available job names and exit',
|
|
1027
|
-
),
|
|
1028
|
-
pipeline: str | None = typer.Option(
|
|
1029
|
-
None,
|
|
1030
|
-
'--pipeline',
|
|
1031
|
-
help='Run a specific pipeline by name',
|
|
1032
|
-
),
|
|
1033
|
-
) -> int:
|
|
1034
|
-
"""
|
|
1035
|
-
Deprecated wrapper to inspect or run a pipeline YAML configuration.
|
|
1036
|
-
|
|
1037
|
-
Parameters
|
|
1038
|
-
----------
|
|
1039
|
-
ctx : typer.Context
|
|
1040
|
-
Typer execution context provided to the command.
|
|
1041
|
-
config : PipelineConfigOption
|
|
1042
|
-
Path to pipeline YAML configuration file.
|
|
1043
|
-
job : str | None, optional
|
|
1044
|
-
Name of a specific job to run.
|
|
1045
|
-
jobs : bool, optional
|
|
1046
|
-
If True, list available job names and exit.
|
|
1047
|
-
pipeline : str | None, optional
|
|
1048
|
-
Name of a specific pipeline to run.
|
|
1049
|
-
|
|
1050
|
-
Returns
|
|
1051
|
-
-------
|
|
1052
|
-
int
|
|
1053
|
-
Zero on success.
|
|
1054
|
-
"""
|
|
1055
|
-
state = _ensure_state(ctx)
|
|
1056
|
-
run_target = job or pipeline
|
|
1057
|
-
ns = _stateful_namespace(
|
|
1058
|
-
state,
|
|
1059
|
-
command='pipeline',
|
|
1060
|
-
config=config,
|
|
1061
|
-
list=jobs,
|
|
1062
|
-
run=run_target,
|
|
1063
|
-
)
|
|
1064
|
-
return int(pipeline_handler(ns))
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
1012
|
@app.command('render')
|
|
1068
1013
|
def render_cmd(
|
|
1069
1014
|
ctx: typer.Context,
|
etlplus/cli/handlers.py
CHANGED
|
@@ -27,6 +27,7 @@ from ..load import load
|
|
|
27
27
|
from ..run import run
|
|
28
28
|
from ..transform import transform
|
|
29
29
|
from ..types import JSONData
|
|
30
|
+
from ..types import TemplateKey
|
|
30
31
|
from ..utils import json_type
|
|
31
32
|
from ..utils import print_json
|
|
32
33
|
from ..validate import validate
|
|
@@ -39,7 +40,6 @@ __all__ = [
|
|
|
39
40
|
'extract_handler',
|
|
40
41
|
'check_handler',
|
|
41
42
|
'load_handler',
|
|
42
|
-
'pipeline_handler',
|
|
43
43
|
'render_handler',
|
|
44
44
|
'run_handler',
|
|
45
45
|
'transform_handler',
|
|
@@ -72,7 +72,7 @@ def _collect_table_specs(
|
|
|
72
72
|
specs: list[dict[str, Any]] = []
|
|
73
73
|
|
|
74
74
|
if spec_path:
|
|
75
|
-
specs.append(load_table_spec(Path(spec_path)))
|
|
75
|
+
specs.append(dict(load_table_spec(Path(spec_path))))
|
|
76
76
|
|
|
77
77
|
if config_path:
|
|
78
78
|
cfg = load_pipeline_config(config_path, substitute=True)
|
|
@@ -552,57 +552,13 @@ def load_handler(
|
|
|
552
552
|
return 0
|
|
553
553
|
|
|
554
554
|
|
|
555
|
-
def pipeline_handler(
|
|
556
|
-
args: argparse.Namespace,
|
|
557
|
-
) -> int:
|
|
558
|
-
"""
|
|
559
|
-
Inspect or run a pipeline YAML configuration.
|
|
560
|
-
|
|
561
|
-
Parameters
|
|
562
|
-
----------
|
|
563
|
-
args : argparse.Namespace
|
|
564
|
-
Parsed command-line arguments.
|
|
565
|
-
|
|
566
|
-
Returns
|
|
567
|
-
-------
|
|
568
|
-
int
|
|
569
|
-
Zero on success.
|
|
570
|
-
"""
|
|
571
|
-
print(
|
|
572
|
-
'DEPRECATED: use "etlplus check --summary|--jobs" or '
|
|
573
|
-
'"etlplus run --job/--pipeline" instead of "etlplus pipeline".',
|
|
574
|
-
file=sys.stderr,
|
|
575
|
-
)
|
|
576
|
-
|
|
577
|
-
cfg = load_pipeline_config(args.config, substitute=True)
|
|
578
|
-
|
|
579
|
-
list_flag = getattr(args, 'list', False) or getattr(args, 'jobs', False)
|
|
580
|
-
run_target = (
|
|
581
|
-
getattr(args, 'run', None)
|
|
582
|
-
or getattr(args, 'job', None)
|
|
583
|
-
or getattr(args, 'pipeline', None)
|
|
584
|
-
)
|
|
585
|
-
|
|
586
|
-
if list_flag and not run_target:
|
|
587
|
-
print_json({'jobs': _pipeline_summary(cfg)['jobs']})
|
|
588
|
-
return 0
|
|
589
|
-
|
|
590
|
-
if run_target:
|
|
591
|
-
result = run(job=run_target, config_path=args.config)
|
|
592
|
-
print_json({'status': 'ok', 'result': result})
|
|
593
|
-
return 0
|
|
594
|
-
|
|
595
|
-
print_json(_pipeline_summary(cfg))
|
|
596
|
-
return 0
|
|
597
|
-
|
|
598
|
-
|
|
599
555
|
def render_handler(
|
|
600
556
|
args: argparse.Namespace,
|
|
601
557
|
) -> int:
|
|
602
558
|
"""Render SQL DDL statements from table schema specs."""
|
|
603
559
|
_, quiet = _presentation_flags(args)
|
|
604
560
|
|
|
605
|
-
template_value = getattr(args, 'template', 'ddl') or 'ddl'
|
|
561
|
+
template_value: TemplateKey = getattr(args, 'template', 'ddl') or 'ddl'
|
|
606
562
|
template_path = getattr(args, 'template_path', None)
|
|
607
563
|
table_filter = getattr(args, 'table', None)
|
|
608
564
|
spec_path = getattr(args, 'spec', None)
|
|
@@ -610,7 +566,7 @@ def render_handler(
|
|
|
610
566
|
|
|
611
567
|
# If the provided template points to a file, treat it as a path override.
|
|
612
568
|
file_override = template_path
|
|
613
|
-
template_key = template_value
|
|
569
|
+
template_key: TemplateKey | None = template_value
|
|
614
570
|
if template_path is None:
|
|
615
571
|
candidate_path = Path(template_value)
|
|
616
572
|
if candidate_path.exists():
|
etlplus/cli/main.py
CHANGED
|
@@ -27,7 +27,6 @@ from .app import app
|
|
|
27
27
|
from .handlers import check_handler
|
|
28
28
|
from .handlers import extract_handler
|
|
29
29
|
from .handlers import load_handler
|
|
30
|
-
from .handlers import pipeline_handler
|
|
31
30
|
from .handlers import render_handler
|
|
32
31
|
from .handlers import run_handler
|
|
33
32
|
from .handlers import transform_handler
|
|
@@ -420,28 +419,6 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
420
419
|
_add_format_options(load_parser, context='target')
|
|
421
420
|
load_parser.set_defaults(func=load_handler)
|
|
422
421
|
|
|
423
|
-
pipe_parser = subparsers.add_parser(
|
|
424
|
-
'pipeline',
|
|
425
|
-
help=(
|
|
426
|
-
'DEPRECATED: use "list" (for summary/jobs) or "run" (to execute); '
|
|
427
|
-
'see '
|
|
428
|
-
f'{PROJECT_URL}/blob/main/docs/pipeline-guide.md'
|
|
429
|
-
),
|
|
430
|
-
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
431
|
-
)
|
|
432
|
-
_add_config_option(pipe_parser)
|
|
433
|
-
pipe_parser.add_argument(
|
|
434
|
-
'--list',
|
|
435
|
-
action='store_true',
|
|
436
|
-
help='List available job names and exit',
|
|
437
|
-
)
|
|
438
|
-
pipe_parser.add_argument(
|
|
439
|
-
'--run',
|
|
440
|
-
metavar='JOB',
|
|
441
|
-
help='Run a specific job by name',
|
|
442
|
-
)
|
|
443
|
-
pipe_parser.set_defaults(func=pipeline_handler)
|
|
444
|
-
|
|
445
422
|
render_parser = subparsers.add_parser(
|
|
446
423
|
'render',
|
|
447
424
|
help='Render SQL DDL from table schema specs',
|
etlplus/database/ddl.py
CHANGED
|
@@ -25,7 +25,7 @@ from jinja2 import StrictUndefined
|
|
|
25
25
|
from ..file import File
|
|
26
26
|
from ..types import StrAnyMap
|
|
27
27
|
from ..types import StrPath
|
|
28
|
-
from
|
|
28
|
+
from ..types import TemplateKey
|
|
29
29
|
|
|
30
30
|
# SECTION: EXPORTS ========================================================== #
|
|
31
31
|
|
etlplus/database/types.py
CHANGED
|
@@ -7,7 +7,6 @@ Shared type aliases leveraged across :mod:`etlplus.database` modules.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
from collections.abc import Callable
|
|
10
|
-
from typing import Literal
|
|
11
10
|
|
|
12
11
|
from sqlalchemy.orm import DeclarativeBase
|
|
13
12
|
from sqlalchemy.types import TypeEngine
|
|
@@ -18,7 +17,6 @@ from sqlalchemy.types import TypeEngine
|
|
|
18
17
|
__all__ = [
|
|
19
18
|
# Type Aliases
|
|
20
19
|
'ModelRegistry',
|
|
21
|
-
'TemplateKey',
|
|
22
20
|
'TypeFactory',
|
|
23
21
|
]
|
|
24
22
|
|
|
@@ -31,8 +29,5 @@ __all__ = [
|
|
|
31
29
|
# Registry mapping fully qualified table names to declarative classes.
|
|
32
30
|
type ModelRegistry = dict[str, type[DeclarativeBase]]
|
|
33
31
|
|
|
34
|
-
# Allowed template keys for bundled DDL rendering.
|
|
35
|
-
type TemplateKey = Literal['ddl', 'view']
|
|
36
|
-
|
|
37
32
|
# Callable producing a SQLAlchemy TypeEngine from parsed parameters.
|
|
38
33
|
type TypeFactory = Callable[[list[int]], TypeEngine]
|
etlplus/types.py
CHANGED
|
@@ -225,3 +225,8 @@ type Sleeper = Callable[[float], None]
|
|
|
225
225
|
|
|
226
226
|
# Numeric timeout in seconds or ``None`` for no timeout.
|
|
227
227
|
type Timeout = float | None
|
|
228
|
+
|
|
229
|
+
# -- Templates -- #
|
|
230
|
+
|
|
231
|
+
# Allowed template keys for bundled DDL rendering.
|
|
232
|
+
type TemplateKey = Literal['ddl', 'view']
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: etlplus
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: A Swiss Army knife for simple ETL operations
|
|
5
5
|
Home-page: https://github.com/Dagitali/ETLPlus
|
|
6
6
|
Author: ETLPlus Team
|
|
@@ -366,8 +366,6 @@ etlplus check --config examples/configs/pipeline.yml --summary
|
|
|
366
366
|
|
|
367
367
|
# Run a job
|
|
368
368
|
etlplus run --config examples/configs/pipeline.yml --job file_to_file_customers
|
|
369
|
-
|
|
370
|
-
# Deprecated shim (will be removed): etlplus pipeline
|
|
371
369
|
```
|
|
372
370
|
|
|
373
371
|
### Complete ETL Pipeline Example
|
|
@@ -10,7 +10,7 @@ etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
10
10
|
etlplus/run.py,sha256=X4kp5FQlIWVf1_d9oSrchKau7BFDCE1Zkscvu7WPaWw,12340
|
|
11
11
|
etlplus/run_helpers.py,sha256=bj6MkaeFxjl3CeKG1HoXKx5DwAlXNERVW-GX-z1P_qQ,24373
|
|
12
12
|
etlplus/transform.py,sha256=uAUVDDHYCgx7GpVez9IK3OAZM-CnCuMa9iox3vwGGJA,25296
|
|
13
|
-
etlplus/types.py,sha256=
|
|
13
|
+
etlplus/types.py,sha256=1hsDlnF6r76zAwaUYay-i6pCM-Y0IU5nP7Crj8PLCQ4,6157
|
|
14
14
|
etlplus/utils.py,sha256=_fn8b-SAdxiw28VX-Ugr8sZUPZI9mEkWKAGExlgxhJA,13993
|
|
15
15
|
etlplus/validate.py,sha256=7rJoEI_SIILdPpoBqqh2UJqg9oeReDz34mYSlc3t7Qg,12989
|
|
16
16
|
etlplus/api/README.md,sha256=UkK5PiZWXbbnMNP0MaPa56S88PjSqOwhMNCyswOhvKc,7329
|
|
@@ -31,9 +31,9 @@ etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-
|
|
|
31
31
|
etlplus/api/rate_limiting/config.py,sha256=2b4wIynblN-1EyMqI4aXa71SljzSjXYh5N1Nngr3jOg,9406
|
|
32
32
|
etlplus/api/rate_limiting/rate_limiter.py,sha256=Uxozqd_Ej5Lsj-M-mLT2WexChgWh7x35_YP10yqYPQA,7159
|
|
33
33
|
etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
|
|
34
|
-
etlplus/cli/app.py,sha256=
|
|
35
|
-
etlplus/cli/handlers.py,sha256=
|
|
36
|
-
etlplus/cli/main.py,sha256=
|
|
34
|
+
etlplus/cli/app.py,sha256=kQuMmY-RqadRGWS5yDpkKl5bWdOpqbR0jqsXCORLsMA,34532
|
|
35
|
+
etlplus/cli/handlers.py,sha256=5Blk5LSqTVSwTPPGEcbOw2NwNTTbFFOTxlpUdy5r2bA,17928
|
|
36
|
+
etlplus/cli/main.py,sha256=1U2WS-yIxTEr6ZJJkQFtisCxOcZ3cdnjIxUTewapN5M,15878
|
|
37
37
|
etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
|
|
38
38
|
etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
|
|
39
39
|
etlplus/config/jobs.py,sha256=hmzRCqt0OvCEZZR4ONKrd3lvSv0OmayjLc4yOBk3ug8,7399
|
|
@@ -42,19 +42,19 @@ etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,189
|
|
|
42
42
|
etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
|
|
43
43
|
etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
|
|
44
44
|
etlplus/database/__init__.py,sha256=AKJsDl2RHuRGPS-eXgNJeh4aSncJP5Y0yLApBF6i7i8,1052
|
|
45
|
-
etlplus/database/ddl.py,sha256=
|
|
45
|
+
etlplus/database/ddl.py,sha256=z9KvHi1MPhPBLHxMDdqJgLTp3A2-lcz0gqhZ7HIE6kU,7916
|
|
46
46
|
etlplus/database/engine.py,sha256=7rr7YndA8LwyWJL8k1YhQbqxxmW4gWEUQjp0NwQcYtc,4061
|
|
47
47
|
etlplus/database/orm.py,sha256=gCSqH-CjQz6tV9133-VqgiwokK5ylun0BwXaIWfImAo,10008
|
|
48
48
|
etlplus/database/schema.py,sha256=HNTgglI8qvQLInr7gq--2lLmLKHzAZTL2MJUOIw9DlY,7025
|
|
49
|
-
etlplus/database/types.py,sha256=
|
|
49
|
+
etlplus/database/types.py,sha256=_pkQyC14TzAlgyeIqZG4F5LWYknZbHw3TW68Auk7Ya0,795
|
|
50
50
|
etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
|
|
51
51
|
etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
|
|
52
52
|
etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
|
|
53
53
|
etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
|
|
54
54
|
etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
|
|
55
|
-
etlplus-0.
|
|
56
|
-
etlplus-0.
|
|
57
|
-
etlplus-0.
|
|
58
|
-
etlplus-0.
|
|
59
|
-
etlplus-0.
|
|
60
|
-
etlplus-0.
|
|
55
|
+
etlplus-0.8.0.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
|
|
56
|
+
etlplus-0.8.0.dist-info/METADATA,sha256=VjENPm-4JBGKuJ_NIjRLjce135eB7_ZTOFY6ZmAdS30,19328
|
|
57
|
+
etlplus-0.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
58
|
+
etlplus-0.8.0.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
59
|
+
etlplus-0.8.0.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
60
|
+
etlplus-0.8.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|