etlplus 0.5.4__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/cli/app.py +16 -16
- etlplus/cli/handlers.py +21 -22
- etlplus/cli/main.py +16 -16
- etlplus/database/__init__.py +23 -0
- etlplus/{ddl.py → database/ddl.py} +160 -46
- etlplus/run.py +2 -4
- {etlplus-0.5.4.dist-info → etlplus-0.6.1.dist-info}/METADATA +1 -1
- {etlplus-0.5.4.dist-info → etlplus-0.6.1.dist-info}/RECORD +12 -11
- {etlplus-0.5.4.dist-info → etlplus-0.6.1.dist-info}/WHEEL +0 -0
- {etlplus-0.5.4.dist-info → etlplus-0.6.1.dist-info}/entry_points.txt +0 -0
- {etlplus-0.5.4.dist-info → etlplus-0.6.1.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.5.4.dist-info → etlplus-0.6.1.dist-info}/top_level.txt +0 -0
etlplus/cli/app.py
CHANGED
|
@@ -57,14 +57,14 @@ from .. import __version__
|
|
|
57
57
|
from ..enums import DataConnectorType
|
|
58
58
|
from ..enums import FileFormat
|
|
59
59
|
from ..utils import json_type
|
|
60
|
-
from .handlers import
|
|
61
|
-
from .handlers import
|
|
62
|
-
from .handlers import
|
|
63
|
-
from .handlers import
|
|
64
|
-
from .handlers import
|
|
65
|
-
from .handlers import
|
|
66
|
-
from .handlers import
|
|
67
|
-
from .handlers import
|
|
60
|
+
from .handlers import check_handler
|
|
61
|
+
from .handlers import extract_handler
|
|
62
|
+
from .handlers import load_handler
|
|
63
|
+
from .handlers import pipeline_handler
|
|
64
|
+
from .handlers import render_handler
|
|
65
|
+
from .handlers import run_handler
|
|
66
|
+
from .handlers import transform_handler
|
|
67
|
+
from .handlers import validate_handler
|
|
68
68
|
|
|
69
69
|
# SECTION: EXPORTS ========================================================== #
|
|
70
70
|
|
|
@@ -819,7 +819,7 @@ def check_cmd(
|
|
|
819
819
|
targets=targets,
|
|
820
820
|
transforms=transforms,
|
|
821
821
|
)
|
|
822
|
-
return int(
|
|
822
|
+
return int(check_handler(ns))
|
|
823
823
|
|
|
824
824
|
|
|
825
825
|
@app.command('extract')
|
|
@@ -905,7 +905,7 @@ def extract_cmd(
|
|
|
905
905
|
source=resolved_source,
|
|
906
906
|
**format_kwargs,
|
|
907
907
|
)
|
|
908
|
-
return int(
|
|
908
|
+
return int(extract_handler(ns))
|
|
909
909
|
|
|
910
910
|
|
|
911
911
|
@app.command('load')
|
|
@@ -1007,7 +1007,7 @@ def load_cmd(
|
|
|
1007
1007
|
target=resolved_target,
|
|
1008
1008
|
**format_kwargs,
|
|
1009
1009
|
)
|
|
1010
|
-
return int(
|
|
1010
|
+
return int(load_handler(ns))
|
|
1011
1011
|
|
|
1012
1012
|
|
|
1013
1013
|
@app.command('pipeline')
|
|
@@ -1061,7 +1061,7 @@ def pipeline_cmd(
|
|
|
1061
1061
|
list=jobs,
|
|
1062
1062
|
run=run_target,
|
|
1063
1063
|
)
|
|
1064
|
-
return int(
|
|
1064
|
+
return int(pipeline_handler(ns))
|
|
1065
1065
|
|
|
1066
1066
|
|
|
1067
1067
|
@app.command('render')
|
|
@@ -1110,7 +1110,7 @@ def render_cmd(
|
|
|
1110
1110
|
template_path=template_path,
|
|
1111
1111
|
output=output,
|
|
1112
1112
|
)
|
|
1113
|
-
return int(
|
|
1113
|
+
return int(render_handler(ns))
|
|
1114
1114
|
|
|
1115
1115
|
|
|
1116
1116
|
@app.command('run')
|
|
@@ -1157,7 +1157,7 @@ def run_cmd(
|
|
|
1157
1157
|
job=job,
|
|
1158
1158
|
pipeline=pipeline,
|
|
1159
1159
|
)
|
|
1160
|
-
return int(
|
|
1160
|
+
return int(run_handler(ns))
|
|
1161
1161
|
|
|
1162
1162
|
|
|
1163
1163
|
@app.command('transform')
|
|
@@ -1294,7 +1294,7 @@ def transform_cmd(
|
|
|
1294
1294
|
target_format=target_format_kwargs['format'],
|
|
1295
1295
|
**target_format_kwargs,
|
|
1296
1296
|
)
|
|
1297
|
-
return int(
|
|
1297
|
+
return int(transform_handler(ns))
|
|
1298
1298
|
|
|
1299
1299
|
|
|
1300
1300
|
@app.command('validate')
|
|
@@ -1364,4 +1364,4 @@ def validate_cmd(
|
|
|
1364
1364
|
source_format=source_format,
|
|
1365
1365
|
**source_format_kwargs,
|
|
1366
1366
|
)
|
|
1367
|
-
return int(
|
|
1367
|
+
return int(validate_handler(ns))
|
etlplus/cli/handlers.py
CHANGED
|
@@ -18,8 +18,8 @@ from typing import cast
|
|
|
18
18
|
|
|
19
19
|
from ..config import PipelineConfig
|
|
20
20
|
from ..config import load_pipeline_config
|
|
21
|
-
from ..
|
|
22
|
-
from ..
|
|
21
|
+
from ..database import load_table_spec
|
|
22
|
+
from ..database import render_tables
|
|
23
23
|
from ..enums import FileFormat
|
|
24
24
|
from ..extract import extract
|
|
25
25
|
from ..file import File
|
|
@@ -36,14 +36,14 @@ from ..validate import validate
|
|
|
36
36
|
|
|
37
37
|
__all__ = [
|
|
38
38
|
# Functions
|
|
39
|
-
'
|
|
40
|
-
'
|
|
41
|
-
'
|
|
42
|
-
'
|
|
43
|
-
'
|
|
44
|
-
'
|
|
45
|
-
'
|
|
46
|
-
'
|
|
39
|
+
'extract_handler',
|
|
40
|
+
'check_handler',
|
|
41
|
+
'load_handler',
|
|
42
|
+
'pipeline_handler',
|
|
43
|
+
'render_handler',
|
|
44
|
+
'run_handler',
|
|
45
|
+
'transform_handler',
|
|
46
|
+
'validate_handler',
|
|
47
47
|
]
|
|
48
48
|
|
|
49
49
|
|
|
@@ -423,7 +423,7 @@ def _write_json_output(
|
|
|
423
423
|
# SECTION: FUNCTIONS ======================================================== #
|
|
424
424
|
|
|
425
425
|
|
|
426
|
-
def
|
|
426
|
+
def check_handler(
|
|
427
427
|
args: argparse.Namespace,
|
|
428
428
|
) -> int:
|
|
429
429
|
"""
|
|
@@ -448,7 +448,7 @@ def cmd_check(
|
|
|
448
448
|
return 0
|
|
449
449
|
|
|
450
450
|
|
|
451
|
-
def
|
|
451
|
+
def extract_handler(
|
|
452
452
|
args: argparse.Namespace,
|
|
453
453
|
) -> int:
|
|
454
454
|
"""
|
|
@@ -493,7 +493,7 @@ def cmd_extract(
|
|
|
493
493
|
return 0
|
|
494
494
|
|
|
495
495
|
|
|
496
|
-
def
|
|
496
|
+
def load_handler(
|
|
497
497
|
args: argparse.Namespace,
|
|
498
498
|
) -> int:
|
|
499
499
|
"""
|
|
@@ -552,7 +552,7 @@ def cmd_load(
|
|
|
552
552
|
return 0
|
|
553
553
|
|
|
554
554
|
|
|
555
|
-
def
|
|
555
|
+
def pipeline_handler(
|
|
556
556
|
args: argparse.Namespace,
|
|
557
557
|
) -> int:
|
|
558
558
|
"""
|
|
@@ -596,12 +596,11 @@ def cmd_pipeline(
|
|
|
596
596
|
return 0
|
|
597
597
|
|
|
598
598
|
|
|
599
|
-
def
|
|
599
|
+
def render_handler(
|
|
600
600
|
args: argparse.Namespace,
|
|
601
601
|
) -> int:
|
|
602
602
|
"""Render SQL DDL statements from table schema specs."""
|
|
603
|
-
|
|
604
|
-
_pretty, quiet = _presentation_flags(args)
|
|
603
|
+
_, quiet = _presentation_flags(args)
|
|
605
604
|
|
|
606
605
|
template_value = getattr(args, 'template', 'ddl') or 'ddl'
|
|
607
606
|
template_path = getattr(args, 'template_path', None)
|
|
@@ -657,7 +656,7 @@ def cmd_render(
|
|
|
657
656
|
return 0
|
|
658
657
|
|
|
659
658
|
|
|
660
|
-
def
|
|
659
|
+
def run_handler(
|
|
661
660
|
args: argparse.Namespace,
|
|
662
661
|
) -> int:
|
|
663
662
|
"""
|
|
@@ -685,7 +684,7 @@ def cmd_run(
|
|
|
685
684
|
return 0
|
|
686
685
|
|
|
687
686
|
|
|
688
|
-
def
|
|
687
|
+
def transform_handler(
|
|
689
688
|
args: argparse.Namespace,
|
|
690
689
|
) -> int:
|
|
691
690
|
"""
|
|
@@ -701,7 +700,7 @@ def cmd_transform(
|
|
|
701
700
|
int
|
|
702
701
|
Zero on success.
|
|
703
702
|
"""
|
|
704
|
-
pretty,
|
|
703
|
+
pretty, _ = _presentation_flags(args)
|
|
705
704
|
format_hint: str | None = getattr(args, 'source_format', None)
|
|
706
705
|
format_explicit: bool = format_hint is not None
|
|
707
706
|
|
|
@@ -726,7 +725,7 @@ def cmd_transform(
|
|
|
726
725
|
return 0
|
|
727
726
|
|
|
728
727
|
|
|
729
|
-
def
|
|
728
|
+
def validate_handler(
|
|
730
729
|
args: argparse.Namespace,
|
|
731
730
|
) -> int:
|
|
732
731
|
"""
|
|
@@ -742,7 +741,7 @@ def cmd_validate(
|
|
|
742
741
|
int
|
|
743
742
|
Zero on success.
|
|
744
743
|
"""
|
|
745
|
-
pretty,
|
|
744
|
+
pretty, _ = _presentation_flags(args)
|
|
746
745
|
format_explicit: bool = getattr(args, '_format_explicit', False)
|
|
747
746
|
format_hint: str | None = getattr(args, 'source_format', None)
|
|
748
747
|
payload = cast(
|
etlplus/cli/main.py
CHANGED
|
@@ -24,14 +24,14 @@ from ..enums import FileFormat
|
|
|
24
24
|
from ..utils import json_type
|
|
25
25
|
from .app import PROJECT_URL
|
|
26
26
|
from .app import app
|
|
27
|
-
from .handlers import
|
|
28
|
-
from .handlers import
|
|
29
|
-
from .handlers import
|
|
30
|
-
from .handlers import
|
|
31
|
-
from .handlers import
|
|
32
|
-
from .handlers import
|
|
33
|
-
from .handlers import
|
|
34
|
-
from .handlers import
|
|
27
|
+
from .handlers import check_handler
|
|
28
|
+
from .handlers import extract_handler
|
|
29
|
+
from .handlers import load_handler
|
|
30
|
+
from .handlers import pipeline_handler
|
|
31
|
+
from .handlers import render_handler
|
|
32
|
+
from .handlers import run_handler
|
|
33
|
+
from .handlers import transform_handler
|
|
34
|
+
from .handlers import validate_handler
|
|
35
35
|
|
|
36
36
|
# SECTION: EXPORTS ========================================================== #
|
|
37
37
|
|
|
@@ -329,7 +329,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
329
329
|
),
|
|
330
330
|
)
|
|
331
331
|
_add_format_options(extract_parser, context='source')
|
|
332
|
-
extract_parser.set_defaults(func=
|
|
332
|
+
extract_parser.set_defaults(func=extract_handler)
|
|
333
333
|
|
|
334
334
|
validate_parser = subparsers.add_parser(
|
|
335
335
|
'validate',
|
|
@@ -346,7 +346,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
346
346
|
default={},
|
|
347
347
|
help='Validation rules as JSON string',
|
|
348
348
|
)
|
|
349
|
-
validate_parser.set_defaults(func=
|
|
349
|
+
validate_parser.set_defaults(func=validate_handler)
|
|
350
350
|
|
|
351
351
|
transform_parser = subparsers.add_parser(
|
|
352
352
|
'transform',
|
|
@@ -394,7 +394,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
394
394
|
'File targets infer format from the extension.'
|
|
395
395
|
),
|
|
396
396
|
)
|
|
397
|
-
transform_parser.set_defaults(func=
|
|
397
|
+
transform_parser.set_defaults(func=transform_handler)
|
|
398
398
|
|
|
399
399
|
load_parser = subparsers.add_parser(
|
|
400
400
|
'load',
|
|
@@ -418,7 +418,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
418
418
|
),
|
|
419
419
|
)
|
|
420
420
|
_add_format_options(load_parser, context='target')
|
|
421
|
-
load_parser.set_defaults(func=
|
|
421
|
+
load_parser.set_defaults(func=load_handler)
|
|
422
422
|
|
|
423
423
|
pipe_parser = subparsers.add_parser(
|
|
424
424
|
'pipeline',
|
|
@@ -440,7 +440,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
440
440
|
metavar='JOB',
|
|
441
441
|
help='Run a specific job by name',
|
|
442
442
|
)
|
|
443
|
-
pipe_parser.set_defaults(func=
|
|
443
|
+
pipe_parser.set_defaults(func=pipeline_handler)
|
|
444
444
|
|
|
445
445
|
render_parser = subparsers.add_parser(
|
|
446
446
|
'render',
|
|
@@ -476,7 +476,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
476
476
|
'Explicit path to a Jinja template file (overrides template key).'
|
|
477
477
|
),
|
|
478
478
|
)
|
|
479
|
-
render_parser.set_defaults(func=
|
|
479
|
+
render_parser.set_defaults(func=render_handler)
|
|
480
480
|
|
|
481
481
|
check_parser = subparsers.add_parser(
|
|
482
482
|
'check',
|
|
@@ -516,7 +516,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
516
516
|
name='transforms',
|
|
517
517
|
help_text='List data transforms',
|
|
518
518
|
)
|
|
519
|
-
check_parser.set_defaults(func=
|
|
519
|
+
check_parser.set_defaults(func=check_handler)
|
|
520
520
|
|
|
521
521
|
run_parser = subparsers.add_parser(
|
|
522
522
|
'run',
|
|
@@ -537,7 +537,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
537
537
|
'--pipeline',
|
|
538
538
|
help='Name of the pipeline to run',
|
|
539
539
|
)
|
|
540
|
-
run_parser.set_defaults(func=
|
|
540
|
+
run_parser.set_defaults(func=run_handler)
|
|
541
541
|
|
|
542
542
|
return parser
|
|
543
543
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""
|
|
2
|
+
:mod:`etlplus.database` package.
|
|
3
|
+
|
|
4
|
+
This package defines database-related utilities for ETLPlus, including:
|
|
5
|
+
- DDL rendering and schema management.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from .ddl import load_table_spec
|
|
11
|
+
from .ddl import render_table_sql
|
|
12
|
+
from .ddl import render_tables
|
|
13
|
+
from .ddl import render_tables_to_string
|
|
14
|
+
|
|
15
|
+
# SECTION: EXPORTS ========================================================== #
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
'load_table_spec',
|
|
20
|
+
'render_table_sql',
|
|
21
|
+
'render_tables',
|
|
22
|
+
'render_tables_to_string',
|
|
23
|
+
]
|
|
@@ -11,18 +11,20 @@ can emit DDLs without shelling out to that script.
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
13
|
import importlib.resources
|
|
14
|
-
import json
|
|
15
14
|
import os
|
|
16
15
|
from collections.abc import Iterable
|
|
17
16
|
from collections.abc import Mapping
|
|
18
17
|
from pathlib import Path
|
|
19
18
|
from typing import Any
|
|
19
|
+
from typing import Final
|
|
20
20
|
|
|
21
21
|
from jinja2 import DictLoader
|
|
22
22
|
from jinja2 import Environment
|
|
23
23
|
from jinja2 import FileSystemLoader
|
|
24
24
|
from jinja2 import StrictUndefined
|
|
25
25
|
|
|
26
|
+
from ..file import File
|
|
27
|
+
|
|
26
28
|
# SECTION: EXPORTS ========================================================== #
|
|
27
29
|
|
|
28
30
|
|
|
@@ -31,13 +33,26 @@ __all__ = [
|
|
|
31
33
|
'load_table_spec',
|
|
32
34
|
'render_table_sql',
|
|
33
35
|
'render_tables',
|
|
36
|
+
'render_tables_to_string',
|
|
34
37
|
]
|
|
35
38
|
|
|
36
39
|
|
|
40
|
+
# SECTION: INTERNAL CONSTANTS =============================================== #
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
_SUPPORTED_SPEC_SUFFIXES: Final[frozenset[str]] = frozenset(
|
|
44
|
+
{
|
|
45
|
+
'.json',
|
|
46
|
+
'.yml',
|
|
47
|
+
'.yaml',
|
|
48
|
+
},
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
37
52
|
# SECTION: CONSTANTS ======================================================== #
|
|
38
53
|
|
|
39
54
|
|
|
40
|
-
TEMPLATES = {
|
|
55
|
+
TEMPLATES: Final[dict[str, str]] = {
|
|
41
56
|
'ddl': 'ddl.sql.j2',
|
|
42
57
|
'view': 'view.sql.j2',
|
|
43
58
|
}
|
|
@@ -46,12 +61,68 @@ TEMPLATES = {
|
|
|
46
61
|
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
47
62
|
|
|
48
63
|
|
|
49
|
-
def
|
|
64
|
+
def _load_template_text(
|
|
65
|
+
filename: str,
|
|
66
|
+
) -> str:
|
|
67
|
+
"""Return the bundled template text.
|
|
68
|
+
|
|
69
|
+
Parameters
|
|
70
|
+
----------
|
|
71
|
+
filename : str
|
|
72
|
+
Template filename located inside the package data folder.
|
|
73
|
+
|
|
74
|
+
Returns
|
|
75
|
+
-------
|
|
76
|
+
str
|
|
77
|
+
Raw template contents.
|
|
78
|
+
|
|
79
|
+
Raises
|
|
80
|
+
------
|
|
81
|
+
FileNotFoundError
|
|
82
|
+
If the template file cannot be located in package data.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
return (
|
|
87
|
+
importlib.resources.files(
|
|
88
|
+
'etlplus.templates',
|
|
89
|
+
)
|
|
90
|
+
.joinpath(filename)
|
|
91
|
+
.read_text(encoding='utf-8')
|
|
92
|
+
)
|
|
93
|
+
except FileNotFoundError as exc: # pragma: no cover - deployment guard
|
|
94
|
+
raise FileNotFoundError(
|
|
95
|
+
f'Could not load template {filename} '
|
|
96
|
+
f'from etlplus.templates package data.',
|
|
97
|
+
) from exc
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _resolve_template(
|
|
50
101
|
*,
|
|
51
102
|
template_key: str | None,
|
|
52
103
|
template_path: str | None,
|
|
53
|
-
) -> Environment:
|
|
54
|
-
"""Return
|
|
104
|
+
) -> tuple[Environment, str]:
|
|
105
|
+
"""Return environment and template name for rendering.
|
|
106
|
+
|
|
107
|
+
Parameters
|
|
108
|
+
----------
|
|
109
|
+
template_key : str | None
|
|
110
|
+
Named template key bundled with the package.
|
|
111
|
+
template_path : str | None
|
|
112
|
+
Explicit template file override.
|
|
113
|
+
|
|
114
|
+
Returns
|
|
115
|
+
-------
|
|
116
|
+
tuple[Environment, str]
|
|
117
|
+
Pair of configured Jinja environment and the template identifier.
|
|
118
|
+
|
|
119
|
+
Raises
|
|
120
|
+
------
|
|
121
|
+
FileNotFoundError
|
|
122
|
+
If the provided template path does not exist.
|
|
123
|
+
ValueError
|
|
124
|
+
If the template key is unknown.
|
|
125
|
+
"""
|
|
55
126
|
file_override = template_path or os.environ.get('TEMPLATE_NAME')
|
|
56
127
|
if file_override:
|
|
57
128
|
path = Path(file_override)
|
|
@@ -64,8 +135,7 @@ def _build_env(
|
|
|
64
135
|
trim_blocks=True,
|
|
65
136
|
lstrip_blocks=True,
|
|
66
137
|
)
|
|
67
|
-
env
|
|
68
|
-
return env
|
|
138
|
+
return env, path.name
|
|
69
139
|
|
|
70
140
|
key = (template_key or 'ddl').strip()
|
|
71
141
|
if key not in TEMPLATES:
|
|
@@ -84,51 +154,59 @@ def _build_env(
|
|
|
84
154
|
trim_blocks=True,
|
|
85
155
|
lstrip_blocks=True,
|
|
86
156
|
)
|
|
87
|
-
env
|
|
88
|
-
return env
|
|
157
|
+
return env, key
|
|
89
158
|
|
|
90
159
|
|
|
91
|
-
|
|
92
|
-
"""Return the raw template text bundled with the package."""
|
|
160
|
+
# SECTION: FUNCTIONS ======================================================== #
|
|
93
161
|
|
|
94
|
-
try:
|
|
95
|
-
return (
|
|
96
|
-
importlib.resources.files(
|
|
97
|
-
'etlplus.templates',
|
|
98
|
-
)
|
|
99
|
-
.joinpath(filename)
|
|
100
|
-
.read_text(encoding='utf-8')
|
|
101
|
-
)
|
|
102
|
-
except FileNotFoundError as exc: # pragma: no cover - deployment guard
|
|
103
|
-
raise FileNotFoundError(
|
|
104
|
-
f'Could not load template {filename} '
|
|
105
|
-
f'from etlplus.templates package data.',
|
|
106
|
-
) from exc
|
|
107
162
|
|
|
163
|
+
def load_table_spec(
|
|
164
|
+
path: Path | str,
|
|
165
|
+
) -> dict[str, Any]:
|
|
166
|
+
"""
|
|
167
|
+
Load a table specification from disk.
|
|
108
168
|
|
|
109
|
-
|
|
169
|
+
Parameters
|
|
170
|
+
----------
|
|
171
|
+
path : Path | str
|
|
172
|
+
Path to the JSON or YAML specification file.
|
|
110
173
|
|
|
174
|
+
Returns
|
|
175
|
+
-------
|
|
176
|
+
dict[str, Any]
|
|
177
|
+
Parsed table specification mapping.
|
|
111
178
|
|
|
112
|
-
|
|
113
|
-
|
|
179
|
+
Raises
|
|
180
|
+
------
|
|
181
|
+
ImportError
|
|
182
|
+
If the file cannot be read due to missing dependencies.
|
|
183
|
+
RuntimeError
|
|
184
|
+
If the YAML dependency is missing for YAML specs.
|
|
185
|
+
TypeError
|
|
186
|
+
If the loaded spec is not a mapping.
|
|
187
|
+
ValueError
|
|
188
|
+
If the file suffix is not supported.
|
|
189
|
+
"""
|
|
114
190
|
|
|
115
191
|
spec_path = Path(path)
|
|
116
|
-
text = spec_path.read_text(encoding='utf-8')
|
|
117
192
|
suffix = spec_path.suffix.lower()
|
|
118
193
|
|
|
119
|
-
if suffix
|
|
120
|
-
|
|
194
|
+
if suffix not in _SUPPORTED_SPEC_SUFFIXES:
|
|
195
|
+
raise ValueError('Spec must be .json, .yml, or .yaml')
|
|
121
196
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
197
|
+
try:
|
|
198
|
+
spec = File.read_file(spec_path)
|
|
199
|
+
except ImportError as e:
|
|
200
|
+
if suffix in {'.yml', '.yaml'}:
|
|
126
201
|
raise RuntimeError(
|
|
127
202
|
'Missing dependency: pyyaml is required for YAML specs.',
|
|
128
|
-
) from
|
|
129
|
-
|
|
203
|
+
) from e
|
|
204
|
+
raise
|
|
130
205
|
|
|
131
|
-
|
|
206
|
+
if not isinstance(spec, Mapping):
|
|
207
|
+
raise TypeError('Table spec must be a mapping')
|
|
208
|
+
|
|
209
|
+
return dict(spec)
|
|
132
210
|
|
|
133
211
|
|
|
134
212
|
def render_table_sql(
|
|
@@ -153,16 +231,11 @@ def render_table_sql(
|
|
|
153
231
|
-------
|
|
154
232
|
str
|
|
155
233
|
Rendered SQL string.
|
|
156
|
-
|
|
157
|
-
Raises
|
|
158
|
-
------
|
|
159
|
-
TypeError
|
|
160
|
-
If the loaded template name is not a string.
|
|
161
234
|
"""
|
|
162
|
-
env =
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
235
|
+
env, template_name = _resolve_template(
|
|
236
|
+
template_key=template,
|
|
237
|
+
template_path=template_path,
|
|
238
|
+
)
|
|
166
239
|
tmpl = env.get_template(template_name)
|
|
167
240
|
return tmpl.render(spec=spec).rstrip() + '\n'
|
|
168
241
|
|
|
@@ -195,3 +268,44 @@ def render_tables(
|
|
|
195
268
|
render_table_sql(spec, template=template, template_path=template_path)
|
|
196
269
|
for spec in specs
|
|
197
270
|
]
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def render_tables_to_string(
|
|
274
|
+
spec_paths: Iterable[Path | str],
|
|
275
|
+
*,
|
|
276
|
+
template: str | None = 'ddl',
|
|
277
|
+
template_path: Path | str | None = None,
|
|
278
|
+
) -> str:
|
|
279
|
+
"""
|
|
280
|
+
Render one or more specs and concatenate the SQL payloads.
|
|
281
|
+
|
|
282
|
+
Parameters
|
|
283
|
+
----------
|
|
284
|
+
spec_paths : Iterable[Path | str]
|
|
285
|
+
Paths to table specification files.
|
|
286
|
+
template : str | None, optional
|
|
287
|
+
Template key bundled with ETLPlus. Defaults to ``'ddl'``.
|
|
288
|
+
template_path : Path | str | None, optional
|
|
289
|
+
Custom Jinja template to override the bundled templates.
|
|
290
|
+
|
|
291
|
+
Returns
|
|
292
|
+
-------
|
|
293
|
+
str
|
|
294
|
+
Concatenated SQL payload suitable for writing to disk or stdout.
|
|
295
|
+
"""
|
|
296
|
+
|
|
297
|
+
resolved_template_path = (
|
|
298
|
+
str(template_path) if template_path is not None else None
|
|
299
|
+
)
|
|
300
|
+
rendered_sql: list[str] = []
|
|
301
|
+
for spec_path in spec_paths:
|
|
302
|
+
spec = load_table_spec(spec_path)
|
|
303
|
+
rendered_sql.append(
|
|
304
|
+
render_table_sql(
|
|
305
|
+
spec,
|
|
306
|
+
template=template,
|
|
307
|
+
template_path=resolved_template_path,
|
|
308
|
+
),
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
return ''.join(rendered_sql)
|
etlplus/run.py
CHANGED
|
@@ -142,10 +142,8 @@ def run(
|
|
|
142
142
|
"""
|
|
143
143
|
Run a pipeline job defined in a YAML configuration.
|
|
144
144
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
from ``in/pipeline.yml``, but callers can provide an explicit
|
|
148
|
-
``config_path`` to override this.
|
|
145
|
+
By default it reads the configuration from ``in/pipeline.yml``, but callers
|
|
146
|
+
can provide an explicit ``config_path`` to override this.
|
|
149
147
|
|
|
150
148
|
Parameters
|
|
151
149
|
----------
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
etlplus/__init__.py,sha256=M2gScnyir6WOMAh_EuoQIiAzdcTls0_5hbd_Q6of8I0,1021
|
|
2
2
|
etlplus/__main__.py,sha256=btoROneNiigyfBU7BSzPKZ1R9gzBMpxcpsbPwmuHwTM,479
|
|
3
3
|
etlplus/__version__.py,sha256=1E0GMK_yUWCMQFKxXjTvyMwofi0qT2k4CDNiHWiymWE,327
|
|
4
|
-
etlplus/ddl.py,sha256=uYkiMTx1uDlUypnXCYy0K5ARnHRMHFVzzg8PizBQRLg,5306
|
|
5
4
|
etlplus/enums.py,sha256=V_j18Ud2BCXpFsBk2pZGrvCVrvAMJ7uja1z9fppFGso,10175
|
|
6
5
|
etlplus/extract.py,sha256=f44JdHhNTACxgn44USx05paKTwq7LQY-V4wANCW9hVM,6173
|
|
7
6
|
etlplus/file.py,sha256=RxIAsGDN4f_vNA2B5-ct88JNd_ISAyYbooIRE5DstS8,17972
|
|
8
7
|
etlplus/load.py,sha256=BwF3gT4gIr-5CvNMz_aLTCl-w2ihWSTxNVd4X92XFwI,8737
|
|
9
8
|
etlplus/mixins.py,sha256=ifGpHwWv7U00yqGf-kN93vJax2IiK4jaGtTsPsO3Oak,1350
|
|
10
9
|
etlplus/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
-
etlplus/run.py,sha256=
|
|
10
|
+
etlplus/run.py,sha256=X4kp5FQlIWVf1_d9oSrchKau7BFDCE1Zkscvu7WPaWw,12340
|
|
12
11
|
etlplus/run_helpers.py,sha256=bj6MkaeFxjl3CeKG1HoXKx5DwAlXNERVW-GX-z1P_qQ,24373
|
|
13
12
|
etlplus/transform.py,sha256=uAUVDDHYCgx7GpVez9IK3OAZM-CnCuMa9iox3vwGGJA,25296
|
|
14
13
|
etlplus/types.py,sha256=SJiZ7wJiSnV4CEvF-9E5nSFLBo4DT9OqHQqj1GSHkv8,6042
|
|
@@ -32,9 +31,9 @@ etlplus/api/rate_limiting/__init__.py,sha256=ZySB1dZettEDnWvI1EHf_TZ9L08M_kKsNR-
|
|
|
32
31
|
etlplus/api/rate_limiting/config.py,sha256=2b4wIynblN-1EyMqI4aXa71SljzSjXYh5N1Nngr3jOg,9406
|
|
33
32
|
etlplus/api/rate_limiting/rate_limiter.py,sha256=Uxozqd_Ej5Lsj-M-mLT2WexChgWh7x35_YP10yqYPQA,7159
|
|
34
33
|
etlplus/cli/__init__.py,sha256=J97-Rv931IL1_b4AXnB7Fbbd7HKnHBpx18NQfC_kE6c,299
|
|
35
|
-
etlplus/cli/app.py,sha256=
|
|
36
|
-
etlplus/cli/handlers.py,sha256=
|
|
37
|
-
etlplus/cli/main.py,sha256=
|
|
34
|
+
etlplus/cli/app.py,sha256=SYPO-NDwXgymJrACw39jZ_NJrSKAs0O8anuWR5o42WM,35893
|
|
35
|
+
etlplus/cli/handlers.py,sha256=nFMvqHQhJ8kJZPisDCiUHeOhjlqAO6hJvRjXiJTcU74,18951
|
|
36
|
+
etlplus/cli/main.py,sha256=ijYOy72SEsxrTEBan5yADW8CZyr0yddVF8HeMgFw6Zg,16576
|
|
38
37
|
etlplus/config/__init__.py,sha256=VZWzOg7d2YR9NT6UwKTv44yf2FRUMjTHynkm1Dl5Qzo,1486
|
|
39
38
|
etlplus/config/connector.py,sha256=0-TIwevHbKRHVmucvyGpPd-3tB1dKHB-dj0yJ6kq5eY,9809
|
|
40
39
|
etlplus/config/jobs.py,sha256=hmzRCqt0OvCEZZR4ONKrd3lvSv0OmayjLc4yOBk3ug8,7399
|
|
@@ -42,14 +41,16 @@ etlplus/config/pipeline.py,sha256=Va4MQY6KEyKqHGMKPmh09ZcGpx95br-iNUjpkqtzVbw,95
|
|
|
42
41
|
etlplus/config/profile.py,sha256=Ss2zedQGjkaGSpvBLTD4SZaWViMJ7TJPLB8Q2_BTpPg,1898
|
|
43
42
|
etlplus/config/types.py,sha256=a0epJ3z16HQ5bY3Ctf8s_cQPa3f0HHcwdOcjCP2xoG4,4954
|
|
44
43
|
etlplus/config/utils.py,sha256=4SUHMkt5bKBhMhiJm-DrnmE2Q4TfOgdNCKz8PJDS27o,3443
|
|
44
|
+
etlplus/database/__init__.py,sha256=MNaqpiPNTMbwbHxdh865GXS3q4H4dkAn2YLl3GFQU8E,525
|
|
45
|
+
etlplus/database/ddl.py,sha256=Hvg1PwwaIMU3y8emMWml4CzvQGvvg6KZfsHo3-EWJjg,7738
|
|
45
46
|
etlplus/templates/__init__.py,sha256=tsniN7XJYs3NwYxJ6c2HD5upHP3CDkLx-bQCMt97UOM,106
|
|
46
47
|
etlplus/templates/ddl.sql.j2,sha256=s8fMWvcb4eaJVXkifuib1aQPljtZ8buuyB_uA-ZdU3Q,4734
|
|
47
48
|
etlplus/templates/view.sql.j2,sha256=Iy8DHfhq5yyvrUKDxqp_aHIEXY4Tm6j4wT7YDEFWAhk,2180
|
|
48
49
|
etlplus/validation/__init__.py,sha256=Pe5Xg1_EA4uiNZGYu5WTF3j7odjmyxnAJ8rcioaplSQ,1254
|
|
49
50
|
etlplus/validation/utils.py,sha256=Mtqg449VIke0ziy_wd2r6yrwJzQkA1iulZC87FzXMjo,10201
|
|
50
|
-
etlplus-0.
|
|
51
|
-
etlplus-0.
|
|
52
|
-
etlplus-0.
|
|
53
|
-
etlplus-0.
|
|
54
|
-
etlplus-0.
|
|
55
|
-
etlplus-0.
|
|
51
|
+
etlplus-0.6.1.dist-info/licenses/LICENSE,sha256=MuNO63i6kWmgnV2pbP2SLqP54mk1BGmu7CmbtxMmT-U,1069
|
|
52
|
+
etlplus-0.6.1.dist-info/METADATA,sha256=NX1ADs4_MZ3J9QLGj7bPRyXc43HJJxfOzfcmIg3VYHc,19288
|
|
53
|
+
etlplus-0.6.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
54
|
+
etlplus-0.6.1.dist-info/entry_points.txt,sha256=6w-2-jzuPa55spzK34h-UKh2JTEShh38adFRONNP9QE,45
|
|
55
|
+
etlplus-0.6.1.dist-info/top_level.txt,sha256=aWWF-udn_sLGuHTM6W6MLh99ArS9ROkUWO8Mi8y1_2U,8
|
|
56
|
+
etlplus-0.6.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|