etlplus 0.9.2__py3-none-any.whl → 0.10.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/__init__.py +26 -1
- etlplus/api/README.md +3 -51
- etlplus/api/__init__.py +0 -10
- etlplus/api/config.py +28 -39
- etlplus/api/endpoint_client.py +3 -3
- etlplus/api/pagination/client.py +1 -1
- etlplus/api/rate_limiting/config.py +1 -13
- etlplus/api/rate_limiting/rate_limiter.py +11 -8
- etlplus/api/request_manager.py +6 -11
- etlplus/api/transport.py +2 -14
- etlplus/api/types.py +6 -96
- etlplus/cli/commands.py +43 -76
- etlplus/cli/constants.py +1 -1
- etlplus/cli/handlers.py +12 -40
- etlplus/cli/io.py +2 -2
- etlplus/cli/main.py +1 -1
- etlplus/cli/state.py +7 -4
- etlplus/{workflow → config}/__init__.py +23 -10
- etlplus/{workflow → config}/connector.py +44 -58
- etlplus/{workflow → config}/jobs.py +32 -105
- etlplus/{workflow → config}/pipeline.py +51 -59
- etlplus/{workflow → config}/profile.py +5 -8
- etlplus/config/types.py +204 -0
- etlplus/config/utils.py +120 -0
- etlplus/database/ddl.py +1 -1
- etlplus/database/engine.py +3 -19
- etlplus/database/orm.py +0 -2
- etlplus/database/schema.py +1 -1
- etlplus/enums.py +266 -0
- etlplus/{ops/extract.py → extract.py} +99 -81
- etlplus/file.py +652 -0
- etlplus/{ops/load.py → load.py} +101 -78
- etlplus/{ops/run.py → run.py} +127 -159
- etlplus/{api/utils.py → run_helpers.py} +153 -209
- etlplus/{ops/transform.py → transform.py} +68 -75
- etlplus/types.py +4 -5
- etlplus/utils.py +2 -136
- etlplus/{ops/validate.py → validate.py} +12 -22
- etlplus/validation/__init__.py +44 -0
- etlplus/{ops → validation}/utils.py +17 -53
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/METADATA +17 -210
- etlplus-0.10.1.dist-info/RECORD +65 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/WHEEL +1 -1
- etlplus/README.md +0 -37
- etlplus/api/enums.py +0 -51
- etlplus/cli/README.md +0 -40
- etlplus/database/README.md +0 -48
- etlplus/file/README.md +0 -105
- etlplus/file/__init__.py +0 -25
- etlplus/file/_imports.py +0 -141
- etlplus/file/_io.py +0 -160
- etlplus/file/accdb.py +0 -78
- etlplus/file/arrow.py +0 -78
- etlplus/file/avro.py +0 -176
- etlplus/file/bson.py +0 -77
- etlplus/file/cbor.py +0 -78
- etlplus/file/cfg.py +0 -79
- etlplus/file/conf.py +0 -80
- etlplus/file/core.py +0 -322
- etlplus/file/csv.py +0 -79
- etlplus/file/dat.py +0 -78
- etlplus/file/dta.py +0 -77
- etlplus/file/duckdb.py +0 -78
- etlplus/file/enums.py +0 -343
- etlplus/file/feather.py +0 -111
- etlplus/file/fwf.py +0 -77
- etlplus/file/gz.py +0 -123
- etlplus/file/hbs.py +0 -78
- etlplus/file/hdf5.py +0 -78
- etlplus/file/ini.py +0 -79
- etlplus/file/ion.py +0 -78
- etlplus/file/jinja2.py +0 -78
- etlplus/file/json.py +0 -98
- etlplus/file/log.py +0 -78
- etlplus/file/mat.py +0 -78
- etlplus/file/mdb.py +0 -78
- etlplus/file/msgpack.py +0 -78
- etlplus/file/mustache.py +0 -78
- etlplus/file/nc.py +0 -78
- etlplus/file/ndjson.py +0 -108
- etlplus/file/numbers.py +0 -75
- etlplus/file/ods.py +0 -79
- etlplus/file/orc.py +0 -111
- etlplus/file/parquet.py +0 -113
- etlplus/file/pb.py +0 -78
- etlplus/file/pbf.py +0 -77
- etlplus/file/properties.py +0 -78
- etlplus/file/proto.py +0 -77
- etlplus/file/psv.py +0 -79
- etlplus/file/rda.py +0 -78
- etlplus/file/rds.py +0 -78
- etlplus/file/sas7bdat.py +0 -78
- etlplus/file/sav.py +0 -77
- etlplus/file/sqlite.py +0 -78
- etlplus/file/stub.py +0 -84
- etlplus/file/sylk.py +0 -77
- etlplus/file/tab.py +0 -81
- etlplus/file/toml.py +0 -78
- etlplus/file/tsv.py +0 -80
- etlplus/file/txt.py +0 -102
- etlplus/file/vm.py +0 -78
- etlplus/file/wks.py +0 -77
- etlplus/file/xls.py +0 -88
- etlplus/file/xlsm.py +0 -79
- etlplus/file/xlsx.py +0 -99
- etlplus/file/xml.py +0 -185
- etlplus/file/xpt.py +0 -78
- etlplus/file/yaml.py +0 -95
- etlplus/file/zip.py +0 -175
- etlplus/file/zsav.py +0 -77
- etlplus/ops/README.md +0 -50
- etlplus/ops/__init__.py +0 -61
- etlplus/templates/README.md +0 -46
- etlplus/workflow/README.md +0 -52
- etlplus/workflow/dag.py +0 -105
- etlplus/workflow/types.py +0 -115
- etlplus-0.9.2.dist-info/RECORD +0 -134
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/entry_points.txt +0 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.9.2.dist-info → etlplus-0.10.1.dist-info}/top_level.txt +0 -0
etlplus/file/fwf.py
DELETED
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.fwf` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Fixed-Width Fields (FWF) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- An FWF file is a text file format where each field has a fixed width.
|
|
9
|
-
- Common cases:
|
|
10
|
-
- Data files from legacy systems.
|
|
11
|
-
- Reports with aligned columns.
|
|
12
|
-
- Data exchange in mainframe environments.
|
|
13
|
-
- Rule of thumb:
|
|
14
|
-
- If the file follows the FWF specification, use this module for
|
|
15
|
-
reading and writing.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
from __future__ import annotations
|
|
19
|
-
|
|
20
|
-
from pathlib import Path
|
|
21
|
-
|
|
22
|
-
from ..types import JSONData
|
|
23
|
-
from ..types import JSONList
|
|
24
|
-
from . import stub
|
|
25
|
-
|
|
26
|
-
# SECTION: EXPORTS ========================================================== #
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
__all__ = [
|
|
30
|
-
'read',
|
|
31
|
-
'write',
|
|
32
|
-
]
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def read(
|
|
39
|
-
path: Path,
|
|
40
|
-
) -> JSONList:
|
|
41
|
-
"""
|
|
42
|
-
Read FWF content from ``path``.
|
|
43
|
-
|
|
44
|
-
Parameters
|
|
45
|
-
----------
|
|
46
|
-
path : Path
|
|
47
|
-
Path to the FWF file on disk.
|
|
48
|
-
|
|
49
|
-
Returns
|
|
50
|
-
-------
|
|
51
|
-
JSONList
|
|
52
|
-
The list of dictionaries read from the FWF file.
|
|
53
|
-
"""
|
|
54
|
-
return stub.read(path, format_name='FWF')
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def write(
|
|
58
|
-
path: Path,
|
|
59
|
-
data: JSONData,
|
|
60
|
-
) -> int:
|
|
61
|
-
"""
|
|
62
|
-
Write ``data`` to FWF file at ``path`` and return record count.
|
|
63
|
-
|
|
64
|
-
Parameters
|
|
65
|
-
----------
|
|
66
|
-
path : Path
|
|
67
|
-
Path to the FWF file on disk.
|
|
68
|
-
data : JSONData
|
|
69
|
-
Data to write as FWF file. Should be a list of dictionaries or a
|
|
70
|
-
single dictionary.
|
|
71
|
-
|
|
72
|
-
Returns
|
|
73
|
-
-------
|
|
74
|
-
int
|
|
75
|
-
The number of rows written to the FWF file.
|
|
76
|
-
"""
|
|
77
|
-
return stub.write(path, data, format_name='FWF')
|
etlplus/file/gz.py
DELETED
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.gz` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing GZ files.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import gzip
|
|
10
|
-
import tempfile
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
|
|
13
|
-
from ..types import JSONData
|
|
14
|
-
from .enums import CompressionFormat
|
|
15
|
-
from .enums import FileFormat
|
|
16
|
-
from .enums import infer_file_format_and_compression
|
|
17
|
-
|
|
18
|
-
# SECTION: EXPORTS ========================================================== #
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
__all__ = [
|
|
22
|
-
'read',
|
|
23
|
-
'write',
|
|
24
|
-
]
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
# SECTION: INTERNAL FUNCTIONS =============================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def _resolve_format(
|
|
31
|
-
path: Path,
|
|
32
|
-
) -> FileFormat:
|
|
33
|
-
"""
|
|
34
|
-
Resolve the inner file format from a .gz filename.
|
|
35
|
-
|
|
36
|
-
Parameters
|
|
37
|
-
----------
|
|
38
|
-
path : Path
|
|
39
|
-
Path to the GZ file on disk.
|
|
40
|
-
|
|
41
|
-
Returns
|
|
42
|
-
-------
|
|
43
|
-
FileFormat
|
|
44
|
-
The inferred inner file format.
|
|
45
|
-
|
|
46
|
-
Raises
|
|
47
|
-
------
|
|
48
|
-
ValueError
|
|
49
|
-
If the file format cannot be inferred from the filename.
|
|
50
|
-
"""
|
|
51
|
-
fmt, compression = infer_file_format_and_compression(path)
|
|
52
|
-
if compression is not CompressionFormat.GZ:
|
|
53
|
-
raise ValueError(f'Not a gzip file: {path}')
|
|
54
|
-
if fmt is None:
|
|
55
|
-
raise ValueError(
|
|
56
|
-
f'Cannot infer file format from compressed file {path!r}',
|
|
57
|
-
)
|
|
58
|
-
return fmt
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def read(
|
|
65
|
-
path: Path,
|
|
66
|
-
) -> JSONData:
|
|
67
|
-
"""
|
|
68
|
-
Read GZ content from ``path`` and parse the inner payload.
|
|
69
|
-
|
|
70
|
-
Parameters
|
|
71
|
-
----------
|
|
72
|
-
path : Path
|
|
73
|
-
Path to the GZ file on disk.
|
|
74
|
-
|
|
75
|
-
Returns
|
|
76
|
-
-------
|
|
77
|
-
JSONData
|
|
78
|
-
Parsed payload.
|
|
79
|
-
"""
|
|
80
|
-
fmt = _resolve_format(path)
|
|
81
|
-
with gzip.open(path, 'rb') as handle:
|
|
82
|
-
payload = handle.read()
|
|
83
|
-
|
|
84
|
-
with tempfile.TemporaryDirectory() as tmpdir:
|
|
85
|
-
tmp_path = Path(tmpdir) / f'payload.{fmt.value}'
|
|
86
|
-
tmp_path.write_bytes(payload)
|
|
87
|
-
from .core import File
|
|
88
|
-
|
|
89
|
-
return File(tmp_path, fmt).read()
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
def write(
|
|
93
|
-
path: Path,
|
|
94
|
-
data: JSONData,
|
|
95
|
-
) -> int:
|
|
96
|
-
"""
|
|
97
|
-
Write ``data`` to GZ at ``path`` and return record count.
|
|
98
|
-
|
|
99
|
-
Parameters
|
|
100
|
-
----------
|
|
101
|
-
path : Path
|
|
102
|
-
Path to the GZ file on disk.
|
|
103
|
-
data : JSONData
|
|
104
|
-
Data to write.
|
|
105
|
-
|
|
106
|
-
Returns
|
|
107
|
-
-------
|
|
108
|
-
int
|
|
109
|
-
Number of records written.
|
|
110
|
-
"""
|
|
111
|
-
fmt = _resolve_format(path)
|
|
112
|
-
with tempfile.TemporaryDirectory() as tmpdir:
|
|
113
|
-
tmp_path = Path(tmpdir) / f'payload.{fmt.value}'
|
|
114
|
-
from .core import File
|
|
115
|
-
|
|
116
|
-
count = File(tmp_path, fmt).write(data)
|
|
117
|
-
payload = tmp_path.read_bytes()
|
|
118
|
-
|
|
119
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
120
|
-
with gzip.open(path, 'wb') as handle:
|
|
121
|
-
handle.write(payload)
|
|
122
|
-
|
|
123
|
-
return count
|
etlplus/file/hbs.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.hbs` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Handlebars (HBS) template files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A Handlebars (HBS) template file is a text file used for generating HTML or
|
|
9
|
-
other text formats by combining templates with data.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- HTML templates.
|
|
12
|
-
- Email templates.
|
|
13
|
-
- Configuration files.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If you need to work with Handlebars template files, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read ZSAV content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the HBS file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the HBS file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='HBS')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to HBS file at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the HBS file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as HBS file. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the HBS file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='HBS')
|
etlplus/file/hdf5.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.hdf5` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Hierarchical Data Format (HDF5) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A HDF5 file is a binary file format designed to store and organize large
|
|
9
|
-
amounts of data.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Scientific data storage and sharing.
|
|
12
|
-
- Large-scale data analysis.
|
|
13
|
-
- Hierarchical data organization.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the HDF5 specification, use this module for reading
|
|
16
|
-
and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read HDF5 content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the HDF5 file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the HDF5 file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='HDF5')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to HDF5 file at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the HDF5 file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as HDF5 file. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the HDF5 file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='HDF5')
|
etlplus/file/ini.py
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.ini` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing initialization (INI) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- An INI file is a simple configuration file format that uses sections,
|
|
9
|
-
properties, and values.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Sections are denoted by square brackets (e.g., ``[section]``).
|
|
12
|
-
- Properties are key-value pairs (e.g., ``key=value``).
|
|
13
|
-
- Comments are often indicated by semicolons (``;``) or hash symbols
|
|
14
|
-
(``#``).
|
|
15
|
-
- Rule of thumb:
|
|
16
|
-
- If the file follows the INI specification, use this module for
|
|
17
|
-
reading and writing.
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
from __future__ import annotations
|
|
21
|
-
|
|
22
|
-
from pathlib import Path
|
|
23
|
-
|
|
24
|
-
from ..types import JSONData
|
|
25
|
-
from ..types import JSONList
|
|
26
|
-
from . import stub
|
|
27
|
-
|
|
28
|
-
# SECTION: EXPORTS ========================================================== #
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
__all__ = [
|
|
32
|
-
'read',
|
|
33
|
-
'write',
|
|
34
|
-
]
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def read(
|
|
41
|
-
path: Path,
|
|
42
|
-
) -> JSONList:
|
|
43
|
-
"""
|
|
44
|
-
Read INI content from ``path``.
|
|
45
|
-
|
|
46
|
-
Parameters
|
|
47
|
-
----------
|
|
48
|
-
path : Path
|
|
49
|
-
Path to the INI file on disk.
|
|
50
|
-
|
|
51
|
-
Returns
|
|
52
|
-
-------
|
|
53
|
-
JSONList
|
|
54
|
-
The list of dictionaries read from the INI file.
|
|
55
|
-
"""
|
|
56
|
-
return stub.read(path, format_name='INI')
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def write(
|
|
60
|
-
path: Path,
|
|
61
|
-
data: JSONData,
|
|
62
|
-
) -> int:
|
|
63
|
-
"""
|
|
64
|
-
Write ``data`` to INI at ``path`` and return record count.
|
|
65
|
-
|
|
66
|
-
Parameters
|
|
67
|
-
----------
|
|
68
|
-
path : Path
|
|
69
|
-
Path to the INI file on disk.
|
|
70
|
-
data : JSONData
|
|
71
|
-
Data to write as INI. Should be a list of dictionaries or a
|
|
72
|
-
single dictionary.
|
|
73
|
-
|
|
74
|
-
Returns
|
|
75
|
-
-------
|
|
76
|
-
int
|
|
77
|
-
The number of rows written to the INI file.
|
|
78
|
-
"""
|
|
79
|
-
return stub.write(path, data, format_name='INI')
|
etlplus/file/ion.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.ion` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing Amazon Ion (ION) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- An ION file is a richly-typed, self-describing data format developed by
|
|
9
|
-
Amazon, designed for efficient data interchange and storage.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Data serialization for distributed systems.
|
|
12
|
-
- Interoperability between different programming languages.
|
|
13
|
-
- Handling of complex data types beyond standard JSON capabilities.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the Amazon Ion specification, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read ION content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the ION file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the ION file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='ION')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to ION at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the ION file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as ION. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the ION file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='ION')
|
etlplus/file/jinja2.py
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.jinja2` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing compressed Jinja2 (JINJA2) template files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A JINJA2 file is a text file used for generating HTML or other text formats
|
|
9
|
-
by combining templates with data.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- HTML templates.
|
|
12
|
-
- Email templates.
|
|
13
|
-
- Configuration files.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If you need to work with Jinja2 template files, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
from pathlib import Path
|
|
22
|
-
|
|
23
|
-
from ..types import JSONData
|
|
24
|
-
from ..types import JSONList
|
|
25
|
-
from . import stub
|
|
26
|
-
|
|
27
|
-
# SECTION: EXPORTS ========================================================== #
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
__all__ = [
|
|
31
|
-
'read',
|
|
32
|
-
'write',
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def read(
|
|
40
|
-
path: Path,
|
|
41
|
-
) -> JSONList:
|
|
42
|
-
"""
|
|
43
|
-
Read JINJA2 content from ``path``.
|
|
44
|
-
|
|
45
|
-
Parameters
|
|
46
|
-
----------
|
|
47
|
-
path : Path
|
|
48
|
-
Path to the JINJA2 file on disk.
|
|
49
|
-
|
|
50
|
-
Returns
|
|
51
|
-
-------
|
|
52
|
-
JSONList
|
|
53
|
-
The list of dictionaries read from the JINJA2 file.
|
|
54
|
-
"""
|
|
55
|
-
return stub.read(path, format_name='JINJA2')
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def write(
|
|
59
|
-
path: Path,
|
|
60
|
-
data: JSONData,
|
|
61
|
-
) -> int:
|
|
62
|
-
"""
|
|
63
|
-
Write ``data`` to JINJA2 file at ``path`` and return record count.
|
|
64
|
-
|
|
65
|
-
Parameters
|
|
66
|
-
----------
|
|
67
|
-
path : Path
|
|
68
|
-
Path to the JINJA2 file on disk.
|
|
69
|
-
data : JSONData
|
|
70
|
-
Data to write as JINJA2 file. Should be a list of dictionaries or a
|
|
71
|
-
single dictionary.
|
|
72
|
-
|
|
73
|
-
Returns
|
|
74
|
-
-------
|
|
75
|
-
int
|
|
76
|
-
The number of rows written to the JINJA2 file.
|
|
77
|
-
"""
|
|
78
|
-
return stub.write(path, data, format_name='JINJA2')
|
etlplus/file/json.py
DELETED
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
:mod:`etlplus.file.json` module.
|
|
3
|
-
|
|
4
|
-
Helpers for reading/writing JavaScript Object Notation (JSON) files.
|
|
5
|
-
|
|
6
|
-
Notes
|
|
7
|
-
-----
|
|
8
|
-
- A JSON file is a widely used data interchange format that uses
|
|
9
|
-
human-readable text to represent structured data.
|
|
10
|
-
- Common cases:
|
|
11
|
-
- Data interchange between web applications and servers.
|
|
12
|
-
- Configuration files for applications.
|
|
13
|
-
- Data storage for NoSQL databases.
|
|
14
|
-
- Rule of thumb:
|
|
15
|
-
- If the file follows the JSON specification, use this module for
|
|
16
|
-
reading and writing.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
import json
|
|
22
|
-
from pathlib import Path
|
|
23
|
-
|
|
24
|
-
from ..types import JSONData
|
|
25
|
-
from ..utils import count_records
|
|
26
|
-
from ._io import coerce_record_payload
|
|
27
|
-
|
|
28
|
-
# SECTION: EXPORTS ========================================================== #
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
__all__ = [
|
|
32
|
-
'read',
|
|
33
|
-
'write',
|
|
34
|
-
]
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
# SECTION: FUNCTIONS ======================================================== #
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def read(
|
|
41
|
-
path: Path,
|
|
42
|
-
) -> JSONData:
|
|
43
|
-
"""
|
|
44
|
-
Read JSON content from ``path``.
|
|
45
|
-
|
|
46
|
-
Validates that the JSON root is a dict or a list of dicts.
|
|
47
|
-
|
|
48
|
-
Parameters
|
|
49
|
-
----------
|
|
50
|
-
path : Path
|
|
51
|
-
Path to the JSON file on disk.
|
|
52
|
-
|
|
53
|
-
Returns
|
|
54
|
-
-------
|
|
55
|
-
JSONData
|
|
56
|
-
The structured data read from the JSON file.
|
|
57
|
-
|
|
58
|
-
Raises
|
|
59
|
-
------
|
|
60
|
-
TypeError
|
|
61
|
-
If the JSON root is not an object or an array of objects.
|
|
62
|
-
"""
|
|
63
|
-
with path.open('r', encoding='utf-8') as handle:
|
|
64
|
-
loaded = json.load(handle)
|
|
65
|
-
|
|
66
|
-
return coerce_record_payload(loaded, format_name='JSON')
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def write(
|
|
70
|
-
path: Path,
|
|
71
|
-
data: JSONData,
|
|
72
|
-
) -> int:
|
|
73
|
-
"""
|
|
74
|
-
Write ``data`` as formatted JSON to ``path``.
|
|
75
|
-
|
|
76
|
-
Parameters
|
|
77
|
-
----------
|
|
78
|
-
path : Path
|
|
79
|
-
Path to the JSON file on disk.
|
|
80
|
-
data : JSONData
|
|
81
|
-
Data to serialize as JSON.
|
|
82
|
-
|
|
83
|
-
Returns
|
|
84
|
-
-------
|
|
85
|
-
int
|
|
86
|
-
The number of records written to the JSON file.
|
|
87
|
-
"""
|
|
88
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
89
|
-
with path.open('w', encoding='utf-8') as handle:
|
|
90
|
-
json.dump(
|
|
91
|
-
data,
|
|
92
|
-
handle,
|
|
93
|
-
indent=2,
|
|
94
|
-
ensure_ascii=False,
|
|
95
|
-
)
|
|
96
|
-
handle.write('\n')
|
|
97
|
-
|
|
98
|
-
return count_records(data)
|